diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt
index 61b9142ad5..e3dfa77801 100644
--- a/.cspell-wordlist.txt
+++ b/.cspell-wordlist.txt
@@ -277,4 +277,9 @@ thresholding
binarization
bugprone
NOLINTNEXTLINE
+preemphasis
+Silero
+autoregressive
+Autoregressive
+preemphasized
nullopt
diff --git a/apps/computer-vision/app/_layout.tsx b/apps/computer-vision/app/_layout.tsx
index f269b27aae..1a53311ad9 100644
--- a/apps/computer-vision/app/_layout.tsx
+++ b/apps/computer-vision/app/_layout.tsx
@@ -27,6 +27,13 @@ export default function Layout() {
title: 'Classification',
}}
/>
+
{
+ let s = 0;
+ for (let i = 0; i < a.length; i++) {
+ s += a[i]! * b[i]!;
+ }
+ return s;
+};
+
+function ImageEmbeddingsContent() {
+ const [selectedImageModel, setSelectedImageModel] = useState(IMAGE_MODEL_OPTIONS[0].value);
+ const [imageUri, setImageUri] = useState(null);
+ const [labels, setLabels] = useState(DEFAULT_LABELS);
+ const [newLabel, setNewLabel] = useState('');
+ const [results, setResults] = useState<{ label: string; score: number }[]>([]);
+ const [latency, setLatency] = useState(null);
+ const [isProcessing, setIsProcessing] = useState(false);
+ const [error, setError] = useState(null);
+
+ const insets = useSafeAreaInsets();
+ const skiaImage = useImage(imageUri, (err) => setError(err.message || String(err)));
+
+ // Zero-shot classification pairs a CLIP image encoder with the CLIP text
+ // encoder and scores the image against each text label by embedding similarity.
+ const imageModel = useImageEmbeddings(selectedImageModel);
+ const textModel = useTextEmbeddings(models.textEmbeddings.CLIP_VIT_BASE_PATCH32_TEXT);
+
+ const ready = imageModel.isReady && textModel.isReady;
+
+ const pickImage = async () => {
+ setError(null);
+ try {
+ const uri = await getImage(false);
+ if (!uri) return;
+ setImageUri(uri);
+ setResults([]);
+ setLatency(null);
+ } catch (e: any) {
+ setError(e.message || String(e));
+ }
+ };
+
+ const classify = async () => {
+ if (!skiaImage || !ready || !imageModel.forward || !textModel.forward) return;
+ setIsProcessing(true);
+ setError(null);
+ try {
+ const start = Date.now();
+ const imageEmbedding = await imageModel.forward(skImageToBuffer(skiaImage));
+ const scored: { label: string; score: number }[] = [];
+ for (const label of labels) {
+ const textEmbedding = await textModel.forward(label);
+ scored.push({ label, score: dot(imageEmbedding, textEmbedding) });
+ }
+ scored.sort((a, b) => b.score - a.score);
+ setLatency(Date.now() - start);
+ setResults(scored);
+ } catch (e: any) {
+ setError(e.message || String(e));
+ } finally {
+ setIsProcessing(false);
+ }
+ };
+
+ const addLabel = () => {
+ const trimmed = newLabel.trim();
+ if (!trimmed || labels.includes(trimmed)) return;
+ setLabels((prev) => [...prev, trimmed]);
+ setNewLabel('');
+ setResults([]);
+ };
+
+ const removeLabel = (label: string) => {
+ setLabels((prev) => prev.filter((l) => l !== label));
+ setResults((prev) => prev.filter((r) => r.label !== label));
+ };
+
+ const activeError = imageModel.error
+ ? String(imageModel.error)
+ : textModel.error
+ ? String(textModel.error)
+ : error;
+
+ return (
+
+
+ Pick an image, then rank text labels by how well CLIP matches them to it (zero-shot
+ classification).
+
+
+ {
+ setSelectedImageModel(model);
+ setResults([]);
+ setLatency(null);
+ }}
+ />
+
+
+
+
+
+
+
+
+
+
+
+
+ {results.length > 0 && (
+
+ Results
+ {results.map((r, i) => (
+
+
+ {i === 0 ? '🥇 ' : ''}
+ {r.label}
+
+ {r.score.toFixed(3)}
+
+ ))}
+
+ )}
+
+
+ Labels
+ {labels.map((label) => (
+
+
+ {label}
+
+ removeLabel(label)} hitSlop={8}>
+ ✕
+
+
+ ))}
+
+
+
+
+
+
+ );
+}
+
+export default function ImageEmbeddingsScreen() {
+ return (
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ card: {
+ width: '100%',
+ backgroundColor: '#fff',
+ borderRadius: 12,
+ padding: 16,
+ borderWidth: 1,
+ borderColor: '#e9ecef',
+ marginTop: 16,
+ },
+ cardTitle: {
+ fontSize: 16,
+ fontWeight: '600',
+ color: ColorPalette.strongPrimary,
+ marginBottom: 8,
+ },
+ row: {
+ flexDirection: 'row',
+ justifyContent: 'space-between',
+ alignItems: 'center',
+ paddingVertical: 8,
+ borderBottomWidth: 1,
+ borderBottomColor: '#f1f3f5',
+ },
+ rowLabel: { fontSize: 14, color: '#334155', flex: 1, marginRight: 8 },
+ topLabel: { fontWeight: '700', color: ColorPalette.strongPrimary },
+ rowScore: { fontSize: 13, fontWeight: '600', color: ColorPalette.primary },
+ remove: { fontSize: 16, color: '#94A3B8', paddingHorizontal: 4 },
+ addRow: { flexDirection: 'row', alignItems: 'center', gap: 8, marginTop: 12 },
+ input: {
+ flex: 1,
+ backgroundColor: '#f1f3f5',
+ borderRadius: 10,
+ paddingHorizontal: 12,
+ paddingVertical: 10,
+ fontSize: 14,
+ color: '#0F172A',
+ },
+});
diff --git a/apps/computer-vision/app/index.tsx b/apps/computer-vision/app/index.tsx
index cc44604e67..14fa922e57 100644
--- a/apps/computer-vision/app/index.tsx
+++ b/apps/computer-vision/app/index.tsx
@@ -14,6 +14,9 @@ export default function Home() {
router.navigate('classification/')}>
Classification
+ router.navigate('imageEmbeddings/')}>
+ Image Embeddings
+
router.navigate('detection/')}>
Object Detection
diff --git a/apps/computer-vision/package.json b/apps/computer-vision/package.json
index 01cebec75c..4a47a35f3f 100644
--- a/apps/computer-vision/package.json
+++ b/apps/computer-vision/package.json
@@ -5,6 +5,8 @@
"react-native-executorch": {
"features": [
"classification",
+ "imageEmbeddings",
+ "textEmbeddings",
"instanceSegmentation",
"keypointDetection",
"objectDetection",
diff --git a/apps/computer-vision/utils.ts b/apps/computer-vision/utils.ts
index 2a73d4b5b5..3a24d0341e 100644
--- a/apps/computer-vision/utils.ts
+++ b/apps/computer-vision/utils.ts
@@ -1,6 +1,32 @@
import { Alert } from 'react-native';
import * as ImagePicker from 'expo-image-picker';
import { ImageManipulator, SaveFormat } from 'expo-image-manipulator';
+import type { SkImage } from '@shopify/react-native-skia';
+
+/**
+ * Converts a Skia image into the raw RGBA/HWC image buffer that
+ * react-native-executorch vision tasks accept. Throws if the pixel data cannot
+ * be read.
+ * @param image - The Skia image to read pixels from.
+ * @returns The RGBA/HWC image buffer with its `data`, `width`, `height`,
+ * `format`, and `layout`.
+ */
+export const skImageToBuffer = (image: SkImage) => {
+ const pixels = image.readPixels();
+ if (!pixels) {
+ throw new Error('Failed to read pixels from image');
+ }
+ if (!(pixels instanceof Uint8Array)) {
+ throw new Error('Expected Uint8Array from readPixels');
+ }
+ return {
+ data: pixels,
+ width: image.width(),
+ height: image.height(),
+ format: 'rgba' as const,
+ layout: 'hwc' as const,
+ };
+};
export const getImage = async (
useCamera: boolean,
diff --git a/apps/nlp/app/_layout.tsx b/apps/nlp/app/_layout.tsx
index bdcfc39660..352cf99d44 100644
--- a/apps/nlp/app/_layout.tsx
+++ b/apps/nlp/app/_layout.tsx
@@ -27,6 +27,13 @@ export default function Layout() {
title: 'Tokenizer',
}}
/>
+
);
}
diff --git a/apps/nlp/app/index.tsx b/apps/nlp/app/index.tsx
index 98ff59ab97..57b5aea6ae 100644
--- a/apps/nlp/app/index.tsx
+++ b/apps/nlp/app/index.tsx
@@ -14,6 +14,9 @@ export default function Home() {
router.navigate('tokenizer/')}>
Tokenizer
+ router.navigate('text-embeddings/')}>
+ Text Embeddings
+
);
diff --git a/apps/nlp/app/text-embeddings/index.tsx b/apps/nlp/app/text-embeddings/index.tsx
new file mode 100644
index 0000000000..d7edde3c68
--- /dev/null
+++ b/apps/nlp/app/text-embeddings/index.tsx
@@ -0,0 +1,398 @@
+import React, { useEffect, useState } from 'react';
+import {
+ View,
+ Text,
+ TextInput,
+ ScrollView,
+ StyleSheet,
+ TouchableOpacity,
+ KeyboardAvoidingView,
+ Platform,
+} from 'react-native';
+import { useTextEmbeddings, models, type TextEmbeddingsModel } from 'react-native-executorch';
+import ScreenWrapper from '../../components/ScreenWrapper';
+import { ModelStatus } from '../../components/ModelStatus';
+import { Button } from '../../components/Button';
+import { theme } from '../../theme';
+
+const MODELS: { label: string; value: TextEmbeddingsModel }[] = [
+ { label: 'MiniLM L6', value: models.textEmbeddings.ALL_MINILM_L6_V2 },
+ { label: 'MPNet Base', value: models.textEmbeddings.ALL_MPNET_BASE_V2 },
+ { label: 'MultiQA MiniLM', value: models.textEmbeddings.MULTI_QA_MINILM_L6_COS_V1 },
+ { label: 'MultiQA MPNet', value: models.textEmbeddings.MULTI_QA_MPNET_BASE_DOT_V1 },
+ { label: 'Paraphrase ML', value: models.textEmbeddings.PARAPHRASE_MULTILINGUAL_MINILM_L12_V2 },
+ { label: 'DistilUSE ML', value: models.textEmbeddings.DISTILUSE_BASE_MULTILINGUAL_CASED_V2 },
+ { label: 'CLIP Text', value: models.textEmbeddings.CLIP_VIT_BASE_PATCH32_TEXT },
+];
+
+const STARTER_SENTENCES = [
+ 'The weather is lovely today.',
+ "It's so sunny outside!",
+ 'He drove to the stadium.',
+ 'A man is eating a piece of bread.',
+ 'The cat sleeps on the warm windowsill.',
+];
+
+// These models output L2-normalized embeddings, so cosine similarity is the dot
+// product.
+const cosine = (a: Float32Array, b: Float32Array) => {
+ let dot = 0;
+ for (let i = 0; i < a.length; i++) {
+ dot += a[i]! * b[i]!;
+ }
+ return dot;
+};
+
+type Entry = { sentence: string; embedding: Float32Array };
+type Match = { sentence: string; similarity: number };
+
+const isDisposedError = (msg: string) => /disposed/i.test(msg);
+
+function TextEmbeddingsContent() {
+ const [selected, setSelected] = useState(0);
+ const { isReady, downloadProgress, error, forward } = useTextEmbeddings(MODELS[selected]!.value);
+
+ const [library, setLibrary] = useState([]);
+ const [input, setInput] = useState('');
+ const [matches, setMatches] = useState(null);
+ const [queryText, setQueryText] = useState('');
+ const [busy, setBusy] = useState(false);
+ const [runError, setRunError] = useState(null);
+ const [embedMs, setEmbedMs] = useState(null);
+
+ const ready = isReady && !!forward;
+
+ // Re-seed the library with starter sentences whenever the model changes.
+ useEffect(() => {
+ if (!ready || !forward) return;
+ let cancelled = false;
+ (async () => {
+ setBusy(true);
+ setRunError(null);
+ try {
+ const entries: Entry[] = [];
+ for (const sentence of STARTER_SENTENCES) {
+ const embedding = await forward(sentence);
+ if (cancelled) return;
+ entries.push({ sentence, embedding });
+ }
+ setLibrary(entries);
+ } catch (e: any) {
+ if (!cancelled) setRunError(e?.message ?? String(e));
+ } finally {
+ if (!cancelled) setBusy(false);
+ }
+ })();
+ return () => {
+ cancelled = true;
+ };
+ }, [forward, ready]);
+
+ const selectModel = (i: number) => {
+ if (i === selected) return;
+ setSelected(i);
+ setLibrary([]);
+ setMatches(null);
+ setQueryText('');
+ setEmbedMs(null);
+ };
+
+ const findSimilar = async () => {
+ if (!forward || !input.trim() || library.length === 0) return;
+ setBusy(true);
+ setRunError(null);
+ try {
+ const start = Date.now();
+ const q = await forward(input.trim());
+ setEmbedMs(Date.now() - start);
+ const ranked = library
+ .map(({ sentence, embedding }) => ({ sentence, similarity: cosine(q, embedding) }))
+ .sort((a, b) => b.similarity - a.similarity);
+ setQueryText(input.trim());
+ setMatches(ranked);
+ } catch (e: any) {
+ const msg = e?.message ?? String(e);
+ if (!isDisposedError(msg)) setRunError(msg);
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ const addToLibrary = async () => {
+ if (!forward || !input.trim()) return;
+ setBusy(true);
+ setRunError(null);
+ try {
+ const start = Date.now();
+ const embedding = await forward(input.trim());
+ setEmbedMs(Date.now() - start);
+ setLibrary((prev) => [...prev, { sentence: input.trim(), embedding }]);
+ setInput('');
+ setMatches(null);
+ } catch (e: any) {
+ const msg = e?.message ?? String(e);
+ if (!isDisposedError(msg)) setRunError(msg);
+ } finally {
+ setBusy(false);
+ }
+ };
+
+ const removeAt = (i: number) => {
+ setLibrary((prev) => prev.filter((_, idx) => idx !== i));
+ setMatches(null);
+ };
+
+ const clearLibrary = () => {
+ setLibrary([]);
+ setMatches(null);
+ setQueryText('');
+ };
+
+ return (
+
+
+
+ Text Embeddings
+
+ Semantic search playground. Build a library of sentences, then find the ones closest in
+ meaning to your query using cosine similarity over the embeddings.
+
+
+ Model
+
+ {MODELS.map((m, i) => (
+ selectModel(i)}
+ >
+
+ {m.label}
+
+
+ ))}
+
+
+
+
+
+ {runError && (
+
+ {runError}
+
+ )}
+
+
+
+ Sentence library ({library.length})
+ {library.length > 0 && (
+
+ Clear
+
+ )}
+
+ {library.length === 0 ? (
+
+ {ready ? 'Library is empty — add a sentence below.' : 'Waiting for the model…'}
+
+ ) : (
+ library.map((item, i) => (
+
+ {item.sentence}
+ removeAt(i)} hitSlop={8}>
+ ✕
+
+
+ ))
+ )}
+
+
+
+ Try your sentence
+
+
+
+
+
+ {embedMs !== null && Embedding time: {embedMs} ms}
+
+
+ {matches && (
+
+ Ranked by similarity
+ to “{queryText}”
+ {matches.map((m, i) => {
+ const pct = Math.max(0, Math.min(1, m.similarity)) * 100;
+ return (
+
+
+
+ {m.sentence}
+
+ {m.similarity.toFixed(3)}
+
+
+
+
+
+ );
+ })}
+
+ )}
+
+
+ );
+}
+
+export default function TextEmbeddingsScreen() {
+ return (
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ flex: { flex: 1 },
+ container: { flex: 1, backgroundColor: theme.colors.background },
+ content: { padding: theme.spacing.large, paddingBottom: 40 },
+ card: {
+ backgroundColor: theme.colors.cardBackground,
+ borderRadius: theme.radius.large,
+ padding: 20,
+ marginBottom: 20,
+ borderWidth: 1,
+ borderColor: theme.colors.lightBorder,
+ },
+ cardTitle: {
+ fontSize: theme.typography.title.fontSize,
+ fontWeight: theme.typography.title.fontWeight,
+ color: theme.colors.strongPrimary,
+ marginBottom: 8,
+ },
+ cardDescription: {
+ fontSize: 14,
+ color: theme.colors.textMuted,
+ lineHeight: 20,
+ marginBottom: 16,
+ },
+ fieldLabel: {
+ fontSize: 12,
+ fontWeight: '600',
+ color: theme.colors.textPlaceholder,
+ textTransform: 'uppercase',
+ letterSpacing: 0.5,
+ marginBottom: 8,
+ },
+ chipRow: { gap: 8, paddingBottom: 4, marginBottom: 12 },
+ chip: {
+ paddingHorizontal: 14,
+ paddingVertical: 8,
+ borderRadius: 20,
+ backgroundColor: '#f1f3f5',
+ borderWidth: 1,
+ borderColor: theme.colors.lightBorder,
+ },
+ chipActive: {
+ backgroundColor: theme.colors.strongPrimary,
+ borderColor: theme.colors.strongPrimary,
+ },
+ chipText: { fontSize: 13, fontWeight: '600', color: theme.colors.textMuted },
+ chipTextActive: { color: '#fff' },
+ sectionTitle: { fontSize: 16, fontWeight: '700', color: '#212529' },
+ libraryHeader: {
+ flexDirection: 'row',
+ justifyContent: 'space-between',
+ alignItems: 'center',
+ marginBottom: 12,
+ },
+ clearLink: { fontSize: 13, fontWeight: '600', color: theme.colors.accent },
+ emptyText: { fontSize: 14, color: theme.colors.textPlaceholder, fontStyle: 'italic' },
+ libraryRow: {
+ flexDirection: 'row',
+ alignItems: 'center',
+ justifyContent: 'space-between',
+ paddingVertical: 8,
+ borderBottomWidth: 1,
+ borderBottomColor: '#f1f3f5',
+ },
+ librarySentence: { flex: 1, fontSize: 14, color: '#495057', marginRight: 10 },
+ removeBtn: { fontSize: 15, color: theme.colors.textPlaceholder, fontWeight: '700' },
+ input: {
+ backgroundColor: '#f1f3f5',
+ borderRadius: theme.radius.small,
+ padding: 12,
+ fontSize: 15,
+ color: '#212529',
+ marginBottom: 16,
+ minHeight: 44,
+ textAlignVertical: 'top',
+ borderWidth: 1,
+ borderColor: theme.colors.lightBorder,
+ },
+ buttonRow: { flexDirection: 'row', gap: theme.spacing.small },
+ statsText: {
+ fontSize: 13,
+ color: theme.colors.textPlaceholder,
+ marginTop: 12,
+ textAlign: 'center',
+ },
+ errorContainer: {
+ backgroundColor: theme.colors.errorBackground,
+ padding: 12,
+ borderRadius: theme.radius.small,
+ marginBottom: 20,
+ },
+ errorText: { color: theme.colors.errorText, fontSize: 14, textAlign: 'center' },
+ queryText: { fontSize: 13, color: theme.colors.textMuted, marginTop: 2, marginBottom: 14 },
+ matchRow: { marginBottom: 14 },
+ matchHeader: {
+ flexDirection: 'row',
+ justifyContent: 'space-between',
+ alignItems: 'center',
+ marginBottom: 6,
+ },
+ matchSentence: { flex: 1, fontSize: 14, color: '#495057', marginRight: 10 },
+ matchTop: { fontWeight: '700', color: theme.colors.strongPrimary },
+ matchScore: { fontSize: 13, fontWeight: '700', color: theme.colors.textMuted },
+ barTrack: {
+ height: 8,
+ borderRadius: 4,
+ backgroundColor: '#f1f3f5',
+ overflow: 'hidden',
+ },
+ barFill: { height: 8, borderRadius: 4, backgroundColor: '#adb5bd' },
+ barFillTop: { backgroundColor: theme.colors.accent },
+});
diff --git a/apps/nlp/package.json b/apps/nlp/package.json
index 361bd18f48..4216dc89cb 100644
--- a/apps/nlp/package.json
+++ b/apps/nlp/package.json
@@ -4,7 +4,8 @@
"main": "expo-router/entry",
"react-native-executorch": {
"features": [
- "tokenizer"
+ "tokenizer",
+ "textEmbeddings"
]
},
"scripts": {
diff --git a/apps/speech/App.tsx b/apps/speech/App.tsx
new file mode 100644
index 0000000000..d9870b4c0e
--- /dev/null
+++ b/apps/speech/App.tsx
@@ -0,0 +1,686 @@
+import React, { useEffect, useRef, useState } from 'react';
+import {
+ StyleSheet,
+ Text,
+ View,
+ TouchableOpacity,
+ ScrollView,
+ ActivityIndicator,
+} from 'react-native';
+import { SafeAreaProvider, useSafeAreaInsets } from 'react-native-safe-area-context';
+import RNFS from 'react-native-fs';
+import { AudioContext, AudioManager, AudioRecorder } from 'react-native-audio-api';
+import {
+ useSpeechToText,
+ models,
+ type WhisperLanguage,
+ type WhisperModel,
+} from 'react-native-executorch';
+
+// Selectable Whisper variants (EN-only models are faster but English-only)
+const MODELS: { name: string; config: WhisperModel }[] = [
+ { name: 'Tiny (EN) CPU', config: models.speechToText.WHISPER_TINY_EN.XNNPACK_FP32 },
+ { name: 'Tiny (EN) CoreML', config: models.speechToText.WHISPER_TINY_EN.COREML_FP16 },
+ { name: 'Tiny CPU', config: models.speechToText.WHISPER_TINY.XNNPACK_FP32 },
+ { name: 'Tiny CoreML', config: models.speechToText.WHISPER_TINY.COREML_FP16 },
+ { name: 'Base (EN) CPU', config: models.speechToText.WHISPER_BASE_EN.XNNPACK_FP32 },
+ { name: 'Base (EN) CoreML', config: models.speechToText.WHISPER_BASE_EN.COREML_FP16 },
+ { name: 'Base CPU', config: models.speechToText.WHISPER_BASE.XNNPACK_FP32 },
+ { name: 'Base CoreML', config: models.speechToText.WHISPER_BASE.COREML_FP16 },
+ { name: 'Small (EN) CPU', config: models.speechToText.WHISPER_SMALL_EN.XNNPACK_FP32 },
+ { name: 'Small (EN) CoreML', config: models.speechToText.WHISPER_SMALL_EN.COREML_FP16 },
+ { name: 'Small CPU', config: models.speechToText.WHISPER_SMALL.XNNPACK_FP32 },
+ { name: 'Small CoreML', config: models.speechToText.WHISPER_SMALL.COREML_FP16 },
+];
+
+// Predefined multilingual speech samples from VoIP Troubleshooter (Open Speech Repository)
+const SAMPLE_AUDIOS = [
+ {
+ name: 'English (US)',
+ url: 'https://www.voiptroubleshooter.com/open_speech/american/OSR_us_000_0010_8k.wav',
+ language: 'en' as WhisperLanguage,
+ },
+ {
+ name: 'English (UK)',
+ url: 'https://www.voiptroubleshooter.com/open_speech/british/OSR_uk_000_0020_8k.wav',
+ language: 'en' as WhisperLanguage,
+ },
+ {
+ name: 'French',
+ url: 'https://www.voiptroubleshooter.com/open_speech/french/OSR_fr_000_0041_8k.wav',
+ language: 'fr' as WhisperLanguage,
+ },
+ {
+ name: 'Mandarin (Chinese)',
+ url: 'https://www.voiptroubleshooter.com/open_speech/chinese/OSR_cn_000_0072_8k.wav',
+ language: 'zh' as WhisperLanguage,
+ },
+ {
+ name: 'English (long)',
+ url: 'https://archive.org/download/1912_shortworks_0906_librivox/010-workerschurch_summers_64kb.mp3',
+ language: 'en' as WhisperLanguage,
+ },
+];
+
+function MainApp() {
+ const insets = useSafeAreaInsets();
+ const [tokens, setTokens] = useState([]);
+ const [committedText, setCommittedText] = useState('');
+ const [nonCommittedText, setNonCommittedText] = useState('');
+ const [status, setStatus] = useState('Idle');
+ const [currentAudioBuffer, setCurrentAudioBuffer] = useState(null);
+ const [isPlaying, setIsPlaying] = useState(false);
+ const [activeSource, setActiveSource] = useState(null);
+ const [audioDownloadProgress, setAudioDownloadProgress] = useState(null);
+ const [selectedModel, setSelectedModel] = useState(MODELS[0]);
+ const [isRecording, setIsRecording] = useState(false);
+ const recorderRef = useRef(null);
+ const isDecodingRef = useRef(false);
+ const [activeTab, setActiveTab] = useState<'file' | 'mic'>('file');
+
+ const { isReady, downloadProgress, error, transcribe, stream, streamInsert, streamStop } =
+ useSpeechToText(selectedModel.config);
+
+ const handleTabChange = (tab: 'file' | 'mic') => {
+ if (status.includes('...')) return;
+ setActiveTab(tab);
+ setTokens([]);
+ setCommittedText('');
+ setNonCommittedText('');
+ isDecodingRef.current = false;
+ setStatus('Idle');
+ setCurrentAudioBuffer(null);
+ };
+
+ // Whisper expects 16 kHz mono PCM
+ const MIC_SAMPLE_RATE = 16000;
+
+ const stopRecording = async () => {
+ recorderRef.current?.stop();
+ recorderRef.current = null;
+ setIsRecording(false);
+ isDecodingRef.current = false;
+
+ if (streamStop) {
+ streamStop();
+ }
+ setStatus('Done');
+ };
+
+ const startRecording = async () => {
+ if (!isReady || !stream || !streamInsert || isRecording) {
+ console.log(`[mic] startRecording blocked: isReady=${isReady} isRecording=${isRecording}`);
+ return;
+ }
+
+ const permission = await AudioManager.requestRecordingPermissions();
+ console.log(`[mic] permission=${permission}`);
+ if (permission !== 'Granted') {
+ setStatus('Microphone permission denied');
+ return;
+ }
+
+ AudioManager.setAudioSessionOptions({
+ iosCategory: 'playAndRecord',
+ iosMode: 'measurement',
+ iosOptions: ['defaultToSpeaker', 'allowBluetoothHFP'],
+ });
+
+ setCommittedText('');
+ setNonCommittedText('');
+ setStatus('Streaming...');
+ setIsRecording(true);
+
+ // Consume the transcription stream in the background
+ (async () => {
+ try {
+ console.log('[App] starting transcription stream...');
+ const textStream = stream({ language: 'en' });
+ for await (const result of textStream) {
+ console.log(`[App] streaming update - committed: "${result}"`);
+ setCommittedText(result);
+ // setNonCommittedText(result.nonCommitted);
+ }
+ } catch (err) {
+ console.error(`Streaming error: ${err}`);
+ }
+ })();
+
+ const recorder = new AudioRecorder();
+ recorderRef.current = recorder;
+ // Feed each captured buffer directly into the stream
+ recorder.onAudioReady(
+ { sampleRate: MIC_SAMPLE_RATE, bufferLength: 4096, channelCount: 1 },
+ (event: any) => {
+ const samples = event.buffer.getChannelData(0);
+ streamInsert(new Float32Array(samples));
+ }
+ );
+ recorder.start();
+ console.log('[mic] recorder started');
+ };
+
+ // Tear down the recorder if the screen unmounts mid-recording
+ useEffect(() => {
+ return () => {
+ recorderRef.current?.stop();
+ recorderRef.current = null;
+ };
+ }, []);
+
+ const togglePlayback = () => {
+ if (isPlaying && activeSource) {
+ try {
+ activeSource.stop();
+ } catch (e) {}
+ setActiveSource(null);
+ setIsPlaying(false);
+ return;
+ }
+ if (!currentAudioBuffer) return;
+ try {
+ const audioContext = new AudioContext({ sampleRate: 16000 });
+ const source = audioContext.createBufferSource();
+ source.buffer = currentAudioBuffer;
+ source.connect(audioContext.destination);
+ source.onEnded = () => {
+ setIsPlaying(false);
+ setActiveSource(null);
+ };
+ source.start();
+ setActiveSource(source);
+ setIsPlaying(true);
+ } catch (err) {}
+ };
+
+ const handleTranscribeUrl = async (url: string, language?: WhisperLanguage) => {
+ if (activeSource) {
+ try {
+ activeSource.stop();
+ } catch (e) {}
+ setActiveSource(null);
+ setIsPlaying(false);
+ }
+ setCurrentAudioBuffer(null);
+
+ if (!isReady || !transcribe) {
+ return;
+ }
+ setStatus('Processing...');
+ setTokens([]);
+ console.log(`Target URL: ${url}`);
+
+ // Preserve the source extension so the decoder (esp. iOS AVAudioFile, which
+ // keys off it) picks the right format for non-WAV samples.
+ const ext = url.split('?')[0]!.split('.').pop()?.toLowerCase() ?? 'wav';
+ const localDest = `${RNFS.CachesDirectoryPath}/test_sample.${ext}`;
+
+ try {
+ // 1. Download audio
+ setStatus('Downloading audio...');
+ setAudioDownloadProgress(0);
+ const downloadRes = await RNFS.downloadFile({
+ fromUrl: url,
+ toFile: localDest,
+ progressInterval: 100,
+ begin: () => setAudioDownloadProgress(0),
+ progress: ({ bytesWritten, contentLength }: any) => {
+ if (contentLength > 0) {
+ setAudioDownloadProgress(Math.round((bytesWritten / contentLength) * 100));
+ }
+ },
+ }).promise;
+ setAudioDownloadProgress(100);
+
+ if (downloadRes.statusCode !== 200) {
+ throw new Error(`Download failed with HTTP status ${downloadRes.statusCode}`);
+ }
+
+ // 2. Decode WAV
+ setAudioDownloadProgress(null);
+ setStatus('Decoding audio...');
+ const audioContext = new AudioContext({ sampleRate: 16000 });
+ const decodedData = await audioContext.decodeAudioData(localDest);
+
+ // Keep the decoded buffer so the user can play back the downloaded audio
+ setCurrentAudioBuffer(decodedData);
+
+ const waveform = decodedData.getChannelData(0);
+
+ await RNFS.unlink(localDest).catch(() => {});
+
+ // 3. Transcribe
+ setStatus('Transcribing...');
+ const result = await transcribe(waveform, { language: language || 'en' }, (token: string) => {
+ setTokens((prev) => [...prev, token]);
+ });
+
+ setStatus('Done');
+ console.log(`Done: "${result}"`);
+ } catch (err) {
+ setStatus('Error');
+ setAudioDownloadProgress(null);
+ console.error(`Error: ${err instanceof Error ? err.message : String(err)}`);
+ await RNFS.unlink(localDest).catch(() => {});
+ }
+ };
+
+ return (
+
+
+ Whisper STT Test Client
+
+
+ {/* Tab Bar */}
+
+ handleTabChange('file')}
+ >
+
+ Audio Files
+
+
+ handleTabChange('mic')}
+ >
+
+ Live Microphone
+
+
+
+
+
+ {/* Model Picker Card */}
+
+ Model
+
+ {MODELS.map((item) => {
+ const isSelected = item.config === selectedModel.config;
+ return (
+ setSelectedModel(item)}
+ disabled={status.includes('...')}
+ >
+
+ {item.name}
+
+
+ );
+ })}
+
+
+
+ {/* Model Status Card */}
+
+ Model: Whisper {selectedModel.name}
+
+ Status: {isReady ? 'Ready' : error ? 'Error loading model' : 'Downloading...'}
+
+ {!isReady && !error && (
+
+
+ Progress: {Math.round(downloadProgress)}%
+
+ )}
+ {error && Error: {error.message}}
+
+
+ {/* Tab Specific Content */}
+ {activeTab === 'file' && isReady && (
+ <>
+ {/* Predefined URLs Card */}
+
+ Predefined Test Audio Files
+ {SAMPLE_AUDIOS.map((item, idx) => (
+ handleTranscribeUrl(item.url, item.language)}
+ disabled={status.includes('...')}
+ >
+
+ {item.name}
+
+ {item.url}
+
+
+ →
+
+ ))}
+
+
+ {/* Status / Playback Card */}
+
+ Current Operation Status: {status}
+ {audioDownloadProgress !== null && (
+
+
+
+
+
+ Downloading audio... {audioDownloadProgress}%
+
+
+ )}
+ {currentAudioBuffer && (
+
+
+ {isPlaying ? 'Stop Audio Input' : 'Play Audio Input'}
+
+
+ )}
+
+
+ {/* Transcription Result Card */}
+
+ Transcription Result:
+
+ {tokens.length > 0 ? tokens.join('') : 'Waiting for transcription...'}
+
+
+
+ )}
+
+ {activeTab === 'mic' && isReady && (
+ <>
+ {/* Microphone Card */}
+
+ Microphone Stream
+
+
+ {isRecording ? 'Stop Recording' : 'Start Recording'}
+
+
+
+
+ {/* Real-time Streamed Tokens Card */}
+
+ Real-time Streamed Tokens:
+
+ {committedText ? (
+ <>
+ {committedText}
+ {nonCommittedText}
+
+ ) : nonCommittedText ? (
+ {nonCommittedText}
+ ) : (
+ 'Waiting for tokens...'
+ )}
+
+
+
+ )}
+
+
+ );
+}
+
+export default function App() {
+ return (
+
+
+
+ );
+}
+
+const styles = StyleSheet.create({
+ container: {
+ flex: 1,
+ backgroundColor: '#f5f5f7',
+ },
+ tabBar: {
+ flexDirection: 'row',
+ backgroundColor: '#ffffff',
+ borderBottomWidth: 1,
+ borderBottomColor: '#e5e5ea',
+ },
+ tabButton: {
+ flex: 1,
+ alignItems: 'center',
+ paddingVertical: 12,
+ borderBottomWidth: 2,
+ borderBottomColor: 'transparent',
+ },
+ tabButtonActive: {
+ borderBottomColor: '#007aff',
+ },
+ tabButtonText: {
+ fontSize: 14,
+ fontWeight: '600',
+ color: '#8e8e93',
+ },
+ tabButtonTextActive: {
+ color: '#007aff',
+ },
+ header: {
+ alignItems: 'center',
+ paddingVertical: 15,
+ backgroundColor: '#ffffff',
+ borderBottomWidth: 1,
+ borderBottomColor: '#e5e5ea',
+ },
+ headerTitle: {
+ fontSize: 18,
+ fontWeight: '700',
+ color: '#1c1c1e',
+ },
+ content: {
+ flex: 1,
+ },
+ contentContainer: {
+ padding: 20,
+ },
+ card: {
+ backgroundColor: '#ffffff',
+ borderRadius: 12,
+ padding: 16,
+ marginBottom: 16,
+ shadowColor: '#000000',
+ shadowOpacity: 0.05,
+ shadowOffset: { width: 0, height: 2 },
+ shadowRadius: 4,
+ elevation: 2,
+ },
+ label: {
+ fontSize: 15,
+ fontWeight: '600',
+ color: '#3a3a3c',
+ marginBottom: 4,
+ },
+ value: {
+ fontSize: 14,
+ color: '#8e8e93',
+ },
+ progressContainer: {
+ flexDirection: 'row',
+ alignItems: 'center',
+ marginTop: 8,
+ },
+ progress: {
+ fontSize: 14,
+ color: '#007aff',
+ fontWeight: '600',
+ },
+ downloadProgress: {
+ marginBottom: 12,
+ },
+ progressBarTrack: {
+ height: 8,
+ borderRadius: 4,
+ backgroundColor: '#e5e5ea',
+ overflow: 'hidden',
+ },
+ progressBarFill: {
+ height: 8,
+ borderRadius: 4,
+ backgroundColor: '#007aff',
+ },
+ progressBarLabel: {
+ fontSize: 12,
+ color: '#007aff',
+ fontWeight: '600',
+ marginTop: 4,
+ },
+ errorText: {
+ fontSize: 14,
+ color: '#ff3b30',
+ marginTop: 8,
+ },
+ sectionTitle: {
+ fontSize: 15,
+ fontWeight: '600',
+ color: '#1c1c1e',
+ marginBottom: 12,
+ },
+ modelPickerScroll: {
+ flexDirection: 'row',
+ gap: 8,
+ paddingVertical: 4,
+ },
+ modelChip: {
+ paddingHorizontal: 14,
+ paddingVertical: 8,
+ borderRadius: 20,
+ backgroundColor: '#f2f2f7',
+ borderWidth: 1,
+ borderColor: '#e5e5ea',
+ },
+ modelChipSelected: {
+ backgroundColor: '#007aff',
+ borderColor: '#007aff',
+ },
+ modelChipText: {
+ fontSize: 13,
+ fontWeight: '600',
+ color: '#3a3a3c',
+ },
+ modelChipTextSelected: {
+ color: '#ffffff',
+ },
+ sampleButton: {
+ flexDirection: 'row',
+ alignItems: 'center',
+ justifyContent: 'space-between',
+ paddingVertical: 10,
+ borderBottomWidth: 1,
+ borderBottomColor: '#f2f2f7',
+ },
+ sampleName: {
+ fontSize: 14,
+ fontWeight: '600',
+ color: '#1c1c1e',
+ },
+ sampleUrl: {
+ fontSize: 11,
+ color: '#8e8e93',
+ maxWidth: 250,
+ },
+ arrowText: {
+ fontSize: 16,
+ color: '#007aff',
+ fontWeight: '600',
+ },
+ input: {
+ backgroundColor: '#f2f2f7',
+ borderRadius: 8,
+ paddingHorizontal: 12,
+ paddingVertical: 8,
+ fontSize: 14,
+ color: '#1c1c1e',
+ marginBottom: 12,
+ },
+ row: {
+ flexDirection: 'row',
+ alignItems: 'center',
+ },
+ langSelect: {
+ flexDirection: 'row',
+ alignItems: 'center',
+ },
+ inputLabel: {
+ fontSize: 13,
+ color: '#3a3a3c',
+ marginRight: 6,
+ },
+ inputLang: {
+ backgroundColor: '#f2f2f7',
+ borderRadius: 6,
+ paddingHorizontal: 8,
+ paddingVertical: 4,
+ fontSize: 13,
+ color: '#1c1c1e',
+ width: 45,
+ textAlign: 'center',
+ },
+ actionButton: {
+ backgroundColor: '#007aff',
+ borderRadius: 8,
+ paddingVertical: 10,
+ alignItems: 'center',
+ justifyContent: 'center',
+ },
+ disabledButton: {
+ backgroundColor: '#a2a2a7',
+ },
+ buttonText: {
+ color: '#ffffff',
+ fontSize: 14,
+ fontWeight: '600',
+ },
+ resultBox: {
+ fontSize: 16,
+ color: '#3a3a3c',
+ backgroundColor: '#f2f2f7',
+ padding: 12,
+ borderRadius: 8,
+ fontStyle: 'italic',
+ },
+ nonCommittedText: {
+ color: '#8e8e93',
+ },
+ tokensContainer: {
+ flexDirection: 'row',
+ flexWrap: 'wrap',
+ backgroundColor: '#f2f2f7',
+ padding: 12,
+ borderRadius: 8,
+ minHeight: 50,
+ },
+ tokenBadge: {
+ backgroundColor: '#34c759',
+ borderRadius: 6,
+ paddingHorizontal: 8,
+ paddingVertical: 4,
+ marginRight: 6,
+ marginBottom: 6,
+ },
+ tokenText: {
+ color: '#ffffff',
+ fontSize: 12,
+ fontWeight: '600',
+ },
+});
diff --git a/apps/speech/app.json b/apps/speech/app.json
new file mode 100644
index 0000000000..af766a9fb3
--- /dev/null
+++ b/apps/speech/app.json
@@ -0,0 +1,38 @@
+{
+ "expo": {
+ "name": "speech",
+ "slug": "speech",
+ "version": "1.0.0",
+ "orientation": "portrait",
+ "userInterfaceStyle": "light",
+ "newArchEnabled": true,
+ "scheme": "rne-speech",
+ "ios": {
+ "supportsTablet": true,
+ "bundleIdentifier": "com.anonymous.speech",
+ "infoPlist": {
+ "NSMicrophoneUsageDescription": "This app uses the microphone for live speech-to-text transcription."
+ }
+ },
+ "android": {
+ "adaptiveIcon": {
+ "backgroundColor": "#ffffff"
+ },
+ "package": "com.anonymous.speech",
+ "permissions": ["android.permission.RECORD_AUDIO"]
+ },
+ "plugins": [
+ [
+ "expo-build-properties",
+ {
+ "android": {
+ "minSdkVersion": 26
+ },
+ "ios": {
+ "deploymentTarget": "17.0"
+ }
+ }
+ ]
+ ]
+ }
+}
diff --git a/apps/speech/assets/adaptive-icon.png b/apps/speech/assets/adaptive-icon.png
new file mode 100644
index 0000000000..03d6f6b6c6
Binary files /dev/null and b/apps/speech/assets/adaptive-icon.png differ
diff --git a/apps/speech/assets/favicon.png b/apps/speech/assets/favicon.png
new file mode 100644
index 0000000000..e75f697b18
Binary files /dev/null and b/apps/speech/assets/favicon.png differ
diff --git a/apps/speech/assets/icon.png b/apps/speech/assets/icon.png
new file mode 100644
index 0000000000..a0b1526fc7
Binary files /dev/null and b/apps/speech/assets/icon.png differ
diff --git a/apps/speech/assets/splash-icon.png b/apps/speech/assets/splash-icon.png
new file mode 100644
index 0000000000..03d6f6b6c6
Binary files /dev/null and b/apps/speech/assets/splash-icon.png differ
diff --git a/apps/speech/babel.config.js b/apps/speech/babel.config.js
new file mode 100644
index 0000000000..6b2006979c
--- /dev/null
+++ b/apps/speech/babel.config.js
@@ -0,0 +1,7 @@
+module.exports = function (api) {
+ api.cache(true);
+ return {
+ presets: ['babel-preset-expo'],
+ plugins: ['react-native-worklets/plugin'],
+ };
+};
diff --git a/apps/speech/index.ts b/apps/speech/index.ts
new file mode 100644
index 0000000000..5fd059fd06
--- /dev/null
+++ b/apps/speech/index.ts
@@ -0,0 +1,4 @@
+import { registerRootComponent } from 'expo';
+import App from './App';
+
+registerRootComponent(App);
diff --git a/apps/speech/metro.config.js b/apps/speech/metro.config.js
new file mode 100644
index 0000000000..36ba2036f5
--- /dev/null
+++ b/apps/speech/metro.config.js
@@ -0,0 +1,27 @@
+// Learn more https://docs.expo.io/guides/customizing-metro
+const { getDefaultConfig } = require('expo/metro-config');
+const path = require('path');
+
+/** @type {import('expo/metro-config').MetroConfig} */
+const config = getDefaultConfig(__dirname);
+
+const { transformer, resolver } = config;
+
+config.transformer = {
+ ...transformer,
+ babelTransformerPath: require.resolve('react-native-svg-transformer/expo'),
+};
+config.resolver = {
+ ...resolver,
+ assetExts: resolver.assetExts.filter((ext) => ext !== 'svg'),
+ sourceExts: [...resolver.sourceExts, 'svg'],
+};
+
+config.resolver.assetExts.push('pte');
+
+// Watch the react-native-executorch package source directly so edits are
+// reflected immediately without a `yarn prepare` rebuild.
+const pkgSrc = path.resolve(__dirname, '../../packages/react-native-executorch/src');
+config.watchFolders = [...(config.watchFolders ?? []), pkgSrc];
+
+module.exports = config;
diff --git a/apps/speech/package.json b/apps/speech/package.json
new file mode 100644
index 0000000000..dd62677b4d
--- /dev/null
+++ b/apps/speech/package.json
@@ -0,0 +1,40 @@
+{
+ "name": "speech",
+ "version": "1.0.0",
+ "main": "index.ts",
+ "react-native-executorch": {
+ "features": [
+ "speechToText"
+ ]
+ },
+ "scripts": {
+ "start": "expo start",
+ "android": "expo run:android",
+ "ios": "expo run:ios",
+ "typecheck": "tsc",
+ "postinstall": "yarn run -T patch-package --patch-dir ../../patches"
+ },
+ "dependencies": {
+ "expo": "~56.0.9",
+ "expo-build-properties": "~56.0.17",
+ "react": "19.2.3",
+ "react-native": "0.85.3",
+ "react-native-audio-api": "0.12.2",
+ "react-native-executorch": "workspace:*",
+ "react-native-fs": "^2.20.0",
+ "react-native-reanimated": "4.4.0",
+ "react-native-safe-area-context": "~5.7.0",
+ "react-native-svg": "15.15.4",
+ "react-native-worklets": "0.9.1"
+ },
+ "devDependencies": {
+ "@babel/core": "^7.29.0",
+ "@react-native/metro-config": "^0.86.0",
+ "@types/react": "~19.2.0",
+ "babel-preset-expo": "~56.0.14",
+ "react-native-svg-transformer": "^1.5.3",
+ "react-refresh": "^0.14.0",
+ "typescript": "~5.9.2"
+ },
+ "private": true
+}
diff --git a/apps/speech/tsconfig.json b/apps/speech/tsconfig.json
new file mode 100644
index 0000000000..47026ce434
--- /dev/null
+++ b/apps/speech/tsconfig.json
@@ -0,0 +1,15 @@
+{
+ "extends": "../../tsconfig.json",
+ "compilerOptions": {
+ "strict": true,
+ "allowJs": true,
+ "module": "preserve",
+ "moduleDetection": "force",
+ "moduleResolution": "bundler",
+ "customConditions": ["react-native"],
+ "noEmit": true,
+ "paths": {
+ "react-native-executorch": ["../../packages/react-native-executorch/src"]
+ }
+ }
+}
diff --git a/packages/react-native-executorch/cpp/core/dtype.cpp b/packages/react-native-executorch/cpp/core/dtype.cpp
index d43418a7c9..b183daf0d6 100644
--- a/packages/react-native-executorch/cpp/core/dtype.cpp
+++ b/packages/react-native-executorch/cpp/core/dtype.cpp
@@ -9,10 +9,13 @@ DType parseDType(const std::string &s) {
if (s == "int32") {
return DType::int32;
}
+ if (s == "int64") {
+ return DType::int64;
+ }
if (s == "float32") {
return DType::float32;
}
- throw std::invalid_argument("Unsupported dtype: '" + s + "'. Expected 'uint8', 'int32', or 'float32'");
+ throw std::invalid_argument("Unsupported dtype: '" + s + "'. Expected 'uint8', 'int32', 'int64', or 'float32'");
}
std::string toString(DType dtype) {
@@ -21,6 +24,8 @@ std::string toString(DType dtype) {
return "uint8";
case DType::int32:
return "int32";
+ case DType::int64:
+ return "int64";
case DType::float32:
return "float32";
}
@@ -32,6 +37,8 @@ executorch::aten::ScalarType toScalarType(DType dtype) {
return executorch::aten::ScalarType::Byte;
case DType::int32:
return executorch::aten::ScalarType::Int;
+ case DType::int64:
+ return executorch::aten::ScalarType::Long;
case DType::float32:
return executorch::aten::ScalarType::Float;
}
@@ -43,6 +50,8 @@ DType fromScalarType(executorch::aten::ScalarType st) {
return DType::uint8;
case executorch::aten::ScalarType::Int:
return DType::int32;
+ case executorch::aten::ScalarType::Long:
+ return DType::int64;
case executorch::aten::ScalarType::Float:
return DType::float32;
default:
@@ -57,6 +66,8 @@ size_t elementSize(DType dtype) {
// NOLINTNEXTLINE(bugprone-branch-clone): int32 and float32 are both 4 bytes; the identical branches are intentional.
case DType::int32:
return 4;
+ case DType::int64:
+ return 8;
case DType::float32:
return 4;
}
diff --git a/packages/react-native-executorch/cpp/core/dtype.h b/packages/react-native-executorch/cpp/core/dtype.h
index 7259b6b18f..fafa0a8429 100644
--- a/packages/react-native-executorch/cpp/core/dtype.h
+++ b/packages/react-native-executorch/cpp/core/dtype.h
@@ -8,6 +8,7 @@ namespace rnexecutorch::core::types {
enum class DType {
uint8,
int32,
+ int64,
float32
};
diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp
index 9e5ca8d9c7..3337d4d2b7 100644
--- a/packages/react-native-executorch/cpp/core/model.cpp
+++ b/packages/react-native-executorch/cpp/core/model.cpp
@@ -1,23 +1,33 @@
#include "model.h"
+#include
#include
+#include
+#include
#include
#include
+#include
+#include
+#include
#include
#include
+#include
#include "dtype.h"
#include "tensor_helpers.h"
#include
+#include
#include
#include
+#include
#include
namespace {
namespace jsi = facebook::jsi;
namespace types = rnexecutorch::core::types;
+namespace tensor = rnexecutorch::core::tensor;
template
T unwrap(jsi::Runtime &rt, const std::string &ctx, executorch::runtime::Result result) {
@@ -27,6 +37,14 @@ T unwrap(jsi::Runtime &rt, const std::string &ctx, executorch::runtime::Result
+T unwrap(const std::string &ctx, executorch::runtime::Result result) {
+ if (!result.ok()) {
+ throw std::runtime_error(std::format("{}: {}", ctx, executorch::runtime::to_string(result.error())));
+ }
+ return std::move(result.get());
+}
+
types::DType
fromScalarType(jsi::Runtime &rt, const std::string &ctx, executorch::aten::ScalarType scalarType) {
try {
@@ -57,6 +75,128 @@ jsi::Object tensorMetaToJs(jsi::Runtime &rt, const executorch::runtime::TensorIn
return jsTensorMeta;
}
+
+/**
+ * Parses compile-time dynamic dimension constraints for the inputs of a module
+ * method.
+ *
+ * @note This is a temporary workaround. ExecuTorch (.pte) model metadata
+ * natively serializes only the static upper-bound limits of dynamic/symbolic
+ * dimensions, and does not expose the active dynamic range (min, max, step) at
+ * runtime.
+ *
+ * To use this feature, the .pte model must be exported with a companion method
+ * named "get_dynamic_dims_" (e.g., "get_dynamic_dims_forward").
+ *
+ * Python export requirements:
+ * 1. The companion method must take no arguments.
+ * 2. It must return a list of outputs matching the number of Tag::Tensor inputs
+ * of the target method (scalar inputs are excluded).
+ * 3. Each output must be a 2D int32 tensor of shape [rank, 3], where each row
+ * represents [min, max, step] constraints for the corresponding dimension of
+ * that input tensor.
+ *
+ * @param module The ExecuTorch extension module to query.
+ * @param methodName The name of the target module method (e.g. "forward").
+ * @return A vector of SymbolicShape objects, or std::nullopt if the companion
+ * method is not defined. If returned, the vector's size is guaranteed
+ * to equal methodMeta.num_inputs(), containing parsed SymbolicShapes
+ * for tensor inputs and empty SymbolicShapes for non-tensor inputs.
+ */
+std::optional>
+parseDynamicInputShapes(executorch::extension::Module &module, const std::string &methodName) {
+ using executorch::aten::ScalarType;
+
+ const auto getDynamicShapesMethodName = std::format("get_dynamic_dims_{}", methodName);
+ const auto ctx = getDynamicShapesMethodName + ": ";
+
+ auto methodNames = unwrap(ctx + "failed to get method names", module.method_names());
+ if (methodName == getDynamicShapesMethodName ||
+ !methodNames.contains(getDynamicShapesMethodName)) {
+ return std::nullopt;
+ }
+
+ auto methodMeta = unwrap(std::format("{}failed to get meta for method '{}'", ctx, methodName),
+ module.method_meta(methodName));
+
+ size_t expectedTensorInputs = 0;
+ for (size_t i = 0; i < methodMeta.num_inputs(); ++i) {
+ auto tag = unwrap(std::format("{}failed to get tag for input [{}]", ctx, i), methodMeta.input_tag(i));
+ if (tag == executorch::runtime::Tag::Tensor) {
+ expectedTensorInputs++;
+ }
+ }
+
+ auto result = unwrap(ctx + "failed to execute", module.execute(getDynamicShapesMethodName));
+ if (result.size() != expectedTensorInputs) {
+ throw std::runtime_error(std::format("{}number of outputs returned ({}) does not match the number of "
+ "tensor inputs declared by method '{}' ({})",
+ ctx, result.size(), methodName, expectedTensorInputs));
+ }
+
+ std::vector dynamicShapes;
+ dynamicShapes.reserve(methodMeta.num_inputs());
+ size_t tensorIndex = 0;
+
+ for (size_t i = 0; i < methodMeta.num_inputs(); ++i) {
+ auto tag = unwrap(std::format("{}failed to get tag for input [{}]", ctx, i),
+ methodMeta.input_tag(i));
+
+ if (tag != executorch::runtime::Tag::Tensor) {
+ // Emplace an empty shape for non-tensor inputs to maintain a 1-to-1
+ // index alignment between dynamicShapes and the model's inputs vector.
+ dynamicShapes.emplace_back();
+ continue;
+ }
+
+ const auto &out = result.at(tensorIndex);
+ if (!out.isTensor()) {
+ throw std::runtime_error(std::format("{}output[{}] is not a tensor", ctx, tensorIndex));
+ }
+
+ auto inputMeta = unwrap(std::format("{}failed to get tensor meta for input [{}]", ctx, i),
+ methodMeta.input_tensor_meta(i));
+ const auto rank = inputMeta.sizes().size();
+ const auto shapeTensor = out.toTensor();
+
+ if (
+ shapeTensor.dim() != 2 ||
+ shapeTensor.size(1) != 3 ||
+ shapeTensor.size(0) != static_cast(rank) ||
+ shapeTensor.scalar_type() != ScalarType::Int) {
+ throw std::runtime_error(std::format("{}output[{}] expected to be a 2D int32_t tensor of shape [{}, 3]",
+ ctx, tensorIndex, rank));
+ }
+
+ const auto *shape = shapeTensor.const_data_ptr();
+ tensor::SymbolicShape symbolicShape;
+ symbolicShape.reserve(rank);
+
+ for (size_t axis = 0; axis < rank; ++axis) {
+ const auto minDim = shape[axis * 3 + 0];
+ const auto maxDim = shape[axis * 3 + 1];
+ const auto step = shape[axis * 3 + 2];
+ if (minDim < 0 || maxDim < minDim || step < 1) {
+ throw std::runtime_error(std::format("{}output[{}], axis {} is invalid: "
+ "expected 0 <= min <= max and step >= 1 but got [{}, {}, {}]",
+ ctx, tensorIndex, axis, minDim, maxDim, step));
+ }
+ if (maxDim > inputMeta.sizes()[axis]) {
+ throw std::runtime_error(std::format("{}output[{}], axis {} max dimension ({}) "
+ "exceeds model metadata upper limit ({})",
+ ctx, tensorIndex, axis, maxDim, inputMeta.sizes()[axis]));
+ }
+
+ symbolicShape.emplace_back(tensor::RangeDim{.min = minDim, .max = maxDim, .step = step});
+ }
+
+ dynamicShapes.push_back(std::move(symbolicShape));
+ ++tensorIndex;
+ }
+
+ return dynamicShapes;
+}
+
} // namespace
namespace rnexecutorch::core::model {
@@ -68,11 +208,20 @@ using rnexecutorch::core::tensor::TensorHostObject;
ModelHostObject::ModelHostObject(const std::string &modelPath)
: modelPath_(modelPath),
etModule_(std::make_unique(modelPath)) {
+
auto error = etModule_->load();
if (!etModule_->is_loaded()) {
const std::string errorMsg = executorch::runtime::to_string(error);
throw std::runtime_error(std::format("Failed to load model: {}", errorMsg));
}
+
+ auto methodNames = unwrap("Failed to get method names", etModule_->method_names());
+ for (const auto &methodName : methodNames) {
+ auto dynamicShapes = parseDynamicInputShapes(*etModule_, methodName);
+ if (dynamicShapes) {
+ dynamicInputShapes_.emplace(methodName, std::move(*dynamicShapes));
+ }
+ }
}
jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) {
@@ -223,7 +372,14 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) {
case executorch::runtime::Tag::Tensor: {
auto tensorMeta = unwrap(rt, ctx + ": tensor meta", methodMeta.input_tensor_meta(i));
auto expectedDtype = fromScalarType(rt, ctx, tensorMeta.scalar_type());
- auto tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype, tensorMeta.sizes());
+
+ std::shared_ptr tensorHostObject;
+ if (self->dynamicInputShapes_.contains(methodName)) {
+ auto expectedShape = self->dynamicInputShapes_[methodName][i];
+ tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype, expectedShape);
+ } else {
+ tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype, tensorMeta.sizes());
+ }
if (!lockedTensors.insert(tensorHostObject.get()).second) {
throw jsi::JSError(rt, "execute: Tensor aliasing detected. "
@@ -285,7 +441,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) {
auto tensorMeta = unwrap(rt, ctx + ": tensor meta", methodMeta.output_tensor_meta(index));
auto expectedDtype = fromScalarType(rt, ctx, tensorMeta.scalar_type());
- auto tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype, tensorMeta.sizes());
+ auto tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype, output.toTensor().sizes());
if (!lockedTensors.insert(tensorHostObject.get()).second) {
throw jsi::JSError(rt, "execute: Tensor aliasing detected. "
diff --git a/packages/react-native-executorch/cpp/core/model.h b/packages/react-native-executorch/cpp/core/model.h
index af8d23dd61..a2c2cf833e 100644
--- a/packages/react-native-executorch/cpp/core/model.h
+++ b/packages/react-native-executorch/cpp/core/model.h
@@ -1,14 +1,19 @@
#pragma once
+#include
+#include
#include
#include
#include
+#include
#include
#include
#include
+#include "core/tensor_helpers.h"
+
namespace rnexecutorch::core::model {
namespace jsi = facebook::jsi;
/**
@@ -31,6 +36,8 @@ class ModelHostObject : public jsi::HostObject,
std::string modelPath_;
std::unique_ptr etModule_;
std::mutex mutex_;
+
+ std::unordered_map> dynamicInputShapes_;
};
void install_loadModel(jsi::Runtime &rt, jsi::Object &module);
diff --git a/packages/react-native-executorch/cpp/extensions/cv/utils.h b/packages/react-native-executorch/cpp/extensions/cv/utils.h
index a29961d0b6..710124b1d9 100644
--- a/packages/react-native-executorch/cpp/extensions/cv/utils.h
+++ b/packages/react-native-executorch/cpp/extensions/cv/utils.h
@@ -14,6 +14,8 @@ inline int dtypeToCvDepth(rnexecutorch::core::types::DType dtype) {
return CV_32S;
case rnexecutorch::core::types::DType::float32:
return CV_32F;
+ case rnexecutorch::core::types::DType::int64:
+ break;
}
throw std::invalid_argument("unsupported dtype");
}
diff --git a/packages/react-native-executorch/src/core/tensor.ts b/packages/react-native-executorch/src/core/tensor.ts
index 8f5c58e10c..4c54c0d32f 100644
--- a/packages/react-native-executorch/src/core/tensor.ts
+++ b/packages/react-native-executorch/src/core/tensor.ts
@@ -6,7 +6,7 @@ declare const tensorBrand: unique symbol;
* Element data type of a {@link Tensor}.
* @category Types
*/
-export type DType = 'float32' | 'uint8' | 'int32';
+export type DType = 'float32' | 'uint8' | 'int32' | 'int64';
/**
* A native ExecuTorch tensor allocated in C++ memory.
@@ -50,10 +50,10 @@ export type Tensor = {
/**
* Writes data from a typed array into this tensor's native buffer.
* @param src The source typed array. Its size in bytes must match the
- * tensor's size.
+ * tensor's size. Use a `BigInt64Array` for `int64` tensors.
* @returns `this` tensor.
*/
- setData(src: Float32Array | Uint8Array | Int32Array): Tensor;
+ setData(src: Float32Array | Uint8Array | Int32Array | BigInt64Array): Tensor;
/**
* Copies data out of this tensor's native buffer into a typed array.
@@ -62,7 +62,7 @@ export type Tensor = {
* tensor's size.
* @returns The same `dst` array, now filled with tensor data.
*/
- getData(dst: T): T;
+ getData(dst: T): T;
/**
* Passes `this` tensor as the first argument to `fn` and returns the result.
@@ -115,7 +115,7 @@ export type Tensor = {
export function tensor(
dtype: DType,
shape: number[],
- src?: Float32Array | Uint8Array | Int32Array
+ src?: Float32Array | Uint8Array | Int32Array | BigInt64Array
): Tensor {
'worklet';
const t: Tensor = rnexecutorchJsi.createTensor(shape, dtype);
diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbeddings.ts b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbeddings.ts
new file mode 100644
index 0000000000..65274c6494
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbeddings.ts
@@ -0,0 +1,88 @@
+import type { WorkletRuntime } from 'react-native-worklets';
+
+import { tensor } from '../../../core/tensor';
+import { loadModel } from '../../../core/model';
+import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
+import { wrapAsync } from '../../../core/runtime';
+
+import type { ImageBuffer } from '../image';
+import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing';
+
+/**
+ * Model configuration required to instantiate an image embeddings task runner.
+ * @category Types
+ */
+export type ImageEmbeddingsModel = {
+ readonly modelPath: string;
+ readonly opts: ImagePreprocessorOptions;
+};
+
+/**
+ * Creates an image embeddings runner for executing local Image Embedding
+ * models (e.g. the image encoder of a CLIP model).
+ *
+ * It validates the model input and output requirements, pre-allocates the
+ * static execution tensors, sets up an image preprocessor, and registers clean
+ * disposal hooks to clear all native memory. Pooling and normalization (if any)
+ * are baked into the exported `.pte`; this runner simply preprocesses the image,
+ * runs the forward pass, and returns the raw embedding vector.
+ * @category Typescript API
+ * @param config Image embeddings task configuration containing path and options.
+ * @param runtime Optional worklet runtime thread on which to run the model
+ * execution.
+ * @returns A promise resolving to an object containing the embedding and
+ * disposal controls.
+ */
+export async function createImageEmbeddings(
+ config: ImageEmbeddingsModel,
+ runtime?: WorkletRuntime
+): Promise<{
+ /**
+ * Releases all allocated native resources.
+ */
+ dispose: () => void;
+ /**
+ * Asynchronously computes the embedding vector for the given input image.
+ * @param input The input image buffer.
+ * @returns A promise resolving to the embedding vector.
+ */
+ forward: (input: ImageBuffer) => Promise;
+ /**
+ * Synchronous version of {@link forward} to be executed directly on the
+ * caller or worklet thread.
+ */
+ forwardWorklet: (input: ImageBuffer) => Float32Array;
+}> {
+ const { modelPath, opts } = config;
+ const model = await wrapAsync(loadModel, runtime)(modelPath);
+
+ const meta = validateModelSchema(
+ model,
+ 'forward',
+ [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])],
+ [SymbolicTensor('float32', [1, 'D'], ['D'])]
+ );
+ const inpShape = meta.inputTensorMeta[0]!.shape;
+ const outShape = meta.outputTensorMeta[0]!.shape;
+
+ const tensors = [tensor('float32', outShape)] as const;
+ const [tEmbedding] = tensors;
+ const preprocessor = createImagePreprocessor(opts, inpShape);
+
+ const dispose = () => {
+ preprocessor.dispose();
+ tensors.forEach((t) => t.dispose());
+ model.dispose();
+ };
+
+ const forwardWorklet = (input: ImageBuffer): Float32Array => {
+ 'worklet';
+ const tInput = preprocessor.process(input);
+ model.execute('forward', [tInput], [tEmbedding]);
+ return tEmbedding.getData(new Float32Array(tEmbedding.numel));
+ };
+
+ const forward = wrapAsync(forwardWorklet, runtime);
+
+ return { forward, forwardWorklet, dispose };
+}
diff --git a/packages/react-native-executorch/src/extensions/nlp/index.ts b/packages/react-native-executorch/src/extensions/nlp/index.ts
index 6195f07395..8d3975ad9c 100644
--- a/packages/react-native-executorch/src/extensions/nlp/index.ts
+++ b/packages/react-native-executorch/src/extensions/nlp/index.ts
@@ -1,2 +1,3 @@
export * from './tokenizer';
export * from './tasks/tokenization';
+export * from './tasks/textEmbeddings';
diff --git a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbeddings.ts b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbeddings.ts
new file mode 100644
index 0000000000..a2bc28fa5e
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbeddings.ts
@@ -0,0 +1,119 @@
+import type { WorkletRuntime } from 'react-native-worklets';
+
+import { tensor } from '../../../core/tensor';
+import { loadModel } from '../../../core/model';
+import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
+import { wrapAsync } from '../../../core/runtime';
+
+import { loadTokenizer } from '../tokenizer';
+
+/**
+ * Model configuration required to instantiate a text embeddings task runner.
+ * @category Types
+ */
+export type TextEmbeddingsModel = {
+ readonly modelPath: string;
+ readonly tokenizerPath: string;
+};
+
+/**
+ * Creates a text embeddings runner for executing local Text Embedding models
+ * (e.g. sentence-transformers like all-MiniLM-L6-v2).
+ *
+ * It loads the tokenizer and model, validates the model input and output
+ * requirements, pre-allocates the static execution tensors, and registers clean
+ * disposal hooks to clear all native memory. The input text is tokenized and fed
+ * at its exact token length (no padding), truncated only when it exceeds the
+ * model's maximum sequence length; the attention mask is all ones. Pooling and
+ * normalization are baked into the exported `.pte`; this runner runs the forward
+ * pass and returns the raw embedding vector.
+ * @category Typescript API
+ * @param config Text embeddings task configuration containing the model and
+ * tokenizer paths.
+ * @param runtime Optional worklet runtime thread on which to run the model
+ * execution.
+ * @returns A promise resolving to an object containing the embedding and
+ * disposal controls.
+ */
+export async function createTextEmbeddings(
+ config: TextEmbeddingsModel,
+ runtime?: WorkletRuntime
+): Promise<{
+ /**
+ * Releases all allocated native resources.
+ */
+ dispose: () => void;
+ /**
+ * Asynchronously computes the embedding vector for the given input text.
+ * @param input The input text to embed.
+ * @returns A promise resolving to the embedding vector.
+ */
+ forward: (input: string) => Promise;
+ /**
+ * Synchronous version of {@link forward} to be executed directly on the
+ * caller or worklet thread.
+ */
+ forwardWorklet: (input: string) => Float32Array;
+}> {
+ const { modelPath, tokenizerPath } = config;
+ const [model, tokenizer] = await Promise.all([
+ wrapAsync(loadModel, runtime)(modelPath),
+ wrapAsync(loadTokenizer, runtime)(tokenizerPath),
+ ]);
+
+ // Text embedding models take two int64 inputs: the token ids and the
+ // attention mask, both of shape [1, sequence_length].
+ const meta = validateModelSchema(
+ model,
+ 'forward',
+ [SymbolicTensor('int64', [1, 'L']), SymbolicTensor('int64', [1, 'L'])],
+ [SymbolicTensor('float32', [1, 'D'], ['D'])]
+ );
+ // The models are exported with a dynamic sequence dimension; the declared size
+ // is the upper bound, used only to truncate over-long inputs.
+ const maxSeqLen = meta.inputTensorMeta[0]!.shape[1]!;
+ const outShape = meta.outputTensorMeta[0]!.shape;
+
+ const tensors = [tensor('float32', outShape)] as const;
+ const [tEmbedding] = tensors;
+
+ const dispose = () => {
+ tensors.forEach((t) => t.dispose());
+ tokenizer.dispose();
+ model.dispose();
+ };
+
+ const forwardWorklet = (input: string): Float32Array => {
+ 'worklet';
+ const ids = tokenizer.encode(input);
+ if (ids.length === 0) {
+ throw new Error('createTextEmbeddings: input tokenized to zero tokens');
+ }
+ const len = Math.min(ids.length, maxSeqLen);
+
+ // Feed the exact token length with no padding. The model resizes its dynamic
+ // sequence input to match. Padding would change the result for pooling heads
+ // that are sensitive to it (e.g. DistilUSE's tanh projection). The attention
+ // mask is all ones since every position is a real token.
+ const idsData = new BigInt64Array(len);
+ const maskData = new BigInt64Array(len);
+ for (let i = 0; i < len; i++) {
+ idsData[i] = BigInt(ids[i]!);
+ maskData[i] = 1n;
+ }
+
+ const tokenIds = tensor('int64', [1, len], idsData);
+ const attentionMask = tensor('int64', [1, len], maskData);
+ try {
+ model.execute('forward', [tokenIds, attentionMask], [tEmbedding]);
+ return tEmbedding.getData(new Float32Array(tEmbedding.numel));
+ } finally {
+ tokenIds.dispose();
+ attentionMask.dispose();
+ }
+ };
+
+ const forward = wrapAsync(forwardWorklet, runtime);
+
+ return { forward, forwardWorklet, dispose };
+}
diff --git a/packages/react-native-executorch/src/extensions/speech/index.ts b/packages/react-native-executorch/src/extensions/speech/index.ts
new file mode 100644
index 0000000000..eed42f8da9
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/speech/index.ts
@@ -0,0 +1 @@
+export * from './tasks/whisperSpeechToText';
diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts
new file mode 100644
index 0000000000..e6f700ad85
--- /dev/null
+++ b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts
@@ -0,0 +1,222 @@
+import type { WorkletRuntime } from 'react-native-worklets';
+import { scheduleOnRN } from 'react-native-worklets';
+
+import { tensor } from '../../../core/tensor';
+import { loadModel } from '../../../core/model';
+import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema';
+import { wrapAsync } from '../../../core/runtime';
+import { loadTokenizer } from '../../nlp/tokenizer';
+import { argmax } from '../../../extensions/math';
+
+export const WHISPER_SAMPLE_RATE_HZ = 16000;
+
+const WHISPER_MAX_SEQ_LEN = 128;
+const MIN_CHUNK_SIZE = 201;
+const CHUNK_LENGTH_SECONDS = 29;
+const CHUNK_SIZE = CHUNK_LENGTH_SECONDS * WHISPER_SAMPLE_RATE_HZ;
+const BUFFER_SIZE = 5 * WHISPER_SAMPLE_RATE_HZ;
+const STRIDE_SIZE = WHISPER_SAMPLE_RATE_HZ; // 1-second hop between windows
+
+// prettier-ignore
+export const WHISPER_LANGUAGES = [
+ 'en', 'zh', 'de', 'es', 'ru', 'ko', 'fr', 'ja', 'pt', 'tr',
+ 'pl', 'ca', 'nl', 'ar', 'sv', 'it', 'id', 'hi', 'fi', 'vi',
+ 'he', 'uk', 'el', 'ms', 'cs', 'ro', 'da', 'hu', 'ta', 'no',
+ 'th', 'ur', 'hr', 'bg', 'lt', 'la', 'mi', 'ml', 'cy', 'sk',
+ 'te', 'fa', 'lv', 'bn', 'sr', 'az', 'sl', 'kn', 'et', 'mk',
+ 'br', 'eu', 'is', 'hy', 'ne', 'mn', 'bs', 'kk', 'sq', 'sw',
+ 'gl', 'mr', 'pa', 'si', 'km', 'sn', 'yo', 'so', 'af', 'oc',
+ 'ka', 'be', 'tg', 'sd', 'gu', 'am', 'yi', 'lo', 'uz', 'fo',
+ 'ht', 'ps', 'tk', 'nn', 'mt', 'sa', 'lb', 'my', 'bo', 'tl',
+ 'mg', 'as', 'tt', 'haw', 'ln', 'ha', 'ba', 'jw', 'su', 'yue'
+] as const;
+
+export type WhisperLanguage = (typeof WHISPER_LANGUAGES)[number];
+
+export type WhisperOptions = { readonly language: L };
+
+export type WhisperModel = {
+ readonly modelPath: string;
+ readonly tokenizerPath: string;
+ readonly supportedLanguages: readonly L[];
+};
+
+export async function createWhisperSpeechToText(
+ config: WhisperModel,
+ runtime?: WorkletRuntime
+): Promise<{
+ dispose: () => void;
+
+ transcribe: (
+ audio: Float32Array,
+ options: WhisperOptions,
+ onToken?: (token: string) => void
+ ) => Promise;
+
+ transcribeWorklet: (
+ audio: Float32Array,
+ options: WhisperOptions,
+ onToken?: (token: string) => void
+ ) => string;
+
+ stream: (options: WhisperOptions) => AsyncGenerator;
+
+ streamStop: () => void;
+
+ streamInsert: (chunk: Float32Array) => void;
+}> {
+ const { modelPath, tokenizerPath, supportedLanguages } = config;
+ const model = await wrapAsync(loadModel, runtime)(modelPath);
+ const tokenizer = await wrapAsync(loadTokenizer, runtime)(tokenizerPath);
+
+ const eotToken = tokenizer.tokenToId('<|endoftext|>')!;
+ const isEnglishOnly = supportedLanguages.length === 1 && supportedLanguages[0] === 'en';
+
+ const encMeta = validateModelSchema(
+ model,
+ 'encode',
+ [SymbolicTensor('float32', ['T_audio'])],
+ [SymbolicTensor('float32', [1, 'Seq', 'State'])]
+ );
+
+ const encSeqLen = encMeta.outputTensorMeta[0]!.shape[1]!;
+ const encStateDim = encMeta.outputTensorMeta[0]!.shape[2]!;
+
+ validateModelSchema(
+ model,
+ 'decode',
+ [
+ SymbolicTensor('int64', [1, 'Tokens']),
+ SymbolicTensor('int64', ['Tokens']),
+ SymbolicTensor('float32', [1, encSeqLen, encStateDim]),
+ ],
+ [SymbolicTensor('float32', [1, 'Tokens', 'Vocab'])]
+ );
+
+ const tensors = [
+ tensor('int64', [1]),
+ tensor('int64', [1, 1]),
+ tensor('int32', [1, 1, 1]),
+ tensor('float32', [1, encSeqLen, encStateDim]),
+ tensor('float32', [1, 1, tokenizer.getVocabSize()]),
+ ] as const;
+
+ const [tPosition, tToken, tArgmax, tEncodings, tLogits] = tensors;
+
+ const dispose = () => {
+ tensors.forEach((t) => t.dispose());
+ tokenizer.dispose();
+ model.dispose();
+ };
+
+ const decode = (token: number, position: number): number => {
+ 'worklet';
+ tPosition.setData(new BigInt64Array([BigInt(position)]));
+ tToken.setData(new BigInt64Array([BigInt(token)]));
+ model.execute('decode', [tToken, tPosition, tEncodings], [tLogits]);
+ return tLogits.through(argmax, tArgmax).getData(new Int32Array(1))[0]!;
+ };
+
+ const transcribeWorklet = (
+ audio: Float32Array,
+ options: WhisperOptions,
+ onToken?: (token: string) => void
+ ): string => {
+ 'worklet';
+
+ const promptTokenStrings = isEnglishOnly
+ ? ['<|startoftranscript|>', '<|notimestamps|>']
+ : ['<|startoftranscript|>', `<|${options.language}|>`, '<|transcribe|>', '<|notimestamps|>'];
+ const promptTokens = promptTokenStrings.map((token) => tokenizer.tokenToId(token)!);
+ const maxNewTokens = WHISPER_MAX_SEQ_LEN - promptTokens.length;
+
+ let text = '';
+ let offset = 0;
+
+ while (offset < audio.length) {
+ const audioChunk = audio.slice(offset, Math.min(offset + CHUNK_SIZE, audio.length));
+ if (audioChunk.length < MIN_CHUNK_SIZE) {
+ break;
+ }
+
+ const tAudioInput = tensor('float32', [audioChunk.length], audioChunk);
+ try {
+ model.execute('encode', [tAudioInput], [tEncodings]);
+ } finally {
+ tAudioInput.dispose();
+ }
+
+ let nextToken = eotToken;
+ let position = promptTokens.length;
+
+ promptTokens.forEach((token, pos) => (nextToken = decode(token, pos)));
+ const generated: number[] = [];
+
+ while (generated.length < maxNewTokens && nextToken !== eotToken) {
+ generated.push(nextToken);
+ if (onToken) {
+ scheduleOnRN(onToken, tokenizer.decode([nextToken]));
+ }
+ nextToken = decode(nextToken, position);
+ position++;
+ }
+
+ text += tokenizer.decode(generated);
+ offset += CHUNK_SIZE;
+ }
+
+ return text.trim();
+ };
+
+ const transcribe = wrapAsync(transcribeWorklet, runtime);
+
+ // ======================================================
+ // Streaming
+ // ======================================================
+ let isStreaming = false;
+ let audioBuffer = new Float32Array(0);
+ let signal: (() => void) | null = null;
+
+ const streamInsert = (audioChunk: Float32Array): void => {
+ const next = new Float32Array(audioBuffer.length + audioChunk.length);
+ next.set(audioBuffer);
+ next.set(audioChunk, audioBuffer.length);
+ audioBuffer = next;
+ signal?.();
+ signal = null;
+ };
+
+ const streamStop = (): void => {
+ isStreaming = false;
+ signal?.();
+ signal = null;
+ };
+
+ async function* stream(options: WhisperOptions): AsyncGenerator {
+ isStreaming = true;
+ audioBuffer = new Float32Array(0);
+ let transcription = '';
+ let lastTranscribedSize = 0;
+
+ while (isStreaming) {
+ if (audioBuffer.length - lastTranscribedSize < STRIDE_SIZE) {
+ await new Promise((resolve) => (signal = resolve));
+ continue;
+ }
+
+ const audio = audioBuffer.slice(0, Math.min(audioBuffer.length, BUFFER_SIZE));
+
+ if (audioBuffer.length >= BUFFER_SIZE) {
+ audioBuffer = audioBuffer.slice(STRIDE_SIZE);
+ }
+ lastTranscribedSize = audioBuffer.length;
+
+ const text = await transcribe(audio, options);
+ transcription += !text.startsWith('[BLANK_AUDIO]') ? text + ' ' : '';
+ console.log(text);
+ yield transcription;
+ }
+ }
+
+ return { dispose, transcribe, transcribeWorklet, stream, streamInsert, streamStop };
+}
diff --git a/packages/react-native-executorch/src/hooks/useImageEmbeddings.ts b/packages/react-native-executorch/src/hooks/useImageEmbeddings.ts
new file mode 100644
index 0000000000..476bd5c64b
--- /dev/null
+++ b/packages/react-native-executorch/src/hooks/useImageEmbeddings.ts
@@ -0,0 +1,45 @@
+import { useModel } from './useModel';
+import { useResourceDownload } from './useResourceDownload';
+import {
+ createImageEmbeddings,
+ type ImageEmbeddingsModel,
+} from '../extensions/cv/tasks/imageEmbeddings';
+
+/**
+ * React hook to load and run an image embeddings model.
+ *
+ * This hook manages downloading (if it's a remote URL) and loading the model
+ * file, compiling it, tracking download progress and compilation errors, and
+ * cleaning up native model memory when the component unmounts or configuration
+ * changes.
+ * @category Hooks
+ * @param config The image embeddings model configuration.
+ * @param options Hook options.
+ * @param options.preventLoad If true, prevents downloading and compiling the
+ * model.
+ * @returns An object containing the model's loading state, error, download
+ * progress, and embedding functions.
+ */
+export function useImageEmbeddings(
+ config: ImageEmbeddingsModel,
+ options?: { preventLoad?: boolean }
+) {
+ const { localPath, downloadProgress, downloadError } = useResourceDownload(
+ config.modelPath,
+ options?.preventLoad
+ );
+ const { model, error } = useModel(
+ createImageEmbeddings,
+ localPath ? { ...config, modelPath: localPath } : null,
+ [localPath]
+ );
+
+ return {
+ isReady: !!model,
+ error: downloadError || error,
+ downloadProgress,
+ localPath,
+ forward: model?.forward,
+ forwardWorklet: model?.forwardWorklet,
+ };
+}
diff --git a/packages/react-native-executorch/src/hooks/useSpeechToText.ts b/packages/react-native-executorch/src/hooks/useSpeechToText.ts
new file mode 100644
index 0000000000..252bfb2068
--- /dev/null
+++ b/packages/react-native-executorch/src/hooks/useSpeechToText.ts
@@ -0,0 +1,39 @@
+import { useModel } from './useModel';
+import { useResourceDownload } from './useResourceDownload';
+import {
+ createWhisperSpeechToText,
+ type WhisperModel,
+ type WhisperLanguage,
+} from '../extensions/speech/tasks/whisperSpeechToText';
+
+export function useSpeechToText(
+ config: WhisperModel,
+ options?: { preventLoad?: boolean }
+) {
+ const modelResource = useResourceDownload(config.modelPath, options?.preventLoad);
+ const tokenizerResource = useResourceDownload(config.tokenizerPath, options?.preventLoad);
+
+ const localModelPath = modelResource.localPath;
+ const localTokenizerPath = tokenizerResource.localPath;
+
+ const whisperConfig =
+ localModelPath && localTokenizerPath
+ ? { ...config, modelPath: localModelPath, tokenizerPath: localTokenizerPath }
+ : null;
+
+ const { model, error } = useModel(createWhisperSpeechToText, whisperConfig, [
+ localModelPath,
+ localTokenizerPath,
+ ]);
+
+ return {
+ isReady: !!model,
+ error: modelResource.downloadError || tokenizerResource.downloadError || error,
+ downloadProgress: modelResource.downloadProgress,
+ transcribe: model?.transcribe,
+ transcribeWorklet: model?.transcribeWorklet,
+ stream: model?.stream,
+ streamInsert: model?.streamInsert,
+ streamStop: model?.streamStop,
+ };
+}
diff --git a/packages/react-native-executorch/src/hooks/useTextEmbeddings.ts b/packages/react-native-executorch/src/hooks/useTextEmbeddings.ts
new file mode 100644
index 0000000000..588ebda012
--- /dev/null
+++ b/packages/react-native-executorch/src/hooks/useTextEmbeddings.ts
@@ -0,0 +1,51 @@
+import { useModel } from './useModel';
+import { useResourceDownload } from './useResourceDownload';
+import {
+ createTextEmbeddings,
+ type TextEmbeddingsModel,
+} from '../extensions/nlp/tasks/textEmbeddings';
+
+/**
+ * React hook to load and run a text embeddings model.
+ *
+ * This hook manages downloading (if they are remote URLs) and loading both the
+ * model file and its `tokenizer.json`, tracking download progress and errors,
+ * and cleaning up native memory when the component unmounts or the configuration
+ * changes.
+ * @category Hooks
+ * @param config The text embeddings model configuration (model and tokenizer
+ * paths).
+ * @param options Hook options.
+ * @param options.preventLoad If true, prevents downloading and compiling the
+ * model.
+ * @returns An object containing the model's loading state, error, download
+ * progress, and embedding functions.
+ */
+export function useTextEmbeddings(
+ config: TextEmbeddingsModel,
+ options?: { preventLoad?: boolean }
+) {
+ const modelResource = useResourceDownload(config.modelPath, options?.preventLoad);
+ const tokenizerResource = useResourceDownload(config.tokenizerPath, options?.preventLoad);
+
+ const localModelPath = modelResource.localPath;
+ const localTokenizerPath = tokenizerResource.localPath;
+
+ const { model, error } = useModel(
+ createTextEmbeddings,
+ localModelPath && localTokenizerPath
+ ? { modelPath: localModelPath, tokenizerPath: localTokenizerPath }
+ : null,
+ [localModelPath, localTokenizerPath]
+ );
+
+ return {
+ isReady: !!model,
+ error: modelResource.downloadError || tokenizerResource.downloadError || error,
+ downloadProgress: (modelResource.downloadProgress + tokenizerResource.downloadProgress) / 2,
+ localPath: localModelPath,
+ tokenizerPath: localTokenizerPath,
+ forward: model?.forward,
+ forwardWorklet: model?.forwardWorklet,
+ };
+}
diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts
index 00557be78b..eea6bb4935 100644
--- a/packages/react-native-executorch/src/index.ts
+++ b/packages/react-native-executorch/src/index.ts
@@ -6,8 +6,11 @@ export * from './hooks/useInstanceSegmenter';
export * from './hooks/useKeypointDetector';
export * from './hooks/useObjectDetector';
export * from './hooks/useTokenizer';
+export * from './hooks/useTextEmbeddings';
+export * from './hooks/useImageEmbeddings';
export * from './hooks/useResourceDownload';
export * from './hooks/useModel';
+export * from './hooks/useSpeechToText';
// Constants
export { models } from './models';
@@ -20,7 +23,10 @@ export * from './extensions/cv/tasks/semanticSegmentation';
export * from './extensions/cv/tasks/instanceSegmentation';
export * from './extensions/cv/tasks/keypointDetection';
export * from './extensions/cv/tasks/objectDetection';
+export * from './extensions/cv/tasks/imageEmbeddings';
export * from './extensions/nlp/tasks/tokenization';
+export * from './extensions/nlp/tasks/textEmbeddings';
+export * from './extensions/speech/tasks/whisperSpeechToText';
// Core primitives — for library builders and power users
export { tensor } from './core/tensor';
@@ -44,6 +50,7 @@ export { defaultWorkletRuntime, wrapAsync } from './core/runtime';
export * as math from './extensions/math';
export * as cv from './extensions/cv';
export * as nlp from './extensions/nlp';
+export * as speech from './extensions/speech';
// Utils
export * from './utils';
diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts
index 3397288e23..eaf0c03f4c 100644
--- a/packages/react-native-executorch/src/models.ts
+++ b/packages/react-native-executorch/src/models.ts
@@ -4,6 +4,12 @@ import type { StyleTransferModel } from './extensions/cv/tasks/styleTransfer';
import type { SemanticSegmentationModel } from './extensions/cv/tasks/semanticSegmentation';
import type { KeypointDetectorModel } from './extensions/cv/tasks/keypointDetection';
import type { InstanceSegmenterModel } from './extensions/cv/tasks/instanceSegmentation';
+import type { ImageEmbeddingsModel } from './extensions/cv/tasks/imageEmbeddings';
+import type { TextEmbeddingsModel } from './extensions/nlp/tasks/textEmbeddings';
+import {
+ type WhisperModel,
+ WHISPER_LANGUAGES,
+} from './extensions/speech/tasks/whisperSpeechToText';
import {
IMAGENET_NORM,
IMAGENET1K_LABELS,
@@ -528,6 +534,125 @@ const YOLO26_XLARGE_SEG_640_XNNPACK_FP32: InstanceSegmenterModel<'xyxy', CocoCla
opts: YOLO26_SEG_OPTS,
};
+// =============================================================================
+// Text Embeddings
+// =============================================================================
+const ALL_MINILM_L6_V2_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-all-MiniLM-L6-v2/${VERSION_TAG}/xnnpack/all_minilm_l6_v2_xnnpack_fp32.pte`,
+ tokenizerPath: `${BASE_URL}-all-MiniLM-L6-v2/${VERSION_TAG}/tokenizer.json`,
+};
+const ALL_MPNET_BASE_V2_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-all-mpnet-base-v2/${VERSION_TAG}/xnnpack/all_mpnet_base_v2_xnnpack_fp32.pte`,
+ tokenizerPath: `${BASE_URL}-all-mpnet-base-v2/${VERSION_TAG}/tokenizer.json`,
+};
+const MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${VERSION_TAG}/xnnpack/multi_qa_minilm_l6_cos_v1_xnnpack_fp32.pte`,
+ tokenizerPath: `${BASE_URL}-multi-qa-MiniLM-L6-cos-v1/${VERSION_TAG}/tokenizer.json`,
+};
+const MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${VERSION_TAG}/xnnpack/multi_qa_mpnet_base_dot_v1_xnnpack_fp32.pte`,
+ tokenizerPath: `${BASE_URL}-multi-qa-mpnet-base-dot-v1/${VERSION_TAG}/tokenizer.json`,
+};
+const PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${VERSION_TAG}/xnnpack/paraphrase_multilingual_minilm_l12_v2_xnnpack_8da4w.pte`,
+ tokenizerPath: `${BASE_URL}-paraphrase-multilingual-MiniLM-L12-v2/${VERSION_TAG}/tokenizer.json`,
+};
+const DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/xnnpack/distiluse_base_multilingual_cased_v2_xnnpack_8da4w.pte`,
+ tokenizerPath: `${BASE_URL}-distiluse-base-multilingual-cased-v2/${NEXT_VERSION_TAG}/tokenizer.json`,
+};
+const CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS: TextEmbeddingsModel = {
+ modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_text_xnnpack_fp32.pte`,
+ tokenizerPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/tokenizer.json`,
+};
+
+// =============================================================================
+// Image Embeddings
+// =============================================================================
+const CLIP_IMAGE_EMBEDDINGS_OPTS = {
+ resizeMode: 'stretch' as const,
+ interpolation: 'linear' as const,
+ alpha: 1 / 255.0,
+ beta: 0.0,
+};
+const CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32: ImageEmbeddingsModel = {
+ modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_image_xnnpack_fp32.pte`,
+ opts: CLIP_IMAGE_EMBEDDINGS_OPTS,
+};
+const CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_INT8: ImageEmbeddingsModel = {
+ modelPath: `${BASE_URL}-clip-vit-base-patch32/${NEXT_VERSION_TAG}/xnnpack/clip_vit_base_patch32_image_xnnpack_int8.pte`,
+ opts: CLIP_IMAGE_EMBEDDINGS_OPTS,
+};
+
+// =============================================================================
+// Speech-To-Text
+// =============================================================================
+const WHISPER_TINY_EN_XNNPACK_FP32: WhisperModel<'en'> = {
+ modelPath: 'https://huggingface.co/bhanc/scratch/resolve/main/whisper_tiny_en_xnnpack_fp32.pte',
+ tokenizerPath: `${BASE_URL}-whisper-tiny.en/${VERSION_TAG}/tokenizer.json`,
+ supportedLanguages: ['en'],
+};
+const WHISPER_TINY_EN_COREML_FP16: WhisperModel<'en'> = {
+ modelPath: 'https://huggingface.co/bhanc/scratch/resolve/main/whisper_tiny_en_coreml_fp16.pte',
+ tokenizerPath: `${BASE_URL}-whisper-tiny.en/${VERSION_TAG}/tokenizer.json`,
+ supportedLanguages: ['en'],
+};
+
+const WHISPER_TINY_XNNPACK_FP32: WhisperModel = {
+ modelPath: 'https://huggingface.co/bhanc/scratch/resolve/main/whisper_tiny_xnnpack_fp32.pte',
+ tokenizerPath: `${BASE_URL}-whisper-tiny/${VERSION_TAG}/tokenizer.json`,
+ supportedLanguages: WHISPER_LANGUAGES,
+};
+const WHISPER_TINY_COREML_FP16: WhisperModel = {
+ modelPath: 'https://huggingface.co/bhanc/scratch/resolve/main/whisper_tiny_coreml_fp16.pte',
+ tokenizerPath: `${BASE_URL}-whisper-tiny/${VERSION_TAG}/tokenizer.json`,
+ supportedLanguages: WHISPER_LANGUAGES,
+};
+
+const WHISPER_BASE_EN_XNNPACK_FP32: WhisperModel<'en'> = {
+ modelPath: 'https://huggingface.co/bhanc/scratch/resolve/main/whisper_base_en_xnnpack_fp32.pte',
+ tokenizerPath: `${BASE_URL}-whisper-base-en/${VERSION_TAG}/tokenizer.json`,
+ supportedLanguages: ['en'],
+};
+const WHISPER_BASE_EN_COREML_FP16: WhisperModel<'en'> = {
+ modelPath: 'https://huggingface.co/bhanc/scratch/resolve/main/whisper_base_en_coreml_fp16.pte',
+ tokenizerPath: `${BASE_URL}-whisper-base-en/${VERSION_TAG}/tokenizer.json`,
+ supportedLanguages: ['en'],
+};
+
+const WHISPER_BASE_XNNPACK_FP32: WhisperModel = {
+ modelPath: 'https://huggingface.co/bhanc/scratch/resolve/main/whisper_base_xnnpack_fp32.pte',
+ tokenizerPath: `${BASE_URL}-whisper-base/${VERSION_TAG}/tokenizer.json`,
+ supportedLanguages: WHISPER_LANGUAGES,
+};
+const WHISPER_BASE_COREML_FP16: WhisperModel = {
+ modelPath: 'https://huggingface.co/bhanc/scratch/resolve/main/whisper_base_coreml_fp16.pte',
+ tokenizerPath: `${BASE_URL}-whisper-base/${VERSION_TAG}/tokenizer.json`,
+ supportedLanguages: WHISPER_LANGUAGES,
+};
+
+const WHISPER_SMALL_EN_XNNPACK_FP32: WhisperModel<'en'> = {
+ modelPath: 'https://huggingface.co/bhanc/scratch/resolve/main/whisper_small_en_xnnpack_fp32.pte',
+ tokenizerPath: `${BASE_URL}-whisper-small-en/${VERSION_TAG}/tokenizer.json`,
+ supportedLanguages: ['en'],
+};
+const WHISPER_SMALL_EN_COREML_FP16: WhisperModel<'en'> = {
+ modelPath: 'https://huggingface.co/bhanc/scratch/resolve/main/whisper_small_en_coreml_fp16.pte',
+ tokenizerPath: `${BASE_URL}-whisper-small-en/${VERSION_TAG}/tokenizer.json`,
+ supportedLanguages: ['en'],
+};
+
+const WHISPER_SMALL_XNNPACK_FP32: WhisperModel = {
+ modelPath: 'https://huggingface.co/bhanc/scratch/resolve/main/whisper_small_xnnpack_fp32.pte',
+ tokenizerPath: `${BASE_URL}-whisper-small/${VERSION_TAG}/tokenizer.json`,
+ supportedLanguages: WHISPER_LANGUAGES,
+};
+const WHISPER_SMALL_COREML_FP16: WhisperModel = {
+ modelPath: 'https://huggingface.co/bhanc/scratch/resolve/main/whisper_small_coreml_fp16.pte',
+ tokenizerPath: `${BASE_URL}-whisper-small/${VERSION_TAG}/tokenizer.json`,
+ supportedLanguages: WHISPER_LANGUAGES,
+};
+
// =============================================================================
// Tokenizers
// =============================================================================
@@ -737,4 +862,73 @@ export const models = {
tokenizer: {
ALL_MINILM_L6_V2: ALL_MINILM_L6_V2_TOKENIZER,
},
+ textEmbeddings: {
+ ALL_MINILM_L6_V2: {
+ ...ALL_MINILM_L6_V2_EMBEDDINGS,
+ XNNPACK_FP32: ALL_MINILM_L6_V2_EMBEDDINGS,
+ },
+ ALL_MPNET_BASE_V2: {
+ ...ALL_MPNET_BASE_V2_EMBEDDINGS,
+ XNNPACK_FP32: ALL_MPNET_BASE_V2_EMBEDDINGS,
+ },
+ MULTI_QA_MINILM_L6_COS_V1: {
+ ...MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS,
+ XNNPACK_FP32: MULTI_QA_MINILM_L6_COS_V1_EMBEDDINGS,
+ },
+ MULTI_QA_MPNET_BASE_DOT_V1: {
+ ...MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS,
+ XNNPACK_FP32: MULTI_QA_MPNET_BASE_DOT_V1_EMBEDDINGS,
+ },
+ PARAPHRASE_MULTILINGUAL_MINILM_L12_V2: {
+ ...PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS,
+ XNNPACK_8DA4W: PARAPHRASE_MULTILINGUAL_MINILM_L12_V2_EMBEDDINGS,
+ },
+ DISTILUSE_BASE_MULTILINGUAL_CASED_V2: {
+ ...DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS,
+ XNNPACK_8DA4W: DISTILUSE_BASE_MULTILINGUAL_CASED_V2_EMBEDDINGS,
+ },
+ CLIP_VIT_BASE_PATCH32_TEXT: {
+ ...CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS,
+ XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_TEXT_EMBEDDINGS,
+ },
+ },
+ imageEmbeddings: {
+ CLIP_VIT_BASE_PATCH32: {
+ ...CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32,
+ XNNPACK_FP32: CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_FP32,
+ XNNPACK_INT8: CLIP_VIT_BASE_PATCH32_IMAGE_XNNPACK_INT8,
+ },
+ },
+ speechToText: {
+ WHISPER_TINY_EN: {
+ ...WHISPER_TINY_EN_XNNPACK_FP32,
+ XNNPACK_FP32: WHISPER_TINY_EN_XNNPACK_FP32,
+ COREML_FP16: WHISPER_TINY_EN_COREML_FP16,
+ },
+ WHISPER_TINY: {
+ ...WHISPER_TINY_XNNPACK_FP32,
+ XNNPACK_FP32: WHISPER_TINY_XNNPACK_FP32,
+ COREML_FP16: WHISPER_TINY_COREML_FP16,
+ },
+ WHISPER_BASE_EN: {
+ ...WHISPER_BASE_EN_XNNPACK_FP32,
+ XNNPACK_FP32: WHISPER_BASE_EN_XNNPACK_FP32,
+ COREML_FP16: WHISPER_BASE_EN_COREML_FP16,
+ },
+ WHISPER_BASE: {
+ ...WHISPER_BASE_XNNPACK_FP32,
+ XNNPACK_FP32: WHISPER_BASE_XNNPACK_FP32,
+ COREML_FP16: WHISPER_BASE_COREML_FP16,
+ },
+ WHISPER_SMALL_EN: {
+ ...WHISPER_SMALL_EN_XNNPACK_FP32,
+ XNNPACK_FP32: WHISPER_SMALL_EN_XNNPACK_FP32,
+ COREML_FP16: WHISPER_SMALL_EN_COREML_FP16,
+ },
+ WHISPER_SMALL: {
+ ...WHISPER_SMALL_XNNPACK_FP32,
+ XNNPACK_FP32: WHISPER_SMALL_XNNPACK_FP32,
+ COREML_FP16: WHISPER_SMALL_COREML_FP16,
+ },
+ },
};
diff --git a/yarn.lock b/yarn.lock
index bc05759538..867dd721b3 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -12764,6 +12764,24 @@ __metadata:
languageName: node
linkType: hard
+"react-native-audio-api@npm:0.12.2":
+ version: 0.12.2
+ resolution: "react-native-audio-api@npm:0.12.2"
+ dependencies:
+ semver: "npm:^7.7.3"
+ peerDependencies:
+ react: "*"
+ react-native: "*"
+ react-native-worklets: ">= 0.6.0"
+ peerDependenciesMeta:
+ react-native-worklets:
+ optional: true
+ bin:
+ setup-rn-audio-api-web: scripts/setup-rn-audio-api-web.js
+ checksum: 10/ed495058382188c8beb51ce89f2ef14d846dc0c0a07c65a7b4c71aa106fb7ea14aa8660b05fb33941c038d1a7ab2ba4ab3eb039fe481841938c45396903c6060
+ languageName: node
+ linkType: hard
+
"react-native-builder-bob@npm:^0.40.18":
version: 0.40.18
resolution: "react-native-builder-bob@npm:0.40.18"
@@ -14036,6 +14054,31 @@ __metadata:
languageName: node
linkType: hard
+"speech@workspace:apps/speech":
+ version: 0.0.0-use.local
+ resolution: "speech@workspace:apps/speech"
+ dependencies:
+ "@babel/core": "npm:^7.29.0"
+ "@react-native/metro-config": "npm:^0.86.0"
+ "@types/react": "npm:~19.2.0"
+ babel-preset-expo: "npm:~56.0.14"
+ expo: "npm:~56.0.9"
+ expo-build-properties: "npm:~56.0.17"
+ react: "npm:19.2.3"
+ react-native: "npm:0.85.3"
+ react-native-audio-api: "npm:0.12.2"
+ react-native-executorch: "workspace:*"
+ react-native-fs: "npm:^2.20.0"
+ react-native-reanimated: "npm:4.4.0"
+ react-native-safe-area-context: "npm:~5.7.0"
+ react-native-svg: "npm:15.15.4"
+ react-native-svg-transformer: "npm:^1.5.3"
+ react-native-worklets: "npm:0.9.1"
+ react-refresh: "npm:^0.14.0"
+ typescript: "npm:~5.9.2"
+ languageName: unknown
+ linkType: soft
+
"split-on-first@npm:^1.0.0":
version: 1.1.0
resolution: "split-on-first@npm:1.1.0"