diff --git a/.agents/skills/add-task-pipeline/SKILL.md b/.agents/skills/add-task-pipeline/SKILL.md index 84574c50c8..6eefdabd72 100644 --- a/.agents/skills/add-task-pipeline/SKILL.md +++ b/.agents/skills/add-task-pipeline/SKILL.md @@ -100,7 +100,7 @@ import { loadModel } from '../../../core/model'; import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { type ImageBuffer } from '../image'; -import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing'; +import { createImagePreprocessor, type ImagePreprocessorOptions } from '../utils/imagePreprocessor'; export type MyTaskOptions = ImagePreprocessorOptions & { readonly defaultThreshold: number; @@ -190,28 +190,21 @@ Wrap the task pipeline in a custom React Hook using the core hooks `useResourceD ```typescript import { useModel } from './useModel'; -import { useResourceDownload } from './useResourceDownload'; +import { useResourceDownload, type ResourceOptions } from './useResourceDownload'; import { createMyTask, type MyTaskModel } from '../extensions//tasks/'; -export function useMyTask(config: MyTaskModel, options?: { preventLoad?: boolean }) { +export function useMyTask(config: MyTaskModel, options?: ResourceOptions) { // 1. Resolve remote or local asset model path and download progress - const { localPath, downloadProgress, downloadError } = useResourceDownload( - config.modelPath, - options?.preventLoad - ); + const { resource, downloadProgress, downloadError } = useResourceDownload(config, options); // 2. Instantiate and compile the task pipeline (with automatic lifecycle cleanup) - const { model, error } = useModel( - createMyTask, - localPath ? { ...config, modelPath: localPath } : null, - [localPath] - ); + const { model, error } = useModel(createMyTask, resource); return { isReady: !!model, error: downloadError || error, downloadProgress, - localPath, + resource, runTask: model?.runTask, runTaskWorklet: model?.runTaskWorklet, }; diff --git a/.agents/skills/model-schema-validation/SKILL.md b/.agents/skills/model-schema-validation/SKILL.md index 9b6cfd2254..9b39dd6106 100644 --- a/.agents/skills/model-schema-validation/SKILL.md +++ b/.agents/skills/model-schema-validation/SKILL.md @@ -30,7 +30,7 @@ import { i64, i32, DynamicDim as Dyn, - constr, + constraint, } from '../../../core/schema'; const { variant, dims } = validateSpec(model.schema, { @@ -39,7 +39,7 @@ const { variant, dims } = validateSpec(model.schema, { [i64(1, Dyn('L')), i64(1, Dyn('L'))], [f32(1, 'D')], [ - constr.eq( + constraint.equality( { paramSide: 'input', tensorIdx: 0, dimIdx: 1 }, { paramSide: 'input', tensorIdx: 1, dimIdx: 1 } ), @@ -50,7 +50,7 @@ const { variant, dims } = validateSpec(model.schema, { [i64(Dyn('L')), i64(Dyn('L'))], [f32('D')], [ - constr.eq( + constraint.equality( { paramSide: 'input', tensorIdx: 0, dimIdx: 0 }, { paramSide: 'input', tensorIdx: 1, dimIdx: 0 } ), @@ -68,10 +68,10 @@ const L = dims.range('L'); - **`f32(...)` / `i64(...)` / `i32(...)` / `ui8(...)`**: Shorthand helpers for tensor parameter specs. - **`StaticDim('symbol')` / String Literals**: Strings passed to shape helpers (e.g. `'H'`, `'W'`) automatically map to `StaticDim`, acting as **static dimension wildcards**. They bind strictly to `constant` positive integer dimensions in the exported spec. - **`DynamicDim('symbol')` (or `Dyn('symbol')`)**: Creates a dynamic dimension symbol. Must be used when a dimension genuinely varies at runtime and binds to a `range` or `enum` domain in the exported spec. -- **Constraint Helpers (`constr`)**: +- **Constraint Helpers (`constraint`)**: - **`DimRef` Object Literal (`{ paramSide: 'input' | 'output', tensorIdx, dimIdx }`)**: Explicit reference to a tensor's dimension. - - **`constr.eq(...dims)`**: Creates an equality constraint requiring the referenced dimensions to take the exact same value at runtime. - - **`constr.linear(lhs, rhs, a, b)`**: Creates a linear constraint `lhs = a * rhs + b`. + - **`constraint.equality(...dims)`**: Creates an equality constraint requiring the referenced dimensions to take the exact same value at runtime. + - **`constraint.linear(lhs, rhs, a, b?)`**: Creates a linear constraint `lhs = a * rhs + b`. - **`validateSpec(exportedSchema, allowedVariants)`**: Compares the model's exported schema against named variants and returns `{ variant, dim, dims }`. - **Symbol Accessors (`dims` & `dim`)**: - `dims.constant('N', 'H')`: Extracts constant values for symbols as numbers. @@ -118,24 +118,24 @@ Understanding the distinction between a dimension's **domain** and its **runtime - Dynamic symbols (`DynamicDim('S')`) bind to exported dimension domains. Reusing a symbol (`'S'`) across tensor inputs or outputs requires every occurrence to bind to the **exact same domain**. - ⚠️ **Key Rule**: Binding to the same domain does **NOT** mean runtime values coincide! Two dimensions bound to the same domain (e.g., both having range `1..512`) may take _different_ runtime values in a single execution (e.g. length 10 and length 25). -### 2. Runtime Constraints (`constr.eq` & `constr.linear`) +### 2. Runtime Constraints (`constraint.equality` & `constraint.linear`) - **Runtime Constraints**: Declarations about the **runtime values** of tensor dimensions during execution: - - **Equality Constraint (`constr.eq(...)`)**: Requires all referenced dimensions to take the exact same runtime value in any execution call. + - **Equality Constraint (`constraint.equality(...)`)**: Requires all referenced dimensions to take the exact same runtime value in any execution call. ```typescript method( 'forward', [f32('B', Dyn('S1')), f32('B', Dyn('S2'))], [f32('B', Dyn('S1'))], [ - constr.eq( + constraint.equality( { paramSide: 'input', tensorIdx: 0, dimIdx: 0 }, { paramSide: 'input', tensorIdx: 1, dimIdx: 0 } ), ] ); ``` - - **Linear Constraint (`constr.linear(...)`)**: Requires two dimensions to satisfy `dimLhs = a * dimRhs + b` at runtime. + - **Linear Constraint (`constraint.linear(...)`)**: Requires two dimensions to satisfy `dimLhs = a * dimRhs + b` at runtime. - **Validation & Enforcement**: - `validateSpec` verifies that the exported model spec declares the exact same runtime constraints (1-to-1 declaration match). - Native C++ validates input runtime constraints before invoking `model.execute()`. diff --git a/.eslintrc.js b/.eslintrc.js index e0f544159c..94b518b535 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,14 +1,7 @@ const path = require('path'); +const typedocConfig = require('./docs/typedoc.json'); -const VALID_CATEGORIES = [ - 'Constants', - 'Errors', - 'Hooks', - 'Types', - 'Typescript API', - 'Utils', - 'Utilities - General', -]; +const VALID_CATEGORIES = typedocConfig.categoryOrder.filter((cat) => cat !== '*'); const CATEGORY_TAG_MATCH = `^(${VALID_CATEGORIES.join('|')})$`; diff --git a/apps/computer-vision/app/detection/index.tsx b/apps/computer-vision/app/detection/index.tsx index b2c990ea1c..fdee01d0a1 100644 --- a/apps/computer-vision/app/detection/index.tsx +++ b/apps/computer-vision/app/detection/index.tsx @@ -16,11 +16,11 @@ import { BoundingBox } from '../../components/BoundingBox'; const MODEL_OPTIONS: ModelOption[] = [ { label: 'SSDLite 320 MobileNet V3 Large (XNNPACK FP32)', - value: models.objectDetection.SSDLITE320_MOBILENET_V3_LARGE, + value: models.objectDetection.SSDLITE320_MOBILENET_V3_LARGE.DEFAULT, }, { label: 'RF-DETR Nano (XNNPACK FP32)', - value: models.objectDetection.RFDETR_NANO, + value: models.objectDetection.RFDETR_NANO.DEFAULT, }, { label: 'RF-DETR Nano (CoreML FP16)', diff --git a/apps/computer-vision/app/imageEmbeddings/index.tsx b/apps/computer-vision/app/imageEmbeddings/index.tsx index 47b8e04e7f..2d6dcf6323 100644 --- a/apps/computer-vision/app/imageEmbeddings/index.tsx +++ b/apps/computer-vision/app/imageEmbeddings/index.tsx @@ -71,7 +71,7 @@ function ImageEmbeddingsContent() { // Zero-shot classification pairs a CLIP image encoder with the CLIP text // encoder and scores the image against each text label by embedding similarity. const imageModel = useImageEmbedder(selectedImageModel); - const textModel = useTextEmbedder(models.textEmbeddings.CLIP_VIT_BASE_PATCH32_TEXT); + const textModel = useTextEmbedder(models.textEmbeddings.CLIP_VIT_BASE_PATCH32_TEXT.DEFAULT); const ready = imageModel.isReady && textModel.isReady; diff --git a/apps/computer-vision/app/inspect/index.tsx b/apps/computer-vision/app/inspect/index.tsx index c22bb5a832..9620063905 100644 --- a/apps/computer-vision/app/inspect/index.tsx +++ b/apps/computer-vision/app/inspect/index.tsx @@ -10,13 +10,13 @@ import { Alert, } from 'react-native'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; -import { inspectModel, type ConcreteDim, type ParamSpec } from 'react-native-executorch'; +import { inspectModel, type schema } from 'react-native-executorch'; import ScreenWrapper from '../../components/ScreenWrapper'; import { ColorPalette } from '../../theme'; type InspectionResult = Awaited>; -const formatDim = (dim: ConcreteDim): string => { +const formatDim = (dim: schema.ConcreteDim): string => { switch (dim.kind) { case 'constant': return `${dim.value}`; @@ -55,7 +55,7 @@ function InspectContent() { }; const renderParamList = ( - params: readonly ParamSpec[] | undefined, + params: readonly schema.ParamSpec[] | undefined, title: string ) => { if (!params || params.length === 0) return null; diff --git a/apps/computer-vision/app/keypoint/index.tsx b/apps/computer-vision/app/keypoint/index.tsx index 85a5260ee2..d0649996da 100644 --- a/apps/computer-vision/app/keypoint/index.tsx +++ b/apps/computer-vision/app/keypoint/index.tsx @@ -16,7 +16,7 @@ import { BoundingBox } from '../../components/BoundingBox'; const MODEL_OPTIONS: ModelOption[] = [ { label: 'BlazeFace (XNNPACK FP32)', - value: models.keypointDetection.BLAZEFACE, + value: models.keypointDetection.BLAZEFACE.DEFAULT, }, { label: 'YOLO26 Pose (XNNPACK FP32)', diff --git a/apps/computer-vision/tsconfig.json b/apps/computer-vision/tsconfig.json index 47026ce434..9baba0ef64 100644 --- a/apps/computer-vision/tsconfig.json +++ b/apps/computer-vision/tsconfig.json @@ -9,7 +9,17 @@ "customConditions": ["react-native"], "noEmit": true, "paths": { - "react-native-executorch": ["../../packages/react-native-executorch/src"] + "react-native-executorch": ["../../packages/react-native-executorch/src"], + "react-native-executorch/cv": ["../../packages/react-native-executorch/src/extensions/cv"], + "react-native-executorch/llm": ["../../packages/react-native-executorch/src/extensions/llm"], + "react-native-executorch/nlp": ["../../packages/react-native-executorch/src/extensions/nlp"], + "react-native-executorch/speech": [ + "../../packages/react-native-executorch/src/extensions/speech" + ], + "react-native-executorch/math": [ + "../../packages/react-native-executorch/src/extensions/math" + ], + "react-native-executorch/schema": ["../../packages/react-native-executorch/src/core/schema"] } } } diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index 3955ff2d62..658435e62a 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -15,14 +15,7 @@ import { import { Skia } from '@shopify/react-native-skia'; import RNBlobUtil from 'react-native-blob-util'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; -import { - useLLMChatSession, - type LLMGenerationStats, - type LLMKVCacheState, - type ToolCall, - type ChatMessage, - type cv, -} from 'react-native-executorch'; +import { useLLMChatSession, type llm, type cv } from 'react-native-executorch'; import ScreenWrapper from '../../components/ScreenWrapper'; import { ModelPicker, type ModelOption } from '../../components/ModelPicker'; import { Button } from '../../components/Button'; @@ -33,11 +26,11 @@ type Turn = { role: 'user' | 'assistant' | 'tool'; content: string; imageUri?: string; - stats?: LLMGenerationStats; - toolCalls?: readonly ToolCall[]; + stats?: llm.LLMGenerationStats; + toolCalls?: readonly llm.ToolCall[]; }; -function formatStats(stats: LLMGenerationStats): string { +function formatStats(stats: llm.LLMGenerationStats): string { const decodeMs = stats.inferenceEndMs - stats.firstTokenMs; const tokensPerSec = decodeMs > 0 ? (stats.numGeneratedTokens / decodeMs) * 1000 : 0; const decodeTtftMs = stats.firstTokenMs - stats.inferenceStartMs; @@ -70,7 +63,7 @@ function LLMContent() { [] ); - const initialMessages: ChatMessage[] = useMemo( + const initialMessages: llm.ChatMessage[] = useMemo( () => (activeModel.systemPrompt ? [{ role: 'system', content: activeModel.systemPrompt }] : []), [activeModel] ); @@ -88,7 +81,7 @@ function LLMContent() { const [turns, setTurns] = useState([]); const [streamingResponse, setStreamingResponse] = useState(null); - let kvCacheState: LLMKVCacheState | null = null; + let kvCacheState: llm.LLMKVCacheState | null = null; if (isReady && getKVCacheState) { try { kvCacheState = getKVCacheState(); diff --git a/apps/nlp/app/privacy-filter/index.tsx b/apps/nlp/app/privacy-filter/index.tsx index 9e9227ad1c..e5aed1838d 100644 --- a/apps/nlp/app/privacy-filter/index.tsx +++ b/apps/nlp/app/privacy-filter/index.tsx @@ -9,14 +9,7 @@ import { KeyboardAvoidingView, Platform, } from 'react-native'; -import { - usePrivacyFilter, - models, - piiSegments, - type PiiEntity, - type PiiSegment, - type PrivacyFilterModel, -} from 'react-native-executorch'; +import { usePrivacyFilter, models, nlp, type PrivacyFilterModel } from 'react-native-executorch'; import ScreenWrapper from '../../components/ScreenWrapper'; import { ModelStatus } from '../../components/ModelStatus'; import { Button } from '../../components/Button'; @@ -33,14 +26,18 @@ Reach her at maria.lopez@example.com or +1 (415) 555-0142. Address: 84 Cedar Hil /* cspell:enable */ const MODELS: { label: string; value: PrivacyFilterModel; sample: string; iosOnly?: boolean }[] = [ - { label: 'OpenAI (8 types)', value: models.privacyFilter.OPENAI, sample: OPENAI_SAMPLE }, + { label: 'OpenAI (8 types)', value: models.privacyFilter.OPENAI.DEFAULT, sample: OPENAI_SAMPLE }, { label: 'OpenAI MLX', value: models.privacyFilter.OPENAI.MLX_INT4, sample: OPENAI_SAMPLE, iosOnly: true, }, - { label: 'Nemotron (55 types)', value: models.privacyFilter.NEMOTRON, sample: NEMOTRON_SAMPLE }, + { + label: 'Nemotron (55 types)', + value: models.privacyFilter.NEMOTRON.DEFAULT, + sample: NEMOTRON_SAMPLE, + }, { label: 'Nemotron MLX', value: models.privacyFilter.NEMOTRON.MLX_INT8, @@ -64,14 +61,14 @@ function PrivacyFilterContent() { const { isReady, downloadProgress, error, detectPii } = usePrivacyFilter(active.value); const [text, setText] = useState(active.sample); - const [entities, setEntities] = useState(null); + const [entities, setEntities] = useState(null); const [busy, setBusy] = useState(false); const [runError, setRunError] = useState(null); const [inferenceMs, setInferenceMs] = useState(null); const ready = isReady && !!detectPii; - const segments: PiiSegment[] | null = useMemo( - () => (entities ? piiSegments(text, entities) : null), + const segments: nlp.PiiSegment[] | null = useMemo( + () => (entities ? nlp.piiSegments(text, entities) : null), [text, entities] ); diff --git a/apps/nlp/app/text-embeddings/index.tsx b/apps/nlp/app/text-embeddings/index.tsx index 820f2296cf..7c33edc760 100644 --- a/apps/nlp/app/text-embeddings/index.tsx +++ b/apps/nlp/app/text-embeddings/index.tsx @@ -24,16 +24,22 @@ const MODELS: { docPrompt?: string; iosOnly?: boolean; }[] = [ - { label: 'MiniLM L6', value: models.textEmbeddings.ALL_MINILM_L6_V2 }, - { label: 'MPNet Base', value: models.textEmbeddings.ALL_MPNET_BASE_V2 }, - { label: 'MultiQA MiniLM', value: models.textEmbeddings.MULTI_QA_MINILM_L6_COS_V1 }, - { label: 'MultiQA MPNet', value: models.textEmbeddings.MULTI_QA_MPNET_BASE_DOT_V1 }, - { label: 'Paraphrase ML', value: models.textEmbeddings.PARAPHRASE_MULTILINGUAL_MINILM_L12_V2 }, - { label: 'DistilUSE ML', value: models.textEmbeddings.DISTILUSE_BASE_MULTILINGUAL_CASED_V2 }, - { label: 'CLIP Text', value: models.textEmbeddings.CLIP_VIT_BASE_PATCH32_TEXT }, + { label: 'MiniLM L6', value: models.textEmbeddings.ALL_MINILM_L6_V2.DEFAULT }, + { label: 'MPNet Base', value: models.textEmbeddings.ALL_MPNET_BASE_V2.DEFAULT }, + { label: 'MultiQA MiniLM', value: models.textEmbeddings.MULTI_QA_MINILM_L6_COS_V1.DEFAULT }, + { label: 'MultiQA MPNet', value: models.textEmbeddings.MULTI_QA_MPNET_BASE_DOT_V1.DEFAULT }, + { + label: 'Paraphrase ML', + value: models.textEmbeddings.PARAPHRASE_MULTILINGUAL_MINILM_L12_V2.DEFAULT, + }, + { + label: 'DistilUSE ML', + value: models.textEmbeddings.DISTILUSE_BASE_MULTILINGUAL_CASED_V2.DEFAULT, + }, + { label: 'CLIP Text', value: models.textEmbeddings.CLIP_VIT_BASE_PATCH32_TEXT.DEFAULT }, { label: 'LFM2.5', - value: models.textEmbeddings.LFM2_5_EMBEDDING_350M, + value: models.textEmbeddings.LFM2_5_EMBEDDING_350M.DEFAULT, docPrompt: 'document: ', }, { diff --git a/apps/nlp/constants/llm.ts b/apps/nlp/constants/llm.ts index 19f7892991..405e531a25 100644 --- a/apps/nlp/constants/llm.ts +++ b/apps/nlp/constants/llm.ts @@ -1,14 +1,6 @@ -import { - models, - type LLMModel, - type LLMToolOpts, - type LLMGenerationConfig, - type ToolDefinition, - type ToolParserResult, - type ToolCall, -} from 'react-native-executorch'; - -export const TOOLS: ToolDefinition[] = [ +import { models, type llm, type LLMModel, type LLMToolOpts } from 'react-native-executorch'; + +export const TOOLS: llm.ToolDefinition[] = [ { type: 'function', function: { @@ -139,10 +131,10 @@ export const TOOLS: ToolDefinition[] = [ export const GEMMA_TOOL_STOP_REGEX = /(?:|)/; -export function parseGemmaToolCalls(text: string): ToolParserResult | undefined { +export function parseGemmaToolCalls(text: string): llm.ToolParserResult | undefined { const callRegex = /<\|tool_call>call:([a-zA-Z0-9_-]+)\{([\s\S]*?)\}(?:)?/g; - const toolCalls: ToolCall[] = []; + const toolCalls: llm.ToolCall[] = []; for (const match of text.matchAll(callRegex)) { const name = match[1]!; const rawArgs = match[2]?.trim() ?? ''; @@ -173,7 +165,7 @@ export function parseGemmaToolCalls(text: string): ToolParserResult | undefined export const HAMMER_TOOL_STOP_REGEX = /(?:<\|im_end\|>)/; -export function parseHammerToolCalls(text: string): ToolParserResult | undefined { +export function parseHammerToolCalls(text: string): llm.ToolParserResult | undefined { const jsonMatch = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/) ?? text.match(/\[\s*\{[\s\S]*\}\s*\]/); const jsonStr = jsonMatch ? (jsonMatch[1] ?? jsonMatch[0]) : text.trim(); @@ -182,7 +174,7 @@ export function parseHammerToolCalls(text: string): ToolParserResult | undefined const parsed = JSON.parse(jsonStr); if (!Array.isArray(parsed) || parsed.length === 0) return undefined; - const toolCalls: ToolCall[] = []; + const toolCalls: llm.ToolCall[] = []; for (const item of parsed) { if (item && typeof item === 'object' && typeof item.name === 'string') { toolCalls.push({ @@ -205,9 +197,9 @@ export function parseHammerToolCalls(text: string): ToolParserResult | undefined export const QWEN_TOOL_STOP_REGEX = /(?:<\/tool_call>|<\|im_end\|>)/; -export function parseQwenToolCalls(text: string): ToolParserResult | undefined { +export function parseQwenToolCalls(text: string): llm.ToolParserResult | undefined { const callRegex = /\s*(\{[\s\S]*?\})\s*<\/tool_call>/g; - const toolCalls: ToolCall[] = []; + const toolCalls: llm.ToolCall[] = []; let match; while ((match = callRegex.exec(text)) !== null) { @@ -245,7 +237,7 @@ export function parseQwenToolCalls(text: string): ToolParserResult | undefined { export const LLAMA_TOOL_STOP_REGEX = /(?:<\|eot_id\|>|<\|eom_id\|>)/; -export function parseLlamaToolCalls(text: string): ToolParserResult | undefined { +export function parseLlamaToolCalls(text: string): llm.ToolParserResult | undefined { const cleanText = text .replace(/<\|(?:eot_id|eom_id|start_header_id|end_header_id)\|>[\s\S]*?$/g, '') .trim(); @@ -270,7 +262,7 @@ export function parseLlamaToolCalls(text: string): ToolParserResult | undefined args = {}; } } - const toolCalls: ToolCall[] = [ + const toolCalls: llm.ToolCall[] = [ { type: 'function', function: { @@ -293,7 +285,7 @@ export function parseLlamaToolCalls(text: string): ToolParserResult | undefined // --- Common Configurations --- -export const DEFAULT_GENERATION_CONFIG: LLMGenerationConfig = { +export const DEFAULT_GENERATION_CONFIG: llm.LLMGenerationConfig = { temperature: 0.7, maxNewTokens: 512, echo: false, @@ -327,7 +319,7 @@ export interface LLMModelConfig { id: string; name: string; model: LLMModel; - generationConfig: LLMGenerationConfig; + generationConfig: llm.LLMGenerationConfig; systemPrompt?: string; stopRegex?: RegExp; toolOpts: LLMToolOpts | undefined; diff --git a/apps/nlp/tsconfig.json b/apps/nlp/tsconfig.json index 47026ce434..9baba0ef64 100644 --- a/apps/nlp/tsconfig.json +++ b/apps/nlp/tsconfig.json @@ -9,7 +9,17 @@ "customConditions": ["react-native"], "noEmit": true, "paths": { - "react-native-executorch": ["../../packages/react-native-executorch/src"] + "react-native-executorch": ["../../packages/react-native-executorch/src"], + "react-native-executorch/cv": ["../../packages/react-native-executorch/src/extensions/cv"], + "react-native-executorch/llm": ["../../packages/react-native-executorch/src/extensions/llm"], + "react-native-executorch/nlp": ["../../packages/react-native-executorch/src/extensions/nlp"], + "react-native-executorch/speech": [ + "../../packages/react-native-executorch/src/extensions/speech" + ], + "react-native-executorch/math": [ + "../../packages/react-native-executorch/src/extensions/math" + ], + "react-native-executorch/schema": ["../../packages/react-native-executorch/src/core/schema"] } } } diff --git a/apps/speech/app/text-to-speech/index.tsx b/apps/speech/app/text-to-speech/index.tsx index 7e39eb1e23..b6dcb2be1b 100644 --- a/apps/speech/app/text-to-speech/index.tsx +++ b/apps/speech/app/text-to-speech/index.tsx @@ -3,10 +3,10 @@ import { Platform, View, Text, StyleSheet, ScrollView, TextInput } from 'react-n import { useTextToSpeech, models, + speech, SUPERTONIC_SAMPLE_RATE, - SUPERTONIC_SUPPORTED_LANGUAGES, - constants, - type SupertonicLanguage, + SUPERTONIC_DEFAULT_VOICE_NAMES, + type SupertonicDefaultVoiceName, } from 'react-native-executorch'; import { AudioContext, type AudioBufferQueueSourceNode } from 'react-native-audio-api'; @@ -24,12 +24,12 @@ const SAMPLE_TEXT = 'Each voice style is encoded as a compact embedding that captures the unique timbre, pitch, and speaking patterns of the target speaker. ' + 'This makes it ideal for accessibility applications, voice assistants, and content creation tools that need high-quality speech synthesis without sending data to external servers.'; -const VOICE_OPTIONS = constants.SUPERTONIC_DEFAULT_VOICE_NAMES.map((name) => ({ +const VOICE_OPTIONS = SUPERTONIC_DEFAULT_VOICE_NAMES.map((name) => ({ label: name, - value: name as constants.SupertonicDefaultVoiceName, + value: name as SupertonicDefaultVoiceName, })); -const LANGUAGE_OPTIONS = SUPERTONIC_SUPPORTED_LANGUAGES.map((lang) => ({ +const LANGUAGE_OPTIONS = speech.SUPERTONIC_SUPPORTED_LANGUAGES.map((lang) => ({ label: lang, value: lang, })); @@ -58,8 +58,8 @@ const MODEL_OPTIONS = [ function TTSContent() { const [text, setText] = useState(SAMPLE_TEXT); const [selectedModel, setSelectedModel] = useState<'XNNPACK_FP32' | 'MLX_FP32'>('XNNPACK_FP32'); - const [selectedVoice, setSelectedVoice] = useState('F1'); - const [selectedLang, setSelectedLang] = useState('en'); + const [selectedVoice, setSelectedVoice] = useState('F1'); + const [selectedLang, setSelectedLang] = useState('en'); const [speed, setSpeed] = useState(1.05); const [totalSteps, setTotalSteps] = useState(8); const [isSynthesizing, setIsSynthesizing] = useState(false); diff --git a/apps/speech/app/vad/index.tsx b/apps/speech/app/vad/index.tsx index f8a7e77d09..da3a47d563 100644 --- a/apps/speech/app/vad/index.tsx +++ b/apps/speech/app/vad/index.tsx @@ -15,7 +15,7 @@ const isSimulator = DeviceInfo.isEmulatorSync(); function VADContent() { const { isReady, downloadProgress, error, detectVoiceOnStream, resetStream } = - useVoiceActivityDetector(models.voiceActivityDetection.FSMN_VAD); + useVoiceActivityDetector(models.voiceActivityDetection.FSMN_VAD.DEFAULT); const [isStreaming, setIsStreaming] = useState(false); const [isSpeaking, setIsSpeaking] = useState(false); diff --git a/apps/speech/tsconfig.json b/apps/speech/tsconfig.json index 47026ce434..9baba0ef64 100644 --- a/apps/speech/tsconfig.json +++ b/apps/speech/tsconfig.json @@ -9,7 +9,17 @@ "customConditions": ["react-native"], "noEmit": true, "paths": { - "react-native-executorch": ["../../packages/react-native-executorch/src"] + "react-native-executorch": ["../../packages/react-native-executorch/src"], + "react-native-executorch/cv": ["../../packages/react-native-executorch/src/extensions/cv"], + "react-native-executorch/llm": ["../../packages/react-native-executorch/src/extensions/llm"], + "react-native-executorch/nlp": ["../../packages/react-native-executorch/src/extensions/nlp"], + "react-native-executorch/speech": [ + "../../packages/react-native-executorch/src/extensions/speech" + ], + "react-native-executorch/math": [ + "../../packages/react-native-executorch/src/extensions/math" + ], + "react-native-executorch/schema": ["../../packages/react-native-executorch/src/core/schema"] } } } diff --git a/docs/typedoc.json b/docs/typedoc.json index f68607dbf1..503ddc63d0 100644 --- a/docs/typedoc.json +++ b/docs/typedoc.json @@ -4,5 +4,35 @@ "compilerOptions": { "skipLibCheck": true }, - "excludeExternals": true + "excludeExternals": true, + "categoryOrder": [ + "Hooks", + "Models", + "Modules", + "Core / Functions", + "Core / Types", + "Core / Constants", + "Core / Schema / Functions", + "Core / Schema / Types", + "CV / Tasks", + "CV / Functions", + "CV / Types", + "CV / Constants", + "LLM / Tasks", + "LLM / Functions", + "LLM / Types", + "NLP / Tasks", + "NLP / Functions", + "NLP / Types", + "NLP / Constants", + "Speech / Tasks", + "Speech / Functions", + "Speech / Types", + "Speech / Constants", + "Math / Functions", + "Math / Types", + "Utils / Functions", + "Utils / Types", + "*" + ] } diff --git a/packages/react-native-executorch/package.json b/packages/react-native-executorch/package.json index c15604ccea..a5b62933a2 100644 --- a/packages/react-native-executorch/package.json +++ b/packages/react-native-executorch/package.json @@ -13,6 +13,36 @@ "types": "./lib/typescript/src/index.d.ts", "default": "./lib/module/index.js" }, + "./cv": { + "source": "./src/extensions/cv/index.ts", + "types": "./lib/typescript/src/extensions/cv/index.d.ts", + "default": "./lib/module/extensions/cv/index.js" + }, + "./llm": { + "source": "./src/extensions/llm/index.ts", + "types": "./lib/typescript/src/extensions/llm/index.d.ts", + "default": "./lib/module/extensions/llm/index.js" + }, + "./nlp": { + "source": "./src/extensions/nlp/index.ts", + "types": "./lib/typescript/src/extensions/nlp/index.d.ts", + "default": "./lib/module/extensions/nlp/index.js" + }, + "./speech": { + "source": "./src/extensions/speech/index.ts", + "types": "./lib/typescript/src/extensions/speech/index.d.ts", + "default": "./lib/module/extensions/speech/index.js" + }, + "./math": { + "source": "./src/extensions/math.ts", + "types": "./lib/typescript/src/extensions/math.d.ts", + "default": "./lib/module/extensions/math.js" + }, + "./schema": { + "source": "./src/core/schema.ts", + "types": "./lib/typescript/src/core/schema.d.ts", + "default": "./lib/module/core/schema.js" + }, "./package.json": "./package.json" }, "files": [ diff --git a/packages/react-native-executorch/src/constants.ts b/packages/react-native-executorch/src/constants.ts index 0bb489bfa0..518beaffde 100644 --- a/packages/react-native-executorch/src/constants.ts +++ b/packages/react-native-executorch/src/constants.ts @@ -1,6 +1,13 @@ +/** + * Shared constants, label maps, and default configurations for task pipelines. + * + * This module exports constant arrays, dataset labels, and normalization + * parameters used across task pipelines and model configurations. + */ + /** * ImageNet 1K dataset label array containing the 1000 categories. - * @category Constants + * @category CV / Constants */ export const IMAGENET1K_LABELS = [ 'tench, Tinca tinca', @@ -1007,7 +1014,7 @@ export const IMAGENET1K_LABELS = [ /** * Pascal VOC dataset label array containing the 21 categories. - * @category Constants + * @category CV / Constants */ export const PASCAL_VOC_LABELS = [ 'background', @@ -1035,7 +1042,7 @@ export const PASCAL_VOC_LABELS = [ /** * COCO classes list. - * @category Constants + * @category CV / Constants */ export const COCO_CLASSES = [ 'background', @@ -1133,25 +1140,25 @@ export const COCO_CLASSES = [ /** * Type representing a valid ImageNet 1K label string. - * @category Types + * @category CV / Types */ export type ImageNet1KLabel = (typeof IMAGENET1K_LABELS)[number]; /** * Type representing a valid Pascal VOC label string. - * @category Types + * @category CV / Types */ export type PascalVocLabel = (typeof PASCAL_VOC_LABELS)[number]; /** * Type representing a valid COCO class string. - * @category Types + * @category CV / Types */ export type CocoClass = (typeof COCO_CLASSES)[number]; /** * COCO classes list specifically for YOLO models (80 classes, 0-indexed). - * @category Constants + * @category CV / Constants */ export const COCO_CLASSES_YOLO = [ 'person', @@ -1238,7 +1245,7 @@ export const COCO_CLASSES_YOLO = [ /** * Type representing a valid YOLO COCO class label string. - * @category Types + * @category CV / Types */ export type CocoClassYolo = (typeof COCO_CLASSES_YOLO)[number]; @@ -1246,7 +1253,7 @@ export type CocoClassYolo = (typeof COCO_CLASSES_YOLO)[number]; * ImageNet standard normalization options containing alpha and beta * coefficients. Based on the standard ImageNet mean [0.485, 0.456, 0.406] and * std [0.229, 0.224, 0.225]. - * @category Constants + * @category CV / Constants */ export const IMAGENET_NORM = { alpha: [1 / (255.0 * 0.229), 1 / (255.0 * 0.224), 1 / (255.0 * 0.225)], @@ -1255,7 +1262,7 @@ export const IMAGENET_NORM = { /** * BlazeFace landmarks list. - * @category Constants + * @category CV / Constants */ export const BLAZEFACE_LANDMARKS = [ 'leftEye', @@ -1268,7 +1275,7 @@ export const BLAZEFACE_LANDMARKS = [ /** * COCO human pose landmarks list. - * @category Constants + * @category CV / Constants */ export const COCO_LANDMARKS = [ 'nose', @@ -1292,19 +1299,19 @@ export const COCO_LANDMARKS = [ /** * Type representing a valid BlazeFace landmark string. - * @category Types + * @category CV / Types */ export type BlazeFaceLandmark = (typeof BLAZEFACE_LANDMARKS)[number]; /** * Type representing a valid COCO human pose landmark string. - * @category Types + * @category CV / Types */ export type CocoLandmark = (typeof COCO_LANDMARKS)[number]; /** * Default Supertonic voice names array. - * @category Constants + * @category Speech / Constants */ // prettier-ignore export const SUPERTONIC_DEFAULT_VOICE_NAMES = [ @@ -1314,7 +1321,7 @@ export const SUPERTONIC_DEFAULT_VOICE_NAMES = [ /** * Type representing a valid Supertonic default voice name. - * @category Types + * @category Speech / Types */ export type SupertonicDefaultVoiceName = (typeof SUPERTONIC_DEFAULT_VOICE_NAMES)[number]; @@ -1328,7 +1335,7 @@ const bioesLabels = (entities: readonly Entity[]) => /** * Label space for the openai/privacy-filter base model (8 entity types, 33 * labels). - * @category Constants + * @category NLP / Constants */ export const PRIVACY_FILTER_OPENAI_LABELS = bioesLabels([ 'account_number', @@ -1344,7 +1351,7 @@ export const PRIVACY_FILTER_OPENAI_LABELS = bioesLabels([ /** * A single BIOES label from the openai/privacy-filter label space (`'O'` or a * `B-`/`I-`/`E-`/`S-` prefixed entity). - * @category Constants + * @category NLP / Types */ export type PrivacyFilterOpenaiLabel = (typeof PRIVACY_FILTER_OPENAI_LABELS)[number]; @@ -1352,7 +1359,7 @@ export type PrivacyFilterOpenaiLabel = (typeof PRIVACY_FILTER_OPENAI_LABELS)[num * Label space for the OpenMed/privacy-filter-nemotron model (55 entity types, * 221 labels). Source: * https://huggingface.co/OpenMed/privacy-filter-nemotron/blob/main/config.json - * @category Constants + * @category NLP / Constants */ export const PRIVACY_FILTER_NEMOTRON_LABELS = bioesLabels([ 'account_number', @@ -1415,6 +1422,6 @@ export const PRIVACY_FILTER_NEMOTRON_LABELS = bioesLabels([ /** * A single BIOES label from the OpenMed/privacy-filter-nemotron label space * (`'O'` or a `B-`/`I-`/`E-`/`S-` prefixed entity). - * @category Constants + * @category NLP / Types */ export type PrivacyFilterNemotronLabel = (typeof PRIVACY_FILTER_NEMOTRON_LABELS)[number]; diff --git a/packages/react-native-executorch/src/core/error.ts b/packages/react-native-executorch/src/core/error.ts index 048e722e91..952f290b01 100644 --- a/packages/react-native-executorch/src/core/error.ts +++ b/packages/react-native-executorch/src/core/error.ts @@ -1,14 +1,13 @@ /** - * Errors raised by React Native ExecuTorch. + * Classified error handling for React Native ExecuTorch. * - * This module is the source of truth for the error contract; `cpp/core/error.h` - * mirrors it by hand, the same way the rest of the TS/JSI interface is mirrored. + * All errors raised by the library carry a machine-readable + * {@link RnExecuTorchErrorCode} (e.g. `LOAD_FAILED`, `EXECUTION_FAILED`, + * `RESOURCE_BUSY`, or `SCHEMA_MISMATCH`) allowing applications to inspect and + * handle failures programmatically. * - * Errors are plain `Error` objects with extra fields rather than a class. - * Worklet runtimes are separate JavaScript runtimes and a value thrown on one - * does not keep its class identity or prototype chain when it travels to - * another, so a class would only work on some of the paths that can throw. - * @packageDocumentation + * Use {@link isRnExecuTorchError} to safely narrow caught errors across + * asynchronous calls and worklet runtime boundaries. */ /** @@ -19,7 +18,7 @@ * resource, re-create a disposed one). Everything else is a category that * exists so crash reporters can group failures, and the detail lives in the * message. - * @category Errors + * @category Core / Constants */ export const VALID_ERROR_CODES = [ 'LOAD_FAILED', @@ -38,14 +37,23 @@ export const VALID_ERROR_CODES = [ * Machine-readable classification of an {@link RnExecuTorchError}. Branch on * this rather than on the message, which is written for humans and can be * reworded in any release. - * @category Errors + * @category Core / Types */ export type RnExecuTorchErrorCode = (typeof VALID_ERROR_CODES)[number]; /** - * An error raised by React Native ExecuTorch: a standard `Error` carrying a - * {@link RnExecuTorchErrorCode}. - * @category Errors + * An error raised by React Native ExecuTorch. + * + * Represents a standard `Error` augmented with a machine-readable + * {@link RnExecuTorchErrorCode} and an optional ExecuTorch C++ runtime code + * (`etRuntimeErrorCode`). + * + * When thrown, call as a factory function without `new` (safe across worklet + * threads): + * ```typescript + * throw RnExecuTorchError('INVALID_ARGUMENT', 'Shape dimensions must be positive'); + * ``` + * @category Core / Types * @typeParam C The specific code, narrowed by {@link isRnExecuTorchError}. */ export type RnExecuTorchError = Error & { @@ -60,13 +68,18 @@ export type RnExecuTorchError( code: C, @@ -91,7 +104,7 @@ export function RnExecuTorchError( * * Duck-typed so it holds for errors that crossed a worklet or JSI boundary, * where class identity is gone. - * @category Errors + * @category Core / Functions * @param err The caught value. * @param code When given, also requires the error to carry exactly this code. * @returns Whether `err` is an `RnExecuTorchError` (of code `code`, if given). diff --git a/packages/react-native-executorch/src/core/model.ts b/packages/react-native-executorch/src/core/model.ts index 5fd59bb2d4..bc63ff04fb 100644 --- a/packages/react-native-executorch/src/core/model.ts +++ b/packages/react-native-executorch/src/core/model.ts @@ -1,3 +1,11 @@ +/** + * Low-level ExecuTorch model loading, execution, and lifetime management. + * + * Provides direct access to compiled `.pte` models in native C++ memory. Loaded + * models expose synchronous method execution (`execute`) with pre-allocated + * output buffers and manual native memory cleanup (`dispose`). + */ + import { rnexecutorchJsi } from '../native/bridge'; import type { Tensor } from './tensor'; import type { ModelSpec, ConcreteDim } from './schema'; @@ -6,13 +14,13 @@ declare const modelBrand: unique symbol; /** * A value that can be passed as an input to a model's `execute` method. - * @category Types + * @category Core / Types */ export type ModelInput = Tensor | number | boolean | null; /** * A value returned from a model's `execute` method. - * @category Types + * @category Core / Types */ export type ModelOutput = Tensor | number | boolean | string | null; @@ -24,10 +32,10 @@ export type ModelOutput = Tensor | number | boolean | string | null; * this interface. * * Obtain a `Model` instance via the {@link loadModel} function. When the model - * is no longer needed call {@link Model.dispose} to release native memory. - * @category Types + * is no longer needed, call {@link Model.dispose} to release native memory. + * @category Core / Types */ -export interface Model { +export type Model = { /** The local filesystem path of the `.pte` model file. */ readonly path: string; /** The exported schema of this model. */ @@ -46,7 +54,11 @@ export interface Model { * @param inputs The list of input values to pass to the method, in order. * @param outputTensors Pre-allocated tensors for the method to write outputs * into, in order. - * @returns The list of output values produced by the method, in order. + * @throws {RnExecuTorchError} Thrown with code `EXECUTION_FAILED` if + * inference fails, `SCHEMA_MISMATCH` if runtime constraints fail, + * `RESOURCE_BUSY` if the model or a tensor is in use, `RESOURCE_DISPOSED` if + * disposed, or `INVALID_ARGUMENT` if inputs or output placeholders are + * invalid. */ execute(methodName: string, inputs: ModelInput[], outputTensors: Tensor[]): ModelOutput[]; @@ -62,7 +74,7 @@ export interface Model { * @internal */ readonly [modelBrand]: never; -} +}; /** * Loads and compiles an ExecuTorch `.pte` model from the local filesystem. @@ -70,9 +82,26 @@ export interface Model { * The model is loaded synchronously into native memory. Prefer calling this * inside a worklet runtime thread (via {@link wrapAsync}) to avoid blocking the * JS thread during compilation. - * @category Typescript API + * @category Core / Functions * @param modelPath The absolute local path to the `.pte` model file. * @returns The compiled {@link Model} instance, ready for execution. + * @throws {RnExecuTorchError} Thrown with code `LOAD_FAILED` if the model file + * cannot be opened, has an invalid format, or fails native initialization. + * @see {@link wrapAsync} + * @example + * ```typescript + * const model = loadModel('/path/to/model.pte'); + * const input = tensor('float32', [1, 3, 224, 224]); + * const output = tensor('float32', [1, 1000]); + * try { + * model.execute('forward', [input], [output]); + * // ... + * } finally { + * input.dispose(); + * output.dispose(); + * model.dispose(); + * } + * ``` */ export function loadModel(modelPath: string): Model { 'worklet'; diff --git a/packages/react-native-executorch/src/core/runtime.ts b/packages/react-native-executorch/src/core/runtime.ts index d4f4335003..9d41837dc9 100644 --- a/packages/react-native-executorch/src/core/runtime.ts +++ b/packages/react-native-executorch/src/core/runtime.ts @@ -1,3 +1,11 @@ +/** + * Background worklet execution and thread runtime management. + * + * Provides utilities to dispatch synchronous, heavy native operations (model + * compilation, tensor inference) onto dedicated background worklet threads, + * preventing them from blocking the React Native JavaScript thread. + */ + import { createWorkletRuntime, runOnRuntimeAsync, @@ -10,9 +18,9 @@ import { isRnExecuTorchError, RnExecuTorchError } from './error'; * * This runtime runs on a dedicated thread separate from the React Native JS * thread, preventing model loading and inference from blocking the UI. Pass it - * explicitly (or a custom {@link WorkletRuntime}) to {@link wrapAsync} when you + * explicitly (or a custom `WorkletRuntime`) to {@link wrapAsync} when you * need fine-grained control over which thread work executes on. - * @category Utils + * @category Core / Constants */ export const defaultWorkletRuntime = createWorkletRuntime({ name: 'ExecuTorchDefaultRuntime', @@ -20,13 +28,13 @@ export const defaultWorkletRuntime = createWorkletRuntime({ /** * Wraps a synchronous worklet function so that it runs asynchronously on a - * background {@link WorkletRuntime} thread and returns a `Promise`. + * background `WorkletRuntime` thread and returns a `Promise`. * * The wrapper serializes arguments, dispatches the worklet to the target * runtime, awaits the result, and re-throws any error thrown inside the worklet * as an {@link RnExecuTorchError}. This keeps heavy native operations (model * loading, tensor computation) off the React Native JS thread. - * @category Utils + * @category Core / Functions * @typeParam Args The tuple of argument types of `fn`. * @typeParam R The return type of `fn`. * @param fn A synchronous worklet function to execute on the background @@ -34,7 +42,14 @@ export const defaultWorkletRuntime = createWorkletRuntime({ * @param runtime The worklet runtime to dispatch `fn` to. Defaults to * {@link defaultWorkletRuntime}. * @returns An async function with the same signature as `fn` that resolves to - * `fn`'s return value or rejects with an `RnExecuTorchError` if `fn` throws. + * `fn`'s return value or rejects with an {@link RnExecuTorchError} if `fn` throws. + * @throws {RnExecuTorchError} Propagates any error thrown inside `fn` across the + * worklet thread boundary. + * @example + * ```typescript + * const asyncLoadModel = wrapAsync(loadModel); + * const model = await asyncLoadModel('/path/to/model.pte'); + * ``` */ export function wrapAsync( fn: (...args: Args) => R, diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index 446e5f615c..6e74bbb62e 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -2,75 +2,70 @@ * Model specs and spec validation. * * A model spec is a structural contract describing a model's methods: the - * parameter specs of every input and output (primitive tags, tensor data - * types, and per-dimension domains) and the runtime constraints the method - * declares over its tensor dimensions. A spec is either: + * parameter specs of every input and output (primitive tags, tensor data types, + * and per-dimension domains) and the runtime constraints the method declares + * over its tensor dimensions. A spec is either: * - **allowed** (`SymbolicDim`) — written by a pipeline to state which models - * it can work with. Dimensions may be named symbols: `static` symbols bind - * to constant dimensions, `dynamic` symbols to ranges or enums, and reusing - * a symbol requires every occurrence to bind to the same domain. Several + * it can work with. Dimensions may be named symbols: `static` symbols bind to + * constant dimensions, `dynamic` symbols to ranges or enums, and reusing a + * symbol requires every occurrence to bind to the same domain. Several * allowed specs can be passed as variants; matching any one of them is * enough. * - **exported** (`ConcreteDim`) — derived from an exported model's metadata, * stating what the model actually provides. * * **Dimension domains vs runtime values.** The central distinction of this - * module is between a dimension's *domain* and its *runtime value*. The - * domain is the set of values the dimension may take: `constant` is a - * singleton (the value is fully known statically), while `range` and `enum` - * are proper sets that only narrow the possibilities. The runtime value is - * the actual size of the dimension for a concrete tensor in one given - * execution — a single element drawn from the domain. + * module is between a dimension's *domain* and its *runtime value*. The domain + * is the set of values the dimension may take: `constant` is a singleton (the + * value is fully known statically), while `range` and `enum` are proper sets + * that only narrow the possibilities. The runtime value is the actual size of + * the dimension for a concrete tensor in one given execution — a single element + * drawn from the domain. * * Accordingly, two different validations must not be confused: * - **Spec validation** (this module, {@link validateSpec}) — a static, * load-time check that an exported spec satisfies an allowed spec. It only * ever compares domains: symbols bind to domains (repeated symbols to the * same one), and constraints are matched as declarations. Domain equality - * says nothing about runtime values — two dimensions with the same domain - * may still take different values in an execution. - * - **Runtime validation** (the native runtime, not this module) — at - * execution time the runtime checks that every concrete tensor shape lies - * within the declared domains and enforces the declared runtime - * constraints. + * says nothing about runtime values — two dimensions with the same domain may + * still take different values in an execution. + * - **Runtime validation** (the native runtime, not this module) — at execution + * time the runtime checks that every concrete tensor shape lies within the + * declared domains and enforces the declared runtime constraints. * * Runtime constraints are statements about runtime values, not domains: an - * equality constraint requires its dimensions' values to coincide in any - * given execution, and a linear constraint requires them to satisfy - * `lhs = a * rhs + b`. Since spec validation only sees domains, it cannot - * decide whether such a relation holds — the exported spec must simply - * declare exactly the allowed spec's constraints (1-to-1, no missing, no - * extras). The only exception is degenerate: a constant domain has exactly - * one possible value, so an equality constraint between constants is fully - * decided statically — equal constants are equal at runtime. Linear - * constraints, in contrast, are never evaluated against domains here, even - * between constants. + * equality constraint requires its dimensions' values to coincide in any given + * execution, and a linear constraint requires them to satisfy `lhs = a * rhs + + * b`. Since spec validation only sees domains, it cannot decide whether such a + * relation holds — the exported spec must simply declare exactly the allowed + * spec's constraints (1-to-1, no missing, no extras). The only exception is + * degenerate: a constant domain has exactly one possible value, so an equality + * constraint between constants is fully decided statically — equal constants + * are equal at runtime. Linear constraints, in contrast, are never evaluated + * against domains here, even between constants. * - * **Exported spec source.** An exported model's `ModelSpec` - * is populated at load time from one of two sources: + * **Exported spec source.** An exported model's `ModelSpec` is + * populated at load time from one of two sources: * * 1. **ExecuTorch `MethodMeta`** (default) — when the `.pte` only carries - * static metadata, every dimension domain is `constant`. This is - * sufficient for models whose input/output shapes are fully fixed at - * export time. + * static metadata, every dimension domain is `constant`. This is sufficient + * for models whose input/output shapes are fully fixed at export time. * - * 2. **Companion `get_model_schema` method** — for models whose tensors - * have dynamic or enumerated dimensions (e.g. variable-length sequences) - * or that declare runtime constraints, the `.pte` exports a method named + * 2. **Companion `get_model_schema` method** — for models whose tensors have + * dynamic or enumerated dimensions (e.g. variable-length sequences) or that + * declare runtime constraints, the `.pte` exports a method named * `get_model_schema` that returns a JSON-encoded `ModelSpec` * string. The native loader calls this method after loading the model and - * merges the result into `model.schema`, overlaying precise `range`, - * `enum`, and `RuntimeConstraint` entries onto the base `MethodMeta`. - * Only methods that actually need overrides need to appear in the JSON; - * methods absent from the companion spec are kept as-is from - * `MethodMeta`. + * merges the result into `model.schema`, overlaying precise `range`, `enum`, + * and `RuntimeConstraint` entries onto the base `MethodMeta`. Only methods + * that actually need overrides need to appear in the JSON; methods absent + * from the companion spec are kept as-is from `MethodMeta`. * * **Tip for model export:** * Embed the companion schema method during ExecuTorch compilation in Python * by passing `constant_methods={"get_model_schema": schema_json}` when * lowering with `to_edge_transform_and_lower(...)` where `schema_json` is * the JSON string encoding the model's `ModelSpec`. - * @packageDocumentation */ import type { DType } from './tensor'; import { RnExecuTorchError } from './error'; @@ -82,7 +77,7 @@ import { RnExecuTorchError } from './error'; /** * Inclusive integer domain of a single dynamic dimension — values from `min` * to `max` in increments of `step`. - * @category Types + * @category Core / Schema / Types */ export type Range = { readonly min: number; readonly max: number; readonly step: number }; @@ -91,7 +86,7 @@ export type Range = { readonly min: number; readonly max: number; readonly step: * - `constant` — exactly `value`. * - `range` — any value of a {@link Range}. * - `enum` — one of the listed `choices`. - * @category Types + * @category Core / Schema / Types */ export type ConcreteDim = | { readonly kind: 'constant'; readonly value: number } @@ -104,7 +99,7 @@ export type ConcreteDim = * validation: `static` symbols bind to constants, `dynamic` symbols to ranges * or enums. Reusing a symbol requires every occurrence to bind to the same * domain — it does NOT imply any runtime relation between the dimensions. - * @category Types + * @category Core / Schema / Types */ export type SymbolicDim = | ConcreteDim @@ -114,7 +109,7 @@ export type SymbolicDim = /** * Spec of a tensor parameter: the expected element `dtype` and one * dimension spec per axis. - * @category Types + * @category Core / Schema / Types */ export type TensorSpec = { readonly kind: 'Tensor'; @@ -125,7 +120,7 @@ export type TensorSpec = { /** * The ExecuTorch value-tag that classifies the runtime type of a model input or * output slot. - * @category Types + * @category Core / Schema / Types */ export type ExecuTorchTag = | 'None' @@ -142,7 +137,7 @@ export type ExecuTorchTag = /** * Spec of a single input or output parameter of a method — either a * {@link TensorSpec} or a primitive ExecuTorch value tag (`Int`, `Bool`, ...). - * @category Types + * @category Core / Schema / Types */ export type ParamSpec = | TensorSpec @@ -156,7 +151,7 @@ export type ParamSpec = * Reference to a single tensor dimension of a method's input or output. * `tensorIdx` counts only tensor parameters (skipping primitives), consistent * with ExecuTorch's `inputTensorMeta` / `outputTensorMeta` ordering. - * @category Types + * @category Core / Schema / Types */ export type DimRef = { readonly paramSide: 'input' | 'output'; @@ -167,7 +162,7 @@ export type DimRef = { /** * Runtime constraint declaring that all referenced dimensions must be equal * to each other in any given execution of the method. - * @category Types + * @category Core / Schema / Types */ export type EqualityConstraint = { readonly kind: 'equality'; @@ -178,7 +173,7 @@ export type EqualityConstraint = { * Runtime constraint declaring that two dimensions must satisfy * `dimLhs = coefficients[0] * dimRhs + coefficients[1]` (integer * coefficients) in any given execution of the method. - * @category Types + * @category Core / Schema / Types */ export type LinearConstraint = { readonly kind: 'linear'; @@ -191,7 +186,7 @@ export type LinearConstraint = { * A requirement on the runtime values of a method's tensor dimensions: the * concrete tensors passed to and produced by the method must satisfy it in * any given execution. Matched as a declaration during spec validation. - * @category Types + * @category Core / Schema / Types */ export type RuntimeConstraint = LinearConstraint | EqualityConstraint; @@ -202,7 +197,7 @@ export type RuntimeConstraint = LinearConstraint | EqualityConstraint; /** * Spec of a single model method: the ordered input and output parameter specs * and the runtime constraints the method declares over its tensor dimensions. - * @category Types + * @category Core / Schema / Types */ export type MethodSpec = { inputs: readonly ParamSpec[]; @@ -214,7 +209,7 @@ export type MethodSpec = { * Spec of a whole model, mapping method names to their {@link MethodSpec}. * A `SymbolicDim` spec describes allowed models; a `ConcreteDim` spec * describes an exported model. - * @category Types + * @category Core / Schema / Types */ export type ModelSpec = Record>; @@ -226,14 +221,14 @@ export type ModelSpec = Record> * Shape notation accepted by {@link SymbolicTensor}: numbers become * {@link ConstantDim}, strings become {@link StaticDim}, and * {@link SymbolicDim} values are used as-is. - * @category Types + * @category Core / Schema / Types */ export type SymbolicShape = readonly (number | string | SymbolicDim)[]; /** * Creates a static symbolic dimension. Static symbols bind to constant * dimensions of the exported spec; repeated uses must bind to the same value. - * @category Typescript API + * @category Core / Schema / Functions * @param symbol The symbol name. * @returns The symbolic dimension. */ @@ -244,7 +239,7 @@ export const StaticDim = (symbol: string): SymbolicDim => { /** * Creates a dynamic symbolic dimension. Dynamic symbols bind to range or enum * dimensions of the exported spec; repeated uses must bind to the same domain. - * @category Typescript API + * @category Core / Schema / Functions * @param symbol The symbol name. * @returns The symbolic dimension. */ @@ -254,7 +249,7 @@ export const DynamicDim = (symbol: string): SymbolicDim => { /** * Creates a constant dimension matching exactly `value`. - * @category Typescript API + * @category Core / Schema / Functions * @param value The required dimension size. * @returns The concrete dimension. * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if `value` @@ -272,7 +267,7 @@ export const ConstantDim = (value: number): ConcreteDim => { /** * Creates an enumerated dimension matching one of `choices`. - * @category Typescript API + * @category Core / Schema / Functions * @param choices The allowed dimension sizes. * @returns The concrete dimension. * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if any @@ -288,7 +283,7 @@ export const EnumDim = (choices: readonly number[]): ConcreteDim => { /** * Creates a range dimension matching values from `min` to `max` in increments * of `step`. - * @category Typescript API + * @category Core / Schema / Functions * @param min The smallest allowed dimension size. * @param max The largest allowed dimension size. * @param step The increment between allowed sizes. Defaults to 1. @@ -327,7 +322,7 @@ export const RangeDim = (min: number, max: number, step?: number): ConcreteDim = /** * Creates a {@link TensorSpec} from a dtype and a {@link SymbolicShape}: * numbers become {@link ConstantDim}, strings become {@link StaticDim}. - * @category Typescript API + * @category Core / Schema / Functions * @param dtype The expected element data type. * @param shape The per-dimension specs. * @returns The tensor spec. @@ -341,17 +336,64 @@ export const SymbolicTensor = (dtype: DType, shape: SymbolicShape) => { return { kind: 'Tensor', dtype, shape: typedShape } as TensorSpec; }; +/** + * Shorthand for `SymbolicTensor('float32', shape)`. + * @category Core / Schema / Functions + * @param shape Dimension sizes of the tensor. + * @returns A {@link SymbolicTensor} with `float32` data type. + */ export const f32 = (...shape: SymbolicShape) => SymbolicTensor('float32', shape); +/** + * Shorthand for `SymbolicTensor('int64', shape)`. + * @category Core / Schema / Functions + * @param shape Dimension sizes of the tensor. + * @returns A {@link SymbolicTensor} with `int64` data type. + */ export const i64 = (...shape: SymbolicShape) => SymbolicTensor('int64', shape); +/** + * Shorthand for `SymbolicTensor('int32', shape)`. + * @category Core / Schema / Functions + * @param shape Dimension sizes of the tensor. + * @returns A {@link SymbolicTensor} with `int32` data type. + */ export const i32 = (...shape: SymbolicShape) => SymbolicTensor('int32', shape); +/** + * Shorthand for `SymbolicTensor('uint8', shape)`. + * @category Core / Schema / Functions + * @param shape Dimension sizes of the tensor. + * @returns A {@link SymbolicTensor} with `uint8` data type. + */ export const ui8 = (...shape: SymbolicShape) => SymbolicTensor('uint8', shape); +/** + * Shorthand for `SymbolicTensor('bool', shape)`. + * @category Core / Schema / Functions + * @param shape Dimension sizes of the tensor. + * @returns A {@link SymbolicTensor} with `bool` data type. + */ export const bool = (...shape: SymbolicShape) => SymbolicTensor('bool', shape); -/** Helper namespace for declaring runtime constraints. */ -export const constr = { - eq: (...dims: DimRef[]): EqualityConstraint => { +/** + * Helper namespace for declaring runtime constraints. + * @category Core / Schema / Functions + */ +export const constraint = { + /** + * Declares that all given dimensions must be equal at runtime. + * @param dims Dimensions that must share the same concrete value. + * @returns An {@link EqualityConstraint} across the given dimensions. + */ + equality: (...dims: DimRef[]): EqualityConstraint => { return { kind: 'equality', dims }; }, + /** + * Declares a linear relation between two dimensions: `dimLhs = a * dimRhs + + * b`. + * @param dimLhs The left-hand-side dimension. + * @param dimRhs The right-hand-side dimension. + * @param a Slope coefficient. + * @param b Intercept coefficient (defaults to `0`). + * @returns A {@link LinearConstraint} relating the two dimensions. + */ linear: (dimLhs: DimRef, dimRhs: DimRef, a: number, b: number = 0): LinearConstraint => { return { kind: 'linear', dimLhs, dimRhs, coefficients: [a, b] }; }, @@ -360,6 +402,7 @@ export const constr = { /** * Constructs a method specification mapping a method name to its parameter * specs and runtime constraints. + * @category Core / Schema / Functions * @param name The execution method name (e.g., `'forward'`). * @param inputs Ordered list of parameter specifications for method inputs. * @param outputs Ordered list of parameter specifications for method outputs. @@ -604,8 +647,8 @@ function matchRuntimeConstraints( const unclaimed = [...exportedMethodSpec.runtimeConstraints]; - for (const [idx, constraint] of allowedMethodSpec.runtimeConstraints.entries()) { - const find = unclaimed.findIndex((c) => constraintsEqual(c, constraint)); + for (const [idx, rc] of allowedMethodSpec.runtimeConstraints.entries()) { + const find = unclaimed.findIndex((c) => constraintsEqual(c, rc)); if (find === -1) { throw RnExecuTorchError( 'SCHEMA_MISMATCH', @@ -653,26 +696,26 @@ function validateSymbolKindConsistency(modelSpec: ModelSpec): void function validateConstraintCorrectness(modelSpec: ModelSpec): void { for (const [methodName, methodSpec] of Object.entries(modelSpec)) { - for (const [idx, constraint] of methodSpec.runtimeConstraints.entries()) { + for (const [idx, rc] of methodSpec.runtimeConstraints.entries()) { const ctx = `Method '${methodName}' constraint ${idx}`; - if (constraint.kind === 'linear') { - const [A, B] = constraint.coefficients; + if (rc.kind === 'linear') { + const [A, B] = rc.coefficients; if (!Number.isInteger(A) || !Number.isInteger(B)) { throw RnExecuTorchError('SCHEMA_MISMATCH', `${ctx}: Coefficients must be integers.`); } - resolveDim(methodSpec, constraint.dimLhs); - resolveDim(methodSpec, constraint.dimRhs); + resolveDim(methodSpec, rc.dimLhs); + resolveDim(methodSpec, rc.dimRhs); } - if (constraint.kind === 'equality') { - if (constraint.dims.length < 2) { + if (rc.kind === 'equality') { + if (rc.dims.length < 2) { throw RnExecuTorchError( 'SCHEMA_MISMATCH', `${ctx}: Equality requires at least two dimensions.` ); } - constraint.dims.forEach((ref) => resolveDim(methodSpec, ref)); + rc.dims.forEach((ref) => resolveDim(methodSpec, ref)); } } } @@ -737,6 +780,7 @@ function validateDimDomains(modelSpec: ModelSpec): void { /** * Result of validating an exported model spec against allowed variants. + * @category Core / Schema / Types * @typeParam K The variant key type. */ export type SpecMatch = { @@ -810,6 +854,72 @@ export type SpecMatch = { }; }; +function resolveSymbolDim(bindings: SymbolBindings, name: string, kind?: string): any { + const dim = bindings.get(name); + if (!dim) { + throw RnExecuTorchError('INVALID_ARGUMENT', `Symbol '${name}' not found in bindings.`); + } + if (kind === 'constant') { + if (dim.kind !== 'constant') { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Symbol '${name}' is '${dim.kind}', expected 'constant'.` + ); + } + return dim.value; + } + if (kind === 'range') { + if (dim.kind !== 'range') { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Symbol '${name}' is '${dim.kind}', expected 'range'.` + ); + } + return dim.range; + } + if (kind === 'enum') { + if (dim.kind !== 'enum') { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Symbol '${name}' is '${dim.kind}', expected 'enum'.` + ); + } + return dim.choices; + } + if (kind === 'dynamic') { + if (dim.kind === 'constant') { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Symbol '${name}' is 'constant', expected 'dynamic'.` + ); + } + return dim; + } + return dim; +} + +function createSpecMatch( + variant: K, + bindings: SymbolBindings +): SpecMatch { + const dim: any = (name: string, kind?: string) => resolveSymbolDim(bindings, name, kind); + const createAccessor = (kind?: string) => { + return (...names: string[]): any => names.map((name) => resolveSymbolDim(bindings, name, kind)); + }; + + return { + variant, + dim, + dims: { + any: createAccessor(), + enum: createAccessor('enum'), + range: createAccessor('range'), + dynamic: createAccessor('dynamic'), + constant: createAccessor('constant'), + }, + }; +} + /** * Validates that an exported (concrete) model spec satisfies at least one of * the allowed (symbolic) model specs — variants are tried in order and the @@ -829,6 +939,7 @@ export type SpecMatch = { * accessors. * @throws {RnExecuTorchError} With code `SCHEMA_MISMATCH`, describing * why every variant failed. + * @category Core / Schema / Functions */ export function validateSpec>>( exportedModelSpec: ModelSpec, @@ -852,49 +963,7 @@ export function validateSpec { - const dim = bindings.get(name); - if (!dim) { - throw RnExecuTorchError('INVALID_ARGUMENT', `Symbol '${name}' not found in bindings.`); - } - if (kind) { - if (kind === 'dynamic') { - if (dim.kind === 'constant') { - throw RnExecuTorchError( - 'INVALID_ARGUMENT', - `Symbol '${name}' is 'constant', expected 'dynamic'.` - ); - } - return dim; - } - if (dim.kind !== kind) { - throw RnExecuTorchError( - 'INVALID_ARGUMENT', - `Symbol '${name}' is '${dim.kind}', expected '${kind}'.` - ); - } - } - if (dim.kind === 'constant') return dim.value; - if (dim.kind === 'range') return dim.range; - if (dim.kind === 'enum') return dim.choices; - return dim; - }; - - const createAccessor = (kind?: string) => { - return (...names: string[]): any => names.map((name) => dimFn(name, kind)); - }; - - return { - variant: key, - dim: dimFn, - dims: { - any: createAccessor(), - enum: createAccessor('enum'), - range: createAccessor('range'), - dynamic: createAccessor('dynamic'), - constant: createAccessor('constant'), - }, - }; + return createSpecMatch(key, bindings); } catch (e: any) { errors.push(`Variant '${key}': ${e.message}`); continue; diff --git a/packages/react-native-executorch/src/core/tensor.ts b/packages/react-native-executorch/src/core/tensor.ts index b54df997c5..2fcbeb21ef 100644 --- a/packages/react-native-executorch/src/core/tensor.ts +++ b/packages/react-native-executorch/src/core/tensor.ts @@ -1,10 +1,18 @@ +/** + * Native C++ tensor allocation, data transfers, and memory management. + * + * Tensors are the fundamental data structures used throughout React Native + * ExecuTorch. They hold multidimensional typed arrays allocated in native + * heap memory and provide transfers to and from JavaScript typed arrays. + */ + import { rnexecutorchJsi } from '../native/bridge'; declare const tensorBrand: unique symbol; /** * Element data type of a {@link Tensor}. - * @category Types + * @category Core / Types */ export type DType = 'float32' | 'uint8' | 'int32' | 'int64' | 'bool'; @@ -17,7 +25,7 @@ export type DType = 'float32' | 'uint8' | 'int32' | 'int64' | 'bool'; * {@link Tensor.dispose} when no longer needed to avoid native memory leaks. * * Create tensors with the {@link tensor} factory function. - * @category Types + * @category Core / Types */ export type Tensor = { /** The element data type of the tensor. */ @@ -37,6 +45,10 @@ export type Tensor = { * `numel - offset`, i.e. copies from `offset` to the end of the source * tensor. * @returns The destination tensor `dst`. + * @throws {RnExecuTorchError} Thrown with code `INVALID_ARGUMENT` if the copy + * bounds exceed the tensor size or data types mismatch, `RESOURCE_BUSY` if + * either tensor is in use, or `RESOURCE_DISPOSED` if either tensor was + * disposed. */ copyTo(dst: Tensor, options?: { offset?: number; length?: number }): Tensor; @@ -53,6 +65,9 @@ export type Tensor = { * tensor's size. Use a `BigInt64Array` for `int64` tensors and a * `Uint8Array` for `bool` tensors. * @returns `this` tensor. + * @throws {RnExecuTorchError} Thrown with code `INVALID_ARGUMENT` if `src` + * byte length does not match tensor size, `RESOURCE_BUSY` if the tensor is in + * use, or `RESOURCE_DISPOSED` if disposed. */ setData(src: Float32Array | Uint8Array | Int32Array | BigInt64Array): Tensor; @@ -62,6 +77,9 @@ export type Tensor = { * @param dst The destination typed array. Its size in bytes must match * tensor's size. * @returns The same `dst` array, now filled with tensor data. + * @throws {RnExecuTorchError} Thrown with code `INVALID_ARGUMENT` if `dst` + * byte length does not match tensor size, `RESOURCE_BUSY` if the tensor is in + * use, or `RESOURCE_DISPOSED` if disposed. */ getData(dst: T): T; @@ -106,12 +124,25 @@ export type Tensor = { * `src` is omitted the buffer contents are undefined. The returned tensor * resides in native C++ memory; call {@link Tensor.dispose} when the tensor is * no longer needed. - * @category Typescript API + * @category Core / Functions * @param dtype The element data type of the tensor. * @param shape An array of dimension sizes (e.g. `[1, 3, 224, 224]`). * @param src Optional typed array used to initialize the tensor's data. Its * size in bytes must match tensor's size. * @returns A newly allocated native tensor. + * @throws {RnExecuTorchError} Thrown with code `INVALID_ARGUMENT` if any + * dimension in `shape` is non-positive or if `src` byte length does not match + * the allocated tensor size. + * @example + * ```typescript + * const t = tensor('float32', [1, 4], new Float32Array([1.0, 2.0, 3.0, 4.0])); + * try { + * const data = t.getData(new Float32Array(4)); + * console.log(data); // Float32Array [1, 2, 3, 4] + * } finally { + * t.dispose(); + * } + * ``` */ export function tensor( dtype: DType, diff --git a/packages/react-native-executorch/src/extensions/cv/image.ts b/packages/react-native-executorch/src/extensions/cv/image.ts index 54856b02e4..218974c7d9 100644 --- a/packages/react-native-executorch/src/extensions/cv/image.ts +++ b/packages/react-native-executorch/src/extensions/cv/image.ts @@ -1,12 +1,16 @@ +/** + * Core image buffer types and pixel formats for Computer Vision. + */ + /** * Supported pixel format layouts for image buffers. - * @category Types + * @category CV / Types */ export type ImageFormat = 'rgb' | 'rgba' | 'bgr' | 'bgra' | 'gray'; /** * Represents a raw CPU image buffer in HWC (Height, Width, Channel) layout. - * @category Types + * @category CV / Types */ export type ImageBuffer = { readonly data: Uint8Array; diff --git a/packages/react-native-executorch/src/extensions/cv/index.ts b/packages/react-native-executorch/src/extensions/cv/index.ts index 58aa2b181d..6370cfd125 100644 --- a/packages/react-native-executorch/src/extensions/cv/index.ts +++ b/packages/react-native-executorch/src/extensions/cv/index.ts @@ -1,3 +1,9 @@ +/** + * Computer Vision extension providing image buffers, spatial transformations, + * OpenCV operators, bounding box utilities, and preprocessing pipelines. + */ + export * from './image'; export * from './ops'; export * from './utils/paddleOcrUtils'; +export * from './utils/imagePreprocessor'; diff --git a/packages/react-native-executorch/src/extensions/cv/ops/boxes.ts b/packages/react-native-executorch/src/extensions/cv/ops/box.ts similarity index 68% rename from packages/react-native-executorch/src/extensions/cv/ops/boxes.ts rename to packages/react-native-executorch/src/extensions/cv/ops/box.ts index 1151c4a30a..61da7dae1e 100644 --- a/packages/react-native-executorch/src/extensions/cv/ops/boxes.ts +++ b/packages/react-native-executorch/src/extensions/cv/ops/box.ts @@ -1,51 +1,54 @@ +/** + * Bounding box coordinate decoding, coordinate transforms, and Non-Maximum + * Suppression (NMS). + */ + import { rnexecutorchJsi } from '../../../native/bridge'; import type { Tensor } from '../../../core/tensor'; import type { ResizeMode } from './image'; -import { scalePoint } from './points'; +import { scalePoint } from './point'; /** * Mapping of bounding box formats to their coordinate representations. - * @category Types + * @category CV / Types */ -export type BoxMap = { - xyxy: { - readonly xmin: number; - readonly ymin: number; - readonly xmax: number; - readonly ymax: number; - }; - xywh: { - readonly xmin: number; - readonly ymin: number; - readonly w: number; - readonly h: number; - }; - cxcywh: { - readonly cx: number; - readonly cy: number; - readonly w: number; - readonly h: number; - }; -}; +export type BoxMap = Readonly<{ + xyxy: Readonly<{ xmin: number; ymin: number; xmax: number; ymax: number }>; + xywh: Readonly<{ xmin: number; ymin: number; w: number; h: number }>; + cxcywh: Readonly<{ cx: number; cy: number; w: number; h: number }>; +}>; /** * The formats of bounding boxes. - * @category Types + * @category CV / Types */ export type BoxFormat = keyof BoxMap; /** * Representation of a bounding box under a specific format. - * @category Types + * @category CV / Types */ export type BoundingBox = F extends any - ? { readonly format: F } & Readonly + ? { readonly format: F } & BoxMap[F] : never; +/** + * Configuration options for scaling bounding box coordinates. + * @category CV / Types + */ +export type ScaleBoxOptions = { + /** The source bounds (e.g. model input dimensions). */ + readonly from: { readonly width: number; readonly height: number }; + /** The destination bounds (e.g. original image dimensions). */ + readonly to: { readonly width: number; readonly height: number }; + /** The mode used to resize the image (excluding `'crop'`). */ + readonly resizeMode: Exclude; +}; + /** * Decodes bounding box coordinates from a 4-tuple into a structured BoundingBox * object. - * @category Utils + * @category CV / Functions * @typeParam F Bounding box coordinate format. * @param tuple A 4-tuple array containing coordinates. * @param format The coordinate format to decode into. @@ -69,23 +72,16 @@ export function decodeBox( /** * Scales bounding box coordinates based on scaling options and resize modes. - * @category Utils + * @category CV / Functions * @typeParam F Bounding box coordinate format. * @param box The original BoundingBox. * @param options Options defining dimensions and resize modes. - * @param options.from The source bounds (e.g. model input dimensions). - * @param options.to The destination bounds (e.g. original image dimensions). - * @param options.resizeMode The mode used to resize the image {@link ResizeMode} - * (excluding `'crop'`). + * See {@link ScaleBoxOptions}. * @returns The scaled BoundingBox object. */ export function scaleBox( box: BoundingBox, - options: { - readonly from: { readonly width: number; readonly height: number }; - readonly to: { readonly width: number; readonly height: number }; - readonly resizeMode: Exclude; - } + options: ScaleBoxOptions ): BoundingBox { 'worklet'; const { from, to, resizeMode } = options; @@ -142,7 +138,7 @@ export function scaleBox( /** * Options for Non-Maximum Suppression (NMS). - * @category Types + * @category CV / Types */ export type NmsOptions = { /** How bounding box coordinates are interpreted {@link BoxFormat}. */ @@ -161,16 +157,13 @@ export type NmsOptions = { /** * Executes Non-Maximum Suppression (NMS) on bounding boxes and confidence * scores. - * @category Utils - * @param boxes Bounding boxes coordinate tensor. - * @param scores Bounding boxes confidence scores tensor. + * @category CV / Functions + * @param boxes Bounding boxes coordinate tensor. Expected shape `[N, 4]` and + * data type `float32`. + * @param scores Bounding boxes confidence scores tensor. Expected shape `[N]` + * (1D) and data type `float32`. * @param options Options configuring NMS thresholds and execution mode. - * @param options.boxFormat The bounding box format {@link BoxFormat}. - * @param options.iouThreshold Intersection over Union (IoU) threshold for - * suppression. - * @param options.confidenceThreshold Minimum confidence score for candidate - * selection. - * @param options.nmsType The NMS algorithm variant {@link NmsOptions.nmsType}. + * See {@link NmsOptions}. * @returns The resulting indices of the non-suppressed boxes: * - For `standard` NMS: A 1D array of indices (`number[]`) representing the * selected boxes. @@ -178,6 +171,9 @@ export type NmsOptions = { * groups of overlapping boxes, where the first element of each group is the * top candidate and the group indices are used to calculate the weighted * average of coordinates. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes or + * formats are invalid, `RESOURCE_BUSY` if a tensor is in use, or + * `RESOURCE_DISPOSED` if either tensor was disposed. */ export function nms( boxes: Tensor, @@ -195,18 +191,23 @@ export function nms(boxes: Tensor, scores: Tensor, options: NmsOptions): number[ } /** - * Masks the source tensor by keeping only the elements inside the specified - * bounding box, writing the result to a pre-allocated destination tensor. + * Masks the source image tensor by keeping only the elements inside the specified + * bounding box, writing the result to a pre-allocated destination image tensor. * - * Note: This operation does not change the tensor dimensions (it does not crop + * Note: This operation does not change the image tensor dimensions (it does not crop * the shape). Instead, it copies the elements within the box coordinates from * `src` to `dst`, and sets all elements outside the box to `0`. - * @category Typescript API - * @param src The source tensor of shape [H, W, C]. - * @param dst The pre-allocated destination tensor of shape [H, W, C] and the - * same data type as `src`. + * @category CV / Functions + * @param src The source image tensor in HWC layout. Expected shape `[H, W, C]` + * (channels-last). Supports any numeric data type. + * @param dst The pre-allocated destination image tensor to write masked values to. + * Expected shape `[H, W, C]` in HWC layout and the same data type as `src`. * @param box The bounding box defining the region of interest to copy. - * @returns The destination tensor containing the masked output. + * @returns The destination image tensor containing the masked output of shape + * `[H, W, C]` and matching data type. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes, + * layouts, or data types are invalid, `RESOURCE_BUSY` if a tensor is in use, or + * `RESOURCE_DISPOSED` if either tensor was disposed. */ export function restrictToBox(src: Tensor, dst: Tensor, box: BoundingBox): Tensor { 'worklet'; diff --git a/packages/react-native-executorch/src/extensions/cv/ops/image.ts b/packages/react-native-executorch/src/extensions/cv/ops/image.ts index b1e5a4ee34..e15421db1b 100644 --- a/packages/react-native-executorch/src/extensions/cv/ops/image.ts +++ b/packages/react-native-executorch/src/extensions/cv/ops/image.ts @@ -1,10 +1,18 @@ +/** + * Low-level image manipulation and transformation operators. + * + * Provides native OpenCV-accelerated image operations on tensors, including + * spatial resizing, color space conversion, channel transposition (HWC/CHW), + * pixel normalization, and colormap application. + */ + import { rnexecutorchJsi } from '../../../native/bridge'; import type { Tensor } from '../../../core/tensor'; import type { ImageFormat } from '../image'; /** * Supported color conversion code presets (similar to OpenCV). - * @category Types + * @category CV / Types */ export type ColorConversionCode = | 'RGBA2RGB' @@ -59,40 +67,43 @@ export const FORMAT_CHANNELS: Record = { /** * Modes for resizing an image tensor to match target dimensions. - * @category Types + * @category CV / Types */ export type ResizeMode = 'stretch' | 'letterbox' | 'crop'; /** * Interpolation algorithms used during image resizing. - * @category Types + * @category CV / Types */ export type InterpolationMethod = 'nearest' | 'area' | 'cubic' | 'lanczos' | 'linear'; /** * Configuration options for image resize operations. - * @category Types + * @category CV / Types */ export type ResizeOptions = { - /** How the image is resized {@link ResizeMode}. */ + /** How the image is resized (stretch, letterbox, or crop). */ readonly mode?: ResizeMode; - /** Background fill value used when letterboxing. */ + /** Background fill value used when letterboxing (padding). */ readonly padValue?: number; - /** Pixel interpolation method {@link InterpolationMethod}. */ + /** Pixel interpolation method. */ readonly interpolation?: InterpolationMethod; }; /** * Configuration options for image tensor normalization. - * @category Types + * @category CV / Types */ export type NormalizeOptions = { /** * Multiplicative coefficient applied as `pixel * alpha`. Single value for - * uniform scaling, array for per-channel. + * uniform scaling across all channels, or per-channel array. */ readonly alpha?: number | readonly number[]; - /** Additive offset applied as `pixel * alpha + beta`. Single value or per-channel array. */ + /** + * Additive offset applied as `pixel * alpha + beta`. Single value or + * per-channel array. + */ readonly beta?: number | readonly number[]; }; @@ -100,18 +111,22 @@ export type NormalizeOptions = { * Resizes an image tensor from a source dimension to a destination dimension. * * Supports various {@link ResizeMode} and {@link InterpolationMethod} options. - * @category Typescript API - * @param src The source image tensor in HWC layout. Shape [H,W,C]. - * @param dst The pre-allocated destination tensor to write the resized image - * to. `dst` must be in HWC layout and its number of channels must match `src`. - * Shape [H',W',C]. - * @param options Configuration options for resizing. - * @param options.mode The resize algorithm mode {@link ResizeMode}. Defaults to - * `'stretch'`. - * @param options.interpolation The pixel interpolation method - * {@link InterpolationMethod}. Defaults to `'lanczos'`. - * @param options.padValue Fill value for letterboxing. Defaults to `0`. - * @returns The destination tensor containing the resized image. + * @category CV / Functions + * @param src The source image tensor in HWC layout. Expected shape `[H, W, C]` + * (channels-last). Supports any numeric data type (e.g. `uint8`, `float32`). + * @param dst The pre-allocated destination image tensor to write the resized + * image to. Expected shape `[H', W', C]` in HWC layout (spatial dimensions + * `[H', W']`, channel count `C` matching `src`) and the same data type as + * `src`. + * @param options Configuration options for resizing. When options or any + * individual properties are omitted, defaults to `'stretch'` mode, `'lanczos'` + * interpolation, and `0` padding. + * See {@link ResizeOptions}. + * @returns The destination image tensor containing the resized image of shape + * `[H', W', C]` and matching data type. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes, + * layouts, or data types are invalid, `RESOURCE_BUSY` if a tensor is in use, or + * `RESOURCE_DISPOSED` if either tensor was disposed. */ export function resize(src: Tensor, dst: Tensor, options?: ResizeOptions): Tensor { 'worklet'; @@ -125,14 +140,20 @@ export function resize(src: Tensor, dst: Tensor, options?: ResizeOptions): Tenso /** * Converts the color space of an image tensor using a specified color * conversion code. - * @category Typescript API - * @param src The source image tensor in HWC layout. Shape [H,W,C]. - * @param dst The pre-allocated destination tensor to write the converted image - * to. `dst` must be in HWC layout and its spatial dimensions [H,W] as well as - * dtype must match `src`. Shape [H,W,C']. - * @param code The color conversion code indicating source and target spaces - * (e.g. 'RGBA2RGB'). - * @returns The destination tensor containing the converted image. + * @category CV / Functions + * @param src The source image tensor in HWC layout. Expected shape `[H, W, C]` + * (channels-last). Supports any numeric data type. + * @param dst The pre-allocated destination image tensor to write the converted + * image to. Expected shape `[H, W, C']` in HWC layout (spatial dimensions `[H, + * W]` and data type matching `src`, with target channel count `C'` determined + * by `code`). + * @param code The color conversion code indicating source and target spaces. + * See {@link ColorConversionCode}. + * @returns The destination image tensor containing the converted image of shape + * `[H, W, C']` and matching data type. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes, + * layouts, or data types are invalid, `RESOURCE_BUSY` if a tensor is in use, or + * `RESOURCE_DISPOSED` if either tensor was disposed. */ export function cvtColor(src: Tensor, dst: Tensor, code: ColorConversionCode): Tensor { 'worklet'; @@ -144,12 +165,16 @@ export function cvtColor(src: Tensor, dst: Tensor, code: ColorConversionCode): T * (Channel, Height, Width). * * Commonly required for PyTorch Edge models which expect channels-first inputs. - * @category Typescript API - * @param src The source image tensor in HWC layout. Shape [H,W,C]. - * @param dst The pre-allocated destination tensor in CHW layout. `dst` tensor's - * spatial dimensions [H,W], number of channels and dtype must match `src`. - * Shape [C,H,W]. - * @returns The destination tensor in CHW layout. + * @category CV / Functions + * @param src The source image tensor in HWC layout. Expected shape `[H, W, C]` + * (channels-last). Supports any numeric data type. + * @param dst The pre-allocated destination image tensor in CHW layout. Expected + * shape `[C, H, W]` (channels-first) and the same data type as `src`. + * @returns The destination image tensor in CHW layout of shape `[C, H, W]` and + * matching data type. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes, + * layouts, or data types are invalid, `RESOURCE_BUSY` if a tensor is in use, or + * `RESOURCE_DISPOSED` if either tensor was disposed. */ export function toChannelsFirst(src: Tensor, dst: Tensor): Tensor { 'worklet'; @@ -162,12 +187,16 @@ export function toChannelsFirst(src: Tensor, dst: Tensor): Tensor { * * Useful for post-processing model outputs back into channels-last layouts for * rendering or display. - * @category Typescript API - * @param src The source image tensor in CHW layout. Shape [C,H,W]. - * @param dst The pre-allocated destination tensor in HWC layout. `dst` tensor's - * spatial dimensions [H,W], number of channels and dtype must match `src`. - * Shape [H,W,C]. - * @returns The destination tensor in HWC layout. + * @category CV / Functions + * @param src The source image tensor in CHW layout. Expected shape `[C, H, W]` + * (channels-first). Supports any numeric data type. + * @param dst The pre-allocated destination image tensor in HWC layout. Expected + * shape `[H, W, C]` (channels-last) and the same data type as `src`. + * @returns The destination image tensor in HWC layout of shape `[H, W, C]` and + * matching data type. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes, + * layouts, or data types are invalid, `RESOURCE_BUSY` if a tensor is in use, or + * `RESOURCE_DISPOSED` if either tensor was disposed. */ export function toChannelsLast(src: Tensor, dst: Tensor): Tensor { 'worklet'; @@ -180,15 +209,22 @@ export function toChannelsLast(src: Tensor, dst: Tensor): Tensor { * Computes: `dst[c,h,w] = src[c,h,w] * alpha[c] + beta[c]`. Can normalize * uniformly or channel-wise using array options. The result is cast to `dst` * tensor's dtype. - * @category Typescript API - * @param src The source image tensor in CHW layout. Shape [C,H,W]. - * @param dst The pre-allocated destination tensor to write the normalized - * values to. `dst` must have the same shape as `src`. Shape [C,H,W]. - * @param options Normalization scaling coefficients. - * @param options.alpha Multiplicative scaling coefficient(s). Defaults to - * `1 / 255.0`. - * @param options.beta Additive offset coefficient(s). Defaults to `0.0`. - * @returns The destination tensor containing the normalized image. + * @category CV / Functions + * @param src The source image tensor in CHW layout. Expected shape `[C, H, W]` + * (channels-first). Supports any numeric data type (typically `uint8` or + * `float32`). + * @param dst The pre-allocated destination image tensor to write normalized + * values to. Expected shape `[C, H, W]` matching `src`. The computed values are + * cast to `dst` tensor's target data type (typically `float32` or `uint8`). + * @param options Normalization scaling coefficients. When options or any + * individual properties are omitted, defaults to `alpha: 1 / 255.0` and `beta: + * 0.0`. + * See {@link NormalizeOptions}. + * @returns The destination image tensor containing the normalized image of + * shape `[C, H, W]` and target data type. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes, + * layouts, or data types are invalid, `RESOURCE_BUSY` if a tensor is in use, or + * `RESOURCE_DISPOSED` if either tensor was disposed. */ export function normalize(src: Tensor, dst: Tensor, options?: NormalizeOptions): Tensor { 'worklet'; @@ -206,15 +242,20 @@ export function normalize(src: Tensor, dst: Tensor, options?: NormalizeOptions): * This operation iterates over each index/class ID in the source tensor, looks * up its corresponding RGBA color in the provided colormap palette, and writes * it to the destination tensor. - * @category Typescript API - * @param src The source index/mask tensor. Must be an integer tensor of `int32` - * dtype containing class indices. Shape `[H, W, 1]` (or `[H, W]`). - * @param dst The pre-allocated destination tensor to write the mapped RGBA - * values to. Must be a 3D image tensor in HWC layout and `uint8` dtype. Shape - * `[H, W, 4]`. - * @param colormap An array of RGBA color arrays `[R, G, B, A]` corresponding to each - * class index. The size of this list must cover all class indices present in `src`. - * @returns The destination tensor with the applied colormap. + * @category CV / Functions + * @param src The source index/mask image tensor. Expected shape `[H, W, 1]` in + * HWC layout with data type `int32` containing class indices. + * @param dst The pre-allocated destination image tensor to write the mapped + * RGBA values to. Expected shape `[H, W, 4]` in HWC layout with data type + * `uint8`. + * @param colormap An array of RGBA color arrays `[R, G, B, A]` corresponding to + * each class index. The size of this list must cover all class indices present + * in `src`. + * @returns The destination image tensor with the applied colormap of shape `[H, + * W, 4]` and data type `uint8`. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes, + * layouts, or data types are invalid, `RESOURCE_BUSY` if a tensor is in use, or + * `RESOURCE_DISPOSED` if either tensor was disposed. */ export function applyColormap( src: Tensor, diff --git a/packages/react-native-executorch/src/extensions/cv/ops/index.ts b/packages/react-native-executorch/src/extensions/cv/ops/index.ts index 4128d25527..f3aa2c8ecf 100644 --- a/packages/react-native-executorch/src/extensions/cv/ops/index.ts +++ b/packages/react-native-executorch/src/extensions/cv/ops/index.ts @@ -1,4 +1,7 @@ -export * as image from './image'; -export * as boxes from './boxes'; -export * as points from './points'; -export * as quad from './quad'; +/** + * Aggregated image, box, point, and quad operations. + */ +export * from './image'; +export * from './box'; +export * from './point'; +export * from './quad'; diff --git a/packages/react-native-executorch/src/extensions/cv/ops/points.ts b/packages/react-native-executorch/src/extensions/cv/ops/point.ts similarity index 68% rename from packages/react-native-executorch/src/extensions/cv/ops/points.ts rename to packages/react-native-executorch/src/extensions/cv/ops/point.ts index 4f6218e46c..54b0b2587b 100644 --- a/packages/react-native-executorch/src/extensions/cv/ops/points.ts +++ b/packages/react-native-executorch/src/extensions/cv/ops/point.ts @@ -1,8 +1,12 @@ +/** + * 2D point representation and spatial scaling utilities. + */ + import type { ResizeMode } from './image'; /** * Represents a 2D coordinate point with x and y values. - * @category Types + * @category CV / Types */ export type Point = { readonly x: number; @@ -11,7 +15,7 @@ export type Point = { /** * Euclidean distance between two points. - * @category Utils + * @category CV / Functions * @param a The first point. * @param b The second point. * @returns The distance between `a` and `b`. @@ -24,7 +28,7 @@ export function distance(a: Point, b: Point): number { /** * Linearly interpolates between two points: `t = 0` returns `a`, `t = 1` * returns `b`, values in between interpolate along the segment. - * @category Utils + * @category CV / Functions * @param a The start point. * @param b The end point. * @param t The interpolation factor. @@ -35,26 +39,29 @@ export function interpolatePoint(a: Point, b: Point, t: number): Point { return { x: a.x + (b.x - a.x) * t, y: a.y + (b.y - a.y) * t }; } +/** + * Configuration options for scaling 2D point coordinates. + * @category CV / Types + */ +export type ScalePointOptions = { + /** The source bounds (e.g. model input dimensions). */ + readonly from: { readonly width: number; readonly height: number }; + /** The destination bounds (e.g. original image dimensions). */ + readonly to: { readonly width: number; readonly height: number }; + /** The mode used to resize the image (excluding `'crop'`). */ + readonly resizeMode: Exclude; +}; + /** * Helper function to scale a 2D point based on resize mode and resolution * changes. - * @category Utils + * @category CV / Functions * @param point The original coordinate point to scale. * @param options Options detailing the scaling factors and resize mode. - * @param options.from The source bounds (e.g. model input dimensions). - * @param options.to The destination bounds (e.g. original image dimensions). - * @param options.resizeMode The mode used to resize the image {@link ResizeMode} - * (excluding `'crop'`). + * See {@link ScalePointOptions}. * @returns The scaled coordinate point. */ -export function scalePoint( - point: Point, - options: { - readonly from: { readonly width: number; readonly height: number }; - readonly to: { readonly width: number; readonly height: number }; - readonly resizeMode: Exclude; - } -): Point { +export function scalePoint(point: Point, options: ScalePointOptions): Point { 'worklet'; const { from, to, resizeMode } = options; switch (resizeMode) { diff --git a/packages/react-native-executorch/src/extensions/cv/ops/quad.ts b/packages/react-native-executorch/src/extensions/cv/ops/quad.ts index 1955e11cb7..79b705aced 100644 --- a/packages/react-native-executorch/src/extensions/cv/ops/quad.ts +++ b/packages/react-native-executorch/src/extensions/cv/ops/quad.ts @@ -1,8 +1,13 @@ +/** + * Quadrilateral geometry, bounding box extraction, and image rectification + * utilities. + */ + import { rnexecutorchJsi } from '../../../native/bridge'; import type { Tensor } from '../../../core/tensor'; import { RnExecuTorchError } from '../../../core/error'; -import { distance, scalePoint, type Point } from './points'; -import type { BoundingBox, BoxFormat } from './boxes'; +import { distance, scalePoint, type Point } from './point'; +import type { BoundingBox, BoxFormat } from './box'; import type { ResizeMode } from './image'; /** @@ -10,18 +15,20 @@ import type { ResizeMode } from './image'; * Helpers that need them as top-left, top-right, bottom-right, bottom-left say * so on their `ordered` parameter; pass the quad through {@link orderQuad} * first. Never assume a `Quad` you were handed is already ordered. - * @category Types + * @category CV / Types */ export type Quad = readonly [Point, Point, Point, Point]; /** * Computes the axis-aligned bounding box enclosing a set of points, in the * requested box format. Returns a zero box for empty input. - * @category Typescript API + * @category CV / Functions * @typeParam F Bounding box coordinate format. * @param points The points to enclose. * @param format The coordinate format of the returned box. * @returns The enclosing {@link BoundingBox} in `format`. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if the bounding box + * format is unsupported. */ export function boundingBoxOfPoints( points: readonly Point[], @@ -66,7 +73,7 @@ export function boundingBoxOfPoints( * Reorders a quad's corners into the top-left, top-right, bottom-right, * bottom-left order the rest of this module assumes, using their * coordinate-sum and coordinate-difference extremes. - * @category Typescript API + * @category CV / Functions * @param quad The quad whose corners may be in any order. * @returns The same corners, ordered TL, TR, BR, BL. */ @@ -91,7 +98,7 @@ export function orderQuad(quad: Quad): Quad { /** * Computes the width and height (in pixels) of an ordered TL,TR,BR,BL quad, taking * the longer of each pair of opposite sides. - * @category Typescript API + * @category CV / Functions * @param ordered The quad corners ordered TL, TR, BR, BL. * @returns The quad's width and height in pixels. */ @@ -103,24 +110,29 @@ export function quadSize(ordered: Quad): { width: number; height: number } { return { width, height }; } +/** + * Configuration options for scaling quad coordinates. + * @category CV / Types + */ +export type ScaleQuadOptions = { + /** The source bounds (e.g. model input dimensions). */ + readonly from: { readonly width: number; readonly height: number }; + /** The destination bounds (e.g. original image dimensions). */ + readonly to: { readonly width: number; readonly height: number }; + /** The mode used to resize the image (excluding `'crop'`). */ + readonly resizeMode?: Exclude; +}; + /** * Rescales a quad from one frame to another, clamping the result to the target * bounds. The counterpart of {@link scaleBox} for quads. - * @category Typescript API + * @category CV / Functions * @param quad The quad, expressed in the `from` frame. - * @param options `from` is the frame the quad is expressed in, `to` the frame to - * express it in, and `resizeMode` how the two were fitted (default - * `'letterbox'`). + * @param options Options detailing the scaling factors and resize mode. + * See {@link ScaleQuadOptions}. * @returns The four corners in `to` pixels. */ -export function scaleQuad( - quad: Quad, - options: { - readonly from: { readonly width: number; readonly height: number }; - readonly to: { readonly width: number; readonly height: number }; - readonly resizeMode?: Exclude; - } -): Quad { +export function scaleQuad(quad: Quad, options: ScaleQuadOptions): Quad { 'worklet'; const { from, to, resizeMode } = options; const map = (p: Point): Point => { @@ -132,7 +144,7 @@ export function scaleQuad( /** * Options for {@link rectifyQuad}. - * @category Types + * @category CV / Types */ export type RectifyQuadOptions = { /** Width in px the rectified content occupies inside the destination canvas. */ @@ -148,13 +160,16 @@ export type RectifyQuadOptions = { * `dst`: perspective crop, resize to the canvas height, and pad, in one native * pass. An axis-aligned bbox is a 4-corner quad, so pass its corners to * rectify a box. - * @category Typescript API + * @category CV / Functions * @param src The source image, `uint8` `[H, W, C]`. * @param dst The pre-allocated destination canvas, `uint8` `[H', W', C]`, with * the same channel count as `src`. Must not alias `src`. * @param quad The region corners (TL, TR, BR, BL) in `src` pixels. - * @param options Content width, alignment, and padding. + * @param options Content width, alignment, and padding. See {@link RectifyQuadOptions}. * @returns The destination tensor `dst`. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes, + * data types, or quadrilateral coordinates are invalid, `RESOURCE_BUSY` if a + * tensor is in use, or `RESOURCE_DISPOSED` if either tensor was disposed. */ export function rectifyQuad( src: Tensor, diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts b/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts index 4d5e2b180f..bb4c059c3e 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts @@ -1,3 +1,8 @@ +/** + * Image classification task pipeline with integrated preprocessing and softmax + * decoding. + */ + import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; @@ -7,13 +12,13 @@ import { wrapAsync } from '../../../core/runtime'; import { softmax } from '../../math'; import type { ImageBuffer } from '../image'; -import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing'; +import { createImagePreprocessor, type ImagePreprocessorOptions } from '../utils/imagePreprocessor'; import { RnExecuTorchError } from '../../../core/error'; /** * Options for configuring an image classifier preprocessor and label * vocabulary. - * @category Types + * @category CV / Types */ export type ClassifierOptions = ImagePreprocessorOptions & { /** Array of class labels matching the model's output vocabulary. */ @@ -22,22 +27,34 @@ export type ClassifierOptions = ImagePreprocessorOptions & { /** * Model configuration required to instantiate a classifier task runner. - * @category Types + * @category CV / Types */ export type ClassifierModel = { /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; /** - * Image preprocessing and label vocabulary - * {@link ClassifierOptions}. The `labels` array length must + * Image preprocessing and label vocabulary. The `labels` array length must * match the model's output dimension. + * See {@link ClassifierOptions}. */ readonly modelOpts: ClassifierOptions; }; +/** + * Optional configuration parameters for classification inference. + * @category CV / Types + */ +export type ClassifyOptions = { + /** + * Number of top-scoring classification results to return. If omitted, all + * classes are returned. + */ + readonly topk?: number; +}; + /** * Result structure representing a single classification prediction. - * @category Types + * @category CV / Types */ export type Classification = { /** Predicted class label. */ @@ -47,47 +64,62 @@ export type Classification = { }; /** - * Creates an image classifier runner for executing local Image Classification - * models. - * - * It validates the model inputs and outputs requirements, asserts that the - * labels array length matches the model's output vocabulary size, pre-allocates - * the necessary static execution tensors, sets up an image preprocessor, and - * registers clean disposal hooks to clear all native memory. - * @category Typescript API + * Image classification task runner. + * @category CV / Types * @typeParam L The type representing the classification labels. - * @param config Classifier task configuration containing path and options. - * @param runtime Optional worklet runtime thread on which to run the model - * execution. - * @returns A promise resolving to an object containing classification and - * disposal controls. */ -export async function createClassifier( - config: ClassifierModel, - runtime?: WorkletRuntime -): Promise<{ +export type Classifier = { /** * Releases all allocated native resources. */ - dispose: () => void; + readonly dispose: () => void; /** * Performs asynchronous image classification on the given input image. * @param input The input image buffer. * @param options Configuration options for classification. - * @param options.topk The number of top-scoring classification results to - * return. If omitted, all classes are returned. Must be non-negative. + * See {@link ClassifyOptions}. * @returns A promise resolving to the list of classifications sorted by * confidence. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if `topk` is + * negative, `RESOURCE_BUSY` if the model is in use, or + * `RESOURCE_DISPOSED` if disposed. */ - classify: (input: ImageBuffer, options?: { topk?: number }) => Promise[]>; + readonly classify: ( + input: ImageBuffer, + options?: ClassifyOptions + ) => Promise[]>; /** * Synchronous version of {@link classify} to be executed directly on the * caller or worklet thread. */ - classifyWorklet: (input: ImageBuffer, options?: { topk?: number }) => Classification[]; -}> { + readonly classifyWorklet: (input: ImageBuffer, options?: ClassifyOptions) => Classification[]; +}; + +/** + * Creates an image classifier runner for executing local Image Classification + * models. + * + * It validates the model inputs and outputs requirements, asserts that the + * labels array length matches the model's output vocabulary size, pre-allocates + * the necessary static execution tensors, sets up an image preprocessor, and + * registers clean disposal hooks to clear all native memory. + * @category CV / Tasks + * @typeParam L The type representing the classification labels. + * @param config Classifier task configuration containing path and options. + * See {@link ClassifierModel}. + * @param runtime Optional worklet runtime thread on which to run the model + * execution. + * @returns A promise resolving to the instantiated {@link Classifier} runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if model fails to load, + * `SCHEMA_MISMATCH` if model schema does not match classification spec, or + * `INVALID_ARGUMENT` if labels length does not match model output classes. + */ +export async function createClassifier( + config: ClassifierModel, + runtime?: WorkletRuntime +): Promise> { const { modelPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); @@ -129,10 +161,7 @@ export async function createClassifier( model.dispose(); }; - const classifyWorklet = ( - input: ImageBuffer, - options?: { topk?: number } - ): Classification[] => { + const classifyWorklet = (input: ImageBuffer, options?: ClassifyOptions): Classification[] => { 'worklet'; if (options?.topk !== undefined && options.topk < 0) { throw RnExecuTorchError( diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts index cb85e7bdb9..868e3bdada 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts @@ -1,3 +1,7 @@ +/** + * Image embedding and visual feature extraction task pipeline. + */ + import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; @@ -6,60 +10,71 @@ import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import type { ImageBuffer } from '../image'; -import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing'; +import { createImagePreprocessor, type ImagePreprocessorOptions } from '../utils/imagePreprocessor'; /** * Model configuration required to instantiate an image embedder task runner. - * @category Types + * @category CV / Types */ export type ImageEmbedderModel = { /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; /** - * Image preprocessing (resize, color conversion, normalization) - * for embedding models {@link ImagePreprocessorOptions}. + * Image preprocessing (resize, color conversion, normalization) for embedding + * models. + * See {@link ImagePreprocessorOptions}. */ readonly modelOpts: ImagePreprocessorOptions; }; /** - * Creates an image embedder for executing local Image Embedding - * models (e.g. the image encoder of a CLIP model). - * - * It validates the model input and output requirements, pre-allocates the - * static execution tensors, sets up an image preprocessor, and registers clean - * disposal hooks to clear all native memory. Pooling and normalization (if any) - * are baked into the exported `.pte`; this runner simply preprocesses the image, - * runs the forward pass, and returns the raw embedding vector. - * @category Typescript API - * @param config Image embedder task configuration containing path and options. - * @param runtime Optional worklet runtime thread on which to run the model - * execution. - * @returns A promise resolving to an object containing the embedding and - * disposal controls. + * Image embedding task runner. + * @category CV / Types */ -export async function createImageEmbedder( - config: ImageEmbedderModel, - runtime?: WorkletRuntime -): Promise<{ +export type ImageEmbedder = { /** * Releases all allocated native resources. */ - dispose: () => void; + readonly dispose: () => void; /** * Asynchronously computes the embedding vector for the given input image. * @param input The input image buffer. * @returns A promise resolving to the embedding vector. + * @throws {RnExecuTorchError} With code `RESOURCE_BUSY` if the model is in + * use, or `RESOURCE_DISPOSED` if disposed. */ - embed: (input: ImageBuffer) => Promise; + readonly embed: (input: ImageBuffer) => Promise; /** * Synchronous version of {@link embed} to be executed directly on the * caller or worklet thread. */ - embedWorklet: (input: ImageBuffer) => Float32Array; -}> { + readonly embedWorklet: (input: ImageBuffer) => Float32Array; +}; + +/** + * Creates an image embedder for executing local Image Embedding + * models (e.g. the image encoder of a CLIP model). + * + * It validates the model input and output requirements, pre-allocates the + * static execution tensors, sets up an image preprocessor, and registers clean + * disposal hooks to clear all native memory. Pooling and normalization (if any) + * are baked into the exported `.pte`; this runner simply preprocesses the image, + * runs the forward pass, and returns the raw embedding vector. + * @category CV / Tasks + * @param config Image embedder task configuration containing path and options. + * See {@link ImageEmbedderModel}. + * @param runtime Optional worklet runtime thread on which to run the model + * execution. + * @returns A promise resolving to the instantiated {@link ImageEmbedder} runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if model fails to load, + * or `SCHEMA_MISMATCH` if model schema does not match embedding spec. + */ +export async function createImageEmbedder( + config: ImageEmbedderModel, + runtime?: WorkletRuntime +): Promise { const { modelPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts index 68bc1fa00e..1390ab185c 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts @@ -1,3 +1,8 @@ +/** + * Instance segmentation task pipeline with NMS, bounding box scaling, and mask + * extraction. + */ + import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; @@ -6,7 +11,7 @@ import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import type { ImageBuffer } from '../image'; -import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing'; +import { createImagePreprocessor, type ImagePreprocessorOptions } from '../utils/imagePreprocessor'; import { threshold } from '../../math'; import { resize, normalize } from '../ops/image'; import { @@ -16,15 +21,13 @@ import { restrictToBox, type BoundingBox, type BoxFormat, -} from '../ops/boxes'; +} from '../ops/box'; import { RnExecuTorchError } from '../../../core/error'; -export type { BoxFormat }; - /** * Options for configuring an instance segmenter preprocessor, label * vocabulary, and threshold parameters. - * @category Types + * @category CV / Types * @typeParam F The format type of the bounding box. * @typeParam L The label type. */ @@ -48,7 +51,7 @@ export type InstanceSegmenterOptions = Omit< /** * Model configuration required to instantiate an instance segmenter task runner. - * @category Types + * @category CV / Types * @typeParam F The format type of the bounding box. * @typeParam L The label type. */ @@ -56,17 +59,39 @@ export type InstanceSegmenterModel = { /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; /** - * Image preprocessing, label vocabulary, bounding box format, - * and default NMS/mask/confidence thresholds - * {@link InstanceSegmenterOptions}. + * Image preprocessing, label vocabulary, bounding box format, and default + * NMS/mask/confidence thresholds. + * See {@link InstanceSegmenterOptions}. */ readonly modelOpts: InstanceSegmenterOptions; }; +/** + * Optional configuration parameters for instance segmentation inference. + * @category CV / Types + */ +export type SegmentInstancesOptions = { + /** + * Minimum confidence threshold. If omitted, uses + * {@link InstanceSegmenterOptions.defaultConfidenceThreshold}. + */ + readonly confidenceThreshold?: number; + /** + * Intersection over Union (IoU) threshold in NMS. If omitted, uses + * {@link InstanceSegmenterOptions.defaultIouThreshold}. + */ + readonly iouThreshold?: number; + /** + * Mask binarization probability threshold. If omitted, uses + * {@link InstanceSegmenterOptions.defaultMaskThreshold}. + */ + readonly maskThreshold?: number; +}; + /** * Result structure representing a single detected instance with its bounding box, * segmentation mask, label, and confidence score. - * @category Types + * @category CV / Types * @typeParam F The format type of the bounding box. * @typeParam L The label type. */ @@ -82,56 +107,65 @@ export type InstanceSegmentationResult = { }; /** - * Creates an instance segmenter runner for executing local Instance - * Segmentation models. - * - * It validates model input/output tensor shapes and types, pre-allocates - * execution and auxiliary tensors, sets up an image preprocessor, and returns - * execution and resource management controls. - * @category Typescript API - * @typeParam F The bounding box format type. + * Instance segmentation task runner. + * @category CV / Types + * @typeParam F The format type of the bounding box. * @typeParam L The label type. - * @param config Model configuration containing path and options. - * @param runtime Optional worklet runtime thread on which to run the model - * execution. - * @returns A promise resolving to an object containing instance segmentation - * and disposal controls. */ -export async function createInstanceSegmenter( - config: InstanceSegmenterModel, - runtime?: WorkletRuntime -): Promise<{ +export type InstanceSegmenter = { /** * Releases all allocated native resources. */ - dispose: () => void; + readonly dispose: () => void; /** * Performs asynchronous instance segmentation on the given input image. * @param input The input image buffer. * @param options Execution override options. - * @param options.confidenceThreshold Minimum confidence threshold. If - * omitted, uses `modelOpts.defaultConfidenceThreshold`. - * @param options.iouThreshold Intersection over Union (IoU) threshold in NMS. If omitted, uses - * `modelOpts.defaultIouThreshold`. - * @param options.maskThreshold Mask binarization threshold. If omitted, - * uses `modelOpts.defaultMaskThreshold`. + * See {@link SegmentInstancesOptions}. * @returns A promise resolving to a list of detected instances. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if predicted class + * index is out of bounds, `RESOURCE_BUSY` if the model is in use, or + * `RESOURCE_DISPOSED` if disposed. */ - segmentInstances: ( + readonly segmentInstances: ( input: ImageBuffer, - options?: { confidenceThreshold?: number; iouThreshold?: number; maskThreshold?: number } + options?: SegmentInstancesOptions ) => Promise[]>; /** * Synchronous version of {@link segmentInstances} to be executed directly on * the caller or worklet thread. */ - segmentInstancesWorklet: ( + readonly segmentInstancesWorklet: ( input: ImageBuffer, - options?: { confidenceThreshold?: number; iouThreshold?: number; maskThreshold?: number } + options?: SegmentInstancesOptions ) => InstanceSegmentationResult[]; -}> { +}; + +/** + * Creates an instance segmenter runner for executing local Instance + * Segmentation models. + * + * It validates model input/output tensor shapes and types, pre-allocates + * execution and auxiliary tensors, sets up an image preprocessor, and returns + * execution and resource management controls. + * @category CV / Tasks + * @typeParam F The bounding box format type. + * @typeParam L The label type. + * @param config Model configuration containing path and options. + * See {@link InstanceSegmenterModel}. + * @param runtime Optional worklet runtime thread on which to run the model + * execution. + * @returns A promise resolving to the instantiated {@link InstanceSegmenter} runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if model fails to load, + * or `SCHEMA_MISMATCH` if model schema does not match instance segmentation + * spec. + */ +export async function createInstanceSegmenter( + config: InstanceSegmenterModel, + runtime?: WorkletRuntime +): Promise> { const { modelPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts b/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts index 66ac81adb9..7b168625a4 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts @@ -1,3 +1,8 @@ +/** + * Keypoint and pose detection task pipeline with weighted NMS and landmark + * scaling. + */ + import type { WorkletRuntime } from 'react-native-worklets'; import { tensor, type Tensor } from '../../../core/tensor'; @@ -6,23 +11,21 @@ import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import type { ImageBuffer } from '../image'; -import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing'; +import { createImagePreprocessor, type ImagePreprocessorOptions } from '../utils/imagePreprocessor'; import type { ResizeMode } from '../ops/image'; -import { scalePoint, type Point } from '../ops/points'; -import { nms, type BoundingBox, type BoxFormat, decodeBox, scaleBox } from '../ops/boxes'; - -export type { BoxFormat }; +import { scalePoint, type Point } from '../ops/point'; +import { nms, type BoundingBox, type BoxFormat, decodeBox, scaleBox } from '../ops/box'; /** * Options for configuring a keypoint detector runner. - * @category Types + * @category CV / Types */ export type KeypointDetectorOptions = Omit< ImagePreprocessorOptions, 'resizeMode' > & { - /** Resize mode for preprocessing input images {@link ResizeMode} (excluding `'crop'`). */ + /** Resize mode for preprocessing input images (excluding `'crop'`). */ readonly resizeMode: Exclude; /** How bounding box coordinates are interpreted {@link BoxFormat}. */ readonly boxFormat: F; @@ -36,30 +39,47 @@ export type KeypointDetectorOptions /** * Model configuration required to instantiate a keypoint detector task runner. - * @category Types + * @category CV / Types */ export type KeypointDetectorModel = { /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; /** - * Image preprocessing, landmark names, bounding box format, - * and default NMS/confidence thresholds - * {@link KeypointDetectorOptions}. + * Image preprocessing, landmark names, bounding box format, and default + * NMS/confidence thresholds. + * See {@link KeypointDetectorOptions}. */ readonly modelOpts: KeypointDetectorOptions; }; +/** + * Optional configuration parameters for keypoint detection inference. + * @category CV / Types + */ +export type DetectKeypointsOptions = { + /** + * Minimum confidence score threshold for detections. If omitted, uses + * {@link KeypointDetectorOptions.defaultConfidenceThreshold}. + */ + readonly confidenceThreshold?: number; + /** + * Intersection over Union (IoU) threshold for NMS. If omitted, uses + * {@link KeypointDetectorOptions.defaultIouThreshold}. + */ + readonly iouThreshold?: number; +}; + /** * Plural landmarks mapped by their names to coordinates and detection * confidence. - * @category Types + * @category CV / Types */ export type Landmarks = Record; /** * Result structure representing a single detected bounding box and its * associated landmarks. - * @category Types + * @category CV / Types */ export type KeypointDetection = { /** Scaled bounding box coordinates matching the input image resolution. */ @@ -70,10 +90,46 @@ export type KeypointDetection = { readonly landmarks: Landmarks; }; +/** + * Keypoint and pose detection task runner. + * @category CV / Types + * @typeParam F The bounding box format. + * @typeParam L The landmark labels type. + */ +export type KeypointDetector = { + /** + * Releases all allocated native resources. + */ + readonly dispose: () => void; + + /** + * Performs asynchronous keypoint and bounding box detection on the given + * input image. + * @param input The input image buffer. + * @param options Configuration options for keypoint detection. + * See {@link DetectKeypointsOptions}. + * @returns A promise resolving to the list of keypoint detections. + * @throws {RnExecuTorchError} With code `RESOURCE_BUSY` if the model is in + * use, or `RESOURCE_DISPOSED` if disposed. + */ + readonly detectKeypoints: ( + input: ImageBuffer, + options?: DetectKeypointsOptions + ) => Promise[]>; + + /** + * Synchronous version of {@link detectKeypoints} to be executed directly on + * the caller or worklet thread. + */ + readonly detectKeypointsWorklet: ( + input: ImageBuffer, + options?: DetectKeypointsOptions + ) => KeypointDetection[]; +}; + /** * Post-processes model outputs by applying Non-Maximum Suppression (NMS) and * scaling coordinates. - * @category Utils * @param tBoxes Bounding boxes tensor output from inference. * @param tScores Scores tensor output from inference. * @param tKeypoints Keypoints tensor output from inference. @@ -151,49 +207,21 @@ function postprocess( * It validates model inputs and output shapes (bounding boxes, confidence * scores, and landmark coordinates), pre-allocates execution tensors, setups * preprocessing, and sets up lifecycle disposals. - * @category Typescript API + * @category CV / Tasks * @typeParam F The bounding box format. * @typeParam L The landmark labels type. * @param config Keypoint task configuration containing path and options. + * See {@link KeypointDetectorModel}. * @param runtime Optional worklet runtime thread on which to run the model * execution. - * @returns A promise resolving to an object containing keypoint detection and - * disposal bindings. + * @returns A promise resolving to the instantiated {@link KeypointDetector} runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if model fails to load, + * or `SCHEMA_MISMATCH` if model schema does not match keypoint detection spec. */ export async function createKeypointDetector( config: KeypointDetectorModel, runtime?: WorkletRuntime -): Promise<{ - /** - * Releases all allocated native resources. - */ - dispose: () => void; - - /** - * Performs asynchronous keypoint and bounding box detection on the given - * input image. - * @param input The input image buffer. - * @param options Configuration options for keypoint detection. - * @param options.confidenceThreshold Minimum confidence score for a - * detection. If omitted, uses `modelOpts.defaultConfidenceThreshold`. - * @param options.iouThreshold Intersection over Union (IoU) threshold for - * NMS. If omitted, uses `modelOpts.defaultIouThreshold`. - * @returns A promise resolving to the list of keypoint detections. - */ - detectKeypoints: ( - input: ImageBuffer, - options?: { confidenceThreshold?: number; iouThreshold?: number } - ) => Promise[]>; - - /** - * Synchronous version of {@link detectKeypoints} to be executed directly on - * the caller or worklet thread. - */ - detectKeypointsWorklet: ( - input: ImageBuffer, - options?: { confidenceThreshold?: number; iouThreshold?: number } - ) => KeypointDetection[]; -}> { +): Promise> { const { modelPath, modelOpts } = config; const { landmarks } = modelOpts; const model = await wrapAsync(loadModel, runtime)(modelPath); diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/objectDetection.ts b/packages/react-native-executorch/src/extensions/cv/tasks/objectDetection.ts index 60cff1b272..833890527a 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/objectDetection.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/objectDetection.ts @@ -1,3 +1,8 @@ +/** + * Object detection task pipeline with integrated preprocessing, NMS, and box + * scaling. + */ + import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; @@ -7,22 +12,20 @@ import { wrapAsync } from '../../../core/runtime'; import type { ResizeMode } from '../ops/image'; import type { ImageBuffer } from '../image'; -import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing'; -import { nms, scaleBox, decodeBox, type BoundingBox, type BoxFormat } from '../ops/boxes'; +import { createImagePreprocessor, type ImagePreprocessorOptions } from '../utils/imagePreprocessor'; +import { nms, scaleBox, decodeBox, type BoundingBox, type BoxFormat } from '../ops/box'; import { RnExecuTorchError } from '../../../core/error'; -export type { BoxFormat }; - /** * Options for configuring an object detector preprocessor, label vocabulary, * and detection thresholds. - * @category Types + * @category CV / Types */ export type ObjectDetectorOptions = Omit< ImagePreprocessorOptions, 'resizeMode' > & { - /** Resize mode for preprocessing input images {@link ResizeMode} (excluding `'crop'`). */ + /** Resize mode for preprocessing input images (excluding `'crop'`). */ readonly resizeMode: Exclude; /** Array of class labels matching the model's output vocabulary. */ readonly labels: readonly L[]; @@ -36,22 +39,39 @@ export type ObjectDetectorOptions = Omit< /** * Model configuration required to instantiate an object detector task runner. - * @category Types + * @category CV / Types */ export type ObjectDetectorModel = { /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; /** - * Image preprocessing, label vocabulary, and default - * NMS/confidence thresholds {@link ObjectDetectorOptions}. - * Used as fallbacks when per-call overrides are omitted. + * Image preprocessing, label vocabulary, and default NMS/confidence + * thresholds. Used as fallbacks when per-call overrides are omitted. + * See {@link ObjectDetectorOptions}. */ readonly modelOpts: ObjectDetectorOptions; }; +/** + * Optional configuration parameters for object detection inference. + * @category CV / Types + */ +export type DetectObjectsOptions = { + /** + * Minimum confidence score threshold. If omitted, uses + * {@link ObjectDetectorOptions.defaultConfidenceThreshold}. + */ + readonly confidenceThreshold?: number; + /** + * Intersection over Union (IoU) threshold for NMS. If omitted, uses + * {@link ObjectDetectorOptions.defaultIouThreshold}. + */ + readonly iouThreshold?: number; +}; + /** * Result structure representing a single object detection prediction. - * @category Types + * @category CV / Types */ export type ObjectDetection = { /** Scaled bounding box coordinates matching the input image resolution. */ @@ -63,53 +83,64 @@ export type ObjectDetection = { }; /** - * Creates an object detector runner for executing local Object Detection - * models. - * - * It validates the model inputs and outputs requirements, pre-allocates the - * necessary static execution tensors (boxes, scores, classes), sets up an image - * preprocessor, and registers clean disposal hooks to clear all native memory. - * @category Typescript API + * Object detection task runner. + * @category CV / Types * @typeParam F The bounding box format. * @typeParam L The type representing the class labels. - * @param config Object detector task configuration containing path and options. - * @param runtime Optional worklet runtime thread on which to run the model - * execution. - * @returns A promise resolving to an object containing object detection and - * disposal controls. */ -export async function createObjectDetector( - config: ObjectDetectorModel, - runtime?: WorkletRuntime -): Promise<{ +export type ObjectDetector = { /** * Releases all allocated native resources. */ - dispose: () => void; + readonly dispose: () => void; /** + * Asynchronously performs object detection on the input image. * @param input The input image buffer. * @param options Configuration options for object detection. - * @param options.confidenceThreshold Minimum confidence score threshold. If - * omitted, uses `modelOpts.defaultConfidenceThreshold`. - * @param options.iouThreshold Intersection over Union (IoU) threshold. If - * omitted, uses `modelOpts.defaultIouThreshold`. + * See {@link DetectObjectsOptions}. * @returns A promise resolving to the list of object detections. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if predicted class + * index is out of bounds, `RESOURCE_BUSY` if the model is in use, or + * `RESOURCE_DISPOSED` if disposed. */ - detectObjects: ( + readonly detectObjects: ( input: ImageBuffer, - options?: { confidenceThreshold?: number; iouThreshold?: number } + options?: DetectObjectsOptions ) => Promise[]>; /** * Synchronous version of {@link detectObjects} to be executed directly on the * caller or worklet thread. */ - detectObjectsWorklet: ( + readonly detectObjectsWorklet: ( input: ImageBuffer, - options?: { confidenceThreshold?: number; iouThreshold?: number } + options?: DetectObjectsOptions ) => ObjectDetection[]; -}> { +}; + +/** + * Creates an object detector runner for executing local Object Detection + * models. + * + * It validates the model inputs and outputs requirements, pre-allocates the + * necessary static execution tensors (boxes, scores, classes), sets up an image + * preprocessor, and registers clean disposal hooks to clear all native memory. + * @category CV / Tasks + * @typeParam F The bounding box format. + * @typeParam L The type representing the class labels. + * @param config Object detector task configuration containing path and options. + * See {@link ObjectDetectorModel}. + * @param runtime Optional worklet runtime thread on which to run the model + * execution. + * @returns A promise resolving to the instantiated {@link ObjectDetector} runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if model fails to load, + * or `SCHEMA_MISMATCH` if model schema does not match object detection spec. + */ +export async function createObjectDetector( + config: ObjectDetectorModel, + runtime?: WorkletRuntime +): Promise> { const { modelPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/paddleOcr.ts b/packages/react-native-executorch/src/extensions/cv/tasks/paddleOcr.ts index 1b7482499e..6d5f4bf7cf 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/paddleOcr.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/paddleOcr.ts @@ -1,7 +1,7 @@ -// PP-OCRv6: a DBNet text detector and an SVTR recognizer fused into one PTE. -// One pass locates every text line on the page, warps each to the recognizer -// canvas and reads it. Worklet source order matters here: a referenced worklet -// must be defined above its callers. +/** + * PP-OCRv6 optical character recognition (DBNet text detection + SVTR text + * recognition). + */ import type { WorkletRuntime } from 'react-native-worklets'; import RNBlobUtil from 'react-native-blob-util'; @@ -13,7 +13,7 @@ import { tensor, type Tensor } from '../../../core/tensor'; import { validateSpec, method, - constr, + constraint, f32, DynamicDim, type ConcreteDim, @@ -28,7 +28,7 @@ import { FORMAT_CHANNELS, FORMAT_CONVERSION, } from '../ops/image'; -import { interpolatePoint } from '../ops/points'; +import { interpolatePoint } from '../ops/point'; import { boundingBoxOfPoints, orderQuad, @@ -39,11 +39,11 @@ import { } from '../ops/quad'; import { argmax, gather } from '../../math'; import { extractDbnetTextQuads } from '../utils/paddleOcrUtils'; -import { createImagePreprocessor } from './preprocessing'; +import { createImagePreprocessor } from '../utils/imagePreprocessor'; /** * A single recognized text region. - * @category Types + * @category CV / Types */ export type OcrDetection = { /** The recognized text. */ @@ -53,14 +53,14 @@ export type OcrDetection = { /** * The region's corners, ordered top-left, top-right, bottom-right, bottom-left, * in original image pixels. Oriented, so a rotated line keeps its angle; take - * `boundingBoxOfPoints(quad, 'xyxy')` for the axis-aligned box. + * {@link boundingBoxOfPoints} for the axis-aligned box. */ readonly quad: Quad; }; /** * Options for the PP-OCRv6 pipeline. - * @category Types + * @category CV / Types */ export type PaddleOcrModelOptions = { /** @@ -70,10 +70,22 @@ export type PaddleOcrModelOptions = { readonly defaultConfidenceThreshold: number; }; +/** + * Optional configuration parameters for optical character recognition inference. + * @category CV / Types + */ +export type RecognizeCharactersOptions = { + /** + * Minimum confidence threshold to retain recognized text regions, in `[0, 1]`. + * Overrides the model's `defaultConfidenceThreshold` for this call. + */ + readonly confidenceThreshold?: number; +}; + /** * Model configuration for the PP-OCRv6 pipeline: one fused detect/recognize PTE, * the charset published beside it, and the run options. - * @category Types + * @category CV / Types */ export type PaddleOcrModel = { /** The fused detect/recognize PTE. Resolved to a local path by the fetcher. */ @@ -88,6 +100,38 @@ export type PaddleOcrModel = { readonly modelOpts: PaddleOcrModelOptions; }; +/** + * PP-OCRv6 optical character recognition task runner. + * @category CV / Types + */ +export type PaddleOcr = { + /** + * Releases all allocated native resources. + */ + readonly dispose: () => void; + + /** + * Detects and recognizes every text line in the given image. + * @param input The input image buffer. + * @param options Per-call overrides. See {@link RecognizeCharactersOptions}. + * @returns A promise resolving to the recognized lines in reading order + * (leftmost column top to bottom, then the next column). + */ + readonly recognizeCharacters: ( + input: ImageBuffer, + options?: RecognizeCharactersOptions + ) => Promise; + + /** + * Synchronous version of {@link recognizeCharacters} to be executed directly + * on the caller or worklet thread. + */ + readonly recognizeCharactersWorklet: ( + input: ImageBuffer, + options?: RecognizeCharactersOptions + ) => OcrDetection[]; +}; + // Fixed by the export: the detector was trained on ImageNet-normalized RGB and // the recognizer on (x/255 - 0.5)/0.5 over a gray-padded canvas. const DETECTOR_PREPROCESSOR_OPTS = { @@ -341,46 +385,21 @@ function greedyCtcDecode( * Creates the PP-OCRv6 runner: one pass detects text quads on the whole page, * warps each to the recognizer canvas and reads it, returning the lines in * reading order. - * @category Typescript API - * @param config Model path, charset path, and run options. - * @param runtime Optional worklet runtime thread. - * @returns A promise resolving to an object containing recognition and disposal - * controls. - * @throws {RnExecuTorchError} With code `SCHEMA_MISMATCH` if the loaded model - * does not match the PP-OCRv6 detect/recognize contract, or if the charset does - * not match the recognizer's vocabulary. + * @category CV / Tasks + * @param config PaddleOCR task configuration containing the model, charset, + * and detection thresholds. See {@link PaddleOcrModel}. + * @param runtime Optional worklet runtime thread on which to run detection and + * recognition. + * @returns A promise resolving to the instantiated {@link PaddleOcr} runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if the model or charset + * fails to load, `SCHEMA_MISMATCH` if the loaded model does not match the + * PP-OCRv6 detect/recognize contract, or if the charset does not match the + * recognizer's vocabulary. */ export async function createPaddleOcr( config: PaddleOcrModel, runtime?: WorkletRuntime -): Promise<{ - /** - * Releases all allocated native resources. - */ - dispose: () => void; - - /** - * Detects and recognizes every text line in the given image. - * @param input The input image buffer. - * @param options Per-call overrides. `confidenceThreshold` replaces the - * model's `defaultConfidenceThreshold` for this call. - * @returns A promise resolving to the recognized lines in reading order - * (leftmost column top to bottom, then the next column). - */ - recognizeCharacters: ( - input: ImageBuffer, - options?: { confidenceThreshold?: number } - ) => Promise; - - /** - * Synchronous version of {@link recognizeCharacters} to be executed directly - * on the caller or worklet thread. - */ - recognizeCharactersWorklet: ( - input: ImageBuffer, - options?: { confidenceThreshold?: number } - ) => OcrDetection[]; -}> { +): Promise { const { modelPath, charsetPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); @@ -399,7 +418,7 @@ export async function createPaddleOcr( [f32(1, 3, 'recH', DynamicDim('recW'))], [f32(1, DynamicDim('recT'), 'vocab')], [ - constr.linear( + constraint.linear( { paramSide: 'input', tensorIdx: 0, dimIdx: 3 }, { paramSide: 'output', tensorIdx: 0, dimIdx: 1 }, SVTR_CTC_STRIDE @@ -430,7 +449,7 @@ export async function createPaddleOcr( const recognizeCharactersWorklet = ( input: ImageBuffer, - options?: { confidenceThreshold?: number } + options?: RecognizeCharactersOptions ): OcrDetection[] => { 'worklet'; const confidenceThreshold = diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts b/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts index 0130e6cd07..37512c737d 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts @@ -1,3 +1,8 @@ +/** + * SDXS single-step text-to-image generation pipeline with latent diffusion and + * TAESD decoding. + */ + import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; @@ -35,7 +40,7 @@ const NOISE_COEFF = -14.579279; /** * Model configuration required to instantiate the SDXS text-to-image runner. - * @category Types + * @category CV / Types */ export type SdxsTextToImageModel = { /** Local path to the model `.pte`. */ @@ -45,21 +50,14 @@ export type SdxsTextToImageModel = { }; /** - * Creates an SDXS text-to-image runner. - * - * It validates the exported method schemas, pre-allocates the static execution - * tensors, and registers disposal hooks that release all native memory. - * @category Typescript API - * @param config SDXS pipeline configuration containing the model and tokenizer paths. - * @param runtime Optional worklet runtime thread on which to run generation. - * @returns A promise resolving to an object with generation and disposal controls. + * SDXS single-step text-to-image generation task runner. + * @category CV / Types */ -export async function createSdxsTextToImage( - config: SdxsTextToImageModel, - runtime?: WorkletRuntime -): Promise<{ - /** Releases all allocated native resources. */ - dispose: () => void; +export type SdxsTextToImage = { + /** + * Releases all allocated native resources. + */ + readonly dispose: () => void; /** * Generates an image from a text prompt. @@ -67,15 +65,35 @@ export async function createSdxsTextToImage( * @param seed Seed for the initial latent noise (same seed → same image). * Defaults to a time-based value so omitting it yields a fresh image each call. * @returns A promise resolving to the generated RGBA image buffer. + * @throws {RnExecuTorchError} With code `RESOURCE_BUSY` if the model is in + * use, or `RESOURCE_DISPOSED` if disposed. */ - generate: (prompt: string, seed?: number) => Promise; + readonly generate: (prompt: string, seed?: number) => Promise; /** * Synchronous version of {@link generate} to be executed directly on the * caller or worklet thread. */ - generateWorklet: (prompt: string, seed?: number) => ImageBuffer; -}> { + readonly generateWorklet: (prompt: string, seed?: number) => ImageBuffer; +}; + +/** + * Creates an SDXS text-to-image runner. + * + * It validates the exported method schemas, pre-allocates the static execution + * tensors, and registers disposal hooks that release all native memory. + * @category CV / Tasks + * @param config SDXS pipeline configuration containing the model and tokenizer paths. + * See {@link SdxsTextToImageModel}. + * @param runtime Optional worklet runtime thread on which to run generation. + * @returns A promise resolving to the instantiated {@link SdxsTextToImage} runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if model or tokenizer + * fails to load, or `SCHEMA_MISMATCH` if model schema does not match SDXS spec. + */ +export async function createSdxsTextToImage( + config: SdxsTextToImageModel, + runtime?: WorkletRuntime +): Promise { const { modelPath, tokenizerPath } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); const tokenizer = await wrapAsync(loadTokenizer, runtime)(tokenizerPath); diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts index 85961ff662..7cf25928b6 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts @@ -1,3 +1,8 @@ +/** + * Semantic segmentation task pipeline with colormap mapping and output mask + * rendering. + */ + import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; @@ -6,7 +11,7 @@ import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import type { ImageBuffer } from '../image'; -import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing'; +import { createImagePreprocessor, type ImagePreprocessorOptions } from '../utils/imagePreprocessor'; import { toChannelsLast, normalize, @@ -21,7 +26,7 @@ import { RnExecuTorchError } from '../../../core/error'; /** * Options for configuring a semantic segmenter preprocessor and label * vocabulary. - * @category Types + * @category CV / Types */ export type SemanticSegmenterOptions = Omit & { /** Resize mode for input images. Must be `'stretch'`. */ @@ -34,28 +39,28 @@ export type SemanticSegmenterOptions = Omit = { /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; /** - * Image preprocessing, output mask interpolation, and label - * vocabulary {@link SemanticSegmenterOptions}. `resizeMode` - * is fixed to `'stretch'`. + * Image preprocessing, output mask interpolation, and label vocabulary. + * `resizeMode` is fixed to `'stretch'`. + * See {@link SemanticSegmenterOptions}. */ readonly modelOpts: SemanticSegmenterOptions; }; /** * Maps each class label to its assigned RGBA color. - * @category Types + * @category CV / Types */ export type ColorMap = Record; /** * Result structure representing the output of a semantic segmentation task. - * @category Types + * @category CV / Types */ export type SemanticSegmentationResult = { /** Generated output RGBA image buffer containing the colored segmentation mask. */ @@ -64,38 +69,16 @@ export type SemanticSegmentationResult = { readonly colormap?: ColorMap; }; -function hslToRgb(h: number, s: number, l: number): [number, number, number] { - s /= 100; - l /= 100; - const k = (n: number) => (n + h / 30) % 12; - const a = s * Math.min(l, 1 - l); - const f = (n: number) => l - a * Math.max(-1, Math.min(k(n) - 3, 9 - k(n), 1)); - return [Math.round(255 * f(0)), Math.round(255 * f(8)), Math.round(255 * f(4))]; -} - /** - * Creates a semantic segmenter runner for executing local Semantic Segmentation - * models. - * - * It validates the model inputs and outputs, asserts that the labels array - * length matches the model's output vocabulary size, pre-allocates the - * necessary static execution tensors, sets up an image preprocessor, and - * registers clean disposal hooks to clear all native memory. - * @category Typescript API + * Semantic segmentation task runner. + * @category CV / Types * @typeParam L The type representing the segmentation labels. - * @param config Segmenter task configuration containing path and options. - * @param runtime Optional worklet runtime thread environment context. - * @returns A promise resolving to an object containing segmentation and - * disposal controls. */ -export async function createSemanticSegmenter( - config: SemanticSegmenterModel, - runtime?: WorkletRuntime -): Promise<{ +export type SemanticSegmenter = { /** * Releases all allocated native resources. */ - dispose: () => void; + readonly dispose: () => void; /** * Runs semantic segmentation asynchronously. @@ -117,21 +100,55 @@ export async function createSemanticSegmenter( * provided, any labels omitted from it will default to being rendered as * fully transparent. * @returns A promise resolving to the segmentation result. + * @throws {RnExecuTorchError} With code `RESOURCE_BUSY` if the model is in + * use, or `RESOURCE_DISPOSED` if disposed. */ - segment: ( + readonly segment: ( input: ImageBuffer, colormap?: Partial> ) => Promise>; /** - * Runs semantic segmentation synchronously. - * @see {@link segment} for details. + * Synchronous version of {@link segment} to be executed directly on the + * caller or worklet thread. */ - segmentWorklet: ( + readonly segmentWorklet: ( input: ImageBuffer, colormap?: Partial> ) => SemanticSegmentationResult; -}> { +}; + +function hslToRgb(h: number, s: number, l: number): [number, number, number] { + s /= 100; + l /= 100; + const k = (n: number) => (n + h / 30) % 12; + const a = s * Math.min(l, 1 - l); + const f = (n: number) => l - a * Math.max(-1, Math.min(k(n) - 3, 9 - k(n), 1)); + return [Math.round(255 * f(0)), Math.round(255 * f(8)), Math.round(255 * f(4))]; +} + +/** + * Creates a semantic segmenter runner for executing local Semantic Segmentation + * models. + * + * It validates the model inputs and outputs, asserts that the labels array + * length matches the model's output vocabulary size, pre-allocates the + * necessary static execution tensors, sets up an image preprocessor, and + * registers clean disposal hooks to clear all native memory. + * @category CV / Tasks + * @typeParam L The type representing the segmentation labels. + * @param config Segmenter task configuration containing path and options. + * See {@link SemanticSegmenterModel}. + * @param runtime Optional worklet runtime thread environment context. + * @returns A promise resolving to the instantiated {@link SemanticSegmenter} runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if model fails to load, + * `SCHEMA_MISMATCH` if model schema does not match segmentation spec, or + * `INVALID_ARGUMENT` if labels length does not match model output classes. + */ +export async function createSemanticSegmenter( + config: SemanticSegmenterModel, + runtime?: WorkletRuntime +): Promise> { const { modelPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts b/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts index a15d537ec2..2232c0eff6 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts @@ -1,3 +1,8 @@ +/** + * Neural style transfer task pipeline with output rendering and colorspace + * conversion. + */ + import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; @@ -6,7 +11,7 @@ import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import type { ImageBuffer } from '../image'; -import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing'; +import { createImagePreprocessor, type ImagePreprocessorOptions } from '../utils/imagePreprocessor'; import { toChannelsLast, normalize, @@ -18,7 +23,7 @@ import { /** * Options for configuring the style transfer preprocessor and postprocessor. - * @category Types + * @category CV / Types */ export type StyleTransferOptions = Omit & { /** Resize mode for input images. Must be `'stretch'`. */ @@ -31,52 +36,63 @@ export type StyleTransferOptions = Omit /** * Model configuration required to instantiate a style transfer task runner. - * @category Types + * @category CV / Types */ export type StyleTransferModel = { /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; /** - * Input preprocessing and output postprocessing - * {@link StyleTransferOptions} (normalization back to uint8, + * Input preprocessing and output postprocessing (normalization back to uint8, * interpolation). `resizeMode` is fixed to `'stretch'`. + * See {@link StyleTransferOptions}. */ readonly modelOpts: StyleTransferOptions; }; /** - * Creates an image style transfer runner for executing local style transfer models. - * - * It validates the model inputs and outputs requirements, pre-allocates - * the necessary static execution tensors, sets up an image preprocessor, and - * registers clean disposal hooks to clear all native memory. - * @category Typescript API - * @param config Style transfer task configuration containing path and options. - * @param runtime Optional worklet runtime thread on which to run the model execution. - * @returns A promise resolving to an object containing style transfer and disposal controls. + * Image style transfer task runner. + * @category CV / Types */ -export async function createStyleTransfer( - config: StyleTransferModel, - runtime?: WorkletRuntime -): Promise<{ +export type StyleTransfer = { /** * Releases all allocated native resources. */ - dispose: () => void; + readonly dispose: () => void; /** * Performs asynchronous image style transfer on the given input image. * @param input The input image buffer. * @returns A promise resolving to the styled image buffer. + * @throws {RnExecuTorchError} With code `RESOURCE_BUSY` if the model is in + * use, or `RESOURCE_DISPOSED` if disposed. */ - transferStyle: (input: ImageBuffer) => Promise; + readonly transferStyle: (input: ImageBuffer) => Promise; /** * Synchronous version of {@link transferStyle} to be executed directly on the * caller or worklet thread. */ - transferStyleWorklet: (input: ImageBuffer) => ImageBuffer; -}> { + readonly transferStyleWorklet: (input: ImageBuffer) => ImageBuffer; +}; + +/** + * Creates an image style transfer runner for executing local style transfer models. + * + * It validates the model inputs and outputs requirements, pre-allocates + * the necessary static execution tensors, sets up an image preprocessor, and + * registers clean disposal hooks to clear all native memory. + * @category CV / Tasks + * @param config Style transfer task configuration containing path and options. + * See {@link StyleTransferModel}. + * @param runtime Optional worklet runtime thread on which to run the model execution. + * @returns A promise resolving to the instantiated {@link StyleTransfer} runner. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if model fails to load, + * or `SCHEMA_MISMATCH` if model schema does not match style transfer spec. + */ +export async function createStyleTransfer( + config: StyleTransferModel, + runtime?: WorkletRuntime +): Promise { const { modelPath, modelOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts b/packages/react-native-executorch/src/extensions/cv/utils/imagePreprocessor.ts similarity index 74% rename from packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts rename to packages/react-native-executorch/src/extensions/cv/utils/imagePreprocessor.ts index fcda285da8..e2a5a75a80 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts +++ b/packages/react-native-executorch/src/extensions/cv/utils/imagePreprocessor.ts @@ -1,3 +1,7 @@ +/** + * Reusable image preprocessing pipeline for neural network inputs. + */ + import { tensor, type Tensor } from '../../../core/tensor'; import type { ImageBuffer } from '../image'; @@ -16,41 +20,29 @@ import { RnExecuTorchError } from '../../../core/error'; /** * Options for configuring the image preprocessor pipeline. - * @category Types + * @category CV / Types */ export type ImagePreprocessorOptions = { - /** - * How the input image is resized to match the model's expected - * dimensions {@link ResizeMode}. - */ + /** How the input image is resized to match the model's expected dimensions. */ readonly resizeMode: ResizeMode; - /** Algorithm used when resizing {@link InterpolationMethod}. `'linear'` is a good default. */ + /** Algorithm used when resizing (e.g. `'linear'`, `'lanczos'`). */ readonly interpolation: InterpolationMethod; /** Normalization scaling coefficients. */ readonly normalizeOpts: NormalizeOptions; - /** Optional background fill value used when letterboxing. */ + /** Optional background fill value used when letterboxing (padding). */ readonly padValue?: number; }; /** - * Creates a reusable image preprocessor pipeline. - * - * Configures a pipeline to resize, color convert, convert layout (HWC to CHW), - * normalize, and copy raw image buffers into target tensors matching model - * input shapes. All intermediate scratch tensors are pre-allocated and safely - * disposed of when calling `dispose()`. - * @category Typescript API - * @param options Normalization scaling coefficients, interpolation algorithms, and - * crop/resize modes. - * @param outputShape Expected output shape of the model input tensor (must - * match `[1, 3, H, W]` or `[3, H, W]`). - * @returns An object containing the `process` runner function and a `dispose` - * method. + * Image preprocessor runner for transforming image buffers into model input tensors. + * @category CV / Types */ -export function createImagePreprocessor( - options: ImagePreprocessorOptions, - outputShape: number[] -): { +export type ImagePreprocessor = { + /** + * Releases all allocated native resources. + */ + readonly dispose: () => void; + /** * Preprocesses the input image by resizing, converting color space, changing * format layout, and normalizing values, copying the output directly to the @@ -60,14 +52,32 @@ export function createImagePreprocessor( * need to dispose of it manually. * @param input The input image buffer to preprocess. * @returns A reference to the output tensor containing preprocessed float32 - * data. + * data of shape `[3, H, W]` (or `[1, 3, H, W]`) and data type `float32`. */ - process: (input: ImageBuffer) => Tensor; - /** - * Releases all allocated native resources. - */ - dispose: () => void; -} { + readonly process: (input: ImageBuffer) => Tensor; +}; + +/** + * Creates a reusable image preprocessor pipeline. + * + * Configures a pipeline to resize, color convert, convert layout (HWC to CHW), + * normalize, and copy raw image buffers into target tensors matching model + * input shapes. All intermediate scratch tensors are pre-allocated and safely + * disposed of when calling `dispose()`. + * @category CV / Functions + * @param options Normalization scaling coefficients, interpolation algorithms, and + * resize modes. + * See {@link ImagePreprocessorOptions}. + * @param outputShape Expected output shape of the preprocessed model input + * tensor (must match rank-3 `[3, H, W]` or rank-4 `[1, 3, H, W]`). + * @returns An instantiated {@link ImagePreprocessor} pipeline. + * @throws {RnExecuTorchError} With code `SCHEMA_MISMATCH` if `outputShape` does + * not match rank-3 `[3, H, W]` or rank-4 `[1, 3, H, W]`. + */ +export function createImagePreprocessor( + options: ImagePreprocessorOptions, + outputShape: number[] +): ImagePreprocessor { 'worklet'; const numRgbChannels = 3; const isRank3 = outputShape.length === 3 && outputShape[0] === numRgbChannels; @@ -91,10 +101,8 @@ export function createImagePreprocessor( const [tColor, tChanFirst, tNorm, tOutput] = tensors; const { resizeMode, interpolation, normalizeOpts, padValue } = options; - const dispose = () => { - 'worklet'; - tensors.forEach((t) => t.dispose()); - }; + const dispose = () => tensors.forEach((t) => t.dispose()); + const process = (input: ImageBuffer): Tensor => { 'worklet'; const { data, width, height, format } = input; diff --git a/packages/react-native-executorch/src/extensions/cv/utils/paddleOcrUtils.ts b/packages/react-native-executorch/src/extensions/cv/utils/paddleOcrUtils.ts index b7f2ec3b10..388cf065dd 100644 --- a/packages/react-native-executorch/src/extensions/cv/utils/paddleOcrUtils.ts +++ b/packages/react-native-executorch/src/extensions/cv/utils/paddleOcrUtils.ts @@ -1,6 +1,6 @@ -// Native decode helper for the PP-OCRv6 pipeline. Wraps a fused C++ op: tracing -// contours in TypeScript would mean pulling the whole detector output across the -// bridge per call. +/** + * DBNet contour tracing and text quad extraction utilities for PP-OCRv6. + */ import { rnexecutorchJsi } from '../../../native/bridge'; import type { Tensor } from '../../../core/tensor'; @@ -30,7 +30,7 @@ function quadsFromFlat(flat: ArrayLike): Quad[] { /** * Thresholds for {@link extractDbnetTextQuads}. - * @category Types + * @category CV / Types */ export type DbnetDecodeOptions = { /** Binarization threshold on the probability map. */ @@ -49,10 +49,14 @@ export type DbnetDecodeOptions = { * Decodes a DBNet probability map into oriented text quads: binarizes the map, * traces contours, scores each candidate by its mean probability and unclips the * survivors back to their unshrunk size. - * @category Typescript API + * @category CV / Functions * @param probabilityMap The `detect` output, shape `[1, 1, H, W]`, post-sigmoid. - * @param options Decode thresholds. + * @param options Decode thresholds. See {@link DbnetDecodeOptions}. * @returns The decoded quads, in detector-input pixel space and arbitrary order. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shape or + * data type is invalid, `RESOURCE_BUSY` if the tensor is in use, + * `RESOURCE_DISPOSED` if the tensor was disposed, or `EXECUTION_FAILED` if + * native decode returns invalid output. */ export function extractDbnetTextQuads(probabilityMap: Tensor, options: DbnetDecodeOptions): Quad[] { 'worklet'; diff --git a/packages/react-native-executorch/src/extensions/llm/index.ts b/packages/react-native-executorch/src/extensions/llm/index.ts index ba276aba05..bae6c85305 100644 --- a/packages/react-native-executorch/src/extensions/llm/index.ts +++ b/packages/react-native-executorch/src/extensions/llm/index.ts @@ -1,3 +1,8 @@ +/** + * Large Language Model (LLM) runners, multi-turn chat sessions, and + * tool-calling utilities. + */ + export * from './llmRunner'; export * from './utils/chatPreprocessor'; export * from './utils/tokenizerConfig'; diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 6cca8b176f..ede9ca1c85 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -1,3 +1,7 @@ +/** + * Low-level native ExecuTorch LLM runner types and factory. + */ + import type { Tensor } from '../../core/tensor'; import { rnexecutorchJsi } from '../../native/bridge'; @@ -6,7 +10,7 @@ declare const llmRunnerBrand: unique symbol; /** * Configuration options for LLM text generation. * @experimental This API is experimental and might change in future releases. - * @category Types + * @category LLM / Types */ export type LLMGenerationConfig = { /** Whether to echo the prompt in the generated output. */ @@ -22,7 +26,7 @@ export type LLMGenerationConfig = { /** * Execution and performance statistics for a generation call. * @experimental This API is experimental and might change in future releases. - * @category Types + * @category LLM / Types */ export type LLMGenerationStats = { /** Number of tokens in the input prompt. */ @@ -46,7 +50,7 @@ export type LLMGenerationStats = { /** * Low-level non-text media input tensor payloads. * @experimental This API is experimental and might change in future releases. - * @category Types + * @category LLM / Types */ export type MediaInput = | { readonly kind: 'image'; readonly image: Tensor } @@ -55,21 +59,21 @@ export type MediaInput = /** * Supported non-text input modality keys (e.g. `'image'`, `'audio'`). * @experimental This API is experimental and might change in future releases. - * @category Types + * @category LLM / Types */ export type Modality = MediaInput['kind']; /** * Text or interleaved multimodal prompt input for a low-level LLM runner. * @experimental This API is experimental and might change in future releases. - * @category Types + * @category LLM / Types */ export type Prompt = string | readonly (string | MediaInput)[]; /** * Current KV cache state and capacity metrics for an LLM runner. * @experimental This API is experimental and might change in future releases. - * @category Types + * @category LLM / Types */ export type LLMKVCacheState = { /** Current token position index / number of occupied tokens in the KV cache. */ @@ -87,7 +91,7 @@ export type LLMKVCacheState = { * @experimental This API is experimental and might change in future releases. It * relies on experimental ExecuTorch runtime extensions and injected member-pointer * accessors to manage KV cache state that may evolve across releases. - * @category Types + * @category LLM / Types */ export type LLMRunner = { /** Path to the local model file. */ @@ -98,7 +102,7 @@ export type LLMRunner = { readonly modalities: readonly Modality[]; /** - * Disposes the native LLM runner and releases the loaded model memory. + * Releases all allocated native resources. */ dispose(): void; @@ -150,12 +154,12 @@ export type LLMRunner = { * @experimental This API is experimental and might change in future releases. It * relies on experimental ExecuTorch runtime extensions and injected member-pointer * accessors to manage KV cache state that may evolve across releases. - * @category Typescript API + * @category LLM / Functions * @param modelPath Path to the local `.pte` model file. * @param tokenizerPath Path to the local tokenizer configuration file (e.g. `tokenizer.json`). - * @param modalities List of supported input non-text modalities (e.g. `['image']`). - * Defaults to text-only. - * @returns A native LLMRunner instance. + * @param modalities List of supported input non-text modalities (e.g. + * `['image']`). When omitted, defaults to text-only. + * @returns A native {@link LLMRunner} instance. */ export function createLLMRunner( modelPath: string, diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index f936514aef..a75ee7704b 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -1,3 +1,9 @@ +/** + * Multi-turn LLM chat session with history management, tool calling, and KV + * cache prefilling. + * @module LLM/Tasks/LLMChatSession + */ + import { scheduleOnRN, type WorkletRuntime } from 'react-native-worklets'; import RNBlobUtil from 'react-native-blob-util'; @@ -14,35 +20,15 @@ import { import { parseTokenizerConfig } from '../utils/tokenizerConfig'; import { createChatPreprocessor, - type ChatMediaInput, type ChatMessageContent, type ChatMessage, - type LLMImagePreprocessorConfig, - type LLMAudioPreprocessorConfig, type LLMMediaPreprocessorConfig, } from '../utils/chatPreprocessor'; -import type { ToolDefinition, ToolCall, ToolParser, ToolParserResult } from '../utils/toolCalling'; - -export type { - LLMKVCacheState, - LLMGenerationConfig, - LLMGenerationStats, - Modality, - ChatMediaInput, - ChatMessageContent, - ChatMessage, - LLMImagePreprocessorConfig, - LLMAudioPreprocessorConfig, - LLMMediaPreprocessorConfig, - ToolDefinition, - ToolCall, - ToolParser, - ToolParserResult, -}; +import type { ToolDefinition, ToolParser } from '../utils/toolCalling'; /** * Model configuration required to instantiate an LLM chat session. - * @category Types + * @category LLM / Types */ export type LLMModel = { /** Local path or remote URL of the `.pte` model file. */ @@ -59,7 +45,7 @@ export type LLMModel = { /** * Configuration options for tool calling in an LLM chat session. - * @category Types + * @category LLM / Types */ export type LLMToolOpts = { /** Tool definitions available to the model. */ @@ -72,7 +58,7 @@ export type LLMToolOpts = { /** * Options for configuring an LLM chat session. - * @category Types + * @category LLM / Types */ export type LLMChatSessionOptions = { /** Default generation configuration options. */ @@ -92,7 +78,7 @@ export type LLMChatSessionOptions = { /** * Result returned by an LLM chat turn. - * @category Types + * @category LLM / Types */ export type LLMChatTurnResult = { /** The messages added to history during this chat turn. */ @@ -109,7 +95,7 @@ export type LLMChatTurnResult = { /** * Handle to an active LLM chat session. - * @category Types + * @category LLM / Types */ export type LLMChatSession = { /** @@ -138,6 +124,8 @@ export type LLMChatSession = { * @param onToken Callback fired on the RN thread for each decoded token. * @param genConfig Generation options overriding session defaults. * @returns A promise resolving to the generated messages and turn stats. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if the message is + * malformed, or `INVALID_STATE` if the chat template is non-monotonic. */ sendMessage( message: ChatMessageContent, @@ -178,11 +166,11 @@ const DEFAULT_MAX_TURNS = 5; /** * Instantiates an LLM chat session using background thread execution. - * @category Typescript API + * @category LLM / Tasks * @param config Model configuration containing model, tokenizer, and tokenizer config paths. * @param options Custom generation, tool calling, and state options. * @param runtime The worklet runtime thread to run native generation on. - * @returns A Promise resolving to an LLMChatSession instance. + * @returns A promise resolving to the instantiated {@link LLMChatSession} session. */ export async function createLLMChatSession( config: LLMModel, diff --git a/packages/react-native-executorch/src/extensions/llm/utils/chatPreprocessor.ts b/packages/react-native-executorch/src/extensions/llm/utils/chatPreprocessor.ts index 2e33654522..77b5b37ab0 100644 --- a/packages/react-native-executorch/src/extensions/llm/utils/chatPreprocessor.ts +++ b/packages/react-native-executorch/src/extensions/llm/utils/chatPreprocessor.ts @@ -1,3 +1,8 @@ +/** + * Multimodal chat message template rendering, incremental prompt diffing, and + * media tensor preprocessing for LLM runners. + */ + import { Template } from '@huggingface/jinja'; import { tensor, type Tensor } from '../../../core/tensor'; @@ -6,14 +11,14 @@ import type { ImageBuffer } from '../../cv'; import { createImagePreprocessor, type ImagePreprocessorOptions, -} from '../../cv/tasks/preprocessing'; +} from '../../cv/utils/imagePreprocessor'; import type { Modality, Prompt, MediaInput } from '../llmRunner'; import type { ToolDefinition, ToolCall } from './toolCalling'; /** * High-level media payload input for chat turns. - * @category Types + * @category LLM / Types */ export type ChatMediaInput = | { readonly kind: 'image'; readonly image: ImageBuffer } @@ -21,13 +26,13 @@ export type ChatMediaInput = /** * Interleaved text and media content for a chat turn. - * @category Types + * @category LLM / Types */ export type ChatMessageContent = string | readonly (string | ChatMediaInput)[]; /** * Conversation turn representing system, user, assistant, or tool execution messages. - * @category Types + * @category LLM / Types */ export type ChatMessage = Readonly< | { role: 'system' | 'user'; content: ChatMessageContent } @@ -37,7 +42,7 @@ export type ChatMessage = Readonly< /** * Image preprocessing and sentinel token config for vision-language LLMs. - * @category Types + * @category LLM / Types */ export type LLMImagePreprocessorConfig = { /** Sentinel token delimiters inserted into Jinja prompts. */ @@ -50,7 +55,7 @@ export type LLMImagePreprocessorConfig = { /** * Audio preprocessing and sentinel token config for audio-language LLMs. - * @category Types + * @category LLM / Types */ export type LLMAudioPreprocessorConfig = { /** Sentinel token delimiters inserted into Jinja prompts. */ @@ -59,28 +64,34 @@ export type LLMAudioPreprocessorConfig = { /** * Preprocessor configuration for media modalities. - * @category Types + * @category LLM / Types */ export type LLMMediaPreprocessorConfig = { + /** Image preprocessing configuration for vision-language models. */ readonly image?: LLMImagePreprocessorConfig; + /** Audio preprocessing configuration for audio-language models. */ readonly audio?: LLMAudioPreprocessorConfig; }; /** * Options for instantiating a ChatPreprocessor. - * @category Types + * @category LLM / Types */ export type ChatPreprocessorConfig = { + /** Jinja chat template string for prompt rendering. */ readonly chatTemplate: string; + /** Tool definitions available to the model. */ readonly tools?: readonly ToolDefinition[]; + /** Supported input modalities (e.g. `['image']`). */ readonly modalities?: readonly Modality[]; + /** Media preprocessing configuration for non-text inputs. */ readonly preprocessorConfig?: LLMMediaPreprocessorConfig; }; /** * Chat preprocessor object for Jinja template rendering, prompt diffing, and * media tensor preprocessing. - * @category Types + * @category LLM / Types */ export type ChatPreprocessor = { /** @@ -161,6 +172,8 @@ const MEDIA_SENTINEL_REGEX = /\uFFFC__ET_MEDIA_(\d+)__/g; * @param content Chat message text or media input array. * @param options Sentinel tokens, supported modalities, and starting media index. * @returns Object containing serialized text and a map of media items. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if a media kind is + * unsupported or a required sentinel token is missing. */ function chatContentToString( content?: ChatMessageContent, @@ -282,7 +295,7 @@ function prepareStringMessages( /** * Handles Jinja template formatting and media tensor preprocessing for chat * turns. - * @category Typescript API + * @category LLM / Functions * @param config Preprocessor configuration including template and preprocessor * settings. * @returns A ChatPreprocessor object containing process, render, buildPrompt, diff --git a/packages/react-native-executorch/src/extensions/llm/utils/tokenizerConfig.ts b/packages/react-native-executorch/src/extensions/llm/utils/tokenizerConfig.ts index ce13438868..5f3f11d346 100644 --- a/packages/react-native-executorch/src/extensions/llm/utils/tokenizerConfig.ts +++ b/packages/react-native-executorch/src/extensions/llm/utils/tokenizerConfig.ts @@ -1,11 +1,18 @@ +/** + * Parser for HuggingFace `tokenizer_config.json` chat templates and special + * tokens. + */ + import { RnExecuTorchError } from '../../../core/error'; /** * Model chat template configuration resolved from tokenizer config file. - * @category Types + * @category LLM / Types */ export type TokenizerChatConfig = { + /** Jinja chat template string for prompt rendering. */ readonly chatTemplate: string; + /** End-of-sequence token string. */ readonly eosToken: string; }; @@ -19,9 +26,11 @@ function resolveToken(token: unknown): string | undefined { /** * Parses raw JSON configuration from `tokenizer_config.json` into a normalized format. - * @category Utils + * @category LLM / Functions * @param config Raw JSON object from tokenizer_config.json. * @returns A parsed TokenizerChatConfig object. + * @throws {RnExecuTorchError} With code `LOAD_FAILED` if `chat_template` is not + * a string or `eos_token` is missing. */ export function parseTokenizerConfig(config: any): TokenizerChatConfig { let chatTemplate = config.chat_template; diff --git a/packages/react-native-executorch/src/extensions/llm/utils/toolCalling.ts b/packages/react-native-executorch/src/extensions/llm/utils/toolCalling.ts index d591fa4167..cbd968309f 100644 --- a/packages/react-native-executorch/src/extensions/llm/utils/toolCalling.ts +++ b/packages/react-native-executorch/src/extensions/llm/utils/toolCalling.ts @@ -1,25 +1,37 @@ +/** + * Tool call definition, XML/JSON parsing, and schema generation for LLM + * function calling. + */ + import type { ChatMessageContent } from './chatPreprocessor'; /** * JSON Schema definition for tool parameter inputs. - * @category Types + * @category LLM / Types */ export type ToolParameters = { + /** JSON Schema type, typically `'object'`. */ readonly type?: 'object' | string; + /** JSON Schema properties mapping parameter names to their schemas. */ readonly properties?: Record; + /** Names of required parameters. */ readonly required?: readonly string[]; readonly [key: string]: unknown; }; /** * Declaration of a tool available to the model. - * @category Types + * @category LLM / Types */ export type ToolDefinition = Record> = { + /** Tool type discriminator, typically `'function'`. */ readonly type: 'function' | string; readonly function: { + /** Function name the model should invoke. */ readonly name: string; + /** Human-readable description of what the tool does. */ readonly description?: string; + /** JSON Schema describing the function's parameters. */ readonly parameters?: ToolParameters; readonly [key: string]: unknown; }; @@ -32,29 +44,35 @@ export type ToolDefinition = Record; }; }; /** * Structured tool parsing result containing detected tool calls and remaining text. - * @category Types + * @category LLM / Types */ export type ToolParserResult = { + /** Detected tool calls extracted from the model output. */ readonly toolCalls: readonly ToolCall[]; + /** Remaining text content after tool call extraction. */ readonly textContent?: string; }; /** * Function signature for parsing tool calls from model output text. * Returns undefined if no tool call was detected. - * @category Types + * @category LLM / Types */ export type ToolParser = (text: string) => ToolParserResult | undefined; diff --git a/packages/react-native-executorch/src/extensions/math.ts b/packages/react-native-executorch/src/extensions/math.ts index a9f15764c7..a3feccc823 100644 --- a/packages/react-native-executorch/src/extensions/math.ts +++ b/packages/react-native-executorch/src/extensions/math.ts @@ -1,15 +1,40 @@ +/** + * Native C++ and JavaScript tensor math and random number generation utilities. + */ + import { rnexecutorchJsi } from '../native/bridge'; import type { Tensor } from '../core/tensor'; import { RnExecuTorchError } from '../core/error'; +/** + * Configuration options for normal distribution random number generation. + * @category Math / Types + */ +export type RandomNormalOptions = { + /** The mean of the distribution. */ + readonly mean?: number; + /** The standard deviation of the distribution. */ + readonly std?: number; + /** + * The seed for the underlying generator. When omitted, a random seed is + * generated so different values are produced each call. + */ + readonly seed?: number; +}; + /** * Computes the element-wise sigmoid activation on a float32 source tensor and * writes the result to a destination tensor. - * @category Typescript API - * @param src The input float32 source tensor. Shape [d1,...,dn]. - * @param dst The pre-allocated float32 destination tensor to write the result - * to. `dst` tensor must have the same shape as `src`. Shape [d1,...,dn]. - * @returns The destination tensor containing the sigmoid output. + * @category Math / Functions + * @param src The input source tensor. Expected shape `[d1, ..., dn]` with data + * type `float32`. + * @param dst The pre-allocated destination tensor to write the result to. + * Expected shape `[d1, ..., dn]` matching `src` with data type `float32`. + * @returns The destination tensor `dst` containing the sigmoid output of shape + * `[d1, ..., dn]` and data type `float32`. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes or + * data types are invalid, `RESOURCE_BUSY` if a tensor is in use, or + * `RESOURCE_DISPOSED` if either tensor was disposed. */ export function sigmoid(src: Tensor, dst: Tensor): Tensor { 'worklet'; @@ -19,13 +44,18 @@ export function sigmoid(src: Tensor, dst: Tensor): Tensor { /** * Computes the softmax activation along a specified axis on a float32 source * tensor and writes the result to a destination tensor. - * @category Typescript API - * @param src The input float32 source tensor. Shape [d1,...,dn]. - * @param dst The pre-allocated float32 destination tensor to write the result - * to. `dst` tensor must have the same shape as `src`. Shape [d1,...,dn]. - * @param axis The dimension along which softmax is computed. Defaults to -1 - * (last dimension). - * @returns The destination tensor containing the softmax output. + * @category Math / Functions + * @param src The input source tensor. Expected shape `[d1, ..., dn]` with data + * type `float32`. + * @param dst The pre-allocated destination tensor to write the result to. + * Expected shape `[d1, ..., dn]` matching `src` with data type `float32`. + * @param axis The dimension along which softmax is computed. Negative indexing + * is supported (e.g. `-1` for the last dimension). Defaults to `-1`. + * @returns The destination tensor `dst` containing the softmax output of shape + * `[d1, ..., dn]` and data type `float32`. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes, + * data types, or `axis` are invalid, `RESOURCE_BUSY` if a tensor is in use, or + * `RESOURCE_DISPOSED` if either tensor was disposed. */ export function softmax(src: Tensor, dst: Tensor, axis: number = -1): Tensor { 'worklet'; @@ -35,13 +65,19 @@ export function softmax(src: Tensor, dst: Tensor, axis: number = -1): Tensor { /** * Computes the indices of the maximum values along a specified axis on a * float32 source tensor and writes the result to an int32 destination tensor. - * @category Typescript API - * @param src The input float32 source tensor. Shape [d1,...,dk,...,dn]. - * @param dst The pre-allocated int32 destination tensor to write the indices - * to. Shape [d1,...,1,...,dn]. - * @param axis The dimension along which argmax is computed. Defaults to -1 - * (last dimension). - * @returns The destination tensor containing the argmax output. + * @category Math / Functions + * @param src The input source tensor. Expected shape `[d1, ..., dk, ..., dn]` + * with data type `float32`. + * @param dst The pre-allocated destination tensor to write the indices to. + * Expected shape `[d1, ..., 1, ..., dn]` (same rank as `src` with dimension 1 + * along `axis`) and data type `int32`. + * @param axis The dimension along which argmax is computed. Negative indexing + * is supported (e.g. `-1` for the last dimension). Defaults to `-1`. + * @returns The destination tensor `dst` containing the argmax indices of shape + * `[d1, ..., 1, ..., dn]` and data type `int32`. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes, + * data types, or `axis` are invalid, `RESOURCE_BUSY` if a tensor is in use, or + * `RESOURCE_DISPOSED` if either tensor was disposed. */ export function argmax(src: Tensor, dst: Tensor, axis: number = -1): Tensor { 'worklet'; @@ -53,7 +89,7 @@ export function argmax(src: Tensor, dst: Tensor, axis: number = -1): Tensor { * given by an int32 index tensor. Pairs with {@link argmax}, whose output has * exactly the shape this expects, so `argmax` then `gather` yields the maximum * values alongside their indices. - * @category Typescript API + * @category Math / Functions * @param src The input float32 source tensor. Shape [d1,...,dk,...,dn]. * @param indices The int32 index tensor, one index per lane. Shape * [d1,...,1,...,dn]. @@ -73,12 +109,18 @@ export function gather(src: Tensor, indices: Tensor, dst: Tensor, axis: number = /** * Applies the element-wise threshold step function on a float32 source tensor and * writes the result to a destination tensor. - * @category Typescript API - * @param src The input float32 source tensor. Shape [d1,...,dn]. + * @category Math / Functions + * @param src The input source tensor. Expected shape `[d1, ..., dn]` with data + * type `float32`. * @param dst The pre-allocated destination tensor to write the result to. - * `dst` tensor must have the same shape as `src` and have dtype float32. - * @param thresholdVal The threshold value above or equal to which elements are mapped to 1.0. - * @returns The destination tensor containing the threshold step output. + * Expected shape `[d1, ..., dn]` matching `src` with data type `float32`. + * @param thresholdVal The threshold value above or equal to which elements are + * mapped to 1.0. + * @returns The destination tensor `dst` containing the threshold step output of + * shape `[d1, ..., dn]` and data type `float32`. + * @throws {RnExecuTorchError} With code `INVALID_ARGUMENT` if tensor shapes or + * data types are invalid, `RESOURCE_BUSY` if a tensor is in use, or + * `RESOURCE_DISPOSED` if either tensor was disposed. */ export function threshold(src: Tensor, dst: Tensor, thresholdVal: number): Tensor { 'worklet'; @@ -89,9 +131,9 @@ export function threshold(src: Tensor, dst: Tensor, thresholdVal: number): Tenso * Creates a mulberry32 pseudo-random generator producing uniform values in * `[0, 1)`. Unlike `Math.random` it accepts a seed, so a fixed seed yields a * reproducible sequence. - * @category Typescript API - * @param seed The 32-bit seed for the generator. - * @returns A function returning the next uniform value in `[0, 1)`. + * @category Math / Functions + * @param seed The 32-bit integer seed for the generator. + * @returns A function returning the next uniform pseudo-random number in `[0, 1)`. */ export function mulberry32(seed: number): () => number { 'worklet'; @@ -112,23 +154,19 @@ export function mulberry32(seed: number): () => number { /** * Draws normally distributed values using the Box–Muller transform, seeded via * {@link mulberry32} so a fixed `seed` reproduces the same sequence. - * @category Typescript API + * @category Math / Functions * @param size The number of values to draw. - * @param options Distribution parameters. - * @param options.mean The mean of the distribution. Defaults to 0. - * @param options.std The standard deviation of the distribution. Defaults to 1. - * @param options.seed The seed for the underlying generator. Defaults to a - * time-based value, so omitting it produces different values each call. + * @param options Distribution parameters. When options or any individual + * properties are omitted, defaults to `mean: 0`, `std: 1`, and a random seed. + * See {@link RandomNormalOptions}. * @returns A `Float32Array` of `size` normally distributed values. */ -export function randomNormal( - size: number, - options?: { mean?: number; std?: number; seed?: number } -): Float32Array { +export function randomNormal(size: number, options?: RandomNormalOptions): Float32Array { 'worklet'; const mean = options?.mean ?? 0; const std = options?.std ?? 1; - const uniform = mulberry32(options?.seed ?? Date.now()); + const seed = options?.seed ?? Math.floor(Math.random() * 0x100000000); + const uniform = mulberry32(seed); const out = new Float32Array(size); for (let i = 0; i < size; i += 2) { // Guard against log(0) when the generator returns exactly 0. @@ -147,7 +185,7 @@ export function randomNormal( * kind — the equivalent of PyTorch's `repeat_interleave` over a 1-D input. * * Non-positive repeat counts drop their element. - * @category Typescript API + * @category Math / Functions * @typeParam T The array kind of `values` (any typed array or a plain array), * preserved in the result. * @param values The values to repeat. diff --git a/packages/react-native-executorch/src/extensions/nlp/index.ts b/packages/react-native-executorch/src/extensions/nlp/index.ts index b210cb0086..476fe02d37 100644 --- a/packages/react-native-executorch/src/extensions/nlp/index.ts +++ b/packages/react-native-executorch/src/extensions/nlp/index.ts @@ -1,2 +1,7 @@ +/** + * Natural Language Processing (NLP) tokenization, text embedding, and privacy + * filtering pipelines. + */ + export * from './tokenizer'; export * from './utils/privacyFilterUtils'; diff --git a/packages/react-native-executorch/src/extensions/nlp/tasks/privacyFilter.ts b/packages/react-native-executorch/src/extensions/nlp/tasks/privacyFilter.ts index 895600224a..7230f5581f 100644 --- a/packages/react-native-executorch/src/extensions/nlp/tasks/privacyFilter.ts +++ b/packages/react-native-executorch/src/extensions/nlp/tasks/privacyFilter.ts @@ -1,8 +1,21 @@ +/** + * Privacy Filter task pipeline for detecting personally identifiable + * information (PII). + * @module NLP/Tasks/PrivacyFilter + */ + import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateSpec, DynamicDim as Dyn, method, i64, f32, constr } from '../../../core/schema'; +import { + validateSpec, + DynamicDim as Dyn, + method, + i64, + f32, + constraint, +} from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { RnExecuTorchError } from '../../../core/error'; @@ -11,18 +24,16 @@ import { buildGrammar, computeCharOffsets, extractSpans, - piiSegments, viterbiDecode, type PiiEntity, type PiiEntityType, - type PiiSegment, type ViterbiBiases, } from '../utils/privacyFilterUtils'; /** * Options describing a privacy filter model's label space and decoding * behavior. - * @category Types + * @category NLP / Types * @typeParam Label The model's BIOES label space, defined alongside the model * in the `models` registry. */ @@ -47,7 +58,7 @@ export type PrivacyFilterOptions