diff --git a/.agents/skills/model-schema-validation/SKILL.md b/.agents/skills/model-schema-validation/SKILL.md index 21a2f70bc3..b074e3dd82 100644 --- a/.agents/skills/model-schema-validation/SKILL.md +++ b/.agents/skills/model-schema-validation/SKILL.md @@ -62,6 +62,20 @@ The `SymbolicTensor` helper supports specifying **Symbolic Shapes** where intege --- +## ⚙️ Runtime Dynamic Dimensions: the `get_dynamic_dims_` companion + +`SymbolicTensor` string symbols let the **validator** accept a range of shapes, but an ExecuTorch `.pte`'s metadata only serializes the **static upper bound** of a dynamic dimension — not its active `[min, max, step]` range. So for an input dimension that genuinely varies at runtime (e.g. a text model's sequence length), the model must be **exported with a companion method** that re-exposes the range to the runtime: + +* **Name**: `get_dynamic_dims_` — e.g. `get_dynamic_dims_forward` for `forward`. +* **Signature**: takes no arguments. +* **Returns**: a list of outputs, one per `Tensor` input of the target method (scalar inputs are skipped), each a **2D `int32` tensor of shape `[rank, 3]`** whose rows are `[min, max, step]` constraints for that input's dimensions — e.g. `[1, 1, 1]` for a fixed dimension and `[1, maxTokens, 1]` for the dynamic one. + +At load time the C++ core (`Model::parseDynamicInputShapes`) reads this companion and validates inputs against the range; `model.execute` then accepts tensors at their exact runtime length. **Without the companion, a method falls back to exact per-dimension validation** — it only accepts the single shape it was exported with. So a `.pte` whose metadata reports `[1, 512]` but is meant to accept `[1, 1..512]` MUST ship `get_dynamic_dims_forward`, or variable-length inputs are rejected at runtime. + +This is an **export-side contract** (the export-scripts repo provides a `build_dynamic_dims_program` helper that emits the companion). The TypeScript task only declares the symbol via `SymbolicTensor` and reads the resulting upper bound from `meta.inputTensorMeta`. + +--- + ## 📋 Common Validation Recipes ### 1. Classification diff --git a/apps/computer-vision/app/_layout.tsx b/apps/computer-vision/app/_layout.tsx index f269b27aae..1a53311ad9 100644 --- a/apps/computer-vision/app/_layout.tsx +++ b/apps/computer-vision/app/_layout.tsx @@ -27,6 +27,13 @@ export default function Layout() { title: 'Classification', }} /> + (MODEL_OPTIONS[0].value); const [imageUri, setImageUri] = useState(null); const [isProcessing, setIsProcessing] = useState(false); @@ -97,7 +99,10 @@ function ClassificationContent() { return ( Upload or capture an image to identify objects using a classifier. diff --git a/apps/computer-vision/app/imageEmbeddings/index.tsx b/apps/computer-vision/app/imageEmbeddings/index.tsx new file mode 100644 index 0000000000..972c063bd6 --- /dev/null +++ b/apps/computer-vision/app/imageEmbeddings/index.tsx @@ -0,0 +1,250 @@ +import React, { useState } from 'react'; +import { View, Text, StyleSheet, ScrollView, TextInput, TouchableOpacity } from 'react-native'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; +import { commonStyles, ColorPalette } from '../../theme'; +import { useImage } from '@shopify/react-native-skia'; +import { useImageEmbedder, useTextEmbedder, models } from 'react-native-executorch'; +import ScreenWrapper from '../../components/ScreenWrapper'; +import { getImage, skImageToBuffer } from '../../utils'; +import { ModelPicker, type ModelOption } from '../../components/ModelPicker'; +import { ImageViewport } from '../../components/ImageViewport'; +import { ModelStatus } from '../../components/ModelStatus'; +import { LatencyIndicator } from '../../components/LatencyIndicator'; +import { Button } from '../../components/Button'; + +const IMAGE_MODEL_OPTIONS: ModelOption[] = [ + { + label: 'CLIP ViT-B/32 (INT8)', + value: models.imageEmbeddings.CLIP_VIT_BASE_PATCH32.XNNPACK_INT8, + }, + { + label: 'CLIP ViT-B/32 (FP32)', + value: models.imageEmbeddings.CLIP_VIT_BASE_PATCH32.XNNPACK_FP32, + }, +]; + +const DEFAULT_LABELS = [ + 'a photo of a dog', + 'a photo of a cat', + 'a landscape photo', + 'a photo of food', + 'a photo of people', +]; + +// CLIP text and image embeddings are L2-normalized, so their cosine similarity +// is the dot product. +const dot = (a: Float32Array, b: Float32Array) => { + let s = 0; + for (let i = 0; i < a.length; i++) { + s += a[i]! * b[i]!; + } + return s; +}; + +function ImageEmbeddingsContent() { + const [selectedImageModel, setSelectedImageModel] = useState(IMAGE_MODEL_OPTIONS[0].value); + const [imageUri, setImageUri] = useState(null); + const [labels, setLabels] = useState(DEFAULT_LABELS); + const [newLabel, setNewLabel] = useState(''); + const [results, setResults] = useState<{ label: string; score: number }[]>([]); + const [latency, setLatency] = useState(null); + const [isProcessing, setIsProcessing] = useState(false); + const [error, setError] = useState(null); + + const insets = useSafeAreaInsets(); + const skiaImage = useImage(imageUri, (err) => setError(err.message || String(err))); + + // Zero-shot classification pairs a CLIP image encoder with the CLIP text + // encoder and scores the image against each text label by embedding similarity. + const imageModel = useImageEmbedder(selectedImageModel); + const textModel = useTextEmbedder(models.textEmbeddings.CLIP_VIT_BASE_PATCH32_TEXT); + + const ready = imageModel.isReady && textModel.isReady; + + const pickImage = async () => { + setError(null); + try { + const uri = await getImage(false); + if (!uri) return; + setImageUri(uri); + setResults([]); + setLatency(null); + } catch (e: any) { + setError(e.message || String(e)); + } + }; + + const classify = async () => { + if (!skiaImage || !ready || !imageModel.embed || !textModel.embed) return; + setIsProcessing(true); + setError(null); + try { + const start = Date.now(); + const imageEmbedding = await imageModel.embed(skImageToBuffer(skiaImage)); + const scored: { label: string; score: number }[] = []; + for (const label of labels) { + const textEmbedding = await textModel.embed(label); + scored.push({ label, score: dot(imageEmbedding, textEmbedding) }); + } + scored.sort((a, b) => b.score - a.score); + setLatency(Date.now() - start); + setResults(scored); + } catch (e: any) { + setError(e.message || String(e)); + } finally { + setIsProcessing(false); + } + }; + + const addLabel = () => { + const trimmed = newLabel.trim(); + if (!trimmed || labels.includes(trimmed)) return; + setLabels((prev) => [...prev, trimmed]); + setNewLabel(''); + setResults([]); + }; + + const removeLabel = (label: string) => { + setLabels((prev) => prev.filter((l) => l !== label)); + setResults((prev) => prev.filter((r) => r.label !== label)); + }; + + const activeError = imageModel.error + ? String(imageModel.error) + : textModel.error + ? String(textModel.error) + : error; + + return ( + + + Pick an image, then rank text labels by how well CLIP matches them to it (zero-shot + classification). + + + { + setSelectedImageModel(model); + setResults([]); + setLatency(null); + }} + /> + + + + + + +