From a1bfd8d7553587b71a51cd92103fdbe807933f54 Mon Sep 17 00:00:00 2001 From: IgorSwat Date: Tue, 23 Jun 2026 15:07:54 +0200 Subject: [PATCH 1/8] Enable int64 tensor dtype --- packages/react-native-executorch/cpp/core/dtype.cpp | 13 ++++++++++++- packages/react-native-executorch/cpp/core/dtype.h | 1 + packages/react-native-executorch/src/core/tensor.ts | 8 ++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/dtype.cpp b/packages/react-native-executorch/cpp/core/dtype.cpp index d43418a7c9..b183daf0d6 100644 --- a/packages/react-native-executorch/cpp/core/dtype.cpp +++ b/packages/react-native-executorch/cpp/core/dtype.cpp @@ -9,10 +9,13 @@ DType parseDType(const std::string &s) { if (s == "int32") { return DType::int32; } + if (s == "int64") { + return DType::int64; + } if (s == "float32") { return DType::float32; } - throw std::invalid_argument("Unsupported dtype: '" + s + "'. Expected 'uint8', 'int32', or 'float32'"); + throw std::invalid_argument("Unsupported dtype: '" + s + "'. Expected 'uint8', 'int32', 'int64', or 'float32'"); } std::string toString(DType dtype) { @@ -21,6 +24,8 @@ std::string toString(DType dtype) { return "uint8"; case DType::int32: return "int32"; + case DType::int64: + return "int64"; case DType::float32: return "float32"; } @@ -32,6 +37,8 @@ executorch::aten::ScalarType toScalarType(DType dtype) { return executorch::aten::ScalarType::Byte; case DType::int32: return executorch::aten::ScalarType::Int; + case DType::int64: + return executorch::aten::ScalarType::Long; case DType::float32: return executorch::aten::ScalarType::Float; } @@ -43,6 +50,8 @@ DType fromScalarType(executorch::aten::ScalarType st) { return DType::uint8; case executorch::aten::ScalarType::Int: return DType::int32; + case executorch::aten::ScalarType::Long: + return DType::int64; case executorch::aten::ScalarType::Float: return DType::float32; default: @@ -57,6 +66,8 @@ size_t elementSize(DType dtype) { // NOLINTNEXTLINE(bugprone-branch-clone): int32 and float32 are both 4 bytes; the identical branches are intentional. case DType::int32: return 4; + case DType::int64: + return 8; case DType::float32: return 4; } diff --git a/packages/react-native-executorch/cpp/core/dtype.h b/packages/react-native-executorch/cpp/core/dtype.h index 7259b6b18f..fafa0a8429 100644 --- a/packages/react-native-executorch/cpp/core/dtype.h +++ b/packages/react-native-executorch/cpp/core/dtype.h @@ -8,6 +8,7 @@ namespace rnexecutorch::core::types { enum class DType { uint8, int32, + int64, float32 }; diff --git a/packages/react-native-executorch/src/core/tensor.ts b/packages/react-native-executorch/src/core/tensor.ts index 8f5c58e10c..ca12609188 100644 --- a/packages/react-native-executorch/src/core/tensor.ts +++ b/packages/react-native-executorch/src/core/tensor.ts @@ -6,7 +6,7 @@ declare const tensorBrand: unique symbol; * Element data type of a {@link Tensor}. * @category Types */ -export type DType = 'float32' | 'uint8' | 'int32'; +export type DType = 'float32' | 'uint8' | 'int32' | 'int64'; /** * A native ExecuTorch tensor allocated in C++ memory. @@ -53,7 +53,7 @@ export type Tensor = { * tensor's size. * @returns `this` tensor. */ - setData(src: Float32Array | Uint8Array | Int32Array): Tensor; + setData(src: Float32Array | Uint8Array | Int32Array | BigInt64Array): Tensor; /** * Copies data out of this tensor's native buffer into a typed array. @@ -62,7 +62,7 @@ export type Tensor = { * tensor's size. * @returns The same `dst` array, now filled with tensor data. */ - getData(dst: T): T; + getData(dst: T): T; /** * Passes `this` tensor as the first argument to `fn` and returns the result. @@ -115,7 +115,7 @@ export type Tensor = { export function tensor( dtype: DType, shape: number[], - src?: Float32Array | Uint8Array | Int32Array + src?: Float32Array | Uint8Array | Int32Array | BigInt64Array ): Tensor { 'worklet'; const t: Tensor = rnexecutorchJsi.createTensor(shape, dtype); From 30aec624ffb26a633239035cfe19c20e14722899 Mon Sep 17 00:00:00 2001 From: IgorSwat Date: Wed, 24 Jun 2026 13:11:27 +0200 Subject: [PATCH 2/8] Add tensor view implementation --- .../cpp/core/model.cpp | 6 +- .../cpp/core/tensor.cpp | 110 ++++++++++++++++-- .../react-native-executorch/cpp/core/tensor.h | 13 +++ .../cpp/extensions/cv/image_ops.cpp | 2 +- .../cpp/extensions/math/operations.cpp | 8 +- .../src/core/tensor.ts | 14 ++- .../third-party/common/phonemis | 2 +- .../third-party/scripts/vendor_headers.sh | 35 ++++++ .../third-party/scripts/vendor_opencv.sh | 74 ++++++++++++ 9 files changed, 246 insertions(+), 18 deletions(-) create mode 100755 packages/react-native-executorch/third-party/scripts/vendor_headers.sh create mode 100755 packages/react-native-executorch/third-party/scripts/vendor_opencv.sh diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index ba50d9d12b..c9ce16a80d 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -273,7 +273,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { } auto tensorHostObject = val.asObject(rt).getHostObject(rt); - if (!tensorHostObject->data_) { + if (!tensorHostObject->tensor_) { throw jsi::JSError(rt, "execute: inputs[" + std::to_string(i) + "] has been disposed"); } @@ -374,7 +374,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { } auto tensorHostObject = val.asObject(rt).getHostObject(rt); - if (!tensorHostObject->data_) { + if (!tensorHostObject->tensor_) { throw jsi::JSError(rt, "execute: outputTensors[" + std::to_string(tensorOutputIdx) + "] has been disposed"); } @@ -399,7 +399,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { validateTensor(rt, tensorHostObject.get(), tensorMeta, "outputTensors[" + std::to_string(tensorOutputIdx) + "]"); - std::memcpy(tensorHostObject->data_.get(), + std::memcpy(tensorHostObject->data(), output.toTensor().const_data_ptr(), output.toTensor().nbytes()); diff --git a/packages/react-native-executorch/cpp/core/tensor.cpp b/packages/react-native-executorch/cpp/core/tensor.cpp index bdc66f2f60..3f38e10558 100644 --- a/packages/react-native-executorch/cpp/core/tensor.cpp +++ b/packages/react-native-executorch/cpp/core/tensor.cpp @@ -17,6 +17,43 @@ TensorHostObject::TensorHostObject(const std::vector &shape, rnexe tensor_ = executorch::extension::from_blob(data_.get(), shape_, rnexecutorch::core::types::toScalarType(dtype)); } +TensorHostObject::TensorHostObject(std::shared_ptr parent, + const std::vector &shape, size_t offset) { + if (!parent) { + throw std::invalid_argument("Tensor view: parent must not be null"); + } + + if (shape.empty()) { + throw std::invalid_argument("Tensor view: shape must not be empty"); + } + + if (!parent->tensor_) { + throw std::invalid_argument("Tensor view: parent tensor has been disposed"); + } + + dtype_ = parent->dtype_; + shape_ = shape; + numel_ = std::accumulate(shape.begin(), shape.end(), size_t(1), std::multiplies()); + + const auto elemSize = rnexecutorch::core::types::elementSize(dtype_); + size_ = numel_ * elemSize; + + if (offset + size_ > parent->size_) { + throw std::invalid_argument( + "Tensor view: view size (" + std::to_string(size_) + " bytes) at offset " + + std::to_string(offset) + " exceeds parent buffer (" + + std::to_string(parent->size_) + " bytes)"); + } + + parent_ = std::move(parent); + offset_ = offset; + + tensor_ = executorch::extension::from_blob( + parent_->data() + offset_, + shape_, + rnexecutorch::core::types::toScalarType(dtype_)); +} + jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { auto nameStr = name.utf8(rt); @@ -63,11 +100,11 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) throw jsi::JSError(rt, "copyTo: dst tensor is currently in use"); } - if (!self->data_) { + if (!self->tensor_) { throw jsi::JSError(rt, "copyTo: src tensor has been disposed"); } - if (!dst->data_) { + if (!dst->tensor_) { throw jsi::JSError(rt, "copyTo: dst tensor has been disposed"); } @@ -143,7 +180,7 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) throw jsi::JSError(rt, "setData: Tensor is currently in use and cannot be written to"); } - if (!self->data_) { + if (!self->tensor_) { throw jsi::JSError(rt, "setData: Tensor has been disposed"); } @@ -154,7 +191,7 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) throw jsi::JSError(rt, errorMsg); } - std::memcpy(self->data_.get(), buffer.data(rt) + byteOffset, byteLength); + std::memcpy(self->data(), buffer.data(rt) + byteOffset, byteLength); return jsi::Value(rt, thisVal.asObject(rt)); }; @@ -202,7 +239,7 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) throw jsi::JSError(rt, "getData: Tensor is currently in use and cannot be read"); } - if (!self->data_) { + if (!self->tensor_) { throw jsi::JSError(rt, "getData: Tensor has been disposed"); } @@ -213,7 +250,7 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) throw jsi::JSError(rt, errorMsg); } - std::memcpy(buffer.data(rt) + byteOffset, self->data_.get(), byteLength); + std::memcpy(buffer.data(rt) + byteOffset, self->data(), byteLength); return jsi::Value(rt, args[0].asObject(rt)); }; @@ -286,18 +323,74 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) std::unique_lock lock(self->mutex_); - if (!self->data_) { + if (!self->tensor_) { throw jsi::JSError(rt, "dispose: Tensor has already been disposed"); } self->tensor_.reset(); - self->data_.reset(); + + if (self->data_) { + self->data_.reset(); + } + + // Unbind the parent tensor (for tensor views) + if (self->parent_) { + self->parent_.reset(); + } return jsi::Value::undefined(); }; return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "dispose"), 0, fnBody); } + if (nameStr == "view") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime &rt, const jsi::Value &thisVal, const jsi::Value *args, size_t count) -> jsi::Value { + if (count < 1 || count > 2) { + throw jsi::JSError(rt, "view: Usage: view(shape, offset?)"); + } + + if (!self->tensor_) { + throw jsi::JSError(rt, "view: Tensor has been disposed"); + } + + if (!args[0].isObject() || !args[0].asObject(rt).isArray(rt)) { + throw jsi::JSError(rt, "view: Expected shape as an array of integers"); + } + + auto shapeArray = args[0].asObject(rt).asArray(rt); + std::vector shape; + for (size_t i = 0; i < shapeArray.length(rt); ++i) { + auto dimValue = shapeArray.getValueAtIndex(rt, i); + if (!dimValue.isNumber()) { + throw jsi::JSError(rt, "view: Shape array must contain only numbers"); + } + + if (dimValue.asNumber() <= 0) { + throw jsi::JSError(rt, "view: Shape dimensions must be positive integers"); + } + + shape.push_back(static_cast(dimValue.asNumber())); + } + + size_t offset = 0; + if (count == 2) { + if (!args[1].isNumber()) { + throw jsi::JSError(rt, "view: Expected offset to be a number"); + } + offset = static_cast(args[1].asNumber()); + } + + try { + auto viewTensor = std::make_shared(self, shape, offset); + return jsi::Object::createFromHostObject(rt, viewTensor); + } catch (const std::exception &e) { + throw jsi::JSError(rt, "view: " + std::string(e.what())); + } + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "view"), 2, fnBody); + } + return jsi::Value::undefined(); } @@ -312,6 +405,7 @@ std::vector TensorHostObject::getPropertyNames(jsi::R properties.push_back(jsi::PropNameID::forAscii(rt, "through")); properties.push_back(jsi::PropNameID::forAscii(rt, "throughIf")); properties.push_back(jsi::PropNameID::forAscii(rt, "dispose")); + properties.push_back(jsi::PropNameID::forAscii(rt, "view")); return properties; } diff --git a/packages/react-native-executorch/cpp/core/tensor.h b/packages/react-native-executorch/cpp/core/tensor.h index 27c9af1199..1390c7e58b 100644 --- a/packages/react-native-executorch/cpp/core/tensor.h +++ b/packages/react-native-executorch/cpp/core/tensor.h @@ -33,8 +33,21 @@ class TensorHostObject : public facebook::jsi::HostObject, public std::enable_sh std::shared_mutex mutex_; + // Parent tensor connection - for tensor views. + std::shared_ptr parent_; // null for owner tensors + size_t offset_ = 0; + + // Helper function to cover data access in tensor / view situation. + uint8_t *data() const { return data_ ? data_.get() : parent_->data() + offset_; } + + // Owner tensor initializagtion TensorHostObject(const std::vector &shape, rnexecutorch::core::types::DType dtype); + // Tensor view initialization + // Does not allocate the data, only points out for a parent tensor. + TensorHostObject(std::shared_ptr parent, + const std::vector &shape, size_t offset); + facebook::jsi::Value get(facebook::jsi::Runtime &rt, const facebook::jsi::PropNameID &name) override; std::vector getPropertyNames(facebook::jsi::Runtime &rt) override; }; diff --git a/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp b/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp index 13bf66cd81..ca1342eeb0 100644 --- a/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp +++ b/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp @@ -494,7 +494,7 @@ void install_toChannelsLast(jsi::Runtime &rt, jsi::Object &module) { channels.emplace_back(srcH, srcW, cvDepth, srcPtr + i * hw * elemSize); } - ::cv::Mat dstMat(dstH, dstW, CV_MAKETYPE(cvDepth, dstC), dst->data_.get()); + ::cv::Mat dstMat(dstH, dstW, CV_MAKETYPE(cvDepth, dstC), dst->data()); ::cv::merge(channels, dstMat); return jsi::Value(rt, args[1]); diff --git a/packages/react-native-executorch/cpp/extensions/math/operations.cpp b/packages/react-native-executorch/cpp/extensions/math/operations.cpp index 4607af49b9..123a13887b 100644 --- a/packages/react-native-executorch/cpp/extensions/math/operations.cpp +++ b/packages/react-native-executorch/cpp/extensions/math/operations.cpp @@ -64,8 +64,8 @@ void install_sigmoid(jsi::Runtime &rt, jsi::Object &module) { } const auto countElements = src->numel_; - const auto *srcData = reinterpret_cast(src->data_.get()); - auto *dstData = reinterpret_cast(dst->data_.get()); + const auto *srcData = reinterpret_cast(src->data()); + auto *dstData = reinterpret_cast(dst->data()); for (size_t i = 0; i < countElements; ++i) { dstData[i] = 1.0f / (1.0f + std::exp(-srcData[i])); @@ -150,8 +150,8 @@ void install_softmax(jsi::Runtime &rt, jsi::Object &module) { throw jsi::JSError(rt, "softmax: dst tensor has been disposed"); } - const auto *srcData = reinterpret_cast(src->data_.get()); - auto *dstData = reinterpret_cast(dst->data_.get()); + const auto *srcData = reinterpret_cast(src->data()); + auto *dstData = reinterpret_cast(dst->data()); const auto axisDim = static_cast(src->shape_[axisIdx]); if (axisDim == 0) { diff --git a/packages/react-native-executorch/src/core/tensor.ts b/packages/react-native-executorch/src/core/tensor.ts index ca12609188..d1b159e5a5 100644 --- a/packages/react-native-executorch/src/core/tensor.ts +++ b/packages/react-native-executorch/src/core/tensor.ts @@ -90,6 +90,18 @@ export type Tensor = { ...args: Args ): Tensor; + /** + * Creates a view over a sub-region of this tensor's buffer without copying. + * + * The view shares the parent's memory, so writes through either tensor are + * visible to the other. The view must be {@link Tensor.dispose disposed} + * independently, which only releases the view — not the parent buffer. + * @param shape The logical shape of the view (dimensions <= parent). + * @param offset Byte offset into the parent buffer (defaults to 0). + * @returns A new tensor view sharing the parent's memory. + */ + view(shape: readonly number[], offset?: number): Tensor; + /** * Prevents plain JS objects from being cast as Tensors. Tensors should only * be created via the `tensor` function exported from this module. @@ -114,7 +126,7 @@ export type Tensor = { */ export function tensor( dtype: DType, - shape: number[], + shape: readonly number[], src?: Float32Array | Uint8Array | Int32Array | BigInt64Array ): Tensor { 'worklet'; diff --git a/packages/react-native-executorch/third-party/common/phonemis b/packages/react-native-executorch/third-party/common/phonemis index a72ef9092c..71eb1ce33b 160000 --- a/packages/react-native-executorch/third-party/common/phonemis +++ b/packages/react-native-executorch/third-party/common/phonemis @@ -1 +1 @@ -Subproject commit a72ef9092c80a86d963b590b4aa0a4e813270bc3 +Subproject commit 71eb1ce33bd586d38cbac037843b8539d7829c3b diff --git a/packages/react-native-executorch/third-party/scripts/vendor_headers.sh b/packages/react-native-executorch/third-party/scripts/vendor_headers.sh new file mode 100755 index 0000000000..1c4697d4c8 --- /dev/null +++ b/packages/react-native-executorch/third-party/scripts/vendor_headers.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# 1. Define paths +# Adjust these if your folder structure is different +SOURCE_DIR="third-party/executorch" +DEST_DIR="third-party/include" + +# 2. Clean up old headers to ensure a fresh sync +echo "Cleaning $DEST_DIR..." +rm -rf "$DEST_DIR" +mkdir -p "$DEST_DIR/executorch" + +echo "Syncing headers from $SOURCE_DIR..." + +# 3. Copy ExecuTorch core headers +# We use rsync with filters to only grab .h and .hpp files while keeping the structure +rsync -amv --include='*/' --include='*.h' --include='*.hpp' --exclude='*' \ + "$SOURCE_DIR/runtime" \ + "$SOURCE_DIR/extension" \ + "$SOURCE_DIR/util" \ + "$DEST_DIR/executorch/" + +# 4. Copy c10 utility headers (often nested in pytorch submodule) +# ExecuTorch depends on a subset of c10 +PYTORCH_CORE_DIR="$SOURCE_DIR/third-party/pytorch" +if [ -d "$PYTORCH_CORE_DIR/c10" ]; then + echo "Found c10 headers in PyTorch submodule, copying..." + mkdir -p "$DEST_DIR/c10" + rsync -amv --include='*/' --include='*.h' --include='*.hpp' --exclude='*' \ + "$PYTORCH_CORE_DIR/c10" \ + "$DEST_DIR/" +fi + +echo "✅ Header vendoring complete!" +echo "Headers are now in the '$DEST_DIR' folder." \ No newline at end of file diff --git a/packages/react-native-executorch/third-party/scripts/vendor_opencv.sh b/packages/react-native-executorch/third-party/scripts/vendor_opencv.sh new file mode 100755 index 0000000000..0e2bcc7c2f --- /dev/null +++ b/packages/react-native-executorch/third-party/scripts/vendor_opencv.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Exit on error +set -e + +# Print usage instructions if the correct arguments are not passed +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo "Example: $0 ~/Downloads/OpenCV-android-sdk ~/Downloads/opencv2.xcframework" + exit 1 +fi + +ANDROID_SDK_DIR="$1" +IOS_FRAMEWORK="$2" + +DEST_INCLUDE="third-party/include" +DEST_ANDROID="third-party/android/jniLibs" +DEST_IOS="third-party/ios/Frameworks" + +# 1. Copy Android / Core C++ headers to third-party/include/opencv2 +if [ -d "$ANDROID_SDK_DIR/sdk/native/jni/include/opencv2" ]; then + echo "Syncing OpenCV header files to $DEST_INCLUDE..." + mkdir -p "$DEST_INCLUDE" + rm -rf "$DEST_INCLUDE/opencv2" + cp -R "$ANDROID_SDK_DIR/sdk/native/jni/include/opencv2" "$DEST_INCLUDE/" +else + echo "Error: Could not find opencv2 headers in '$ANDROID_SDK_DIR/sdk/native/jni/include/'" + exit 1 +fi + +# 2. Copy Android shared libraries to third-party/android/jniLibs +if [ -d "$ANDROID_SDK_DIR/sdk/native/libs" ]; then + echo "Syncing Android OpenCV precompiled libraries to $DEST_ANDROID..." + mkdir -p "$DEST_ANDROID" + + # Sync arm64-v8a (Physical arm64 Android devices) + if [ -d "$ANDROID_SDK_DIR/sdk/native/libs/arm64-v8a" ]; then + mkdir -p "$DEST_ANDROID/arm64-v8a" + cp "$ANDROID_SDK_DIR/sdk/native/libs/arm64-v8a/libopencv_java4.so" "$DEST_ANDROID/arm64-v8a/" + echo " - Added arm64-v8a library" + fi + + # Sync x86_64 (Intel Android emulators) + if [ -d "$ANDROID_SDK_DIR/sdk/native/libs/x86_64" ]; then + mkdir -p "$DEST_ANDROID/x86_64" + cp "$ANDROID_SDK_DIR/sdk/native/libs/x86_64/libopencv_java4.so" "$DEST_ANDROID/x86_64/" + echo " - Added x86_64 library" + fi + + # Sync armeabi-v7a (Older arm Android devices) + if [ -d "$ANDROID_SDK_DIR/sdk/native/libs/armeabi-v7a" ]; then + mkdir -p "$DEST_ANDROID/armeabi-v7a" + cp "$ANDROID_SDK_DIR/sdk/native/libs/armeabi-v7a/libopencv_java4.so" "$DEST_ANDROID/armeabi-v7a/" + echo " - Added armeabi-v7a library" + fi +else + echo "Error: Could not find libraries folder in '$ANDROID_SDK_DIR/sdk/native/libs'" + exit 1 +fi + +# 3. Copy iOS framework to third-party/ios/Frameworks +if [ -d "$IOS_FRAMEWORK" ]; then + FRAMEWORK_NAME=$(basename "$IOS_FRAMEWORK") + echo "Syncing iOS framework ($FRAMEWORK_NAME) to $DEST_IOS..." + mkdir -p "$DEST_IOS" + rm -rf "$DEST_IOS/$FRAMEWORK_NAME" + cp -R "$IOS_FRAMEWORK" "$DEST_IOS/" + echo " - Copied $FRAMEWORK_NAME" +else + echo "Error: Could not find iOS framework directory at '$IOS_FRAMEWORK'" + exit 1 +fi + +echo "✅ OpenCV headers and libraries successfully vendored!" From d7f82625ca3ba3d2d06adba56b093b9157ff0cab Mon Sep 17 00:00:00 2001 From: IgorSwat Date: Thu, 25 Jun 2026 14:06:30 +0200 Subject: [PATCH 3/8] Reorganize cpp core & add explicit tensor view --- .agents/skills/add-native-extension/SKILL.md | 4 +- .../cpp/core/dtype.cpp | 84 ++--- .../react-native-executorch/cpp/core/dtype.h | 35 +- .../cpp/core/model.cpp | 14 +- .../cpp/core/tensor.cpp | 330 +++++++++++------- .../react-native-executorch/cpp/core/tensor.h | 57 ++- .../cpp/core/tensor_view.cpp | 37 ++ .../cpp/core/tensor_view.h | 45 +++ .../react-native-executorch/cpp/core/types.h | 17 + .../react-native-executorch/cpp/core/value.h | 33 ++ .../cpp/extensions/cv/box_ops.cpp | 8 +- .../cpp/extensions/cv/image_ops.cpp | 33 +- .../cpp/extensions/math/operations.cpp | 24 +- .../src/core/tensor.ts | 4 +- 14 files changed, 462 insertions(+), 263 deletions(-) create mode 100644 packages/react-native-executorch/cpp/core/tensor_view.cpp create mode 100644 packages/react-native-executorch/cpp/core/tensor_view.h create mode 100644 packages/react-native-executorch/cpp/core/types.h create mode 100644 packages/react-native-executorch/cpp/core/value.h diff --git a/.agents/skills/add-native-extension/SKILL.md b/.agents/skills/add-native-extension/SKILL.md index bb83842b2e..63b88969e9 100644 --- a/.agents/skills/add-native-extension/SKILL.md +++ b/.agents/skills/add-native-extension/SKILL.md @@ -120,8 +120,8 @@ namespace rnexecutorch::extensions:: } // 6. Perform the computation - const float *srcData = reinterpret_cast(src->data_.get()); - float *dstData = reinterpret_cast(dst->data_.get()); + const float *srcData = reinterpret_cast(src->data_); + float *dstData = reinterpret_cast(dst->data_); size_t size = src->size(); for (size_t i = 0; i < size; ++i) diff --git a/packages/react-native-executorch/cpp/core/dtype.cpp b/packages/react-native-executorch/cpp/core/dtype.cpp index b183daf0d6..c9e35dbd2e 100644 --- a/packages/react-native-executorch/cpp/core/dtype.cpp +++ b/packages/react-native-executorch/cpp/core/dtype.cpp @@ -1,38 +1,44 @@ #include "dtype.h" #include -namespace rnexecutorch::core::types { -DType parseDType(const std::string &s) { +namespace rnexecutorch::core { + +DType::DType(const std::string &s) { if (s == "uint8") { - return DType::uint8; - } - if (s == "int32") { - return DType::int32; - } - if (s == "int64") { - return DType::int64; + v_ = DType::uint8; + } else if (s == "int32") { + v_ = DType::int32; + } else if (s == "int64") { + v_ = DType::int64; + } else if (s == "float32") { + v_ = DType::float32; + } else { + throw std::invalid_argument( + "Unsupported dtype: '" + s + "'. Expected 'uint8', 'int32', 'int64', or 'float32'"); } - if (s == "float32") { - return DType::float32; - } - throw std::invalid_argument("Unsupported dtype: '" + s + "'. Expected 'uint8', 'int32', 'int64', or 'float32'"); } -std::string toString(DType dtype) { - switch (dtype) { - case DType::uint8: - return "uint8"; - case DType::int32: - return "int32"; - case DType::int64: - return "int64"; - case DType::float32: - return "float32"; +DType::DType(executorch::aten::ScalarType st) { + switch (st) { + case executorch::aten::ScalarType::Byte: + v_ = DType::uint8; + break; + case executorch::aten::ScalarType::Int: + v_ = DType::int32; + break; + case executorch::aten::ScalarType::Long: + v_ = DType::int64; + break; + case executorch::aten::ScalarType::Float: + v_ = DType::float32; + break; + default: + throw std::invalid_argument("Unsupported ScalarType"); } } -executorch::aten::ScalarType toScalarType(DType dtype) { - switch (dtype) { +DType::operator executorch::aten::ScalarType() const { + switch (v_) { case DType::uint8: return executorch::aten::ScalarType::Byte; case DType::int32: @@ -44,23 +50,21 @@ executorch::aten::ScalarType toScalarType(DType dtype) { } } -DType fromScalarType(executorch::aten::ScalarType st) { - switch (st) { - case executorch::aten::ScalarType::Byte: - return DType::uint8; - case executorch::aten::ScalarType::Int: - return DType::int32; - case executorch::aten::ScalarType::Long: - return DType::int64; - case executorch::aten::ScalarType::Float: - return DType::float32; - default: - throw std::invalid_argument("Unsupported ScalarType"); +DType::operator std::string() const { + switch (v_) { + case DType::uint8: + return "uint8"; + case DType::int32: + return "int32"; + case DType::int64: + return "int64"; + case DType::float32: + return "float32"; } } -size_t elementSize(DType dtype) { - switch (dtype) { +size_t DType::size() const { + switch (v_) { case DType::uint8: return 1; // NOLINTNEXTLINE(bugprone-branch-clone): int32 and float32 are both 4 bytes; the identical branches are intentional. @@ -73,4 +77,4 @@ size_t elementSize(DType dtype) { } } -} // namespace rnexecutorch::core::types +} // namespace rnexecutorch::core diff --git a/packages/react-native-executorch/cpp/core/dtype.h b/packages/react-native-executorch/cpp/core/dtype.h index fafa0a8429..5c8b63b2f5 100644 --- a/packages/react-native-executorch/cpp/core/dtype.h +++ b/packages/react-native-executorch/cpp/core/dtype.h @@ -4,20 +4,29 @@ #include #include -namespace rnexecutorch::core::types { -enum class DType { - uint8, - int32, - int64, - float32 -}; +namespace rnexecutorch::core { + +class DType { +public: + enum Value : uint8_t { uint8, + int32, + int64, + float32 }; -DType parseDType(const std::string &s); -std::string toString(DType dtype); + DType(Value v) : v_(v) {} // Direct initialization + DType(executorch::aten::ScalarType st); // Initialization from corresponding ET type + DType(const std::string &s); // Initialization from name (for JSI applications) -executorch::aten::ScalarType toScalarType(DType dtype); -DType fromScalarType(executorch::aten::ScalarType st); + // Returns the size (in bytes) of the type. + size_t size() const; -size_t elementSize(DType dtype); + // Hidden operator-like conversions provide a good layer of abstraction. + operator Value() const { return v_; } + explicit operator executorch::aten::ScalarType() const; + operator std::string() const; + +private: + Value v_; +}; -} // namespace rnexecutorch::core::types +} // namespace rnexecutorch::core diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index c9ce16a80d..d845f7989d 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -123,7 +123,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { jsTensorMeta.setProperty(rt, "nbytes", static_cast(tensorMeta.nbytes())); try { - const std::string dtypeStr = rnexecutorch::core::types::toString(rnexecutorch::core::types::fromScalarType(tensorMeta.scalar_type())); + std::string dtypeStr = DType(tensorMeta.scalar_type()); jsTensorMeta.setProperty(rt, "dtype", jsi::String::createFromUtf8(rt, dtypeStr)); } catch (const std::exception &) { jsTensorMeta.setProperty(rt, "dtype", jsi::String::createFromUtf8(rt, "not supported")); @@ -222,7 +222,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { const TensorHostObject *tensorHostObject, const executorch::runtime::Result &tensorMeta, const std::string &identifier) { - if (tensorMeta->scalar_type() != tensorHostObject->tensor_->dtype()) { + if (tensorMeta->scalar_type() != tensorHostObject->et_tensor_->dtype()) { throw jsi::JSError(rt, "execute: Tensor dtype mismatch for " + identifier); } @@ -232,7 +232,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { " but got " + std::to_string(tensorHostObject->shape_.size())); } - auto ndim = tensorHostObject->tensor_->sizes().size(); + auto ndim = tensorHostObject->et_tensor_->sizes().size(); for (size_t j = 0; j < ndim; ++j) { if (tensorMeta->sizes()[j] != tensorHostObject->shape_[j]) { throw jsi::JSError(rt, "execute: Tensor shape mismatch for " + identifier + @@ -273,7 +273,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { } auto tensorHostObject = val.asObject(rt).getHostObject(rt); - if (!tensorHostObject->tensor_) { + if (!tensorHostObject->et_tensor_) { throw jsi::JSError(rt, "execute: inputs[" + std::to_string(i) + "] has been disposed"); } @@ -297,7 +297,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { validateTensor(rt, tensorHostObject.get(), tensorMeta, "inputs[" + std::to_string(i) + "]"); - inputs[i] = tensorHostObject->tensor_; + inputs[i] = tensorHostObject->et_tensor_; break; } case executorch::runtime::Tag::Double: { @@ -374,7 +374,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { } auto tensorHostObject = val.asObject(rt).getHostObject(rt); - if (!tensorHostObject->tensor_) { + if (!tensorHostObject->et_tensor_) { throw jsi::JSError(rt, "execute: outputTensors[" + std::to_string(tensorOutputIdx) + "] has been disposed"); } @@ -399,7 +399,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { validateTensor(rt, tensorHostObject.get(), tensorMeta, "outputTensors[" + std::to_string(tensorOutputIdx) + "]"); - std::memcpy(tensorHostObject->data(), + std::memcpy(tensorHostObject->data_, output.toTensor().const_data_ptr(), output.toTensor().nbytes()); diff --git a/packages/react-native-executorch/cpp/core/tensor.cpp b/packages/react-native-executorch/cpp/core/tensor.cpp index 3f38e10558..496dcec823 100644 --- a/packages/react-native-executorch/cpp/core/tensor.cpp +++ b/packages/react-native-executorch/cpp/core/tensor.cpp @@ -1,60 +1,24 @@ #include "tensor.h" #include -#include +#include namespace rnexecutorch::core::tensor { namespace jsi = facebook::jsi; -TensorHostObject::TensorHostObject(const std::vector &shape, rnexecutorch::core::types::DType dtype) - : dtype_(dtype), - shape_(shape), - numel_(std::accumulate(shape.begin(), shape.end(), static_cast(1), std::multiplies<>())) { - const auto elemSize = rnexecutorch::core::types::elementSize(dtype); - - size_ = numel_ * elemSize; - // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays): owning runtime-sized byte buffer - data_ = std::make_unique(size_); - tensor_ = executorch::extension::from_blob(data_.get(), shape_, rnexecutorch::core::types::toScalarType(dtype)); +TensorHostObject::TensorHostObject(Shape shape, DType dtype) + : TensorView(nullptr, dtype, std::move(shape)) { + storage_ = std::make_unique(size_); + data_ = storage_.get(); + et_tensor_ = executorch::extension::from_blob(data_, shape_, static_cast(dtype_)); } -TensorHostObject::TensorHostObject(std::shared_ptr parent, - const std::vector &shape, size_t offset) { - if (!parent) { - throw std::invalid_argument("Tensor view: parent must not be null"); - } - - if (shape.empty()) { - throw std::invalid_argument("Tensor view: shape must not be empty"); - } - - if (!parent->tensor_) { - throw std::invalid_argument("Tensor view: parent tensor has been disposed"); - } - - dtype_ = parent->dtype_; - shape_ = shape; - numel_ = std::accumulate(shape.begin(), shape.end(), size_t(1), std::multiplies()); - - const auto elemSize = rnexecutorch::core::types::elementSize(dtype_); - size_ = numel_ * elemSize; - - if (offset + size_ > parent->size_) { - throw std::invalid_argument( - "Tensor view: view size (" + std::to_string(size_) + " bytes) at offset " + - std::to_string(offset) + " exceeds parent buffer (" + - std::to_string(parent->size_) + " bytes)"); - } - - parent_ = std::move(parent); - offset_ = offset; - - tensor_ = executorch::extension::from_blob( - parent_->data() + offset_, - shape_, - rnexecutorch::core::types::toScalarType(dtype_)); +TensorHostObject::TensorHostObject(uint8_t *data, Shape shape, DType dtype) + : TensorView(data, dtype, std::move(shape)), storage_(nullptr) { + et_tensor_ = executorch::extension::from_blob(data_, shape_, static_cast(dtype_)); } -jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { +jsi::Value TensorHostObject::get(jsi::Runtime &rt, + const jsi::PropNameID &name) { auto nameStr = name.utf8(rt); if (nameStr == "shape") { @@ -66,7 +30,7 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) } if (nameStr == "dtype") { - return jsi::String::createFromUtf8(rt, rnexecutorch::core::types::toString(dtype_)); + return jsi::String::createFromUtf8(rt, dtype_); } if (nameStr == "numel") { @@ -80,32 +44,42 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) throw jsi::JSError(rt, "copyTo: Usage: copyTo(dst, options?)"); } - if (!args[0].isObject() || !args[0].asObject(rt).isHostObject(rt)) { + if (!args[0].isObject() || + !args[0].asObject(rt).isHostObject(rt)) { throw jsi::JSError(rt, "copyTo: Expected dst to be a Tensor"); } auto dst = args[0].asObject(rt).getHostObject(rt); if (self.get() == dst.get()) { - throw jsi::JSError(rt, "copyTo: In-place operations (src == dst) are not supported."); + throw jsi::JSError( + rt, + "copyTo: In-place operations (src == dst) are not " + "supported."); } - std::shared_lock srcLock(self->mutex_, std::try_to_lock); + std::shared_lock srcLock(self->mutex_, + std::try_to_lock); if (!srcLock.owns_lock()) { - throw jsi::JSError(rt, "copyTo: src tensor is currently in use"); + throw jsi::JSError(rt, + "copyTo: src tensor is currently in use"); } - std::unique_lock dstLock(dst->mutex_, std::try_to_lock); + std::unique_lock dstLock(dst->mutex_, + std::try_to_lock); if (!dstLock.owns_lock()) { - throw jsi::JSError(rt, "copyTo: dst tensor is currently in use"); + throw jsi::JSError(rt, + "copyTo: dst tensor is currently in use"); } - if (!self->tensor_) { - throw jsi::JSError(rt, "copyTo: src tensor has been disposed"); + if (!self->data_) { + throw jsi::JSError(rt, + "copyTo: src tensor has been disposed"); } - if (!dst->tensor_) { - throw jsi::JSError(rt, "copyTo: dst tensor has been disposed"); + if (!dst->data_) { + throw jsi::JSError(rt, + "copyTo: dst tensor has been disposed"); } size_t srcOffset = 0; @@ -132,27 +106,35 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) throw jsi::JSError(rt, "copyTo: size mismatch between copy byte size and dst tensor size"); } - std::memcpy(dst->data_.get(), self->data_.get() + (srcOffset * elemSize), copyLen * elemSize); + std::memcpy(dst->data_, self->data_ + (srcOffset * elemSize), copyLen * elemSize); return jsi::Value(rt, args[0].asObject(rt)); }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "copyTo"), 1, fnBody); + return jsi::Function::createFromHostFunction( + rt, jsi::PropNameID::forAscii(rt, "copyTo"), 1, fnBody); } if (nameStr == "setData") { auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime &rt, const jsi::Value &thisVal, const jsi::Value *args, size_t count) -> jsi::Value { + auto fnBody = [self]( + jsi::Runtime &rt, const jsi::Value &thisVal, + const jsi::Value *args, + size_t count) -> jsi::Value { if (count != 1) { throw jsi::JSError(rt, "setData: Usage: setData(array)"); } if (!args[0].isObject()) { - throw jsi::JSError(rt, "setData: Expected array to be an object (TypedArray)"); + throw jsi::JSError( + rt, + "setData: Expected array to be an object (TypedArray)"); } const jsi::Object dataObj = args[0].asObject(rt); if (!dataObj.hasProperty(rt, "buffer")) { - throw jsi::JSError(rt, "setData: Expected a TypedArray with a 'buffer' property"); + throw jsi::JSError( + rt, + "setData: Expected a TypedArray with a 'buffer' property"); } const jsi::ArrayBuffer buffer = dataObj.getProperty(rt, "buffer").asObject(rt).getArrayBuffer(rt); @@ -162,25 +144,35 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) if (dataObj.hasProperty(rt, "byteOffset")) { auto byteOffsetValue = dataObj.getProperty(rt, "byteOffset"); if (!byteOffsetValue.isNumber()) { - throw jsi::JSError(rt, "setData: Expected 'byteOffset' to be a number"); + throw jsi::JSError( + rt, + "setData: Expected 'byteOffset' to be a number"); } - byteOffset = static_cast(byteOffsetValue.asNumber()); + byteOffset = + static_cast(byteOffsetValue.asNumber()); } if (dataObj.hasProperty(rt, "byteLength")) { auto byteLengthValue = dataObj.getProperty(rt, "byteLength"); if (!byteLengthValue.isNumber()) { - throw jsi::JSError(rt, "setData: Expected 'byteLength' to be a number"); + throw jsi::JSError( + rt, + "setData: Expected 'byteLength' to be a number"); } - byteLength = static_cast(byteLengthValue.asNumber()); + byteLength = + static_cast(byteLengthValue.asNumber()); } - std::unique_lock lock(self->mutex_, std::try_to_lock); + std::unique_lock lock(self->mutex_, + std::try_to_lock); if (!lock.owns_lock()) { - throw jsi::JSError(rt, "setData: Tensor is currently in use and cannot be written to"); + throw jsi::JSError( + rt, + "setData: Tensor is currently in use and cannot be " + "written to"); } - if (!self->tensor_) { + if (!self->data_) { throw jsi::JSError(rt, "setData: Tensor has been disposed"); } @@ -191,11 +183,13 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) throw jsi::JSError(rt, errorMsg); } - std::memcpy(self->data(), buffer.data(rt) + byteOffset, byteLength); + std::memcpy(self->data_, buffer.data(rt) + byteOffset, + byteLength); return jsi::Value(rt, thisVal.asObject(rt)); }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "setData"), 1, fnBody); + return jsi::Function::createFromHostFunction( + rt, jsi::PropNameID::forAscii(rt, "setData"), 1, fnBody); } if (nameStr == "getData") { @@ -206,12 +200,16 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) } if (!args[0].isObject()) { - throw jsi::JSError(rt, "getData: Expected array to be an object (TypedArray)"); + throw jsi::JSError( + rt, + "getData: Expected array to be an object (TypedArray)"); } const jsi::Object dataObj = args[0].asObject(rt); if (!dataObj.hasProperty(rt, "buffer")) { - throw jsi::JSError(rt, "getData: Expected a TypedArray with a 'buffer' property"); + throw jsi::JSError( + rt, + "getData: Expected a TypedArray with a 'buffer' property"); } const jsi::ArrayBuffer buffer = dataObj.getProperty(rt, "buffer").asObject(rt).getArrayBuffer(rt); @@ -221,25 +219,34 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) if (dataObj.hasProperty(rt, "byteOffset")) { auto byteOffsetValue = dataObj.getProperty(rt, "byteOffset"); if (!byteOffsetValue.isNumber()) { - throw jsi::JSError(rt, "getData: Expected 'byteOffset' to be a number"); + throw jsi::JSError( + rt, + "getData: Expected 'byteOffset' to be a number"); } - byteOffset = static_cast(byteOffsetValue.asNumber()); + byteOffset = + static_cast(byteOffsetValue.asNumber()); } if (dataObj.hasProperty(rt, "byteLength")) { auto byteLengthValue = dataObj.getProperty(rt, "byteLength"); if (!byteLengthValue.isNumber()) { - throw jsi::JSError(rt, "getData: Expected 'byteLength' to be a number"); + throw jsi::JSError( + rt, + "getData: Expected 'byteLength' to be a number"); } - byteLength = static_cast(byteLengthValue.asNumber()); + byteLength = + static_cast(byteLengthValue.asNumber()); } - std::shared_lock lock(self->mutex_, std::try_to_lock); + std::shared_lock lock(self->mutex_, + std::try_to_lock); if (!lock.owns_lock()) { - throw jsi::JSError(rt, "getData: Tensor is currently in use and cannot be read"); + throw jsi::JSError( + rt, + "getData: Tensor is currently in use and cannot be read"); } - if (!self->tensor_) { + if (!self->data_) { throw jsi::JSError(rt, "getData: Tensor has been disposed"); } @@ -250,22 +257,30 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) throw jsi::JSError(rt, errorMsg); } - std::memcpy(buffer.data(rt) + byteOffset, self->data(), byteLength); + std::memcpy(buffer.data(rt) + byteOffset, self->data_, + byteLength); return jsi::Value(rt, args[0].asObject(rt)); }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "getData"), 1, fnBody); + return jsi::Function::createFromHostFunction( + rt, jsi::PropNameID::forAscii(rt, "getData"), 1, fnBody); } if (nameStr == "through") { auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime &rt, const jsi::Value &thisVal, const jsi::Value *args, size_t count) -> jsi::Value { + auto fnBody = [self]( + jsi::Runtime &rt, const jsi::Value &thisVal, + const jsi::Value *args, + size_t count) -> jsi::Value { if (count < 1) { - throw jsi::JSError(rt, "through: Usage: through(fn, ...args)"); + throw jsi::JSError(rt, + "through: Usage: through(fn, ...args)"); } - if (!args[0].isObject() || !args[0].asObject(rt).isFunction(rt)) { - throw jsi::JSError(rt, "through: First argument must be a function"); + if (!args[0].isObject() || + !args[0].asObject(rt).isFunction(rt)) { + throw jsi::JSError( + rt, "through: First argument must be a function"); } auto fn = args[0].asObject(rt).asFunction(rt); @@ -277,17 +292,25 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) fnArgs.emplace_back(rt, args[i]); } - return fn.call(rt, static_cast(fnArgs.data()), fnArgs.size()); + return fn.call(rt, + static_cast(fnArgs.data()), + fnArgs.size()); }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "through"), 1, fnBody); + return jsi::Function::createFromHostFunction( + rt, jsi::PropNameID::forAscii(rt, "through"), 1, fnBody); } if (nameStr == "throughIf") { auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime &rt, const jsi::Value &thisVal, const jsi::Value *args, size_t count) -> jsi::Value { + auto fnBody = [self]( + jsi::Runtime &rt, const jsi::Value &thisVal, + const jsi::Value *args, + size_t count) -> jsi::Value { if (count < 2) { - throw jsi::JSError(rt, "throughIf: Usage: throughIf(pred, fn, ...args)"); + throw jsi::JSError( + rt, + "throughIf: Usage: throughIf(pred, fn, ...args)"); } const bool pred = args[0].asBool(); @@ -295,8 +318,10 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) return jsi::Value(rt, thisVal); } - if (!args[1].isObject() || !args[1].asObject(rt).isFunction(rt)) { - throw jsi::JSError(rt, "throughIf: Second argument must be a function"); + if (!args[1].isObject() || + !args[1].asObject(rt).isFunction(rt)) { + throw jsi::JSError( + rt, "throughIf: Second argument must be a function"); } auto fn = args[1].asObject(rt).asFunction(rt); @@ -308,10 +333,13 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) fnArgs.emplace_back(rt, args[i]); } - return fn.call(rt, static_cast(fnArgs.data()), fnArgs.size()); + return fn.call(rt, + static_cast(fnArgs.data()), + fnArgs.size()); }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "throughIf"), 2, fnBody); + return jsi::Function::createFromHostFunction( + rt, jsi::PropNameID::forAscii(rt, "throughIf"), 2, fnBody); } if (nameStr == "dispose") { @@ -323,79 +351,96 @@ jsi::Value TensorHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) std::unique_lock lock(self->mutex_); - if (!self->tensor_) { - throw jsi::JSError(rt, "dispose: Tensor has already been disposed"); + if (!self->data_) { + throw jsi::JSError( + rt, "dispose: Tensor has already been disposed"); } - self->tensor_.reset(); - - if (self->data_) { - self->data_.reset(); + if (self->storage_) { + self->storage_.reset(); } - // Unbind the parent tensor (for tensor views) - if (self->parent_) { - self->parent_.reset(); - } + self->data_ = nullptr; + self->et_tensor_.reset(); return jsi::Value::undefined(); }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "dispose"), 0, fnBody); + return jsi::Function::createFromHostFunction( + rt, jsi::PropNameID::forAscii(rt, "dispose"), 0, fnBody); } if (nameStr == "view") { auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime &rt, const jsi::Value &thisVal, const jsi::Value *args, size_t count) -> jsi::Value { + auto fnBody = [self]( + jsi::Runtime &rt, const jsi::Value &thisVal, + const jsi::Value *args, + size_t count) -> jsi::Value { if (count < 1 || count > 2) { throw jsi::JSError(rt, "view: Usage: view(shape, offset?)"); } - if (!self->tensor_) { - throw jsi::JSError(rt, "view: Tensor has been disposed"); + if (!self->data_) { + throw jsi::JSError(rt, + "view: Tensor has been disposed"); } - if (!args[0].isObject() || !args[0].asObject(rt).isArray(rt)) { - throw jsi::JSError(rt, "view: Expected shape as an array of integers"); + if (!args[0].isObject() || + !args[0].asObject(rt).isArray(rt)) { + throw jsi::JSError( + rt, "view: Expected shape as an array of integers"); } auto shapeArray = args[0].asObject(rt).asArray(rt); - std::vector shape; + Shape shape; for (size_t i = 0; i < shapeArray.length(rt); ++i) { auto dimValue = shapeArray.getValueAtIndex(rt, i); if (!dimValue.isNumber()) { - throw jsi::JSError(rt, "view: Shape array must contain only numbers"); + throw jsi::JSError( + rt, + "view: Shape array must contain only numbers"); } if (dimValue.asNumber() <= 0) { - throw jsi::JSError(rt, "view: Shape dimensions must be positive integers"); + throw jsi::JSError( + rt, + "view: Shape dimensions must be positive integers"); } - shape.push_back(static_cast(dimValue.asNumber())); + shape.push_back( + static_cast(dimValue.asNumber())); } size_t offset = 0; if (count == 2) { if (!args[1].isNumber()) { - throw jsi::JSError(rt, "view: Expected offset to be a number"); + throw jsi::JSError(rt, + "view: Expected offset to be a " + "number"); } offset = static_cast(args[1].asNumber()); } try { - auto viewTensor = std::make_shared(self, shape, offset); - return jsi::Object::createFromHostObject(rt, viewTensor); + auto viewHost = std::make_shared( + self->data_ + offset, + std::move(shape), + self->dtype_); + return jsi::Object::createFromHostObject(rt, viewHost); } catch (const std::exception &e) { - throw jsi::JSError(rt, "view: " + std::string(e.what())); + throw jsi::JSError(rt, + "view: " + std::string(e.what())); } }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "view"), 2, fnBody); + return jsi::Function::createFromHostFunction( + rt, jsi::PropNameID::forAscii(rt, "view"), 2, fnBody); } return jsi::Value::undefined(); } -std::vector TensorHostObject::getPropertyNames(jsi::Runtime &rt) { - std::vector properties; +std::vector TensorHostObject::getPropertyNames( + jsi::Runtime &rt) { + std::vector properties; properties.push_back(jsi::PropNameID::forAscii(rt, "shape")); properties.push_back(jsi::PropNameID::forAscii(rt, "dtype")); properties.push_back(jsi::PropNameID::forAscii(rt, "numel")); @@ -413,42 +458,59 @@ void install_createTensor(jsi::Runtime &rt, jsi::Object &module) { const auto *name = "createTensor"; auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { if (count != 2) { - throw jsi::JSError(rt, "createTensor: Usage: createTensor(shape, dtype)"); + throw jsi::JSError(rt, + "createTensor: Usage: createTensor(shape, " + "dtype)"); } - if (!args[0].isObject() || !args[0].asObject(rt).isArray(rt)) { - throw jsi::JSError(rt, "createTensor: Expected shape as an array of integers"); + if (!args[0].isObject() || + !args[0].asObject(rt).isArray(rt)) { + throw jsi::JSError( + rt, "createTensor: Expected shape as an array of integers"); } if (!args[1].isString()) { - throw jsi::JSError(rt, "createTensor: Expected dtype as a string"); + throw jsi::JSError( + rt, "createTensor: Expected dtype as a string"); } auto shapeArray = args[0].asObject(rt).asArray(rt); - std::vector shape; + Shape shape; for (size_t i = 0; i < shapeArray.length(rt); ++i) { auto dimValue = shapeArray.getValueAtIndex(rt, i); if (!dimValue.isNumber()) { - throw jsi::JSError(rt, "createTensor: Shape array must contain only numbers"); + throw jsi::JSError( + rt, + "createTensor: Shape array must contain only numbers"); } if (dimValue.asNumber() <= 0) { - throw jsi::JSError(rt, "createTensor: Shape dimensions must be positive integers"); + throw jsi::JSError( + rt, + "createTensor: Shape dimensions must be positive " + "integers"); } - shape.push_back(static_cast(dimValue.asNumber())); + shape.push_back( + static_cast(dimValue.asNumber())); } try { - const auto dtype = rnexecutorch::core::types::parseDType(args[1].asString(rt).utf8(rt)); - auto tensorHostObject = std::make_shared(shape, dtype); + DType dtype(args[1].asString(rt).utf8(rt)); + auto tensorHostObject = + std::make_shared(std::move(shape), dtype); return jsi::Object::createFromHostObject(rt, tensorHostObject); } catch (const std::exception &e) { - throw jsi::JSError(rt, "createTensor: Error creating tensor: " + std::string(e.what())); + throw jsi::JSError( + rt, + "createTensor: Error creating tensor: " + + std::string(e.what())); } }; - auto fn = jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 2, fnBody); + auto fn = jsi::Function::createFromHostFunction( + rt, jsi::PropNameID::forAscii(rt, name), 2, fnBody); module.setProperty(rt, name, fn); } + } // namespace rnexecutorch::core::tensor diff --git a/packages/react-native-executorch/cpp/core/tensor.h b/packages/react-native-executorch/cpp/core/tensor.h index 1390c7e58b..61603247b9 100644 --- a/packages/react-native-executorch/cpp/core/tensor.h +++ b/packages/react-native-executorch/cpp/core/tensor.h @@ -1,56 +1,47 @@ #pragma once -#include #include #include #include -#include #include #include -#include -#include "dtype.h" +#include "tensor_view.h" +#include "types.h" namespace rnexecutorch::core::tensor { + /** - * JSI HostObject wrapping an ExecuTorch TensorPtr instance. - * - * Exposes methods to JavaScript for copying data, accessing properties (shape, - * dtype, numel), writing data from array buffers, reading data to array - * buffers, and disposing of underlying memory. + * A JSI HostObject that wraps an owning or non-owning tensor view. */ -class TensorHostObject : public facebook::jsi::HostObject, public std::enable_shared_from_this { +class TensorHostObject : public facebook::jsi::HostObject, + public TensorView, + public std::enable_shared_from_this { public: - rnexecutorch::core::types::DType dtype_; - std::vector shape_; - size_t numel_; - - size_t size_; - std::unique_ptr data_; // NOLINT(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays): owning runtime-sized byte buffer - executorch::extension::TensorPtr tensor_; - - std::shared_mutex mutex_; + // Owning tensor allocation. + TensorHostObject(Shape shape, DType dtype); - // Parent tensor connection - for tensor views. - std::shared_ptr parent_; // null for owner tensors - size_t offset_ = 0; + // Non-owning tensor (view) wrapping external data. + TensorHostObject(uint8_t *data, Shape shape, DType dtype); - // Helper function to cover data access in tensor / view situation. - uint8_t *data() const { return data_ ? data_.get() : parent_->data() + offset_; } + // JSI bridge methods. + facebook::jsi::Value get(facebook::jsi::Runtime &rt, + const facebook::jsi::PropNameID &name) override; + std::vector getPropertyNames( + facebook::jsi::Runtime &rt) override; - // Owner tensor initializagtion - TensorHostObject(const std::vector &shape, rnexecutorch::core::types::DType dtype); + // Owned data storage — null for views. + std::unique_ptr storage_; - // Tensor view initialization - // Does not allocate the data, only points out for a parent tensor. - TensorHostObject(std::shared_ptr parent, - const std::vector &shape, size_t offset); + // ExecuTorch entry point. + executorch::extension::TensorPtr et_tensor_; - facebook::jsi::Value get(facebook::jsi::Runtime &rt, const facebook::jsi::PropNameID &name) override; - std::vector getPropertyNames(facebook::jsi::Runtime &rt) override; + std::shared_mutex mutex_; }; -void install_createTensor(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +void install_createTensor(facebook::jsi::Runtime &rt, + facebook::jsi::Object &module); + } // namespace rnexecutorch::core::tensor diff --git a/packages/react-native-executorch/cpp/core/tensor_view.cpp b/packages/react-native-executorch/cpp/core/tensor_view.cpp new file mode 100644 index 0000000000..a90e9ce119 --- /dev/null +++ b/packages/react-native-executorch/cpp/core/tensor_view.cpp @@ -0,0 +1,37 @@ +#include "tensor_view.h" +#include +#include +#include + +namespace rnexecutorch::core::tensor { + +TensorView::TensorView(uint8_t *data, DType dtype, Shape shape) + : dtype_(dtype), shape_(std::move(shape)), data_(data) { + if (shape_.empty()) { + throw std::invalid_argument("TensorView: shape must not be empty"); + } + + numel_ = std::accumulate(shape_.begin(), shape_.end(), size_t(1), std::multiplies()); + size_ = numel_ * dtype_.size(); +} + +Value TensorView::operator[](std::initializer_list indices) { + size_t flat = flatten(indices); + return Value(data_ + flat * dtype_.size(), dtype_); +} + +size_t TensorView::flatten(std::initializer_list indices) const { + assert(indices.size() == shape_.size()); + + size_t flat = 0; + auto shape_it = shape_.begin(); + for (auto idx : indices) { + assert(idx < static_cast(*shape_it)); + flat = flat * static_cast(*shape_it) + idx; + ++shape_it; + } + + return flat; +} + +} // namespace rnexecutorch::core::tensor diff --git a/packages/react-native-executorch/cpp/core/tensor_view.h b/packages/react-native-executorch/cpp/core/tensor_view.h new file mode 100644 index 0000000000..4c4647cfe5 --- /dev/null +++ b/packages/react-native-executorch/cpp/core/tensor_view.h @@ -0,0 +1,45 @@ +#pragma once + +#include + +#include "dtype.h" +#include "types.h" +#include "value.h" + +namespace rnexecutorch::core::tensor { + +class TensorView { +public: + TensorView(uint8_t *data, DType dtype, Shape shape); + + virtual ~TensorView() {} + + /** + * Python-like indexing. Returns a Value proxy for reading or writing the + * element at the given multi-dimensional indices. The caller's declared type + * selects the correct element width; a runtime guard asserts it matches the + * tensor's dtype. + * + * Sample usage: + * @code + * TensorView tensor(buffer, DType::int32, {1, 10}); + * tensor[{0, 5}] = 42; // write + * int32_t token = tensor[{0, 7}]; // read + * @endcode + */ + Value operator[](std::initializer_list indices); + + // Metadata + DType dtype_; + Shape shape_; + size_t numel_; // Number of elements (values) in a tensor + size_t size_; // Size of a tensor (numel * dtype.size()) + + // Data (pointer - non-owning) + uint8_t *data_ = nullptr; + +private: + size_t flatten(std::initializer_list indices) const; +}; + +} // namespace rnexecutorch::core::tensor diff --git a/packages/react-native-executorch/cpp/core/types.h b/packages/react-native-executorch/cpp/core/types.h new file mode 100644 index 0000000000..6cb03635b1 --- /dev/null +++ b/packages/react-native-executorch/cpp/core/types.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +namespace rnexecutorch::core { + +// Dimension index must be strictly non-negative. +using Dimension = uint32_t; + +// Defines number of elements along a specific dimension (in a tensor). +using DSize = int32_t; + +// Shape = sequence of sizes (one per dimension). +using Shape = std::vector; + +} // namespace rnexecutorch::core diff --git a/packages/react-native-executorch/cpp/core/value.h b/packages/react-native-executorch/cpp/core/value.h new file mode 100644 index 0000000000..292522e5ff --- /dev/null +++ b/packages/react-native-executorch/cpp/core/value.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include "dtype.h" + +namespace rnexecutorch::core::tensor { + +// An abstract wrapper representing a single tensor element. +// Designed to work as a flexible runtime multi-type, limited only by the available dtypes. +class Value { +public: + Value(uint8_t *ptr, DType dtype) : ptr_(ptr), dtype_(dtype) {} + + template + Value &operator=(const T &val) { + assert(sizeof(T) == dtype_.size()); // runtime guard + *reinterpret_cast(ptr_) = val; + return *this; + } + + template + operator T() const { + assert(sizeof(T) == dtype_.size()); // runtime guard + return *reinterpret_cast(ptr_); + } + +private: + uint8_t *ptr_; + DType dtype_; +}; + +} // namespace rnexecutorch::core::tensor diff --git a/packages/react-native-executorch/cpp/extensions/cv/box_ops.cpp b/packages/react-native-executorch/cpp/extensions/cv/box_ops.cpp index 501b9245c1..c78549fff4 100644 --- a/packages/react-native-executorch/cpp/extensions/cv/box_ops.cpp +++ b/packages/react-native-executorch/cpp/extensions/cv/box_ops.cpp @@ -142,8 +142,8 @@ void install_nms(jsi::Runtime &rt, jsi::Object &module) { throw jsi::JSError(rt, "nms: boxes and scores must have dtype float32"); } - const auto *boxesPtr = reinterpret_cast(boxes->data_.get()); - const auto *scoresPtr = reinterpret_cast(scores->data_.get()); + const auto *boxesPtr = reinterpret_cast(boxes->data_); + const auto *scoresPtr = reinterpret_cast(scores->data_); std::vector> candidates; candidates.reserve(static_cast(numAnchors)); @@ -342,8 +342,8 @@ void install_restrictToBox(jsi::Runtime &rt, jsi::Object &module) { throw jsi::JSError(rt, "restrictToBox: " + std::string(e.what())); } - ::cv::Mat srcMat(H, W, cvType, src->data_.get()); - ::cv::Mat dstMat(H, W, cvType, dst->data_.get()); + ::cv::Mat srcMat(H, W, cvType, src->data_); + ::cv::Mat dstMat(H, W, cvType, dst->data_); dstMat.setTo(::cv::Scalar::all(0)); if (!isEmpty) { diff --git a/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp b/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp index ca1342eeb0..a5c362e429 100644 --- a/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp +++ b/packages/react-native-executorch/cpp/extensions/cv/image_ops.cpp @@ -15,6 +15,7 @@ namespace rnexecutorch::extensions::cv::image_ops { namespace jsi = facebook::jsi; using TensorHostObject = rnexecutorch::core::tensor::TensorHostObject; +using DType = rnexecutorch::core::DType; namespace { int interpToFlag(const std::string &interp) { @@ -146,8 +147,8 @@ void install_resize(jsi::Runtime &rt, jsi::Object &module) { throw jsi::JSError(rt, "resize: " + std::string(e.what())); } - const ::cv::Mat srcMat(srcH, srcW, cvType, src->data_.get()); - ::cv::Mat dstMat(dstH, dstW, cvType, dst->data_.get()); + const ::cv::Mat srcMat(srcH, srcW, cvType, src->data_); + ::cv::Mat dstMat(dstH, dstW, cvType, dst->data_); if (mode == "stretch") { ::cv::resize(srcMat, dstMat, dstMat.size(), 0, 0, interpFlag); @@ -313,8 +314,8 @@ void install_cvtColor(jsi::Runtime &rt, jsi::Object &module) { cvDstType = CV_MAKETYPE(dtypeToCvDepth(dst->dtype_), dstC); flag = codeToColorConversionFlag(code); - const ::cv::Mat srcMat(srcH, srcW, cvSrcType, src->data_.get()); - ::cv::Mat dstMat(srcH, srcW, cvDstType, dst->data_.get()); + const ::cv::Mat srcMat(srcH, srcW, cvSrcType, src->data_); + ::cv::Mat dstMat(srcH, srcW, cvDstType, dst->data_); ::cv::cvtColor(srcMat, dstMat, flag); } catch (const std::invalid_argument &e) { @@ -397,13 +398,13 @@ void install_toChannelsFirst(jsi::Runtime &rt, jsi::Object &module) { throw jsi::JSError(rt, "toChannelsFirst: " + std::string(e.what())); } - const ::cv::Mat srcMat(srcH, srcW, cvType, src->data_.get()); + const ::cv::Mat srcMat(srcH, srcW, cvType, src->data_); std::vector<::cv::Mat> channels; ::cv::split(srcMat, channels); const size_t hw = static_cast(srcH) * static_cast(srcW); - const size_t elemSize = rnexecutorch::core::types::elementSize(src->dtype_); - uint8_t *dstPtr = dst->data_.get(); + const size_t elemSize =src->dtype_.size(); + uint8_t *dstPtr = dst->data_; for (size_t i = 0; std::cmp_less(i, srcC); ++i) { std::memcpy(dstPtr + i * hw * elemSize, channels[i].data, hw * elemSize); @@ -486,15 +487,15 @@ void install_toChannelsLast(jsi::Runtime &rt, jsi::Object &module) { } const size_t hw = static_cast(srcH) * static_cast(srcW); - const size_t elemSize = rnexecutorch::core::types::elementSize(src->dtype_); - uint8_t *srcPtr = src->data_.get(); + const size_t elemSize = src->dtype_.type(); + uint8_t *srcPtr = src->data_; std::vector<::cv::Mat> channels; for (size_t i = 0; std::cmp_less(i, srcC); ++i) { channels.emplace_back(srcH, srcW, cvDepth, srcPtr + i * hw * elemSize); } - ::cv::Mat dstMat(dstH, dstW, CV_MAKETYPE(cvDepth, dstC), dst->data()); + ::cv::Mat dstMat(dstH, dstW, CV_MAKETYPE(cvDepth, dstC), dst->data_); ::cv::merge(channels, dstMat); return jsi::Value(rt, args[1]); @@ -607,10 +608,10 @@ void install_normalize(jsi::Runtime &rt, jsi::Object &module) { throw jsi::JSError(rt, "normalize: " + std::string(e.what())); } - const size_t srcElemSize = rnexecutorch::core::types::elementSize(src->dtype_); - const size_t dstElemSize = rnexecutorch::core::types::elementSize(dst->dtype_); - uint8_t *srcPtr = src->data_.get(); - uint8_t *dstPtr = dst->data_.get(); + const size_t srcElemSize = src->dtype_.size(); + const size_t dstElemSize = dst->dtype_.size(); + uint8_t *srcPtr = src->data_; + uint8_t *dstPtr = dst->data_; const size_t plane = static_cast(h) * static_cast(w); for (size_t ch = 0; std::cmp_less(ch, c); ++ch) { @@ -695,8 +696,8 @@ void install_applyColormap(jsi::Runtime &rt, jsi::Object &module) { const size_t pixels = src->numel_; - const auto *srcData = reinterpret_cast(src->data_.get()); - uint8_t *dstData = dst->data_.get(); + const auto *srcData = reinterpret_cast(src->data_); + uint8_t *dstData = dst->data_; for (size_t i = 0; i < pixels; ++i) { const int32_t idx = srcData[i]; diff --git a/packages/react-native-executorch/cpp/extensions/math/operations.cpp b/packages/react-native-executorch/cpp/extensions/math/operations.cpp index 123a13887b..b824ba1580 100644 --- a/packages/react-native-executorch/cpp/extensions/math/operations.cpp +++ b/packages/react-native-executorch/cpp/extensions/math/operations.cpp @@ -41,7 +41,7 @@ void install_sigmoid(jsi::Runtime &rt, jsi::Object &module) { throw jsi::JSError(rt, "sigmoid: src and dst must have the same dtype"); } - if (src->dtype_ != rnexecutorch::core::types::DType::float32) { + if (src->dtype_ != rnexecutorch::core::DType::float32) { throw jsi::JSError(rt, "sigmoid: only float32 tensors are supported"); } @@ -64,8 +64,8 @@ void install_sigmoid(jsi::Runtime &rt, jsi::Object &module) { } const auto countElements = src->numel_; - const auto *srcData = reinterpret_cast(src->data()); - auto *dstData = reinterpret_cast(dst->data()); + const auto *srcData = reinterpret_cast(src->data_); + auto *dstData = reinterpret_cast(dst->data_); for (size_t i = 0; i < countElements; ++i) { dstData[i] = 1.0f / (1.0f + std::exp(-srcData[i])); @@ -107,7 +107,7 @@ void install_softmax(jsi::Runtime &rt, jsi::Object &module) { throw jsi::JSError(rt, "softmax: src and dst must have the same dtype"); } - if (src->dtype_ != rnexecutorch::core::types::DType::float32) { + if (src->dtype_ != rnexecutorch::core::DType::float32) { throw jsi::JSError(rt, "softmax: only float32 tensors are supported"); } @@ -150,8 +150,8 @@ void install_softmax(jsi::Runtime &rt, jsi::Object &module) { throw jsi::JSError(rt, "softmax: dst tensor has been disposed"); } - const auto *srcData = reinterpret_cast(src->data()); - auto *dstData = reinterpret_cast(dst->data()); + const auto *srcData = reinterpret_cast(src->data_); + auto *dstData = reinterpret_cast(dst->data_); const auto axisDim = static_cast(src->shape_[axisIdx]); if (axisDim == 0) { @@ -218,11 +218,11 @@ void install_argmax(jsi::Runtime &rt, jsi::Object &module) { throw jsi::JSError(rt, "argmax: In-place operations (src == dst) are not supported."); } - if (src->dtype_ != rnexecutorch::core::types::DType::float32) { + if (src->dtype_ != rnexecutorch::core::DType::float32) { throw jsi::JSError(rt, "argmax: src must be float32"); } - if (dst->dtype_ != rnexecutorch::core::types::DType::int32) { + if (dst->dtype_ != rnexecutorch::core::DType::int32) { throw jsi::JSError(rt, "argmax: dst must be int32"); } @@ -263,7 +263,7 @@ void install_argmax(jsi::Runtime &rt, jsi::Object &module) { throw jsi::JSError(rt, "argmax: dst tensor has been disposed"); } - const auto *srcData = reinterpret_cast(src->data_.get()); + const auto *srcData = reinterpret_cast(src->data_); const auto axisDim = static_cast(src->shape_[axisIdx]); if (axisDim == 0) { @@ -279,7 +279,7 @@ void install_argmax(jsi::Runtime &rt, jsi::Object &module) { inner *= static_cast(src->shape_[i]); } - auto *dstData = reinterpret_cast(dst->data_.get()); + auto *dstData = reinterpret_cast(dst->data_); // DO NOT swap loop order. This structure intentionally prioritizes the // most common case (axis = -1, inner = 1) for sequential access. @@ -356,8 +356,8 @@ void install_threshold(jsi::Runtime &rt, jsi::Object &module) { throw jsi::JSError(rt, "threshold: dst tensor has been disposed"); } - const auto *srcData = reinterpret_cast(src->data_.get()); - auto *dstData = reinterpret_cast(dst->data_.get()); + const auto *srcData = reinterpret_cast(src->data_); + auto *dstData = reinterpret_cast(dst->data_); for (size_t i = 0; i < src->numel_; ++i) { dstData[i] = (srcData[i] >= thresholdVal) ? 1.0f : 0.0f; diff --git a/packages/react-native-executorch/src/core/tensor.ts b/packages/react-native-executorch/src/core/tensor.ts index d1b159e5a5..20200ca0da 100644 --- a/packages/react-native-executorch/src/core/tensor.ts +++ b/packages/react-native-executorch/src/core/tensor.ts @@ -94,8 +94,8 @@ export type Tensor = { * Creates a view over a sub-region of this tensor's buffer without copying. * * The view shares the parent's memory, so writes through either tensor are - * visible to the other. The view must be {@link Tensor.dispose disposed} - * independently, which only releases the view — not the parent buffer. + * visible to the other. The view does not own its memory — dispose the + * parent tensor when no views are needed. * @param shape The logical shape of the view (dimensions <= parent). * @param offset Byte offset into the parent buffer (defaults to 0). * @returns A new tensor view sharing the parent's memory. From 7ccc574bf21e6da64eb7740dca94ddbdf42eeb13 Mon Sep 17 00:00:00 2001 From: IgorSwat Date: Mon, 29 Jun 2026 11:17:35 +0200 Subject: [PATCH 4/8] First, buggy implementation --- apps/speech/app.json | 45 ++++ apps/speech/app/_layout.tsx | 32 +++ apps/speech/app/index.tsx | 49 +++++ apps/speech/app/text-to-speech/index.tsx | 150 +++++++++++++ apps/speech/assets/icons/adaptive-icon.png | Bin 0 -> 17547 bytes apps/speech/assets/icons/executorch.svg | 9 + apps/speech/assets/icons/favicon.png | Bin 0 -> 1466 bytes apps/speech/assets/icons/icon.png | Bin 0 -> 22380 bytes apps/speech/assets/icons/splash.png | Bin 0 -> 47346 bytes apps/speech/babel.config.js | 7 + apps/speech/components/Button.tsx | 102 +++++++++ apps/speech/components/LatencyIndicator.tsx | 34 +++ apps/speech/components/ModelPicker.tsx | 98 +++++++++ apps/speech/components/ModelStatus.tsx | 69 ++++++ apps/speech/components/ScreenWrapper.tsx | 8 + apps/speech/index.ts | 5 + apps/speech/metro.config.js | 8 + apps/speech/package.json | 39 ++++ apps/speech/theme.ts | 76 +++++++ apps/speech/tsconfig.json | 16 ++ .../android/CMakeLists.txt | 8 + .../cpp/RnExecutorch.cpp | 1 + .../cpp/core/tensor.cpp | 3 + .../react-native-executorch/cpp/core/tensor.h | 3 +- .../cpp/extensions/speech/audio_ops.cpp | 101 +++++++++ .../cpp/extensions/speech/audio_ops.h | 8 + .../cpp/extensions/speech/install.cpp | 36 ++++ .../cpp/extensions/speech/install.h | 7 + .../cpp/extensions/speech/kokoro/duration.cpp | 193 +++++++++++++++++ .../cpp/extensions/speech/kokoro/duration.h | 11 + .../extensions/speech/kokoro/partitioner.cpp | 190 ++++++++++++++++ .../extensions/speech/kokoro/partitioner.h | 36 ++++ .../extensions/speech/kokoro/tokenizer.cpp | 76 +++++++ .../cpp/extensions/speech/kokoro/tokenizer.h | 8 + .../cpp/extensions/speech/kokoro/voice.cpp | 93 ++++++++ .../cpp/extensions/speech/kokoro/voice.h | 7 + .../cpp/extensions/speech/phonemizer.cpp | 113 ++++++++++ .../cpp/extensions/speech/phonemizer.h | 29 +++ .../src/extensions/speech/ops/audio.ts | 16 ++ .../extensions/speech/ops/kokoro/duration.ts | 42 ++++ .../extensions/speech/ops/kokoro/partition.ts | 35 +++ .../extensions/speech/ops/kokoro/tokenize.ts | 14 ++ .../src/extensions/speech/ops/kokoro/voice.ts | 14 ++ .../extensions/speech/tasks/textToSpeech.ts | 204 ++++++++++++++++++ .../src/extensions/speech/utils/phonemizer.ts | 45 ++++ .../src/hooks/useTextToSpeech.ts | 107 +++++++++ packages/react-native-executorch/src/index.ts | 2 + .../react-native-executorch/src/models.ts | 192 +++++++++++++++++ yarn.lock | 27 +++ 49 files changed, 2367 insertions(+), 1 deletion(-) create mode 100644 apps/speech/app.json create mode 100644 apps/speech/app/_layout.tsx create mode 100644 apps/speech/app/index.tsx create mode 100644 apps/speech/app/text-to-speech/index.tsx create mode 100644 apps/speech/assets/icons/adaptive-icon.png create mode 100644 apps/speech/assets/icons/executorch.svg create mode 100644 apps/speech/assets/icons/favicon.png create mode 100644 apps/speech/assets/icons/icon.png create mode 100644 apps/speech/assets/icons/splash.png create mode 100644 apps/speech/babel.config.js create mode 100644 apps/speech/components/Button.tsx create mode 100644 apps/speech/components/LatencyIndicator.tsx create mode 100644 apps/speech/components/ModelPicker.tsx create mode 100644 apps/speech/components/ModelStatus.tsx create mode 100644 apps/speech/components/ScreenWrapper.tsx create mode 100644 apps/speech/index.ts create mode 100644 apps/speech/metro.config.js create mode 100644 apps/speech/package.json create mode 100644 apps/speech/theme.ts create mode 100644 apps/speech/tsconfig.json create mode 100644 packages/react-native-executorch/cpp/extensions/speech/audio_ops.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/speech/audio_ops.h create mode 100644 packages/react-native-executorch/cpp/extensions/speech/install.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/speech/install.h create mode 100644 packages/react-native-executorch/cpp/extensions/speech/kokoro/duration.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/speech/kokoro/duration.h create mode 100644 packages/react-native-executorch/cpp/extensions/speech/kokoro/partitioner.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/speech/kokoro/partitioner.h create mode 100644 packages/react-native-executorch/cpp/extensions/speech/kokoro/tokenizer.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/speech/kokoro/tokenizer.h create mode 100644 packages/react-native-executorch/cpp/extensions/speech/kokoro/voice.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/speech/kokoro/voice.h create mode 100644 packages/react-native-executorch/cpp/extensions/speech/phonemizer.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/speech/phonemizer.h create mode 100644 packages/react-native-executorch/src/extensions/speech/ops/audio.ts create mode 100644 packages/react-native-executorch/src/extensions/speech/ops/kokoro/duration.ts create mode 100644 packages/react-native-executorch/src/extensions/speech/ops/kokoro/partition.ts create mode 100644 packages/react-native-executorch/src/extensions/speech/ops/kokoro/tokenize.ts create mode 100644 packages/react-native-executorch/src/extensions/speech/ops/kokoro/voice.ts create mode 100644 packages/react-native-executorch/src/extensions/speech/tasks/textToSpeech.ts create mode 100644 packages/react-native-executorch/src/extensions/speech/utils/phonemizer.ts create mode 100644 packages/react-native-executorch/src/hooks/useTextToSpeech.ts diff --git a/apps/speech/app.json b/apps/speech/app.json new file mode 100644 index 0000000000..002040822f --- /dev/null +++ b/apps/speech/app.json @@ -0,0 +1,45 @@ +{ + "expo": { + "name": "speech", + "slug": "speech", + "version": "1.0.0", + "orientation": "portrait", + "icon": "./assets/icons/icon.png", + "userInterfaceStyle": "light", + "newArchEnabled": true, + "scheme": "rne-speech", + "splash": { + "image": "./assets/icons/splash.png", + "resizeMode": "contain", + "backgroundColor": "#ffffff" + }, + "ios": { + "supportsTablet": true, + "bundleIdentifier": "com.anonymous.speech" + }, + "android": { + "adaptiveIcon": { + "foregroundImage": "./assets/icons/adaptive-icon.png", + "backgroundColor": "#ffffff" + }, + "package": "com.anonymous.speech" + }, + "web": { + "favicon": "./assets/icons/favicon.png" + }, + "plugins": [ + "expo-router", + [ + "expo-build-properties", + { + "android": { + "minSdkVersion": 26 + }, + "ios": { + "deploymentTarget": "17.0" + } + } + ] + ] + } +} diff --git a/apps/speech/app/_layout.tsx b/apps/speech/app/_layout.tsx new file mode 100644 index 0000000000..0788f41329 --- /dev/null +++ b/apps/speech/app/_layout.tsx @@ -0,0 +1,32 @@ +import { Drawer } from 'expo-router/drawer'; +import { ColorPalette } from '../theme'; +import React from 'react'; + +export default function Layout() { + return ( + + null, + title: 'Main Menu', + drawerItemStyle: { display: 'none' }, + }} + /> + + + ); +} diff --git a/apps/speech/app/index.tsx b/apps/speech/app/index.tsx new file mode 100644 index 0000000000..7aa06e823f --- /dev/null +++ b/apps/speech/app/index.tsx @@ -0,0 +1,49 @@ +import { useRouter } from 'expo-router'; +import { View, Text, StyleSheet, TouchableOpacity } from 'react-native'; +import { ColorPalette } from '../theme'; + +export default function Home() { + const router = useRouter(); + + return ( + + Select a demo model + + router.navigate('text-to-speech/')}> + Text to Speech + + + + ); +} + +const styles = StyleSheet.create({ + container: { + flex: 1, + justifyContent: 'center', + alignItems: 'center', + backgroundColor: '#fff', + }, + headerText: { + fontSize: 18, + color: ColorPalette.strongPrimary, + margin: 20, + }, + buttonContainer: { + width: '80%', + justifyContent: 'space-evenly', + marginBottom: 20, + }, + button: { + backgroundColor: ColorPalette.strongPrimary, + borderRadius: 8, + padding: 14, + alignItems: 'center', + marginBottom: 12, + }, + buttonText: { + color: 'white', + fontSize: 16, + fontWeight: '600', + }, +}); diff --git a/apps/speech/app/text-to-speech/index.tsx b/apps/speech/app/text-to-speech/index.tsx new file mode 100644 index 0000000000..a30b0476a9 --- /dev/null +++ b/apps/speech/app/text-to-speech/index.tsx @@ -0,0 +1,150 @@ +import React, { useState, useCallback } from 'react'; +import { View, Text, TextInput, StyleSheet, ScrollView } from 'react-native'; +import { commonStyles, theme, ColorPalette } from '../../theme'; +import { useTextToSpeech, models } from 'react-native-executorch'; +import ScreenWrapper from '../../components/ScreenWrapper'; +import { ModelPicker, type ModelOption } from '../../components/ModelPicker'; +import { ModelStatus } from '../../components/ModelStatus'; +import { LatencyIndicator } from '../../components/LatencyIndicator'; +import { Button } from '../../components/Button'; +import type { TextToSpeechModel } from 'react-native-executorch'; + +const VOICE_OPTIONS: ModelOption[] = [ + { label: 'AF Heart (en-us)', value: models.textToSpeech.AF_HEART }, + { label: 'AF River (en-us)', value: models.textToSpeech.AF_RIVER }, + { label: 'AF Sarah (en-us)', value: models.textToSpeech.AF_SARAH }, + { label: 'AM Adam (en-us)', value: models.textToSpeech.AM_ADAM }, + { label: 'AM Michael (en-us)', value: models.textToSpeech.AM_MICHAEL }, + { label: 'BF Emma (en-gb)', value: models.textToSpeech.BF_EMMA }, + { label: 'BM Daniel (en-gb)', value: models.textToSpeech.BM_DANIEL }, + { label: 'FF Siwis (fr)', value: models.textToSpeech.FF_SIWIS }, + { label: 'EF Dora (es)', value: models.textToSpeech.EF_DORA }, + { label: 'EM Alex (es)', value: models.textToSpeech.EM_ALEX }, + { label: 'IF Sara (it)', value: models.textToSpeech.IF_SARA }, + { label: 'IM Nicola (it)', value: models.textToSpeech.IM_NICOLA }, + { label: 'PF Dora (pt)', value: models.textToSpeech.PF_DORA }, + { label: 'PM Santa (pt)', value: models.textToSpeech.PM_SANTA }, + { label: 'HF Alpha (hi)', value: models.textToSpeech.HF_ALPHA }, + { label: 'HM Omega (hi)', value: models.textToSpeech.HM_OMEGA }, + { label: 'PM Mateusz (pl)', value: models.textToSpeech.PM_MATEUSZ }, + { label: 'DF Anna (de)', value: models.textToSpeech.DF_ANNA }, +]; + +function TextToSpeechContent() { + const [selectedVoice, setSelectedVoice] = useState(VOICE_OPTIONS[0].value); + const [text, setText] = useState(''); + const [isPlaying, setIsPlaying] = useState(false); + const [latency, setLatency] = useState(null); + const [error, setError] = useState(null); + + const { + isReady, + downloadProgress, + error: loadError, + streamWorklet, + } = useTextToSpeech(selectedVoice); + + const handleSpeak = useCallback(() => { + if (!streamWorklet || !text.trim()) return; + setIsPlaying(true); + setError(null); + const start = Date.now(); + + try { + streamWorklet({ + text: text.trim(), + speed: 1.0, + onBegin: () => { + setLatency(null); + }, + onNext: () => { + // Audio chunk delivered + }, + onEnd: () => { + setLatency(Date.now() - start); + setIsPlaying(false); + }, + }); + } catch (e: any) { + setError(e.message || String(e)); + setIsPlaying(false); + } + }, [streamWorklet, text]); + + const activeError = loadError ? String(loadError) : error; + + return ( + + + Enter text and select a voice to generate speech. + + + { + setSelectedVoice(model); + setError(null); + setLatency(null); + }} + /> + + + + + + +