From f215c0c7b876372865a42168bce7111bc3f97658 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Sat, 11 Jul 2026 01:58:47 -0700 Subject: [PATCH 01/17] add torch-tensorrt-executorch-delegate wheel --- .github/workflows/executorch-static-linux.yml | 11 +- .gitignore | 3 + MODULE.bazel | 6 +- examples/executorch_reference_runner/BUILD | 1 + .../executorch_reference_runner/README.md | 22 ++++ .../executorch_reference_runner/load_model.py | 61 ++++++++++ packaging/executorch_delegate/README.md | 38 ++++++ .../executorch_delegate/native/CMakeLists.txt | 68 +++++++++++ packaging/executorch_delegate/pyproject.toml | 7 ++ packaging/executorch_delegate/setup.py | 110 ++++++++++++++++++ .../__init__.py | 55 +++++++++ py/torch_tensorrt/executorch/__init__.py | 3 + py/torch_tensorrt/executorch/runtime.py | 58 +++++++++ setup.py | 16 ++- tests/py/executorch/test_python_runtime.py | 64 ++++++++++ 15 files changed, 513 insertions(+), 10 deletions(-) create mode 100644 examples/executorch_reference_runner/load_model.py create mode 100644 packaging/executorch_delegate/README.md create mode 100644 packaging/executorch_delegate/native/CMakeLists.txt create mode 100644 packaging/executorch_delegate/pyproject.toml create mode 100644 packaging/executorch_delegate/setup.py create mode 100644 packaging/executorch_delegate/torch_tensorrt_executorch_delegate/__init__.py create mode 100644 py/torch_tensorrt/executorch/runtime.py create mode 100644 tests/py/executorch/test_python_runtime.py diff --git a/.github/workflows/executorch-static-linux.yml b/.github/workflows/executorch-static-linux.yml index fe53fdb12c..d0e4bab2f5 100644 --- a/.github/workflows/executorch-static-linux.yml +++ b/.github/workflows/executorch-static-linux.yml @@ -68,6 +68,7 @@ jobs: build-matrix: ${{ needs.select-matrix.outputs.matrix }} pre-script: packaging/pre_build_script.sh fail-on-empty: false + upload-artifact: torch-tensorrt-executorch-delegate script: | set -euo pipefail BAZELISK_VERSION="1.26.0" @@ -96,5 +97,13 @@ jobs: export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}" # this is to verify the end user's workflow - python -m pip install pyyaml "executorch>=1.3.1" + python -m pip install pyyaml executorch + + # Build the no-compile-for-users Python runtime/delegate wheel. + export TORCH_TENSORRT_EXECUTORCH_DELEGATE_VERSION="$(python -c 'import torch_tensorrt; print(torch_tensorrt.__version__)')" + python -m pip wheel --no-build-isolation --no-deps --wheel-dir dist packaging/executorch_delegate + python -m pip install --no-deps --force-reinstall dist/torch_tensorrt_executorch_delegate-*.whl + python -c 'from torch_tensorrt_executorch_delegate import runtime; assert runtime().backend_registry.is_available("TensorRTBackend")' + python examples/torchtrt_executorch_example/export_static_shape.py --model_path="${RUNNER_TEMP}/torchtrt-python.pte" + python examples/executorch_reference_runner/load_model.py --model_path="${RUNNER_TEMP}/torchtrt-python.pte" --num_runs=1 .github/scripts/verify-executorch-reference-runner.sh diff --git a/.gitignore b/.gitignore index a9fb3b5ebf..477056ed82 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,6 @@ CLAUDE.md /compile_commands.json /external/ build-executorch/ +packaging/executorch_delegate/build/ +packaging/executorch_delegate/dist/ + diff --git a/MODULE.bazel b/MODULE.bazel index 2f384e8066..99c22600bc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -42,12 +42,12 @@ new_git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl" local_torch = use_repo_rule("//toolchains:local_torch.bzl", "local_torch") -# Pinned to the ExecuTorch release/1.3 branch head. +# Pinned to ExecuTorch main for PyTorch 2.14 native ABI compatibility. new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # latest commit in release/1.3 branch - commit = "6118688a095fd9697224f5cad72ce42db641c9cd", + # PyTorch 2.14-compatible ExecuTorch main commit + commit = "a6d812a082df57898b8608f56c867140cc9da32c", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;", diff --git a/examples/executorch_reference_runner/BUILD b/examples/executorch_reference_runner/BUILD index 4f53bc6b91..8335092725 100644 --- a/examples/executorch_reference_runner/BUILD +++ b/examples/executorch_reference_runner/BUILD @@ -7,6 +7,7 @@ filegroup( srcs = [ "CMakeLists.txt", "README.md", + "load_model.py", "main.cpp", ], ) diff --git a/examples/executorch_reference_runner/README.md b/examples/executorch_reference_runner/README.md index a518353bb6..171fd7c246 100644 --- a/examples/executorch_reference_runner/README.md +++ b/examples/executorch_reference_runner/README.md @@ -79,6 +79,28 @@ build-executorch-reference-runner/lib/libexecutorch_trt_backend.a ## Load And Run A `.pte` Model +### Python + +Install the complete prebuilt Python runtime and delegate: + +```bash +pip install "torch-tensorrt[executorch]" +``` + +Load and run the model without an ExecuTorch checkout or native build: + +```bash +python examples/executorch_reference_runner/load_model.py \ + --model_path=model.pte \ + --num_runs=1 +``` + +The extra installs `executorch` and the matching +`torch-tensorrt-executorch-delegate` wheel. That wheel contains an ExecuTorch +Python runtime with `TensorRTBackend` linked into its backend registry. + +### C++ + Run the reference runner against a Torch-TensorRT compiled ExecuTorch model: ```bash diff --git a/examples/executorch_reference_runner/load_model.py b/examples/executorch_reference_runner/load_model.py new file mode 100644 index 0000000000..7497744743 --- /dev/null +++ b/examples/executorch_reference_runner/load_model.py @@ -0,0 +1,61 @@ +"""Load and optionally run a Torch-TensorRT ExecuTorch ``.pte`` model. + +The default input matches ``export_static_shape.py``: one CUDA float tensor +with shape ``(2, 3, 4, 4)`` filled with ones. +""" + +import argparse +from pathlib import Path + +import torch +from torch_tensorrt.executorch.runtime import load + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--model_path", type=Path, default=Path("model.pte")) + parser.add_argument("--method", default="forward") + parser.add_argument("--num_runs", type=int, default=1) + parser.add_argument( + "--load_only", + action="store_true", + help="Parse the program and print its methods without loading/executing one", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if not args.model_path.is_file(): + raise FileNotFoundError(f"ExecuTorch model not found: {args.model_path}") + if args.num_runs < 1: + raise ValueError("--num_runs must be at least 1") + + program = load(args.model_path) + print(f"Loaded {args.model_path}") + print(f"Program methods: {sorted(program.method_names)}") + + if args.load_only: + return + if args.method not in program.method_names: + raise ValueError( + f"Method {args.method!r} is not in the program; " + f"available methods: {sorted(program.method_names)}" + ) + inputs = (torch.ones((2, 3, 4, 4), dtype=torch.float32),) + for run in range(args.num_runs): + outputs = program.run(inputs, args.method) + print(f"Run {run + 1} outputs:") + for index, output in enumerate(outputs): + if isinstance(output, torch.Tensor): + print( + f" output[{index}]: shape={tuple(output.shape)}, " + f"dtype={output.dtype}, device={output.device}, " + f"sample={output.flatten()[:8]}" + ) + else: + print(f" output[{index}]: {output!r}") + + +if __name__ == "__main__": + main() diff --git a/packaging/executorch_delegate/README.md b/packaging/executorch_delegate/README.md new file mode 100644 index 0000000000..b4c4e65b6d --- /dev/null +++ b/packaging/executorch_delegate/README.md @@ -0,0 +1,38 @@ +# Torch-TensorRT ExecuTorch Delegate Wheel + +This directory builds `torch-tensorrt-executorch-delegate`. The Linux wheel +contains an ExecuTorch `_portable_lib` Python runtime with `TensorRTBackend` +force-linked into the same native module that owns the backend registry. + +The wheel must use the same Python, PyTorch, ExecuTorch, CUDA, TensorRT, and +C++ ABI as its matching Torch-TensorRT wheel. + +## Build + +```bash +executorch_cmake_location="$(bazel query \ + @executorch//:executorch/CMakeLists.txt --output=location)" +export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" +export TensorRT_ROOT=/path/to/TensorRT + +python -m pip install executorch==1.3.1 +python -m pip wheel --no-build-isolation --no-deps \ + --wheel-dir dist packaging/executorch_delegate +``` + +The static ExecuTorch and delegate archives are intermediate build inputs; +users receive the final native Python module and do not compile anything. + +## Use + +```bash +pip install "torch-tensorrt[executorch]" +``` + +```python +import torch +from torch_tensorrt.executorch.runtime import load + +program = load("model.pte") +outputs = program.forward(torch.ones((2, 3, 4, 4))) +``` diff --git a/packaging/executorch_delegate/native/CMakeLists.txt b/packaging/executorch_delegate/native/CMakeLists.txt new file mode 100644 index 0000000000..1b61d3f767 --- /dev/null +++ b/packaging/executorch_delegate/native/CMakeLists.txt @@ -0,0 +1,68 @@ +cmake_minimum_required(VERSION 3.24) +project(torch_tensorrt_executorch_delegate LANGUAGES CXX) + +if(NOT EXECUTORCH_SOURCE_DIR OR NOT TORCH_TENSORRT_SOURCE_DIR) + message(FATAL_ERROR "EXECUTORCH_SOURCE_DIR and TORCH_TENSORRT_SOURCE_DIR are required") +endif() + +list(APPEND CMAKE_MODULE_PATH "${TORCH_TENSORRT_SOURCE_DIR}/cmake/Modules") +find_package(TensorRT REQUIRED) +find_package(CUDAToolkit REQUIRED) +find_package(Threads REQUIRED) + +set(BUILD_TESTING OFF CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_PYBIND ON CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_EXTENSION_MODULE ON CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER ON CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR ON CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP ON CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_EXTENSION_TENSOR ON CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_KERNELS_OPTIMIZED OFF CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_XNNPACK OFF CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +add_subdirectory("${EXECUTORCH_SOURCE_DIR}" executorch) + +add_library(torch_tensorrt_executorch_backend STATIC + "${TORCH_TENSORRT_SOURCE_DIR}/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp" + "${TORCH_TENSORRT_SOURCE_DIR}/cpp/src/torch_tensorrt/executorch/TensorRTBlobHeader.cpp") +target_compile_features(torch_tensorrt_executorch_backend PUBLIC cxx_std_17) +target_compile_definitions(torch_tensorrt_executorch_backend + PRIVATE C10_USING_CUSTOM_GENERATED_MACROS) +target_include_directories(torch_tensorrt_executorch_backend PRIVATE + "${TORCH_TENSORRT_SOURCE_DIR}" + "${TORCH_TENSORRT_SOURCE_DIR}/cpp/include" + "${EXECUTORCH_SOURCE_DIR}/.." + "${EXECUTORCH_SOURCE_DIR}/runtime/core/portable_type/c10") +target_link_libraries(torch_tensorrt_executorch_backend PRIVATE + CUDA::cudart TensorRT::nvinfer Threads::Threads executorch) + +if(NOT TARGET portable_lib) + message(FATAL_ERROR "ExecuTorch did not define portable_lib") +endif() + +# ExecuTorch portable c10 delegates selected headers to torch/headeronly. The +# pybind bridge must resolve those delegated headers from the exact PyTorch +# wheel it links, not ExecuTorch's vendored snapshot. +file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/torch_header_shim/torch") +file(CREATE_LINK + "${TORCH_PRIMARY_INCLUDE_DIR}/torch/headeronly" + "${CMAKE_BINARY_DIR}/torch_header_shim/torch/headeronly" + SYMBOLIC COPY_ON_ERROR) +target_include_directories(util BEFORE PRIVATE "${CMAKE_BINARY_DIR}/torch_header_shim") +target_include_directories(portable_lib BEFORE PRIVATE "${CMAKE_BINARY_DIR}/torch_header_shim") + +# TensorRTBackend registers through a static initializer. Force its complete +# archive into the same pybind module that owns ExecuTorch's backend registry. +target_link_libraries(portable_lib PRIVATE + "$" + extension_threadpool + CUDA::cudart TensorRT::nvinfer Threads::Threads) +set_target_properties(portable_lib PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${DELEGATE_LIBRARY_OUTPUT_DIRECTORY}" + OUTPUT_NAME "_portable_lib") +set_target_properties(data_loader PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${DELEGATE_LIBRARY_OUTPUT_DIRECTORY}") + +add_custom_target(torch_tensorrt_executorch_portable_lib DEPENDS portable_lib data_loader) diff --git a/packaging/executorch_delegate/pyproject.toml b/packaging/executorch_delegate/pyproject.toml new file mode 100644 index 0000000000..5142ec285d --- /dev/null +++ b/packaging/executorch_delegate/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = [ + "setuptools>=68", + "wheel>=0.40", + "torch", +] +build-backend = "setuptools.build_meta" diff --git a/packaging/executorch_delegate/setup.py b/packaging/executorch_delegate/setup.py new file mode 100644 index 0000000000..f2ce8a014f --- /dev/null +++ b/packaging/executorch_delegate/setup.py @@ -0,0 +1,110 @@ +"""Build the precompiled Torch-TensorRT backend for ExecuTorch Python.""" + +from __future__ import annotations + +import importlib.metadata +import os +import pathlib +import re +import subprocess +import sys + +import torch +from torch.utils.cpp_extension import include_paths +from setuptools import Extension, find_packages, setup +from setuptools.command.build_ext import build_ext + +HERE = pathlib.Path(__file__).resolve().parent +REPO_ROOT = HERE.parents[1] + + +def torchtrt_version() -> str: + if value := os.getenv("TORCH_TENSORRT_EXECUTORCH_DELEGATE_VERSION"): + return value + source = (REPO_ROOT / "py/torch_tensorrt/_version.py").read_text() + match = re.search(r'^__version__\s*=\s*["\']([^"\']+)', source, re.MULTILINE) + if not match: + raise RuntimeError("Could not determine the Torch-TensorRT version") + return match.group(1) + + +class CMakeExtension(Extension): + def __init__(self, name: str) -> None: + super().__init__(name, sources=[]) + + +class CMakeBuild(build_ext): + def build_extension(self, ext: Extension) -> None: + if sys.platform != "linux": + raise RuntimeError("The ExecuTorch TensorRT delegate supports Linux only") + executorch_source = os.getenv("EXECUTORCH_SOURCE_DIR") + if not executorch_source: + raise RuntimeError( + "Set EXECUTORCH_SOURCE_DIR to the pinned ExecuTorch source tree" + ) + + output = pathlib.Path(self.get_ext_fullpath(ext.name)).resolve() + if output.exists(): + return + build_dir = pathlib.Path(self.build_temp) / "executorch_delegate_native" + output.parent.mkdir(parents=True, exist_ok=True) + build_dir.mkdir(parents=True, exist_ok=True) + configure = [ + "cmake", + "-S", + str(HERE / "native"), + "-B", + str(build_dir), + f"-DEXECUTORCH_SOURCE_DIR={pathlib.Path(executorch_source).resolve()}", + f"-DTORCH_TENSORRT_SOURCE_DIR={REPO_ROOT}", + f"-DPYTHON_EXECUTABLE={sys.executable}", + f"-DTORCH_PRIMARY_INCLUDE_DIR={include_paths()[0]}", + f"-DDELEGATE_LIBRARY_OUTPUT_DIRECTORY={output.parent}", + f"-DCMAKE_BUILD_TYPE={'Debug' if self.debug else 'Release'}", + ] + if value := os.getenv("TensorRT_ROOT"): + configure.append(f"-DTensorRT_ROOT={value}") + configure.extend(os.getenv("CMAKE_ARGS", "").split()) + subprocess.run(configure, check=True) + subprocess.run( + [ + "cmake", + "--build", + str(build_dir), + "--target", + "torch_tensorrt_executorch_portable_lib", + "--parallel", + str(self.parallel or os.cpu_count() or 1), + ], + check=True, + ) + library_stem = ( + "_portable_lib" if ext.name.endswith("._portable_lib") else "data_loader" + ) + built = next(output.parent.glob(f"{library_stem}*.so"), None) + if built is None: + raise RuntimeError( + f"CMake did not produce {library_stem} in {output.parent}" + ) + if built != output: + built.replace(output) + + +executorch_version = importlib.metadata.version("executorch") +setup( + name="torch-tensorrt-executorch-delegate", + version=torchtrt_version(), + description="Torch-TensorRT delegate for the ExecuTorch Python runtime", + packages=find_packages(), + ext_modules=[ + CMakeExtension("torch_tensorrt_executorch_delegate._portable_lib"), + CMakeExtension("torch_tensorrt_executorch_delegate.data_loader"), + ], + cmdclass={"build_ext": CMakeBuild}, + python_requires=">=3.10", + install_requires=[ + f"torch=={torch.__version__}", + f"executorch=={executorch_version}", + ], + zip_safe=False, +) diff --git a/packaging/executorch_delegate/torch_tensorrt_executorch_delegate/__init__.py b/packaging/executorch_delegate/torch_tensorrt_executorch_delegate/__init__.py new file mode 100644 index 0000000000..6772477933 --- /dev/null +++ b/packaging/executorch_delegate/torch_tensorrt_executorch_delegate/__init__.py @@ -0,0 +1,55 @@ +"""Activate the TensorRT delegate-enabled ExecuTorch Python runtime.""" + +from __future__ import annotations + +import importlib +import sys +from types import ModuleType + +BACKEND_NAME = "TensorRTBackend" +_NATIVE_NAME = "executorch.extension.pybindings._portable_lib" +_WRAPPER_NAME = "executorch.extension.pybindings.portable_lib" +_DATA_LOADER_NAME = "executorch.extension.pybindings.data_loader" + + +class DelegateCompatibilityError(ImportError): + """The delegate wheel is incompatible with the active native runtime.""" + + +def activate() -> ModuleType: + """Make the delegate-enabled portable runtime back ``executorch.runtime``.""" + existing = sys.modules.get(_NATIVE_NAME) + if existing is not None: + if existing.__name__ != __name__ + "._portable_lib": + raise DelegateCompatibilityError( + "ExecuTorch's stock runtime was imported first. Import " + "torch_tensorrt.executorch.runtime before executorch.runtime." + ) + return existing + try: + data_loader = importlib.import_module(__name__ + ".data_loader") + sys.modules[_DATA_LOADER_NAME] = data_loader + native = importlib.import_module(__name__ + "._portable_lib") + except ImportError as error: + raise DelegateCompatibilityError( + "Could not load the prebuilt Torch-TensorRT ExecuTorch runtime. " + "Install torch, executorch, torch-tensorrt, and the delegate from " + "the same release matrix." + ) from error + sys.modules[_NATIVE_NAME] = native + sys.modules.pop(_WRAPPER_NAME, None) + return native + + +def runtime(): + """Return the activated ExecuTorch Runtime singleton.""" + activate() + from executorch.runtime import Runtime + + value = Runtime.get() + if not value.backend_registry.is_available(BACKEND_NAME): + raise DelegateCompatibilityError(f"{BACKEND_NAME} is not registered") + return value + + +__all__ = ["BACKEND_NAME", "DelegateCompatibilityError", "activate", "runtime"] diff --git a/py/torch_tensorrt/executorch/__init__.py b/py/torch_tensorrt/executorch/__init__.py index 123eee846d..a068329e8b 100644 --- a/py/torch_tensorrt/executorch/__init__.py +++ b/py/torch_tensorrt/executorch/__init__.py @@ -29,6 +29,7 @@ def __getattr__(name: str) -> NoReturn: else: from torch_tensorrt.executorch.backend import TensorRTBackend from torch_tensorrt.executorch.partitioner import TensorRTPartitioner + from torch_tensorrt.executorch.runtime import Program, load def get_edge_compile_config() -> "EdgeCompileConfig": """Return the EdgeCompileConfig used for Torch-TensorRT ExecuTorch export.""" @@ -40,4 +41,6 @@ def get_edge_compile_config() -> "EdgeCompileConfig": "get_edge_compile_config", "TensorRTPartitioner", "TensorRTBackend", + "Program", + "load", ] diff --git a/py/torch_tensorrt/executorch/runtime.py b/py/torch_tensorrt/executorch/runtime.py new file mode 100644 index 0000000000..5be97b7e11 --- /dev/null +++ b/py/torch_tensorrt/executorch/runtime.py @@ -0,0 +1,58 @@ +"""Python inference API for Torch-TensorRT ExecuTorch programs.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Sequence, Union + + +def _runtime(): + try: + from torch_tensorrt_executorch_delegate import runtime + except ImportError as error: + raise ImportError( + "ExecuTorch Python inference requires the prebuilt delegate. " + 'Install it with: pip install "torch-tensorrt[executorch]"' + ) from error + return runtime() + + +class Program: + """A loaded ExecuTorch program backed by TensorRTBackend.""" + + def __init__(self, program: Any) -> None: + self._program = program + + @property + def method_names(self): + return self._program.method_names + + def run(self, inputs: Sequence[Any], method: str = "forward") -> Sequence[Any]: + import torch + + inputs = tuple( + value.cpu() if isinstance(value, torch.Tensor) and value.is_cuda else value + for value in inputs + ) + if method not in self.method_names: + raise ValueError( + f"Unknown method {method!r}; available methods: {sorted(self.method_names)}" + ) + loaded = self._program.load_method(method) + if loaded is None: + raise RuntimeError(f"ExecuTorch failed to load method {method!r}") + return loaded.execute(inputs) + + def forward(self, *inputs: Any) -> Sequence[Any]: + return self.run(inputs, "forward") + + +def load(path: Union[str, Path]) -> Program: + """Load a `.pte` with the delegate-enabled ExecuTorch Python runtime.""" + model_path = Path(path) + if not model_path.is_file(): + raise FileNotFoundError(f"ExecuTorch model not found: {model_path}") + return Program(_runtime().load_program(model_path.read_bytes())) + + +__all__ = ["Program", "load"] diff --git a/setup.py b/setup.py index 91651c7e15..1563d41c33 100644 --- a/setup.py +++ b/setup.py @@ -99,12 +99,6 @@ def load_dep_info(): CI_BUILD = False USE_TRT_RTX = False -EXECUTORCH_REQUIREMENT = "executorch>=1.3.1" -EXTRAS_REQUIRE = { - "executorch": [EXECUTORCH_REQUIREMENT], - "all": [EXECUTORCH_REQUIREMENT], -} - if "--use-rtx" in sys.argv: USE_TRT_RTX = True @@ -189,6 +183,16 @@ def load_dep_info(): else: __version__ = f"{get_base_version()}.dev0+{get_git_revision_short_hash()}" +EXECUTORCH_REQUIREMENT = "executorch>=1.3.1" +EXECUTORCH_DELEGATE_REQUIREMENT = ( + f"torch-tensorrt-executorch-delegate=={__version__}; " + "platform_system == 'Linux' and platform_machine == 'x86_64'" +) +EXTRAS_REQUIRE = { + "executorch": [EXECUTORCH_REQUIREMENT, EXECUTORCH_DELEGATE_REQUIREMENT], + "all": [EXECUTORCH_REQUIREMENT, EXECUTORCH_DELEGATE_REQUIREMENT], +} + if "--ci" in sys.argv: sys.argv.remove("--ci") if RELEASE: diff --git a/tests/py/executorch/test_python_runtime.py b/tests/py/executorch/test_python_runtime.py new file mode 100644 index 0000000000..c4c70a0bc7 --- /dev/null +++ b/tests/py/executorch/test_python_runtime.py @@ -0,0 +1,64 @@ +import importlib.util +import sys +import types +from pathlib import Path + +import pytest + +RUNTIME_PATH = Path(__file__).parents[3] / "py/torch_tensorrt/executorch/runtime.py" + + +def load_runtime_module(): + spec = importlib.util.spec_from_file_location( + "torchtrt_et_runtime_test", RUNTIME_PATH + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FakeMethod: + def execute(self, inputs): + return [inputs[0] + 1] + + +class FakeProgram: + method_names = {"forward"} + + def load_method(self, name): + return FakeMethod() if name == "forward" else None + + +class FakeRuntime: + def load_program(self, path): + self.path = path + return FakeProgram() + + +def test_load_and_forward(monkeypatch, tmp_path): + delegate = types.ModuleType("torch_tensorrt_executorch_delegate") + delegate.runtime = FakeRuntime + monkeypatch.setitem(sys.modules, delegate.__name__, delegate) + model = tmp_path / "model.pte" + model.write_bytes(b"pte") + + program = load_runtime_module().load(model) + + assert program.forward(2) == [3] + + +def test_unknown_method(monkeypatch, tmp_path): + delegate = types.ModuleType("torch_tensorrt_executorch_delegate") + delegate.runtime = FakeRuntime + monkeypatch.setitem(sys.modules, delegate.__name__, delegate) + model = tmp_path / "model.pte" + model.write_bytes(b"pte") + + with pytest.raises(ValueError, match="Unknown method"): + load_runtime_module().load(model).run([], "missing") + + +def test_missing_model(): + with pytest.raises(FileNotFoundError): + load_runtime_module().load("does-not-exist.pte") From f1339a08150b7fae7e3577f0bf9f000a47b25298 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Sat, 11 Jul 2026 16:41:42 -0700 Subject: [PATCH 02/17] fix the test error --- .../verify-executorch-reference-runner.sh | 65 +++++-------------- .github/workflows/executorch-static-linux.yml | 8 ++- 2 files changed, 23 insertions(+), 50 deletions(-) diff --git a/.github/scripts/verify-executorch-reference-runner.sh b/.github/scripts/verify-executorch-reference-runner.sh index 19e00b4843..b59ed905ec 100755 --- a/.github/scripts/verify-executorch-reference-runner.sh +++ b/.github/scripts/verify-executorch-reference-runner.sh @@ -6,11 +6,12 @@ set +x # # 1. Build //:libtorchtrt first so bazel-bin/libtorchtrt.tar.gz exists. # 2. Provide an ExecuTorch source checkout with EXECUTORCH_SOURCE_DIR. -# 3. This script exports a small Torch-TensorRT ExecuTorch .pte model, -# unpacks libtorchtrt.tar.gz, configures the packaged CMake runner, -# builds example_executorch_runner, and runs one inference. +# 3. Provide a Torch-TensorRT ExecuTorch .pte model. +# 4. This script unpacks libtorchtrt.tar.gz, configures and builds the +# packaged CMake runner, and runs one inference. # # Required: +# First argument: path to an existing .pte model. # EXECUTORCH_SOURCE_DIR=/path/to/executorch # # Optional: @@ -28,6 +29,16 @@ set +x repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "${repo_root}" +if [[ $# -ne 1 ]]; then + echo "Usage: $0 PATH_TO_MODEL.pte" >&2 + exit 1 +fi +model_path="$1" +if [[ ! -f "${model_path}" ]]; then + echo "ExecuTorch model not found: ${model_path}" >&2 + exit 1 +fi + python_executable="${PYTHON_EXECUTABLE:-}" if [[ -z "${python_executable}" ]]; then python_executable="$(command -v python || true)" @@ -201,14 +212,13 @@ else export LD_LIBRARY_PATH="${torch_lib_dir}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" fi -# Fail early if the Python environment cannot export a Torch-TensorRT -# ExecuTorch model or run ExecuTorch's CMake codegen. +# Fail early if the Python environment cannot run ExecuTorch's CMake codegen. if ! "${python_executable}" - <<'PY' import importlib import importlib.util missing = [] -for name in ("yaml", "torch", "torch_tensorrt", "executorch.exir"): +for name in ("yaml", "torch", "executorch.exir"): try: spec = importlib.util.find_spec(name) except ModuleNotFoundError: @@ -217,56 +227,17 @@ for name in ("yaml", "torch", "torch_tensorrt", "executorch.exir"): missing.append(name) if missing: raise SystemExit( - "Missing Python package(s) required to export the .pte and build the runner: " + "Missing Python package(s) required to build the runner: " + ", ".join(missing) ) -for name in ("yaml", "torch", "torch_tensorrt", "executorch.exir"): +for name in ("yaml", "torch", "executorch.exir"): importlib.import_module(name) PY then exit 1 fi -model_path="${verify_root}/model.pte" -"${python_executable}" - "${model_path}" <<'PY' -import importlib.util -import runpy -import sys -from pathlib import Path - -model_path = sys.argv[1] -repo_root = Path.cwd() - -# Use the installed package for native extensions and the in-tree ExecuTorch -# route for the serializer/backend under test. -import torch_tensorrt # noqa: F401 - - -def overlay_module(name: str, path: Path) -> None: - spec = importlib.util.spec_from_file_location(name, path) - if spec is None or spec.loader is None: - raise RuntimeError(f"Could not load {name} from {path}") - module = importlib.util.module_from_spec(spec) - sys.modules[name] = module - spec.loader.exec_module(module) - - -overlay_module( - "torch_tensorrt.executorch.serialization", - repo_root / "py/torch_tensorrt/executorch/serialization.py", -) -overlay_module( - "torch_tensorrt.executorch.backend", - repo_root / "py/torch_tensorrt/executorch/backend.py", -) - -export_script = repo_root / "examples/torchtrt_executorch_example/export_static_shape.py" -sys.argv = [str(export_script), "--model_path", model_path] -runpy.run_path(str(export_script), run_name="__main__") -PY -test -f "${model_path}" - if [[ -n "${TensorRT_ROOT:-}" && -d "${TensorRT_ROOT}/lib" ]]; then export LD_LIBRARY_PATH="${TensorRT_ROOT}/lib${original_ld_library_path:+:${original_ld_library_path}}" else diff --git a/.github/workflows/executorch-static-linux.yml b/.github/workflows/executorch-static-linux.yml index d0e4bab2f5..2e6a592573 100644 --- a/.github/workflows/executorch-static-linux.yml +++ b/.github/workflows/executorch-static-linux.yml @@ -99,11 +99,13 @@ jobs: # this is to verify the end user's workflow python -m pip install pyyaml executorch + # export the model to a .pte file + python examples/torchtrt_executorch_example/export_static_shape.py --model_path="${RUNNER_TEMP}/torchtrt-python.pte" + # verify the c++ reference runner can run the model + .github/scripts/verify-executorch-reference-runner.sh "${RUNNER_TEMP}/torchtrt-python.pte" + # Build the no-compile-for-users Python runtime/delegate wheel. - export TORCH_TENSORRT_EXECUTORCH_DELEGATE_VERSION="$(python -c 'import torch_tensorrt; print(torch_tensorrt.__version__)')" python -m pip wheel --no-build-isolation --no-deps --wheel-dir dist packaging/executorch_delegate python -m pip install --no-deps --force-reinstall dist/torch_tensorrt_executorch_delegate-*.whl python -c 'from torch_tensorrt_executorch_delegate import runtime; assert runtime().backend_registry.is_available("TensorRTBackend")' - python examples/torchtrt_executorch_example/export_static_shape.py --model_path="${RUNNER_TEMP}/torchtrt-python.pte" python examples/executorch_reference_runner/load_model.py --model_path="${RUNNER_TEMP}/torchtrt-python.pte" --num_runs=1 - .github/scripts/verify-executorch-reference-runner.sh From e3a8f27c4d7c4cd0fea69b1428adc855fe666a4c Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Sat, 11 Jul 2026 22:27:29 -0700 Subject: [PATCH 03/17] test --- .github/workflows/_linux-x86_64-core.yml | 1 + .github/workflows/build_linux.yml | 45 +++++++++++++++++++++--- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_linux-x86_64-core.yml b/.github/workflows/_linux-x86_64-core.yml index c0011470b2..c32fc55bdc 100644 --- a/.github/workflows/_linux-x86_64-core.yml +++ b/.github/workflows/_linux-x86_64-core.yml @@ -96,6 +96,7 @@ jobs: trigger-event: ${{ github.event_name }} architecture: "x86_64" use-rtx: ${{ inputs.use-rtx }} + build-executorch-delegate: ${{ !inputs.use-rtx }} pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" # Standard-TRT only: ExecuTorch static build is not part of the RTX matrix. diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml index 90333aedf8..6b27a3ff21 100644 --- a/.github/workflows/build_linux.yml +++ b/.github/workflows/build_linux.yml @@ -90,6 +90,11 @@ on: required: false default: "python -m build --wheel" type: string + build-executorch-delegate: + description: Build and publish the matching ExecuTorch delegate wheel + required: false + default: false + type: boolean pip-install-torch-extra-args: # NOTE: Why does this exist? # Well setuptools / python packaging doesn't actually allow you to specify dependencies @@ -152,6 +157,7 @@ jobs: ARCH: ${{ inputs.architecture }} BUILD_TARGET: ${{ inputs.build-target }} USE_TRT_RTX: ${{ inputs.use-rtx }} + BUILD_EXECUTORCH_DELEGATE: ${{ inputs.build-executorch-delegate }} name: build-wheel-${{ matrix.python_version }}-${{ matrix.desired_cuda }}-${{ matrix.gpu_arch_type }}-${{ inputs.architecture }}-${{ inputs.use-rtx }}-${{ inputs.is-jetpack }} runs-on: ${{ matrix.validation_runner }} environment: ${{(inputs.trigger-event == 'schedule' || (inputs.trigger-event == 'push' && (startsWith(github.event.ref, 'refs/heads/nightly') || startsWith(github.event.ref, 'refs/tags/v')))) && 'pytorchbot-env' || ''}} @@ -313,6 +319,19 @@ jobs: ${CONDA_RUN} python setup.py bdist_wheel ${param} fi fi + - name: Build the ExecuTorch delegate wheel + if: ${{ inputs.build-executorch-delegate && inputs.use-rtx == false && inputs.is-jetpack == false && inputs.is-release-tarball == false }} + working-directory: ${{ inputs.repository }} + shell: bash -l {0} + run: | + set -euxo pipefail + source "${BUILD_ENV_FILE}" + ${CONDA_RUN} python -m pip install pyyaml executorch + executorch_cmake_location="$(bazel query @executorch//:executorch/CMakeLists.txt --output=location)" + export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" + export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}" + ${CONDA_RUN} python -m pip wheel --no-build-isolation --no-deps \ + --wheel-dir dist packaging/executorch_delegate - name: Repair Manylinux_2_28 Wheel shell: bash -l {0} env: @@ -346,16 +365,31 @@ jobs: run: | set -euxo pipefail source "${BUILD_ENV_FILE}" - WHEEL_NAME=$(ls "${{ inputs.repository }}/dist/") - echo "$WHEEL_NAME" + mapfile -t WHEEL_PATHS < <(find "${{ inputs.repository }}/dist" -maxdepth 1 -name "*.whl" -print | sort) + if [[ ${#WHEEL_PATHS[@]} -eq 0 ]]; then + echo "No wheels found in ${{ inputs.repository }}/dist" >&2 + exit 1 + fi + printf "Wheel: %s\n" "${WHEEL_PATHS[@]}" + MAIN_WHEEL="" + for wheel_path in "${WHEEL_PATHS[@]}"; do + if [[ $(basename "${wheel_path}") == torch_tensorrt-[0-9]*.whl ]]; then + MAIN_WHEEL="${wheel_path}" + break + fi + done + if [[ -z "${MAIN_WHEEL}" ]]; then + echo "Could not identify the torch_tensorrt wheel" >&2 + exit 1 + fi if [[ ${{ inputs.architecture }} == "aarch64" ]]; then echo "Skipping smoke test for aarch64, since it is not an actual gpu runner" else - ${CONDA_RUN} pip install "${{ inputs.repository }}/dist/$WHEEL_NAME" --use-deprecated=legacy-resolver --extra-index-url https://download.pytorch.org/${CHANNEL}/${CU_VERSION} + ${CONDA_RUN} pip install "${WHEEL_PATHS[@]}" --use-deprecated=legacy-resolver --extra-index-url https://download.pytorch.org/${CHANNEL}/${CU_VERSION} # Checking that we have a pinned version of torch in our dependency tree ( pushd "${RUNNER_TEMP}" - unzip -o "${GITHUB_WORKSPACE}/${{ inputs.repository }}/dist/$WHEEL_NAME" + unzip -o "${MAIN_WHEEL}" # Ensure that pytorch version is pinned, should output file where it was found grep "Requires-Dist: torch (==.*)" -r . ) @@ -374,6 +408,9 @@ jobs: echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found" ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" fi + if [[ "${BUILD_EXECUTORCH_DELEGATE}" == "true" ]]; then + ${CONDA_RUN} python -c 'from torch_tensorrt_executorch_delegate import runtime; assert runtime().backend_registry.is_available("TensorRTBackend")' + fi fi # NB: Only upload to GitHub after passing smoke tests From 14c9c71bf612d0e7e7ac7b637e30639df5ea08ca Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Fri, 17 Jul 2026 12:30:05 -0700 Subject: [PATCH 04/17] resolve review comments --- .github/workflows/build_linux.yml | 2 +- .github/workflows/executorch-static-linux.yml | 2 +- .gitignore | 5 +- packaging/executorch_delegate/README.md | 38 ------ .../executorch_delegate/native/CMakeLists.txt | 68 ----------- packaging/executorch_delegate/pyproject.toml | 7 -- packaging/executorch_delegate/setup.py | 110 ------------------ .../__init__.py | 55 --------- py/torch_tensorrt/executorch/runtime.py | 7 +- tests/py/executorch/test_python_runtime.py | 64 ---------- toolchains/ci_workspaces/MODULE.bazel.tmpl | 8 +- 11 files changed, 13 insertions(+), 353 deletions(-) delete mode 100644 packaging/executorch_delegate/README.md delete mode 100644 packaging/executorch_delegate/native/CMakeLists.txt delete mode 100644 packaging/executorch_delegate/pyproject.toml delete mode 100644 packaging/executorch_delegate/setup.py delete mode 100644 packaging/executorch_delegate/torch_tensorrt_executorch_delegate/__init__.py delete mode 100644 tests/py/executorch/test_python_runtime.py diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml index 6b27a3ff21..109c439ab5 100644 --- a/.github/workflows/build_linux.yml +++ b/.github/workflows/build_linux.yml @@ -331,7 +331,7 @@ jobs: export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}" ${CONDA_RUN} python -m pip wheel --no-build-isolation --no-deps \ - --wheel-dir dist packaging/executorch_delegate + --wheel-dir dist py/torch-tensorrt-executorch-delegate - name: Repair Manylinux_2_28 Wheel shell: bash -l {0} env: diff --git a/.github/workflows/executorch-static-linux.yml b/.github/workflows/executorch-static-linux.yml index 2e6a592573..b98c1a4dac 100644 --- a/.github/workflows/executorch-static-linux.yml +++ b/.github/workflows/executorch-static-linux.yml @@ -105,7 +105,7 @@ jobs: .github/scripts/verify-executorch-reference-runner.sh "${RUNNER_TEMP}/torchtrt-python.pte" # Build the no-compile-for-users Python runtime/delegate wheel. - python -m pip wheel --no-build-isolation --no-deps --wheel-dir dist packaging/executorch_delegate + python -m pip wheel --no-build-isolation --no-deps --wheel-dir dist py/torch-tensorrt-executorch-delegate python -m pip install --no-deps --force-reinstall dist/torch_tensorrt_executorch_delegate-*.whl python -c 'from torch_tensorrt_executorch_delegate import runtime; assert runtime().backend_registry.is_available("TensorRTBackend")' python examples/executorch_reference_runner/load_model.py --model_path="${RUNNER_TEMP}/torchtrt-python.pte" --num_runs=1 diff --git a/.gitignore b/.gitignore index 477056ed82..662a6fc267 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,5 @@ CLAUDE.md /compile_commands.json /external/ build-executorch/ -packaging/executorch_delegate/build/ -packaging/executorch_delegate/dist/ - +py/torch-tensorrt-executorch-delegate/build/ +py/torch-tensorrt-executorch-delegate/dist/ \ No newline at end of file diff --git a/packaging/executorch_delegate/README.md b/packaging/executorch_delegate/README.md deleted file mode 100644 index b4c4e65b6d..0000000000 --- a/packaging/executorch_delegate/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# Torch-TensorRT ExecuTorch Delegate Wheel - -This directory builds `torch-tensorrt-executorch-delegate`. The Linux wheel -contains an ExecuTorch `_portable_lib` Python runtime with `TensorRTBackend` -force-linked into the same native module that owns the backend registry. - -The wheel must use the same Python, PyTorch, ExecuTorch, CUDA, TensorRT, and -C++ ABI as its matching Torch-TensorRT wheel. - -## Build - -```bash -executorch_cmake_location="$(bazel query \ - @executorch//:executorch/CMakeLists.txt --output=location)" -export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" -export TensorRT_ROOT=/path/to/TensorRT - -python -m pip install executorch==1.3.1 -python -m pip wheel --no-build-isolation --no-deps \ - --wheel-dir dist packaging/executorch_delegate -``` - -The static ExecuTorch and delegate archives are intermediate build inputs; -users receive the final native Python module and do not compile anything. - -## Use - -```bash -pip install "torch-tensorrt[executorch]" -``` - -```python -import torch -from torch_tensorrt.executorch.runtime import load - -program = load("model.pte") -outputs = program.forward(torch.ones((2, 3, 4, 4))) -``` diff --git a/packaging/executorch_delegate/native/CMakeLists.txt b/packaging/executorch_delegate/native/CMakeLists.txt deleted file mode 100644 index 1b61d3f767..0000000000 --- a/packaging/executorch_delegate/native/CMakeLists.txt +++ /dev/null @@ -1,68 +0,0 @@ -cmake_minimum_required(VERSION 3.24) -project(torch_tensorrt_executorch_delegate LANGUAGES CXX) - -if(NOT EXECUTORCH_SOURCE_DIR OR NOT TORCH_TENSORRT_SOURCE_DIR) - message(FATAL_ERROR "EXECUTORCH_SOURCE_DIR and TORCH_TENSORRT_SOURCE_DIR are required") -endif() - -list(APPEND CMAKE_MODULE_PATH "${TORCH_TENSORRT_SOURCE_DIR}/cmake/Modules") -find_package(TensorRT REQUIRED) -find_package(CUDAToolkit REQUIRED) -find_package(Threads REQUIRED) - -set(BUILD_TESTING OFF CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_PYBIND ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_EXTENSION_MODULE ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_EXTENSION_TENSOR ON CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_KERNELS_OPTIMIZED OFF CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_XNNPACK OFF CACHE BOOL "" FORCE) -set(EXECUTORCH_BUILD_TESTS OFF CACHE BOOL "" FORCE) -set(CMAKE_POSITION_INDEPENDENT_CODE ON) - -add_subdirectory("${EXECUTORCH_SOURCE_DIR}" executorch) - -add_library(torch_tensorrt_executorch_backend STATIC - "${TORCH_TENSORRT_SOURCE_DIR}/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp" - "${TORCH_TENSORRT_SOURCE_DIR}/cpp/src/torch_tensorrt/executorch/TensorRTBlobHeader.cpp") -target_compile_features(torch_tensorrt_executorch_backend PUBLIC cxx_std_17) -target_compile_definitions(torch_tensorrt_executorch_backend - PRIVATE C10_USING_CUSTOM_GENERATED_MACROS) -target_include_directories(torch_tensorrt_executorch_backend PRIVATE - "${TORCH_TENSORRT_SOURCE_DIR}" - "${TORCH_TENSORRT_SOURCE_DIR}/cpp/include" - "${EXECUTORCH_SOURCE_DIR}/.." - "${EXECUTORCH_SOURCE_DIR}/runtime/core/portable_type/c10") -target_link_libraries(torch_tensorrt_executorch_backend PRIVATE - CUDA::cudart TensorRT::nvinfer Threads::Threads executorch) - -if(NOT TARGET portable_lib) - message(FATAL_ERROR "ExecuTorch did not define portable_lib") -endif() - -# ExecuTorch portable c10 delegates selected headers to torch/headeronly. The -# pybind bridge must resolve those delegated headers from the exact PyTorch -# wheel it links, not ExecuTorch's vendored snapshot. -file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/torch_header_shim/torch") -file(CREATE_LINK - "${TORCH_PRIMARY_INCLUDE_DIR}/torch/headeronly" - "${CMAKE_BINARY_DIR}/torch_header_shim/torch/headeronly" - SYMBOLIC COPY_ON_ERROR) -target_include_directories(util BEFORE PRIVATE "${CMAKE_BINARY_DIR}/torch_header_shim") -target_include_directories(portable_lib BEFORE PRIVATE "${CMAKE_BINARY_DIR}/torch_header_shim") - -# TensorRTBackend registers through a static initializer. Force its complete -# archive into the same pybind module that owns ExecuTorch's backend registry. -target_link_libraries(portable_lib PRIVATE - "$" - extension_threadpool - CUDA::cudart TensorRT::nvinfer Threads::Threads) -set_target_properties(portable_lib PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${DELEGATE_LIBRARY_OUTPUT_DIRECTORY}" - OUTPUT_NAME "_portable_lib") -set_target_properties(data_loader PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${DELEGATE_LIBRARY_OUTPUT_DIRECTORY}") - -add_custom_target(torch_tensorrt_executorch_portable_lib DEPENDS portable_lib data_loader) diff --git a/packaging/executorch_delegate/pyproject.toml b/packaging/executorch_delegate/pyproject.toml deleted file mode 100644 index 5142ec285d..0000000000 --- a/packaging/executorch_delegate/pyproject.toml +++ /dev/null @@ -1,7 +0,0 @@ -[build-system] -requires = [ - "setuptools>=68", - "wheel>=0.40", - "torch", -] -build-backend = "setuptools.build_meta" diff --git a/packaging/executorch_delegate/setup.py b/packaging/executorch_delegate/setup.py deleted file mode 100644 index f2ce8a014f..0000000000 --- a/packaging/executorch_delegate/setup.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Build the precompiled Torch-TensorRT backend for ExecuTorch Python.""" - -from __future__ import annotations - -import importlib.metadata -import os -import pathlib -import re -import subprocess -import sys - -import torch -from torch.utils.cpp_extension import include_paths -from setuptools import Extension, find_packages, setup -from setuptools.command.build_ext import build_ext - -HERE = pathlib.Path(__file__).resolve().parent -REPO_ROOT = HERE.parents[1] - - -def torchtrt_version() -> str: - if value := os.getenv("TORCH_TENSORRT_EXECUTORCH_DELEGATE_VERSION"): - return value - source = (REPO_ROOT / "py/torch_tensorrt/_version.py").read_text() - match = re.search(r'^__version__\s*=\s*["\']([^"\']+)', source, re.MULTILINE) - if not match: - raise RuntimeError("Could not determine the Torch-TensorRT version") - return match.group(1) - - -class CMakeExtension(Extension): - def __init__(self, name: str) -> None: - super().__init__(name, sources=[]) - - -class CMakeBuild(build_ext): - def build_extension(self, ext: Extension) -> None: - if sys.platform != "linux": - raise RuntimeError("The ExecuTorch TensorRT delegate supports Linux only") - executorch_source = os.getenv("EXECUTORCH_SOURCE_DIR") - if not executorch_source: - raise RuntimeError( - "Set EXECUTORCH_SOURCE_DIR to the pinned ExecuTorch source tree" - ) - - output = pathlib.Path(self.get_ext_fullpath(ext.name)).resolve() - if output.exists(): - return - build_dir = pathlib.Path(self.build_temp) / "executorch_delegate_native" - output.parent.mkdir(parents=True, exist_ok=True) - build_dir.mkdir(parents=True, exist_ok=True) - configure = [ - "cmake", - "-S", - str(HERE / "native"), - "-B", - str(build_dir), - f"-DEXECUTORCH_SOURCE_DIR={pathlib.Path(executorch_source).resolve()}", - f"-DTORCH_TENSORRT_SOURCE_DIR={REPO_ROOT}", - f"-DPYTHON_EXECUTABLE={sys.executable}", - f"-DTORCH_PRIMARY_INCLUDE_DIR={include_paths()[0]}", - f"-DDELEGATE_LIBRARY_OUTPUT_DIRECTORY={output.parent}", - f"-DCMAKE_BUILD_TYPE={'Debug' if self.debug else 'Release'}", - ] - if value := os.getenv("TensorRT_ROOT"): - configure.append(f"-DTensorRT_ROOT={value}") - configure.extend(os.getenv("CMAKE_ARGS", "").split()) - subprocess.run(configure, check=True) - subprocess.run( - [ - "cmake", - "--build", - str(build_dir), - "--target", - "torch_tensorrt_executorch_portable_lib", - "--parallel", - str(self.parallel or os.cpu_count() or 1), - ], - check=True, - ) - library_stem = ( - "_portable_lib" if ext.name.endswith("._portable_lib") else "data_loader" - ) - built = next(output.parent.glob(f"{library_stem}*.so"), None) - if built is None: - raise RuntimeError( - f"CMake did not produce {library_stem} in {output.parent}" - ) - if built != output: - built.replace(output) - - -executorch_version = importlib.metadata.version("executorch") -setup( - name="torch-tensorrt-executorch-delegate", - version=torchtrt_version(), - description="Torch-TensorRT delegate for the ExecuTorch Python runtime", - packages=find_packages(), - ext_modules=[ - CMakeExtension("torch_tensorrt_executorch_delegate._portable_lib"), - CMakeExtension("torch_tensorrt_executorch_delegate.data_loader"), - ], - cmdclass={"build_ext": CMakeBuild}, - python_requires=">=3.10", - install_requires=[ - f"torch=={torch.__version__}", - f"executorch=={executorch_version}", - ], - zip_safe=False, -) diff --git a/packaging/executorch_delegate/torch_tensorrt_executorch_delegate/__init__.py b/packaging/executorch_delegate/torch_tensorrt_executorch_delegate/__init__.py deleted file mode 100644 index 6772477933..0000000000 --- a/packaging/executorch_delegate/torch_tensorrt_executorch_delegate/__init__.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Activate the TensorRT delegate-enabled ExecuTorch Python runtime.""" - -from __future__ import annotations - -import importlib -import sys -from types import ModuleType - -BACKEND_NAME = "TensorRTBackend" -_NATIVE_NAME = "executorch.extension.pybindings._portable_lib" -_WRAPPER_NAME = "executorch.extension.pybindings.portable_lib" -_DATA_LOADER_NAME = "executorch.extension.pybindings.data_loader" - - -class DelegateCompatibilityError(ImportError): - """The delegate wheel is incompatible with the active native runtime.""" - - -def activate() -> ModuleType: - """Make the delegate-enabled portable runtime back ``executorch.runtime``.""" - existing = sys.modules.get(_NATIVE_NAME) - if existing is not None: - if existing.__name__ != __name__ + "._portable_lib": - raise DelegateCompatibilityError( - "ExecuTorch's stock runtime was imported first. Import " - "torch_tensorrt.executorch.runtime before executorch.runtime." - ) - return existing - try: - data_loader = importlib.import_module(__name__ + ".data_loader") - sys.modules[_DATA_LOADER_NAME] = data_loader - native = importlib.import_module(__name__ + "._portable_lib") - except ImportError as error: - raise DelegateCompatibilityError( - "Could not load the prebuilt Torch-TensorRT ExecuTorch runtime. " - "Install torch, executorch, torch-tensorrt, and the delegate from " - "the same release matrix." - ) from error - sys.modules[_NATIVE_NAME] = native - sys.modules.pop(_WRAPPER_NAME, None) - return native - - -def runtime(): - """Return the activated ExecuTorch Runtime singleton.""" - activate() - from executorch.runtime import Runtime - - value = Runtime.get() - if not value.backend_registry.is_available(BACKEND_NAME): - raise DelegateCompatibilityError(f"{BACKEND_NAME} is not registered") - return value - - -__all__ = ["BACKEND_NAME", "DelegateCompatibilityError", "activate", "runtime"] diff --git a/py/torch_tensorrt/executorch/runtime.py b/py/torch_tensorrt/executorch/runtime.py index 5be97b7e11..fb78b54d77 100644 --- a/py/torch_tensorrt/executorch/runtime.py +++ b/py/torch_tensorrt/executorch/runtime.py @@ -20,7 +20,9 @@ def _runtime(): class Program: """A loaded ExecuTorch program backed by TensorRTBackend.""" - def __init__(self, program: Any) -> None: + def __init__(self, program: Any, data: bytes) -> None: + # ExecuTorch's BufferDataLoader references this memory without copying it. + self._data = data self._program = program @property @@ -52,7 +54,8 @@ def load(path: Union[str, Path]) -> Program: model_path = Path(path) if not model_path.is_file(): raise FileNotFoundError(f"ExecuTorch model not found: {model_path}") - return Program(_runtime().load_program(model_path.read_bytes())) + data = model_path.read_bytes() + return Program(_runtime().load_program(data), data) __all__ = ["Program", "load"] diff --git a/tests/py/executorch/test_python_runtime.py b/tests/py/executorch/test_python_runtime.py deleted file mode 100644 index c4c70a0bc7..0000000000 --- a/tests/py/executorch/test_python_runtime.py +++ /dev/null @@ -1,64 +0,0 @@ -import importlib.util -import sys -import types -from pathlib import Path - -import pytest - -RUNTIME_PATH = Path(__file__).parents[3] / "py/torch_tensorrt/executorch/runtime.py" - - -def load_runtime_module(): - spec = importlib.util.spec_from_file_location( - "torchtrt_et_runtime_test", RUNTIME_PATH - ) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -class FakeMethod: - def execute(self, inputs): - return [inputs[0] + 1] - - -class FakeProgram: - method_names = {"forward"} - - def load_method(self, name): - return FakeMethod() if name == "forward" else None - - -class FakeRuntime: - def load_program(self, path): - self.path = path - return FakeProgram() - - -def test_load_and_forward(monkeypatch, tmp_path): - delegate = types.ModuleType("torch_tensorrt_executorch_delegate") - delegate.runtime = FakeRuntime - monkeypatch.setitem(sys.modules, delegate.__name__, delegate) - model = tmp_path / "model.pte" - model.write_bytes(b"pte") - - program = load_runtime_module().load(model) - - assert program.forward(2) == [3] - - -def test_unknown_method(monkeypatch, tmp_path): - delegate = types.ModuleType("torch_tensorrt_executorch_delegate") - delegate.runtime = FakeRuntime - monkeypatch.setitem(sys.modules, delegate.__name__, delegate) - model = tmp_path / "model.pte" - model.write_bytes(b"pte") - - with pytest.raises(ValueError, match="Unknown method"): - load_runtime_module().load(model).run([], "missing") - - -def test_missing_model(): - with pytest.raises(FileNotFoundError): - load_runtime_module().load("does-not-exist.pte") diff --git a/toolchains/ci_workspaces/MODULE.bazel.tmpl b/toolchains/ci_workspaces/MODULE.bazel.tmpl index 46479dc114..9c22f41291 100644 --- a/toolchains/ci_workspaces/MODULE.bazel.tmpl +++ b/toolchains/ci_workspaces/MODULE.bazel.tmpl @@ -172,13 +172,12 @@ new_local_repository( build_file = "third_party/libtorch/BUILD" ) -# Pinned to the ExecuTorch release/1.3 branch head. +# Pinned to ExecuTorch main for PyTorch 2.14 native ABI compatibility. new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # latest commit in release/1.3 branch - commit = "6118688a095fd9697224f5cad72ce42db641c9cd", - recursive_init_submodules = True, + # PyTorch 2.14-compatible ExecuTorch main commit + commit = "a6d812a082df57898b8608f56c867140cc9da32c", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;", @@ -187,6 +186,7 @@ new_git_repository( "Get-ChildItem -Recurse -Include BUILD,BUILD.bazel | Where-Object { $_.DirectoryName -ne (Get-Location).Path } | Remove-Item -Force", "New-Item -ItemType Directory executorch | Out-Null; Get-ChildItem -Force | Where-Object { $_.Name -notin @('executorch','BUILD','BUILD.bazel','REPO.bazel') } | Copy-Item -Destination executorch -Recurse -Force", ], + recursive_init_submodules = True, remote = "https://github.com/pytorch/executorch.git", ) From e6a6f7ac93d2d583efce43ab852b9663ab5fca4c Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Fri, 17 Jul 2026 12:30:21 -0700 Subject: [PATCH 05/17] resolve review commits --- .../README.md | 44 +++++ .../native/CMakeLists.txt | 68 ++++++++ .../pyproject.toml | 9 ++ .../setup.py | 109 +++++++++++++ .../__init__.py | 62 ++++++++ .../dynamo/executorch/test_python_runtime.py | 150 ++++++++++++++++++ 6 files changed, 442 insertions(+) create mode 100644 py/torch-tensorrt-executorch-delegate/README.md create mode 100644 py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt create mode 100644 py/torch-tensorrt-executorch-delegate/pyproject.toml create mode 100644 py/torch-tensorrt-executorch-delegate/setup.py create mode 100644 py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py create mode 100644 tests/py/dynamo/executorch/test_python_runtime.py diff --git a/py/torch-tensorrt-executorch-delegate/README.md b/py/torch-tensorrt-executorch-delegate/README.md new file mode 100644 index 0000000000..c2f93dcb19 --- /dev/null +++ b/py/torch-tensorrt-executorch-delegate/README.md @@ -0,0 +1,44 @@ +# Torch-TensorRT ExecuTorch Delegate Wheel + +This directory builds `torch-tensorrt-executorch-delegate`. The Linux wheel +contains an ExecuTorch `_portable_lib` Python runtime with `TensorRTBackend` +force-linked into the same native module that owns the backend registry. + +The wheel must use the same Python, PyTorch, ExecuTorch, CUDA, TensorRT, and +C++ ABI as its matching Torch-TensorRT wheel. + +## Build + +> [!IMPORTANT] +> Build this wheel with `--no-build-isolation`. Its native extension must use +> the exact PyTorch installation that the matching Torch-TensorRT artifacts +> were built against. An isolated build may download a newer, ABI-incompatible +> PyTorch version. + +```bash +executorch_cmake_location="$(bazel query \ + @executorch//:executorch/CMakeLists.txt --output=location)" +export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" +export TensorRT_ROOT=/path/to/TensorRT + +python -m pip install executorch==1.3.1 +python -m pip wheel --no-build-isolation --no-deps \ + --wheel-dir dist py/torch-tensorrt-executorch-delegate +``` + +The static ExecuTorch and delegate archives are intermediate build inputs; +users receive the final native Python module and do not compile anything. + +## Use + +```bash +pip install "torch-tensorrt[executorch]" +``` + +```python +import torch +from torch_tensorrt.executorch.runtime import load + +program = load("model.pte") +outputs = program.forward(torch.ones((2, 3, 4, 4))) +``` diff --git a/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt b/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt new file mode 100644 index 0000000000..1b61d3f767 --- /dev/null +++ b/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt @@ -0,0 +1,68 @@ +cmake_minimum_required(VERSION 3.24) +project(torch_tensorrt_executorch_delegate LANGUAGES CXX) + +if(NOT EXECUTORCH_SOURCE_DIR OR NOT TORCH_TENSORRT_SOURCE_DIR) + message(FATAL_ERROR "EXECUTORCH_SOURCE_DIR and TORCH_TENSORRT_SOURCE_DIR are required") +endif() + +list(APPEND CMAKE_MODULE_PATH "${TORCH_TENSORRT_SOURCE_DIR}/cmake/Modules") +find_package(TensorRT REQUIRED) +find_package(CUDAToolkit REQUIRED) +find_package(Threads REQUIRED) + +set(BUILD_TESTING OFF CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_PYBIND ON CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_EXTENSION_MODULE ON CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER ON CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR ON CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP ON CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_EXTENSION_TENSOR ON CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_KERNELS_OPTIMIZED OFF CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_XNNPACK OFF CACHE BOOL "" FORCE) +set(EXECUTORCH_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +add_subdirectory("${EXECUTORCH_SOURCE_DIR}" executorch) + +add_library(torch_tensorrt_executorch_backend STATIC + "${TORCH_TENSORRT_SOURCE_DIR}/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp" + "${TORCH_TENSORRT_SOURCE_DIR}/cpp/src/torch_tensorrt/executorch/TensorRTBlobHeader.cpp") +target_compile_features(torch_tensorrt_executorch_backend PUBLIC cxx_std_17) +target_compile_definitions(torch_tensorrt_executorch_backend + PRIVATE C10_USING_CUSTOM_GENERATED_MACROS) +target_include_directories(torch_tensorrt_executorch_backend PRIVATE + "${TORCH_TENSORRT_SOURCE_DIR}" + "${TORCH_TENSORRT_SOURCE_DIR}/cpp/include" + "${EXECUTORCH_SOURCE_DIR}/.." + "${EXECUTORCH_SOURCE_DIR}/runtime/core/portable_type/c10") +target_link_libraries(torch_tensorrt_executorch_backend PRIVATE + CUDA::cudart TensorRT::nvinfer Threads::Threads executorch) + +if(NOT TARGET portable_lib) + message(FATAL_ERROR "ExecuTorch did not define portable_lib") +endif() + +# ExecuTorch portable c10 delegates selected headers to torch/headeronly. The +# pybind bridge must resolve those delegated headers from the exact PyTorch +# wheel it links, not ExecuTorch's vendored snapshot. +file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/torch_header_shim/torch") +file(CREATE_LINK + "${TORCH_PRIMARY_INCLUDE_DIR}/torch/headeronly" + "${CMAKE_BINARY_DIR}/torch_header_shim/torch/headeronly" + SYMBOLIC COPY_ON_ERROR) +target_include_directories(util BEFORE PRIVATE "${CMAKE_BINARY_DIR}/torch_header_shim") +target_include_directories(portable_lib BEFORE PRIVATE "${CMAKE_BINARY_DIR}/torch_header_shim") + +# TensorRTBackend registers through a static initializer. Force its complete +# archive into the same pybind module that owns ExecuTorch's backend registry. +target_link_libraries(portable_lib PRIVATE + "$" + extension_threadpool + CUDA::cudart TensorRT::nvinfer Threads::Threads) +set_target_properties(portable_lib PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${DELEGATE_LIBRARY_OUTPUT_DIRECTORY}" + OUTPUT_NAME "_portable_lib") +set_target_properties(data_loader PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${DELEGATE_LIBRARY_OUTPUT_DIRECTORY}") + +add_custom_target(torch_tensorrt_executorch_portable_lib DEPENDS portable_lib data_loader) diff --git a/py/torch-tensorrt-executorch-delegate/pyproject.toml b/py/torch-tensorrt-executorch-delegate/pyproject.toml new file mode 100644 index 0000000000..90cff97b26 --- /dev/null +++ b/py/torch-tensorrt-executorch-delegate/pyproject.toml @@ -0,0 +1,9 @@ +[build-system] +requires = [ + "setuptools>=68", + "wheel>=0.40", + # Intentionally supplied by the active Torch-TensorRT build environment. + # Builds must use --no-build-isolation; see README.md. + "torch", +] +build-backend = "setuptools.build_meta" diff --git a/py/torch-tensorrt-executorch-delegate/setup.py b/py/torch-tensorrt-executorch-delegate/setup.py new file mode 100644 index 0000000000..83a645a721 --- /dev/null +++ b/py/torch-tensorrt-executorch-delegate/setup.py @@ -0,0 +1,109 @@ +"""Build the precompiled Torch-TensorRT backend for ExecuTorch Python.""" + +from __future__ import annotations + +import importlib.metadata +import os +import pathlib +import re +import subprocess +import sys + +import torch +from torch.utils.cpp_extension import include_paths +from setuptools import Extension, find_packages, setup +from setuptools.command.build_ext import build_ext + +HERE = pathlib.Path(__file__).resolve().parent +REPO_ROOT = HERE.parents[1] + +def torchtrt_version() -> str: + if value := os.getenv("TORCH_TENSORRT_EXECUTORCH_DELEGATE_VERSION"): + return value + version_py = REPO_ROOT / "py/torch_tensorrt/_version.py" + if version_py.exists(): + if m := re.search(r'__version__\s*=\s*["\']([^"\']+)', version_py.read_text()): + return m.group(1) + return (REPO_ROOT / "version.txt").read_text().strip() + + +class CMakeExtension(Extension): + def __init__(self, name: str) -> None: + super().__init__(name, sources=[]) + + +class CMakeBuild(build_ext): + def build_extension(self, ext: Extension) -> None: + if sys.platform != "linux": + raise RuntimeError("The ExecuTorch TensorRT delegate supports Linux only") + executorch_source = os.getenv("EXECUTORCH_SOURCE_DIR") + if not executorch_source: + raise RuntimeError( + "Set EXECUTORCH_SOURCE_DIR to the pinned ExecuTorch source tree" + ) + + output = pathlib.Path(self.get_ext_fullpath(ext.name)).resolve() + if output.exists(): + return + build_dir = pathlib.Path(self.build_temp) / "executorch_delegate_native" + output.parent.mkdir(parents=True, exist_ok=True) + build_dir.mkdir(parents=True, exist_ok=True) + configure = [ + "cmake", + "-S", + str(HERE / "native"), + "-B", + str(build_dir), + f"-DEXECUTORCH_SOURCE_DIR={pathlib.Path(executorch_source).resolve()}", + f"-DTORCH_TENSORRT_SOURCE_DIR={REPO_ROOT}", + f"-DPYTHON_EXECUTABLE={sys.executable}", + f"-DTORCH_PRIMARY_INCLUDE_DIR={include_paths()[0]}", + f"-DDELEGATE_LIBRARY_OUTPUT_DIRECTORY={output.parent}", + f"-DCMAKE_BUILD_TYPE={'Debug' if self.debug else 'Release'}", + ] + if value := os.getenv("TensorRT_ROOT"): + configure.append(f"-DTensorRT_ROOT={value}") + configure.extend(os.getenv("CMAKE_ARGS", "").split()) + subprocess.run(configure, check=True) + subprocess.run( + [ + "cmake", + "--build", + str(build_dir), + "--target", + "torch_tensorrt_executorch_portable_lib", + "--parallel", + str(self.parallel or os.cpu_count() or 1), + ], + check=True, + ) + library_stem = ( + "_portable_lib" if ext.name.endswith("._portable_lib") else "data_loader" + ) + built = next(output.parent.glob(f"{library_stem}*.so"), None) + if built is None: + raise RuntimeError( + f"CMake did not produce {library_stem} in {output.parent}" + ) + if built != output: + built.replace(output) + + +executorch_version = importlib.metadata.version("executorch") +setup( + name="torch-tensorrt-executorch-delegate", + version=torchtrt_version(), + description="Torch-TensorRT delegate for the ExecuTorch Python runtime", + packages=find_packages(), + ext_modules=[ + CMakeExtension("torch_tensorrt_executorch_delegate._portable_lib"), + CMakeExtension("torch_tensorrt_executorch_delegate.data_loader"), + ], + cmdclass={"build_ext": CMakeBuild}, + python_requires=">=3.10", + install_requires=[ + f"torch=={torch.__version__}", + f"executorch=={executorch_version}", + ], + zip_safe=False, +) diff --git a/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py b/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py new file mode 100644 index 0000000000..343873d789 --- /dev/null +++ b/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py @@ -0,0 +1,62 @@ +"""Activate the TensorRT delegate-enabled ExecuTorch Python runtime.""" + +from __future__ import annotations + +import importlib +import sys +from types import ModuleType + +BACKEND_NAME = "TensorRTBackend" +_NATIVE_NAME = "executorch.extension.pybindings._portable_lib" +_WRAPPER_NAME = "executorch.extension.pybindings.portable_lib" +_DATA_LOADER_NAME = "executorch.extension.pybindings.data_loader" + + +class DelegateCompatibilityError(ImportError): + """The delegate wheel is incompatible with the active native runtime.""" + + +def activate() -> ModuleType: + """Make the delegate-enabled portable runtime back ``executorch.runtime``.""" + existing = sys.modules.get(_NATIVE_NAME) + if existing is not None and existing.__name__ == __name__ + "._portable_lib": + return existing + if existing is not None or _WRAPPER_NAME in sys.modules: + raise DelegateCompatibilityError( + "ExecuTorch's stock runtime was imported first. Call " + "torch_tensorrt.executorch.load(...) before importing " + "executorch.runtime." + ) + missing = object() + previous_data_loader = sys.modules.get(_DATA_LOADER_NAME, missing) + try: + data_loader = importlib.import_module(__name__ + ".data_loader") + native = importlib.import_module(__name__ + "._portable_lib") + except ImportError as error: + if previous_data_loader is missing: + sys.modules.pop(_DATA_LOADER_NAME, None) + else: + sys.modules[_DATA_LOADER_NAME] = previous_data_loader + raise DelegateCompatibilityError( + "Could not load the prebuilt Torch-TensorRT ExecuTorch runtime. " + "Install torch, executorch, torch-tensorrt, and the delegate from " + "the same release matrix." + ) from error + sys.modules[_DATA_LOADER_NAME] = data_loader + sys.modules[_NATIVE_NAME] = native + sys.modules.pop(_WRAPPER_NAME, None) + return native + + +def runtime(): + """Return the activated ExecuTorch Runtime singleton.""" + activate() + from executorch.runtime import Runtime + + value = Runtime.get() + if not value.backend_registry.is_available(BACKEND_NAME): + raise DelegateCompatibilityError(f"{BACKEND_NAME} is not registered") + return value + + +__all__ = ["BACKEND_NAME", "DelegateCompatibilityError", "activate", "runtime"] diff --git a/tests/py/dynamo/executorch/test_python_runtime.py b/tests/py/dynamo/executorch/test_python_runtime.py new file mode 100644 index 0000000000..648607fa54 --- /dev/null +++ b/tests/py/dynamo/executorch/test_python_runtime.py @@ -0,0 +1,150 @@ +import importlib.util +import sys +import types +from pathlib import Path + +import pytest + +RUNTIME_PATH = Path(__file__).parents[4] / "py/torch_tensorrt/executorch/runtime.py" +DELEGATE_PATH = ( + Path(__file__).parents[4] + / "py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py" +) + + +def load_runtime_module(): + spec = importlib.util.spec_from_file_location( + "torchtrt_et_runtime_test", RUNTIME_PATH + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_delegate_module(): + spec = importlib.util.spec_from_file_location( + "torchtrt_et_delegate_test", DELEGATE_PATH + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class FakeMethod: + def execute(self, inputs): + return [inputs[0] + 1] + + +class FakeProgram: + method_names = {"forward"} + + def load_method(self, name): + return FakeMethod() if name == "forward" else None + + +class FakeRuntime: + def load_program(self, data): + self.data = data + return FakeProgram() + + +def test_load_and_forward(monkeypatch, tmp_path): + delegate = types.ModuleType("torch_tensorrt_executorch_delegate") + fake_runtime = FakeRuntime() + delegate.runtime = lambda: fake_runtime + monkeypatch.setitem(sys.modules, delegate.__name__, delegate) + model = tmp_path / "model.pte" + model.write_bytes(b"pte") + + program = load_runtime_module().load(model) + + assert program.forward(2) == [3] + assert program._data is fake_runtime.data + + +def test_unknown_method(monkeypatch, tmp_path): + delegate = types.ModuleType("torch_tensorrt_executorch_delegate") + delegate.runtime = FakeRuntime + monkeypatch.setitem(sys.modules, delegate.__name__, delegate) + model = tmp_path / "model.pte" + model.write_bytes(b"pte") + + with pytest.raises(ValueError, match="Unknown method"): + load_runtime_module().load(model).run([], "missing") + + +def test_missing_model(): + with pytest.raises(FileNotFoundError): + load_runtime_module().load("does-not-exist.pte") + + +def test_activate_twice_is_safe(monkeypatch): + delegate = load_delegate_module() + data_loader = types.ModuleType(delegate.__name__ + ".data_loader") + native = types.ModuleType(delegate.__name__ + "._portable_lib") + imported = [] + + def fake_import(name): + imported.append(name) + return { + data_loader.__name__: data_loader, + native.__name__: native, + }[name] + + monkeypatch.setattr( + delegate, "importlib", types.SimpleNamespace(import_module=fake_import) + ) + assert delegate.activate() is native + wrapper = types.ModuleType(delegate._WRAPPER_NAME) + monkeypatch.setitem(sys.modules, delegate._WRAPPER_NAME, wrapper) + assert delegate.activate() is native + assert imported == [data_loader.__name__, native.__name__] + assert sys.modules[delegate._NATIVE_NAME] is native + assert sys.modules[delegate._DATA_LOADER_NAME] is data_loader + assert sys.modules[delegate._WRAPPER_NAME] is wrapper + + +def test_activate_rejects_preloaded_stock_runtime(monkeypatch): + delegate = load_delegate_module() + stock_runtime = types.ModuleType(delegate._NATIVE_NAME) + monkeypatch.setitem(sys.modules, delegate._NATIVE_NAME, stock_runtime) + + with pytest.raises(delegate.DelegateCompatibilityError, match="stock runtime"): + delegate.activate() + + +def test_activate_rejects_preloaded_stock_wrapper(monkeypatch): + delegate = load_delegate_module() + stock_wrapper = types.ModuleType(delegate._WRAPPER_NAME) + monkeypatch.delitem(sys.modules, delegate._NATIVE_NAME, raising=False) + monkeypatch.setitem(sys.modules, delegate._WRAPPER_NAME, stock_wrapper) + + with pytest.raises( + delegate.DelegateCompatibilityError, + match=r"Call torch_tensorrt\.executorch\.load", + ): + delegate.activate() + + +def test_activate_cleans_up_data_loader_when_native_import_fails(monkeypatch): + delegate = load_delegate_module() + data_loader = types.ModuleType(delegate.__name__ + ".data_loader") + + def fake_import(name): + if name == data_loader.__name__: + return data_loader + assert delegate._DATA_LOADER_NAME not in sys.modules + raise ImportError("native module failed to load") + + monkeypatch.setattr( + delegate, "importlib", types.SimpleNamespace(import_module=fake_import) + ) + monkeypatch.delitem(sys.modules, delegate._NATIVE_NAME, raising=False) + monkeypatch.delitem(sys.modules, delegate._DATA_LOADER_NAME, raising=False) + + with pytest.raises(delegate.DelegateCompatibilityError): + delegate.activate() + + assert delegate._DATA_LOADER_NAME not in sys.modules From 680b0f4eca3215ceef577cda44f4b44442fe73e6 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Mon, 27 Jul 2026 09:36:36 -0700 Subject: [PATCH 06/17] change cmake build to bazel build --- .../executorch/TensorRTBackend.cpp | 27 +++++- .../native/BUILD.bazel | 43 +++++++++ .../native/CMakeLists.txt | 24 +++-- .../setup.py | 92 +++++++++---------- 4 files changed, 129 insertions(+), 57 deletions(-) create mode 100644 py/torch-tensorrt-executorch-delegate/native/BUILD.bazel diff --git a/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp b/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp index b2e3b08232..718117f2ff 100644 --- a/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp +++ b/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp @@ -51,6 +51,16 @@ using ::executorch::runtime::Span; namespace { thread_local cudaStream_t g_user_stream = nullptr; thread_local bool g_user_stream_set = false; + +extern const Error kRegistrationResult; + +Error check_registration() { + if (kRegistrationResult != Error::Ok) { + ET_LOG( + Error, "TensorRTBackend registration failed: %s", ::executorch::runtime::error_to_string(kRegistrationResult)); + } + return kRegistrationResult; +} } // namespace CudaStreamGuard::CudaStreamGuard(cudaStream_t stream) : prev_stream_(g_user_stream), prev_set_(g_user_stream_set) { @@ -209,6 +219,10 @@ bool is_cuda_accessible_ptr(const void* ptr) { // is_available // --------------------------------------------------------------------------- bool TensorRTBackend::is_available() const { + if (check_registration() != Error::Ok) { + return false; + } + TRTLogger logger; TRTUniquePtr runtime(nvinfer1::createInferRuntime(logger)); return runtime != nullptr; @@ -227,6 +241,11 @@ Result TensorRTBackend::init( ArrayRef compile_specs) const { (void)compile_specs; + const Error registration_result = check_registration(); + if (registration_result != Error::Ok) { + return registration_result; + } + TORCHTRT_ET_CHECK_NOT_NULL(processed, Error::InvalidArgument, "TensorRTBackend::init: null processed buffer"); TORCHTRT_ET_CHECK_NOT_NULL(processed->data(), Error::InvalidArgument, "TensorRTBackend::init: null processed buffer"); @@ -642,14 +661,18 @@ void TensorRTBackend::destroy(DelegateHandle* handle) const { // Static registration – links the name "TensorRTBackend" used in the .pte // file to this implementation at program startup. // --------------------------------------------------------------------------- +namespace torch_tensorrt { +namespace executorch_backend { namespace { -torch_tensorrt::executorch_backend::TensorRTBackend& get_backend() { +TensorRTBackend& get_backend() { static torch_tensorrt::executorch_backend::TensorRTBackend backend; return backend; } const ::executorch::runtime::Backend kBackendId{"TensorRTBackend", &get_backend()}; -const auto kRegistered = ::executorch::runtime::register_backend(kBackendId); +const Error kRegistrationResult = ::executorch::runtime::register_backend(kBackendId); } // namespace +} // namespace executorch_backend +} // namespace torch_tensorrt diff --git a/py/torch-tensorrt-executorch-delegate/native/BUILD.bazel b/py/torch-tensorrt-executorch-delegate/native/BUILD.bazel new file mode 100644 index 0000000000..3b9ed04556 --- /dev/null +++ b/py/torch-tensorrt-executorch-delegate/native/BUILD.bazel @@ -0,0 +1,43 @@ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake") + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "delegate_sources", + srcs = glob( + ["**"], + exclude = ["BUILD.bazel"], + ) + [ + "//cpp:executorch_backend_source_files", + "//:torchtrt_cmake_modules", + "@executorch//:executorch_sources", + ], +) + +# ExecuTorch exposes its Python portable runtime through CMake. Keep that +# upstream build behind Bazel so both Torch-TensorRT wheels have one build +# entry point and share Bazel's dependency/toolchain selection. +cmake( + name = "delegate_native", + cache_entries = { + "CMAKE_BUILD_TYPE": "Release", + "EXECUTORCH_CMAKE_FILE": "$(execpath @executorch//:executorch/CMakeLists.txt)", + "PYTHON_EXECUTABLE": "$${PYTHON_BIN_PATH}", + "TORCH_HEADER_MARKER": "$(execpath @libtorch//:include/torch/headeronly/util/TypeTraits.h)", + "TORCH_TENSORRT_SOURCE_DIR": "$$EXT_BUILD_ROOT$$", + }, + data = [ + "@executorch//:executorch/CMakeLists.txt", + "@libtorch//:include/torch/headeronly/util/TypeTraits.h", + ], + deps = [ + "@cuda//:cudart", + "@tensorrt//:nvinfer", + ], + lib_source = ":delegate_sources", + out_include_dir = "", + out_shared_libs = [ + "_portable_lib.so", + "data_loader.so", + ], +) diff --git a/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt b/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt index 1b61d3f767..2cbf6b261f 100644 --- a/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt +++ b/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt @@ -1,8 +1,15 @@ cmake_minimum_required(VERSION 3.24) project(torch_tensorrt_executorch_delegate LANGUAGES CXX) -if(NOT EXECUTORCH_SOURCE_DIR OR NOT TORCH_TENSORRT_SOURCE_DIR) - message(FATAL_ERROR "EXECUTORCH_SOURCE_DIR and TORCH_TENSORRT_SOURCE_DIR are required") +if(NOT EXECUTORCH_SOURCE_DIR AND EXECUTORCH_CMAKE_FILE) + get_filename_component(EXECUTORCH_SOURCE_DIR "${EXECUTORCH_CMAKE_FILE}" DIRECTORY) +endif() +if(NOT TORCH_PRIMARY_INCLUDE_DIR AND TORCH_HEADER_MARKER) + get_filename_component(TORCH_PRIMARY_INCLUDE_DIR "${TORCH_HEADER_MARKER}/../../../.." ABSOLUTE) +endif() +if(NOT EXECUTORCH_SOURCE_DIR OR NOT TORCH_TENSORRT_SOURCE_DIR OR NOT TORCH_PRIMARY_INCLUDE_DIR) + message(FATAL_ERROR + "EXECUTORCH_SOURCE_DIR, TORCH_TENSORRT_SOURCE_DIR, and TORCH_PRIMARY_INCLUDE_DIR are required") endif() list(APPEND CMAKE_MODULE_PATH "${TORCH_TENSORRT_SOURCE_DIR}/cmake/Modules") @@ -45,6 +52,10 @@ endif() # ExecuTorch portable c10 delegates selected headers to torch/headeronly. The # pybind bridge must resolve those delegated headers from the exact PyTorch # wheel it links, not ExecuTorch's vendored snapshot. +if(NOT EXISTS "${TORCH_PRIMARY_INCLUDE_DIR}/torch/headeronly") + message(FATAL_ERROR + "torch/headeronly not found under TORCH_PRIMARY_INCLUDE_DIR=${TORCH_PRIMARY_INCLUDE_DIR}") +endif() file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/torch_header_shim/torch") file(CREATE_LINK "${TORCH_PRIMARY_INCLUDE_DIR}/torch/headeronly" @@ -60,9 +71,10 @@ target_link_libraries(portable_lib PRIVATE extension_threadpool CUDA::cudart TensorRT::nvinfer Threads::Threads) set_target_properties(portable_lib PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${DELEGATE_LIBRARY_OUTPUT_DIRECTORY}" - OUTPUT_NAME "_portable_lib") -set_target_properties(data_loader PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${DELEGATE_LIBRARY_OUTPUT_DIRECTORY}") + OUTPUT_NAME "_portable_lib" + SUFFIX ".so") +set_target_properties(data_loader PROPERTIES SUFFIX ".so") + +install(TARGETS portable_lib data_loader LIBRARY DESTINATION lib) add_custom_target(torch_tensorrt_executorch_portable_lib DEPENDS portable_lib data_loader) diff --git a/py/torch-tensorrt-executorch-delegate/setup.py b/py/torch-tensorrt-executorch-delegate/setup.py index 83a645a721..f6e7f259bc 100644 --- a/py/torch-tensorrt-executorch-delegate/setup.py +++ b/py/torch-tensorrt-executorch-delegate/setup.py @@ -6,16 +6,19 @@ import os import pathlib import re +import shlex +import shutil import subprocess import sys import torch -from torch.utils.cpp_extension import include_paths from setuptools import Extension, find_packages, setup from setuptools.command.build_ext import build_ext HERE = pathlib.Path(__file__).resolve().parent REPO_ROOT = HERE.parents[1] +BAZEL_TARGET = "//py/torch-tensorrt-executorch-delegate/native:delegate_native" + def torchtrt_version() -> str: if value := os.getenv("TORCH_TENSORRT_EXECUTORCH_DELEGATE_VERSION"): @@ -27,66 +30,57 @@ def torchtrt_version() -> str: return (REPO_ROOT / "version.txt").read_text().strip() -class CMakeExtension(Extension): +class BazelExtension(Extension): def __init__(self, name: str) -> None: super().__init__(name, sources=[]) -class CMakeBuild(build_ext): +class BazelBuild(build_ext): def build_extension(self, ext: Extension) -> None: if sys.platform != "linux": raise RuntimeError("The ExecuTorch TensorRT delegate supports Linux only") - executorch_source = os.getenv("EXECUTORCH_SOURCE_DIR") - if not executorch_source: - raise RuntimeError( - "Set EXECUTORCH_SOURCE_DIR to the pinned ExecuTorch source tree" - ) output = pathlib.Path(self.get_ext_fullpath(ext.name)).resolve() - if output.exists(): - return - build_dir = pathlib.Path(self.build_temp) / "executorch_delegate_native" output.parent.mkdir(parents=True, exist_ok=True) - build_dir.mkdir(parents=True, exist_ok=True) - configure = [ - "cmake", - "-S", - str(HERE / "native"), - "-B", - str(build_dir), - f"-DEXECUTORCH_SOURCE_DIR={pathlib.Path(executorch_source).resolve()}", - f"-DTORCH_TENSORRT_SOURCE_DIR={REPO_ROOT}", - f"-DPYTHON_EXECUTABLE={sys.executable}", - f"-DTORCH_PRIMARY_INCLUDE_DIR={include_paths()[0]}", - f"-DDELEGATE_LIBRARY_OUTPUT_DIRECTORY={output.parent}", - f"-DCMAKE_BUILD_TYPE={'Debug' if self.debug else 'Release'}", + + bazel = shutil.which("bazelisk") or shutil.which("bazel") + if bazel is None: + raise RuntimeError("Could not find bazelisk or bazel in PATH") + + command = [ + bazel, + "build", + BAZEL_TARGET, + "--config=linux", + "--config=python", + f"--compilation_mode={'dbg' if self.debug else 'opt'}", + f"--action_env=PYTHON_BIN_PATH={sys.executable}", ] - if value := os.getenv("TensorRT_ROOT"): - configure.append(f"-DTensorRT_ROOT={value}") - configure.extend(os.getenv("CMAKE_ARGS", "").split()) - subprocess.run(configure, check=True) - subprocess.run( - [ - "cmake", - "--build", - str(build_dir), - "--target", - "torch_tensorrt_executorch_portable_lib", - "--parallel", - str(self.parallel or os.cpu_count() or 1), - ], - check=True, + dist_dir = REPO_ROOT / "third_party/dist_dir/x86_64-linux-gnu" + if dist_dir.is_dir(): + command.append(f"--distdir={dist_dir}") + command.extend(shlex.split(os.getenv("BAZEL_ARGS", ""))) + + env = os.environ.copy() + env.setdefault("TORCH_PATH", str(pathlib.Path(torch.__file__).resolve().parent)) + subprocess.run(command, cwd=REPO_ROOT, env=env, check=True) + + bazel_bin = pathlib.Path( + subprocess.check_output( + [bazel, "info", "bazel-bin"], cwd=REPO_ROOT, env=env, text=True + ).strip() ) library_stem = ( "_portable_lib" if ext.name.endswith("._portable_lib") else "data_loader" ) - built = next(output.parent.glob(f"{library_stem}*.so"), None) - if built is None: - raise RuntimeError( - f"CMake did not produce {library_stem} in {output.parent}" - ) - if built != output: - built.replace(output) + built = ( + bazel_bin + / "py/torch-tensorrt-executorch-delegate/native/delegate_native/lib" + / f"{library_stem}.so" + ) + if not built.is_file(): + raise RuntimeError(f"Bazel did not produce {built}") + shutil.copy2(built, output) executorch_version = importlib.metadata.version("executorch") @@ -96,10 +90,10 @@ def build_extension(self, ext: Extension) -> None: description="Torch-TensorRT delegate for the ExecuTorch Python runtime", packages=find_packages(), ext_modules=[ - CMakeExtension("torch_tensorrt_executorch_delegate._portable_lib"), - CMakeExtension("torch_tensorrt_executorch_delegate.data_loader"), + BazelExtension("torch_tensorrt_executorch_delegate._portable_lib"), + BazelExtension("torch_tensorrt_executorch_delegate.data_loader"), ], - cmdclass={"build_ext": CMakeBuild}, + cmdclass={"build_ext": BazelBuild}, python_requires=">=3.10", install_requires=[ f"torch=={torch.__version__}", From e77dcbede6fb9642af8bcb058a56a3e4a50e2125 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Mon, 27 Jul 2026 11:49:51 -0700 Subject: [PATCH 07/17] resolve comments: seperate the save and load in the different package --- BUILD.bazel | 8 +++ MODULE.bazel | 7 +- .../executorch/TensorRTBackend.cpp | 2 +- .../README.md | 7 +- .../native/BUILD.bazel | 3 + .../native/CMakeLists.txt | 18 +++-- .../pyproject.toml | 4 +- .../setup.py | 15 +++- .../__init__.py | 6 +- .../runtime.py | 8 +-- py/torch_tensorrt/_compile.py | 42 +++++++++-- py/torch_tensorrt/executorch/__init__.py | 10 ++- setup.py | 5 +- tests/py/dynamo/executorch/test_api.py | 71 +++++++++++++++---- .../dynamo/executorch/test_python_runtime.py | 11 +-- 15 files changed, 166 insertions(+), 51 deletions(-) rename py/{torch_tensorrt/executorch => torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate}/runtime.py (91%) diff --git a/BUILD.bazel b/BUILD.bazel index a0123ff3ee..e234f30d1b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -53,6 +53,14 @@ config_setting( }, ) +filegroup( + name = "torchtrt_cmake_modules", + srcs = [ + "cmake/Modules/FindTensorRT.cmake", + ], + visibility = ["//visibility:public"], +) + pkg_tar( name = "include_core", package_dir = "include/torch_tensorrt", diff --git a/MODULE.bazel b/MODULE.bazel index 99c22600bc..12895c9cef 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -42,12 +42,13 @@ new_git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl" local_torch = use_repo_rule("//toolchains:local_torch.bzl", "local_torch") -# Pinned to ExecuTorch main for PyTorch 2.14 native ABI compatibility. +# Keep this pin synchronized with the ExecuTorch release installed in +# py/torch-tensorrt-executorch-delegate/README.md. new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # PyTorch 2.14-compatible ExecuTorch main commit - commit = "a6d812a082df57898b8608f56c867140cc9da32c", + # executorch==1.3.1 + commit = "e2f18eb23c45bd22ca332b0b8b49a81de304b472", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;", diff --git a/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp b/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp index 718117f2ff..648fb88232 100644 --- a/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp +++ b/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp @@ -57,7 +57,7 @@ extern const Error kRegistrationResult; Error check_registration() { if (kRegistrationResult != Error::Ok) { ET_LOG( - Error, "TensorRTBackend registration failed: %s", ::executorch::runtime::error_to_string(kRegistrationResult)); + Error, "TensorRTBackend registration failed: %s", ::executorch::runtime::to_string(kRegistrationResult)); } return kRegistrationResult; } diff --git a/py/torch-tensorrt-executorch-delegate/README.md b/py/torch-tensorrt-executorch-delegate/README.md index c2f93dcb19..2f46960b0c 100644 --- a/py/torch-tensorrt-executorch-delegate/README.md +++ b/py/torch-tensorrt-executorch-delegate/README.md @@ -26,6 +26,9 @@ python -m pip wheel --no-build-isolation --no-deps \ --wheel-dir dist py/torch-tensorrt-executorch-delegate ``` +The ExecuTorch source commit pinned in `MODULE.bazel` is the revision recorded +by the `executorch==1.3.1` wheel. + The static ExecuTorch and delegate archives are intermediate build inputs; users receive the final native Python module and do not compile anything. @@ -37,8 +40,8 @@ pip install "torch-tensorrt[executorch]" ```python import torch -from torch_tensorrt.executorch.runtime import load +import torch_tensorrt -program = load("model.pte") +program = torch_tensorrt.load("model.pte", format="executorch") outputs = program.forward(torch.ones((2, 3, 4, 4))) ``` diff --git a/py/torch-tensorrt-executorch-delegate/native/BUILD.bazel b/py/torch-tensorrt-executorch-delegate/native/BUILD.bazel index 3b9ed04556..6c63942591 100644 --- a/py/torch-tensorrt-executorch-delegate/native/BUILD.bazel +++ b/py/torch-tensorrt-executorch-delegate/native/BUILD.bazel @@ -8,6 +8,8 @@ filegroup( ["**"], exclude = ["BUILD.bazel"], ) + [ + "//core/runtime:include_files", + "//cpp:executorch_api_headers", "//cpp:executorch_backend_source_files", "//:torchtrt_cmake_modules", "@executorch//:executorch_sources", @@ -32,6 +34,7 @@ cmake( ], deps = [ "@cuda//:cudart", + "@libtorch//:torch", "@tensorrt//:nvinfer", ], lib_source = ":delegate_sources", diff --git a/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt b/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt index 2cbf6b261f..1e2b8bc24d 100644 --- a/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt +++ b/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt @@ -49,18 +49,22 @@ if(NOT TARGET portable_lib) message(FATAL_ERROR "ExecuTorch did not define portable_lib") endif() -# ExecuTorch portable c10 delegates selected headers to torch/headeronly. The -# pybind bridge must resolve those delegated headers from the exact PyTorch -# wheel it links, not ExecuTorch's vendored snapshot. +# The pybind bridge includes both ATen and portable ExecuTorch headers. Keep the +# ATen/c10/torch header set internally consistent by resolving all of it from +# the exact PyTorch wheel that supplies the linked libraries. if(NOT EXISTS "${TORCH_PRIMARY_INCLUDE_DIR}/torch/headeronly") message(FATAL_ERROR "torch/headeronly not found under TORCH_PRIMARY_INCLUDE_DIR=${TORCH_PRIMARY_INCLUDE_DIR}") endif() file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/torch_header_shim/torch") -file(CREATE_LINK - "${TORCH_PRIMARY_INCLUDE_DIR}/torch/headeronly" - "${CMAKE_BINARY_DIR}/torch_header_shim/torch/headeronly" - SYMBOLIC COPY_ON_ERROR) +file(COPY "${TORCH_PRIMARY_INCLUDE_DIR}/c10/" + DESTINATION "${CMAKE_BINARY_DIR}/torch_header_shim/c10") +file(COPY "${TORCH_PRIMARY_INCLUDE_DIR}/torch/headeronly/" + DESTINATION "${CMAKE_BINARY_DIR}/torch_header_shim/torch/headeronly") +if(NOT EXISTS "${CMAKE_BINARY_DIR}/torch_header_shim/c10/util/complex.h" OR + NOT EXISTS "${CMAKE_BINARY_DIR}/torch_header_shim/torch/headeronly/macros/Macros.h") + message(FATAL_ERROR "Failed to stage the PyTorch header shim") +endif() target_include_directories(util BEFORE PRIVATE "${CMAKE_BINARY_DIR}/torch_header_shim") target_include_directories(portable_lib BEFORE PRIVATE "${CMAKE_BINARY_DIR}/torch_header_shim") diff --git a/py/torch-tensorrt-executorch-delegate/pyproject.toml b/py/torch-tensorrt-executorch-delegate/pyproject.toml index 90cff97b26..a500209126 100644 --- a/py/torch-tensorrt-executorch-delegate/pyproject.toml +++ b/py/torch-tensorrt-executorch-delegate/pyproject.toml @@ -2,8 +2,10 @@ requires = [ "setuptools>=68", "wheel>=0.40", - # Intentionally supplied by the active Torch-TensorRT build environment. + # Native ABI dependencies supplied by the active Torch-TensorRT build + # environment. # Builds must use --no-build-isolation; see README.md. "torch", + "executorch==1.3.1", ] build-backend = "setuptools.build_meta" diff --git a/py/torch-tensorrt-executorch-delegate/setup.py b/py/torch-tensorrt-executorch-delegate/setup.py index f6e7f259bc..8e39c8bcc6 100644 --- a/py/torch-tensorrt-executorch-delegate/setup.py +++ b/py/torch-tensorrt-executorch-delegate/setup.py @@ -47,13 +47,14 @@ def build_extension(self, ext: Extension) -> None: if bazel is None: raise RuntimeError("Could not find bazelisk or bazel in PATH") + compilation_mode = "dbg" if self.debug else "opt" command = [ bazel, "build", BAZEL_TARGET, "--config=linux", "--config=python", - f"--compilation_mode={'dbg' if self.debug else 'opt'}", + f"--compilation_mode={compilation_mode}", f"--action_env=PYTHON_BIN_PATH={sys.executable}", ] dist_dir = REPO_ROOT / "third_party/dist_dir/x86_64-linux-gnu" @@ -67,7 +68,15 @@ def build_extension(self, ext: Extension) -> None: bazel_bin = pathlib.Path( subprocess.check_output( - [bazel, "info", "bazel-bin"], cwd=REPO_ROOT, env=env, text=True + [ + bazel, + "info", + "bazel-bin", + f"--compilation_mode={compilation_mode}", + ], + cwd=REPO_ROOT, + env=env, + text=True, ).strip() ) library_stem = ( @@ -80,6 +89,7 @@ def build_extension(self, ext: Extension) -> None: ) if not built.is_file(): raise RuntimeError(f"Bazel did not produce {built}") + output.unlink(missing_ok=True) shutil.copy2(built, output) @@ -98,6 +108,7 @@ def build_extension(self, ext: Extension) -> None: install_requires=[ f"torch=={torch.__version__}", f"executorch=={executorch_version}", + f"torch-tensorrt=={torchtrt_version()}", ], zip_safe=False, ) diff --git a/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py b/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py index 343873d789..44ce05be92 100644 --- a/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py +++ b/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py @@ -24,7 +24,7 @@ def activate() -> ModuleType: if existing is not None or _WRAPPER_NAME in sys.modules: raise DelegateCompatibilityError( "ExecuTorch's stock runtime was imported first. Call " - "torch_tensorrt.executorch.load(...) before importing " + 'torch_tensorrt.load(..., format="executorch") before importing ' "executorch.runtime." ) missing = object() @@ -48,7 +48,7 @@ def activate() -> ModuleType: return native -def runtime(): +def get_runtime(): """Return the activated ExecuTorch Runtime singleton.""" activate() from executorch.runtime import Runtime @@ -59,4 +59,4 @@ def runtime(): return value -__all__ = ["BACKEND_NAME", "DelegateCompatibilityError", "activate", "runtime"] +__all__ = ["BACKEND_NAME", "DelegateCompatibilityError", "activate", "get_runtime"] diff --git a/py/torch_tensorrt/executorch/runtime.py b/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/runtime.py similarity index 91% rename from py/torch_tensorrt/executorch/runtime.py rename to py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/runtime.py index fb78b54d77..948a231089 100644 --- a/py/torch_tensorrt/executorch/runtime.py +++ b/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/runtime.py @@ -6,15 +6,15 @@ from typing import Any, Sequence, Union -def _runtime(): +def _get_runtime(): try: - from torch_tensorrt_executorch_delegate import runtime + from torch_tensorrt_executorch_delegate import get_runtime except ImportError as error: raise ImportError( "ExecuTorch Python inference requires the prebuilt delegate. " 'Install it with: pip install "torch-tensorrt[executorch]"' ) from error - return runtime() + return get_runtime() class Program: @@ -55,7 +55,7 @@ def load(path: Union[str, Path]) -> Program: if not model_path.is_file(): raise FileNotFoundError(f"ExecuTorch model not found: {model_path}") data = model_path.read_bytes() - return Program(_runtime().load_program(data), data) + return Program(_get_runtime().load_program(data), data) __all__ = ["Program", "load"] diff --git a/py/torch_tensorrt/_compile.py b/py/torch_tensorrt/_compile.py index a4d1feaabf..9b35e9178e 100644 --- a/py/torch_tensorrt/_compile.py +++ b/py/torch_tensorrt/_compile.py @@ -89,6 +89,15 @@ def _has_executorch_exir() -> bool: return False +def _has_executorch_delegate() -> bool: + try: + return ( + importlib.util.find_spec("torch_tensorrt_executorch_delegate") is not None + ) + except ModuleNotFoundError: + return False + + def _non_fx_input_interface( inputs: Sequence[Input | torch.Tensor], ) -> TypeGuard[List[Input | torch.Tensor]]: @@ -587,16 +596,23 @@ def load_cross_compiled_exported_program(file_path: str = "") -> Any: def load( - file_path: str = "", extra_files: Optional[dict[str, Any]] = None, **kwargs: Any + file_path: str = "", + extra_files: Optional[dict[str, Any]] = None, + *, + format: Optional[str] = None, + **kwargs: Any, ) -> Any: """ - Load either a Torchscript model or ExportedProgram. + Load a TorchScript, ExportedProgram, or ExecuTorch program. - Loads a TorchScript or ExportedProgram file from disk. File type will be detect the type using try, except. + By default, detects TorchScript and ExportedProgram files. Set + ``format="executorch"`` explicitly for an ExecuTorch ``.pte`` file. Arguments: file_path (str): Path to file on the disk extra_files (dict[str, Any]): Extra files to load with the model + format (Optional[str]): Set to ``"executorch"`` to load a ``.pte`` file + using the separately installed ExecuTorch delegate package. Example: # Load with extra files. @@ -605,8 +621,26 @@ def load( print(extra_files["foo.txt"]) Raises: - ValueError: If there is no file or the file is not either a TorchScript file or ExportedProgram file + ImportError: If ExecuTorch format is requested without the delegate package + ValueError: If the format is unsupported or the file is not a TorchScript or ExportedProgram file """ + if format == "executorch": + if not _has_executorch_delegate(): + raise ImportError( + "Loading an ExecuTorch program requires the prebuilt " + "Torch-TensorRT ExecuTorch delegate. Install it with: " + 'pip install "torch-tensorrt[executorch]"' + ) + from torch_tensorrt_executorch_delegate.runtime import ( + load as load_executorch, + ) + + return load_executorch(file_path) + if format is not None: + raise ValueError( + f"Unsupported format {format!r}; expected None or 'executorch'" + ) + # Ensure Python TRT engine ops are registered so torch.export.load can # resolve tensorrt::execute_engine when the C++ runtime is absent. if not ENABLED_FEATURES.torch_tensorrt_runtime: diff --git a/py/torch_tensorrt/executorch/__init__.py b/py/torch_tensorrt/executorch/__init__.py index a068329e8b..e9aa87d656 100644 --- a/py/torch_tensorrt/executorch/__init__.py +++ b/py/torch_tensorrt/executorch/__init__.py @@ -1,3 +1,10 @@ +"""ExecuTorch compilation and export integration. + +Runtime loading is provided by the optional +``torch-tensorrt-executorch-delegate`` distribution and dispatched through +``torch_tensorrt.load(..., format="executorch")``. +""" + import importlib from typing import TYPE_CHECKING, NoReturn @@ -29,7 +36,6 @@ def __getattr__(name: str) -> NoReturn: else: from torch_tensorrt.executorch.backend import TensorRTBackend from torch_tensorrt.executorch.partitioner import TensorRTPartitioner - from torch_tensorrt.executorch.runtime import Program, load def get_edge_compile_config() -> "EdgeCompileConfig": """Return the EdgeCompileConfig used for Torch-TensorRT ExecuTorch export.""" @@ -41,6 +47,4 @@ def get_edge_compile_config() -> "EdgeCompileConfig": "get_edge_compile_config", "TensorRTPartitioner", "TensorRTBackend", - "Program", - "load", ] diff --git a/setup.py b/setup.py index 1563d41c33..4ab3256cff 100644 --- a/setup.py +++ b/setup.py @@ -186,7 +186,7 @@ def load_dep_info(): EXECUTORCH_REQUIREMENT = "executorch>=1.3.1" EXECUTORCH_DELEGATE_REQUIREMENT = ( f"torch-tensorrt-executorch-delegate=={__version__}; " - "platform_system == 'Linux' and platform_machine == 'x86_64'" + "platform_system == 'Linux'" ) EXTRAS_REQUIRE = { "executorch": [EXECUTORCH_REQUIREMENT, EXECUTORCH_DELEGATE_REQUIREMENT], @@ -616,7 +616,6 @@ def run(self): ) package_data = {} -executorch_header_package_data = ["include/torch_tensorrt/executorch/*.h"] if not (PY_ONLY or NO_TS): tensorrt_x86_64_external_dir = ( @@ -783,7 +782,6 @@ def run(self): { "torch_tensorrt": [ "include/torch_tensorrt/*.h", - *executorch_header_package_data, "include/torch_tensorrt/core/*.h", "include/torch_tensorrt/core/conversion/*.h", "include/torch_tensorrt/core/conversion/conversionctx/*.h", @@ -813,7 +811,6 @@ def run(self): { "torch_tensorrt": [ "include/torch_tensorrt/*.h", - *executorch_header_package_data, "include/torch_tensorrt/core/*.h", "include/torch_tensorrt/core/runtime/*.h", "lib/*", diff --git a/tests/py/dynamo/executorch/test_api.py b/tests/py/dynamo/executorch/test_api.py index 5d88822716..d7782ed60c 100644 --- a/tests/py/dynamo/executorch/test_api.py +++ b/tests/py/dynamo/executorch/test_api.py @@ -53,18 +53,59 @@ def fake_find_spec(name, package=None): ) +@pytest.mark.unit +def test_load_executorch_error_when_delegate_missing(monkeypatch): + from torch_tensorrt import _compile + + monkeypatch.setattr(_compile, "_has_executorch_delegate", lambda: False) + + with pytest.raises(ImportError, match=r"torch-tensorrt\[executorch\]"): + _compile.load("model.pte", format="executorch") + + +@pytest.mark.unit +def test_load_executorch_dispatches_to_delegate(monkeypatch): + from torch_tensorrt import _compile + + delegate = types.ModuleType("torch_tensorrt_executorch_delegate") + delegate.__path__ = [] + runtime = types.ModuleType("torch_tensorrt_executorch_delegate.runtime") + sentinel = object() + runtime.load = lambda path: (sentinel, path) + monkeypatch.setitem(sys.modules, delegate.__name__, delegate) + monkeypatch.setitem(sys.modules, runtime.__name__, runtime) + monkeypatch.setattr(_compile, "_has_executorch_delegate", lambda: True) + + assert _compile.load("model.pte", format="executorch") == ( + sentinel, + "model.pte", + ) + + @pytest.mark.unit def test_public_api_symbols_present(): module = importlib.import_module("torch_tensorrt.executorch") assert "get_edge_compile_config" in module.__all__ assert "TensorRTPartitioner" in module.__all__ assert "TensorRTBackend" in module.__all__ + assert "Program" not in module.__all__ + assert "load" not in module.__all__ _REPO_ROOT = Path(__file__).resolve().parents[4] _SETUP_PY = _REPO_ROOT / "setup.py" +@pytest.mark.unit +def test_runtime_implementation_is_owned_by_delegate_package(): + assert not (_REPO_ROOT / "py/torch_tensorrt/executorch/runtime.py").exists() + assert ( + _REPO_ROOT + / "py/torch-tensorrt-executorch-delegate" + / "torch_tensorrt_executorch_delegate/runtime.py" + ).is_file() + + def _setup_tree(): return ast.parse(_SETUP_PY.read_text(encoding="utf-8")) @@ -101,11 +142,14 @@ def test_packaging_declares_executorch_extra(): assert extra_name in extras_by_name requirements = extras_by_name[extra_name] assert isinstance(requirements, ast.List) - assert any( - isinstance(requirement, ast.Name) - and requirement.id == "EXECUTORCH_REQUIREMENT" - for requirement in requirements.elts - ) + for requirement_name in ( + "EXECUTORCH_REQUIREMENT", + "EXECUTORCH_DELEGATE_REQUIREMENT", + ): + assert any( + isinstance(requirement, ast.Name) and requirement.id == requirement_name + for requirement in requirements.elts + ) setup_call = next( node @@ -134,19 +178,20 @@ def test_executorch_is_not_base_install_requirement(): ): function = _function_def(tree, function_name) assert not any( - isinstance(node, ast.Name) and node.id == "EXECUTORCH_REQUIREMENT" + isinstance(node, ast.Name) + and node.id + in { + "EXECUTORCH_REQUIREMENT", + "EXECUTORCH_DELEGATE_REQUIREMENT", + } for node in ast.walk(function) ) @pytest.mark.unit -def test_executorch_headers_are_not_dlfw_gated(): - tree = _setup_tree() - header_package_data = _assignment_value(tree, "executorch_header_package_data") - assert isinstance(header_package_data, ast.List) - assert not any( - isinstance(node, ast.Name) and node.id == "IS_DLFW_CI" - for node in ast.walk(header_package_data) +def test_main_wheel_does_not_package_executorch_delegate_headers(): + assert "include/torch_tensorrt/executorch/*.h" not in _SETUP_PY.read_text( + encoding="utf-8" ) diff --git a/tests/py/dynamo/executorch/test_python_runtime.py b/tests/py/dynamo/executorch/test_python_runtime.py index 648607fa54..66a4443ff5 100644 --- a/tests/py/dynamo/executorch/test_python_runtime.py +++ b/tests/py/dynamo/executorch/test_python_runtime.py @@ -5,7 +5,10 @@ import pytest -RUNTIME_PATH = Path(__file__).parents[4] / "py/torch_tensorrt/executorch/runtime.py" +RUNTIME_PATH = ( + Path(__file__).parents[4] + / "py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/runtime.py" +) DELEGATE_PATH = ( Path(__file__).parents[4] / "py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py" @@ -53,7 +56,7 @@ def load_program(self, data): def test_load_and_forward(monkeypatch, tmp_path): delegate = types.ModuleType("torch_tensorrt_executorch_delegate") fake_runtime = FakeRuntime() - delegate.runtime = lambda: fake_runtime + delegate.get_runtime = lambda: fake_runtime monkeypatch.setitem(sys.modules, delegate.__name__, delegate) model = tmp_path / "model.pte" model.write_bytes(b"pte") @@ -66,7 +69,7 @@ def test_load_and_forward(monkeypatch, tmp_path): def test_unknown_method(monkeypatch, tmp_path): delegate = types.ModuleType("torch_tensorrt_executorch_delegate") - delegate.runtime = FakeRuntime + delegate.get_runtime = FakeRuntime monkeypatch.setitem(sys.modules, delegate.__name__, delegate) model = tmp_path / "model.pte" model.write_bytes(b"pte") @@ -123,7 +126,7 @@ def test_activate_rejects_preloaded_stock_wrapper(monkeypatch): with pytest.raises( delegate.DelegateCompatibilityError, - match=r"Call torch_tensorrt\.executorch\.load", + match=r"torch_tensorrt\.load", ): delegate.activate() From aa33e13c9302827ad00752a39ba781377f70ba44 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Mon, 27 Jul 2026 13:10:28 -0700 Subject: [PATCH 08/17] fix load error --- .../executorch_reference_runner/load_model.py | 86 +++++++------------ .../setup.py | 3 + .../__init__.py | 5 +- .../dynamo/executorch/test_python_runtime.py | 2 +- 4 files changed, 37 insertions(+), 59 deletions(-) diff --git a/examples/executorch_reference_runner/load_model.py b/examples/executorch_reference_runner/load_model.py index 7497744743..1e365c6cc8 100644 --- a/examples/executorch_reference_runner/load_model.py +++ b/examples/executorch_reference_runner/load_model.py @@ -1,61 +1,33 @@ -"""Load and optionally run a Torch-TensorRT ExecuTorch ``.pte`` model. - -The default input matches ``export_static_shape.py``: one CUDA float tensor -with shape ``(2, 3, 4, 4)`` filled with ones. -""" - import argparse from pathlib import Path import torch -from torch_tensorrt.executorch.runtime import load - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() - parser.add_argument("--model_path", type=Path, default=Path("model.pte")) - parser.add_argument("--method", default="forward") - parser.add_argument("--num_runs", type=int, default=1) - parser.add_argument( - "--load_only", - action="store_true", - help="Parse the program and print its methods without loading/executing one", - ) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - if not args.model_path.is_file(): - raise FileNotFoundError(f"ExecuTorch model not found: {args.model_path}") - if args.num_runs < 1: - raise ValueError("--num_runs must be at least 1") - - program = load(args.model_path) - print(f"Loaded {args.model_path}") - print(f"Program methods: {sorted(program.method_names)}") - - if args.load_only: - return - if args.method not in program.method_names: - raise ValueError( - f"Method {args.method!r} is not in the program; " - f"available methods: {sorted(program.method_names)}" - ) - inputs = (torch.ones((2, 3, 4, 4), dtype=torch.float32),) - for run in range(args.num_runs): - outputs = program.run(inputs, args.method) - print(f"Run {run + 1} outputs:") - for index, output in enumerate(outputs): - if isinstance(output, torch.Tensor): - print( - f" output[{index}]: shape={tuple(output.shape)}, " - f"dtype={output.dtype}, device={output.device}, " - f"sample={output.flatten()[:8]}" - ) - else: - print(f" output[{index}]: {output!r}") - - -if __name__ == "__main__": - main() +import torch_tensorrt + +parser = argparse.ArgumentParser() +parser.add_argument( + "--model_path", + type=Path, + required=True, + help="Path to the ExecuTorch .pte model", +) +parser.add_argument("--num_runs", type=int, default=1) +args = parser.parse_args() +if args.num_runs < 1: + raise ValueError("--num_runs must be at least 1") + +model_path = args.model_path +x = torch.ones((2, 3, 4, 4), dtype=torch.float32) + +program = torch_tensorrt.load(model_path, format="executorch") +for _ in range(args.num_runs): + outputs = program.forward(x) +y = outputs[0] + +expected = x + 1 +torch.testing.assert_close(y.cpu(), expected) + +print("methods:", sorted(program.method_names)) +print("output shape:", tuple(y.shape)) +print("output device:", y.device) +print("PASS: ExecuTorch TensorRT delegate output matches x + 1") diff --git a/py/torch-tensorrt-executorch-delegate/setup.py b/py/torch-tensorrt-executorch-delegate/setup.py index 8e39c8bcc6..08b8f4ee4c 100644 --- a/py/torch-tensorrt-executorch-delegate/setup.py +++ b/py/torch-tensorrt-executorch-delegate/setup.py @@ -10,6 +10,7 @@ import shutil import subprocess import sys +import uuid import torch from setuptools import Extension, find_packages, setup @@ -18,6 +19,7 @@ HERE = pathlib.Path(__file__).resolve().parent REPO_ROOT = HERE.parents[1] BAZEL_TARGET = "//py/torch-tensorrt-executorch-delegate/native:delegate_native" +BUILD_NONCE = os.getenv("TORCH_TENSORRT_EXECUTORCH_BUILD_NONCE", uuid.uuid4().hex) def torchtrt_version() -> str: @@ -56,6 +58,7 @@ def build_extension(self, ext: Extension) -> None: "--config=python", f"--compilation_mode={compilation_mode}", f"--action_env=PYTHON_BIN_PATH={sys.executable}", + f"--action_env=TORCH_TENSORRT_EXECUTORCH_BUILD_NONCE={BUILD_NONCE}", ] dist_dir = REPO_ROOT / "third_party/dist_dir/x86_64-linux-gnu" if dist_dir.is_dir(): diff --git a/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py b/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py index 44ce05be92..ee85575eca 100644 --- a/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py +++ b/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py @@ -31,6 +31,10 @@ def activate() -> ModuleType: previous_data_loader = sys.modules.get(_DATA_LOADER_NAME, missing) try: data_loader = importlib.import_module(__name__ + ".data_loader") + # _portable_lib imports this canonical name while its module initializer + # runs. Install our binding first so Python does not load ExecuTorch's + # stock data_loader and register PyDataLoader a second time. + sys.modules[_DATA_LOADER_NAME] = data_loader native = importlib.import_module(__name__ + "._portable_lib") except ImportError as error: if previous_data_loader is missing: @@ -42,7 +46,6 @@ def activate() -> ModuleType: "Install torch, executorch, torch-tensorrt, and the delegate from " "the same release matrix." ) from error - sys.modules[_DATA_LOADER_NAME] = data_loader sys.modules[_NATIVE_NAME] = native sys.modules.pop(_WRAPPER_NAME, None) return native diff --git a/tests/py/dynamo/executorch/test_python_runtime.py b/tests/py/dynamo/executorch/test_python_runtime.py index 66a4443ff5..28d90ae387 100644 --- a/tests/py/dynamo/executorch/test_python_runtime.py +++ b/tests/py/dynamo/executorch/test_python_runtime.py @@ -138,7 +138,7 @@ def test_activate_cleans_up_data_loader_when_native_import_fails(monkeypatch): def fake_import(name): if name == data_loader.__name__: return data_loader - assert delegate._DATA_LOADER_NAME not in sys.modules + assert sys.modules[delegate._DATA_LOADER_NAME] is data_loader raise ImportError("native module failed to load") monkeypatch.setattr( From 8f83c8ef8d7350f192477069be372ac6958420c4 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Mon, 27 Jul 2026 13:40:34 -0700 Subject: [PATCH 09/17] fix lint --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4ab3256cff..fe1be76f5d 100644 --- a/setup.py +++ b/setup.py @@ -185,8 +185,7 @@ def load_dep_info(): EXECUTORCH_REQUIREMENT = "executorch>=1.3.1" EXECUTORCH_DELEGATE_REQUIREMENT = ( - f"torch-tensorrt-executorch-delegate=={__version__}; " - "platform_system == 'Linux'" + f"torch-tensorrt-executorch-delegate=={__version__}; " "platform_system == 'Linux'" ) EXTRAS_REQUIRE = { "executorch": [EXECUTORCH_REQUIREMENT, EXECUTORCH_DELEGATE_REQUIREMENT], From bdf3d737cc2c18a57a3f7e2224c39fccd1fab1f3 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Mon, 27 Jul 2026 14:14:42 -0700 Subject: [PATCH 10/17] add aarch64 support --- .../executorch/TensorRTBackend.cpp | 3 +-- .../native/BUILD.bazel | 22 ++++++++++++++----- .../setup.py | 8 ++++++- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp b/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp index 648fb88232..e335748b33 100644 --- a/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp +++ b/cpp/src/torch_tensorrt/executorch/TensorRTBackend.cpp @@ -56,8 +56,7 @@ extern const Error kRegistrationResult; Error check_registration() { if (kRegistrationResult != Error::Ok) { - ET_LOG( - Error, "TensorRTBackend registration failed: %s", ::executorch::runtime::to_string(kRegistrationResult)); + ET_LOG(Error, "TensorRTBackend registration failed: %s", ::executorch::runtime::to_string(kRegistrationResult)); } return kRegistrationResult; } diff --git a/py/torch-tensorrt-executorch-delegate/native/BUILD.bazel b/py/torch-tensorrt-executorch-delegate/native/BUILD.bazel index 6c63942591..3844ca869a 100644 --- a/py/torch-tensorrt-executorch-delegate/native/BUILD.bazel +++ b/py/torch-tensorrt-executorch-delegate/native/BUILD.bazel @@ -2,16 +2,24 @@ load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake") package(default_visibility = ["//visibility:public"]) +config_setting( + name = "aarch64_linux", + constraint_values = [ + "@platforms//cpu:aarch64", + "@platforms//os:linux", + ], +) + filegroup( name = "delegate_sources", srcs = glob( ["**"], exclude = ["BUILD.bazel"], ) + [ + "//:torchtrt_cmake_modules", "//core/runtime:include_files", "//cpp:executorch_api_headers", "//cpp:executorch_backend_source_files", - "//:torchtrt_cmake_modules", "@executorch//:executorch_sources", ], ) @@ -32,15 +40,17 @@ cmake( "@executorch//:executorch/CMakeLists.txt", "@libtorch//:include/torch/headeronly/util/TypeTraits.h", ], - deps = [ - "@cuda//:cudart", - "@libtorch//:torch", - "@tensorrt//:nvinfer", - ], lib_source = ":delegate_sources", out_include_dir = "", out_shared_libs = [ "_portable_lib.so", "data_loader.so", ], + deps = [ + "@cuda//:cudart", + "@libtorch//:torch", + ] + select({ + ":aarch64_linux": ["@tensorrt_sbsa//:nvinfer"], + "//conditions:default": ["@tensorrt//:nvinfer"], + }), ) diff --git a/py/torch-tensorrt-executorch-delegate/setup.py b/py/torch-tensorrt-executorch-delegate/setup.py index 08b8f4ee4c..cc8fdae266 100644 --- a/py/torch-tensorrt-executorch-delegate/setup.py +++ b/py/torch-tensorrt-executorch-delegate/setup.py @@ -5,6 +5,7 @@ import importlib.metadata import os import pathlib +import platform import re import shlex import shutil @@ -60,7 +61,12 @@ def build_extension(self, ext: Extension) -> None: f"--action_env=PYTHON_BIN_PATH={sys.executable}", f"--action_env=TORCH_TENSORRT_EXECUTORCH_BUILD_NONCE={BUILD_NONCE}", ] - dist_dir = REPO_ROOT / "third_party/dist_dir/x86_64-linux-gnu" + dist_dir_arch = ( + "aarch64-linux-gnu" + if platform.machine() in {"aarch64", "arm64"} + else "x86_64-linux-gnu" + ) + dist_dir = REPO_ROOT / "third_party/dist_dir" / dist_dir_arch if dist_dir.is_dir(): command.append(f"--distdir={dist_dir}") command.extend(shlex.split(os.getenv("BAZEL_ARGS", ""))) From 91e7eab175f74c26b390622f4dd169e7c257918e Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Tue, 28 Jul 2026 12:01:29 -0700 Subject: [PATCH 11/17] move build executorch-delegate as a downstream job --- .github/workflows/_linux-x86_64-core.yml | 20 +++- .github/workflows/build_linux.yml | 45 +-------- ...c-linux.yml => executorch-build-linux.yml} | 16 +--- .github/workflows/executorch-test-linux.yml | 94 +++++++++++++++++++ .github/workflows/linux-test.yml | 10 ++ tests/py/utils/ci_helpers.sh | 12 +-- 6 files changed, 133 insertions(+), 64 deletions(-) rename .github/workflows/{executorch-static-linux.yml => executorch-build-linux.yml} (76%) create mode 100644 .github/workflows/executorch-test-linux.yml diff --git a/.github/workflows/_linux-x86_64-core.yml b/.github/workflows/_linux-x86_64-core.yml index c32fc55bdc..90b857422d 100644 --- a/.github/workflows/_linux-x86_64-core.yml +++ b/.github/workflows/_linux-x86_64-core.yml @@ -96,14 +96,24 @@ jobs: trigger-event: ${{ github.event_name }} architecture: "x86_64" use-rtx: ${{ inputs.use-rtx }} - build-executorch-delegate: ${{ !inputs.use-rtx }} pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - # Standard-TRT only: ExecuTorch static build is not part of the RTX matrix. - executorch-static-build: + # Standard-TRT only: ExecuTorch delegate build is not part of the RTX matrix. + executorch-delegate-build: needs: [filter-matrix, build] if: ${{ !inputs.use-rtx }} - uses: ./.github/workflows/executorch-static-linux.yml + uses: ./.github/workflows/executorch-build-linux.yml + with: + repository: "pytorch/tensorrt" + ref: "" + test-infra-repository: pytorch/test-infra + test-infra-ref: main + build-matrix: ${{ needs.filter-matrix.outputs.matrix }} + + executorch-delegate-test: + needs: [filter-matrix, executorch-delegate-build] + if: ${{ !inputs.use-rtx && needs.executorch-delegate-build.result == 'success' }} + uses: ./.github/workflows/executorch-test-linux.yml with: repository: "pytorch/tensorrt" ref: "" @@ -640,6 +650,8 @@ jobs: needs: [ build, + executorch-delegate-build, + executorch-delegate-test, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests, diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml index 109c439ab5..90333aedf8 100644 --- a/.github/workflows/build_linux.yml +++ b/.github/workflows/build_linux.yml @@ -90,11 +90,6 @@ on: required: false default: "python -m build --wheel" type: string - build-executorch-delegate: - description: Build and publish the matching ExecuTorch delegate wheel - required: false - default: false - type: boolean pip-install-torch-extra-args: # NOTE: Why does this exist? # Well setuptools / python packaging doesn't actually allow you to specify dependencies @@ -157,7 +152,6 @@ jobs: ARCH: ${{ inputs.architecture }} BUILD_TARGET: ${{ inputs.build-target }} USE_TRT_RTX: ${{ inputs.use-rtx }} - BUILD_EXECUTORCH_DELEGATE: ${{ inputs.build-executorch-delegate }} name: build-wheel-${{ matrix.python_version }}-${{ matrix.desired_cuda }}-${{ matrix.gpu_arch_type }}-${{ inputs.architecture }}-${{ inputs.use-rtx }}-${{ inputs.is-jetpack }} runs-on: ${{ matrix.validation_runner }} environment: ${{(inputs.trigger-event == 'schedule' || (inputs.trigger-event == 'push' && (startsWith(github.event.ref, 'refs/heads/nightly') || startsWith(github.event.ref, 'refs/tags/v')))) && 'pytorchbot-env' || ''}} @@ -319,19 +313,6 @@ jobs: ${CONDA_RUN} python setup.py bdist_wheel ${param} fi fi - - name: Build the ExecuTorch delegate wheel - if: ${{ inputs.build-executorch-delegate && inputs.use-rtx == false && inputs.is-jetpack == false && inputs.is-release-tarball == false }} - working-directory: ${{ inputs.repository }} - shell: bash -l {0} - run: | - set -euxo pipefail - source "${BUILD_ENV_FILE}" - ${CONDA_RUN} python -m pip install pyyaml executorch - executorch_cmake_location="$(bazel query @executorch//:executorch/CMakeLists.txt --output=location)" - export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" - export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}" - ${CONDA_RUN} python -m pip wheel --no-build-isolation --no-deps \ - --wheel-dir dist py/torch-tensorrt-executorch-delegate - name: Repair Manylinux_2_28 Wheel shell: bash -l {0} env: @@ -365,31 +346,16 @@ jobs: run: | set -euxo pipefail source "${BUILD_ENV_FILE}" - mapfile -t WHEEL_PATHS < <(find "${{ inputs.repository }}/dist" -maxdepth 1 -name "*.whl" -print | sort) - if [[ ${#WHEEL_PATHS[@]} -eq 0 ]]; then - echo "No wheels found in ${{ inputs.repository }}/dist" >&2 - exit 1 - fi - printf "Wheel: %s\n" "${WHEEL_PATHS[@]}" - MAIN_WHEEL="" - for wheel_path in "${WHEEL_PATHS[@]}"; do - if [[ $(basename "${wheel_path}") == torch_tensorrt-[0-9]*.whl ]]; then - MAIN_WHEEL="${wheel_path}" - break - fi - done - if [[ -z "${MAIN_WHEEL}" ]]; then - echo "Could not identify the torch_tensorrt wheel" >&2 - exit 1 - fi + WHEEL_NAME=$(ls "${{ inputs.repository }}/dist/") + echo "$WHEEL_NAME" if [[ ${{ inputs.architecture }} == "aarch64" ]]; then echo "Skipping smoke test for aarch64, since it is not an actual gpu runner" else - ${CONDA_RUN} pip install "${WHEEL_PATHS[@]}" --use-deprecated=legacy-resolver --extra-index-url https://download.pytorch.org/${CHANNEL}/${CU_VERSION} + ${CONDA_RUN} pip install "${{ inputs.repository }}/dist/$WHEEL_NAME" --use-deprecated=legacy-resolver --extra-index-url https://download.pytorch.org/${CHANNEL}/${CU_VERSION} # Checking that we have a pinned version of torch in our dependency tree ( pushd "${RUNNER_TEMP}" - unzip -o "${MAIN_WHEEL}" + unzip -o "${GITHUB_WORKSPACE}/${{ inputs.repository }}/dist/$WHEEL_NAME" # Ensure that pytorch version is pinned, should output file where it was found grep "Requires-Dist: torch (==.*)" -r . ) @@ -408,9 +374,6 @@ jobs: echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found" ${CONDA_RUN} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}" fi - if [[ "${BUILD_EXECUTORCH_DELEGATE}" == "true" ]]; then - ${CONDA_RUN} python -c 'from torch_tensorrt_executorch_delegate import runtime; assert runtime().backend_registry.is_available("TensorRTBackend")' - fi fi # NB: Only upload to GitHub after passing smoke tests diff --git a/.github/workflows/executorch-static-linux.yml b/.github/workflows/executorch-build-linux.yml similarity index 76% rename from .github/workflows/executorch-static-linux.yml rename to .github/workflows/executorch-build-linux.yml index b98c1a4dac..916484d866 100644 --- a/.github/workflows/executorch-static-linux.yml +++ b/.github/workflows/executorch-build-linux.yml @@ -1,4 +1,4 @@ -name: ExecuTorch Static Linux Build +name: ExecuTorch Delegate Linux Build on: workflow_call: @@ -60,7 +60,7 @@ jobs: needs: select-matrix uses: ./.github/workflows/linux-test.yml with: - job-name: executorch-static-build + job-name: executorch-delegate-build repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} test-infra-repository: ${{ inputs.test-infra-repository }} @@ -91,21 +91,11 @@ jobs: export PATH="${RUNNER_TEMP}/bin:${PATH}" bazel --version - # this is to build the libtorchtrt.tar.gz - bazel build //:libtorchtrt --compilation_mode opt --config=linux executorch_cmake_location="$(bazel query @executorch//:executorch/CMakeLists.txt --output=location)" export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}" - # this is to verify the end user's workflow python -m pip install pyyaml executorch - - # export the model to a .pte file - python examples/torchtrt_executorch_example/export_static_shape.py --model_path="${RUNNER_TEMP}/torchtrt-python.pte" - # verify the c++ reference runner can run the model - .github/scripts/verify-executorch-reference-runner.sh "${RUNNER_TEMP}/torchtrt-python.pte" + export TORCH_TENSORRT_EXECUTORCH_DELEGATE_VERSION="$(python -c 'import importlib.metadata; print(importlib.metadata.version("torch-tensorrt"))')" # Build the no-compile-for-users Python runtime/delegate wheel. python -m pip wheel --no-build-isolation --no-deps --wheel-dir dist py/torch-tensorrt-executorch-delegate - python -m pip install --no-deps --force-reinstall dist/torch_tensorrt_executorch_delegate-*.whl - python -c 'from torch_tensorrt_executorch_delegate import runtime; assert runtime().backend_registry.is_available("TensorRTBackend")' - python examples/executorch_reference_runner/load_model.py --model_path="${RUNNER_TEMP}/torchtrt-python.pte" --num_runs=1 diff --git a/.github/workflows/executorch-test-linux.yml b/.github/workflows/executorch-test-linux.yml new file mode 100644 index 0000000000..c80d38049d --- /dev/null +++ b/.github/workflows/executorch-test-linux.yml @@ -0,0 +1,94 @@ +name: ExecuTorch Delegate Linux Test + +on: + workflow_call: + inputs: + build-matrix: + description: "Build matrix to utilize" + default: "" + type: string + repository: + description: 'Repository to checkout, defaults to ""' + default: "" + type: string + ref: + description: 'Reference to checkout, defaults to ""' + default: "" + type: string + test-infra-repository: + description: "Test infra repository to use" + default: "pytorch/test-infra" + type: string + test-infra-ref: + description: "Test infra reference to use" + default: "" + type: string + +jobs: + select-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.pick.outputs.matrix }} + steps: + - id: pick + env: + FULL_MATRIX: ${{ inputs.build-matrix }} + run: | + set -euo pipefail + python - <<'PY' >> "${GITHUB_OUTPUT}" + import json + import os + + matrix = json.loads(os.environ["FULL_MATRIX"]) + include = matrix.get("include", []) + if not include: + raise SystemExit("build-matrix include[] is empty") + + preferred = None + for entry in include: + if entry.get("python_version") == "3.11": + preferred = entry + break + + if preferred is None: + preferred = include[0] + + print("matrix=" + json.dumps({"include": [preferred]})) + PY + + test: + needs: select-matrix + uses: ./.github/workflows/linux-test.yml + with: + job-name: executorch-delegate-test + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + test-infra-repository: ${{ inputs.test-infra-repository }} + test-infra-ref: ${{ inputs.test-infra-ref }} + build-matrix: ${{ needs.select-matrix.outputs.matrix }} + pre-script: packaging/pre_build_script.sh + additional-artifact: torch-tensorrt-executorch-delegate + script: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/bin" + curl -L "https://github.com/bazelbuild/bazelisk/releases/download/v1.26.0/bazelisk-linux-amd64" \ + -o "${RUNNER_TEMP}/bin/bazel" + chmod +x "${RUNNER_TEMP}/bin/bazel" + export PATH="${RUNNER_TEMP}/bin:${PATH}" + + python -m pip install pyyaml executorch + python -c 'from torch_tensorrt_executorch_delegate import runtime; assert runtime().backend_registry.is_available("TensorRTBackend")' + + source tests/py/utils/ci_helpers.sh + trt_tier_executorch + + bazel build //:libtorchtrt --compilation_mode opt --config=linux + executorch_cmake_location="$(bazel query @executorch//:executorch/CMakeLists.txt --output=location)" + export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" + export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}" + python examples/torchtrt_executorch_example/export_static_shape.py \ + --model_path="${RUNNER_TEMP}/torchtrt-python.pte" + .github/scripts/verify-executorch-reference-runner.sh \ + "${RUNNER_TEMP}/torchtrt-python.pte" + python examples/executorch_reference_runner/load_model.py \ + --model_path="${RUNNER_TEMP}/torchtrt-python.pte" --num_runs=1 diff --git a/.github/workflows/linux-test.yml b/.github/workflows/linux-test.yml index d055276143..c872c6f8f0 100644 --- a/.github/workflows/linux-test.yml +++ b/.github/workflows/linux-test.yml @@ -53,6 +53,10 @@ on: description: 'Name to give artifacts uploaded from ${RUNNER_ARTIFACT_DIR}' default: '' type: string + additional-artifact: + description: "Optional additional artifact to install alongside the core wheel" + default: "" + type: string use-rtx: description: "Whether to use TensorRT-RTX" default: false @@ -146,6 +150,12 @@ jobs: # with: # repository: ${{ inputs.repository }} # script: .github/scripts/install-torch-tensorrt.sh + - name: Download additional artifacts + if: ${{ inputs.additional-artifact != '' }} + uses: actions/download-artifact@v7 + with: + name: ${{ inputs.additional-artifact }} + path: /opt/torch-tensorrt-builds/ - name: Pack script continue-on-error: ${{ inputs.continue-on-error }} working-directory: ${{ inputs.repository }} diff --git a/tests/py/utils/ci_helpers.sh b/tests/py/utils/ci_helpers.sh index 466f864f7f..7f3b4e5861 100755 --- a/tests/py/utils/ci_helpers.sh +++ b/tests/py/utils/ci_helpers.sh @@ -155,12 +155,12 @@ trt_tier_l2_dynamo_compile() { trt_tier_l2_dynamo_core() { ( cd "${TRT_REPO_ROOT}/tests/py/dynamo" - _trt_py -m pytest -ra $(_trt_nproc auto) --junitxml="$(_trt_xml l2_dynamo_core_tests_results)" -k "not test_000_ and not test_001_" runtime/* "$@" - if [ "${USE_TRT_RTX:-false}" != "true" ]; then - # ExecuTorch integration is standard-TRT only. - _trt_py -m pip install pyyaml "executorch>=1.3.1" - _trt_py -m pytest -ra $(_trt_nproc auto) --junitxml="$(_trt_xml l2_dynamo_executorch_tests_results)" executorch/ "$@" - fi ) + _trt_py -m pytest -ra $(_trt_nproc auto) --junitxml="$(_trt_xml l2_dynamo_core_tests_results)" -k "not test_000_ and not test_001_" runtime/* "$@" ) +} + +trt_tier_executorch() { + ( cd "${TRT_REPO_ROOT}/tests/py/dynamo" + _trt_py -m pytest -ra $(_trt_nproc auto) --junitxml="$(_trt_xml executorch_tests_results)" executorch/ "$@" ) } trt_tier_l2_plugin() { From cc8035b59d8d22d1aa6ba3916508a7505476579a Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Tue, 28 Jul 2026 14:56:31 -0700 Subject: [PATCH 12/17] fix the executorch commit mismatch in MODULE.bazel.tmpl --- toolchains/ci_workspaces/MODULE.bazel.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/toolchains/ci_workspaces/MODULE.bazel.tmpl b/toolchains/ci_workspaces/MODULE.bazel.tmpl index 9c22f41291..1aa929c506 100644 --- a/toolchains/ci_workspaces/MODULE.bazel.tmpl +++ b/toolchains/ci_workspaces/MODULE.bazel.tmpl @@ -176,8 +176,8 @@ new_local_repository( new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", - # PyTorch 2.14-compatible ExecuTorch main commit - commit = "a6d812a082df57898b8608f56c867140cc9da32c", + # executorch==1.3.1 + commit = "e2f18eb23c45bd22ca332b0b8b49a81de304b472", patch_cmds = [ "find . -mindepth 2 \\( -name BUILD -o -name BUILD.bazel \\) -delete", "mkdir executorch && find . -mindepth 1 -maxdepth 1 ! -name executorch ! -name BUILD ! -name BUILD.bazel ! -name REPO.bazel -exec cp -a {} executorch/ \\;", From 32d7dd9f7017a7bcda27374b04e8d333f66f56e1 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Wed, 29 Jul 2026 11:11:51 -0700 Subject: [PATCH 13/17] rename delegate to runtime, add rpath --- .github/workflows/_linux-x86_64-core.yml | 14 +++--- .github/workflows/executorch-build-linux.yml | 10 ++--- .github/workflows/executorch-test-linux.yml | 6 +-- .gitignore | 4 +- MODULE.bazel | 4 +- .../executorch_reference_runner/README.md | 2 +- .../README.md | 25 +++++++++-- .../native/BUILD.bazel | 0 .../native/CMakeLists.txt | 15 ++++++- .../pyproject.toml | 0 .../setup.py | 12 ++--- .../__init__.py | 24 +++++++--- .../runtime.py | 15 ++++--- py/torch_tensorrt/_compile.py | 44 +++++++++---------- py/torch_tensorrt/executorch/__init__.py | 2 +- setup.py | 8 ++-- tests/py/dynamo/executorch/test_api.py | 29 ++++++++---- .../dynamo/executorch/test_python_runtime.py | 8 ++-- 18 files changed, 139 insertions(+), 83 deletions(-) rename py/{torch-tensorrt-executorch-delegate => torch-tensorrt-executorch-runtime}/README.md (58%) rename py/{torch-tensorrt-executorch-delegate => torch-tensorrt-executorch-runtime}/native/BUILD.bazel (100%) rename py/{torch-tensorrt-executorch-delegate => torch-tensorrt-executorch-runtime}/native/CMakeLists.txt (85%) rename py/{torch-tensorrt-executorch-delegate => torch-tensorrt-executorch-runtime}/pyproject.toml (100%) rename py/{torch-tensorrt-executorch-delegate => torch-tensorrt-executorch-runtime}/setup.py (89%) rename py/{torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate => torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime}/__init__.py (81%) rename py/{torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate => torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime}/runtime.py (80%) diff --git a/.github/workflows/_linux-x86_64-core.yml b/.github/workflows/_linux-x86_64-core.yml index 90b857422d..d164a48c50 100644 --- a/.github/workflows/_linux-x86_64-core.yml +++ b/.github/workflows/_linux-x86_64-core.yml @@ -98,8 +98,8 @@ jobs: use-rtx: ${{ inputs.use-rtx }} pip-install-torch-extra-args: "--extra-index-url https://pypi.org/simple" - # Standard-TRT only: ExecuTorch delegate build is not part of the RTX matrix. - executorch-delegate-build: + # Standard-TRT only: ExecuTorch runtime build is not part of the RTX matrix. + executorch-runtime-build: needs: [filter-matrix, build] if: ${{ !inputs.use-rtx }} uses: ./.github/workflows/executorch-build-linux.yml @@ -110,9 +110,9 @@ jobs: test-infra-ref: main build-matrix: ${{ needs.filter-matrix.outputs.matrix }} - executorch-delegate-test: - needs: [filter-matrix, executorch-delegate-build] - if: ${{ !inputs.use-rtx && needs.executorch-delegate-build.result == 'success' }} + executorch-runtime-test: + needs: [filter-matrix, executorch-runtime-build] + if: ${{ !inputs.use-rtx && needs.executorch-runtime-build.result == 'success' }} uses: ./.github/workflows/executorch-test-linux.yml with: repository: "pytorch/tensorrt" @@ -650,8 +650,8 @@ jobs: needs: [ build, - executorch-delegate-build, - executorch-delegate-test, + executorch-runtime-build, + executorch-runtime-test, L0-dynamo-converter-tests, L0-dynamo-core-tests, L0-py-core-tests, diff --git a/.github/workflows/executorch-build-linux.yml b/.github/workflows/executorch-build-linux.yml index 916484d866..03bb9e7dd3 100644 --- a/.github/workflows/executorch-build-linux.yml +++ b/.github/workflows/executorch-build-linux.yml @@ -60,7 +60,7 @@ jobs: needs: select-matrix uses: ./.github/workflows/linux-test.yml with: - job-name: executorch-delegate-build + job-name: executorch-runtime-build repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} test-infra-repository: ${{ inputs.test-infra-repository }} @@ -68,7 +68,7 @@ jobs: build-matrix: ${{ needs.select-matrix.outputs.matrix }} pre-script: packaging/pre_build_script.sh fail-on-empty: false - upload-artifact: torch-tensorrt-executorch-delegate + upload-artifact: torch-tensorrt-executorch-runtime script: | set -euo pipefail BAZELISK_VERSION="1.26.0" @@ -95,7 +95,7 @@ jobs: export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}" python -m pip install pyyaml executorch - export TORCH_TENSORRT_EXECUTORCH_DELEGATE_VERSION="$(python -c 'import importlib.metadata; print(importlib.metadata.version("torch-tensorrt"))')" + export TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION="$(python -c 'import importlib.metadata; print(importlib.metadata.version("torch-tensorrt"))')" - # Build the no-compile-for-users Python runtime/delegate wheel. - python -m pip wheel --no-build-isolation --no-deps --wheel-dir dist py/torch-tensorrt-executorch-delegate + # Build the no-compile-for-users Python runtime wheel. + python -m pip wheel --no-build-isolation --no-deps --wheel-dir dist py/torch-tensorrt-executorch-runtime diff --git a/.github/workflows/executorch-test-linux.yml b/.github/workflows/executorch-test-linux.yml index c80d38049d..80ac314e34 100644 --- a/.github/workflows/executorch-test-linux.yml +++ b/.github/workflows/executorch-test-linux.yml @@ -60,14 +60,14 @@ jobs: needs: select-matrix uses: ./.github/workflows/linux-test.yml with: - job-name: executorch-delegate-test + job-name: executorch-runtime-test repository: ${{ inputs.repository }} ref: ${{ inputs.ref }} test-infra-repository: ${{ inputs.test-infra-repository }} test-infra-ref: ${{ inputs.test-infra-ref }} build-matrix: ${{ needs.select-matrix.outputs.matrix }} pre-script: packaging/pre_build_script.sh - additional-artifact: torch-tensorrt-executorch-delegate + additional-artifact: torch-tensorrt-executorch-runtime script: | set -euo pipefail mkdir -p "${RUNNER_TEMP}/bin" @@ -77,7 +77,7 @@ jobs: export PATH="${RUNNER_TEMP}/bin:${PATH}" python -m pip install pyyaml executorch - python -c 'from torch_tensorrt_executorch_delegate import runtime; assert runtime().backend_registry.is_available("TensorRTBackend")' + python -c 'from torch_tensorrt_executorch_runtime import get_runtime; assert get_runtime().backend_registry.is_available("TensorRTBackend")' source tests/py/utils/ci_helpers.sh trt_tier_executorch diff --git a/.gitignore b/.gitignore index de9f7e5336..51871e0053 100644 --- a/.gitignore +++ b/.gitignore @@ -94,5 +94,5 @@ CLAUDE.md /compile_commands.json /external/ build-executorch/ -py/torch-tensorrt-executorch-delegate/build/ -py/torch-tensorrt-executorch-delegate/dist/ \ No newline at end of file +py/torch-tensorrt-executorch-runtime/build/ +py/torch-tensorrt-executorch-runtime/dist/ \ No newline at end of file diff --git a/MODULE.bazel b/MODULE.bazel index 12895c9cef..93d491acd8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -32,8 +32,8 @@ git_override( bazel_dep(name = "hedron_compile_commands", dev_dependency = True) git_override( module_name = "hedron_compile_commands", - remote = "https://github.com/hedronvision/bazel-compile-commands-extractor.git", commit = "0e990032f3c5a866e72615cf67e5ce22186dcb97", + remote = "https://github.com/hedronvision/bazel-compile-commands-extractor.git", ) new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") @@ -43,7 +43,7 @@ new_git_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:git.bzl" local_torch = use_repo_rule("//toolchains:local_torch.bzl", "local_torch") # Keep this pin synchronized with the ExecuTorch release installed in -# py/torch-tensorrt-executorch-delegate/README.md. +# py/torch-tensorrt-executorch-runtime/README.md. new_git_repository( name = "executorch", build_file = "@//third_party/executorch:BUILD", diff --git a/examples/executorch_reference_runner/README.md b/examples/executorch_reference_runner/README.md index 171fd7c246..ddf635f297 100644 --- a/examples/executorch_reference_runner/README.md +++ b/examples/executorch_reference_runner/README.md @@ -96,7 +96,7 @@ python examples/executorch_reference_runner/load_model.py \ ``` The extra installs `executorch` and the matching -`torch-tensorrt-executorch-delegate` wheel. That wheel contains an ExecuTorch +`torch-tensorrt-executorch-runtime` wheel. That wheel contains an ExecuTorch Python runtime with `TensorRTBackend` linked into its backend registry. ### C++ diff --git a/py/torch-tensorrt-executorch-delegate/README.md b/py/torch-tensorrt-executorch-runtime/README.md similarity index 58% rename from py/torch-tensorrt-executorch-delegate/README.md rename to py/torch-tensorrt-executorch-runtime/README.md index 2f46960b0c..5499ddcb85 100644 --- a/py/torch-tensorrt-executorch-delegate/README.md +++ b/py/torch-tensorrt-executorch-runtime/README.md @@ -1,12 +1,31 @@ -# Torch-TensorRT ExecuTorch Delegate Wheel +# Torch-TensorRT ExecuTorch Runtime Wheel -This directory builds `torch-tensorrt-executorch-delegate`. The Linux wheel +This directory builds `torch-tensorrt-executorch-runtime`. The Linux wheel contains an ExecuTorch `_portable_lib` Python runtime with `TensorRTBackend` force-linked into the same native module that owns the backend registry. The wheel must use the same Python, PyTorch, ExecuTorch, CUDA, TensorRT, and C++ ABI as its matching Torch-TensorRT wheel. +## Runtime libraries + +The wheel does not bundle PyTorch, c10, TensorRT, or CUDA shared libraries. +Its `_portable_lib.so` has origin-relative runtime search paths for the +TensorRT and CUDA library locations installed by their Python packages: + +- `tensorrt_libs` +- `nvidia/cuda_runtime/lib` (CUDA 12) +- `nvidia/cu13/lib` (CUDA 13) + +These packages are installed transitively with the matching `torch-tensorrt` +wheel. For a system TensorRT or CUDA installation outside these standard +locations, its `lib` directory must be available through the system dynamic +loader configuration or `LD_LIBRARY_PATH`. + +The CI manylinux repair step changes the wheel platform tag; it does not +bundle these external libraries. The origin-relative paths are therefore part +of the wheel runtime contract. + ## Build > [!IMPORTANT] @@ -23,7 +42,7 @@ export TensorRT_ROOT=/path/to/TensorRT python -m pip install executorch==1.3.1 python -m pip wheel --no-build-isolation --no-deps \ - --wheel-dir dist py/torch-tensorrt-executorch-delegate + --wheel-dir dist py/torch-tensorrt-executorch-runtime ``` The ExecuTorch source commit pinned in `MODULE.bazel` is the revision recorded diff --git a/py/torch-tensorrt-executorch-delegate/native/BUILD.bazel b/py/torch-tensorrt-executorch-runtime/native/BUILD.bazel similarity index 100% rename from py/torch-tensorrt-executorch-delegate/native/BUILD.bazel rename to py/torch-tensorrt-executorch-runtime/native/BUILD.bazel diff --git a/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt b/py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt similarity index 85% rename from py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt rename to py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt index 1e2b8bc24d..78c95da1ea 100644 --- a/py/torch-tensorrt-executorch-delegate/native/CMakeLists.txt +++ b/py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.24) -project(torch_tensorrt_executorch_delegate LANGUAGES CXX) +project(torch_tensorrt_executorch_runtime LANGUAGES CXX) if(NOT EXECUTORCH_SOURCE_DIR AND EXECUTORCH_CMAKE_FILE) get_filename_component(EXECUTORCH_SOURCE_DIR "${EXECUTORCH_CMAKE_FILE}" DIRECTORY) @@ -74,7 +74,20 @@ target_link_libraries(portable_lib PRIVATE "$" extension_threadpool CUDA::cudart TensorRT::nvinfer Threads::Threads) + +# The runtime wheel intentionally does not bundle PyTorch, TensorRT, or CUDA. +# TensorRT and CUDA pip packages install their shared libraries in sibling +# directories under site-packages, relative to this extension package. +# Preserve these paths in both the Bazel-collected build output and installed +# extension so importing the wheel does not depend on LD_LIBRARY_PATH. +set(_torch_tensorrt_executorch_runtime_rpath + "$ORIGIN" + "$ORIGIN/../tensorrt_libs" + "$ORIGIN/../nvidia/cuda_runtime/lib" + "$ORIGIN/../nvidia/cu13/lib") set_target_properties(portable_lib PROPERTIES + BUILD_WITH_INSTALL_RPATH ON + INSTALL_RPATH "${_torch_tensorrt_executorch_runtime_rpath}" OUTPUT_NAME "_portable_lib" SUFFIX ".so") set_target_properties(data_loader PROPERTIES SUFFIX ".so") diff --git a/py/torch-tensorrt-executorch-delegate/pyproject.toml b/py/torch-tensorrt-executorch-runtime/pyproject.toml similarity index 100% rename from py/torch-tensorrt-executorch-delegate/pyproject.toml rename to py/torch-tensorrt-executorch-runtime/pyproject.toml diff --git a/py/torch-tensorrt-executorch-delegate/setup.py b/py/torch-tensorrt-executorch-runtime/setup.py similarity index 89% rename from py/torch-tensorrt-executorch-delegate/setup.py rename to py/torch-tensorrt-executorch-runtime/setup.py index cc8fdae266..08eee30759 100644 --- a/py/torch-tensorrt-executorch-delegate/setup.py +++ b/py/torch-tensorrt-executorch-runtime/setup.py @@ -19,12 +19,12 @@ HERE = pathlib.Path(__file__).resolve().parent REPO_ROOT = HERE.parents[1] -BAZEL_TARGET = "//py/torch-tensorrt-executorch-delegate/native:delegate_native" +BAZEL_TARGET = "//py/torch-tensorrt-executorch-runtime/native:delegate_native" BUILD_NONCE = os.getenv("TORCH_TENSORRT_EXECUTORCH_BUILD_NONCE", uuid.uuid4().hex) def torchtrt_version() -> str: - if value := os.getenv("TORCH_TENSORRT_EXECUTORCH_DELEGATE_VERSION"): + if value := os.getenv("TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION"): return value version_py = REPO_ROOT / "py/torch_tensorrt/_version.py" if version_py.exists(): @@ -93,7 +93,7 @@ def build_extension(self, ext: Extension) -> None: ) built = ( bazel_bin - / "py/torch-tensorrt-executorch-delegate/native/delegate_native/lib" + / "py/torch-tensorrt-executorch-runtime/native/delegate_native/lib" / f"{library_stem}.so" ) if not built.is_file(): @@ -104,13 +104,13 @@ def build_extension(self, ext: Extension) -> None: executorch_version = importlib.metadata.version("executorch") setup( - name="torch-tensorrt-executorch-delegate", + name="torch-tensorrt-executorch-runtime", version=torchtrt_version(), description="Torch-TensorRT delegate for the ExecuTorch Python runtime", packages=find_packages(), ext_modules=[ - BazelExtension("torch_tensorrt_executorch_delegate._portable_lib"), - BazelExtension("torch_tensorrt_executorch_delegate.data_loader"), + BazelExtension("torch_tensorrt_executorch_runtime._portable_lib"), + BazelExtension("torch_tensorrt_executorch_runtime.data_loader"), ], cmdclass={"build_ext": BazelBuild}, python_requires=">=3.10", diff --git a/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/__init__.py similarity index 81% rename from py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py rename to py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/__init__.py index ee85575eca..50738b8cdb 100644 --- a/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py +++ b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/__init__.py @@ -5,6 +5,7 @@ import importlib import sys from types import ModuleType +from typing import Any, Protocol, cast BACKEND_NAME = "TensorRTBackend" _NATIVE_NAME = "executorch.extension.pybindings._portable_lib" @@ -12,8 +13,18 @@ _DATA_LOADER_NAME = "executorch.extension.pybindings.data_loader" +class _BackendRegistry(Protocol): + def is_available(self, name: str) -> bool: ... + + +class _Runtime(Protocol): + backend_registry: _BackendRegistry + + def load_program(self, data: bytes) -> Any: ... + + class DelegateCompatibilityError(ImportError): - """The delegate wheel is incompatible with the active native runtime.""" + """The runtime wheel is incompatible with the active native runtime.""" def activate() -> ModuleType: @@ -27,8 +38,7 @@ def activate() -> ModuleType: 'torch_tensorrt.load(..., format="executorch") before importing ' "executorch.runtime." ) - missing = object() - previous_data_loader = sys.modules.get(_DATA_LOADER_NAME, missing) + previous_data_loader = sys.modules.get(_DATA_LOADER_NAME) try: data_loader = importlib.import_module(__name__ + ".data_loader") # _portable_lib imports this canonical name while its module initializer @@ -37,13 +47,13 @@ def activate() -> ModuleType: sys.modules[_DATA_LOADER_NAME] = data_loader native = importlib.import_module(__name__ + "._portable_lib") except ImportError as error: - if previous_data_loader is missing: + if previous_data_loader is None: sys.modules.pop(_DATA_LOADER_NAME, None) else: sys.modules[_DATA_LOADER_NAME] = previous_data_loader raise DelegateCompatibilityError( "Could not load the prebuilt Torch-TensorRT ExecuTorch runtime. " - "Install torch, executorch, torch-tensorrt, and the delegate from " + "Install torch, executorch, torch-tensorrt, and the runtime package from " "the same release matrix." ) from error sys.modules[_NATIVE_NAME] = native @@ -51,12 +61,12 @@ def activate() -> ModuleType: return native -def get_runtime(): +def get_runtime() -> _Runtime: """Return the activated ExecuTorch Runtime singleton.""" activate() from executorch.runtime import Runtime - value = Runtime.get() + value = cast(_Runtime, Runtime.get()) if not value.backend_registry.is_available(BACKEND_NAME): raise DelegateCompatibilityError(f"{BACKEND_NAME} is not registered") return value diff --git a/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/runtime.py b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py similarity index 80% rename from py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/runtime.py rename to py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py index 948a231089..e93b149188 100644 --- a/py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/runtime.py +++ b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py @@ -3,12 +3,15 @@ from __future__ import annotations from pathlib import Path -from typing import Any, Sequence, Union +from typing import TYPE_CHECKING, Any, Collection, Sequence, Union, cast +if TYPE_CHECKING: + from torch_tensorrt_executorch_runtime import _Runtime -def _get_runtime(): + +def _get_runtime() -> _Runtime: try: - from torch_tensorrt_executorch_delegate import get_runtime + from torch_tensorrt_executorch_runtime import get_runtime except ImportError as error: raise ImportError( "ExecuTorch Python inference requires the prebuilt delegate. " @@ -26,8 +29,8 @@ def __init__(self, program: Any, data: bytes) -> None: self._program = program @property - def method_names(self): - return self._program.method_names + def method_names(self) -> Collection[str]: + return cast(Collection[str], self._program.method_names) def run(self, inputs: Sequence[Any], method: str = "forward") -> Sequence[Any]: import torch @@ -43,7 +46,7 @@ def run(self, inputs: Sequence[Any], method: str = "forward") -> Sequence[Any]: loaded = self._program.load_method(method) if loaded is None: raise RuntimeError(f"ExecuTorch failed to load method {method!r}") - return loaded.execute(inputs) + return cast(Sequence[Any], loaded.execute(inputs)) def forward(self, *inputs: Any) -> Sequence[Any]: return self.run(inputs, "forward") diff --git a/py/torch_tensorrt/_compile.py b/py/torch_tensorrt/_compile.py index 9b35e9178e..f34384d900 100644 --- a/py/torch_tensorrt/_compile.py +++ b/py/torch_tensorrt/_compile.py @@ -18,6 +18,7 @@ Set, Tuple, Union, + cast, ) import torch @@ -89,11 +90,9 @@ def _has_executorch_exir() -> bool: return False -def _has_executorch_delegate() -> bool: +def _has_executorch_runtime() -> bool: try: - return ( - importlib.util.find_spec("torch_tensorrt_executorch_delegate") is not None - ) + return importlib.util.find_spec("torch_tensorrt_executorch_runtime") is not None except ModuleNotFoundError: return False @@ -325,7 +324,7 @@ def _fx_input_interface( "'arg_inputs' and 'inputs' should not be used at the same time." ) if inputs is not None: - arg_inputs = inputs # type: ignore[assignment] + arg_inputs = inputs if kwarg_inputs is None: kwarg_inputs = {} @@ -424,7 +423,7 @@ def cross_compile_for_windows( "'arg_inputs' and 'inputs' should not be used at the same time." ) - arg_inputs = inputs or arg_inputs # type: ignore[assignment] + arg_inputs = inputs or arg_inputs if kwarg_inputs is None: kwarg_inputs = {} @@ -524,7 +523,7 @@ def convert_method_to_trt_engine( raise AssertionError( "'arg_inputs' and 'inputs' should not be used at the same time." ) - arg_inputs = arg_inputs or inputs # type: ignore[assignment] + arg_inputs = arg_inputs or inputs module_type = _parse_module_type(module) target_ir = _get_target_fe(module_type, ir) @@ -568,11 +567,14 @@ def convert_method_to_trt_engine( module, torchtrt_arg_inputs, kwarg_inputs=torchtrt_kwarg_inputs, **kwargs ) - return dynamo_convert_exported_program_to_serialized_trt_engine( - exp_program, - arg_inputs=tuple(normalized_arg_inputs), - kwarg_inputs=torchtrt_kwarg_inputs, - **kwargs, + return cast( + bytes, + dynamo_convert_exported_program_to_serialized_trt_engine( + exp_program, + arg_inputs=tuple(normalized_arg_inputs), + kwarg_inputs=torchtrt_kwarg_inputs, + **kwargs, + ), ) elif target_ir == _IRType.torch_compile: raise RuntimeError( @@ -612,7 +614,7 @@ def load( file_path (str): Path to file on the disk extra_files (dict[str, Any]): Extra files to load with the model format (Optional[str]): Set to ``"executorch"`` to load a ``.pte`` file - using the separately installed ExecuTorch delegate package. + using the separately installed ExecuTorch runtime package. Example: # Load with extra files. @@ -621,19 +623,17 @@ def load( print(extra_files["foo.txt"]) Raises: - ImportError: If ExecuTorch format is requested without the delegate package + ImportError: If ExecuTorch format is requested without the runtime package ValueError: If the format is unsupported or the file is not a TorchScript or ExportedProgram file """ if format == "executorch": - if not _has_executorch_delegate(): + if not _has_executorch_runtime(): raise ImportError( "Loading an ExecuTorch program requires the prebuilt " "Torch-TensorRT ExecuTorch delegate. Install it with: " 'pip install "torch-tensorrt[executorch]"' ) - from torch_tensorrt_executorch_delegate.runtime import ( - load as load_executorch, - ) + from torch_tensorrt_executorch_runtime.runtime import load as load_executorch return load_executorch(file_path) if format is not None: @@ -790,7 +790,7 @@ def save( CUDA ``.pte``. See :ref:`the ExecuTorch save guide ` for the ``CudaPartitioner`` recipe, its export-time requirements (CUDA backend + nvcc), and the external - ``.ptd`` weight caveats. + ``.pdf`` weight caveats. """ if isinstance(module, CudaGraphsTorchTensorRTModule): module = module.compiled_module @@ -1370,14 +1370,14 @@ def _replace_execute_engine_for_executorch(exp_program: Any) -> Any: def _write_external_tensor_data(executorch_program: Any, file_path: str) -> None: - """Write an ExecuTorch program's external named tensor data (``.ptd``) next to the ``.pte``. + """Write an ExecuTorch program's external named tensor data (``.pdf``) next to the ``.pte``. The CUDA (AOTInductor) backend emits its weights as external named data (``save_data_externally``), which ExecuTorch serializes only via ``write_tensor_data_to_file`` -- ``ExecutorchProgram.write_to_file`` does not persist it. So a partition that carries external weights (e.g. a ``CudaPartitioner`` delegate) would lose its blob and the ``.pte`` could not - load. This writes the ``.ptd`` data file(s) into the ``.pte``'s directory when + load. This writes the ``.pdf`` data file(s) into the ``.pte``'s directory when the program has any; the runtime must then be given those data file(s) (e.g. via the ExecuTorch ``Module`` data-files argument) to load the weights. """ @@ -1386,7 +1386,7 @@ def _write_external_tensor_data(executorch_program: Any, file_path: str) -> None out_dir = os.path.dirname(os.path.abspath(file_path)) executorch_program.write_tensor_data_to_file(out_dir) logger.info( - "Wrote external delegate weights (.ptd) to %s; point the runtime's " + "Wrote external delegate weights (.pdf) to %s; point the runtime's " "data-files at this directory to load them.", out_dir, ) diff --git a/py/torch_tensorrt/executorch/__init__.py b/py/torch_tensorrt/executorch/__init__.py index e9aa87d656..28a84a6878 100644 --- a/py/torch_tensorrt/executorch/__init__.py +++ b/py/torch_tensorrt/executorch/__init__.py @@ -1,7 +1,7 @@ """ExecuTorch compilation and export integration. Runtime loading is provided by the optional -``torch-tensorrt-executorch-delegate`` distribution and dispatched through +``torch-tensorrt-executorch-runtime`` distribution and dispatched through ``torch_tensorrt.load(..., format="executorch")``. """ diff --git a/setup.py b/setup.py index fe1be76f5d..6007ef15cc 100644 --- a/setup.py +++ b/setup.py @@ -184,12 +184,12 @@ def load_dep_info(): __version__ = f"{get_base_version()}.dev0+{get_git_revision_short_hash()}" EXECUTORCH_REQUIREMENT = "executorch>=1.3.1" -EXECUTORCH_DELEGATE_REQUIREMENT = ( - f"torch-tensorrt-executorch-delegate=={__version__}; " "platform_system == 'Linux'" +EXECUTORCH_RUNTIME_REQUIREMENT = ( + f"torch-tensorrt-executorch-runtime=={__version__}; " "platform_system == 'Linux'" ) EXTRAS_REQUIRE = { - "executorch": [EXECUTORCH_REQUIREMENT, EXECUTORCH_DELEGATE_REQUIREMENT], - "all": [EXECUTORCH_REQUIREMENT, EXECUTORCH_DELEGATE_REQUIREMENT], + "executorch": [EXECUTORCH_REQUIREMENT, EXECUTORCH_RUNTIME_REQUIREMENT], + "all": [EXECUTORCH_REQUIREMENT, EXECUTORCH_RUNTIME_REQUIREMENT], } if "--ci" in sys.argv: diff --git a/tests/py/dynamo/executorch/test_api.py b/tests/py/dynamo/executorch/test_api.py index d7782ed60c..04821e1154 100644 --- a/tests/py/dynamo/executorch/test_api.py +++ b/tests/py/dynamo/executorch/test_api.py @@ -57,7 +57,7 @@ def fake_find_spec(name, package=None): def test_load_executorch_error_when_delegate_missing(monkeypatch): from torch_tensorrt import _compile - monkeypatch.setattr(_compile, "_has_executorch_delegate", lambda: False) + monkeypatch.setattr(_compile, "_has_executorch_runtime", lambda: False) with pytest.raises(ImportError, match=r"torch-tensorrt\[executorch\]"): _compile.load("model.pte", format="executorch") @@ -67,14 +67,14 @@ def test_load_executorch_error_when_delegate_missing(monkeypatch): def test_load_executorch_dispatches_to_delegate(monkeypatch): from torch_tensorrt import _compile - delegate = types.ModuleType("torch_tensorrt_executorch_delegate") + delegate = types.ModuleType("torch_tensorrt_executorch_runtime") delegate.__path__ = [] - runtime = types.ModuleType("torch_tensorrt_executorch_delegate.runtime") + runtime = types.ModuleType("torch_tensorrt_executorch_runtime.runtime") sentinel = object() runtime.load = lambda path: (sentinel, path) monkeypatch.setitem(sys.modules, delegate.__name__, delegate) monkeypatch.setitem(sys.modules, runtime.__name__, runtime) - monkeypatch.setattr(_compile, "_has_executorch_delegate", lambda: True) + monkeypatch.setattr(_compile, "_has_executorch_runtime", lambda: True) assert _compile.load("model.pte", format="executorch") == ( sentinel, @@ -97,15 +97,26 @@ def test_public_api_symbols_present(): @pytest.mark.unit -def test_runtime_implementation_is_owned_by_delegate_package(): +def test_runtime_implementation_is_owned_by_runtime_package(): assert not (_REPO_ROOT / "py/torch_tensorrt/executorch/runtime.py").exists() assert ( _REPO_ROOT - / "py/torch-tensorrt-executorch-delegate" - / "torch_tensorrt_executorch_delegate/runtime.py" + / "py/torch-tensorrt-executorch-runtime" + / "torch_tensorrt_executorch_runtime/runtime.py" ).is_file() +@pytest.mark.unit +def test_runtime_extension_has_dependency_wheel_rpaths(): + cmake = ( + _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/native/CMakeLists.txt" + ).read_text(encoding="utf-8") + assert "BUILD_WITH_INSTALL_RPATH ON" in cmake + assert "$ORIGIN/../tensorrt_libs" in cmake + assert "$ORIGIN/../nvidia/cuda_runtime/lib" in cmake + assert "$ORIGIN/../nvidia/cu13/lib" in cmake + + def _setup_tree(): return ast.parse(_SETUP_PY.read_text(encoding="utf-8")) @@ -144,7 +155,7 @@ def test_packaging_declares_executorch_extra(): assert isinstance(requirements, ast.List) for requirement_name in ( "EXECUTORCH_REQUIREMENT", - "EXECUTORCH_DELEGATE_REQUIREMENT", + "EXECUTORCH_RUNTIME_REQUIREMENT", ): assert any( isinstance(requirement, ast.Name) and requirement.id == requirement_name @@ -182,7 +193,7 @@ def test_executorch_is_not_base_install_requirement(): and node.id in { "EXECUTORCH_REQUIREMENT", - "EXECUTORCH_DELEGATE_REQUIREMENT", + "EXECUTORCH_RUNTIME_REQUIREMENT", } for node in ast.walk(function) ) diff --git a/tests/py/dynamo/executorch/test_python_runtime.py b/tests/py/dynamo/executorch/test_python_runtime.py index 28d90ae387..6520dd6159 100644 --- a/tests/py/dynamo/executorch/test_python_runtime.py +++ b/tests/py/dynamo/executorch/test_python_runtime.py @@ -7,11 +7,11 @@ RUNTIME_PATH = ( Path(__file__).parents[4] - / "py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/runtime.py" + / "py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py" ) DELEGATE_PATH = ( Path(__file__).parents[4] - / "py/torch-tensorrt-executorch-delegate/torch_tensorrt_executorch_delegate/__init__.py" + / "py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/__init__.py" ) @@ -54,7 +54,7 @@ def load_program(self, data): def test_load_and_forward(monkeypatch, tmp_path): - delegate = types.ModuleType("torch_tensorrt_executorch_delegate") + delegate = types.ModuleType("torch_tensorrt_executorch_runtime") fake_runtime = FakeRuntime() delegate.get_runtime = lambda: fake_runtime monkeypatch.setitem(sys.modules, delegate.__name__, delegate) @@ -68,7 +68,7 @@ def test_load_and_forward(monkeypatch, tmp_path): def test_unknown_method(monkeypatch, tmp_path): - delegate = types.ModuleType("torch_tensorrt_executorch_delegate") + delegate = types.ModuleType("torch_tensorrt_executorch_runtime") delegate.get_runtime = FakeRuntime monkeypatch.setitem(sys.modules, delegate.__name__, delegate) model = tmp_path / "model.pte" From 9a466e68fd4cab1be8b2eb2384a731c3f828330a Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Wed, 29 Jul 2026 11:51:28 -0700 Subject: [PATCH 14/17] add the documentation suggested by anthony --- .../README.md | 11 ++++++++++ .../runtime.py | 21 +++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/py/torch-tensorrt-executorch-runtime/README.md b/py/torch-tensorrt-executorch-runtime/README.md index 5499ddcb85..44c0e02eea 100644 --- a/py/torch-tensorrt-executorch-runtime/README.md +++ b/py/torch-tensorrt-executorch-runtime/README.md @@ -51,6 +51,17 @@ by the `executorch==1.3.1` wheel. The static ExecuTorch and delegate archives are intermediate build inputs; users receive the final native Python module and do not compile anything. +## Python tensor placement + +The ExecuTorch Python portable runtime uses CPU tensors at its API boundary. +CUDA tensor inputs passed to `Program.run()` or `Program.forward()` are copied +to CPU before dispatch. TensorRT executes the delegated graph on GPU, but the +runtime copies inputs to the device and returns outputs on CPU. + +Consequently, the Python API does not use the backend's device-resident +input/output fast path. Applications that need to keep inputs and outputs on +GPU should use the ExecuTorch C++ runner. + ## Use ```bash diff --git a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py index e93b149188..deb9600e64 100644 --- a/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py +++ b/py/torch-tensorrt-executorch-runtime/torch_tensorrt_executorch_runtime/runtime.py @@ -21,7 +21,14 @@ def _get_runtime() -> _Runtime: class Program: - """A loaded ExecuTorch program backed by TensorRTBackend.""" + """A loaded ExecuTorch program backed by TensorRTBackend. + + The ExecuTorch Python portable runtime executes across a CPU tensor + boundary: CUDA inputs are copied to CPU before dispatch and outputs are + returned on CPU. TensorRT still executes the delegated graph on GPU, but + the device-resident input/output fast path is available only through the + ExecuTorch C++ runner. + """ def __init__(self, program: Any, data: bytes) -> None: # ExecuTorch's BufferDataLoader references this memory without copying it. @@ -33,6 +40,12 @@ def method_names(self) -> Collection[str]: return cast(Collection[str], self._program.method_names) def run(self, inputs: Sequence[Any], method: str = "forward") -> Sequence[Any]: + """Run a method using CPU inputs and return CPU outputs. + + CUDA tensor inputs are copied to CPU before entering the portable + Python runtime. Use the C++ runner when inputs and outputs must remain + device-resident. + """ import torch inputs = tuple( @@ -53,7 +66,11 @@ def forward(self, *inputs: Any) -> Sequence[Any]: def load(path: Union[str, Path]) -> Program: - """Load a `.pte` with the delegate-enabled ExecuTorch Python runtime.""" + """Load a `.pte` with the delegate-enabled ExecuTorch Python runtime. + + External `.pdf` weight files are not supported; weights must be embedded + in the `.pte` file. + """ model_path = Path(path) if not model_path.is_file(): raise FileNotFoundError(f"ExecuTorch model not found: {model_path}") From bf3fa6250402903665b917941c17a2270d672816 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Wed, 29 Jul 2026 13:10:28 -0700 Subject: [PATCH 15/17] resolve comments --- py/torch-tensorrt-executorch-runtime/setup.py | 36 +++++++++++++++++-- tests/py/dynamo/executorch/test_api.py | 30 ++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/py/torch-tensorrt-executorch-runtime/setup.py b/py/torch-tensorrt-executorch-runtime/setup.py index 08eee30759..f8d676ac80 100644 --- a/py/torch-tensorrt-executorch-runtime/setup.py +++ b/py/torch-tensorrt-executorch-runtime/setup.py @@ -21,6 +21,8 @@ REPO_ROOT = HERE.parents[1] BAZEL_TARGET = "//py/torch-tensorrt-executorch-runtime/native:delegate_native" BUILD_NONCE = os.getenv("TORCH_TENSORRT_EXECUTORCH_BUILD_NONCE", uuid.uuid4().hex) +TENSORRT_DISTRIBUTION = "tensorrt-cu13" +CUDA_RUNTIME_DISTRIBUTION = "nvidia-cuda-runtime" def torchtrt_version() -> str: @@ -33,6 +35,31 @@ def torchtrt_version() -> str: return (REPO_ROOT / "version.txt").read_text().strip() +def public_version(version: str) -> str: + """Drop a PEP 440 local suffix that may not be present on package indexes.""" + return version.partition("+")[0] + + +def installed_version(distribution: str) -> str: + """Return the version of a dependency in the native build environment.""" + try: + return importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError as error: + raise RuntimeError( + f"{distribution} must be installed to build the ExecuTorch runtime wheel" + ) from error + + +def require_cuda_13() -> None: + """Reject builds whose native dependencies do not use supported CUDA 13.""" + cuda_version = torch.version.cuda + if cuda_version is None or not cuda_version.startswith("13."): + raise RuntimeError( + "CUDA 13-enabled PyTorch is required to build this wheel " + f"(found CUDA {cuda_version or 'None'})" + ) + + class BazelExtension(Extension): def __init__(self, name: str) -> None: super().__init__(name, sources=[]) @@ -102,7 +129,10 @@ def build_extension(self, ext: Extension) -> None: shutil.copy2(built, output) -executorch_version = importlib.metadata.version("executorch") +require_cuda_13() +executorch_version = installed_version("executorch") +tensorrt_version = installed_version(TENSORRT_DISTRIBUTION) +cuda_runtime_version = installed_version(CUDA_RUNTIME_DISTRIBUTION) setup( name="torch-tensorrt-executorch-runtime", version=torchtrt_version(), @@ -115,9 +145,11 @@ def build_extension(self, ext: Extension) -> None: cmdclass={"build_ext": BazelBuild}, python_requires=">=3.10", install_requires=[ - f"torch=={torch.__version__}", + f"torch=={public_version(torch.__version__)}", f"executorch=={executorch_version}", f"torch-tensorrt=={torchtrt_version()}", + f"{TENSORRT_DISTRIBUTION}=={tensorrt_version}", + f"{CUDA_RUNTIME_DISTRIBUTION}=={cuda_runtime_version}", ], zip_safe=False, ) diff --git a/tests/py/dynamo/executorch/test_api.py b/tests/py/dynamo/executorch/test_api.py index 04821e1154..4ac9cdcf04 100644 --- a/tests/py/dynamo/executorch/test_api.py +++ b/tests/py/dynamo/executorch/test_api.py @@ -94,6 +94,7 @@ def test_public_api_symbols_present(): _REPO_ROOT = Path(__file__).resolve().parents[4] _SETUP_PY = _REPO_ROOT / "setup.py" +_RUNTIME_SETUP_PY = _REPO_ROOT / "py/torch-tensorrt-executorch-runtime/setup.py" @pytest.mark.unit @@ -121,6 +122,10 @@ def _setup_tree(): return ast.parse(_SETUP_PY.read_text(encoding="utf-8")) +def _runtime_setup_tree(): + return ast.parse(_RUNTIME_SETUP_PY.read_text(encoding="utf-8")) + + def _assignment_value(tree, name): for node in tree.body: if isinstance(node, ast.Assign) and any( @@ -138,6 +143,31 @@ def _function_def(tree, name): raise AssertionError(f"Could not find function {name}") +@pytest.mark.unit +def test_runtime_wheel_uses_public_torch_version(): + function = _function_def(_runtime_setup_tree(), "public_version") + namespace = {} + exec( + compile(ast.Module(body=[function], type_ignores=[]), "", "exec"), + namespace, + ) + + assert namespace["public_version"]("2.14.0.dev20260726+cu132") == ( + "2.14.0.dev20260726" + ) + + +@pytest.mark.unit +def test_runtime_wheel_pins_cuda_13_native_dependencies(): + setup_source = _RUNTIME_SETUP_PY.read_text(encoding="utf-8") + assert 'TENSORRT_DISTRIBUTION = "tensorrt-cu13"' in setup_source + assert 'CUDA_RUNTIME_DISTRIBUTION = "nvidia-cuda-runtime"' in setup_source + assert "torch=={public_version(torch.__version__)}" in setup_source + assert "{TENSORRT_DISTRIBUTION}=={tensorrt_version}" in setup_source + assert "{CUDA_RUNTIME_DISTRIBUTION}=={cuda_runtime_version}" in setup_source + assert "nvidia-cuda-runtime-cu12" not in setup_source + + @pytest.mark.unit def test_packaging_declares_executorch_extra(): tree = _setup_tree() From 8aab31b4d584e5db89fca026c1007421b17b4ba8 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Wed, 29 Jul 2026 13:21:29 -0700 Subject: [PATCH 16/17] fix the upload executorch-runtime package path --- .github/workflows/linux-test.yml | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/.github/workflows/linux-test.yml b/.github/workflows/linux-test.yml index c872c6f8f0..a79c869799 100644 --- a/.github/workflows/linux-test.yml +++ b/.github/workflows/linux-test.yml @@ -50,7 +50,7 @@ on: default: false type: boolean upload-artifact: - description: 'Name to give artifacts uploaded from ${RUNNER_ARTIFACT_DIR}' + description: 'Name to give wheel artifacts uploaded from the repository dist directory' default: '' type: string additional-artifact: @@ -184,23 +184,10 @@ jobs: display-options: fEs fail-on-empty: ${{ inputs.fail-on-empty }} - - name: Prepare artifacts for upload + - name: Check documentation for upload working-directory: ${{ inputs.repository }} id: check-artifacts - env: - UPLOAD_ARTIFACT_NAME: ${{ inputs.upload-artifact }} run: | - # Only do these steps if we actually want to upload an artifact - if [[ -n "${UPLOAD_ARTIFACT_NAME}" ]]; then - # If the default execution path is followed then we should get a wheel in the dist/ folder - # attempt to just grab whatever is in there and scoop it all up - if find "dist/" -name "*.whl" >/dev/null 2>/dev/null; then - mv -v dist/*.whl "${RUNNER_ARTIFACT_DIR}/" - fi - # Set to fail upload step if there are no files for upload and expected files for upload - echo 'if-no-files-found=error' >> "${GITHUB_OUTPUT}" - fi - upload_docs=0 # Check if there are things in the documentation folder to upload if find "${RUNNER_DOCS_DIR}" -mindepth 1 -maxdepth 1 | read -r; then @@ -215,8 +202,8 @@ jobs: if: ${{ inputs.upload-artifact != '' }} with: name: ${{ inputs.upload-artifact }} - path: ${{ runner.temp }}/artifacts/ - if-no-files-found: ${{ steps.check-artifacts.outputs.if-no-files-found }} + path: ${{ inputs.repository }}/dist/ + if-no-files-found: error - name: Upload documentation to S3 (if any) uses: seemethere/upload-artifact-s3@v5 From 589df362bb294901dc919de35317164ad4b9d81e Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Wed, 29 Jul 2026 18:52:00 -0700 Subject: [PATCH 17/17] fix the test issue --- .github/workflows/executorch-build-linux.yml | 5 +---- py/torch-tensorrt-executorch-runtime/README.md | 11 +++++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/executorch-build-linux.yml b/.github/workflows/executorch-build-linux.yml index 03bb9e7dd3..f94b60256c 100644 --- a/.github/workflows/executorch-build-linux.yml +++ b/.github/workflows/executorch-build-linux.yml @@ -91,10 +91,7 @@ jobs: export PATH="${RUNNER_TEMP}/bin:${PATH}" bazel --version - executorch_cmake_location="$(bazel query @executorch//:executorch/CMakeLists.txt --output=location)" - export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" - export EXECUTORCH_ROOT="${EXECUTORCH_SOURCE_DIR}" - python -m pip install pyyaml executorch + python -m pip install pyyaml "executorch==1.3.1" export TORCH_TENSORRT_EXECUTORCH_RUNTIME_VERSION="$(python -c 'import importlib.metadata; print(importlib.metadata.version("torch-tensorrt"))')" # Build the no-compile-for-users Python runtime wheel. diff --git a/py/torch-tensorrt-executorch-runtime/README.md b/py/torch-tensorrt-executorch-runtime/README.md index 44c0e02eea..9a524d69b6 100644 --- a/py/torch-tensorrt-executorch-runtime/README.md +++ b/py/torch-tensorrt-executorch-runtime/README.md @@ -35,18 +35,17 @@ of the wheel runtime contract. > PyTorch version. ```bash -executorch_cmake_location="$(bazel query \ - @executorch//:executorch/CMakeLists.txt --output=location)" -export EXECUTORCH_SOURCE_DIR="$(dirname "${executorch_cmake_location%%:*}")" export TensorRT_ROOT=/path/to/TensorRT -python -m pip install executorch==1.3.1 +python -m pip install pyyaml "executorch==1.3.1" python -m pip wheel --no-build-isolation --no-deps \ --wheel-dir dist py/torch-tensorrt-executorch-runtime ``` -The ExecuTorch source commit pinned in `MODULE.bazel` is the revision recorded -by the `executorch==1.3.1` wheel. +The native build obtains the ExecuTorch source through Bazel; no separate +source checkout or `EXECUTORCH_SOURCE_DIR` setting is required. The source +commit pinned in `MODULE.bazel` is the revision recorded by the +`executorch==1.3.1` wheel. The static ExecuTorch and delegate archives are intermediate build inputs; users receive the final native Python module and do not compile anything.