Let a C++ application use the ExecuTorch TensorRT delegate from an install

.github/scripts/verify-executorch-wheel-consumer.sh

@@ -0,0 +1,213 @@
+#!/usr/bin/env bash
+# Verify that an application can consume the ExecuTorch TensorRT delegate from
+# INSTALLED packages, with no source checkout.
+#
+# The existing reference-runner check builds ExecuTorch from source via
+# add_subdirectory, which is the right thing to verify for that path but says
+# nothing about whether the installed package is usable. This script covers the
+# other half: install the delegate, then build and run an application that finds
+# everything through find_package.
+#
+# Exports its own model so it does not depend on another script's scratch
+# directory, and asserts TensorRT actually claimed part of the graph: without that
+# a program with no TensorRT delegate would still load and run, and the check would
+# prove nothing.
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+work_dir="$(mktemp -d)"
+trap 'rm -rf "${work_dir}"' EXIT
+
+python_executable="${PYTHON_EXECUTABLE:-python}"
+pte_path="${work_dir}/model.pte"
+expected_path="${work_dir}/expected.txt"
+
+# The example needs CMake 3.28, which is what the ExecuTorch package it consumes
+# requires. Say so plainly rather than letting a version error
+# surface from three layers down.
+if ! command -v cmake >/dev/null 2>&1; then
+  echo "SKIP: cmake is not available, cannot build the example" >&2
+  exit 0
+fi
+cmake_version="$(cmake --version | head -1 | awk '{print $3}')"
+if [ "$(printf '%s\n3.28\n' "${cmake_version}" | sort -V | head -1)" != "3.28" ]; then
+  echo "SKIP: cmake ${cmake_version} is older than the 3.28 the example needs" >&2
+  exit 0
+fi
+echo "using cmake ${cmake_version}"
+
+echo "=== locating the installed ExecuTorch package ==="
+executorch_cmake_dir="$(
+  cd "${work_dir}" && "${python_executable}" - <<'PY'
+import importlib.util
+import sys
+from pathlib import Path
+
+spec = importlib.util.find_spec("executorch")
+locations = list(getattr(spec, "submodule_search_locations", []) or []) if spec else []
+if not locations:
+    print("SKIP no installed ExecuTorch package")
+    raise SystemExit(0)
+package = Path(locations[0])
+config = package / "share" / "cmake" / "executorch-config.cmake"
+if not config.is_file():
+    print("SKIP the installed package ships no CMake config")
+    raise SystemExit(0)
+# A released wheel predating the separately shipped runtime has the config but no
+# linkable runtime, so there is nothing for an application to consume yet. That is
+# a missing feature upstream rather than a failure of this check.
+if "executorch::runtime" not in config.read_text():
+    print("SKIP the installed package offers no shared runtime target")
+    raise SystemExit(0)
+print(config.parent)
+PY
+)"
+
+case "${executorch_cmake_dir}" in
+  SKIP*)
+    echo "${executorch_cmake_dir#SKIP }: nothing to verify against, skipping"
+    exit 0
+    ;;
+esac
+echo "found: ${executorch_cmake_dir}"
+
+echo "=== exporting a model with a TensorRT delegate ==="
+(cd "${work_dir}" && "${python_executable}" - "${pte_path}" "${expected_path}" <<'PY'
+import sys
+
+import torch
+import torch_tensorrt
+
+pte_path, expected_path = sys.argv[1], sys.argv[2]
+
+
+class Model(torch.nn.Module):
+    def forward(self, x):
+        return torch.tanh(x * 2.0 + 1.0)
+
+
+model = Model().eval().cuda()
+example = torch.ones((2, 3, 4, 4)).cuda()
+compiled = torch_tensorrt.dynamo.compile(
+    torch.export.export(model, (example,)),
+    arg_inputs=[torch_tensorrt.Input(shape=tuple(example.shape), dtype=example.dtype)],
+    min_block_size=1,
+)
+torch_tensorrt.save(
+    compiled, pte_path, output_format="executorch", arg_inputs=(example,), retrace=False
+)
+
+from executorch.exir._serialize._program import deserialize_pte_binary
+
+with open(pte_path, "rb") as handle:
+    program = deserialize_pte_binary(handle.read()).program
+ids = [d.id for plan in program.execution_plan for d in plan.delegates]
+if ids.count("TensorRTBackend") < 1:
+    sys.exit(f"TensorRT claimed no part of the graph, so this proves nothing: {ids}")
+print("delegates:", ids)
+
+# The application fills inputs with ones, so the reference uses the same input.
+with torch.no_grad():
+    reference = model(torch.ones_like(example))
+with open(expected_path, "w") as handle:
+    handle.write(" ".join(f"{v:.6f}" for v in reference.detach().cpu().flatten().tolist()))
+PY
+)
+
+
+echo "=== locating the TensorRT SDK ==="
+# The delegate links TensorRT directly, and the installed library carries only
+# $ORIGIN entries, so the SDK has to be supplied at configure time and its
+# library directory again when the application links. On distributions where
+# TensorRT is a system library this is already satisfied and the search finds
+# nothing, which is why a failure here is not fatal.
+tensorrt_root="${TensorRT_ROOT:-}"
+if [ -z "${tensorrt_root}" ] && command -v bazel >/dev/null 2>&1; then
+  output_base="$(bazel info output_base 2>/dev/null || true)"
+  if [ -n "${output_base}" ]; then
+    trt_header="$(
+      find -L "${output_base}/external" \
+        \( -path "*/+*tensorrt/include/NvInfer.h" \
+           -o -path "*/tensorrt/include/NvInfer.h" \) \
+        -print -quit 2>/dev/null || true
+    )"
+    if [ -n "${trt_header}" ]; then
+      tensorrt_root="$(dirname "$(dirname "${trt_header}")")"
+    fi
+  fi
+fi
+
+delegate_cmake_args=()
+consumer_cmake_args=()
+if [ -n "${tensorrt_root}" ]; then
+  echo "using the TensorRT SDK at ${tensorrt_root}"
+  delegate_cmake_args+=("-DTensorRT_ROOT=${tensorrt_root}")
+  consumer_cmake_args+=(
+    # --disable-new-dtags so the path lands in DT_RPATH rather than DT_RUNPATH. A
+    # DT_RUNPATH on the application is not used when resolving what its own libraries
+    # need, so the delegate's transitive TensorRT dependency would go unfound at load
+    # time even though the link succeeded. The delegate target itself already forces
+    # DT_RPATH for the same reason.
+    "-DCMAKE_EXE_LINKER_FLAGS=-L${tensorrt_root}/lib -Wl,-rpath,${tensorrt_root}/lib -Wl,--disable-new-dtags"
+  )
+else
+  echo "no separate TensorRT SDK found, assuming it is installed system wide"
+fi
+
+echo "=== installing the shared delegate ==="
+delegate_prefix="${work_dir}/delegate-install"
+cmake -S "${repo_root}/cpp/src/torch_tensorrt/executorch" \
+  -B "${work_dir}/delegate-build" \
+  -DTORCHTRT_EXECUTORCH_BUILD_SHARED_DELEGATE=ON \
+  -DCMAKE_PREFIX_PATH="${executorch_cmake_dir}" \
+  -DCMAKE_INSTALL_PREFIX="${delegate_prefix}" \
+  -DCMAKE_INSTALL_LIBDIR=lib \
+  "${delegate_cmake_args[@]}"
+cmake --build "${work_dir}/delegate-build" -j"${MAX_JOBS:-$(nproc)}"
+cmake --install "${work_dir}/delegate-build"
+
+# Pinned to lib above, but check both names anyway: the default library directory
+# is lib64 on several distributions, and a wrong path here would report a failed
+# install after a completely successful build.
+delegate_library="$(
+  find "${delegate_prefix}" -name "libexecutorch_backend_tensorrt.so*" -type f | head -1
+)"
+if [ -z "${delegate_library}" ]; then
+  echo "ERROR: the delegate did not install a shared library under ${delegate_prefix}" >&2
+  find "${delegate_prefix}" -type f | head -20 >&2
+  exit 1
+fi
+echo "installed: ${delegate_library#"${delegate_prefix}"/}"
+
+echo "=== building the application against installed packages only ==="
+app_build="${work_dir}/app-build"
+cmake -S "${repo_root}/examples/executorch_wheel_runner" \
+  -B "${app_build}" \
+  -DCMAKE_PREFIX_PATH="${executorch_cmake_dir};${delegate_prefix}" \
+  "${consumer_cmake_args[@]}"
+cmake --build "${app_build}" -j"${MAX_JOBS:-$(nproc)}"
+
+# A source build would have configured ExecuTorch itself, leaving a second cache
+# behind. Exactly one means nothing was built from source.
+cache_count="$(find "${app_build}" -name CMakeCache.txt | wc -l)"
+if [ "${cache_count}" -ne 1 ]; then
+  echo "ERROR: expected one CMake cache, found ${cache_count}; something was built from source" >&2
+  exit 1
+fi
+
+# The delegate registers itself from a static initializer, so nothing in the
+# application references it and a linker may drop it. If that happens the backend
+# is missing at runtime, so check the dependency is recorded.
+app_path="${app_build}/executorch_wheel_runner"
+if command -v readelf >/dev/null 2>&1; then
+  if ! readelf -d "${app_path}" | grep -q "libexecutorch_backend_tensorrt"; then
+    echo "ERROR: the delegate was dropped from the application's dependencies" >&2
+    exit 1
+  fi
+  echo "the delegate is recorded in the application's dependencies"
+fi
+
+echo "=== running it ==="
+"${app_path}" --model "${pte_path}" --expected "${expected_path}" --tolerance 1e-3
+
+echo "SUCCESS: the delegate is usable from an installed package"

.github/workflows/executorch-static-linux.yml

@@ -98,3 +98,7 @@ jobs:
         # this is to verify the end user's workflow
         python -m pip install pyyaml "executorch>=1.3.1"
         .github/scripts/verify-executorch-reference-runner.sh
+        # The check above builds ExecuTorch from source. This one verifies the other
+        # half: that an application can consume the delegate from an installed
+        # package, with no source checkout.
+        .github/scripts/verify-executorch-wheel-consumer.sh

BUILD.bazel

@@ -143,9 +143,9 @@ genrule(
 )
 
 genrule(
-    name = "executorch_trt_backend_archive",
+    name = "executorch_backend_tensorrt_archive",
     srcs = ["//cpp:tensorrt_executorch_backend"],
-    outs = ["libexecutorch_trt_backend.a"],
+    outs = ["libexecutorch_backend_tensorrt.a"],
     cmd = """
 set -e
 for f in $(locations //cpp:tensorrt_executorch_backend); do
@@ -159,15 +159,15 @@ exit 1
 )
 
 alias(
-    name = "executorch_trt_backend",
-    actual = ":executorch_trt_backend_archive",
+    name = "executorch_backend_tensorrt",
+    actual = ":executorch_backend_tensorrt_archive",
 )
 
 pkg_files(
     name = "executorch_lib_pkg_files",
     srcs = [
         ":executorch_core_archive",
-        ":executorch_trt_backend_archive",
+        ":executorch_backend_tensorrt_archive",
     ],
     prefix = "lib/",
     visibility = ["//visibility:public"],

cmake/torch_tensorrt_executorchConfig.cmake.in

@@ -0,0 +1,67 @@
+@PACKAGE_INIT@
+
+# Package config for the ExecuTorch TensorRT delegate.
+#
+# A C++ application that wants to run a .pte containing TensorRT partitions needs
+# three things: the ExecuTorch runtime, the TensorRT libraries, and this delegate.
+# The delegate registers itself from a static initializer, so the application
+# never references any of its symbols directly. Consuming it through an imported
+# target keeps that link correct without the application knowing the details.
+
+include(CMakeFindDependencyMacro)
+
+# The delegate resolves the ExecuTorch runtime at load time, so a consumer needs the same
+# runtime package this was built against. The version found at build time is recorded here
+# so a prefix that resolves a different ExecuTorch is refused at configure time rather
+# than producing a delegate paired with a runtime it cannot work with.
+set(_torchtrt_executorch_build_version "@TORCHTRT_EXECUTORCH_VERSION@")
+if(_torchtrt_executorch_build_version)
+  # EXACT because there is no promise of a stable C++ ABI across ExecuTorch
+  # releases. A newer same-major runtime would satisfy an inexact request and then
+  # fail to load the delegate, so the version this was built against is required
+  # rather than preferred.
+  find_dependency(executorch "${_torchtrt_executorch_build_version}" EXACT CONFIG)
+else()
+  find_dependency(executorch CONFIG)
+endif()
+unset(_torchtrt_executorch_build_version)
+
+include("${CMAKE_CURRENT_LIST_DIR}/torchtrtExecuTorchTargets.cmake")
+
+# The export set names the concrete library. Offer the stable name consumers use,
+# so an application does not depend on which of the two library flavors it got.
+if(NOT TARGET torchtrt::backend_tensorrt)
+    if(TARGET torchtrt::backend_tensorrt_shared)
+        # The delegate is useful because a static initializer registers the
+        # backend. Nothing in an application references a symbol from it, so a
+        # normal link drops it and the backend is never registered. The target
+        # carries the retention itself, so the documented two-line usage is
+        # enough and a consumer does not have to know this.
+        #
+        # The library goes inside the single option: a SHELL: string would split
+        # on spaces, and separate options repeat identical text that CMake
+        # de-duplicates, which would unscope every library after the first.
+        if(NOT (APPLE OR MSVC))
+            set_property(
+                TARGET torchtrt::backend_tensorrt_shared
+                APPEND
+                PROPERTY
+                    INTERFACE_LINK_OPTIONS
+                    "LINKER:--push-state,--no-as-needed,$<TARGET_FILE:torchtrt::backend_tensorrt_shared>,--pop-state"
+            )
+        elseif(APPLE)
+            set_property(
+                TARGET torchtrt::backend_tensorrt_shared
+                APPEND
+                PROPERTY
+                    INTERFACE_LINK_OPTIONS
+                    "SHELL:LINKER:-force_load,$<TARGET_FILE:torchtrt::backend_tensorrt_shared>"
+            )
+        endif()
+        add_library(torchtrt::backend_tensorrt ALIAS
+                    torchtrt::backend_tensorrt_shared
+        )
+    endif()
+endif()
+
+check_required_components(torch_tensorrt_executorch)