From 656de46310619496faadf71dadca90acee2a2138 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Wed, 4 Nov 2026 11:08:42 -0800 Subject: [PATCH 1/3] Ignore a torch that is not ready when the Python runtime reads inputs The Python runtime checks each input against torch.Tensor through the torch it finds in sys.modules, and keeps the first one it finds. That can be a torch that another thread is still importing, which has no Tensor yet, or the None that blocks an import. If torch was not imported when the bindings loaded, for example with EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT set to executorch, calls then raise AttributeError, even ones with only NumPy inputs. Now the runtime uses torch only once it is fully imported, and keeps looking until then. Tested on Linux x86_64 with a source build: a new test makes the call while another thread is halfway through importing a torch package, and with sys.modules["torch"] set to None. Without this change both cases fail with AttributeError. --- .../pybindings/pybindings_executorch_result.h | 13 +++-- extension/pybindings/test/test_pybindings.py | 54 +++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/extension/pybindings/pybindings_executorch_result.h b/extension/pybindings/pybindings_executorch_result.h index 1397c46186a..5b863bdc685 100644 --- a/extension/pybindings/pybindings_executorch_result.h +++ b/extension/pybindings/pybindings_executorch_result.h @@ -97,10 +97,17 @@ inline PyObject* loaded_torch_module() { static PyObject* torch_module = nullptr; if (torch_module == nullptr) { PyObject* loaded = PyDict_GetItemString(PyImport_GetModuleDict(), "torch"); - if (loaded != nullptr) { - Py_INCREF(loaded); - torch_module = loaded; + if (loaded == nullptr || loaded == Py_None) { + return nullptr; } + // A torch that another thread is still importing is not usable yet. + const auto spec = py::getattr(py::handle(loaded), "__spec__", py::none()); + if (!spec.is_none() && + py::getattr(spec, "_initializing", py::bool_(false)).cast()) { + return nullptr; + } + Py_INCREF(loaded); + torch_module = loaded; } return torch_module; } diff --git a/extension/pybindings/test/test_pybindings.py b/extension/pybindings/test/test_pybindings.py index 74d719be0a7..a4575bfdfa6 100644 --- a/extension/pybindings/test/test_pybindings.py +++ b/extension/pybindings/test/test_pybindings.py @@ -273,6 +273,60 @@ def find_spec(self, fullname, path=None, target=None): completed.stdout + completed.stderr, ) + def test_torch_that_is_not_ready_is_ignored(self): + if self.kernel_mode != "portable": + self.skipTest("only portable bindings look torch up in sys.modules") + program, _ = create_program(ModuleAddSingleInput()) + script = """ +import os +import sys +import threading + +import numpy as np + +program_path, case = sys.argv[1:] +if case == "none": + sys.modules["torch"] = None +else: + # A torch that another thread is still importing. + package = os.path.join(os.path.dirname(program_path), "partial", "torch") + os.makedirs(package) + with open(os.path.join(package, "__init__.py"), "w") as source: + source.write("import __main__\\n__main__.started.set()\\n__main__.release.wait()\\n") + sys.path.insert(0, os.path.dirname(package)) + started, release = threading.Event(), threading.Event() + importer = threading.Thread(target=__import__, args=("torch",), daemon=True) + importer.start() + started.wait() +from executorch.extension.pybindings import portable_lib +with open(program_path, "rb") as source: + module = portable_lib._load_for_executorch_from_buffer(source.read()) +output = module(np.ones((2, 2), dtype=np.float32))[0] +assert isinstance(output, portable_lib.ExecuTorchResult), type(output) +if case != "none": + release.set() + importer.join() +""" + with tempfile.TemporaryDirectory() as directory: + program_path = os.path.join(directory, "add.pte") + with open(program_path, "wb") as handle: + handle.write(program.buffer) + environment = dict( + os.environ, EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT="executorch" + ) + for case in ("none", "importing"): + with self.subTest(case=case): + completed = subprocess.run( + [sys.executable, "-c", script, program_path, case], + env=environment, + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual( + completed.returncode, 0, completed.stdout + completed.stderr + ) + def test_empty_numpy_buffer_input(self): exported_program, inputs = create_program(ModuleAddEmpty()) array = np.zeros((3, 0), dtype=np.float32) From c17afa31cb8cdaaf18ab02a401a65c0ad1b56d4c Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Wed, 4 Nov 2026 11:08:42 -0800 Subject: [PATCH 2/3] Make bool(), int() and float() of an ExecuTorchResult raise TypeError An ExecuTorchResult supports the buffer protocol but has no truth value or number conversion of its own. So Python falls back to defaults that can give wrong answers without an error: bool(result) is always True, and int() and float() parse the raw bytes as text, so a uint8 result that holds 55 (the character "7") gives 7. Now all three raise TypeError and say how to convert the result first: bool(result) # TypeError: ExecuTorchResult has no truth value. Convert it first, for # example with numpy.asarray(result) or torch.from_dlpack(result). Tested with the standalone no-ATen build: a new test checks that all three raise TypeError, and it fails without this change. --- extension/pybindings/README.md | 4 +++- extension/pybindings/pybindings.cpp | 15 +++++++++++++++ .../pybindings/test/test_pybindings_no_aten.py | 15 +++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/extension/pybindings/README.md b/extension/pybindings/README.md index 62ccff747c4..bccfc52548b 100644 --- a/extension/pybindings/README.md +++ b/extension/pybindings/README.md @@ -103,7 +103,9 @@ from executorch.extension.pybindings import portable_lib ``` `ExecuTorchResult` preserves shape, strides, dtype, and device. CPU results expose -the Python buffer protocol so NumPy can read them without another copy. Without +the Python buffer protocol so NumPy can read them without another copy. A result +is not a number or a truth value: `bool()`, `int()`, and `float()` raise +`TypeError`, so convert it first, for example with `np.asarray(output)`. Without the environment setting, the bindings fall back to `ExecuTorchResult` if PyTorch cannot be imported. Explicitly selecting `torch` requires it to import successfully. ATen-mode bindings continue to return native PyTorch tensors diff --git a/extension/pybindings/pybindings.cpp b/extension/pybindings/pybindings.cpp index 17b4935a73d..b002d4956d2 100644 --- a/extension/pybindings/pybindings.cpp +++ b/extension/pybindings/pybindings.cpp @@ -5447,6 +5447,15 @@ py::dict get_option(const std::string& backend_name, const py::dict& options) { return result; } +auto refuse_scalar_conversion(const char* problem) { + return [problem](const PyExecuTorchResult&) -> py::object { + throw py::type_error( + std::string("ExecuTorchResult ") + problem + + ". Convert it first, for example with numpy.asarray(result) or " + "torch.from_dlpack(result)."); + }; +} + } // namespace PYBIND11_MODULE(EXECUTORCH_PYTHON_MODULE_NAME, m) { @@ -5579,6 +5588,12 @@ PYBIND11_MODULE(EXECUTORCH_PYTHON_MODULE_NAME, m) { "Copy a dense CPU buffer, or a CPU ExecuTorchResult, to a CUDA " "device, for use as a method input. Requires bindings built with " "CUDA support.") + // Python would otherwise treat a result as always true, and int() and + // float() would parse its bytes as text. + .def("__bool__", refuse_scalar_conversion("has no truth value")) + .def("__int__", refuse_scalar_conversion("cannot be converted to int")) + .def( + "__float__", refuse_scalar_conversion("cannot be converted to float")) .def("__repr__", &PyExecuTorchResult::repr); #ifndef USE_ATEN_LIB py::class_< diff --git a/extension/pybindings/test/test_pybindings_no_aten.py b/extension/pybindings/test/test_pybindings_no_aten.py index 8488ac01e6c..0b8ee2673ba 100644 --- a/extension/pybindings/test/test_pybindings_no_aten.py +++ b/extension/pybindings/test/test_pybindings_no_aten.py @@ -211,6 +211,21 @@ def test_executorch_result_to_host_and_repr(self) -> None: self.assertFalse(np.shares_memory(np.asarray(copy), np.asarray(output))) np.testing.assert_array_equal(np.asarray(copy), np.array([3.0])) + def test_executorch_result_is_not_a_bool_or_number(self) -> None: + if runtime._tensor_output != "executorch": + self.skipTest("ExecuTorchResult output was not selected") + with open(os.environ["EXECUTORCH_PYBIND_TEST_PTE"], "rb") as program_file: + program_data = program_file.read() + module = runtime._load_for_executorch_from_buffer(program_data) + output = module( + (np.array([1.0], dtype=np.float32), np.array([2.0], dtype=np.float32)) + )[0] + + for convert in (bool, int, float): + with self.subTest(convert=convert.__name__): + with self.assertRaisesRegex(TypeError, r"numpy\.asarray\(result\)"): + convert(output) + def test_executorch_result_from_host_requires_cuda(self) -> None: # A build with CUDA support uploads instead; the CUDA suite covers it. if "CudaBackend" in runtime._get_registered_backend_names(): From 00ffa2e663c188cc8fb92313b26940af6d651e65 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Wed, 4 Nov 2026 11:17:45 -0800 Subject: [PATCH 3/3] Import torch only when the Python runtime returns torch tensors, and warn before outputs follow inputs Importing executorch.runtime imports PyTorch whenever it is installed, even for a program that only passes NumPy arrays. Only setting EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT=executorch avoids that, and never loading PyTorch for NumPy callers would change what they get back. So that change has to be announced first. Without the setting, importing executorch.runtime no longer imports torch. The outputs stay as they are today: torch.Tensor whenever PyTorch is installed, and ExecuTorchResult without it. So a program with only NumPy inputs still loads torch, at its first call that returns a tensor. A future release will make tensor outputs follow each call's tensor inputs, including inputs kept from an earlier call: - any torch.Tensor input: torch.Tensor outputs; - only NumPy, other buffer, DLPack or ExecuTorchResult inputs: ExecuTorchResult outputs, even when PyTorch is installed; - no tensor inputs: torch.Tensor outputs only if torch is already imported. get_attribute() and bundled program verification count as calls without tensor inputs. The first call whose outputs will change shows one FutureWarning, which says how to keep torch.Tensor outputs or get ExecuTorchResult outputs now. A call that runs the method picks the type first, so with an error filter for this FutureWarning, such a call raises before it runs the method (its inputs are already installed). A torch that is installed but fails to import, even with an ImportError, raises its own error, with a note on how to skip torch. EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT still fixes the type: "executorch" never imports torch, and "torch" imports it when ExecuTorch is imported. Tested on Linux x86_64 with Python 3.12 and a CUDA build of PyTorch: the pybindings, runtime, bundled program and CUDA DLPack suites give the same results as on main, and the standalone no-ATen job passes with and without PyTorch. Importing executorch.runtime takes 0.04 s and 18 MB of peak memory instead of 1.9 s and 506 MB. A new test runs each case of the rule in a fresh process, and each of its checks fails when its part of the change is removed. --- docs/source/runtime-python-api-reference.rst | 16 +- extension/pybindings/README.md | 38 +- extension/pybindings/portable_lib.py | 27 +- extension/pybindings/pybindings.cpp | 261 ++++++++-- .../pybindings/pybindings_executorch_result.h | 4 +- extension/pybindings/test/test_pybindings.py | 461 ++++++++++++++---- .../test/test_pybindings_no_aten.py | 8 +- runtime/__init__.py | 16 +- 8 files changed, 646 insertions(+), 185 deletions(-) diff --git a/docs/source/runtime-python-api-reference.rst b/docs/source/runtime-python-api-reference.rst index 87c8c8facd1..7979496a124 100644 --- a/docs/source/runtime-python-api-reference.rst +++ b/docs/source/runtime-python-api-reference.rst @@ -43,7 +43,21 @@ Other buffers, such as ``memoryview``, work as inputs too. NumPy has no bfloat16 For data on a GPU, use DLPack, the standard way to share tensors between Python libraries. This needs ExecuTorch built with CUDA support. Pass a CuPy array to a method that takes CUDA inputs. Read a CUDA result with ``cupy.from_dlpack(out)``, or copy it to the CPU with ``out.to_host()``. ``ExecuTorchResult.from_host()`` copies CPU data to a CUDA device. -To get ``ExecuTorchResult`` values even when PyTorch is installed, set ``EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT=executorch`` before importing ExecuTorch. +Output types +~~~~~~~~~~~~ + +Importing ``executorch.runtime`` does not import PyTorch, unless the setting below asks for ``torch``. When PyTorch is installed, ``Method.execute`` returns ``torch.Tensor`` outputs, and the first call that returns one imports ``torch``. Without PyTorch, it returns ``ExecuTorchResult`` outputs. If PyTorch is installed but fails to import, every call that returns a tensor raises the import error; set the setting below to ``executorch`` to run without PyTorch. + +A future release will make the outputs follow the tensor inputs of each call. A call with any ``torch.Tensor`` input will return ``torch.Tensor`` outputs. A call whose tensor inputs are all NumPy arrays, other buffers, DLPack objects, or ``ExecuTorchResult`` values will return ``ExecuTorchResult`` outputs, even when PyTorch is installed. A call with no tensor inputs will return ``torch.Tensor`` outputs only if ``torch`` is already imported. The first call whose outputs will change shows a ``FutureWarning``. + +To choose the output type now, set ``EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT`` before importing ExecuTorch. With ``executorch``, every call returns ``ExecuTorchResult`` outputs and ExecuTorch never imports PyTorch. With ``torch``, every call returns ``torch.Tensor`` outputs, and importing ExecuTorch imports PyTorch. + +.. code-block:: python + + import os + + os.environ["EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT"] = "executorch" + from executorch.runtime import Runtime CUDA streams ~~~~~~~~~~~~ diff --git a/extension/pybindings/README.md b/extension/pybindings/README.md index bccfc52548b..86119a300e1 100644 --- a/extension/pybindings/README.md +++ b/extension/pybindings/README.md @@ -41,10 +41,14 @@ execution can share the caller's device allocation. input `i`, as `"cpu"` or `"cuda:N"`, and `output_tensor_meta(i).device()` does the same for its outputs. -When PyTorch is available, importing `portable_lib` imports it and portable -bindings return `torch.Tensor` outputs. When it cannot be imported, the default -output type is `ExecuTorchResult`. Output conversion uses Python APIs and -DLPack; the extension does not link against PyTorch. +Unless `EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT` is `torch`, importing +`portable_lib` does not import PyTorch. When PyTorch is installed, portable +bindings return `torch.Tensor` outputs, and the first call that returns one +imports `torch`. When PyTorch is not installed, outputs are `ExecuTorchResult` +values. When it is installed but fails to import, every call that returns a +tensor raises the import error; set the variable to `executorch` to run +without PyTorch. Output conversion uses Python APIs and DLPack; the extension +does not link against PyTorch. `clone_outputs=True` returns storage that later executions do not overwrite, on the output's original device. With `clone_outputs=False`, results share runtime or input storage and later execution may overwrite that storage. @@ -91,9 +95,16 @@ with `clone_outputs=False` lives in the method's output memory, so passing it back in as an input means each run reads what the run before it wrote there; pass a copied result instead. -To select `ExecuTorchResult` outputs even when PyTorch is installed, set the -environment variable before importing the bindings. The setting is fixed when -the extension loads, so importing PyTorch later does not change the output type. +A future release will make tensor outputs follow the tensor inputs of each +call, including inputs kept from an earlier call. A call with any torch tensor +input will return torch tensors. A call whose tensor inputs are all NumPy +arrays, other buffers, DLPack objects, or `ExecuTorchResult` values will return +`ExecuTorchResult` values, even when PyTorch is installed. A call with no tensor +inputs will return torch tensors only if `torch` is already imported. The first +call whose outputs will change shows a `FutureWarning`. To choose the output +type now, set `EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT` to `executorch` or `torch` +before importing the bindings. The setting is fixed when the extension loads, +and `executorch` never imports PyTorch. ```python import os @@ -105,11 +116,9 @@ from executorch.extension.pybindings import portable_lib `ExecuTorchResult` preserves shape, strides, dtype, and device. CPU results expose the Python buffer protocol so NumPy can read them without another copy. A result is not a number or a truth value: `bool()`, `int()`, and `float()` raise -`TypeError`, so convert it first, for example with `np.asarray(output)`. Without -the environment setting, the bindings fall back to `ExecuTorchResult` if -PyTorch cannot be imported. Explicitly selecting `torch` requires it to import -successfully. ATen-mode bindings continue to return native PyTorch tensors -directly. +`TypeError`, so convert it first, for example with `np.asarray(output)`. +Explicitly selecting `torch` requires it to import successfully. ATen-mode +bindings continue to return native PyTorch tensors directly. ```python output = module.forward([numpy_input])[0] @@ -138,8 +147,9 @@ result = np.asarray(output.to_host()) Copying CUDA memory requires CUDA-enabled bindings: `from_host()`, `to_host()` of a CUDA result, and a CUDA `ExecuTorchResult` with `clone_outputs=True`. Bindings built without CUDA support synchronize through Python Torch instead, -so while torch is loaded they still read CUDA inputs, torch tensors or DLPack, -and return borrowed CUDA results. +so once torch is imported, by you or by a call that returns torch tensors, they +still read CUDA inputs, torch tensors or DLPack, and return borrowed CUDA +results. Execution is synchronous at the Python boundary: a call returns after its device work has finished, so its outputs are ready on any stream. How long a call with diff --git a/extension/pybindings/portable_lib.py b/extension/pybindings/portable_lib.py index b936acaeb2f..3dc40d0d455 100644 --- a/extension/pybindings/portable_lib.py +++ b/extension/pybindings/portable_lib.py @@ -28,23 +28,19 @@ class _LightweightExperimentalWarning(DeprecationWarning): _TENSOR_OUTPUT_ENV = "EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT" _tensor_output = os.environ.get(_TENSOR_OUTPUT_ENV) -if _tensor_output in (None, "torch"): +# Without the setting, the first call that returns torch.Tensor outputs imports +# torch, so importing ExecuTorch does not load it. +if _tensor_output == "torch": try: - # Preserve the existing API: importing portable_lib also imports Torch, - # and tensor outputs are torch.Tensor objects. - import torch as _torch + _import_module("torch") except ImportError as error: - if _tensor_output == "torch": - raise ImportError( - "ExecuTorch portable pybindings were configured to return " - "torch.Tensor outputs, but PyTorch could not be imported. " - f"Install PyTorch or set {_TENSOR_OUTPUT_ENV}=executorch " - "before importing portable_lib." - ) from error - _torch = None -elif _tensor_output == "executorch": - _torch = None -else: + raise ImportError( + "ExecuTorch portable pybindings were configured to return " + "torch.Tensor outputs, but PyTorch could not be imported. " + f"Install PyTorch or set {_TENSOR_OUTPUT_ENV}=executorch " + "before importing portable_lib." + ) from error +elif _tensor_output not in (None, "executorch"): raise ValueError( f"{_TENSOR_OUTPUT_ENV} must be either 'torch' or 'executorch', " f"but was {_tensor_output!r}" @@ -152,7 +148,6 @@ class _LightweightExperimentalWarning(DeprecationWarning): del _bindings del _tensor_output del _TENSOR_OUTPUT_ENV -del _torch del _warning_category del _import_module del _exir_warnings diff --git a/extension/pybindings/pybindings.cpp b/extension/pybindings/pybindings.cpp index b002d4956d2..21dd211f2c8 100644 --- a/extension/pybindings/pybindings.cpp +++ b/extension/pybindings/pybindings.cpp @@ -728,38 +728,149 @@ struct RuntimeErrorScope final { std::unique_ptr logs_; }; +// The Python type of a call's tensor outputs. ATen bindings always return +// torch tensors. +enum class PortableTensorOutput { Torch, ExecuTorch }; + #ifndef USE_ATEN_LIB constexpr char kTensorOutputEnvironmentVariable[] = "EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT"; -enum class PortableTensorOutput { Torch, ExecuTorch }; - -PortableTensorOutput& portable_tensor_output() { - static PortableTensorOutput output = PortableTensorOutput::Torch; +constexpr char kTensorOutputWarning[] = + "ExecuTorch returned torch.Tensor outputs for a call without torch.Tensor " + "inputs. A future release will return ExecuTorchResult outputs for such a " + "call, unless it has no tensor inputs and torch is already imported. To " + "keep torch.Tensor outputs, pass torch.Tensor inputs, or import torch " + "before a call without tensor inputs. To choose the type for every call, " + "set EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT to torch or executorch before " + "importing ExecuTorch. With executorch, PyTorch is not loaded."; + +// Set from EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT when the extension loads. +std::optional& tensor_output_override() { + static std::optional output; return output; } void configure_portable_tensor_output() { const char* value = std::getenv(kTensorOutputEnvironmentVariable); - if (value != nullptr && std::strcmp(value, "executorch") == 0) { - portable_tensor_output() = PortableTensorOutput::ExecuTorch; + if (value == nullptr) { return; } - if (value != nullptr && std::strcmp(value, "torch") != 0) { + if (std::strcmp(value, "executorch") == 0) { + tensor_output_override() = PortableTensorOutput::ExecuTorch; + return; + } + if (std::strcmp(value, "torch") != 0) { throw py::value_error( std::string(kTensorOutputEnvironmentVariable) + " must be either 'torch' or 'executorch', but was '" + value + "'"); } - try { - py::module_::import("torch"); - (void)loaded_torch_module(); - portable_tensor_output() = PortableTensorOutput::Torch; - } catch (py::error_already_set& error) { - if (value != nullptr || !error.matches(PyExc_ImportError)) { - throw; + py::module_::import("torch"); + tensor_output_override() = PortableTensorOutput::Torch; +} + +// Points a warning at the code that called ExecuTorch rather than at +// ExecuTorch's own Python wrappers, such as Method.execute. +int caller_stacklevel() { + int level = 1; + for (auto frame = py::reinterpret_borrow( + reinterpret_cast(PyEval_GetFrame())); + frame && !frame.attr("f_back").is_none() && + py::str(frame.attr("f_globals").attr("get")("__name__", "")) + .cast() + .rfind("executorch.", 0) == 0; + frame = frame.attr("f_back")) { + ++level; + } + return level; +} + +// Until a later release makes outputs follow inputs, a call that would then +// return ExecuTorchResult outputs keeps returning torch.Tensor outputs +// whenever torch can be imported, as before, and the first one warns. +bool keep_torch_outputs() { + // Read and written with the GIL held. An import inside a function-local + // static's initializer could deadlock, since the import lets other threads + // run and one of them could then wait for that static while holding the GIL. + static bool torch_missing = false; + static bool warned = false; + if (torch_missing) { + return false; + } + if (loaded_torch_module() == nullptr) { + try { + py::module_::import("torch"); + } catch (py::error_already_set& error) { + // Only a torch that is not there counts as missing. A torch that is + // there but fails to import, even with an ImportError, is reported. + if (!error.matches(PyExc_ModuleNotFoundError) || + py::str(error.value().attr("name")).cast() != "torch") { + try { + error.value().attr("add_note")( + "ExecuTorch imports torch here to return torch.Tensor outputs. " + "Set EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT=executorch before " + "importing ExecuTorch to skip it."); + } catch (...) { + // add_note is best effort; never hide the original error. + } + throw; + } + torch_missing = true; + return false; + } + } + if (!warned) { + warned = true; + if (PyErr_WarnEx( + PyExc_FutureWarning, kTensorOutputWarning, caller_stacklevel()) != + 0) { + // An error filter made the warning an exception, so the next call + // that gets here warns again, as plain Python warnings do. Calls pick + // the type before they run the method, so a call that raises here has + // not run it. + warned = false; + throw py::error_already_set(); + } + } + return true; +} + +// Picks the type of a call's tensor outputs. A later release will follow the +// call's tensor inputs, including inputs kept from earlier calls: torch.Tensor +// if any of them is one, ExecuTorchResult if all are other tensors, and with +// no tensor inputs, torch.Tensor only if torch is imported. Until then, +// keep_torch_outputs() keeps torch.Tensor where that rule would change it. A +// call that returns no tensor never imports torch. +PortableTensorOutput portable_tensor_output( + bool returns_tensor, + bool torch_input, + bool other_tensor_input) { + if (const auto& output = tensor_output_override()) { + return *output; + } + if (torch_input || + (!other_tensor_input && loaded_torch_module() != nullptr)) { + return PortableTensorOutput::Torch; + } + return returns_tensor && keep_torch_outputs() + ? PortableTensorOutput::Torch + : PortableTensorOutput::ExecuTorch; +} + +// From the declared outputs, so the type is picked before the method runs. +// An output bound to a caller's tensor comes back as that tensor, so it does +// not count. +bool returns_tensor( + const MethodMeta& meta, + const std::vector& bound_outputs = {}) { + for (size_t i = 0; i < meta.num_outputs(); ++i) { + const auto tag = meta.output_tag(i); + const bool bound = i < bound_outputs.size() && bound_outputs[i]; + if (tag.ok() && tag.get() == Tag::Tensor && !bound) { + return true; } - portable_tensor_output() = PortableTensorOutput::ExecuTorch; } + return false; } #endif @@ -2079,9 +2190,9 @@ py::object dlpack_output_owner( py::object module_aliased_torch_output( const executorch::aten::Tensor& tensor, const std::vector>& torch_inputs, - const std::string& method_name) { - if (torch_inputs.empty() || - portable_tensor_output() != PortableTensorOutput::Torch || + const std::string& method_name, + PortableTensorOutput output) { + if (torch_inputs.empty() || output != PortableTensorOutput::Torch || tensor.nbytes() == 0) { return py::none(); } @@ -2515,9 +2626,10 @@ class PyCallerStreamScope final { py::object portable_tensor_result( const executorch::aten::Tensor& tensor, + PortableTensorOutput output, bool clone = true, py::object owner = py::none()) { - if (portable_tensor_output() == PortableTensorOutput::ExecuTorch) { + if (output == PortableTensorOutput::ExecuTorch) { return py::cast(std::make_shared( tensor, clone, clone ? py::none() : std::move(owner))); } @@ -2759,6 +2871,7 @@ inline std::unique_ptr load_module_from_data_loader( inline py::list get_outputs_as_py_list( const std::vector& outputs, + PortableTensorOutput output, bool clone_outputs = true, py::object owner = py::none(), const std::vector& output_owners = {}, @@ -2791,9 +2904,10 @@ inline py::list get_outputs_as_py_list( i < handed_over.size() && handed_over[i]; list[i] = returns_without_copy ? portable_tensor_result( - v.toTensor(), /*clone=*/false, handed_over[i]) + v.toTensor(), output, /*clone=*/false, handed_over[i]) : portable_tensor_result( v.toTensor(), + output, clone_outputs, output_owners.empty() ? owner : output_owners[i]); #endif @@ -2849,6 +2963,21 @@ struct PyBundledModule : public BundledModule { size_t testset_idx, double rtol = 1e-5, double atol = 1e-8) { + auto tensor_output = PortableTensorOutput::Torch; +#ifndef USE_ATEN_LIB + { + const auto meta = this->method_meta(method_name); + THROW_IF_ERROR( + meta.error(), + "Failed to get metadata for method %s", + method_name.c_str()); + // The inputs come from the bundled program, so Python passed no tensors. + tensor_output = portable_tensor_output( + returns_tensor(meta.get()), + /*torch_input=*/false, + /*other_tensor_input=*/false); + } +#endif // Execute the method auto result = BundledModule::execute(method_name, testset_idx); if (!result.ok()) { @@ -2857,6 +2986,7 @@ struct PyBundledModule : public BundledModule { "Method execution failed with status 0x%" PRIx32, static_cast(result.error())); } + const auto& outputs = result.get(); #ifndef USE_ATEN_LIB { const auto method = this->method(method_name); @@ -2869,8 +2999,7 @@ struct PyBundledModule : public BundledModule { #endif // Convert outputs to py::list - const auto& outputs = result.get(); - py::list py_outputs = get_outputs_as_py_list(outputs); + py::list py_outputs = get_outputs_as_py_list(outputs, tensor_output); Error status = BundledModule::verify_method_outputs( method_name, testset_idx, rtol, atol); @@ -3593,9 +3722,11 @@ struct PyModule final { for (size_t i = 0; i < inputs_size; ++i) { retained[i] = std::move(new_input_owners[i]); } + auto tensor_output = PortableTensorOutput::Torch; #ifndef USE_ATEN_LIB - // The run reads inputs kept from earlier calls too, so the wait and the - // output owners below cover every installed input, not only this call's. + // The run reads inputs kept from earlier calls too, so the output type, + // the wait and the output owners below cover every installed input, not + // only this call's. buffer_inputs.clear(); torch_inputs.clear(); dlpack_inputs.clear(); @@ -3610,6 +3741,10 @@ struct PyModule final { dlpack_inputs.push_back(owner.dlpack); } } + tensor_output = portable_tensor_output( + returns_tensor(method_meta), + !torch_inputs.empty(), + !buffer_inputs.empty() || !dlpack_inputs.empty()); synchronize_tensor_devices(method); #endif auto outputs = module_->execute(method_name); @@ -3641,6 +3776,7 @@ struct PyModule final { #endif auto result = get_outputs_as_py_list( outputs.get(), + tensor_output, clone_outputs, std::move(output_owner), output_owners, @@ -3652,7 +3788,7 @@ struct PyModule final { const auto& output = (*outputs)[i]; if (output.isTensor()) { auto alias = module_aliased_torch_output( - output.toTensor(), torch_inputs, method_name); + output.toTensor(), torch_inputs, method_name, tensor_output); if (!alias.is_none()) { result[i] = std::move(alias); } @@ -4633,6 +4769,18 @@ struct PyMethod final { "' has no outputs to return: it has not executed since its last " "set_inputs(), or that execution failed. Call execute() first."); } + return read_outputs(tensor_output(caller_outputs_), clone_outputs); + } + + py::list call(const py::object& inputs, bool clone_outputs = true) { + ObjectLock lock(mutex_); + set_inputs(inputs); + const auto output = tensor_output(bound_outputs_); + execute_installed(/*inputs_just_installed=*/true); + return read_outputs(output, clone_outputs); + } + + py::list read_outputs(PortableTensorOutput output, bool clone_outputs) { std::vector result(storage_->method->outputs_size()); Error get_outputs_status = storage_->method->get_outputs( @@ -4669,7 +4817,7 @@ struct PyMethod final { } // Retrieve outputs - py::list outputs = get_outputs_as_py_list(result, clone_outputs); + py::list outputs = get_outputs_as_py_list(result, output, clone_outputs); for (size_t i = 0; i < caller_backed.size(); ++i) { if (caller_backed[i]) { outputs[i] = caller_backed[i]; @@ -4678,13 +4826,6 @@ struct PyMethod final { return outputs; } - py::list call(const py::object& inputs, bool clone_outputs = true) { - ObjectLock lock(mutex_); - set_inputs(inputs); - execute_installed(/*inputs_just_installed=*/true); - return get_outputs(clone_outputs); - } - py::object get_attribute(const std::string& name) { ObjectLock lock(mutex_); Result attr = @@ -4707,11 +4848,17 @@ struct PyMethod final { // writable: a torch tensor cannot be read-only, so torch gets a copy. const bool constant = attr->nbytes() != 0 && !storage_->memory->owns_data(attr->const_data_ptr()); - if (constant && portable_tensor_output() == PortableTensorOutput::Torch) { - return portable_tensor_result(attr.get(), /*clone=*/true); + // An attribute does not come from the inputs of set_inputs(). + const auto output = portable_tensor_output( + /*returns_tensor=*/true, + /*torch_input=*/false, + /*other_tensor_input=*/false); + if (constant && output == PortableTensorOutput::Torch) { + return portable_tensor_result(attr.get(), output, /*clone=*/true); } auto result = portable_tensor_result( attr.get(), + output, /*clone=*/false, borrowed_output_owner(attr.get())); if (constant) { @@ -4776,6 +4923,20 @@ struct PyMethod final { return index < bound_outputs_.size() && bool(bound_outputs_[index]); } + // Uses the inputs of the last successful set_inputs(), and the outputs bound + // for the run that the outputs come from. + PortableTensorOutput tensor_output( + const std::vector& bound_outputs) { +#ifdef USE_ATEN_LIB + return PortableTensorOutput::Torch; +#else + return portable_tensor_output( + returns_tensor(storage_->method->method_meta(), bound_outputs), + !torch_inputs_.empty(), + !buffer_inputs_.empty() || !dlpack_inputs_.empty()); +#endif + } + std::optional aliased_input(size_t index) const { return index < inputs_aliased_by_outputs_.size() ? inputs_aliased_by_outputs_[index] @@ -4855,9 +5016,10 @@ struct PyMethod final { std::vector> dlpack_inputs; }; - py::object aliased_torch_output(const executorch::aten::Tensor& tensor) { - if (torch_inputs_.empty() || - portable_tensor_output() != PortableTensorOutput::Torch || + py::object aliased_torch_output( + const executorch::aten::Tensor& tensor, + PortableTensorOutput output) { + if (torch_inputs_.empty() || output != PortableTensorOutput::Torch || tensor.nbytes() == 0) { return py::none(); } @@ -4909,7 +5071,8 @@ struct PyMethod final { // result stays independent. std::optional hand_over_device_output( size_t index, - const executorch::aten::Tensor& tensor) { + const executorch::aten::Tensor& tensor, + PortableTensorOutput output) { const auto slot = device_buffer_slot(tensor); if (!slot) { return std::nullopt; @@ -4925,10 +5088,11 @@ struct PyMethod final { std::make_shared>(buffer); const std::shared_ptr memory(holder, buffer->data()); py::object result; - if (portable_tensor_output() == PortableTensorOutput::Torch) { + if (output == PortableTensorOutput::Torch) { auto* retained = new std::shared_ptr(memory); result = portable_tensor_result( tensor, + output, /*clone=*/false, py::capsule(retained, [](void* pointer) { delete static_cast*>(pointer); @@ -5109,6 +5273,7 @@ struct PyMethod final { py::list get_outputs_as_py_list( const std::vector& outputs, + PortableTensorOutput output, bool clone_outputs = true) { const auto outputs_size = outputs.size(); py::list list(outputs_size); @@ -5143,13 +5308,14 @@ struct PyMethod final { continue; } if (clone_outputs) { - if (const auto handed = hand_over_device_output(i, v.toTensor())) { + if (const auto handed = + hand_over_device_output(i, v.toTensor(), output)) { list[i] = *handed; continue; } } if (!clone_outputs) { - auto alias = aliased_torch_output(v.toTensor()); + auto alias = aliased_torch_output(v.toTensor(), output); if (!alias.is_none()) { list[i] = std::move(alias); continue; @@ -5157,6 +5323,7 @@ struct PyMethod final { } list[i] = portable_tensor_result( v.toTensor(), + output, clone_outputs, clone_outputs && v.toTensor().device().is_cpu() ? py::none() @@ -5500,9 +5667,13 @@ PYBIND11_MODULE(EXECUTORCH_PYTHON_MODULE_NAME, m) { #else configure_portable_tensor_output(); m.attr("_uses_aten") = false; - m.attr("_tensor_output") = - portable_tensor_output() == PortableTensorOutput::Torch ? "torch" - : "executorch"; + // None when the setting is unset. + const auto& tensor_output = tensor_output_override(); + m.attr("_tensor_output") = !tensor_output + ? py::object(py::none()) + : py::str( + *tensor_output == PortableTensorOutput::Torch ? "torch" + : "executorch"); #endif py::class_>( m, diff --git a/extension/pybindings/pybindings_executorch_result.h b/extension/pybindings/pybindings_executorch_result.h index 5b863bdc685..aff44df868d 100644 --- a/extension/pybindings/pybindings_executorch_result.h +++ b/extension/pybindings/pybindings_executorch_result.h @@ -119,8 +119,8 @@ inline void synchronize_torch_cuda_device(int index) { } // Waits for every queued operation on a device. Bindings built without CUDA -// support have no CUDA runtime of their own, so they wait through torch's, -// which is loaded whenever such bindings see CUDA memory. +// support have no CUDA runtime of their own, so they wait through torch's +// once torch is imported. inline void synchronize_device(int index) { #ifndef EXECUTORCH_PYBINDINGS_CUDA if (loaded_torch_module() != nullptr) { diff --git a/extension/pybindings/test/test_pybindings.py b/extension/pybindings/test/test_pybindings.py index a4575bfdfa6..b9d89935311 100644 --- a/extension/pybindings/test/test_pybindings.py +++ b/extension/pybindings/test/test_pybindings.py @@ -14,6 +14,7 @@ import subprocess import sys import tempfile +import textwrap import time import unittest import weakref @@ -163,115 +164,373 @@ def test_numpy_float16_buffer_input(self): self.assertTrue(torch.equal(output, inputs[0] + inputs[0])) - def test_buffer_input_does_not_import_torch(self): - exported_program, inputs = create_program(ModuleAddSingleInput()) - executorch_module = self.load_fn(exported_program.buffer) - torch_module = sys.modules.pop("torch") - try: - output = executorch_module(inputs[0].numpy())[0] - self.assertNotIn("torch", sys.modules) - finally: - sys.modules["torch"] = torch_module - - self.assertTrue(torch.equal(output, inputs[0] + inputs[0])) - - def test_tensor_output_import_policy(self): + def test_tensor_output_type(self): if self.kernel_mode != "portable": - self.skipTest("only portable bindings configure Python tensor outputs") - program, _ = create_program(ModuleAddSingleInput()) - script = """ -import importlib + self.skipTest("only portable bindings choose Python tensor outputs") + from executorch.devtools.bundled_program.config import ( + MethodTestCase, + MethodTestSuite, + ) + from executorch.devtools.bundled_program.core import BundledProgram + from executorch.devtools.bundled_program.serialize import ( + serialize_from_bundled_program_to_flatbuffer, + ) + + inputs = ModuleAddWithAttributes().get_inputs() + program = to_edge( + export(ModuleAddWithAttributes(), inputs, strict=True), + constant_methods={"tensor_constant": torch.ones(2), "int_constant": 3}, + ).to_executorch(ExecutorchBackendConfig(emit_mutable_buffer_names=True)) + bundled_program, _ = create_program(ModuleAdd()) + bundled = serialize_from_bundled_program_to_flatbuffer( + BundledProgram( + bundled_program, + [ + MethodTestSuite( + method_name="forward", + test_cases=[ + MethodTestCase( + inputs=inputs, expected_outputs=(inputs[0] + inputs[1],) + ) + ], + ) + ], + ) + ) + prelude = """ import importlib.abc -import os import sys -from pathlib import Path +import threading +import warnings + +import numpy as np + +program_path, bundled_path = sys.argv[1:] +x = np.ones((2, 2), dtype=np.float32) class BlockTorch(importlib.abc.MetaPathFinder): - def find_spec(self, fullname, path=None, target=None): - if fullname == "torch" or fullname.startswith("torch."): - raise ImportError("torch is unavailable") - -entrypoint, native_name, policy, available, program_path, native_path = sys.argv[1:] -if sys.platform == "win32": - native_directory = Path(native_path).parent - os.add_dll_directory(str(native_directory)) - library_directory = native_directory.parent.parent / "lib" - if library_directory.is_dir(): - os.add_dll_directory(str(library_directory)) -blocker = BlockTorch() -if available == "no": - sys.meta_path.insert(0, blocker) -try: - bindings = importlib.import_module(entrypoint) -except (ImportError, SystemError) as error: - # Built-in extension loaders may wrap initialization errors. - assert policy == "torch" and available == "no" - while error.__cause__ is not None: - error = error.__cause__ - assert isinstance(error, ImportError) - assert "torch is unavailable" in str(error) -else: - assert policy != "torch" or available == "yes" - native = sys.modules[native_name] - expected = "torch" if policy != "executorch" and available == "yes" else "executorch" - assert native._tensor_output == expected - assert ("torch" in sys.modules) == (expected == "torch") - import numpy as np - program_data = Path(program_path).read_bytes() - module = bindings._load_for_executorch_from_buffer(program_data) - output = module(np.ones((2, 2), dtype=np.float32))[0] - if expected == "executorch": - assert isinstance(output, bindings.ExecuTorchResult) - np.testing.assert_array_equal(np.asarray(output), np.full((2, 2), 2.0)) - if available == "no": - sys.meta_path.remove(blocker) - import torch - assert isinstance(module(torch.ones(2, 2))[0], bindings.ExecuTorchResult) - else: - import torch - assert isinstance(output, torch.Tensor) - torch.testing.assert_close(output, torch.full((2, 2), 2.0)) + def __init__(self, error=ModuleNotFoundError): + self.error = error + # The module the error names: torch itself, or a part of it. + self.missing = "torch" + self.attempts = 0 + + def find_spec(self, name, path=None, target=None): + if name == "torch" or name.startswith("torch."): + self.attempts += 1 + error = self.error("torch is unavailable") + if isinstance(error, ImportError): + error.name = self.missing + raise error + +def kinds(outputs): + torch = sys.modules.get("torch") + return [ + "torch" if torch is not None and isinstance(value, torch.Tensor) + else type(value).__name__ + for value in outputs + ] + +# Importing torch can show warnings of its own, so only this one counts. +OURS = "ExecuTorch returned torch.Tensor outputs" + +def ours(caught): + return [w for w in caught if str(w.message).startswith(OURS)] + +def check(call, expected, warnings_expected=0): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + outputs = call() + future = ours(caught) + assert kinds(outputs) == expected, (kinds(outputs), expected) + assert len(future) == warnings_expected, [str(w.message) for w in future] + assert all(w.category is FutureWarning for w in future) + return future """ - native_name = self.load_fn.__module__ - native_path = sys.modules[native_name].__spec__.origin + # Each runs in a fresh process, since torch imports and the + # warning, shown once per process, are process-wide. + scenarios = { + "NumPy inputs before torch is imported": ( + None, + """ + from executorch.runtime import Runtime + + assert "torch" not in sys.modules + program = Runtime.get().load_program(program_path) + check(lambda: program.load_method("int_constant").execute([]), ["int"]) + assert "torch" not in sys.modules + forward = program.load_method("forward") + (warning,), line = check(lambda: forward.execute([x, x]), ["torch"], 1), sys._getframe().f_lineno + # The warning points at the caller, not at ExecuTorch's wrapper. + assert (warning.filename, warning.lineno) == ("", line), warning + assert "EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT" in str(warning.message) + check(lambda: forward.execute([x, x]), ["torch"]) + """, + ), + "NumPy inputs kept by set_inputs after torch is imported": ( + None, + """ + import torch + from executorch.extension.pybindings import portable_lib + + native = sys.modules["executorch.extension.pybindings._C"] + assert native._tensor_output is None + method = portable_lib._load_program(program_path).load_method("forward") + method.set_inputs([x, x]) + method.execute() + # An attribute does not come from the inputs. + check(lambda: [method.get_attribute("state")], ["torch"]) + check(method.get_outputs, ["torch"], 1) + """, + ), + "NumPy inputs with every output bound to a torch tensor": ( + None, + """ + import torch + from executorch.exir import ExecutorchBackendConfig, to_edge + from executorch.exir.passes import MemoryPlanningPass + from executorch.extension.pybindings import portable_lib + + class Add(torch.nn.Module): + def forward(self, a, b): + return a + b + + t = torch.ones(2, 2) + config = ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass(alloc_graph_output=False) + ) + buffer = to_edge(torch.export.export(Add(), (t, t))).to_executorch(config).buffer + program = portable_lib._load_program_from_buffer(buffer) + method = program.load_method("forward") + bound = torch.empty(2, 2) + method.set_output(bound, 0) + # A bound output is the caller's own tensor, now and later. + check(lambda: method([x, x]), ["torch"]) + assert method([x, x])[0] is bound + # Bound after the run, so the outputs of that run are not. + late = program.load_method("forward") + late.set_inputs([x, x]) + late.execute() + late.set_output(torch.empty(2, 2), 0) + check(late.get_outputs, ["torch"], 1) + """, + ), + "torch inputs, kept or mixed, and no tensor inputs once torch is imported": ( + None, + """ + import torch + from executorch.extension.pybindings import portable_lib + + t = torch.ones(2, 2) + module = portable_lib._load_for_executorch(program_path) + check(lambda: module.forward([t, t]), ["torch"]) + # Input 1 is still the torch tensor of the last call. + check(lambda: module.run_method("forward", [x]), ["torch"]) + check(lambda: module.forward([x, t]), ["torch"]) + check(lambda: module.run_method("tensor_constant"), ["torch"]) + with open(bundled_path, "rb") as source: + bundled = portable_lib._load_bundled_program_from_buffer( + source.read() + ) + check( + lambda: bundled.verify_result_with_bundled_expected_output( + "forward", 0 + ), + ["torch"], + ) + """, + ), + "the warning turned into an error": ( + None, + """ + from executorch.extension.pybindings import portable_lib + + module = portable_lib._load_for_executorch(program_path) + method = portable_lib._load_program(program_path).load_method("forward") + with warnings.catch_warnings(): + warnings.filterwarnings("error", OURS) + for call in (module.forward, method) * 2: + try: + call([x, x]) + except FutureWarning: + pass + else: + raise AssertionError("every such call must raise") + # The raising calls did not run: state is still 0, so 1 + 1 + 1. + for call in (module.forward, method): + assert call([x, x])[0].tolist() == [[3.0, 3.0]] * 2 + """, + ), + "two threads before torch is imported": ( + None, + """ + from executorch.extension.pybindings import portable_lib + + modules = [portable_lib._load_for_executorch(program_path) for _ in range(2)] + barrier = threading.Barrier(2) + outputs = [None, None] + + def run(index): + barrier.wait() + outputs[index] = modules[index].forward([x, x])[0] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + threads = [threading.Thread(target=run, args=(i,)) for i in (0, 1)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert kinds(outputs) == ["torch", "torch"], kinds(outputs) + assert len(ours(caught)) == 1, [str(w.message) for w in caught] + """, + ), + "torch fails to import or is not installed": ( + None, + """ + blocker = BlockTorch(RuntimeError) + sys.meta_path.insert(0, blocker) + from executorch.extension.pybindings import portable_lib + + module = portable_lib._load_for_executorch(program_path) + # A torch that is there but broken, even with an ImportError or + # a missing module of its own. + for error_type, missing in ( + (RuntimeError, "torch"), + (ImportError, "torch"), + (ModuleNotFoundError, "torch._C"), + ): + blocker.error, blocker.missing = error_type, missing + try: + module.forward([x, x]) + except error_type as error: + assert "torch is unavailable" in str(error), error + # add_note is new in Python 3.11. + notes = "".join(getattr(error, "__notes__", [])) + assert sys.version_info < (3, 11) or ( + "EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT" in notes + ), notes + else: + raise AssertionError("a broken torch must not be hidden") + blocker.missing = "torch" + check(lambda: module.forward([x, x]), ["ExecuTorchResult"]) + check(lambda: module.run_method("tensor_constant"), ["ExecuTorchResult"]) + # A missing torch is not looked for again on every call. + assert blocker.attempts == 4, blocker.attempts + assert "torch" not in sys.modules + """, + ), + "the setting asks for ExecuTorchResult": ( + "executorch", + """ + from executorch.extension.pybindings import portable_lib + + native = sys.modules["executorch.extension.pybindings._C"] + assert native._tensor_output == "executorch" + module = portable_lib._load_for_executorch(program_path) + check(lambda: module.forward([x, x]), ["ExecuTorchResult"]) + check(lambda: module.run_method("tensor_constant"), ["ExecuTorchResult"]) + assert "torch" not in sys.modules + import torch + + t = torch.ones(2, 2) + check(lambda: module.forward([t, t]), ["ExecuTorchResult"]) + """, + ), + "the setting asks for torch.Tensor": ( + "torch", + """ + import importlib.util + import os + from pathlib import Path + + # The extension imported on its own, without portable_lib, which + # imports torch itself and on Windows adds the folders of the + # extension's libraries. + name = "executorch.extension.pybindings._C" + if sys.platform == "win32": + native_directory = Path(importlib.util.find_spec(name).origin).parent + for directory in (native_directory, native_directory.parent.parent / "lib"): + if directory.is_dir(): + os.add_dll_directory(str(directory.resolve())) + native = importlib.import_module(name) + assert "torch" in sys.modules + assert native._tensor_output == "torch" + from executorch.extension.pybindings import portable_lib + + module = portable_lib._load_for_executorch(program_path) + check(lambda: module.forward([x, x]), ["torch"]) + """, + ), + "the setting asks for torch.Tensor without torch": ( + "torch", + """ + sys.meta_path.insert(0, BlockTorch()) + try: + from executorch.extension.pybindings import portable_lib + except ImportError as error: + assert "PyTorch could not be imported" in str(error), error + else: + raise AssertionError("importing must fail without torch") + """, + ), + } + # Each of these calls returns torch.Tensor outputs now and will return + # ExecuTorchResult outputs later, so it warns: calls without tensor + # inputs before torch is imported, and calls with tensor inputs of + # other kinds after it is. + for call, torch_imported in ( + ("module.run_method('tensor_constant')", False), + ("[method.get_attribute('state')]", False), + ("bundled.verify_result_with_bundled_expected_output('forward', 0)", False), + ("module.forward([x, x])", True), + ("module.forward(capsules)", True), + ("method(capsules)", True), + ): + scenarios[f"{call}, torch imported: {torch_imported}"] = ( + None, + f""" + {'import torch' if torch_imported else ''} + from executorch.extension.pybindings import portable_lib + + module = portable_lib._load_for_executorch(program_path) + method = portable_lib._load_program(program_path).load_method("forward") + with open(bundled_path, "rb") as source: + bundled = portable_lib._load_bundled_program_from_buffer(source.read()) + capsules = [x.__dlpack__(), x.__dlpack__()] + (warning,), line = check(lambda: {call}, ["torch"], 1), sys._getframe().f_lineno + assert warning.lineno == line, (warning.lineno, line) + """, + ) with tempfile.TemporaryDirectory() as directory: - program_path = os.path.join(directory, "add.pte") + program_path = os.path.join(directory, "program.pte") with open(program_path, "wb") as handle: handle.write(program.buffer) - for entrypoint in dict.fromkeys((self.runtime.__name__, native_name)): - for policy in (None, "torch", "executorch"): - for available in ("yes", "no"): - with self.subTest( - entrypoint=entrypoint, policy=policy, available=available - ): - environment = os.environ.copy() - environment.pop("EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT", None) - if policy is not None: - environment["EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT"] = ( - policy - ) - completed = subprocess.run( - [ - sys.executable, - "-c", - script, - entrypoint, - native_name, - str(policy), - available, - program_path, - native_path, - ], - env=environment, - capture_output=True, - text=True, - timeout=60, - ) - self.assertEqual( - completed.returncode, - 0, - completed.stdout + completed.stderr, - ) + bundled_path = os.path.join(directory, "program.bpte") + with open(bundled_path, "wb") as handle: + handle.write(bundled) + for name, (setting, code) in scenarios.items(): + with self.subTest(name): + environment = os.environ.copy() + environment.pop("EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT", None) + if setting is not None: + environment["EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT"] = setting + completed = subprocess.run( + [ + sys.executable, + "-c", + prelude + textwrap.dedent(code), + program_path, + bundled_path, + ], + env=environment, + capture_output=True, + text=True, + timeout=120, + ) + self.assertEqual( + completed.returncode, 0, completed.stdout + completed.stderr + ) def test_torch_that_is_not_ready_is_ignored(self): if self.kernel_mode != "portable": @@ -1474,7 +1733,7 @@ def test_method_rejects_aliased_input_whose_storage_changed(self): self.skipTest("only portable bindings return a view of the input") from executorch.extension.pybindings import _C - if _C._tensor_output != "torch": + if _C._tensor_output == "executorch": self.skipTest("only torch output mode returns a view of the input") class Identity(torch.nn.Module): diff --git a/extension/pybindings/test/test_pybindings_no_aten.py b/extension/pybindings/test/test_pybindings_no_aten.py index 0b8ee2673ba..acb1ece9a87 100644 --- a/extension/pybindings/test/test_pybindings_no_aten.py +++ b/extension/pybindings/test/test_pybindings_no_aten.py @@ -134,7 +134,7 @@ def test_output_policy_without_torch(self) -> None: import os assert Path(runtime.__file__).resolve() == Path(sys.argv[1]).resolve() - assert runtime._tensor_output == "executorch" + assert runtime._tensor_output is None with open(os.environ["EXECUTORCH_PYBIND_TEST_PTE"], "rb") as source: data = source.read() module = runtime._load_for_executorch_from_buffer(data) @@ -490,9 +490,9 @@ def test_z_torch_tensor_uses_python_api(self) -> None: output = module((torch.tensor([1.0]), torch.tensor([2.0])))[0] expected_type = ( - torch.Tensor - if runtime._tensor_output == "torch" - else runtime.ExecuTorchResult + runtime.ExecuTorchResult + if runtime._tensor_output == "executorch" + else torch.Tensor ) self.assertIsInstance(output, expected_type) np.testing.assert_array_equal(np.asarray(output), np.array([3.0])) diff --git a/runtime/__init__.py b/runtime/__init__.py index 0aa0d89655c..204b18df6b8 100644 --- a/runtime/__init__.py +++ b/runtime/__init__.py @@ -172,8 +172,20 @@ def execute( a later execution may overwrite them, which avoids a copy. Returns: - A list of output values, typically torch.Tensor objects. Without - PyTorch, tensor outputs are ExecuTorchResult objects. + A list of output values. When PyTorch is installed, tensor outputs + are torch.Tensor objects, and the first call that returns one + imports torch. Without PyTorch, they are ExecuTorchResult objects. + + .. warning:: + + A future release will make tensor outputs follow the tensor inputs: + torch.Tensor objects if any input is one, ExecuTorchResult objects + if all are other tensors, even when PyTorch is installed, and with + no tensor inputs, torch.Tensor objects only if torch is already + imported. The first call whose outputs will change shows a + FutureWarning. Set ``EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT`` to + ``torch`` or ``executorch`` before importing ExecuTorch to choose + the type now. """ return self._method(inputs, clone_outputs=clone_outputs)