diff --git a/docs/source/runtime-python-api-reference.rst b/docs/source/runtime-python-api-reference.rst index 87c8c8facd1..7979496a124 100644 --- a/docs/source/runtime-python-api-reference.rst +++ b/docs/source/runtime-python-api-reference.rst @@ -43,7 +43,21 @@ Other buffers, such as ``memoryview``, work as inputs too. NumPy has no bfloat16 For data on a GPU, use DLPack, the standard way to share tensors between Python libraries. This needs ExecuTorch built with CUDA support. Pass a CuPy array to a method that takes CUDA inputs. Read a CUDA result with ``cupy.from_dlpack(out)``, or copy it to the CPU with ``out.to_host()``. ``ExecuTorchResult.from_host()`` copies CPU data to a CUDA device. -To get ``ExecuTorchResult`` values even when PyTorch is installed, set ``EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT=executorch`` before importing ExecuTorch. +Output types +~~~~~~~~~~~~ + +Importing ``executorch.runtime`` does not import PyTorch, unless the setting below asks for ``torch``. When PyTorch is installed, ``Method.execute`` returns ``torch.Tensor`` outputs, and the first call that returns one imports ``torch``. Without PyTorch, it returns ``ExecuTorchResult`` outputs. If PyTorch is installed but fails to import, every call that returns a tensor raises the import error; set the setting below to ``executorch`` to run without PyTorch. + +A future release will make the outputs follow the tensor inputs of each call. A call with any ``torch.Tensor`` input will return ``torch.Tensor`` outputs. A call whose tensor inputs are all NumPy arrays, other buffers, DLPack objects, or ``ExecuTorchResult`` values will return ``ExecuTorchResult`` outputs, even when PyTorch is installed. A call with no tensor inputs will return ``torch.Tensor`` outputs only if ``torch`` is already imported. The first call whose outputs will change shows a ``FutureWarning``. + +To choose the output type now, set ``EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT`` before importing ExecuTorch. With ``executorch``, every call returns ``ExecuTorchResult`` outputs and ExecuTorch never imports PyTorch. With ``torch``, every call returns ``torch.Tensor`` outputs, and importing ExecuTorch imports PyTorch. + +.. code-block:: python + + import os + + os.environ["EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT"] = "executorch" + from executorch.runtime import Runtime CUDA streams ~~~~~~~~~~~~ diff --git a/extension/pybindings/README.md b/extension/pybindings/README.md index 62ccff747c4..86119a300e1 100644 --- a/extension/pybindings/README.md +++ b/extension/pybindings/README.md @@ -41,10 +41,14 @@ execution can share the caller's device allocation. input `i`, as `"cpu"` or `"cuda:N"`, and `output_tensor_meta(i).device()` does the same for its outputs. -When PyTorch is available, importing `portable_lib` imports it and portable -bindings return `torch.Tensor` outputs. When it cannot be imported, the default -output type is `ExecuTorchResult`. Output conversion uses Python APIs and -DLPack; the extension does not link against PyTorch. +Unless `EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT` is `torch`, importing +`portable_lib` does not import PyTorch. When PyTorch is installed, portable +bindings return `torch.Tensor` outputs, and the first call that returns one +imports `torch`. When PyTorch is not installed, outputs are `ExecuTorchResult` +values. When it is installed but fails to import, every call that returns a +tensor raises the import error; set the variable to `executorch` to run +without PyTorch. Output conversion uses Python APIs and DLPack; the extension +does not link against PyTorch. `clone_outputs=True` returns storage that later executions do not overwrite, on the output's original device. With `clone_outputs=False`, results share runtime or input storage and later execution may overwrite that storage. @@ -91,9 +95,16 @@ with `clone_outputs=False` lives in the method's output memory, so passing it back in as an input means each run reads what the run before it wrote there; pass a copied result instead. -To select `ExecuTorchResult` outputs even when PyTorch is installed, set the -environment variable before importing the bindings. The setting is fixed when -the extension loads, so importing PyTorch later does not change the output type. +A future release will make tensor outputs follow the tensor inputs of each +call, including inputs kept from an earlier call. A call with any torch tensor +input will return torch tensors. A call whose tensor inputs are all NumPy +arrays, other buffers, DLPack objects, or `ExecuTorchResult` values will return +`ExecuTorchResult` values, even when PyTorch is installed. A call with no tensor +inputs will return torch tensors only if `torch` is already imported. The first +call whose outputs will change shows a `FutureWarning`. To choose the output +type now, set `EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT` to `executorch` or `torch` +before importing the bindings. The setting is fixed when the extension loads, +and `executorch` never imports PyTorch. ```python import os @@ -103,11 +114,11 @@ from executorch.extension.pybindings import portable_lib ``` `ExecuTorchResult` preserves shape, strides, dtype, and device. CPU results expose -the Python buffer protocol so NumPy can read them without another copy. Without -the environment setting, the bindings fall back to `ExecuTorchResult` if -PyTorch cannot be imported. Explicitly selecting `torch` requires it to import -successfully. ATen-mode bindings continue to return native PyTorch tensors -directly. +the Python buffer protocol so NumPy can read them without another copy. A result +is not a number or a truth value: `bool()`, `int()`, and `float()` raise +`TypeError`, so convert it first, for example with `np.asarray(output)`. +Explicitly selecting `torch` requires it to import successfully. ATen-mode +bindings continue to return native PyTorch tensors directly. ```python output = module.forward([numpy_input])[0] @@ -136,8 +147,9 @@ result = np.asarray(output.to_host()) Copying CUDA memory requires CUDA-enabled bindings: `from_host()`, `to_host()` of a CUDA result, and a CUDA `ExecuTorchResult` with `clone_outputs=True`. Bindings built without CUDA support synchronize through Python Torch instead, -so while torch is loaded they still read CUDA inputs, torch tensors or DLPack, -and return borrowed CUDA results. +so once torch is imported, by you or by a call that returns torch tensors, they +still read CUDA inputs, torch tensors or DLPack, and return borrowed CUDA +results. Execution is synchronous at the Python boundary: a call returns after its device work has finished, so its outputs are ready on any stream. How long a call with diff --git a/extension/pybindings/portable_lib.py b/extension/pybindings/portable_lib.py index b936acaeb2f..3dc40d0d455 100644 --- a/extension/pybindings/portable_lib.py +++ b/extension/pybindings/portable_lib.py @@ -28,23 +28,19 @@ class _LightweightExperimentalWarning(DeprecationWarning): _TENSOR_OUTPUT_ENV = "EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT" _tensor_output = os.environ.get(_TENSOR_OUTPUT_ENV) -if _tensor_output in (None, "torch"): +# Without the setting, the first call that returns torch.Tensor outputs imports +# torch, so importing ExecuTorch does not load it. +if _tensor_output == "torch": try: - # Preserve the existing API: importing portable_lib also imports Torch, - # and tensor outputs are torch.Tensor objects. - import torch as _torch + _import_module("torch") except ImportError as error: - if _tensor_output == "torch": - raise ImportError( - "ExecuTorch portable pybindings were configured to return " - "torch.Tensor outputs, but PyTorch could not be imported. " - f"Install PyTorch or set {_TENSOR_OUTPUT_ENV}=executorch " - "before importing portable_lib." - ) from error - _torch = None -elif _tensor_output == "executorch": - _torch = None -else: + raise ImportError( + "ExecuTorch portable pybindings were configured to return " + "torch.Tensor outputs, but PyTorch could not be imported. " + f"Install PyTorch or set {_TENSOR_OUTPUT_ENV}=executorch " + "before importing portable_lib." + ) from error +elif _tensor_output not in (None, "executorch"): raise ValueError( f"{_TENSOR_OUTPUT_ENV} must be either 'torch' or 'executorch', " f"but was {_tensor_output!r}" @@ -152,7 +148,6 @@ class _LightweightExperimentalWarning(DeprecationWarning): del _bindings del _tensor_output del _TENSOR_OUTPUT_ENV -del _torch del _warning_category del _import_module del _exir_warnings diff --git a/extension/pybindings/pybindings.cpp b/extension/pybindings/pybindings.cpp index 17b4935a73d..21dd211f2c8 100644 --- a/extension/pybindings/pybindings.cpp +++ b/extension/pybindings/pybindings.cpp @@ -728,38 +728,149 @@ struct RuntimeErrorScope final { std::unique_ptr logs_; }; +// The Python type of a call's tensor outputs. ATen bindings always return +// torch tensors. +enum class PortableTensorOutput { Torch, ExecuTorch }; + #ifndef USE_ATEN_LIB constexpr char kTensorOutputEnvironmentVariable[] = "EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT"; -enum class PortableTensorOutput { Torch, ExecuTorch }; - -PortableTensorOutput& portable_tensor_output() { - static PortableTensorOutput output = PortableTensorOutput::Torch; +constexpr char kTensorOutputWarning[] = + "ExecuTorch returned torch.Tensor outputs for a call without torch.Tensor " + "inputs. A future release will return ExecuTorchResult outputs for such a " + "call, unless it has no tensor inputs and torch is already imported. To " + "keep torch.Tensor outputs, pass torch.Tensor inputs, or import torch " + "before a call without tensor inputs. To choose the type for every call, " + "set EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT to torch or executorch before " + "importing ExecuTorch. With executorch, PyTorch is not loaded."; + +// Set from EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT when the extension loads. +std::optional& tensor_output_override() { + static std::optional output; return output; } void configure_portable_tensor_output() { const char* value = std::getenv(kTensorOutputEnvironmentVariable); - if (value != nullptr && std::strcmp(value, "executorch") == 0) { - portable_tensor_output() = PortableTensorOutput::ExecuTorch; + if (value == nullptr) { return; } - if (value != nullptr && std::strcmp(value, "torch") != 0) { + if (std::strcmp(value, "executorch") == 0) { + tensor_output_override() = PortableTensorOutput::ExecuTorch; + return; + } + if (std::strcmp(value, "torch") != 0) { throw py::value_error( std::string(kTensorOutputEnvironmentVariable) + " must be either 'torch' or 'executorch', but was '" + value + "'"); } - try { - py::module_::import("torch"); - (void)loaded_torch_module(); - portable_tensor_output() = PortableTensorOutput::Torch; - } catch (py::error_already_set& error) { - if (value != nullptr || !error.matches(PyExc_ImportError)) { - throw; + py::module_::import("torch"); + tensor_output_override() = PortableTensorOutput::Torch; +} + +// Points a warning at the code that called ExecuTorch rather than at +// ExecuTorch's own Python wrappers, such as Method.execute. +int caller_stacklevel() { + int level = 1; + for (auto frame = py::reinterpret_borrow( + reinterpret_cast(PyEval_GetFrame())); + frame && !frame.attr("f_back").is_none() && + py::str(frame.attr("f_globals").attr("get")("__name__", "")) + .cast() + .rfind("executorch.", 0) == 0; + frame = frame.attr("f_back")) { + ++level; + } + return level; +} + +// Until a later release makes outputs follow inputs, a call that would then +// return ExecuTorchResult outputs keeps returning torch.Tensor outputs +// whenever torch can be imported, as before, and the first one warns. +bool keep_torch_outputs() { + // Read and written with the GIL held. An import inside a function-local + // static's initializer could deadlock, since the import lets other threads + // run and one of them could then wait for that static while holding the GIL. + static bool torch_missing = false; + static bool warned = false; + if (torch_missing) { + return false; + } + if (loaded_torch_module() == nullptr) { + try { + py::module_::import("torch"); + } catch (py::error_already_set& error) { + // Only a torch that is not there counts as missing. A torch that is + // there but fails to import, even with an ImportError, is reported. + if (!error.matches(PyExc_ModuleNotFoundError) || + py::str(error.value().attr("name")).cast() != "torch") { + try { + error.value().attr("add_note")( + "ExecuTorch imports torch here to return torch.Tensor outputs. " + "Set EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT=executorch before " + "importing ExecuTorch to skip it."); + } catch (...) { + // add_note is best effort; never hide the original error. + } + throw; + } + torch_missing = true; + return false; + } + } + if (!warned) { + warned = true; + if (PyErr_WarnEx( + PyExc_FutureWarning, kTensorOutputWarning, caller_stacklevel()) != + 0) { + // An error filter made the warning an exception, so the next call + // that gets here warns again, as plain Python warnings do. Calls pick + // the type before they run the method, so a call that raises here has + // not run it. + warned = false; + throw py::error_already_set(); } - portable_tensor_output() = PortableTensorOutput::ExecuTorch; } + return true; +} + +// Picks the type of a call's tensor outputs. A later release will follow the +// call's tensor inputs, including inputs kept from earlier calls: torch.Tensor +// if any of them is one, ExecuTorchResult if all are other tensors, and with +// no tensor inputs, torch.Tensor only if torch is imported. Until then, +// keep_torch_outputs() keeps torch.Tensor where that rule would change it. A +// call that returns no tensor never imports torch. +PortableTensorOutput portable_tensor_output( + bool returns_tensor, + bool torch_input, + bool other_tensor_input) { + if (const auto& output = tensor_output_override()) { + return *output; + } + if (torch_input || + (!other_tensor_input && loaded_torch_module() != nullptr)) { + return PortableTensorOutput::Torch; + } + return returns_tensor && keep_torch_outputs() + ? PortableTensorOutput::Torch + : PortableTensorOutput::ExecuTorch; +} + +// From the declared outputs, so the type is picked before the method runs. +// An output bound to a caller's tensor comes back as that tensor, so it does +// not count. +bool returns_tensor( + const MethodMeta& meta, + const std::vector& bound_outputs = {}) { + for (size_t i = 0; i < meta.num_outputs(); ++i) { + const auto tag = meta.output_tag(i); + const bool bound = i < bound_outputs.size() && bound_outputs[i]; + if (tag.ok() && tag.get() == Tag::Tensor && !bound) { + return true; + } + } + return false; } #endif @@ -2079,9 +2190,9 @@ py::object dlpack_output_owner( py::object module_aliased_torch_output( const executorch::aten::Tensor& tensor, const std::vector>& torch_inputs, - const std::string& method_name) { - if (torch_inputs.empty() || - portable_tensor_output() != PortableTensorOutput::Torch || + const std::string& method_name, + PortableTensorOutput output) { + if (torch_inputs.empty() || output != PortableTensorOutput::Torch || tensor.nbytes() == 0) { return py::none(); } @@ -2515,9 +2626,10 @@ class PyCallerStreamScope final { py::object portable_tensor_result( const executorch::aten::Tensor& tensor, + PortableTensorOutput output, bool clone = true, py::object owner = py::none()) { - if (portable_tensor_output() == PortableTensorOutput::ExecuTorch) { + if (output == PortableTensorOutput::ExecuTorch) { return py::cast(std::make_shared( tensor, clone, clone ? py::none() : std::move(owner))); } @@ -2759,6 +2871,7 @@ inline std::unique_ptr load_module_from_data_loader( inline py::list get_outputs_as_py_list( const std::vector& outputs, + PortableTensorOutput output, bool clone_outputs = true, py::object owner = py::none(), const std::vector& output_owners = {}, @@ -2791,9 +2904,10 @@ inline py::list get_outputs_as_py_list( i < handed_over.size() && handed_over[i]; list[i] = returns_without_copy ? portable_tensor_result( - v.toTensor(), /*clone=*/false, handed_over[i]) + v.toTensor(), output, /*clone=*/false, handed_over[i]) : portable_tensor_result( v.toTensor(), + output, clone_outputs, output_owners.empty() ? owner : output_owners[i]); #endif @@ -2849,6 +2963,21 @@ struct PyBundledModule : public BundledModule { size_t testset_idx, double rtol = 1e-5, double atol = 1e-8) { + auto tensor_output = PortableTensorOutput::Torch; +#ifndef USE_ATEN_LIB + { + const auto meta = this->method_meta(method_name); + THROW_IF_ERROR( + meta.error(), + "Failed to get metadata for method %s", + method_name.c_str()); + // The inputs come from the bundled program, so Python passed no tensors. + tensor_output = portable_tensor_output( + returns_tensor(meta.get()), + /*torch_input=*/false, + /*other_tensor_input=*/false); + } +#endif // Execute the method auto result = BundledModule::execute(method_name, testset_idx); if (!result.ok()) { @@ -2857,6 +2986,7 @@ struct PyBundledModule : public BundledModule { "Method execution failed with status 0x%" PRIx32, static_cast(result.error())); } + const auto& outputs = result.get(); #ifndef USE_ATEN_LIB { const auto method = this->method(method_name); @@ -2869,8 +2999,7 @@ struct PyBundledModule : public BundledModule { #endif // Convert outputs to py::list - const auto& outputs = result.get(); - py::list py_outputs = get_outputs_as_py_list(outputs); + py::list py_outputs = get_outputs_as_py_list(outputs, tensor_output); Error status = BundledModule::verify_method_outputs( method_name, testset_idx, rtol, atol); @@ -3593,9 +3722,11 @@ struct PyModule final { for (size_t i = 0; i < inputs_size; ++i) { retained[i] = std::move(new_input_owners[i]); } + auto tensor_output = PortableTensorOutput::Torch; #ifndef USE_ATEN_LIB - // The run reads inputs kept from earlier calls too, so the wait and the - // output owners below cover every installed input, not only this call's. + // The run reads inputs kept from earlier calls too, so the output type, + // the wait and the output owners below cover every installed input, not + // only this call's. buffer_inputs.clear(); torch_inputs.clear(); dlpack_inputs.clear(); @@ -3610,6 +3741,10 @@ struct PyModule final { dlpack_inputs.push_back(owner.dlpack); } } + tensor_output = portable_tensor_output( + returns_tensor(method_meta), + !torch_inputs.empty(), + !buffer_inputs.empty() || !dlpack_inputs.empty()); synchronize_tensor_devices(method); #endif auto outputs = module_->execute(method_name); @@ -3641,6 +3776,7 @@ struct PyModule final { #endif auto result = get_outputs_as_py_list( outputs.get(), + tensor_output, clone_outputs, std::move(output_owner), output_owners, @@ -3652,7 +3788,7 @@ struct PyModule final { const auto& output = (*outputs)[i]; if (output.isTensor()) { auto alias = module_aliased_torch_output( - output.toTensor(), torch_inputs, method_name); + output.toTensor(), torch_inputs, method_name, tensor_output); if (!alias.is_none()) { result[i] = std::move(alias); } @@ -4633,6 +4769,18 @@ struct PyMethod final { "' has no outputs to return: it has not executed since its last " "set_inputs(), or that execution failed. Call execute() first."); } + return read_outputs(tensor_output(caller_outputs_), clone_outputs); + } + + py::list call(const py::object& inputs, bool clone_outputs = true) { + ObjectLock lock(mutex_); + set_inputs(inputs); + const auto output = tensor_output(bound_outputs_); + execute_installed(/*inputs_just_installed=*/true); + return read_outputs(output, clone_outputs); + } + + py::list read_outputs(PortableTensorOutput output, bool clone_outputs) { std::vector result(storage_->method->outputs_size()); Error get_outputs_status = storage_->method->get_outputs( @@ -4669,7 +4817,7 @@ struct PyMethod final { } // Retrieve outputs - py::list outputs = get_outputs_as_py_list(result, clone_outputs); + py::list outputs = get_outputs_as_py_list(result, output, clone_outputs); for (size_t i = 0; i < caller_backed.size(); ++i) { if (caller_backed[i]) { outputs[i] = caller_backed[i]; @@ -4678,13 +4826,6 @@ struct PyMethod final { return outputs; } - py::list call(const py::object& inputs, bool clone_outputs = true) { - ObjectLock lock(mutex_); - set_inputs(inputs); - execute_installed(/*inputs_just_installed=*/true); - return get_outputs(clone_outputs); - } - py::object get_attribute(const std::string& name) { ObjectLock lock(mutex_); Result attr = @@ -4707,11 +4848,17 @@ struct PyMethod final { // writable: a torch tensor cannot be read-only, so torch gets a copy. const bool constant = attr->nbytes() != 0 && !storage_->memory->owns_data(attr->const_data_ptr()); - if (constant && portable_tensor_output() == PortableTensorOutput::Torch) { - return portable_tensor_result(attr.get(), /*clone=*/true); + // An attribute does not come from the inputs of set_inputs(). + const auto output = portable_tensor_output( + /*returns_tensor=*/true, + /*torch_input=*/false, + /*other_tensor_input=*/false); + if (constant && output == PortableTensorOutput::Torch) { + return portable_tensor_result(attr.get(), output, /*clone=*/true); } auto result = portable_tensor_result( attr.get(), + output, /*clone=*/false, borrowed_output_owner(attr.get())); if (constant) { @@ -4776,6 +4923,20 @@ struct PyMethod final { return index < bound_outputs_.size() && bool(bound_outputs_[index]); } + // Uses the inputs of the last successful set_inputs(), and the outputs bound + // for the run that the outputs come from. + PortableTensorOutput tensor_output( + const std::vector& bound_outputs) { +#ifdef USE_ATEN_LIB + return PortableTensorOutput::Torch; +#else + return portable_tensor_output( + returns_tensor(storage_->method->method_meta(), bound_outputs), + !torch_inputs_.empty(), + !buffer_inputs_.empty() || !dlpack_inputs_.empty()); +#endif + } + std::optional aliased_input(size_t index) const { return index < inputs_aliased_by_outputs_.size() ? inputs_aliased_by_outputs_[index] @@ -4855,9 +5016,10 @@ struct PyMethod final { std::vector> dlpack_inputs; }; - py::object aliased_torch_output(const executorch::aten::Tensor& tensor) { - if (torch_inputs_.empty() || - portable_tensor_output() != PortableTensorOutput::Torch || + py::object aliased_torch_output( + const executorch::aten::Tensor& tensor, + PortableTensorOutput output) { + if (torch_inputs_.empty() || output != PortableTensorOutput::Torch || tensor.nbytes() == 0) { return py::none(); } @@ -4909,7 +5071,8 @@ struct PyMethod final { // result stays independent. std::optional hand_over_device_output( size_t index, - const executorch::aten::Tensor& tensor) { + const executorch::aten::Tensor& tensor, + PortableTensorOutput output) { const auto slot = device_buffer_slot(tensor); if (!slot) { return std::nullopt; @@ -4925,10 +5088,11 @@ struct PyMethod final { std::make_shared>(buffer); const std::shared_ptr memory(holder, buffer->data()); py::object result; - if (portable_tensor_output() == PortableTensorOutput::Torch) { + if (output == PortableTensorOutput::Torch) { auto* retained = new std::shared_ptr(memory); result = portable_tensor_result( tensor, + output, /*clone=*/false, py::capsule(retained, [](void* pointer) { delete static_cast*>(pointer); @@ -5109,6 +5273,7 @@ struct PyMethod final { py::list get_outputs_as_py_list( const std::vector& outputs, + PortableTensorOutput output, bool clone_outputs = true) { const auto outputs_size = outputs.size(); py::list list(outputs_size); @@ -5143,13 +5308,14 @@ struct PyMethod final { continue; } if (clone_outputs) { - if (const auto handed = hand_over_device_output(i, v.toTensor())) { + if (const auto handed = + hand_over_device_output(i, v.toTensor(), output)) { list[i] = *handed; continue; } } if (!clone_outputs) { - auto alias = aliased_torch_output(v.toTensor()); + auto alias = aliased_torch_output(v.toTensor(), output); if (!alias.is_none()) { list[i] = std::move(alias); continue; @@ -5157,6 +5323,7 @@ struct PyMethod final { } list[i] = portable_tensor_result( v.toTensor(), + output, clone_outputs, clone_outputs && v.toTensor().device().is_cpu() ? py::none() @@ -5447,6 +5614,15 @@ py::dict get_option(const std::string& backend_name, const py::dict& options) { return result; } +auto refuse_scalar_conversion(const char* problem) { + return [problem](const PyExecuTorchResult&) -> py::object { + throw py::type_error( + std::string("ExecuTorchResult ") + problem + + ". Convert it first, for example with numpy.asarray(result) or " + "torch.from_dlpack(result)."); + }; +} + } // namespace PYBIND11_MODULE(EXECUTORCH_PYTHON_MODULE_NAME, m) { @@ -5491,9 +5667,13 @@ PYBIND11_MODULE(EXECUTORCH_PYTHON_MODULE_NAME, m) { #else configure_portable_tensor_output(); m.attr("_uses_aten") = false; - m.attr("_tensor_output") = - portable_tensor_output() == PortableTensorOutput::Torch ? "torch" - : "executorch"; + // None when the setting is unset. + const auto& tensor_output = tensor_output_override(); + m.attr("_tensor_output") = !tensor_output + ? py::object(py::none()) + : py::str( + *tensor_output == PortableTensorOutput::Torch ? "torch" + : "executorch"); #endif py::class_>( m, @@ -5579,6 +5759,12 @@ PYBIND11_MODULE(EXECUTORCH_PYTHON_MODULE_NAME, m) { "Copy a dense CPU buffer, or a CPU ExecuTorchResult, to a CUDA " "device, for use as a method input. Requires bindings built with " "CUDA support.") + // Python would otherwise treat a result as always true, and int() and + // float() would parse its bytes as text. + .def("__bool__", refuse_scalar_conversion("has no truth value")) + .def("__int__", refuse_scalar_conversion("cannot be converted to int")) + .def( + "__float__", refuse_scalar_conversion("cannot be converted to float")) .def("__repr__", &PyExecuTorchResult::repr); #ifndef USE_ATEN_LIB py::class_< diff --git a/extension/pybindings/pybindings_executorch_result.h b/extension/pybindings/pybindings_executorch_result.h index 1397c46186a..aff44df868d 100644 --- a/extension/pybindings/pybindings_executorch_result.h +++ b/extension/pybindings/pybindings_executorch_result.h @@ -97,10 +97,17 @@ inline PyObject* loaded_torch_module() { static PyObject* torch_module = nullptr; if (torch_module == nullptr) { PyObject* loaded = PyDict_GetItemString(PyImport_GetModuleDict(), "torch"); - if (loaded != nullptr) { - Py_INCREF(loaded); - torch_module = loaded; + if (loaded == nullptr || loaded == Py_None) { + return nullptr; } + // A torch that another thread is still importing is not usable yet. + const auto spec = py::getattr(py::handle(loaded), "__spec__", py::none()); + if (!spec.is_none() && + py::getattr(spec, "_initializing", py::bool_(false)).cast()) { + return nullptr; + } + Py_INCREF(loaded); + torch_module = loaded; } return torch_module; } @@ -112,8 +119,8 @@ inline void synchronize_torch_cuda_device(int index) { } // Waits for every queued operation on a device. Bindings built without CUDA -// support have no CUDA runtime of their own, so they wait through torch's, -// which is loaded whenever such bindings see CUDA memory. +// support have no CUDA runtime of their own, so they wait through torch's +// once torch is imported. inline void synchronize_device(int index) { #ifndef EXECUTORCH_PYBINDINGS_CUDA if (loaded_torch_module() != nullptr) { diff --git a/extension/pybindings/test/test_pybindings.py b/extension/pybindings/test/test_pybindings.py index 74d719be0a7..b9d89935311 100644 --- a/extension/pybindings/test/test_pybindings.py +++ b/extension/pybindings/test/test_pybindings.py @@ -14,6 +14,7 @@ import subprocess import sys import tempfile +import textwrap import time import unittest import weakref @@ -163,115 +164,427 @@ def test_numpy_float16_buffer_input(self): self.assertTrue(torch.equal(output, inputs[0] + inputs[0])) - def test_buffer_input_does_not_import_torch(self): - exported_program, inputs = create_program(ModuleAddSingleInput()) - executorch_module = self.load_fn(exported_program.buffer) - torch_module = sys.modules.pop("torch") - try: - output = executorch_module(inputs[0].numpy())[0] - self.assertNotIn("torch", sys.modules) - finally: - sys.modules["torch"] = torch_module + def test_tensor_output_type(self): + if self.kernel_mode != "portable": + self.skipTest("only portable bindings choose Python tensor outputs") + from executorch.devtools.bundled_program.config import ( + MethodTestCase, + MethodTestSuite, + ) + from executorch.devtools.bundled_program.core import BundledProgram + from executorch.devtools.bundled_program.serialize import ( + serialize_from_bundled_program_to_flatbuffer, + ) - self.assertTrue(torch.equal(output, inputs[0] + inputs[0])) + inputs = ModuleAddWithAttributes().get_inputs() + program = to_edge( + export(ModuleAddWithAttributes(), inputs, strict=True), + constant_methods={"tensor_constant": torch.ones(2), "int_constant": 3}, + ).to_executorch(ExecutorchBackendConfig(emit_mutable_buffer_names=True)) + bundled_program, _ = create_program(ModuleAdd()) + bundled = serialize_from_bundled_program_to_flatbuffer( + BundledProgram( + bundled_program, + [ + MethodTestSuite( + method_name="forward", + test_cases=[ + MethodTestCase( + inputs=inputs, expected_outputs=(inputs[0] + inputs[1],) + ) + ], + ) + ], + ) + ) + prelude = """ +import importlib.abc +import sys +import threading +import warnings + +import numpy as np + +program_path, bundled_path = sys.argv[1:] +x = np.ones((2, 2), dtype=np.float32) - def test_tensor_output_import_policy(self): +class BlockTorch(importlib.abc.MetaPathFinder): + def __init__(self, error=ModuleNotFoundError): + self.error = error + # The module the error names: torch itself, or a part of it. + self.missing = "torch" + self.attempts = 0 + + def find_spec(self, name, path=None, target=None): + if name == "torch" or name.startswith("torch."): + self.attempts += 1 + error = self.error("torch is unavailable") + if isinstance(error, ImportError): + error.name = self.missing + raise error + +def kinds(outputs): + torch = sys.modules.get("torch") + return [ + "torch" if torch is not None and isinstance(value, torch.Tensor) + else type(value).__name__ + for value in outputs + ] + +# Importing torch can show warnings of its own, so only this one counts. +OURS = "ExecuTorch returned torch.Tensor outputs" + +def ours(caught): + return [w for w in caught if str(w.message).startswith(OURS)] + +def check(call, expected, warnings_expected=0): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + outputs = call() + future = ours(caught) + assert kinds(outputs) == expected, (kinds(outputs), expected) + assert len(future) == warnings_expected, [str(w.message) for w in future] + assert all(w.category is FutureWarning for w in future) + return future +""" + # Each runs in a fresh process, since torch imports and the + # warning, shown once per process, are process-wide. + scenarios = { + "NumPy inputs before torch is imported": ( + None, + """ + from executorch.runtime import Runtime + + assert "torch" not in sys.modules + program = Runtime.get().load_program(program_path) + check(lambda: program.load_method("int_constant").execute([]), ["int"]) + assert "torch" not in sys.modules + forward = program.load_method("forward") + (warning,), line = check(lambda: forward.execute([x, x]), ["torch"], 1), sys._getframe().f_lineno + # The warning points at the caller, not at ExecuTorch's wrapper. + assert (warning.filename, warning.lineno) == ("", line), warning + assert "EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT" in str(warning.message) + check(lambda: forward.execute([x, x]), ["torch"]) + """, + ), + "NumPy inputs kept by set_inputs after torch is imported": ( + None, + """ + import torch + from executorch.extension.pybindings import portable_lib + + native = sys.modules["executorch.extension.pybindings._C"] + assert native._tensor_output is None + method = portable_lib._load_program(program_path).load_method("forward") + method.set_inputs([x, x]) + method.execute() + # An attribute does not come from the inputs. + check(lambda: [method.get_attribute("state")], ["torch"]) + check(method.get_outputs, ["torch"], 1) + """, + ), + "NumPy inputs with every output bound to a torch tensor": ( + None, + """ + import torch + from executorch.exir import ExecutorchBackendConfig, to_edge + from executorch.exir.passes import MemoryPlanningPass + from executorch.extension.pybindings import portable_lib + + class Add(torch.nn.Module): + def forward(self, a, b): + return a + b + + t = torch.ones(2, 2) + config = ExecutorchBackendConfig( + memory_planning_pass=MemoryPlanningPass(alloc_graph_output=False) + ) + buffer = to_edge(torch.export.export(Add(), (t, t))).to_executorch(config).buffer + program = portable_lib._load_program_from_buffer(buffer) + method = program.load_method("forward") + bound = torch.empty(2, 2) + method.set_output(bound, 0) + # A bound output is the caller's own tensor, now and later. + check(lambda: method([x, x]), ["torch"]) + assert method([x, x])[0] is bound + # Bound after the run, so the outputs of that run are not. + late = program.load_method("forward") + late.set_inputs([x, x]) + late.execute() + late.set_output(torch.empty(2, 2), 0) + check(late.get_outputs, ["torch"], 1) + """, + ), + "torch inputs, kept or mixed, and no tensor inputs once torch is imported": ( + None, + """ + import torch + from executorch.extension.pybindings import portable_lib + + t = torch.ones(2, 2) + module = portable_lib._load_for_executorch(program_path) + check(lambda: module.forward([t, t]), ["torch"]) + # Input 1 is still the torch tensor of the last call. + check(lambda: module.run_method("forward", [x]), ["torch"]) + check(lambda: module.forward([x, t]), ["torch"]) + check(lambda: module.run_method("tensor_constant"), ["torch"]) + with open(bundled_path, "rb") as source: + bundled = portable_lib._load_bundled_program_from_buffer( + source.read() + ) + check( + lambda: bundled.verify_result_with_bundled_expected_output( + "forward", 0 + ), + ["torch"], + ) + """, + ), + "the warning turned into an error": ( + None, + """ + from executorch.extension.pybindings import portable_lib + + module = portable_lib._load_for_executorch(program_path) + method = portable_lib._load_program(program_path).load_method("forward") + with warnings.catch_warnings(): + warnings.filterwarnings("error", OURS) + for call in (module.forward, method) * 2: + try: + call([x, x]) + except FutureWarning: + pass + else: + raise AssertionError("every such call must raise") + # The raising calls did not run: state is still 0, so 1 + 1 + 1. + for call in (module.forward, method): + assert call([x, x])[0].tolist() == [[3.0, 3.0]] * 2 + """, + ), + "two threads before torch is imported": ( + None, + """ + from executorch.extension.pybindings import portable_lib + + modules = [portable_lib._load_for_executorch(program_path) for _ in range(2)] + barrier = threading.Barrier(2) + outputs = [None, None] + + def run(index): + barrier.wait() + outputs[index] = modules[index].forward([x, x])[0] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + threads = [threading.Thread(target=run, args=(i,)) for i in (0, 1)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert kinds(outputs) == ["torch", "torch"], kinds(outputs) + assert len(ours(caught)) == 1, [str(w.message) for w in caught] + """, + ), + "torch fails to import or is not installed": ( + None, + """ + blocker = BlockTorch(RuntimeError) + sys.meta_path.insert(0, blocker) + from executorch.extension.pybindings import portable_lib + + module = portable_lib._load_for_executorch(program_path) + # A torch that is there but broken, even with an ImportError or + # a missing module of its own. + for error_type, missing in ( + (RuntimeError, "torch"), + (ImportError, "torch"), + (ModuleNotFoundError, "torch._C"), + ): + blocker.error, blocker.missing = error_type, missing + try: + module.forward([x, x]) + except error_type as error: + assert "torch is unavailable" in str(error), error + # add_note is new in Python 3.11. + notes = "".join(getattr(error, "__notes__", [])) + assert sys.version_info < (3, 11) or ( + "EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT" in notes + ), notes + else: + raise AssertionError("a broken torch must not be hidden") + blocker.missing = "torch" + check(lambda: module.forward([x, x]), ["ExecuTorchResult"]) + check(lambda: module.run_method("tensor_constant"), ["ExecuTorchResult"]) + # A missing torch is not looked for again on every call. + assert blocker.attempts == 4, blocker.attempts + assert "torch" not in sys.modules + """, + ), + "the setting asks for ExecuTorchResult": ( + "executorch", + """ + from executorch.extension.pybindings import portable_lib + + native = sys.modules["executorch.extension.pybindings._C"] + assert native._tensor_output == "executorch" + module = portable_lib._load_for_executorch(program_path) + check(lambda: module.forward([x, x]), ["ExecuTorchResult"]) + check(lambda: module.run_method("tensor_constant"), ["ExecuTorchResult"]) + assert "torch" not in sys.modules + import torch + + t = torch.ones(2, 2) + check(lambda: module.forward([t, t]), ["ExecuTorchResult"]) + """, + ), + "the setting asks for torch.Tensor": ( + "torch", + """ + import importlib.util + import os + from pathlib import Path + + # The extension imported on its own, without portable_lib, which + # imports torch itself and on Windows adds the folders of the + # extension's libraries. + name = "executorch.extension.pybindings._C" + if sys.platform == "win32": + native_directory = Path(importlib.util.find_spec(name).origin).parent + for directory in (native_directory, native_directory.parent.parent / "lib"): + if directory.is_dir(): + os.add_dll_directory(str(directory.resolve())) + native = importlib.import_module(name) + assert "torch" in sys.modules + assert native._tensor_output == "torch" + from executorch.extension.pybindings import portable_lib + + module = portable_lib._load_for_executorch(program_path) + check(lambda: module.forward([x, x]), ["torch"]) + """, + ), + "the setting asks for torch.Tensor without torch": ( + "torch", + """ + sys.meta_path.insert(0, BlockTorch()) + try: + from executorch.extension.pybindings import portable_lib + except ImportError as error: + assert "PyTorch could not be imported" in str(error), error + else: + raise AssertionError("importing must fail without torch") + """, + ), + } + # Each of these calls returns torch.Tensor outputs now and will return + # ExecuTorchResult outputs later, so it warns: calls without tensor + # inputs before torch is imported, and calls with tensor inputs of + # other kinds after it is. + for call, torch_imported in ( + ("module.run_method('tensor_constant')", False), + ("[method.get_attribute('state')]", False), + ("bundled.verify_result_with_bundled_expected_output('forward', 0)", False), + ("module.forward([x, x])", True), + ("module.forward(capsules)", True), + ("method(capsules)", True), + ): + scenarios[f"{call}, torch imported: {torch_imported}"] = ( + None, + f""" + {'import torch' if torch_imported else ''} + from executorch.extension.pybindings import portable_lib + + module = portable_lib._load_for_executorch(program_path) + method = portable_lib._load_program(program_path).load_method("forward") + with open(bundled_path, "rb") as source: + bundled = portable_lib._load_bundled_program_from_buffer(source.read()) + capsules = [x.__dlpack__(), x.__dlpack__()] + (warning,), line = check(lambda: {call}, ["torch"], 1), sys._getframe().f_lineno + assert warning.lineno == line, (warning.lineno, line) + """, + ) + with tempfile.TemporaryDirectory() as directory: + program_path = os.path.join(directory, "program.pte") + with open(program_path, "wb") as handle: + handle.write(program.buffer) + bundled_path = os.path.join(directory, "program.bpte") + with open(bundled_path, "wb") as handle: + handle.write(bundled) + for name, (setting, code) in scenarios.items(): + with self.subTest(name): + environment = os.environ.copy() + environment.pop("EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT", None) + if setting is not None: + environment["EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT"] = setting + completed = subprocess.run( + [ + sys.executable, + "-c", + prelude + textwrap.dedent(code), + program_path, + bundled_path, + ], + env=environment, + capture_output=True, + text=True, + timeout=120, + ) + self.assertEqual( + completed.returncode, 0, completed.stdout + completed.stderr + ) + + def test_torch_that_is_not_ready_is_ignored(self): if self.kernel_mode != "portable": - self.skipTest("only portable bindings configure Python tensor outputs") + self.skipTest("only portable bindings look torch up in sys.modules") program, _ = create_program(ModuleAddSingleInput()) script = """ -import importlib -import importlib.abc import os import sys -from pathlib import Path +import threading -class BlockTorch(importlib.abc.MetaPathFinder): - def find_spec(self, fullname, path=None, target=None): - if fullname == "torch" or fullname.startswith("torch."): - raise ImportError("torch is unavailable") - -entrypoint, native_name, policy, available, program_path, native_path = sys.argv[1:] -if sys.platform == "win32": - native_directory = Path(native_path).parent - os.add_dll_directory(str(native_directory)) - library_directory = native_directory.parent.parent / "lib" - if library_directory.is_dir(): - os.add_dll_directory(str(library_directory)) -blocker = BlockTorch() -if available == "no": - sys.meta_path.insert(0, blocker) -try: - bindings = importlib.import_module(entrypoint) -except (ImportError, SystemError) as error: - # Built-in extension loaders may wrap initialization errors. - assert policy == "torch" and available == "no" - while error.__cause__ is not None: - error = error.__cause__ - assert isinstance(error, ImportError) - assert "torch is unavailable" in str(error) +import numpy as np + +program_path, case = sys.argv[1:] +if case == "none": + sys.modules["torch"] = None else: - assert policy != "torch" or available == "yes" - native = sys.modules[native_name] - expected = "torch" if policy != "executorch" and available == "yes" else "executorch" - assert native._tensor_output == expected - assert ("torch" in sys.modules) == (expected == "torch") - import numpy as np - program_data = Path(program_path).read_bytes() - module = bindings._load_for_executorch_from_buffer(program_data) - output = module(np.ones((2, 2), dtype=np.float32))[0] - if expected == "executorch": - assert isinstance(output, bindings.ExecuTorchResult) - np.testing.assert_array_equal(np.asarray(output), np.full((2, 2), 2.0)) - if available == "no": - sys.meta_path.remove(blocker) - import torch - assert isinstance(module(torch.ones(2, 2))[0], bindings.ExecuTorchResult) - else: - import torch - assert isinstance(output, torch.Tensor) - torch.testing.assert_close(output, torch.full((2, 2), 2.0)) + # A torch that another thread is still importing. + package = os.path.join(os.path.dirname(program_path), "partial", "torch") + os.makedirs(package) + with open(os.path.join(package, "__init__.py"), "w") as source: + source.write("import __main__\\n__main__.started.set()\\n__main__.release.wait()\\n") + sys.path.insert(0, os.path.dirname(package)) + started, release = threading.Event(), threading.Event() + importer = threading.Thread(target=__import__, args=("torch",), daemon=True) + importer.start() + started.wait() +from executorch.extension.pybindings import portable_lib +with open(program_path, "rb") as source: + module = portable_lib._load_for_executorch_from_buffer(source.read()) +output = module(np.ones((2, 2), dtype=np.float32))[0] +assert isinstance(output, portable_lib.ExecuTorchResult), type(output) +if case != "none": + release.set() + importer.join() """ - native_name = self.load_fn.__module__ - native_path = sys.modules[native_name].__spec__.origin with tempfile.TemporaryDirectory() as directory: program_path = os.path.join(directory, "add.pte") with open(program_path, "wb") as handle: handle.write(program.buffer) - for entrypoint in dict.fromkeys((self.runtime.__name__, native_name)): - for policy in (None, "torch", "executorch"): - for available in ("yes", "no"): - with self.subTest( - entrypoint=entrypoint, policy=policy, available=available - ): - environment = os.environ.copy() - environment.pop("EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT", None) - if policy is not None: - environment["EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT"] = ( - policy - ) - completed = subprocess.run( - [ - sys.executable, - "-c", - script, - entrypoint, - native_name, - str(policy), - available, - program_path, - native_path, - ], - env=environment, - capture_output=True, - text=True, - timeout=60, - ) - self.assertEqual( - completed.returncode, - 0, - completed.stdout + completed.stderr, - ) + environment = dict( + os.environ, EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT="executorch" + ) + for case in ("none", "importing"): + with self.subTest(case=case): + completed = subprocess.run( + [sys.executable, "-c", script, program_path, case], + env=environment, + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual( + completed.returncode, 0, completed.stdout + completed.stderr + ) def test_empty_numpy_buffer_input(self): exported_program, inputs = create_program(ModuleAddEmpty()) @@ -1420,7 +1733,7 @@ def test_method_rejects_aliased_input_whose_storage_changed(self): self.skipTest("only portable bindings return a view of the input") from executorch.extension.pybindings import _C - if _C._tensor_output != "torch": + if _C._tensor_output == "executorch": self.skipTest("only torch output mode returns a view of the input") class Identity(torch.nn.Module): diff --git a/extension/pybindings/test/test_pybindings_no_aten.py b/extension/pybindings/test/test_pybindings_no_aten.py index 8488ac01e6c..acb1ece9a87 100644 --- a/extension/pybindings/test/test_pybindings_no_aten.py +++ b/extension/pybindings/test/test_pybindings_no_aten.py @@ -134,7 +134,7 @@ def test_output_policy_without_torch(self) -> None: import os assert Path(runtime.__file__).resolve() == Path(sys.argv[1]).resolve() - assert runtime._tensor_output == "executorch" + assert runtime._tensor_output is None with open(os.environ["EXECUTORCH_PYBIND_TEST_PTE"], "rb") as source: data = source.read() module = runtime._load_for_executorch_from_buffer(data) @@ -211,6 +211,21 @@ def test_executorch_result_to_host_and_repr(self) -> None: self.assertFalse(np.shares_memory(np.asarray(copy), np.asarray(output))) np.testing.assert_array_equal(np.asarray(copy), np.array([3.0])) + def test_executorch_result_is_not_a_bool_or_number(self) -> None: + if runtime._tensor_output != "executorch": + self.skipTest("ExecuTorchResult output was not selected") + with open(os.environ["EXECUTORCH_PYBIND_TEST_PTE"], "rb") as program_file: + program_data = program_file.read() + module = runtime._load_for_executorch_from_buffer(program_data) + output = module( + (np.array([1.0], dtype=np.float32), np.array([2.0], dtype=np.float32)) + )[0] + + for convert in (bool, int, float): + with self.subTest(convert=convert.__name__): + with self.assertRaisesRegex(TypeError, r"numpy\.asarray\(result\)"): + convert(output) + def test_executorch_result_from_host_requires_cuda(self) -> None: # A build with CUDA support uploads instead; the CUDA suite covers it. if "CudaBackend" in runtime._get_registered_backend_names(): @@ -475,9 +490,9 @@ def test_z_torch_tensor_uses_python_api(self) -> None: output = module((torch.tensor([1.0]), torch.tensor([2.0])))[0] expected_type = ( - torch.Tensor - if runtime._tensor_output == "torch" - else runtime.ExecuTorchResult + runtime.ExecuTorchResult + if runtime._tensor_output == "executorch" + else torch.Tensor ) self.assertIsInstance(output, expected_type) np.testing.assert_array_equal(np.asarray(output), np.array([3.0])) diff --git a/runtime/__init__.py b/runtime/__init__.py index 0aa0d89655c..204b18df6b8 100644 --- a/runtime/__init__.py +++ b/runtime/__init__.py @@ -172,8 +172,20 @@ def execute( a later execution may overwrite them, which avoids a copy. Returns: - A list of output values, typically torch.Tensor objects. Without - PyTorch, tensor outputs are ExecuTorchResult objects. + A list of output values. When PyTorch is installed, tensor outputs + are torch.Tensor objects, and the first call that returns one + imports torch. Without PyTorch, they are ExecuTorchResult objects. + + .. warning:: + + A future release will make tensor outputs follow the tensor inputs: + torch.Tensor objects if any input is one, ExecuTorchResult objects + if all are other tensors, even when PyTorch is installed, and with + no tensor inputs, torch.Tensor objects only if torch is already + imported. The first call whose outputs will change shows a + FutureWarning. Set ``EXECUTORCH_PYBINDINGS_TENSOR_OUTPUT`` to + ``torch`` or ``executorch`` before importing ExecuTorch to choose + the type now. """ return self._method(inputs, clone_outputs=clone_outputs)