diff --git a/.ci/scripts/test_openvino.sh b/.ci/scripts/test_openvino.sh index 2bb2115b1ec..7f5338b3b0a 100755 --- a/.ci/scripts/test_openvino.sh +++ b/.ci/scripts/test_openvino.sh @@ -12,5 +12,8 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" source openvino/setupvars.sh cd backends/openvino/tests -python test_runner.py --test_type ops -python test_runner.py --test_type models +# Run the model tests even when an op test fails, and still fail the job. +status=0 +python test_runner.py --test_type ops || status=1 +python test_runner.py --test_type models || status=1 +exit "${status}" diff --git a/backends/openvino/README.md b/backends/openvino/README.md index 331aebf87ff..883e1e285f6 100644 --- a/backends/openvino/README.md +++ b/backends/openvino/README.md @@ -48,6 +48,63 @@ print(_get_registered_backend_names()) # Should include 'OpenvinoBackend' ``` +## Precision + +When the device is `CPU`, a program computes in float32 by default. Without this default, +OpenVINO would compute in bfloat16 on x86 CPUs with bfloat16 units, which keeps only two to +three significant digits, and in float16 on ARM CPUs, which keeps about three. +Both are faster. To trade accuracy for speed, ask for the faster precision when +lowering: + +```python +from executorch.backends.openvino.partitioner import OpenvinoPartitioner +from executorch.exir.backend.compile_spec_schema import CompileSpec + +OpenvinoPartitioner( + [CompileSpec("device", b"CPU"), CompileSpec("inference_precision", b"bf16")] +) +``` + +On an ARM CPU, ask for `f16` instead: there float32 is about 1.8x slower than +float16, as measured on an Orin Nano. + +The value is OpenVINO's `INFERENCE_PRECISION_HINT`: `f32`, `f16` or `bf16`. An +empty value means the default, and any other value fails at lowering. A CPU +without float16 support computes an `f16` program in float32. OpenVINO has no +bfloat16 on ARM CPUs, so there `bf16` keeps the ARM default, float16 where the +CPU supports it. Which x86 CPUs compute in bfloat16 is up to OpenVINO; see its +[Precision Control](https://docs.openvino.ai/2026/openvino-workflow/running-inference/optimize-inference/precision-control.html) +page. Other devices, such as GPUs, keep OpenVINO's default unless the option is +set. + +The default also applies to a program exported in float16: on a CPU it computes +in float32 unless you lower it with `inference_precision`. + +On x86 CPUs, OpenVINO applies the precision when the runtime loads a program. +So a program lowered without `inference_precision` computes in float32, even +one exported with an ExecuTorch release that did not have this default. On an +x86 CPU with bfloat16 units, float32 is slower and can use more memory; export it +again with `inference_precision` set to `bf16` to get the earlier speed. Other +x86 CPUs already computed such a program in float32. On ARM CPUs, OpenVINO applies the +precision when the program is lowered, so a program exported with such an +earlier release keeps computing in float16 until it is exported again. + +OpenVINO's CPU plugin computes 64-bit integers in 32 bits. An integer result +outside the int32 range wraps around without an error. This applies to every +int64 tensor, including the int64 result of `cumsum` and `prod` over smaller +integer types. In OpenVINO 2026.4.1, the CPU plugin also computes some int32 and +int64 ops in float32. On x86 CPUs these are `sum`, `amax`, `amin`, `max` and +`min`. On ARM CPUs they are element-wise ops such as `add`, `maximum` and +`clamp`, and the same reductions along any dimension but the last. Once a value +or a running total is above 16,777,216 (2^24) or below -16,777,216, such a +result can be off, even inside the int32 range. For example, on x86 `sum` over a +tensor that holds 16,777,216 and 1 gives 16,777,216, and so does `amax` over one +that holds 16,777,217. + +In OpenVINO 2026.4.1, the CPU plugin's `min(dim)` and `max(dim)` can return a +later index on ties than PyTorch, which returns the first one. This happens with +float32 as well as integer dtypes such as int32 and uint8. + ## Directory Structure ``` diff --git a/backends/openvino/_passes/__init__.py b/backends/openvino/_passes/__init__.py index 0db94ca1ec8..bc2853a73e6 100644 --- a/backends/openvino/_passes/__init__.py +++ b/backends/openvino/_passes/__init__.py @@ -4,6 +4,12 @@ # except in compliance with the License. See the license file found in the # LICENSE file in the root directory of this source tree. +from .cast_reduction_input_pass import CastReductionInputPass from .decompose_floor_divide_pass import DecomposeFloorDividePass +from .fold_constant_cumsum_pass import FoldConstantCumsumPass -__all__ = ["DecomposeFloorDividePass"] +__all__ = [ + "CastReductionInputPass", + "DecomposeFloorDividePass", + "FoldConstantCumsumPass", +] diff --git a/backends/openvino/_passes/cast_reduction_input_pass.py b/backends/openvino/_passes/cast_reduction_input_pass.py new file mode 100644 index 00000000000..36e3f3795c0 --- /dev/null +++ b/backends/openvino/_passes/cast_reduction_input_pass.py @@ -0,0 +1,45 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import torch +from executorch.exir.dialects._ops import ops as exir_ops +from torch.fx.passes.infra.pass_base import PassBase, PassResult + +REDUCTION_OPS = { + exir_ops.edge.aten.cumsum.default, + exir_ops.edge.aten.prod.default, + exir_ops.edge.aten.prod.dim_int, +} + + +class CastReductionInputPass(PassBase): + """Cast the input of cumsum and prod to the dtype of their result. + + PyTorch reduces bool and integer tensors in int64, but OpenVINO keeps a + narrower type for these two, though not for sum. A cumsum over a bool mask + is computed in 8 bits and wraps after 255, and a cumsum or prod over int32 + returns int32. + """ + + def call(self, graph_module: torch.fx.GraphModule) -> PassResult: + modified = False + for node in graph_module.graph.nodes: + if node.op != "call_function" or node.target not in REDUCTION_OPS: + continue + source = node.args[0] + dtype = node.meta["val"].dtype + if source.meta["val"].dtype == dtype: + continue + with graph_module.graph.inserting_before(node): + cast = graph_module.graph.call_function( + exir_ops.edge.aten._to_copy.default, (source,), {"dtype": dtype} + ) + cast.meta["val"] = source.meta["val"].to(dtype) + node.replace_input_with(source, cast) + modified = True + if modified: + graph_module.recompile() + return PassResult(graph_module, modified) diff --git a/backends/openvino/_passes/fold_constant_cumsum_pass.py b/backends/openvino/_passes/fold_constant_cumsum_pass.py new file mode 100644 index 00000000000..cae7002f05f --- /dev/null +++ b/backends/openvino/_passes/fold_constant_cumsum_pass.py @@ -0,0 +1,161 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import logging +import operator + +import torch +from executorch.exir.dialects._ops import ops as exir_ops +from executorch.exir.passes.constant_prop_pass import _nodes_written_in_place +from torch.fx.passes.infra.pass_base import PassBase, PassResult +from torch.utils import _pytree as pytree + + +def _reads_only_shape(node: torch.fx.Node) -> bool: + """full_like with a constant fill value reads only the shape and dtype of + its input. With a static shape its result is a constant, and OpenVINO + folds it into one too. + """ + if node.target != exir_ops.edge.aten.full_like.default: + return False + if any( + isinstance(a, torch.fx.Node) for a in (*node.args[1:], *node.kwargs.values()) + ): + return False + val = node.meta.get("val") + return isinstance(val, torch.Tensor) and all(isinstance(d, int) for d in val.shape) + + +def _inputs(node: torch.fx.Node) -> list: + return [] if _reads_only_shape(node) else node.all_input_nodes + + +def _on_cpu(value): + if isinstance(value, torch.Tensor): + return value.cpu() + if isinstance(value, torch.device): + return torch.device("cpu") + return value + + +def _constant_nodes(graph_module: torch.fx.GraphModule) -> set: + """Nodes that depend only on constants. Graph order puts inputs first.""" + constants = set() + written = _nodes_written_in_place(graph_module) + for node in graph_module.graph.nodes: + # Only tensors: a higher-order op such as cond reads its subgraphs, + # which may hold a random op, through get_attr nodes too. + if node.op == "get_attr": + value = operator.attrgetter(node.target)(graph_module) + if isinstance(value, torch.Tensor) and node not in written: + constants.add(node) + elif node.op == "call_function" and ( + _reads_only_shape(node) + or ( + torch.Tag.nondeterministic_seeded + not in getattr(node.target, "tags", ()) + and all(n in constants for n in node.all_input_nodes) + ) + ): + constants.add(node) + return constants + + +def _evaluate( + graph_module: torch.fx.GraphModule, target: torch.fx.Node +) -> torch.Tensor: + """Value of target, computing only what it reads, on the CPU.""" + needed = set() + stack: list = [target] + while stack: + node = stack.pop() + if node not in needed: + needed.add(node) + stack.extend(_inputs(node)) + order = [n for n in graph_module.graph.nodes if n in needed] + last_use = {i: n for n in order for i in _inputs(n)} + values = {} + for node in order: + if node.op == "get_attr": + values[node] = operator.attrgetter(node.target)(graph_module) + continue + if _reads_only_shape(node): + like = node.args[0].meta["val"] + args = ( + torch.empty(like.shape, dtype=like.dtype, device="cpu"), + *node.args[1:], + ) + kwargs = dict(node.kwargs) + else: + args, kwargs = torch.fx.node.map_arg( + (node.args, node.kwargs), lambda n: values[n] + ) + # The values are the same on any device, and lowering for OpenVINO + # must not need the GPU a constant was made for. + args, kwargs = pytree.tree_map(_on_cpu, (args, kwargs)) + if kwargs.get("pin_memory"): + kwargs = {**kwargs, "pin_memory": False} + values[node] = node.target(*args, **kwargs) + for i in _inputs(node): + if last_use[i] is node: + del values[i] + return values[target] + + +class FoldConstantCumsumPass(PassBase): + """Replace an integer cumsum that reads only constants with its result. + + With an integer cumsum over a constant, OpenVINO's CPU plugin (2025.1 to + 2026.4) gives wrong results from the second inference on: masks built from + it keep only what earlier calls allowed. pi0 builds its attention masks + this way. The OpenVINO nightly no longer does this, so the pass can go once + the oldest supported OpenVINO release has the fix. Only these cumsums are + folded; constants that feed no such cumsum, such as compressed weights, are + left as they are. Runs on an unlifted module, where constants are get_attr + nodes; buffers the graph writes to are not constants, and full_like of an + input with a static shape is one. + """ + + def call(self, graph_module: torch.fx.GraphModule) -> PassResult: + graph = graph_module.graph + constants = _constant_nodes(graph_module) + cumsums = [ + n + for n in graph.nodes + if n in constants + and n.target == exir_ops.edge.aten.cumsum.default + and not n.meta["val"].dtype.is_floating_point + and not n.meta["val"].dtype.is_complex + ] + modified = False + # One at a time, so a cumsum that cannot be evaluated does not keep + # the others from being folded. + for node in cumsums: + try: + value = _evaluate(graph_module, node) + except RuntimeError as e: + # Some ops have modes the CPU lacks. + logging.warning( + "OpenVINO: could not fold %s, so on a CPU some OpenVINO " + "releases give wrong results for it from the second " + "inference on: %s", + node.name, + e, + ) + continue + name = f"_folded_cumsum_{node.name}" + while hasattr(graph_module, name): + name += "_" + graph_module.register_buffer(name, value) + with graph.inserting_before(node): + folded = graph.get_attr(name) + folded.meta = node.meta.copy() + node.replace_all_uses_with(folded) + modified = True + if modified: + graph.eliminate_dead_code() + graph_module.recompile() + return PassResult(graph_module, modified) diff --git a/backends/openvino/preprocess.py b/backends/openvino/preprocess.py index 7a3f490b258..8d934dc05a9 100644 --- a/backends/openvino/preprocess.py +++ b/backends/openvino/preprocess.py @@ -9,7 +9,11 @@ from typing import final, List import torch -from executorch.backends.openvino._passes import DecomposeFloorDividePass +from executorch.backends.openvino._passes import ( + CastReductionInputPass, + DecomposeFloorDividePass, + FoldConstantCumsumPass, +) from executorch.exir.backend.backend_details import ( BackendDetails, @@ -23,8 +27,40 @@ from openvino.frontend.pytorch.fx_decoder import ( # type: ignore[import-untyped] TorchFXPythonDecoder, ) +from openvino.frontend.pytorch.utils import ( # type: ignore[import-untyped] + pt_to_ov_type_map, +) from openvino.preprocess import PrePostProcessor # type: ignore[import-untyped] +# The values of the inference_precision compile spec. The runtime +# (OpenvinoBackend.cpp) accepts the same ones. +_PRECISIONS = ("f32", "f16", "bf16") + + +def _match_partition_inputs( + model, module: torch.fx.GraphModule, input_shapes: list, input_types: list +) -> None: + """Make the converted model take the inputs the runtime passes.""" + # The converter takes an unused first input for forward's self and removes + # it, but the runtime passes every partition input. The cumsum fold can + # leave the first input unused. + parameters = list(model.get_parameters()) + placeholders = [n for n in module.graph.nodes if n.op == "placeholder"] + if len(parameters) == len(input_types) - 1 and not placeholders[0].users: + for parameter in parameters: + model.remove_parameter(parameter) + first = opset13.parameter( + input_shapes[0], pt_to_ov_type_map[str(input_types[0])] + ) + model.add_parameters([first, *parameters]) + # The runtime binds inputs by position and refuses a wrong count, so any + # other mismatch fails here instead of on every call. + if len(model.get_parameters()) != len(input_types): + raise RuntimeError( + f"OpenVINO backend: expected {len(input_types)} parameters in the " + f"converted model, got {len(model.get_parameters())}" + ) + @final class OpenvinoBackend(BackendDetails): @@ -43,7 +79,25 @@ def preprocess( Returns: PreprocessResult: The result of preprocessing, including the compiled model bytes. """ - for pass_cls in [DimOrderOpsRevertPass, DecomposeFloorDividePass]: + compile_options = {} + for spec in module_compile_spec: + compile_options[spec.key] = spec.value.decode() + device = compile_options.get("device", "CPU") + requested_precision = compile_options.get("inference_precision") + # Some OpenVINO releases accept an unknown value. Checking before the + # conversion fails fast and keeps it out of the program, where the + # runtime would only fail at load. + if requested_precision and requested_precision not in _PRECISIONS: + raise ValueError( + f"OpenVINO backend: unsupported inference_precision " + f"'{requested_precision}', expected one of {', '.join(_PRECISIONS)}" + ) + + for pass_cls in [ + DimOrderOpsRevertPass, + DecomposeFloorDividePass, + CastReductionInputPass, + ]: result = pass_cls()(edge_program.graph_module) if result and result.graph_module: edge_program._graph_module = result.graph_module @@ -70,23 +124,27 @@ def preprocess( f"{type(val).__name__}; only tensors and ints are supported" ) - compile_options = {} - for spec in module_compile_spec: - compile_options[spec.key] = spec.value.decode() - + module = edge_program.module() + FoldConstantCumsumPass()(module) # The converter chooses some lowerings (e.g. boolean-mask indexing) from # the input types, so they must be known before conversion starts. decoder = TorchFXPythonDecoder( - edge_program.module(), input_shapes=input_shapes, input_types=input_types + module, input_shapes=input_shapes, input_types=input_types ) frontend = FrontEndManager().load_by_framework("pytorch") - ppp = PrePostProcessor(frontend.convert(frontend.load(decoder))) + model = frontend.convert(frontend.load(decoder)) + _match_partition_inputs(model, module, input_shapes, input_types) + ppp = PrePostProcessor(model) # The converter takes an int input, like a SymInt size, as a scalar, but # the runtime passes it as an int64 tensor of shape [1]. for i in int_inputs: ppp.input(i).tensor().set_shape([1]) ppp.input(i).preprocess().custom(lambda x: opset13.squeeze(x, 0).output(0)) - compiled = compile_model(ppp.build(), compile_options.get("device", "CPU")) + # On ARM CPUs OpenVINO fixes the precision here, not when the runtime + # imports the model, so this must follow the same rule as the runtime. + precision = requested_precision or ("f32" if device.startswith("CPU") else None) + config = {"INFERENCE_PRECISION_HINT": precision} if precision else {} + compiled = compile_model(ppp.build(), device, config) model_bytes = compiled.export_model() return PreprocessResult(processed_bytes=model_bytes.getvalue()) diff --git a/backends/openvino/runtime/OpenvinoApi.h b/backends/openvino/runtime/OpenvinoApi.h index e4c3e7f737a..08af221c3c0 100644 --- a/backends/openvino/runtime/OpenvinoApi.h +++ b/backends/openvino/runtime/OpenvinoApi.h @@ -22,6 +22,7 @@ typedef struct ov_core ov_core_t; typedef struct ov_compiled_model ov_compiled_model_t; typedef struct ov_infer_request ov_infer_request_t; typedef struct ov_tensor ov_tensor_t; +typedef struct ov_output_const_port ov_output_const_port_t; // Value types reproduced from openvino/c/ov_shape.h and ov_common.h. // These are stable C ABI — pinned via version constraint in pyproject.toml. @@ -30,6 +31,17 @@ typedef struct { int64_t* dims; } ov_shape_t; +// Reproduced from openvino/c/ov_dimension.h and ov_partial_shape.h. +typedef struct { + int64_t min; + int64_t max; +} ov_dimension_t; + +typedef struct { + ov_dimension_t rank; + ov_dimension_t* dims; +} ov_partial_shape_t; + typedef struct { char** devices; size_t size; @@ -76,6 +88,9 @@ using ov_core_free_fn = void (*)(ov_core_t*); using ov_core_get_available_devices_fn = ov_status_e (*)(const ov_core_t*, ov_available_devices_t*); using ov_available_devices_free_fn = void (*)(ov_available_devices_t*); +// Takes one property key and value, both strings, after the device name. +using ov_core_set_property_fn = + ov_status_e (*)(const ov_core_t*, const char*, ...); using ov_core_import_model_fn = ov_status_e (*)( const ov_core_t*, const char*, @@ -89,18 +104,33 @@ using ov_compiled_model_inputs_size_fn = using ov_compiled_model_outputs_size_fn = ov_status_e (*)(const ov_compiled_model_t*, size_t*); using ov_compiled_model_free_fn = void (*)(ov_compiled_model_t*); +using ov_compiled_model_output_by_index_fn = ov_status_e (*)( + const ov_compiled_model_t*, + size_t, + ov_output_const_port_t**); +using ov_port_get_partial_shape_fn = + ov_status_e (*)(const ov_output_const_port_t*, ov_partial_shape_t*); +using ov_output_const_port_free_fn = void (*)(ov_output_const_port_t*); using ov_infer_request_set_input_tensor_by_index_fn = ov_status_e (*)(ov_infer_request_t*, size_t, const ov_tensor_t*); using ov_infer_request_set_output_tensor_by_index_fn = ov_status_e (*)(ov_infer_request_t*, size_t, const ov_tensor_t*); using ov_infer_request_infer_fn = ov_status_e (*)(ov_infer_request_t*); +using ov_infer_request_get_output_tensor_by_index_fn = + ov_status_e (*)(const ov_infer_request_t*, size_t, ov_tensor_t**); using ov_infer_request_free_fn = void (*)(ov_infer_request_t*); using ov_tensor_create_from_host_ptr_fn = ov_status_e (*)(ov_element_type_e, ov_shape_t, void*, ov_tensor_t**); +using ov_tensor_create_fn = + ov_status_e (*)(ov_element_type_e, ov_shape_t, ov_tensor_t**); using ov_tensor_free_fn = void (*)(ov_tensor_t*); +using ov_tensor_get_shape_fn = ov_status_e (*)(const ov_tensor_t*, ov_shape_t*); +using ov_tensor_get_byte_size_fn = ov_status_e (*)(const ov_tensor_t*, size_t*); +using ov_tensor_data_fn = ov_status_e (*)(const ov_tensor_t*, void**); using ov_shape_create_fn = ov_status_e (*)(int64_t, const int64_t*, ov_shape_t*); using ov_shape_free_fn = ov_status_e (*)(ov_shape_t*); +using ov_partial_shape_free_fn = void (*)(ov_partial_shape_t*); // Defined in OpenvinoBackend.cpp, where the platform's loader API is included. struct DlCloser { @@ -115,22 +145,33 @@ struct OpenvinoFunctions { ov_core_free_fn core_free = nullptr; ov_core_get_available_devices_fn core_get_available_devices = nullptr; ov_available_devices_free_fn available_devices_free = nullptr; + ov_core_set_property_fn core_set_property = nullptr; ov_core_import_model_fn core_import_model = nullptr; ov_compiled_model_create_infer_request_fn compiled_model_create_infer_request = nullptr; ov_compiled_model_inputs_size_fn compiled_model_inputs_size = nullptr; ov_compiled_model_outputs_size_fn compiled_model_outputs_size = nullptr; ov_compiled_model_free_fn compiled_model_free = nullptr; + ov_compiled_model_output_by_index_fn compiled_model_output_by_index = nullptr; + ov_port_get_partial_shape_fn port_get_partial_shape = nullptr; + ov_output_const_port_free_fn output_const_port_free = nullptr; ov_infer_request_set_input_tensor_by_index_fn infer_request_set_input_tensor_by_index = nullptr; ov_infer_request_set_output_tensor_by_index_fn infer_request_set_output_tensor_by_index = nullptr; ov_infer_request_infer_fn infer_request_infer = nullptr; + ov_infer_request_get_output_tensor_by_index_fn + infer_request_get_output_tensor_by_index = nullptr; ov_infer_request_free_fn infer_request_free = nullptr; ov_tensor_create_from_host_ptr_fn tensor_create_from_host_ptr = nullptr; + ov_tensor_create_fn tensor_create = nullptr; ov_tensor_free_fn tensor_free = nullptr; + ov_tensor_get_shape_fn tensor_get_shape = nullptr; + ov_tensor_get_byte_size_fn tensor_get_byte_size = nullptr; + ov_tensor_data_fn tensor_data = nullptr; ov_shape_create_fn shape_create = nullptr; ov_shape_free_fn shape_free = nullptr; + ov_partial_shape_free_fn partial_shape_free = nullptr; }; } // namespace openvino diff --git a/backends/openvino/runtime/OpenvinoBackend.cpp b/backends/openvino/runtime/OpenvinoBackend.cpp index fad6ef91cb1..ff7c33aa4d9 100644 --- a/backends/openvino/runtime/OpenvinoBackend.cpp +++ b/backends/openvino/runtime/OpenvinoBackend.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "OpenvinoBackend.h" @@ -236,12 +237,16 @@ bool OpenvinoBackend::ensure_loaded() const { LOAD_SYM(core_free, ov_core_free) LOAD_SYM(core_get_available_devices, ov_core_get_available_devices) LOAD_SYM(available_devices_free, ov_available_devices_free) + LOAD_SYM(core_set_property, ov_core_set_property) LOAD_SYM(core_import_model, ov_core_import_model) LOAD_SYM( compiled_model_create_infer_request, ov_compiled_model_create_infer_request) LOAD_SYM(compiled_model_inputs_size, ov_compiled_model_inputs_size) LOAD_SYM(compiled_model_outputs_size, ov_compiled_model_outputs_size) + LOAD_SYM(compiled_model_output_by_index, ov_compiled_model_output_by_index) + LOAD_SYM(port_get_partial_shape, ov_port_get_partial_shape) + LOAD_SYM(output_const_port_free, ov_output_const_port_free) LOAD_SYM(compiled_model_free, ov_compiled_model_free) LOAD_SYM( infer_request_set_input_tensor_by_index, @@ -250,11 +255,19 @@ bool OpenvinoBackend::ensure_loaded() const { infer_request_set_output_tensor_by_index, ov_infer_request_set_output_tensor_by_index) LOAD_SYM(infer_request_infer, ov_infer_request_infer) + LOAD_SYM( + infer_request_get_output_tensor_by_index, + ov_infer_request_get_output_tensor_by_index) LOAD_SYM(infer_request_free, ov_infer_request_free) LOAD_SYM(tensor_create_from_host_ptr, ov_tensor_create_from_host_ptr) + LOAD_SYM(tensor_create, ov_tensor_create) LOAD_SYM(tensor_free, ov_tensor_free) + LOAD_SYM(tensor_get_shape, ov_tensor_get_shape) + LOAD_SYM(tensor_get_byte_size, ov_tensor_get_byte_size) + LOAD_SYM(tensor_data, ov_tensor_data) LOAD_SYM(shape_create, ov_shape_create) LOAD_SYM(shape_free, ov_shape_free) + LOAD_SYM(partial_shape_free, ov_partial_shape_free) #undef LOAD_SYM @@ -321,17 +334,53 @@ exr::Result OpenvinoBackend::init( size_t data_size = processed->size(); std::string device = "CPU"; + std::string precision; for (auto& compile_spec : compile_specs) { - if (std::strcmp(compile_spec.key, "device") == 0) { - const char* buf = static_cast(compile_spec.value.buffer); - size_t len = compile_spec.value.nbytes; - // Strip trailing null bytes that may be included in nbytes. - while (len > 0 && buf[len - 1] == '\0') { - --len; - } - if (len > 0) { - device.assign(buf, len); - } + const bool is_device = std::strcmp(compile_spec.key, "device") == 0; + if (!is_device && + std::strcmp(compile_spec.key, "inference_precision") != 0) { + continue; + } + const char* buf = static_cast(compile_spec.value.buffer); + size_t len = compile_spec.value.nbytes; + // Strip trailing null bytes that may be included in nbytes. + while (len > 0 && buf[len - 1] == '\0') { + --len; + } + (is_device ? device : precision).assign(buf, len); + } + + // The same rule as preprocess.py, because on x86 CPUs OpenVINO applies the + // precision here, on import. + if (precision.empty() && device.rfind("CPU", 0) == 0) { + precision = "f32"; + } + // The same values as in preprocess.py. Some OpenVINO releases accept an + // unknown value, and others only reject it on import, with a generic + // error. + if (!precision.empty() && precision != "f32" && precision != "f16" && + precision != "bf16") { + ET_LOG( + Error, + "OpenVINO: unsupported inference_precision '%s', expected one of " + "f32, f16, bf16", + precision.c_str()); + ov_.core_free(core); + return exr::Error::InvalidArgument; + } + if (!precision.empty()) { + status = ov_.core_set_property( + core, device.c_str(), "INFERENCE_PRECISION_HINT", precision.c_str()); + if (status != OV_STATUS_OK) { + ET_LOG( + Error, + "OpenVINO: failed to set inference precision '%s' for device '%s' " + "(status=%d)", + precision.c_str(), + device.c_str(), + status); + ov_.core_free(core); + return exr::Error::InvalidArgument; } } @@ -379,9 +428,26 @@ exr::Result OpenvinoBackend::init( ov_.compiled_model_free(compiled_model); return exr::Error::MemoryAllocationFailed; } + size_t num_outputs = 0; + status = ov_.compiled_model_outputs_size(compiled_model, &num_outputs); + ov_tensor_t** owned_outputs = nullptr; + if (status == OV_STATUS_OK && num_outputs > 0) { + owned_outputs = allocator->allocateList(num_outputs); + } + if (status != OV_STATUS_OK || (num_outputs > 0 && !owned_outputs)) { + ET_LOG(Error, "OpenVINO: failed to set up outputs (status=%d)", status); + ov_.infer_request_free(infer_request); + ov_.compiled_model_free(compiled_model); + return exr::Error::Internal; + } + for (size_t i = 0; i < num_outputs; i++) { + owned_outputs[i] = nullptr; + } new (handle) ExecutionHandle; handle->compiled_model = compiled_model; handle->infer_request = infer_request; + handle->owned_outputs = owned_outputs; + handle->num_outputs = num_outputs; return handle; } @@ -469,16 +535,36 @@ exr::Error OpenvinoBackend::execute( "OpenVINO: expected tensor for output %zu", i); auto output_tensor = args[num_inputs + i]->toTensor(); - auto result = create_ov_tensor(output_tensor); - if (!result.ok()) { - return result.error(); + ov_tensor_t*& owned = execution_handle->owned_outputs[i]; + ov_tensor_t* tensor = nullptr; + if (output_tensor.shape_dynamism() == exa::TensorShapeDynamism::STATIC) { + auto result = create_ov_tensor(output_tensor); + if (!result.ok()) { + return result.error(); + } + tensor = result.get(); + } else { + // OpenVINO cannot write more than a borrowed buffer's shape holds, and + // an earlier call may have shrunk that shape. So an output whose size + // can change goes to a buffer OpenVINO owns and grows, and is copied + // out after inference. + if (!owned) { + auto result = create_owned_output( + execution_handle->compiled_model, i, output_tensor.scalar_type()); + if (!result.ok()) { + return result.error(); + } + owned = result.get(); + } + tensor = owned; } - ov_tensor_t* tensor = result.get(); status = ov_.infer_request_set_output_tensor_by_index( execution_handle->infer_request, i, tensor); // Safe to free: see shared_ptr ownership comment on input tensor above. - ov_.tensor_free(tensor); + if (tensor != owned) { + ov_.tensor_free(tensor); + } if (status != OV_STATUS_OK) { return exr::Error::Internal; } @@ -490,6 +576,27 @@ exr::Error OpenvinoBackend::execute( return exr::Error::Internal; } + for (size_t i = 0; i < num_outputs; i++) { + auto output_tensor = args[num_inputs + i]->toTensor(); + if (output_tensor.shape_dynamism() == exa::TensorShapeDynamism::STATIC) { + continue; + } + ov_tensor_t* ov_output = nullptr; + status = ov_.infer_request_get_output_tensor_by_index( + execution_handle->infer_request, i, &ov_output); + if (status != OV_STATUS_OK) { + ET_LOG( + Error, "OpenVINO: failed to get output %zu (status=%d)", i, status); + return exr::Error::Internal; + } + exr::Error error = copy_output(ov_output, output_tensor); + ov_.tensor_free(ov_output); + if (error != exr::Error::Ok) { + ET_LOG(Error, "OpenVINO: failed to copy output %zu", i); + return error; + } + } + return exr::Error::Ok; } @@ -504,6 +611,13 @@ void OpenvinoBackend::destroy(exr::DelegateHandle* handle) const { ExecutionHandle* execution_handle = static_cast(handle); + for (size_t i = 0; i < execution_handle->num_outputs; i++) { + if (execution_handle->owned_outputs[i] && ov_.tensor_free) { + ov_.tensor_free(execution_handle->owned_outputs[i]); + execution_handle->owned_outputs[i] = nullptr; + } + } + if (execution_handle->infer_request && ov_.infer_request_free) { ov_.infer_request_free(execution_handle->infer_request); execution_handle->infer_request = nullptr; @@ -515,6 +629,102 @@ void OpenvinoBackend::destroy(exr::DelegateHandle* handle) const { } } +exr::Error OpenvinoBackend::copy_output( + const ov_tensor_t* ov_output, + const exa::Tensor& output) const { + ov_shape_t shape = {}; + ov_status_e status = ov_.tensor_get_shape(ov_output, &shape); + if (status != OV_STATUS_OK) { + ET_LOG( + Error, "OpenVINO: failed to get an output shape (status=%d)", status); + return exr::Error::Internal; + } + exr::Error error = exr::resize_tensor( + output, + exa::ArrayRef(shape.dims, static_cast(shape.rank))); + ov_.shape_free(&shape); + if (error != exr::Error::Ok) { + return error; + } + size_t nbytes = 0; + void* data = nullptr; + if (ov_.tensor_get_byte_size(ov_output, &nbytes) != OV_STATUS_OK || + ov_.tensor_data(ov_output, &data) != OV_STATUS_OK) { + ET_LOG(Error, "OpenVINO: failed to read an output"); + return exr::Error::Internal; + } + ET_CHECK_OR_RETURN_ERROR( + nbytes == output.nbytes(), + Internal, + "OpenVINO: output has %zu bytes, expected %zu", + nbytes, + output.nbytes()); + if (nbytes > 0) { + std::memcpy(output.mutable_data_ptr(), data, nbytes); + } + return exr::Error::Ok; +} + +exr::Result OpenvinoBackend::create_owned_output( + const ov_compiled_model_t* compiled_model, + size_t index, + exa::ScalarType scalar_type) const { + ov_element_type_e ov_type = convert_to_openvino_type(scalar_type); + if (ov_type == OV_ELEMENT_UNDEFINED) { + return exr::Error::NotSupported; + } + // The shape comes from the compiled model, not the request: on a handle + // shared by several calls, the request's output can be an earlier call's + // buffer. + ov_output_const_port_t* port = nullptr; + ov_partial_shape_t compiled = {}; + ov_status_e status = + ov_.compiled_model_output_by_index(compiled_model, index, &port); + if (status == OV_STATUS_OK) { + status = ov_.port_get_partial_shape(port, &compiled); + ov_.output_const_port_free(port); + } + if (status != OV_STATUS_OK) { + ET_LOG( + Error, + "OpenVINO: failed to read the shape of output %zu (status=%d)", + index, + status); + return exr::Error::Internal; + } + const int64_t rank = compiled.rank.max; + if (compiled.rank.min != rank) { + ov_.partial_shape_free(&compiled); + ET_LOG( + Error, + "OpenVINO: output %zu was compiled with a dynamic rank, which " + "ExecuTorch does not support. Export the model with static sizes for " + "the inputs this output depends on.", + index); + return exr::Error::NotSupported; + } + // OpenVINO refuses any other shape for an output it compiled with a fixed + // shape, even when ExecuTorch planned it with a dynamic size. Along each + // dynamic dimension the buffer starts empty. + std::unique_ptr dims(new int64_t[rank]); + for (int64_t d = 0; d < rank; d++) { + const ov_dimension_t& dim = compiled.dims[d]; + dims[d] = dim.min == dim.max ? dim.max : 0; + } + ov_.partial_shape_free(&compiled); + ov_tensor_t* owned = nullptr; + status = ov_.tensor_create(ov_type, {rank, dims.get()}, &owned); + if (status != OV_STATUS_OK || !owned) { + ET_LOG( + Error, + "OpenVINO: failed to create a buffer for output %zu (status=%d)", + index, + status); + return exr::Error::Internal; + } + return owned; +} + exr::Result OpenvinoBackend::create_ov_tensor( const exa::Tensor& tensor) const { ov_element_type_e ov_type = convert_to_openvino_type(tensor.scalar_type()); @@ -539,12 +749,20 @@ exr::Result OpenvinoBackend::create_ov_tensor( for (int64_t d = 0; d < rank; d++) { dims[d] = sizes[d]; } - // shape is zero-initialized; shape_free is only needed after a successful - // shape_create (the zero state is safe to skip). + // shape_create refuses rank 0, so a 0-d tensor keeps the zero-initialized + // shape, which describes a scalar. ov_shape_t shape = {}; - ov_status_e status = ov_.shape_create(rank, dims, &shape); + ov_status_e status = OV_STATUS_OK; + if (rank > 0) { + status = ov_.shape_create(rank, dims, &shape); + } dims_heap.reset(); // Release heap dims (no-op if stack was used). if (status != OV_STATUS_OK) { + ET_LOG( + Error, + "OpenVINO: failed to create a shape of rank %" PRId64 " (status=%d)", + rank, + status); return exr::Error::Internal; } @@ -553,6 +771,7 @@ exr::Result OpenvinoBackend::create_ov_tensor( ov_type, shape, tensor.mutable_data_ptr(), &ov_tensor); ov_.shape_free(&shape); if (status != OV_STATUS_OK || !ov_tensor) { + ET_LOG(Error, "OpenVINO: failed to create a tensor (status=%d)", status); return exr::Error::Internal; } return ov_tensor; diff --git a/backends/openvino/runtime/OpenvinoBackend.h b/backends/openvino/runtime/OpenvinoBackend.h index 510e49ee690..22d5fd39724 100644 --- a/backends/openvino/runtime/OpenvinoBackend.h +++ b/backends/openvino/runtime/OpenvinoBackend.h @@ -28,6 +28,10 @@ namespace openvino { struct ExecutionHandle { ov_compiled_model_t* compiled_model = nullptr; ov_infer_request_t* infer_request = nullptr; + // A buffer OpenVINO owns for each output whose size can change, made on + // first use and reused by every call; null for the others. + ov_tensor_t** owned_outputs = nullptr; + size_t num_outputs = 0; }; class OpenvinoBackend final : public ::exr::BackendInterface { @@ -50,6 +54,15 @@ class OpenvinoBackend final : public ::exr::BackendInterface { bool ensure_loaded() const; ov_element_type_e convert_to_openvino_type(exa::ScalarType scalar_type) const; exr::Result create_ov_tensor(const exa::Tensor& tensor) const; + // Creates a buffer OpenVINO owns and grows for an output whose size can + // change. It starts with the shape OpenVINO compiled for that output. + exr::Result create_owned_output( + const ov_compiled_model_t* compiled_model, + size_t index, + exa::ScalarType scalar_type) const; + exr::Error copy_output( + const ov_tensor_t* ov_output, + const exa::Tensor& output) const; mutable DlHandle lib_handle_; mutable OpenvinoFunctions ov_; diff --git a/backends/openvino/tests/ops/base_openvino_op_test.py b/backends/openvino/tests/ops/base_openvino_op_test.py index f6bc928e4d4..4e20b41cbf6 100644 --- a/backends/openvino/tests/ops/base_openvino_op_test.py +++ b/backends/openvino/tests/ops/base_openvino_op_test.py @@ -25,6 +25,8 @@ def execute_layer_test( sample_inputs: tuple[torch.Tensor], expected_partitions: int = 1, assert_output_equal: bool = True, + compile_specs: tuple[CompileSpec, ...] = (), + more_inputs: tuple[tuple[torch.Tensor, ...], ...] = (), ): module = module.eval() @@ -32,7 +34,7 @@ def execute_layer_test( aten_dialect: ExportedProgram = export(module, sample_inputs) # Convert to edge dialect and lower the module to the backend with a custom partitioner - compile_spec = [CompileSpec("device", self.device.encode())] + compile_spec = [CompileSpec("device", self.device.encode()), *compile_specs] lowered_module: EdgeProgramManager = to_edge_transform_and_lower( aten_dialect, partitioner=[ @@ -59,12 +61,8 @@ def execute_layer_test( # Execute the model and compare the outputs with the reference outputs if assert_output_equal: - # Execute the module in eager mode to calculate the reference outputs - ref_output = module(*sample_inputs) - if isinstance(ref_output, torch.Tensor): - ref_output = [ - ref_output, - ] + # Later inputs run on the same loaded method, as an app would. + all_inputs = (sample_inputs, *more_inputs) # Load from a file: unlike a buffer, it really frees the delegate # data when the backend frees it. @@ -75,20 +73,28 @@ def execute_layer_test( program = Runtime.get().load_program(pte_path) method = program.load_method("forward") assert method is not None - outputs = method.execute(sample_inputs) + all_outputs = [method.execute(inputs) for inputs in all_inputs] # Windows cannot delete a mapped file, so release it first. del method, program - # Compare the outputs with the reference outputs - self.assertTrue(len(ref_output) == len(outputs)) - for i in range(len(ref_output)): - self.assertTrue( - torch.allclose( - outputs[i], - ref_output[i], - atol=self.atol, - rtol=self.rtol, - equal_nan=True, - ), - msg=f"ref_output:\n{ref_output[i]}\n\ntest_output:\n{outputs[i]}", - ) + for inputs, outputs in zip(all_inputs, all_outputs): + # Execute the module in eager mode to calculate the reference outputs + ref_output = module(*inputs) + if isinstance(ref_output, torch.Tensor): + ref_output = [ + ref_output, + ] + + # Compare the outputs with the reference outputs + self.assertTrue(len(ref_output) == len(outputs)) + for i in range(len(ref_output)): + self.assertTrue( + torch.allclose( + outputs[i], + ref_output[i], + atol=self.atol, + rtol=self.rtol, + equal_nan=True, + ), + msg=f"ref_output:\n{ref_output[i]}\n\ntest_output:\n{outputs[i]}", + ) diff --git a/backends/openvino/tests/ops/test_add.py b/backends/openvino/tests/ops/test_add.py index 5b68d0ff149..a976f7cc7fd 100644 --- a/backends/openvino/tests/ops/test_add.py +++ b/backends/openvino/tests/ops/test_add.py @@ -20,3 +20,12 @@ def test_add(self): module = self.create_model() sample_input = (torch.randn(2, 5, 1, 3), torch.randn(2, 5, 1, 3)) self.execute_layer_test(module, sample_input) + + def test_add_0d_inputs(self): + module = self.create_model() + for sample_input in ( + (torch.randn(2, 3), torch.tensor(2.5)), + (torch.tensor(1.5), torch.tensor(2.5)), + ): + with self.subTest(shapes=[list(x.shape) for x in sample_input]): + self.execute_layer_test(module, sample_input) diff --git a/backends/openvino/tests/ops/test_addmm.py b/backends/openvino/tests/ops/test_addmm.py index 51c1314db0d..00efb18ae82 100644 --- a/backends/openvino/tests/ops/test_addmm.py +++ b/backends/openvino/tests/ops/test_addmm.py @@ -1,7 +1,15 @@ +# mypy: disable-error-code=import-not-found + + +from unittest import mock + import torch +from executorch.backends.openvino import preprocess from executorch.backends.openvino.tests.ops.base_openvino_op_test import ( BaseOpenvinoOpTest, ) +from executorch.exir.backend.backend_details import CompileSpec +from openvino import Core, Type # type: ignore[import-untyped] class TestAddMMOperator(BaseOpenvinoOpTest): @@ -26,3 +34,67 @@ def test_addmm(self): input_z = torch.randn(4, 4, dtype=torch.float32) sample_input = (input_x, input_y, input_z) self.execute_layer_test(module, sample_input) + + def test_addmm_computes_in_float32(self): + if not self.device.startswith("CPU"): + self.skipTest("only CPUs compute in float32 by default") + if Core().get_property("CPU", "INFERENCE_PRECISION_HINT") == Type.f32: + self.skipTest("OpenVINO already computes in float32 on this CPU") + # OpenVINO would otherwise compute this in bfloat16 on x86 CPUs with + # bfloat16 units, or in float16 on ARM CPUs, far outside 1e-4. + self.atol = self.rtol = 1e-4 + sample_input = tuple(torch.randn(256, 256) for _ in range(3)) + self.execute_layer_test(self.create_model(), sample_input) + # On x86 OpenVINO applies only the precision it gets at load, so this + # fails if the runtime drops an explicit one. + bf16 = CompileSpec("inference_precision", b"bf16") + with self.assertRaisesRegex(AssertionError, "ref_output"): + self.execute_layer_test( + self.create_model(), sample_input, compile_specs=(bf16,) + ) + # The last value wins, as in lowering, and an empty one is the default. + self.execute_layer_test( + self.create_model(), + sample_input, + compile_specs=(bf16, CompileSpec("inference_precision", b"")), + ) + + def test_addmm_takes_the_last_inference_precision(self): + # Lowering keeps only the last value. The runtime must too, or it fails + # to load this invalid first one, on any device. + compile_specs = tuple( + CompileSpec("inference_precision", value) for value in (b"f7", b"") + ) + sample_input = tuple(torch.randn(4, 4) for _ in range(3)) + self.execute_layer_test( + self.create_model(), sample_input, compile_specs=compile_specs + ) + + def test_addmm_passes_the_precision_to_openvino(self): + if not self.device.startswith("CPU"): + self.skipTest("only CPUs compute in float32 by default") + # Checks the lowering half too: on x86 the runtime half alone decides. + self.atol = self.rtol = 0.05 + sample_input = tuple(torch.randn(4, 4) for _ in range(3)) + bf16 = (CompileSpec("inference_precision", b"bf16"),) + for compile_specs, hint in (((), "f32"), (bf16, "bf16")): + with self.subTest(hint=hint), mock.patch.object( + preprocess, "compile_model", wraps=preprocess.compile_model + ) as compile_model: + self.execute_layer_test( + self.create_model(), sample_input, compile_specs=compile_specs + ) + self.assertEqual( + compile_model.call_args.args[2], {"INFERENCE_PRECISION_HINT": hint} + ) + + def test_addmm_refuses_an_unknown_inference_precision_before_converting(self): + sample_input = tuple(torch.randn(4, 4) for _ in range(3)) + with mock.patch.object( + preprocess, "FrontEndManager", side_effect=AssertionError("converted") + ), self.assertRaisesRegex(ValueError, "unsupported inference_precision 'f7'"): + self.execute_layer_test( + self.create_model(), + sample_input, + compile_specs=(CompileSpec("inference_precision", b"f7"),), + ) diff --git a/backends/openvino/tests/ops/test_reductions.py b/backends/openvino/tests/ops/test_reductions.py new file mode 100644 index 00000000000..7b202eac8e6 --- /dev/null +++ b/backends/openvino/tests/ops/test_reductions.py @@ -0,0 +1,300 @@ +# mypy: disable-error-code=import-not-found + +import itertools +import unittest +from unittest import mock + +import torch +from executorch.backends.openvino import preprocess +from executorch.backends.openvino._passes import ( + fold_constant_cumsum_pass, + FoldConstantCumsumPass, +) +from executorch.backends.openvino.tests.ops.base_openvino_op_test import ( + BaseOpenvinoOpTest, +) +from executorch.exir import to_edge +from executorch.exir.dialects._ops import ops as exir_ops +from openvino import Model, opset13, Type # type: ignore[import-untyped] +from openvino.frontend.pytorch.torchdynamo.op_support import ( # type: ignore[import-untyped] + OperatorSupport, +) + + +class TestIntegerReductions(BaseOpenvinoOpTest): + """PyTorch reduces bool and small integer tensors to int64.""" + + def test_cumsum(self): + class CumSum(torch.nn.Module): + def forward(self, x): + return torch.cumsum(x, 1) + + for x in ( + torch.rand(2, 7) > 0.5, + torch.randint(0, 5, (2, 7), dtype=torch.int32), + ): + with self.subTest(dtype=x.dtype): + self.execute_layer_test(CumSum(), (x,)) + + def test_sum(self): + class Sum(torch.nn.Module): + def forward(self, x): + return x.sum(1) + + # More than 255 true values in a row, which 8 bits cannot count. + mask = torch.arange(800).reshape(2, 400) % 4 != 0 + for x in (mask, torch.randint(0, 5, (2, 7), dtype=torch.int32)): + with self.subTest(dtype=x.dtype): + self.execute_layer_test(Sum(), (x,)) + + def test_prod(self): + class Prod(torch.nn.Module): + def __init__(self, dim): + super().__init__() + self.dim = dim + + def forward(self, x): + return x.prod() if self.dim is None else x.prod(self.dim) + + support = OperatorSupport([])._support_dict + x = torch.randint(1, 4, (2, 5), dtype=torch.int32) + for dim, op in ((1, "prod.dim_int"), (None, "prod.default")): + with self.subTest(op=op): + if f"torch.ops.aten.{op}" not in support: + self.skipTest(f"this OpenVINO release does not delegate {op}") + self.execute_layer_test(Prod(dim), (x,)) + + def test_attention_masks_with_more_valid_tokens(self): + class AttentionMasks(torch.nn.Module): + """Masks and positions from padding, as pi0 builds them.""" + + def __init__(self, dtype, from_input): + super().__init__() + self.dtype = dtype + self.from_input = from_input + + def forward(self, pad_masks): + if self.from_input: + # Constant too, but only once OpenVINO folds the shape. + att_masks = torch.ones_like(pad_masks[0]) + else: + att_masks = torch.tensor([0] * 765 + [1] * 51, dtype=torch.bool) + cumsum = torch.cumsum(att_masks[None, :], dim=1, dtype=self.dtype) + att_2d = cumsum[:, None, :] <= cumsum[:, :, None] + pad_2d = pad_masks[:, None, :] * pad_masks[:, :, None] + return att_2d & pad_2d, torch.cumsum(pad_masks, dim=1) - 1 + + def pad_masks(valid): + return (torch.arange(816) < valid)[None, :] + + for dtype, from_input in itertools.product( + (torch.int64, torch.int32), (False, True) + ): + with self.subTest(dtype=dtype, from_input=from_input): + # One loaded program, run again with more valid tokens. + self.execute_layer_test( + AttentionMasks(dtype, from_input), + (pad_masks(300),), + more_inputs=((pad_masks(700),),), + ) + + def test_cumsum_over_ones_like_of_inputs_read_nowhere_else(self): + class Positions(torch.nn.Module): + def forward(self, input_ids, offset, mask): + positions = torch.cumsum(torch.ones_like(input_ids), 1) - 1 + return positions + offset * torch.cumsum(torch.ones_like(mask), 1) + + # Once the cumsums are folded, nothing in the partition reads input_ids + # or mask: the first input and one after it. + input_ids = torch.zeros(1, 8, dtype=torch.int64) + mask = torch.zeros(1, 8, dtype=torch.int32) + self.execute_layer_test( + Positions(), + (input_ids, torch.arange(8)[None], mask), + ) + + + +class TestMatchPartitionInputs(unittest.TestCase): + def test_only_an_unused_first_input_is_put_back(self): + class Add(torch.nn.Module): + def forward(self, x, y): + return x + y + + class Second(torch.nn.Module): + def forward(self, x, y): + return y * 2 + + def converted_without_the_first_input(): + y = opset13.parameter([3], Type.f32) + return Model([opset13.relu(y)], [y]) + + shapes, types = [[1, 3], [3]], [torch.int64, torch.float32] + model = converted_without_the_first_input() + preprocess._match_partition_inputs( + model, torch.fx.symbolic_trace(Second()), shapes, types + ) + # The runtime binds inputs by position. + self.assertEqual( + [ + (list(p.get_partial_shape().to_shape()), p.get_element_type()) + for p in model.get_parameters() + ], + [([1, 3], Type.i64), ([3], Type.f32)], + ) + # Add reads its first input, so losing it fails lowering. + with self.assertRaisesRegex(RuntimeError, "expected 2 parameters"): + preprocess._match_partition_inputs( + converted_without_the_first_input(), + torch.fx.symbolic_trace(Add()), + shapes, + types, + ) + + +class TestFoldConstantCumsumPass(unittest.TestCase): + def fold(self, module, inputs): + """Edge program module of module, with constant cumsums folded.""" + edge = to_edge(torch.export.export(module, inputs)) + folded = edge.exported_program().module() + FoldConstantCumsumPass()(folded) + return folded + + def test_buffer_written_by_the_graph_is_not_folded(self): + class State(torch.nn.Module): + def __init__(self): + super().__init__() + self.register_buffer("s", torch.arange(3)) + + def forward(self, x): + self.s.add_(1) + return torch.cumsum(self.s, 0) + x + + x = torch.zeros(3, dtype=torch.int64) + folded = self.fold(State(), (x,)) + eager = State() + for _ in range(2): + torch.testing.assert_close(folded(x), eager(x)) + + def test_folded_name_does_not_replace_a_buffer(self): + class Model(torch.nn.Module): + def __init__(self): + super().__init__() + self.register_buffer( + "_folded_cumsum_aten_cumsum_default", torch.ones(3) + ) + + def forward(self, x): + counts = torch.cumsum(torch.ones(3, dtype=torch.int64), 0) + return counts + x + self._folded_cumsum_aten_cumsum_default + + x = torch.zeros(3) + folded = self.fold(Model(), (x,)) + # The folded cumsum would have taken this name. + self.assertTrue(hasattr(folded, "_folded_cumsum_aten_cumsum_default_")) + torch.testing.assert_close(folded(x), Model()(x)) + + def test_a_cumsum_that_cannot_be_evaluated_leaves_the_others_folded(self): + class Model(torch.nn.Module): + def forward(self, x): + ones = torch.ones(3, dtype=torch.int64) + return torch.cumsum(ones, 0) + torch.cumsum(ones * 2, 0) + x + + evaluate = fold_constant_cumsum_pass._evaluate + + def fail_on_the_doubled_one(graph_module, target): + if "mul" in str(target.args[0].target): + raise RuntimeError("not on the CPU") + return evaluate(graph_module, target) + + x = torch.zeros(3, dtype=torch.int64) + with mock.patch.object( + fold_constant_cumsum_pass, "_evaluate", side_effect=fail_on_the_doubled_one + ), self.assertLogs(level="WARNING") as logs: + folded = self.fold(Model(), (x,)) + targets = [n.target for n in folded.graph.nodes] + self.assertEqual(targets.count(exir_ops.edge.aten.cumsum.default), 1) + self.assertIn("wrong results", logs.output[0]) + torch.testing.assert_close(folded(x), Model()(x)) + + def test_float_cumsum_is_not_folded(self): + class Model(torch.nn.Module): + def forward(self, x): + return torch.cumsum(torch.ones(3), 0) + x + + folded = self.fold(Model(), (torch.zeros(3),)) + self.assertIn( + exir_ops.edge.aten.cumsum.default, + [n.target for n in folded.graph.nodes], + ) + + def test_random_draws_are_not_folded(self): + class Model(torch.nn.Module): + def forward(self, x): + c = torch.ones(3) + r = torch.cond( + torch.tensor(True), + lambda c: (torch.rand(3) * 100 + c).long(), + lambda c: c.long(), + (c,), + ) + direct = (torch.rand(3) * 100).long() + return torch.cumsum(r, 0) + torch.cumsum(direct, 0) + x + + folded = self.fold(Model(), (torch.zeros(3, dtype=torch.int64),)) + targets = [n.target for n in folded.graph.nodes] + self.assertEqual(targets.count(exir_ops.edge.aten.cumsum.default), 2) + + def test_cumsum_over_a_split_constant_is_folded(self): + class Model(torch.nn.Module): + def __init__(self, dtype): + super().__init__() + self.dtype = dtype + + def forward(self, x): + part = torch.ones(6, dtype=torch.int64).split(3)[1] + return torch.cumsum(part, 0, dtype=self.dtype) + x + + for dtype in (torch.int64, torch.int32): + with self.subTest(dtype=dtype): + # Of the same dtype, so the result keeps the folded dtype. + x = torch.zeros(3, dtype=dtype) + folded = self.fold(Model(dtype), (x,)) + self.assertNotIn( + exir_ops.edge.aten.cumsum.default, + [n.target for n in folded.graph.nodes], + ) + torch.testing.assert_close(folded(x), Model(dtype)(x)) + + def test_constants_are_folded_without_a_gpu(self): + class Model(torch.nn.Module): + def forward(self, x): + ones = torch.ones(3, dtype=torch.int64, device="cuda") + pinned = torch.ones(3, dtype=torch.int64, pin_memory=True) + return torch.cumsum(ones, 0).cpu() + torch.cumsum(pinned, 0) + x + + x = torch.zeros(3, dtype=torch.int64) + folded = self.fold(Model(), (x,)) + self.assertNotIn( + exir_ops.edge.aten.cumsum.default, + [n.target for n in folded.graph.nodes], + ) + torch.testing.assert_close(folded(x), torch.tensor([2, 4, 6])) + + def test_full_like_is_folded_under_another_default_device(self): + class Model(torch.nn.Module): + def forward(self, x): + return torch.cumsum(torch.ones_like(x), 0) + x + + x = torch.zeros(3, dtype=torch.int64) + folded = to_edge(torch.export.export(Model(), (x,))).exported_program().module() + # Without the fix, the meta device keeps the cumsum unfolded on every + # build, because a meta tensor has no data to copy to the CPU. CUDA + # would only show it on a build without CUDA. + with torch.device("meta"): + FoldConstantCumsumPass()(folded) + self.assertNotIn( + exir_ops.edge.aten.cumsum.default, + [n.target for n in folded.graph.nodes], + ) + torch.testing.assert_close(folded(x), torch.tensor([1, 2, 3])) diff --git a/backends/openvino/tests/ops/test_view.py b/backends/openvino/tests/ops/test_view.py index 5ea2f8cb1dc..2cefdc4206b 100644 --- a/backends/openvino/tests/ops/test_view.py +++ b/backends/openvino/tests/ops/test_view.py @@ -1,7 +1,16 @@ +import os +import tempfile + import torch +from executorch.backends.openvino.preprocess import OpenvinoBackend from executorch.backends.openvino.tests.ops.base_openvino_op_test import ( BaseOpenvinoOpTest, ) +from executorch.exir import to_edge +from executorch.exir.backend.backend_api import to_backend +from executorch.exir.backend.backend_details import CompileSpec +from executorch.runtime import Runtime +from torch.export import Dim, export op_params = [ {"input_shape": [2, 3, 2], "target_shape": [2, 6]}, @@ -36,11 +45,86 @@ def test_view(self): def test_view_data_dependent_size(self): class View(torch.nn.Module): + def __init__(self, reduce): + super().__init__() + self.reduce = reduce + def forward(self, x, n): n = n.item() torch._check(n >= 1) torch._check(n <= x.shape[0]) - return x[:n].view(n, -1).sum(0) + y = x[:n].view(n, -1) + return y.sum(0) if self.reduce else y + + # The size reaches the OpenVINO partition as a SymInt input. The second + # call needs a larger output than the first one returned, unless the + # sum makes the output static. + x = torch.randn(4, 8) + for reduce in (False, True): + with self.subTest(reduce=reduce): + self.execute_layer_test( + View(reduce), + (x, torch.tensor(3)), + more_inputs=((x, torch.tensor(4)),), + ) + + def test_dynamic_size_output_that_openvino_makes_static(self): + class Select(torch.nn.Module): + def forward(self, x): + return x[torch.ones(4, dtype=torch.bool)] + + # ExecuTorch plans the output with a dynamic size, but OpenVINO folds + # the mask and compiles a fixed output shape. + self.execute_layer_test(Select(), (torch.randn(4, 8),)) + + def test_one_delegate_called_with_different_output_sizes(self): + class Double(torch.nn.Module): + def forward(self, x): + return x * 2 + + x, y = torch.randn(2, 8), torch.randn(4, 8) + double = to_backend( + OpenvinoBackend.__name__, + to_edge( + export(Double(), (y,), dynamic_shapes=({0: Dim("n", min=1, max=4)},)) + ).exported_program(), + [CompileSpec("device", self.device.encode())], + ) + # Programs from ExecuTorch 0.7 and earlier run every call of a lowered + # module on one delegate handle, each call with its own outputs. + double._share_handle_across_calls = True + + class TwoCalls(torch.nn.Module): + def __init__(self): + super().__init__() + self.lowered = double + + def forward(self, x, y): + return self.lowered(x), self.lowered(y) + + rows_y = {0: Dim("rows_y", min=1, max=4)} + # In the second program the first call's output is static, so that call + # binds its own buffer to the output the dynamic call shares. + for rows_x, later in ( + ({0: Dim("rows_x", min=1, max=2)}, (y[:1], y[:3])), + (None, (x + 1, y[:1])), + ): + with self.subTest(static_first=rows_x is None): + program = to_edge( + export(TwoCalls(), (x, y), dynamic_shapes=(rows_x, rows_y)) + ).to_executorch() + plan = program.executorch_program.execution_plan[0] + self.assertEqual(len(plan.delegates), 1) - # The size reaches the OpenVINO partition as a SymInt input. - self.execute_layer_test(View(), (torch.randn(4, 8), torch.tensor(3))) + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp: + pte_path = os.path.join(tmp, "model.pte") + with open(pte_path, "wb") as f: + program.write_to_file(f) + loaded = Runtime.get().load_program(pte_path) + method = loaded.load_method("forward") + assert method is not None + for inputs in ((x, y), later): + outputs = method.execute(inputs) + for output, value in zip(outputs, inputs): + torch.testing.assert_close(output, value * 2) + del method, loaded diff --git a/backends/openvino/tests/test_runner.py b/backends/openvino/tests/test_runner.py index 7d8c6b968c2..0c839e0c347 100644 --- a/backends/openvino/tests/test_runner.py +++ b/backends/openvino/tests/test_runner.py @@ -1,4 +1,5 @@ import argparse +import sys import unittest @@ -71,3 +72,4 @@ def parse_arguments(): print("OpenVINO backend tests completed successfully") else: print("OpenVINO backend tests completed with failures") + sys.exit(1) diff --git a/docs/source/build-run-openvino.md b/docs/source/build-run-openvino.md index 69a73672ac1..2d842ed2bcd 100644 --- a/docs/source/build-run-openvino.md +++ b/docs/source/build-run-openvino.md @@ -102,6 +102,63 @@ Follow the steps below to setup your build environment: ./openvino_build.sh ``` +## Precision + +When the device is `CPU`, a program computes in float32 by default. Without this default, +OpenVINO would compute in bfloat16 on x86 CPUs with bfloat16 units, which keeps only two to +three significant digits, and in float16 on ARM CPUs, which keeps about three. +Both are faster. To trade accuracy for speed, ask for the faster precision when +lowering: + +```python +from executorch.backends.openvino.partitioner import OpenvinoPartitioner +from executorch.exir.backend.compile_spec_schema import CompileSpec + +OpenvinoPartitioner( + [CompileSpec("device", b"CPU"), CompileSpec("inference_precision", b"bf16")] +) +``` + +On an ARM CPU, ask for `f16` instead: there float32 is about 1.8x slower than +float16, as measured on an Orin Nano. + +The value is OpenVINO's `INFERENCE_PRECISION_HINT`: `f32`, `f16` or `bf16`. An +empty value means the default, and any other value fails at lowering. A CPU +without float16 support computes an `f16` program in float32. OpenVINO has no +bfloat16 on ARM CPUs, so there `bf16` keeps the ARM default, float16 where the +CPU supports it. Which x86 CPUs compute in bfloat16 is up to OpenVINO; see its +[Precision Control](https://docs.openvino.ai/2026/openvino-workflow/running-inference/optimize-inference/precision-control.html) +page. Other devices, such as GPUs, keep OpenVINO's default unless the option is +set. + +The default also applies to a program exported in float16: on a CPU it computes +in float32 unless you lower it with `inference_precision`. + +On x86 CPUs, OpenVINO applies the precision when the runtime loads a program. +So a program lowered without `inference_precision` computes in float32, even +one exported with an ExecuTorch release that did not have this default. On an +x86 CPU with bfloat16 units, float32 is slower and can use more memory; export it +again with `inference_precision` set to `bf16` to get the earlier speed. Other +x86 CPUs already computed such a program in float32. On ARM CPUs, OpenVINO applies the +precision when the program is lowered, so a program exported with such an +earlier release keeps computing in float16 until it is exported again. + +OpenVINO's CPU plugin computes 64-bit integers in 32 bits. An integer result +outside the int32 range wraps around without an error. This applies to every +int64 tensor, including the int64 result of `cumsum` and `prod` over smaller +integer types. In OpenVINO 2026.4.1, the CPU plugin also computes some int32 and +int64 ops in float32. On x86 CPUs these are `sum`, `amax`, `amin`, `max` and +`min`. On ARM CPUs they are element-wise ops such as `add`, `maximum` and +`clamp`, and the same reductions along any dimension but the last. Once a value +or a running total is above 16,777,216 (2^24) or below -16,777,216, such a +result can be off, even inside the int32 range. For example, on x86 `sum` over a +tensor that holds 16,777,216 and 1 gives 16,777,216, and so does `amax` over one +that holds 16,777,217. + +In OpenVINO 2026.4.1, the CPU plugin's `min(dim)` and `max(dim)` can return a +later index on ties than PyTorch, which returns the first one. This happens with +float32 as well as integer dtypes such as int32 and uint8. + ## Build Instructions for Examples ### AOT step: