diff --git a/backends/openvino/runtime/OpenvinoApi.h b/backends/openvino/runtime/OpenvinoApi.h index e4c3e7f737a..c9f8f1dd2e4 100644 --- a/backends/openvino/runtime/OpenvinoApi.h +++ b/backends/openvino/runtime/OpenvinoApi.h @@ -76,6 +76,8 @@ using ov_core_free_fn = void (*)(ov_core_t*); using ov_core_get_available_devices_fn = ov_status_e (*)(const ov_core_t*, ov_available_devices_t*); using ov_available_devices_free_fn = void (*)(ov_available_devices_t*); +using ov_core_set_property_fn = + ov_status_e (*)(const ov_core_t*, const char*, ...); using ov_core_import_model_fn = ov_status_e (*)( const ov_core_t*, const char*, @@ -115,6 +117,7 @@ struct OpenvinoFunctions { ov_core_free_fn core_free = nullptr; ov_core_get_available_devices_fn core_get_available_devices = nullptr; ov_available_devices_free_fn available_devices_free = nullptr; + ov_core_set_property_fn core_set_property = nullptr; ov_core_import_model_fn core_import_model = nullptr; ov_compiled_model_create_infer_request_fn compiled_model_create_infer_request = nullptr; diff --git a/backends/openvino/runtime/OpenvinoBackend.cpp b/backends/openvino/runtime/OpenvinoBackend.cpp index fad6ef91cb1..3a776332faf 100644 --- a/backends/openvino/runtime/OpenvinoBackend.cpp +++ b/backends/openvino/runtime/OpenvinoBackend.cpp @@ -236,6 +236,7 @@ bool OpenvinoBackend::ensure_loaded() const { LOAD_SYM(core_free, ov_core_free) LOAD_SYM(core_get_available_devices, ov_core_get_available_devices) LOAD_SYM(available_devices_free, ov_available_devices_free) + LOAD_SYM(core_set_property, ov_core_set_property) LOAD_SYM(core_import_model, ov_core_import_model) LOAD_SYM( compiled_model_create_infer_request, @@ -321,17 +322,44 @@ exr::Result OpenvinoBackend::init( size_t data_size = processed->size(); std::string device = "CPU"; + std::string inference_precision; for (auto& compile_spec : compile_specs) { + std::string* target = nullptr; if (std::strcmp(compile_spec.key, "device") == 0) { - const char* buf = static_cast(compile_spec.value.buffer); - size_t len = compile_spec.value.nbytes; - // Strip trailing null bytes that may be included in nbytes. - while (len > 0 && buf[len - 1] == '\0') { - --len; - } - if (len > 0) { - device.assign(buf, len); - } + target = &device; + } else if (std::strcmp(compile_spec.key, "inference_precision") == 0) { + target = &inference_precision; + } else { + continue; + } + const char* buf = static_cast(compile_spec.value.buffer); + size_t len = compile_spec.value.nbytes; + // Strip trailing null bytes that may be included in nbytes. + while (len > 0 && buf[len - 1] == '\0') { + --len; + } + if (len > 0) { + target->assign(buf, len); + } + } + + // The device picks its default precision when the model is imported, e.g. + // bf16 for f32 models on CPUs with native bf16 support. + if (!inference_precision.empty()) { + status = ov_.core_set_property( + core, + device.c_str(), + "INFERENCE_PRECISION_HINT", + inference_precision.c_str()); + if (status != OV_STATUS_OK) { + ET_LOG( + Error, + "OpenVINO: failed to set inference precision '%s' for device '%s' (status=%d)", + inference_precision.c_str(), + device.c_str(), + status); + ov_.core_free(core); + return exr::Error::InvalidArgument; } } @@ -343,8 +371,10 @@ exr::Result OpenvinoBackend::init( if (status != OV_STATUS_OK || !compiled_model) { ET_LOG( Error, - "OpenVINO: failed to import model for device '%s' (status=%d)", + "OpenVINO: failed to import model for device '%s'%s%s (status=%d)", device.c_str(), + inference_precision.empty() ? "" : " with inference_precision ", + inference_precision.c_str(), status); return exr::Error::Internal; } @@ -541,8 +571,12 @@ exr::Result OpenvinoBackend::create_ov_tensor( } // shape is zero-initialized; shape_free is only needed after a successful // shape_create (the zero state is safe to skip). + // ov_shape_create rejects rank 0, so 0-d tensors use the zero shape as-is. ov_shape_t shape = {}; - ov_status_e status = ov_.shape_create(rank, dims, &shape); + ov_status_e status = OV_STATUS_OK; + if (rank > 0) { + status = ov_.shape_create(rank, dims, &shape); + } dims_heap.reset(); // Release heap dims (no-op if stack was used). if (status != OV_STATUS_OK) { return exr::Error::Internal; @@ -551,7 +585,9 @@ exr::Result OpenvinoBackend::create_ov_tensor( ov_tensor_t* ov_tensor = nullptr; status = ov_.tensor_create_from_host_ptr( ov_type, shape, tensor.mutable_data_ptr(), &ov_tensor); - ov_.shape_free(&shape); + if (rank > 0) { + ov_.shape_free(&shape); + } if (status != OV_STATUS_OK || !ov_tensor) { return exr::Error::Internal; } diff --git a/backends/openvino/tests/ops/base_openvino_op_test.py b/backends/openvino/tests/ops/base_openvino_op_test.py index f6bc928e4d4..1fd2929f3d3 100644 --- a/backends/openvino/tests/ops/base_openvino_op_test.py +++ b/backends/openvino/tests/ops/base_openvino_op_test.py @@ -33,6 +33,9 @@ def execute_layer_test( # Convert to edge dialect and lower the module to the backend with a custom partitioner compile_spec = [CompileSpec("device", self.device.encode())] + # CPUs with native bf16 otherwise run f32 models in bf16, which misses atol. + if self.device == "CPU": + compile_spec.append(CompileSpec("inference_precision", b"f32")) lowered_module: EdgeProgramManager = to_edge_transform_and_lower( aten_dialect, partitioner=[ diff --git a/backends/openvino/tests/test_runner.py b/backends/openvino/tests/test_runner.py index 7d8c6b968c2..0c839e0c347 100644 --- a/backends/openvino/tests/test_runner.py +++ b/backends/openvino/tests/test_runner.py @@ -1,4 +1,5 @@ import argparse +import sys import unittest @@ -71,3 +72,4 @@ def parse_arguments(): print("OpenVINO backend tests completed successfully") else: print("OpenVINO backend tests completed with failures") + sys.exit(1)