From b70ec42c13222701afefeb41ecca306037254df7 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Thu, 7 May 2026 18:25:04 +0200 Subject: [PATCH 01/12] CVS-169692_enable_on_commit_tests_part_I --- tests/functional/config.py | 305 +++++++- tests/functional/config_template.json | 77 -- .../constants/{constants.py => components.py} | 9 +- tests/functional/constants/core.py | 21 + tests/functional/constants/custom_loader.py | 44 ++ tests/functional/constants/generative_ai.py | 30 + tests/functional/constants/metrics.py | 707 ++++++++++++++++++ tests/functional/constants/os_type.py | 60 ++ 8 files changed, 1160 insertions(+), 93 deletions(-) delete mode 100644 tests/functional/config_template.json rename tests/functional/constants/{constants.py => components.py} (74%) create mode 100644 tests/functional/constants/core.py create mode 100644 tests/functional/constants/custom_loader.py create mode 100644 tests/functional/constants/generative_ai.py create mode 100644 tests/functional/constants/metrics.py create mode 100644 tests/functional/constants/os_type.py diff --git a/tests/functional/config.py b/tests/functional/config.py index 88ece20ae7..65dfe2cc9a 100644 --- a/tests/functional/config.py +++ b/tests/functional/config.py @@ -15,10 +15,23 @@ # import os +import re +from pathlib import Path +from tests.functional.constants.os_type import OsType +from tests.functional.constants.ovms_type import OvmsType from tests.functional.constants.target_device import TargetDevice -from tests.functional.utils.helpers import get_bool, get_int, get_path, get_target_devices -from tests.functional.utils.parametrization import generate_test_object_name +from tests.functional.utils.core import TmpDir +from tests.functional.utils.helpers import ( + generate_test_object_name, + get_bool, + get_int, + get_list, + get_path, + get_target_devices, + validate_supported_values, +) + try: # In user_config.py, user might export custom environment variables @@ -26,6 +39,27 @@ except ImportError: pass + +def get_uses_mapping(): + _uses_mapping = get_list("TT_USES_MAPPING", fallback=[None]) + _uses_mapping = list(set([str(x).upper() for x in _uses_mapping])) # make upper & remove duplicates + # Reduce to True/False/None + _uses_mapping = [x == "TRUE" if x in ["TRUE", "FALSE"] else None for x in _uses_mapping] + validate_supported_values(_uses_mapping, [True, False, None]) + return _uses_mapping + + +""" + TT_USES_MAPPING - use mapping JSON for model inputs/output name aliasing + Possible TT_USES_MAPPING values (case insensitive): + - (empty)/""/NONE - Default leave mapping.json provided alongside model untouched (if exists). + - FALSE - forcibly remove mapping.json if provided with model. + - TRUE - remove any previous mapping and add generic mapping.json + (see: ovms/object_model/ovms_mapping_config.py for details) + - TRUE,FALSE,NONE - Iterate each test case from listed values in single test session. +""" +uses_mapping = get_uses_mapping() + """TEST_DIR - location where models and test data should be copied from TEST_DIR_CACHE and deleted after tests""" test_dir = os.environ.get("TEST_DIR", "/tmp/{}".format(generate_test_object_name(prefix='ovms_models'))) @@ -37,7 +71,54 @@ test_dir_cleanup = test_dir_cleanup.lower() == "true" """BUILD_LOGS - path to dir where artifacts should be stored""" -artifacts_dir = os.environ.get("BUILD_LOGS", "") +artifacts_dir = get_path("BUILD_LOGS", os.path.join("~", "ovms_test_logs")) + +""" TT_NGNIX_CERTS_DIR - Custom nodes directory path""" +nginx_certs_dir = get_path("TT_NGINX_CERTS_DIR", os.path.join("~", "ovms_nginx_certs")) + +""" TT_MODELS_PATH - Models local repo path""" +models_path = get_path("TT_MODELS_PATH", os.path.join("~", "ovms_models")) + +""" TT_DATASETS_PATH - Datasets local repo path""" +datasets_path = get_path("TT_DATASETS_PATH", os.path.join("~", "ovms_datasets")) + +""" TT_CLEAN_ARTIFACTS_DIR """ +clean_artifacts_dir = get_bool("TT_CLEAN_ARTIFACTS_DIR", False) + +""" TT_LANGUAGE_MODELS_ENABLED - model UniversalSentenceEncoder added to various models """ +language_models_enabled = get_bool("TT_LANGUAGE_MODELS_ENABLED", True) + +""" MEDIAPIE_DISABLE - if OVMS image has mediapipe feature """ +mediapipe_disable = bool(get_int("MEDIAPIPE_DISABLE", 0)) + +""" PYTHON_DISABLE - if OVMS image has python feature """ +python_disable = bool(get_int("PYTHON_DISABLE", 0)) + +""" TT_WIN_PY_VERSION - Python version for virtualenv on Windows OS """ +windows_python_version = os.environ.get("TT_WIN_PY_VERSION", "3.12") + +""" TT_DOCKER_REGISTRY - Docker registry""" +docker_registry = os.environ.get("TT_DOCKER_REGISTRY", None) + +""" OVMS_CPP_DOCKER_IMAGE """ +ovms_cpp_docker_image = os.environ.get("OVMS_CPP_DOCKER_IMAGE", None) + +""" TT_OVMS_IMAGE_NAME """ +ovms_image = os.environ.get("TT_OVMS_IMAGE_NAME", None) + +""" OVMS_CPP_IMAGE_TAG - tag of OVMS image to test (compatible with build parameter) """ +ovms_image_tag = os.environ.get("OVMS_CPP_IMAGE_TAG", None) + +""" TT_OVMS_TEST_IMAGE_NAME - image name for cpu extensions and custom nodes """ +ovms_test_image_name = os.environ.get("TT_OVMS_TEST_IMAGE_NAME", None) + +""" TT_FORCE_USE_OVMS_IMAGE - force to use TT_OVMS_IMAGE_NAME """ +force_use_ovms_image = get_bool("TT_FORCE_USE_OVMS_IMAGE", False) + +""" TT_OVMS_C_RELEASE_ARTIFACTS_PATH - path to current release artifacts """ +# multiple local and remote location supported +# example: TT_OVMS_C_RELEASE_ARTIFACTS_PATH="../ubuntu24/ovms.tar.gz,../redat/ovms.tar.gz" +ovms_c_release_artifacts_path = get_list("TT_OVMS_C_RELEASE_ARTIFACTS_PATH") """START_CONTAINER_COMMAND - command to start ovms container""" start_container_command = os.environ.get("START_CONTAINER_COMMAND", "") @@ -58,12 +139,10 @@ path_to_mount_cache = os.path.join(test_dir_cache, "saved_models") -models_path = path_to_mount if ovms_binary_path else "/opt/ml" - """TT_MINIO_IMAGE_NAME - Docker image for Minio""" minio_image = os.environ.get("TT_MINIO_IMAGE_NAME", "minio/minio:latest") -""" TT_TARGET_DEVICE - list of devices separated by a comma "CPU,GPU" """ +""" TT_TARGET_DEVICE - list of devices separated by a comma "CPU,GPU,NPU" """ target_devices = get_target_devices() target_device = target_devices[0] @@ -78,14 +157,16 @@ container_minio_log_line = "Console endpoint is listening on a dynamic port" +# Reservation manager values, for details study tests.functional.utils.reservation_manager """ TT_GRPC_OVMS_STARTING_PORT - Grpc port where ovms should be exposed""" -grpc_ovms_starting_port = get_int("TT_GRPC_OVMS_STARTING_PORT", 9001) +grpc_ovms_starting_port = get_int("TT_GRPC_OVMS_STARTING_PORT", None) """ TT_REST_OVMS_STARTING_PORT - Rest port where ovms should be exposed""" -rest_ovms_starting_port = get_int("TT_REST_OVMS_STARTING_PORT", 18001) +rest_ovms_starting_port = get_int("TT_REST_OVMS_STARTING_PORT", None) """ TT_PORTS_POOL_SIZE- Ports pool size""" -ports_pool_size = get_int("TT_PORTS_POOL_SIZE", 5000) +ports_pool_size = get_int("TT_PORTS_POOL_SIZE", None) +# NOTE: Above values will be validated and could be changed if invalid """ TT_CONVERTED_MODELS_EXPIRE_TIME - Time after converted models are not up-to-date and needs to be refreshed(s) """ converted_models_expire_time = get_int("TT_CONVERTED_MODELS_EXPIRE_TIME", 7*24*3600) # Set default to one week @@ -110,8 +191,8 @@ } infer_timeout = infer_timeouts[target_device] -""" TT_IS_NGINX_MTLS - Specify if given image is OVSA nginx mtls image. If not specified, detect from image name""" -is_nginx_mtls = get_bool("TT_IS_NGINX_MTLS", "nginx-mtls" in image) +""" TT_IS_NGINX_MTLS - Specify if given image is OVSA nginx mtls image. """ +is_nginx_mtls = get_bool("TT_IS_NGINX_MTLS", False) """ TT_SKIP_TEST_IF_IS_NGINX_MTLS """ skip_nginx_test = get_bool("TT_SKIP_TEST_IF_IS_NGINX_MTLS", "True") @@ -122,3 +203,205 @@ """ TT_OVMS_C_REPO_PATH - path to ovms-c repository. Can be relative or absolute. """ ovms_c_repo_path = get_path("TT_OVMS_C_REPO_PATH", get_path("PWD", "./")) + +""" TT_REPOSITORY_NAME - repository name provided by user """ +repository_name = os.environ.get("TT_REPOSITORY_NAME", "ovms-c") + +""" TT_ENVIRON_NAME - Environment name to be used while reporting test results + to be presented on test reports as a environment name.""" +environment_name = os.environ.get("TT_ENVIRONMENT_NAME", "") + +""" TT_PRODUCT_BUILD_NUMBER - Test product build number provided by user (last number from version - 0.8.0.XXXX)""" +product_build_number = os.environ.get("TT_PRODUCT_BUILD_NUMBER", "1") + +""" TT_PRODUCT_BUILD_FROM_ENV - If set to True, environment build number is taken from tested environment, + If set to False environment build number is taken from TT_PRODUCT_BUILD_NUMBER + If TT_PRODUCT_BUILD_NUMBER not set default environment build number is taken""" +product_build_number_from_env = get_bool("TT_PRODUCT_BUILD_FROM_ENV", True) + +""" TT_PRODUCT_VERSION - Environment version provided by user""" +product_version = os.environ.get("TT_PRODUCT_VERSION", "1.0.0") + +""" TT_PRODUCT_VERSION_FROM_ENV - If set to True, version is taken from tested environment, + If set to False version is taken from TT_PRODUCT_VERSION + If TT_PRODUCT_VERSION not set default version is taken""" +product_version_number_from_env = get_bool("TT_PRODUCT_VERSION_FROM_ENV", False) + +""" TT_PRODUCT_VERSION_SUFFIX - Environment version suffix provided by user""" +product_version_suffix = os.environ.get("TT_PRODUCT_VERSION_SUFFIX", "ovms") + +""" TT_DELAY_BETWEEN_TESTS - Time of pause between test case runs""" +delay_between_test = get_int("TT_DELAY_BETWEEN_TESTS", 0) + +""" TEST_TIMEOUT - default timeout (number of hours) for whole test session inherited from CI """ +pytest_global_session_timeout = get_int("TEST_TIMEOUT", 15) + +""" TT_BUILD_TEST_IMAGE - build ovms test image (cpu extensions, custom nodes etc.) """ +build_test_image = get_bool("TT_BUILD_TEST_IMAGE", False) + +""" TT_SAVE_IMAGE_TO_ARTIFACTS - save generated or edited image to artifacts """ +save_image_to_artifacts = get_bool("TT_SAVE_IMAGE_TO_ARTIFACTS", False) + +"""TT_SET_NO_PROXY""" +set_no_proxy = os.environ.get("TT_SET_NO_PROXY", True) +no_proxy = os.environ.get("no_proxy", "") +if set_no_proxy: + os.environ["NO_PROXY"] = no_proxy +http_proxy = os.environ.get("http_proxy", "") +https_proxy = os.environ.get("https_proxy", "") + +""" TT_RUN_OVMS_WITH_VALGRIND - run ovms using Valgrind """ +run_ovms_with_valgrind = get_bool("TT_RUN_OVMS_WITH_VALGRIND", False) + +""" TT_RUN_OVMS_WITH_OPENCL_TRACE - run OVMS with cliloader """ +run_ovms_with_opencl_trace = get_bool("TT_RUN_OVMS_WITH_OPENCL_TRACE", False) + +""" TT_XDIST_WORKERS - number of workers """ +xdist_workers = get_int("TT_XDIST_WORKERS", 0) + +""" TT_DOCKER_CLIENT_TIMEOUT - Docker client timeout""" +docker_client_timeout = get_int("TT_DOCKER_CLIENT_TIMEOUT", 120) + +""" TT_SERVER_ADDRESS - OVMS server address""" +server_address = os.environ.get("TT_SERVER_ADDRESS", "localhost") + +""" TT_RESOURCE_MONITOR_ENABLED - Dump ovms container resource statistics once per second """ +resource_monitor_enabled = get_bool("TT_RESOURCE_MONITOR_ENABLED", False) + +""" TT_TEST_TEMP_DIR - directory path where all temporary files are stored, default is not set """ +test_temp_dir = os.environ.get("TT_TEST_TEMP_DIR", None) +tmp_dir = TmpDir(test_temp_dir) + +"""TT_LOGGING_LEVEL_OVMS - ovms docker default log level, default: INFO""" +logging_level_ovms = os.environ.get("TT_LOGGING_LEVEL_OVMS", "INFO") + +"""TT_CONTAINER_PROXY - Proxy settings to be used in container """ +container_proxy = os.environ.get("TT_CONTAINER_PROXY", os.environ.get("http_proxy", "")) + +"""TT_DISABLE_DMESG_LOG_MONITOR""" +disable_dmesg_log_monitor = get_bool("TT_DISABLE_DMESG_LOG_MONITOR", False) + +""" TT_MACHINE_IS_RESERVED_FOR_TEST_SESSION """ +machine_is_reserved_for_test_session = get_bool("TT_MACHINE_IS_RESERVED_FOR_TEST_SESSION", False) + +""" TT_CLEANUP_ENVIRONMENT_ON_STARTUP """ +cleanup_env_on_startup = get_bool("TT_CLEANUP_ENVIRONMENT_ON_STARTUP", False) + +""" TT_TEARDOWN_DOCKER_IMAGES - at teardown remove docker images build during test session """ +teardown_docker_images = get_bool("TT_TEARDOWN_DOCKER_IMAGES", True) + +""" TT_TEARDOWN_DOCKER_CONTAINERS - at teardown remove stopped docker containers """ +teardown_docker_containers = get_bool("TT_TEARDOWN_DOCKER_CONTAINERS", False) + +""" TT_TEARDOWN_OVMS_PROCESSES - at teardown remove all ovms.exe processes """ +teardown_ovms_processes = get_bool("TT_TEARDOWN_OVMS_PROCESSES", False) + +""" TT_WAIT_FOR_MESSAGES_TIMEOUT - timeout for ovms.wait_for_messages(...) method """ +wait_for_messages_timeout = get_int("TT_WAIT_FOR_MESSAGES_TIMEOUT", 180) + +""" TT_AIRPLANE_MODE - disable connecting to remote resources, disable all downloads and docker pull/build commands and + expect that all required data is available locally. """ +airplane_mode = get_bool("TT_AIRPLANE_MODE", False) + +""" TT_OVMS_IMAGE_LOCAL - ovms image can only be found locally """ +ovms_image_local = get_bool("TT_OVMS_IMAGE_LOCAL", False) + +""" TT_BASE_OS - os type used for calculating ovms_image name (if not given explicitly). + Possible options (case insensitive): + ubuntu22 - use default Ubuntu 22.04 image + ubuntu24 - use default Ubuntu 24.04 image + redhat - use UBI 8.10 based + ubuntu22,ubuntu24,redhat - iterate all tests both for ubuntu and redhat + windows - can't iterate (supports only BINARY ovms type) +""" +__base_os = os.environ.get("BASE_OS", OsType.Ubuntu24) +base_os = get_list("TT_BASE_OS", fallback=[__base_os]) + +""" GLOBAL_TEMP_DIR - global temporary directory """ +global_tmp_dir_default = os.path.join("~", "AppData", "Local", "Temp") if OsType.Windows in base_os else "/tmp" +global_tmp_dir = get_path("GLOBAL_TEMP_DIR", global_tmp_dir_default) + +""" TT_ENABLE_PLUGIN_CONFIG_TARGET_DEVICE - use plugin_config globally set for target devices """ +enable_plugin_config_target_device = get_bool("TT_ENABLE_PLUGIN_CONFIG_TARGET_DEVICE", False) + +"""TT_DISABLE_CUSTOM_LOADER""" +disable_custom_loader = get_bool("TT_DISABLE_CUSTOM_LOADER", True) + +""" TT_CUSTOM_NODES - Custom nodes directory path""" +custom_nodes_path = get_path("TT_CUSTOM_NODES", os.path.join("~", "ovms_custom_nodes")) + +""" TT_BINARY_IO_IMAGES_PATH - Datasets local repo path""" +binary_io_images_path = get_path("TT_BINARY_IO_IMAGES_PATH", os.path.join("~", "ovms_binary_io")) + +""" TT_KV_CACHE_SIZE - memory size in GB for storing KV cache """ +kv_cache_size_value = get_int("TT_KV_CACHE_SIZE", 0) + +""" TT_KV_CACHE_PRECISION - Reduced kv cache precision to u8 lowers the cache size consumption. """ +kv_cache_precision_value = os.environ.get("TT_KV_CACHE_PRECISION", None) + +""" "MEDIAPIPE_REPO_BRANCH" - https://github.com/openvinotoolkit/mediapipe/ branch name """ +mediapipe_repo_branch = os.environ.get("MEDIAPIPE_REPO_BRANCH", "main") + +""" TT_MAX_NUM_BATCHED_TOKENS - max number of tokens processed in a single iteration """ +max_num_batched_tokens = get_int("TT_MAX_NUM_BATCHED_TOKENS", None) + +""" TT_PIPELINE_TYPE - pipeline type in LLM graph node_options, e.g. VLM, LM """ +pipeline_type = os.environ.get("TT_PIPELINE_TYPE", None) + +""" TT_ENABLE_PREFIX_CACHING - enable prefix caching for model """ +enable_prefix_caching_config = get_bool("TT_ENABLE_PREFIX_CACHING", False) + +""" TT_SSL_VALIDATION - if set to True for https request ssl protocol validation is performed, + default False""" +ssl_validation = get_bool("TT_SSL_VALIDATION", False) + +"""TT_LOGGED_RESPONSE_BODY_LENGTH - length of http response logged , default: 1024 """ +logged_response_body_length = os.environ.get("TT_LOGGED_RESPONSE_BODY_LENGTH", 1024) + +""" TT_C_API_WRAPPER_DIR - Cython wrapper files for C_API """ +c_api_wrapper_dir = get_path("TT_C_API_WRAPPER_DIR", os.path.join("~", "ovms_c_api_wrapper_dir")) + +""" TT_OVMS_FILE_LOCKS_DIR """ +ovms_file_locks_dir = get_path("TT_OVMS_FILE_LOCKS_DIR", os.path.join("~", "ovms_locks")) + +""" TT_USE_LEGACY_MODELS """ +use_legacy_models = get_bool("TT_USE_LEGACY_MODELS", True) + + +class StrippingLists: + DEFAULT_SENSITIVE_KEYS_TO_BE_MASKED = [ + r"(?!zabbix_operator_initial_).*pass(word)?", + r".*client_id", + r".*(access)?(_)?(? 0, f"Unable to match interface: {interface} to available values: {Metric.Interface}" + return result[0] + + @staticmethod + def create_method_metrics(model, base_name): + result = [] + for method in Metric.Methods: + for interface in Metric.Interface: + content = { + "api": Metric.MethodsProtocol[method], + "interface": interface, + "method": method, + "name": model.name, + } + + if method not in [Metric.Method_getmodelstatus, Metric.Method_modelready]: + content["version"] = str(model.version) + + result.append(Metric(metric_name=base_name, content=content)) + return result + + @staticmethod + def create_successful_method_metrics(model, ovms_run=None): + return Metric.create_method_metrics(model, Metric.Method_success) + + @staticmethod + def create_fail_method_metrics(model, ovms_run=None): + return Metric.create_method_metrics(model, Metric.Method_fail) + + @staticmethod + def create_stream_metrics(model, ovms_run=None): + value = None + if ovms_run is not None: + ovms_log_monitor = ovms_run.ovms.create_log(True) + value = ovms_log_monitor.get_log_value(msg_to_found=OvmsMessages.OV_NUMBER_STREAMS) + return [ + Metric(metric_name=Metric.Stream, content={"name": model.name, "version": str(model.version)}, value=value) + ] + + @staticmethod + def create_infer_request_active_metrics(model, ovms_run=None): + return [ + Metric( + metric_name=Metric.Infer_req_active, + content={"name": model.name, "version": str(model.version)}, + value=0, + ) + ] + + @staticmethod + def create_infer_request_queue_size_metrics(model, ovms_run=None): + value = None + if ovms_run is not None: + ovms_log_monitor = ovms_run.ovms.create_log(True) + value = ovms_log_monitor.get_log_value(msg_to_found=OvmsMessages.OV_NIREQ) + return [ + Metric( + metric_name=Metric.Infer_req_queue_size, + content={"name": model.name, "version": str(model.version)}, + value=value, + ) + ] + + @staticmethod + def create_current_request_metrics(model, ovms_run=None): + return [Metric(metric_name=Metric.Current_request, content={"name": model.name, "version": str(model.version)})] + + @staticmethod + def create_histogram_metrics(model, base_name, ovms_run=None): + result = [] + result.append( + Metric(metric_name=f"{base_name}_count", content={"name": model.name, "version": str(model.version)}) + ) + result.append( + Metric(metric_name=f"{base_name}_sum", content={"name": model.name, "version": str(model.version)}) + ) + for bucket_len in Metric.Histogram_bucket_len_list: + result.append( + Metric( + metric_name=f"{base_name}_bucket", + content={"name": model.name, "version": str(model.version), "le": bucket_len}, + ) + ) + return result + + @staticmethod + def create_request_histogram_metrics(model, ovms_run=None): + base_name = Metric.Request_histogram + result = [] + for interface in Metric.Interface: + result.append( + Metric( + metric_name=f"{base_name}_count", + content={"interface": interface, "name": model.name, "version": str(model.version)}, + ) + ) + result.append( + Metric( + metric_name=f"{base_name}_sum", + content={"interface": interface, "name": model.name, "version": str(model.version)}, + ) + ) + for bucket_len in Metric.Histogram_bucket_len_list: + result.append( + Metric( + metric_name=f"{base_name}_bucket", + content={ + "interface": interface, + "name": model.name, + "version": str(model.version), + "le": bucket_len, + }, + ) + ) + return result + + @staticmethod + def create_wait_for_infer_histogram(model, ovms_run=None): + return Metric.create_histogram_metrics(model, Metric.Wait_for_inference_histogram) + + @staticmethod + def create_infer_histogram_metrics(model, ovms_run=None): + return Metric.create_histogram_metrics(model, Metric.Inference_histogram) + + def __init__(self, metric_name, content: dict, value=0): + self.name = metric_name + self.content = content + self.keys = [x for x in content] + self.value = value + + def get_type(self): + result = None + if self.name in Metric.Default: + result = Metric.Default[self.name] + elif self.name in Metric.Additional: + result = Metric.Additional[self.name] + else: + histogram_metrics = [ + Metric.Request_histogram, + Metric.Inference_histogram, + Metric.Wait_for_inference_histogram, + ] + if any([x in self.name for x in histogram_metrics]): + result = Metric.Type_histogram + return result + + def to_str(self): + return f"{self.name}[{self.content}] {self.value}" + + def __str__(self): + return self.to_str() + + def compare(self, ref_metric): + return int(self.value) == int(ref_metric.value) + + +class Metrics: + std_regexp = re.compile(r"(\w+)\{([^\}]+)\}\s+(\d+)") + properties_regexp = re.compile(r"(\w+)=\"([\w\+-]+)\"") + + @staticmethod + def create_from_request(request_output): + lines = request_output.splitlines() + metrics = [] + for line in lines: + match = Metrics.std_regexp.search(line) + if match: + name, properties, value = match.groups() + content = {} + for key_val in properties.split(","): + key, v = Metrics.properties_regexp.search(key_val).groups() + content[key] = v + metrics.append(Metric(metric_name=name, content=content, value=value)) + result = Metrics() + result.list = metrics + return result + + _fill_method = { + Metric.Method_success: Metric.create_successful_method_metrics, + Metric.Method_fail: Metric.create_fail_method_metrics, + Metric.Stream: Metric.create_stream_metrics, + Metric.Infer_req_active: Metric.create_infer_request_active_metrics, + Metric.Infer_req_queue_size: Metric.create_infer_request_queue_size_metrics, + Metric.Current_request: Metric.create_current_request_metrics, + Metric.Request_histogram: Metric.create_request_histogram_metrics, + Metric.Inference_histogram: Metric.create_infer_histogram_metrics, + Metric.Wait_for_inference_histogram: Metric.create_wait_for_infer_histogram, + } + + @staticmethod + def create_from_model_list(model_list, ovms_run=None, metrics=None): + metric_list = [] + for model in model_list: + for metric in metrics: + if model.is_pipeline(): + for pipeline_model in model.get_models(): + metric_list += Metrics._fill_method[metric](pipeline_model) + + if metric not in Metric.Models_only: + metric_list += Metrics._fill_method[metric](model) + else: + metric_list += Metrics._fill_method[metric](model=model, ovms_run=ovms_run) + + """ + The following metrics are not multiplied for each model version (should occur once for single model name) + ovms_requests_success[{'api': 'TensorFlowServing', 'interface': 'gRPC', 'method': 'GetModelStatus', 'name': 'resnet-50-tf'}] 0 + ovms_requests_success[{'api': 'TensorFlowServing', 'interface': 'REST', 'method': 'GetModelStatus', 'name': 'resnet-50-tf'}] 0 + ovms_requests_success[{'api': 'KServe', 'interface': 'gRPC', 'method': 'ModelReady', 'name': 'resnet-50-tf'}] 0 + ovms_requests_success[{'api': 'KServe', 'interface': 'REST', 'method': 'ModelReady', 'name': 'resnet-50-tf'}] 0 + ovms_requests_fail[{'api': 'TensorFlowServing', 'interface': 'gRPC', 'method': 'GetModelStatus', 'name': 'resnet-50-tf'}] 0 + ovms_requests_fail[{'api': 'TensorFlowServing', 'interface': 'REST', 'method': 'GetModelStatus', 'name': 'resnet-50-tf'}] 0 + ovms_requests_fail[{'api': 'KServe', 'interface': 'gRPC', 'method': 'ModelReady', 'name': 'resnet-50-tf'}] 0 + ovms_requests_fail[{'api': 'KServe', 'interface': 'REST', 'method': 'ModelReady', 'name': 'resnet-50-tf'}] 0 + """ + metrics_to_remove = [] + model_unique_metrics = [] + for metric in metric_list: + if metric.content.get("method", None) in [Metric.Method_getmodelstatus, Metric.Method_modelready]: + if metric.to_str() in model_unique_metrics: + metrics_to_remove.append(metric) + else: + model_unique_metrics.append(metric.to_str()) + + for metric_to_remove in metrics_to_remove: + metric_list.remove(metric_to_remove) + + result = Metrics() + result.list = metric_list + return result + + def __init__(self): + self.list = [] + + def _search_metrics(self, key): + if isinstance(key, str): + key = [key] + + search_keys = [key] + if isinstance(key, tuple): + search_keys = list(key) + + metric_name = key[0] + search_keys = key[1:] + + result_metric_list = [x for x in self.list if x.name == metric_name] + for item in search_keys: + result_metric_list = [x for x in result_metric_list if item in x.to_str()] + return result_metric_list + + def __getitem__(self, key): + result_metric_list = self._search_metrics(key) + + result = Metrics() + result.list = result_metric_list + return result + + def __setitem__(self, key, value): + result_metric_list = self._search_metrics(key) + assert len(result_metric_list) > 0 + for metric in result_metric_list: + metric.value = value + + def to_str(self): + result = "" + for metric in self.list: + result += f"{metric.to_str()}\n" + + return result + + def search(self, metric): + metric_list = [x for x in self.list if x.name == metric.name and x.content == metric.content] + + if len(metric_list) == 0: + metric_list = [x for x in self.list if x.name == metric.name] + for key, value in metric.content.items(): + tmp_metrics = [x for x in metric_list if key in x.content and x.content[key] == value] + assert ( + len(tmp_metrics) > 0 + ), f"Unable to find metric: {metric.to_str()} (closes: {[x.to_str() for x in metric_list]})" + metric_list = tmp_metrics + return metric_list[0] + + def compare(self, ref_metrics, metrics_title=None, ref_metrics_title=None): + logger.info(f"Metrics::compare - nr of items: {len(self.list)}") + metrics_not_equal = [] + for metric in self.list: + logger.debug(f"\t{metric.to_str()}") + found_metric = ref_metrics.search(metric) + if found_metric.get_type() not in [Metric.Type_histogram] and metric.name not in [Metric.Stream]: + if not metric.compare(found_metric): + error_message = f"Metrics are not equal: {metric.to_str()} != {found_metric.to_str()}" + logger.info(f"Metrics are not equal: {metric.to_str()} != {found_metric.to_str()}") + metrics_not_equal.append(error_message) + + assert len(metrics_not_equal) == 0 + assert len(self.list) == len( + ref_metrics.list + ), (f"Length of {metrics_title} ({len(self.list)}) " + f"and length of {ref_metrics_title} ({len(ref_metrics.list)}) are not equal!") + + def find_metric_specific_value_content(self, metric_name, api, interface, method, model_name, model_version, value): + expected_content = { + "api": api, + "interface": interface, + "method": method, + "name": model_name, + "version": str(model_version), + } + metric_found = False + for metric in self.list: + if metric.content == expected_content and metric.name == metric_name: + if metric.value == str(value): + logger.debug(f"Found expected value={value} in metric {metric_name} for method {method}") + metric_found = True + break + assert metric_found, f"No metric found" + + def verify_metric_values(self, value): + for metric in self.list: + assert metric.value == value + + +class DefaultMetrics(Metrics): + + def __init__(self): + super().__init__() + + @staticmethod + def create_from_model_list(model_list): + return Metrics.create_from_model_list(model_list, metrics=Metric.Default_names) + + +class AdditionalMetrics(Metrics): + Names = ["ovms_infer_req_queue_size", "ovms_infer_req_active"] + + def __init__(self): + super().__init__() + + +# Output example +# +# HELP ovms_requests_success Number of successful requests to a model or a DAG. +# TYPE ovms_requests_success counter +# +# ovms_requests_success{ +# interface="rest", +# method="modelinfer", +# name="ssdlite_mobilenet_v2_ov", +# protocol="kserve", +# version="1"} 0 +# ovms_requests_success{ +# interface="grpc", +# method="modelready", +# name="ssdlite_mobilenet_v2_ov", +# protocol="kserve"} 0 +# ovms_requests_success{ +# interface="grpc", +# method="modelmetadata", +# name="ssdlite_mobilenet_v2_ov", +# protocol="kserve", +# version="1"} 0 +# ovms_requests_success{ +# interface="rest", +# method="getmodelstatus", +# name="ssdlite_mobilenet_v2_ov", +# protocol="tensorflowserving"} 0 +# ovms_requests_success{ +# interface="rest", +# method="getmodelmetadata", +# name="ssdlite_mobilenet_v2_ov", +# protocol="tensorflowserving", +# version="1"} 0 +# ovms_requests_success{ +# interface="rest", +# method="predict", +# name="ssdlite_mobilenet_v2_ov", +# protocol="tensorflowserving", +# version="1"} 0 +# ovms_requests_success{ +# interface="grpc", +# method="modelinfer", +# name="ssdlite_mobilenet_v2_ov", +# protocol="kserve", +# version="1"} 0 +# ovms_requests_success{ +# interface="grpc", +# method="getmodelstatus", +# name="ssdlite_mobilenet_v2_ov", +# protocol="tensorflowserving"} 0 +# ovms_requests_success{ +# interface="grpc", +# method="getmodelmetadata", +# name="ssdlite_mobilenet_v2_ov", +# protocol="tensorflowserving", +# version="1"} 0 +# ovms_requests_success{ +# interface="rest", +# method="modelready", +# name="ssdlite_mobilenet_v2_ov", +# protocol="kserve"} 0 +# ovms_requests_success{ +# interface="rest", +# method="modelmetadata", +# name="ssdlite_mobilenet_v2_ov", +# protocol="kserve", +# version="1"} 0 +# ovms_requests_success{ +# interface="grpc", +# method="predict", +# name="ssdlite_mobilenet_v2_ov", +# protocol="tensorflowserving", +# version="1"} 0 +# # HELP ovms_requests_fail Number of failed requests to a model or a DAG. +# # TYPE ovms_requests_fail counter +# ovms_requests_fail{ +# interface="rest", +# method="modelready", +# name="ssdlite_mobilenet_v2_ov", +# protocol="kserve", +# version="1"} 0 +# ovms_requests_fail{ +# interface="grpc", +# method="modelready", +# name="ssdlite_mobilenet_v2_ov", +# protocol="kserve", +# version="1"} 0 +# ovms_requests_fail{ +# interface="grpc", +# method="modelinfer", +# name="ssdlite_mobilenet_v2_ov", +# protocol="kserve", +# version="1"} 0 +# ovms_requests_fail{ +# interface="rest", +# method="modelinfer", +# name="ssdlite_mobilenet_v2_ov", +# protocol="kserve", +# version="1"} 0 +# ovms_requests_fail{ +# interface="rest", +# method="getmodelstatus", +# name="ssdlite_mobilenet_v2_ov", +# protocol="tensorflowserving", +# version="1"} 0 +# ovms_requests_fail{ +# interface="rest", +# method="getmodelmetadata", +# name="ssdlite_mobilenet_v2_ov", +# protocol="tensorflowserving", +# version="1"} 0 +# ovms_requests_fail{ +# interface="rest", +# method="predict", +# name="ssdlite_mobilenet_v2_ov", +# protocol="tensorflowserving", +# version="1"} 0 +# ovms_requests_fail{ +# interface="grpc", +# method="modelmetadata", +# name="ssdlite_mobilenet_v2_ov", +# protocol="kserve", +# version="1"} 0 +# ovms_requests_fail{ +# interface="grpc", +# method="getmodelstatus", +# name="ssdlite_mobilenet_v2_ov", +# protocol="tensorflowserving", +# version="1"} 0 +# ovms_requests_fail{ +# interface="grpc", +# method="getmodelmetadata", +# name="ssdlite_mobilenet_v2_ov", +# protocol="tensorflowserving", +# version="1"} 0 +# ovms_requests_fail{ +# interface="rest", +# method="modelmetadata", +# name="ssdlite_mobilenet_v2_ov", +# protocol="kserve", +# version="1"} 0 +# ovms_requests_fail{ +# interface="grpc", +# method="predict", +# name="ssdlite_mobilenet_v2_ov", +# protocol="tensorflowserving", +# version="1"} 0 +# # HELP ovms_streams Number of OpenVINO execution streams. +# # TYPE ovms_streams gauge +# ovms_streams{name="ssdlite_mobilenet_v2_ov",version="1"} 4 +# # HELP ovms_infer_req_queue_size Inference request queue size (nireq). +# # TYPE ovms_infer_req_queue_size gauge +# ovms_infer_req_queue_size{name="ssdlite_mobilenet_v2_ov",version="1"} 4 +# # HELP ovms_infer_req_active Number of currently consumed inference request from the processing queue. +# # TYPE ovms_infer_req_active gauge +# ovms_infer_req_active{name="ssdlite_mobilenet_v2_ov",version="1"} 0 +# # HELP ovms_current_requests Number of inference requests currently in process. +# # TYPE ovms_current_requests gauge +# ovms_current_requests{name="ssdlite_mobilenet_v2_ov",version="1"} 0 +# # HELP ovms_request_time_us Processing time of requests to a model or a DAG. +# # TYPE ovms_request_time_us histogram +# ovms_request_time_us_count{interface="rest",name="ssdlite_mobilenet_v2_ov",version="1"} 0 +# ovms_request_time_us_sum{interface="rest",name="ssdlite_mobilenet_v2_ov",version="1"} 0 +# ovms_request_time_us_bucket{interface="rest",name="ssdlite_mobilenet_v2_ov",version="1",le="10"} 0 +# ovms_request_time_us_bucket{interface="rest",name="ssdlite_mobilenet_v2_ov",version="1",le="18"} 0 +# ... +# ovms_request_time_us_bucket{interface="rest",name="ssdlite_mobilenet_v2_ov",version="1",le="1474755971"} 0 +# ovms_request_time_us_bucket{interface="rest",name="ssdlite_mobilenet_v2_ov",version="1",le="+Inf"} 0 +# ovms_request_time_us_count{interface="grpc",name="ssdlite_mobilenet_v2_ov",version="1"} 0 +# ovms_request_time_us_sum{interface="grpc",name="ssdlite_mobilenet_v2_ov",version="1"} 0 +# ovms_request_time_us_bucket{interface="grpc",name="ssdlite_mobilenet_v2_ov",version="1",le="10"} 0 +# ... +# ovms_request_time_us_bucket{interface="grpc",name="ssdlite_mobilenet_v2_ov",version="1",le="+Inf"} 0 +# # HELP ovms_inference_time_us Inference execution time in the OpenVINO backend. +# # TYPE ovms_inference_time_us histogram +# ovms_inference_time_us_count{name="ssdlite_mobilenet_v2_ov",version="1"} 0 +# ovms_inference_time_us_sum{name="ssdlite_mobilenet_v2_ov",version="1"} 0 +# ovms_inference_time_us_bucket{name="ssdlite_mobilenet_v2_ov",version="1",le="10"} 0 +# ... +# ovms_inference_time_us_bucket{name="ssdlite_mobilenet_v2_ov",version="1",le="+Inf"} 0 +# # HELP ovms_wait_for_infer_req_time_us Request waiting time in the scheduling queue. +# # TYPE ovms_wait_for_infer_req_time_us histogram +# ovms_wait_for_infer_req_time_us_count{name="ssdlite_mobilenet_v2_ov",version="1"} 0 +# ovms_wait_for_infer_req_time_us_sum{name="ssdlite_mobilenet_v2_ov",version="1"} 0 +# ovms_wait_for_infer_req_time_us_bucket{name="ssdlite_mobilenet_v2_ov",version="1",le="10"} 0 +# ... +# ovms_wait_for_infer_req_time_us_bucket{name="ssdlite_mobilenet_v2_ov",version="1",le="+Inf"} 0 diff --git a/tests/functional/constants/os_type.py b/tests/functional/constants/os_type.py new file mode 100644 index 0000000000..f45e1d02e5 --- /dev/null +++ b/tests/functional/constants/os_type.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import platform +import distro + + +class OsType: + Redhat = "redhat" + Ubuntu22 = "ubuntu22" + Ubuntu24 = "ubuntu24" + Windows = "windows" + + +UBUNTU = "ubuntu" +UBUNTU_22_OS_VERSION = "22.04" +UBUNTU_24_OS_VERSION = "24.04" +WINDOWS_MSYS_NT = "msys_nt" + + +def get_host_os(): + # pylint: disable=no-else-return + result = distro.id() if distro.id() else platform.system() + if result == UBUNTU: + ubuntu_version = get_host_os_version() + if ubuntu_version == UBUNTU_22_OS_VERSION: + return OsType.Ubuntu22 + elif ubuntu_version == UBUNTU_24_OS_VERSION: + return OsType.Ubuntu24 + elif result == "rhel": + return OsType.Redhat + elif result.lower() == OsType.Windows: + return OsType.Windows + elif WINDOWS_MSYS_NT in result.lower(): + return OsType.Windows + raise NotImplementedError(f"OS not supported: {result}") + + +def get_host_os_version(): + result = distro.version() + return result + + +def get_host_os_details(): + if distro.lsb_release_info(): + return distro.lsb_release_info().get("description", None) + return platform.platform() From d30023e8f0f55155e7d0d1395714bf8239282a9e Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Fri, 8 May 2026 10:14:03 +0200 Subject: [PATCH 02/12] constants + data --- tests/functional/constants/ovms.py | 305 +++ tests/functional/constants/ovms_binaries.py | 84 + tests/functional/constants/ovms_images.py | 174 ++ tests/functional/constants/ovms_messages.py | 491 +++++ tests/functional/constants/ovms_openai.py | 307 +++ tests/functional/constants/ovms_type.py | 71 + tests/functional/constants/ovsa.py | 43 + tests/functional/constants/paths.py | 76 + tests/functional/constants/pipelines.py | 1860 +++++++++++++++++ tests/functional/constants/requirements.py | 68 + tests/functional/constants/target_device.py | 14 + .../constants/target_device_configuration.py | 112 + .../{common_libs => data}/__init__.py | 0 .../data/ovms_capi_wrapper/Makefile | 26 + .../ovms_capi_wrapper}/__init__.py | 2 +- .../data/ovms_capi_wrapper/ovms_autopxd.py | 77 + .../ovms_capi_wrapper/ovms_capi_shared.py} | 12 +- .../ovms_capi_wrapper/ovms_capi_wrapper.pyx | 364 ++++ .../data/ovms_capi_wrapper/setup.py | 72 + .../python_custom_nodes}/__init__.py | 30 +- .../incrementer/__init__.py} | 14 +- .../incrementer/incrementer.py | 42 + .../ovms_basic/__init__.py | 15 + .../ovms_basic/python_model.py | 49 + .../ovms_basic/python_model_loopback.py | 47 + .../ovms_corrupted/__init__.py | 15 + .../python_model_corrupted_import.py | 23 + .../ovms_corrupted/python_model_exceptions.py | 45 + ..._loopback_multiple_use_of_valid_outputs.py | 53 + ..._model_loopback_return_instead_of_yield.py | 50 + .../python_model_missing_execute.py | 18 + ...l_writing_to_loopback_output_in_execute.py | 50 + tests/functional/fixtures/api_type.py | 106 + tests/functional/fixtures/common_fixtures.py | 79 - .../fixtures/model_conversion_fixtures.py | 61 - .../fixtures/model_download_fixtures.py | 70 - tests/functional/fixtures/ovms.py | 148 ++ tests/functional/fixtures/params.py | 82 + tests/functional/fixtures/server.py | 184 ++ .../server_detection_model_fixtures.py | 67 - .../fixtures/server_for_update_fixtures.py | 58 - .../fixtures/server_local_models_fixtures.py | 76 - .../fixtures/server_multi_model_fixtures.py | 55 - .../fixtures/server_remote_models_fixtures.py | 167 -- .../fixtures/server_with_batching_fixtures.py | 95 - .../server_with_version_policy_fixtures.py | 66 - .../fixtures/test_files_fixtures.py | 49 - tests/functional/mapping_config.json | 8 - tests/functional/model/models_information.py | 166 -- .../model_version_policy_config_template.json | 28 - 50 files changed, 5094 insertions(+), 1080 deletions(-) create mode 100644 tests/functional/constants/ovms.py create mode 100644 tests/functional/constants/ovms_binaries.py create mode 100644 tests/functional/constants/ovms_images.py create mode 100644 tests/functional/constants/ovms_messages.py create mode 100644 tests/functional/constants/ovms_openai.py create mode 100644 tests/functional/constants/ovms_type.py create mode 100644 tests/functional/constants/ovsa.py create mode 100644 tests/functional/constants/paths.py create mode 100644 tests/functional/constants/pipelines.py create mode 100644 tests/functional/constants/requirements.py create mode 100644 tests/functional/constants/target_device_configuration.py rename tests/functional/{common_libs => data}/__init__.py (100%) create mode 100644 tests/functional/data/ovms_capi_wrapper/Makefile rename tests/functional/{command_wrappers => data/ovms_capi_wrapper}/__init__.py (93%) create mode 100644 tests/functional/data/ovms_capi_wrapper/ovms_autopxd.py rename tests/functional/{command_wrappers/server.py => data/ovms_capi_wrapper/ovms_capi_shared.py} (66%) create mode 100644 tests/functional/data/ovms_capi_wrapper/ovms_capi_wrapper.pyx create mode 100644 tests/functional/data/ovms_capi_wrapper/setup.py rename tests/functional/{model => data/python_custom_nodes}/__init__.py (91%) rename tests/{reservation_manager_single_port.yml => functional/data/python_custom_nodes/incrementer/__init__.py} (64%) create mode 100644 tests/functional/data/python_custom_nodes/incrementer/incrementer.py create mode 100644 tests/functional/data/python_custom_nodes/ovms_basic/__init__.py create mode 100644 tests/functional/data/python_custom_nodes/ovms_basic/python_model.py create mode 100644 tests/functional/data/python_custom_nodes/ovms_basic/python_model_loopback.py create mode 100644 tests/functional/data/python_custom_nodes/ovms_corrupted/__init__.py create mode 100644 tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_corrupted_import.py create mode 100644 tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_exceptions.py create mode 100644 tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_loopback_multiple_use_of_valid_outputs.py create mode 100644 tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_loopback_return_instead_of_yield.py create mode 100644 tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_missing_execute.py create mode 100644 tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_writing_to_loopback_output_in_execute.py create mode 100644 tests/functional/fixtures/api_type.py delete mode 100644 tests/functional/fixtures/common_fixtures.py delete mode 100644 tests/functional/fixtures/model_conversion_fixtures.py delete mode 100644 tests/functional/fixtures/model_download_fixtures.py create mode 100644 tests/functional/fixtures/ovms.py create mode 100644 tests/functional/fixtures/params.py create mode 100644 tests/functional/fixtures/server.py delete mode 100644 tests/functional/fixtures/server_detection_model_fixtures.py delete mode 100644 tests/functional/fixtures/server_for_update_fixtures.py delete mode 100644 tests/functional/fixtures/server_local_models_fixtures.py delete mode 100644 tests/functional/fixtures/server_multi_model_fixtures.py delete mode 100644 tests/functional/fixtures/server_remote_models_fixtures.py delete mode 100644 tests/functional/fixtures/server_with_batching_fixtures.py delete mode 100644 tests/functional/fixtures/server_with_version_policy_fixtures.py delete mode 100644 tests/functional/fixtures/test_files_fixtures.py delete mode 100644 tests/functional/mapping_config.json delete mode 100644 tests/functional/model/models_information.py delete mode 100644 tests/functional/model_version_policy_config_template.json diff --git a/tests/functional/constants/ovms.py b/tests/functional/constants/ovms.py new file mode 100644 index 0000000000..5482ad13d7 --- /dev/null +++ b/tests/functional/constants/ovms.py @@ -0,0 +1,305 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import re + +from enum import Enum +from pathlib import Path +from tensorflow_serving.apis.get_model_status_pb2 import ModelVersionStatus + +from tests.functional.constants.os_type import OsType + +from tests.functional.constants.target_device import TargetDevice +from tests.functional.constants.ovms_type import OvmsType + + +class Ovms: + + # OPENVINO + OPENVINO = "OpenVINO" + OPENVINO_MODEL_SERVER = "OpenVINO Model Server" + + INFERENCE_PRECISION_HINT = "INFERENCE_PRECISION_HINT" + + # OVMS INTERNAL PARAMS + CPU_THROUGHPUT_AUTO = "CPU_THROUGHPUT_AUTO" + PLUGIN_CONFIG_CPU_STREAMS_THROUGHPUT_AUTO = {"CPU_THROUGHPUT_STREAMS": CPU_THROUGHPUT_AUTO} + + PLUGIN_CONFIG_AUTO = {"PERFORMANCE_HINT": "LATENCY"} + PLUGIN_CONFIG_CPU = PLUGIN_CONFIG_AUTO + PLUGIN_CONFIG_GPU = PLUGIN_CONFIG_AUTO + PLUGIN_CONFIG_NPU = PLUGIN_CONFIG_AUTO + PLUGIN_CONFIG_AUTO_CUMULATIVE_THROUGHPUT = {"PERFORMANCE_HINT": "CUMULATIVE_THROUGHPUT"} + PLUGIN_CONFIG_HETERO = {"MULTI_DEVICE_PRIORITIES": "GPU,CPU"} + + PLUGIN_CONFIG = { + TargetDevice.CPU: PLUGIN_CONFIG_CPU, + TargetDevice.GPU: PLUGIN_CONFIG_GPU, + TargetDevice.NPU: PLUGIN_CONFIG_NPU, + TargetDevice.AUTO: PLUGIN_CONFIG_AUTO, + TargetDevice.HETERO: PLUGIN_CONFIG_HETERO, + } + + PLUGIN_CONFIG_CPU_PARAMS_LIST = [ + PLUGIN_CONFIG_CPU, + {"NUM_STREAMS": "AUTO"}, + {"NUM_STREAMS": "1"}, + {"NUM_STREAMS": "32"}, + {"INFERENCE_NUM_THREADS": "1"}, + {"INFERENCE_NUM_THREADS": "24"}, + {"ENABLE_CPU_PINNING": "false"}, + ] + + PLUGIN_CONFIG_GPU_PARAMS_LIST = [ + PLUGIN_CONFIG_GPU, + {"NUM_STREAMS": "AUTO"}, + {"NUM_STREAMS": "1"}, + {"NUM_STREAMS": "32"}, + {"PERFORMANCE_HINT": "THROUGHPUT"}, + ] + + PLUGIN_CONFIG_NPU_PARAMS_LIST = [ + PLUGIN_CONFIG_NPU, + {"PERFORMANCE_HINT": "THROUGHPUT"}, + ] + + PLUGIN_CONFIG_AUTO_PARAMS_LIST = [ + PLUGIN_CONFIG_AUTO, + {"PERFORMANCE_HINT": "THROUGHPUT"}, + PLUGIN_CONFIG_AUTO_CUMULATIVE_THROUGHPUT + ] + + PLUGIN_CONFIG_HETERO_PARAMS_LIST = [ + # https://docs.openvino.ai/latest/openvino_docs_OV_UG_Hetero_execution.html + PLUGIN_CONFIG_HETERO, + ] + + PLUGIN_CONFIG_PARAMS = { + TargetDevice.CPU: PLUGIN_CONFIG_CPU_PARAMS_LIST, + TargetDevice.GPU: PLUGIN_CONFIG_GPU_PARAMS_LIST, + TargetDevice.NPU: PLUGIN_CONFIG_NPU_PARAMS_LIST, + TargetDevice.AUTO: PLUGIN_CONFIG_AUTO_PARAMS_LIST, + TargetDevice.HETERO: PLUGIN_CONFIG_HETERO_PARAMS_LIST, + } + + PLUGIN_CONFIG_WINDOWS = {"ENABLE_MMAP": "NO"} + + PERFORMANCE_HINT = "PERFORMANCE_HINT" + PERFORMANCE_HINT_VALUES = ["LATENCY", "THROUGHPUT"] + + OVMS_REQUEST_TIMEOUT: int = 240 + + V2_OPERATION_HEALTHY = "healthy" + V2_OPERATION_METADATA = "metadata" + V2_OPERATIONS = [V2_OPERATION_HEALTHY, V2_OPERATION_METADATA] + + OVMS_CONTAINER_NAME_DEFAULT = "ovms_cpp" + ALL_MODEL_VERSION_POLICY = '{"all":{}}' + TARGET_DEVICE_CPU = "CPU" + TARGET_DEVICE_GPU = "GPU" + TARGET_DEVICE_NPU = "NPU" + BATCHSIZE = "1" + BATCHSIZE_2 = "2" + BATCHSIZE_3 = "3" + BATCHSIZE_HUGE = "150" + BATCHSIZE_TOO_LARGE = "1800" + NIREQ = 2 + WINDOWS_GRPC_WORKERS = 1 + GRPC_WORKERS = 4 + REST_WORKERS = 4 + LOG_LEVEL_TRACE = "TRACE" + LOG_LEVEL_DEBUG = "DEBUG" + LOG_LEVEL_INFO = "INFO" + LOG_LEVEL_ERROR = "ERROR" + LOG_LEVEL_WARNING = "WARNING" + DYNAMIC_AUTO_SIZE = "auto" + + SCALAR_BATCH_SIZE = "none" + OUTPUT_FILLER = "\00" * 7 + + # The time in seconds of OVMS verification if models have been changed on disk + OVMS_MODELS_REFRESH_TIMEOUT = 2 + OVMS_DEFAULT_FILE_SYSTEM_POLL_WAIT_SECONDS = 1 + + class ModelStatus(Enum): + UNDEFINED = None + UNKNOWN = ModelVersionStatus.UNKNOWN + START = ModelVersionStatus.START + LOADING = ModelVersionStatus.LOADING + AVAILABLE = ModelVersionStatus.AVAILABLE + UNLOADING = ModelVersionStatus.UNLOADING + END = ModelVersionStatus.END + + LAYOUT_NHWC = "NHWC:NCHW" + LAYOUT_NCHW = "NCHW:NCHW" + + IMAGE_CHANNEL_FORMAT_RGB = "RGB" + + BINARY_IO_LAYOUT_ROW_NAME = "row_name" + BINARY_IO_LAYOUT_COLUMN_NAME = "column_name" + BINARY_IO_LAYOUT_ROW_NONAME = "row_noname" + BINARY_IO_LAYOUT_COLUMN_NONAME = "column_noname" + + TFS_REST_LAYOUT_TYPES = [ + BINARY_IO_LAYOUT_ROW_NAME, + BINARY_IO_LAYOUT_COLUMN_NAME, + BINARY_IO_LAYOUT_ROW_NONAME, + BINARY_IO_LAYOUT_COLUMN_NONAME, + ] + + GRPC_PROTOCOL_NAME = "gRPC" + REST_PROTOCOL_NAME = "REST" + + JPG_IMAGE_FORMAT = "JPEG" + PNG_IMAGE_FORMAT = "PNG" + + # For details take a peek: + # model_server/docs/ovms_docs_streaming_endpoints.html#manual-timestamping + TIMESTAMP_PARAM_NAME = "OVMS_MP_TIMESTAMP" + + SIGTERM_SIGNAL = "SIGTERM" + SIGKILL_SIGNAL = "SIGKILL" + SIGINT_SIGNAL = "SIGINT" + KILL_SIGNAL = "KILL" + TERM_SIGNAL = "TERM" + + STOP_METHOD = "stop" + KILL_METHOD = "kill" + + MAX_THREADS_VALGRIND = 96 * 4 + + @staticmethod + def get_ovms_binary_paths(ovms_type, base_os=None): + if base_os == OsType.Windows: + ovms_binary_path = "ovms\\ovms.exe" + ovms_lib_binary_path = "" + else: + ovms_binary_path = "ovms/bin/ovms" if ovms_type == OvmsType.BINARY else "/ovms/bin/ovms" + ovms_lib_binary_path = "ovms/lib" if ovms_type == OvmsType.BINARY else "/ovms/lib/" + return ovms_binary_path, ovms_lib_binary_path + + +class CurrentTarget: + target_device = None + + is_auto_target = lambda: CurrentTarget.target_device in [TargetDevice.AUTO] + is_hetero_target = lambda: CurrentTarget.target_device in [TargetDevice.HETERO] + is_gpu_target = lambda: CurrentTarget.target_device in [TargetDevice.GPU] + is_npu_target = lambda: CurrentTarget.target_device in [TargetDevice.NPU] + is_cpu_target = lambda: CurrentTarget.target_device in [TargetDevice.CPU] + + @classmethod + def is_plugin_target(cls): + return any([ + cls.is_auto_target(), + cls.is_hetero_target(), + cls.is_gpu_target(), + ]) + + @staticmethod + def is_gpu_based_target(target_device): + return target_device in [ + TargetDevice.GPU, + TargetDevice.NPU, + TargetDevice.AUTO, + TargetDevice.AUTO_CPU_GPU, + TargetDevice.HETERO, + ] + + +class CurrentOvmsType: + ovms_type = None + + is_docker_type = lambda: CurrentOvmsType.ovms_type in [OvmsType.DOCKER] + is_binary_type = lambda: CurrentOvmsType.ovms_type in [OvmsType.BINARY] + is_binary_docker_type = lambda: CurrentOvmsType.ovms_type in [OvmsType.BINARY_DOCKER] + is_kubernetes_type = lambda: CurrentOvmsType.ovms_type in [OvmsType.KUBERNETES] + is_docker_cmd_line_type = lambda: CurrentOvmsType.ovms_type in [OvmsType.DOCKER_CMD_LINE] + is_none_type = lambda: CurrentOvmsType.ovms_type in [OvmsType.NONE] + + +TARGET_DEVICE_PARAM_NAME = "target_device" +OVMS_TYPE_PARAM_NAME = "ovms_type" +API_TYPE_PARAM_NAME = "api_type" +GRPC_API_TYPE_PARAM_NAME = "grpc_api_type" +REST_API_TYPE_PARAM_NAME = "rest_api_type" +MODEL_TYPE_PARAM_NAME = "model_type" +CLOUD_TYPE_PARAM_NAME = "cloud_type" +USES_MAPPING_PARAM_NAME = "use_mapping" +USES_CONFIG_PARAM_NAME = "use_config" +BASE_OS_PARAM_NAME = "base_os" +TEST_RUN_WORKER_ARGUMENT = "test_run_reporters" +TMP_REPOS_DIR_ARGUMENT = "tmp_repos_dir" +CURRENT_TARGET_DEVICE_DICT_ARGUMENT = "current_target_device_dict" + + +class Config: + MODEL_CONFIG_LIST = "model_config_list" + MEDIAPIPE_CONFIG_LIST = "mediapipe_config_list" + PIPELINE_CONFIG_LIST = "pipeline_config_list" + CUSTOM_LOADER_CONFIG_LIST = "custom_loader_config_list" + CUSTOM_NODE_LIBRARY_CONFIG_LIST = "custom_node_library_config_list" + MONITORING = "monitoring" + CONFIG = "config" + PLUGIN_CONFIG = "plugin_config" + + +class MediaPipeConstants: + DEFAULT_INPUT_STREAM = "in" + DEFAULT_OUTPUT_STREAM = "out" + + +class MediapipeIntermediatePacket(Enum): + # In case of whole graph input/output stream packet types accepted tags are: + TENSOR = "TENSOR" + TFTENSOR = "TFTENSOR" + TFLITE_TENSOR = "TFLITE_TENSOR" + OVTENSOR = "OVTENSOR" + IMAGE = "IMAGE" + + +def set_plugin_config_boolean_value(plugin_config_str, config_file=False): + # remove quotation marks for bool plugin_config values + if config_file: + plugin_config_pattern = re.compile(r"(\"plugin_config\":\s\{[\s\"\w]+\:\s)(\"(false|true)\")([\s\"\w]+\})") + match_plugin_config = plugin_config_pattern.search(plugin_config_str) + if match_plugin_config: + return re.sub( + plugin_config_pattern, + f"{match_plugin_config[1]}{match_plugin_config[3]}{match_plugin_config[4]}", + plugin_config_str, + ) + return plugin_config_str + else: + return plugin_config_str.replace('\\"false\\"', "false").replace('\\"true\\"', "true") + + +def get_model_base_path(model_base_path, context, ovms_run): + new_model_base_path = model_base_path if OvmsType.DOCKER in context.ovms_type \ + else os.path.join(ovms_run.ovms.container_folder, *model_base_path.split(os.path.sep)[1:]) + return new_model_base_path + + +class HfImportParams: + PULL = "pull" + LIST_MODELS = "list_models" + ADD_TO_CONFIG = "add_to_config" + REMOVE_FROM_CONFIG = "remove_from_config" + TEXT_GENERATION = "text_generation" + IMAGE_GENERATION = "image_generation" + EMBEDDINGS = "embeddings" + RERANK = "rerank" diff --git a/tests/functional/constants/ovms_binaries.py b/tests/functional/constants/ovms_binaries.py new file mode 100644 index 0000000000..3dc477021b --- /dev/null +++ b/tests/functional/constants/ovms_binaries.py @@ -0,0 +1,84 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import shutil + +from pathlib import Path + +from tests.functional.utils.logger import get_logger +from tests.functional.utils.test_framework import generate_test_object_name +from tests.functional.constants.os_type import OsType +from tests.functional.utils.process import Process + +from tests.functional.config import ovms_c_release_artifacts_path +from tests.functional.constants.ovms import Ovms +from tests.functional.constants.ovms_type import OvmsType +from tests.functional.constants.paths import Paths + + +logger = get_logger(__name__) + + +def calculate_ovms_binary_name(base_os=OsType.Ubuntu22): + if len(ovms_c_release_artifacts_path) == 1: + return ovms_c_release_artifacts_path[0] + for binary_name in ovms_c_release_artifacts_path: + if base_os in binary_name: + return binary_name + return None + + +def get_binaries(base_os, test_object_name, tmp_dir): + """ + Run dummy ovms instance just for copying docker resources to temporary directory + """ + proc = Process() + proc.disable_check_stderr() + package_content_path = Path(Paths.CAPI_WRAPPER_PACKAGE_CONTENT_PATH(base_os)) + test_object_name = test_object_name if test_object_name is not None else generate_test_object_name() + resource_dir = Path(tmp_dir, test_object_name) + resource_dir.mkdir(exist_ok=True, parents=True) + ovms_binary_path, _ = Ovms.get_ovms_binary_paths(OvmsType.BINARY, base_os) + resource_dir_ovms = Path(resource_dir, "ovms") + if not os.path.exists(resource_dir_ovms): + if base_os == OsType.Windows: + shutil.copytree(package_content_path, resource_dir_ovms) + else: + os.symlink(package_content_path, resource_dir_ovms) + return os.path.join(resource_dir, ovms_binary_path), test_object_name + + +def get_ovms_binary_cmd_setup(base_os=None, resources_dir_path=None, environment=None, venv_activate_path=None): + env = {} if environment is None else environment + if base_os == OsType.Windows: + env["SYSTEMROOT"] = os.environ["SYSTEMROOT"] + pre_cmd = f"{resources_dir_path}\\setupvars.bat && " + if venv_activate_path: + pre_cmd += f"{venv_activate_path} && " + else: + # required for GPU + env_neo_read_debug_keys = os.environ.get("NEOReadDebugKeys", None) + if env_neo_read_debug_keys is not None: + env["NEOReadDebugKeys"] = env_neo_read_debug_keys + env_override_gpu_adress_space = os.environ.get("OverrideGpuAddressSpace", None) + if env_override_gpu_adress_space is not None: + env["OverrideGpuAddressSpace"] = env_override_gpu_adress_space + # required for git/git-lfs + env["PATH"] = f"{os.environ['PATH']}:{os.path.join(resources_dir_path, 'bin')}" + pre_cmd = "LD_LIBRARY_PATH=${PWD}/ovms/lib PYTHONPATH=${PWD}/ovms/lib/python/ " + logger.debug(f"Binary environment: {env}") + return pre_cmd, env diff --git a/tests/functional/constants/ovms_images.py b/tests/functional/constants/ovms_images.py new file mode 100644 index 0000000000..1e268965b4 --- /dev/null +++ b/tests/functional/constants/ovms_images.py @@ -0,0 +1,174 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re + +from tests.functional.utils.environment_info import EnvironmentInfo +from tests.functional.constants.os_type import OsType, UBUNTU +from tests.functional.config import ( + docker_registry, + is_nginx_mtls, + ovms_cpp_docker_image, + ovms_image, + ovms_image_tag, + ovms_test_image_name, + force_use_ovms_image, +) +from tests.functional.constants.target_device import TargetDevice +from tests.functional.constants.ovms import CurrentOvmsType +from tests.functional.constants.ovms import CurrentTarget as ct + + +# should be checked periodically with OVMS Dockerfiles +GPU_INSTALL_DRIVER_VERSION = { + "redhat": "24.52.32224", + "ubuntu22": "24.39.31294", + "ubuntu24": "26.09.37435", +} + +GPU_INSTALL_SCRIPTS = { + OsType.Ubuntu22: ["install_ubuntu_gpu_drivers.sh", "install_va.sh"], + OsType.Ubuntu24: ["install_ubuntu_gpu_drivers.sh", "install_va.sh"], + OsType.Redhat: ["install_redhat_gpu_drivers.sh"], +} + + +class OvmsImages: + _os_type = None + _os_version = None + + @classmethod + def get_os_type(cls): + if cls._os_type is None: + if CurrentOvmsType.is_none_type(): + cls._os_type, _ = "", None + else: + cls._os_type, _ = cls._get_os_type_and_version() + return cls._os_type + + @classmethod + def get_os_version(cls): + _, os_verion = cls._get_os_type_and_version() + return os_verion + + @classmethod + def _get_os_type_and_version(cls): + if cls._os_type and cls._os_version: + return cls._os_type, cls._os_version + + # 'Ubuntu 20.04.2 LTS', 'CentOS Linux 7 (Core)', 'Red Hat Enterprise Linux 8.4 (Ootpa)' + os_name = EnvironmentInfo().get_os_distname() + os_name_lower = os_name.lower() + + for os_name_part in os_name_lower.split(): + if str(os_name_part[0]).isdigit(): + cls._os_version = os_name_part + break + + if "red hat" in os_name_lower: + cls._os_type = OsType.Redhat + elif "ubuntu 22" in os_name_lower: + cls._os_type = OsType.Ubuntu22 + elif "ubuntu 24" in os_name_lower: + cls._os_type = OsType.Ubuntu24 + else: + raise NotImplementedError() + + return cls._os_type, cls._os_version + + +NGINX = "nginx" +DEFAULT_OVMS_IMAGE_NAME = "openvino/model_server" +DEFAULT_OVMS_IMAGE_SUFFIXES = { + NGINX: "-nginx-mtls", + TargetDevice.GPU: "-gpu", + TargetDevice.NPU: "-gpu", +} + +DEFAULT_OVMS_IMAGE_TAG = { + OsType.Ubuntu22: "ubuntu22_main", + OsType.Ubuntu24: "ubuntu24_main", + OsType.Redhat: "redhat_main", +} + + +def calculate_ovms_image_suffix(target_device): + if is_nginx_mtls: + return DEFAULT_OVMS_IMAGE_SUFFIXES[NGINX] + elif ct.is_gpu_based_target(target_device) or ct.is_npu_target(): + return DEFAULT_OVMS_IMAGE_SUFFIXES[TargetDevice.GPU] + return "" + + +def prepare_general_os_list(base_os_list): + os_types = set(base_os_list) + if any(elem == OsType.Ubuntu22 or elem == OsType.Ubuntu24 for elem in base_os_list): + os_types.add(UBUNTU) + return os_types + + +def calculate_ovms_image_tag(image_tag_to_check, base_os, base_os_list): + os_types = prepare_general_os_list(base_os_list) + image_tag_base_os = [i for i in image_tag_to_check.split("_") if i in os_types] + if image_tag_base_os and image_tag_base_os[0] not in base_os: + return image_tag_to_check.replace(image_tag_base_os[0], base_os) + return image_tag_to_check + + +def calculate_ovms_image_name(target_device=None, base_os=OsType.Ubuntu22): + assert target_device, "Wrong target_device specified." + base_os_list = [val for key, val in vars(OsType).items() if not key.startswith("__")] + assert base_os in base_os_list, f"Wrong os specified: {base_os}" + assert not ( + is_nginx_mtls and target_device != TargetDevice.CPU + ), "nginx_mtls only available for CPU target_device" + + ct.target_device = target_device + + if force_use_ovms_image and ovms_image: + return ovms_image + elif ovms_image: + image_name = re.sub("|".join(DEFAULT_OVMS_IMAGE_SUFFIXES.values()), "", ovms_image.split(":")[0]) + image_tag = ovms_image.split(":")[1] + image_name = f"{image_name}{calculate_ovms_image_suffix(target_device)}" + image_tag = calculate_ovms_image_tag(image_tag, base_os, base_os_list) + else: + if ovms_cpp_docker_image: + image_name = ovms_cpp_docker_image + elif docker_registry is not None: + image_name = f"{docker_registry}/{DEFAULT_OVMS_IMAGE_NAME}" + else: + image_name = DEFAULT_OVMS_IMAGE_NAME + image_name = f"{image_name}{calculate_ovms_image_suffix(target_device)}" + image_tag = ovms_image_tag if ovms_image_tag else DEFAULT_OVMS_IMAGE_TAG[base_os] + image_tag = calculate_ovms_image_tag(image_tag, base_os, base_os_list) + + return f"{image_name}:{image_tag}" + + +def calculate_ovms_binary_image_name(ovms_image_name): + binary_image_name = f"{ovms_image_name}-binary" + return binary_image_name + + +def calculate_ovms_capi_image_name(ovms_image_name): + capi_image_name = f"{ovms_image_name}-capi" + return capi_image_name + + +def calculate_ovms_test_image_name(ovms_image_name): + test_image_name = ovms_test_image_name if ovms_test_image_name is not None else f"{ovms_image_name}-test" + return test_image_name diff --git a/tests/functional/constants/ovms_messages.py b/tests/functional/constants/ovms_messages.py new file mode 100644 index 0000000000..456b94d753 --- /dev/null +++ b/tests/functional/constants/ovms_messages.py @@ -0,0 +1,491 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re + +from tests.functional.utils.environment_info import CurrentOsInfo +from tests.functional.constants.os_type import OsType +from tests.functional.config import base_os +from tests.functional.constants.ovms import CurrentOvmsType + + +class OvmsMessages: + + MODEL_VERSION_NOT_FOUND = "Model with requested version is not found" + MODEL_NAME_OR_VERSION_NOT_FOUND = "Model with requested name and/or version is not found" + MODEL_VERSION_RETIRED = "Model with requested version is retired" + MODEL_NAME_VERSION_NOT_FOUND = "Model with requested name and/or version is not found" + MODEL_INVALID_INPUT_SHAPE = "Invalid input shape" + + MODEL_INVALID_RESHAPED_REQUEST = "Model could not be reshaped with requested shape" + MODEL_INVALID_NDARRAY = "Could not parse instance content. Not valid ndarray detected" + MODEL_INVALID_NUMBER_SHAPE_DIMS = "Invalid number of shape dimensions" + MODEL_UNEXPECTED_INPUT_TENSOR_ALIAS = "Unexpected input tensor alias" + MODEL_INVALID_BATCH_SIZE = "Wrong batch size parameter provided. Model batch size will be set to default." + MODEL_BATCH_SIZE_OUT_OF_RANGE = "Wrong batch size parameter provided. Model batch size will be set to default." + MODEL_INVALID_NIREQ_BIG_VALUE = "error: Nireq parameter too high - Exceeded allowed nireq value" + MODEL_INVALID_NIREQ_NEGATIVE_VALUE = "error: Nireq parameter cannot be negative value" + MODEL_INVALID_PRECISION = "Invalid precision" + MODEL_INVALID_INPUT_PRECISION = "Invalid input precision" + MODEL_INVALID_INPUT_PRECISION_EXTENDED = "Invalid input precision - Expected: {}; Actual: {}" + MODEL_MISSING_INPUT_SPECIFIC_NAME = "Missing input with specific name" + MODEL_RELOADING = "Will reload model: {}; version:" + MODEL_UNLOADING = "Will unload model: {}; version: {}" + MODEL_INPUT_NAME_MAPPING_NAME = "Input name: {}; mapping_name: {};" + MODEL_OUTPUT_NAME_MAPPING_NAME = "Output name: {}; mapping_name: {};" + MODEL_INPUT_SHAPE_RELOADING = "Input name: {}; mapping_name: {}; shape: {};" + MODEL_OUTPUT_SHAPE_RELOADING = "Output name: {}; mapping_name: {}; shape: {};" + MODEL_INPUT_SHAPE_RELOADING_DETAIL = "Input name: {}; mapping_name: {}; shape: {}; precision: {}; layout: {}" + MODEL_OUTPUT_SHAPE_RELOADING_DETAIL = "Output name: {}; mapping_name: {}; shape: {}; precision: {}; layout: {}" + MODEL_FAILED_TO_LOAD_INTO_DEVICE = "Cannot load network into target device" + MODEL_INVALID_INFERENCE_INPUT = "Invalid number of inputs - Expected: {}; Actual: {}" + MODEL_WITH_REQUESTED_VERSION_NOT_LOADED_YET = "Model with requested version is not loaded yet" + MODEL_STATUS_CHANGE = ( + 'STATUS CHANGE: Version {} of model {} status change. New status: ( "state": "{}", "error_code": "{}" )' + ) + + MODEL_LOADING = "Loading model: {}, version: {}, from path: {}" + PIPELINE_NOT_LOADED_YET = "Pipeline is not loaded yet" + PIPELINE_IS_RETIRED = "Pipeline is retired" + MEDIAPIPE_IS_RETIRED = "Mediapipe is retired" + MEDIAPIPE_EXECUTION_FAILED = "Mediapipe execution failed" + OVMS_PIPELINE_VALIDATION_FAILED_MSG = "Validation of pipeline: {} definition failed" + OVMS_PIPELINE_VALIDATION_FAILED_MSG_MISSING_MODEL = ( + "Validation of pipeline: {} definition failed. Missing model: {};" + ) + OVMS_PIPELINE_VALIDATION_PASS_MSG = "Pipeline: {} state changed to: AVAILABLE after handling: ValidationPassedEvent" + TWO_PIPELINES_WITH_THE_SAME_NAME = "Duplicated pipeline names" + PIPELINE_WITH_THE_SAME_NAME_AS_MODEL_NAME = "Pipeline name: {} is already occupied by model" + PIPELINE_STATE = "Pipeline: {} state changed to: " + PIPELINE_STARTED = ( + PIPELINE_STATE + "AVAILABLE " + ) # adding whitespace to avoid matching state AVAILABLE_REQUIRED_REVALIDATION + PIPELINE_UNLOADED = PIPELINE_STATE + "RETIRED" + PIPELINE_MULTIPLE_DEMULTIPLEXERS = "PipelineDefinition: {} has multiple demultiplexers with at least one dynamic." + PIPELINE_LOADING_PRECONDITION_FAILED = ( + "Pipeline: {} state changed to: LOADING_PRECONDITION_FAILED after handling: ValidationFailedEvent:" + ) + PIPELINE_INVALID_BATCH_SIZE_USED_WITH_DEMULTIPLEXER = ( + "Validation of pipeline: {} definition failed. Shape mismatch between: dependent node" + ) + PIPELINE_INVALID_DIMENSION_DEMULTIPLY_GREATER_THAN_BATCH_SIZE = ( + "Validation of pipeline: {} definition failed. Demultiply count: {} " + "of node: elastic_node does not match tensor first dimension value: {}" + ) + PIPELINE_INVALID_INPUT_SHAPE = ( + "Invalid input shape - Node: {} input: input Invalid shape - Expected: {}; Actual: {}" + ) + PIPELINE_REFERS_TO_INCORRECT_LIBRARY = "Pipeline refers to incorrect library" + + OVMS_BATCH_SIZE_WARNING = "[warn] Unexpected batch_size value" + OVMS_INVALID_BATCH_SIZE = "Invalid batch size - Expected: {}; Actual: {}" + OVMS_INVALID_INPUT_BATCH_SIZE = "Invalid input batch size - Expected: {}; Actual: {}" + OVMS_INVALID_INPUT_BATCH_SIZE_SHORT = "Invalid input batch size" + + OVMS_MODEL_LOADED = "Loaded model {}; version: {}; batch size: {}; No of InferRequests:" + OVMS_MODEL_LOADED_NIREQ = "Loaded model {}; version: {}; batch size: {}; No of InferRequests: {}" + OVMS_MODEL_LOADED_SHORT = "Loaded model" + + OVMS_ERROR_OCCURED_WHILE_LOADING_MODEL_GENERIC = "Error occurred while loading model:" + + OVMS_ERROR_COULD_NOT_FIND_FILE_FOR_MODEL = "Could not find file for model:" + OVMS_ERROR_INCORRECT_WEIGHTS_IN_BIN_FILE = "Error: Incorrect weights in bin file!" + + OVMS_MODEL_FAILED_TO_LOAD = ( + "Error occurred while loading model: {}; version: {}; error: Cannot load network into target device" + ) + + OVMS_SERVER_UNLOADED_MSG = ( + 'Version {} of model {} status change. New status: ( "state": "END", "error_code": "OK" )' + ) + + OVMS_SERVER_RUNNING_MSG = ( + 'Version {} of model {} status change. New status: ( "state": "AVAILABLE", "error_code": "OK" )' + ) + + OVMS_SERVICES_RUNNING_MSG = {"grpc": "Started gRPC server", "rest": "Started REST server"} + OVMS_STOPPING = "Stopping ovms" + + OVMS_LOG_LEVEL_DEBUG = "log level: DEBUG" + OVMS_REST_WORKERS = "Will start {} REST workers" + OVMS_MODEL_CACHE_ENABLED = "Model cache is enabled:" + OVMS_MODEL_CACHE_WITH_CUSTOM_LOADER = "Model: {} has allow cache set to true while using custom loader" + OVMS_DEBUG_CLI_PARAMETERS = "CLI parameters passed to ovms server" + + OVMS_PLUGIN_CONFIG = "OVMS set plugin settings key: {}; value: {};" + OVMS_TARGET_DEVICE = "Model: {}; version: {}; target device: {}" + COMPILED_MODEL_TARGET_DEVICE = "compiled model: {}; version: {}; target device: {}" + OV_NUMBER_STREAMS = "Number of OpenVINO streams:" + OV_NIREQ = "No of InferRequests:" + PLUGIN_CONFIG_FOR_DEVICE = "Plugin config for device: {}" + + ERROR_DURING_LOADING_INPUT_TENSORS = "Error during loading input tensors" + ERROR_TENSOR_INVALID_CONTENT_SIZE = ( + "Invalid content size of tensor proto - Expected: {} bytes; Actual: {} bytes; input name: {}" + ) + ERROR_EMPTY_RAW_INPUT_CONTENT = "Invalid message structure - raw_input_content is empty" + ERROR_ZERO_DIMENSION_IS_NOT_ALLOWED = "has zero dimension which is not allowed" + ERROR_FAILED_TO_SET_BATCH_SIZE = "Failed to set batch size to {}. Possible reasons are:" + ERROR_MODEL_VERSION_POLICY_UNSUPPORTED_KEY = "Model version policy contains unsupported key" + ERROR_NOT_VALID_JSON = "Error: The file is not valid json" + ERROR_MODEL_NOT_FOUND = "Model with requested name is not found" + ERROR_PIPELINE_NOT_FOUND = "Pipeline with requested name is not found" + ERROR_FAILED_TO_PARSE = "error parsing options: Argument '{}' failed to parse" if OsType.Windows in base_os \ + else "error parsing options: Argument ‘{}’ failed to parse" + ERROR_WRONG_BATCH_SIZE = "Wrong batch size parameter provided" + ERROR_FAILED_TO_PARSE_SHAPE_SHORT = "There was an error parsing shape" + ERROR_FAILED_TO_PARSE_SHAPE = "There was an error parsing shape {}" + ERROR_VALUE_HAS_TO_BE_GREATER_THAN_0 = "value has to be greater than 0" + ERROR_STRING_VAL_IS_EMPTY = "String val is empty" + ERROR_BINARY_DATA_SIZE_INVALID = "binary_data_size parameter is invalid and cannot be parsed" + + DETECTION_OUTPUT_HAS_ZERO_DIMENSION = "DetectionOutput has zero dimension which is not allowed" + BATCH_SIZE_IS_ZERO = "Batch size is zero" + DATA_BATCH_NOT_MATCH = "Data batch and filters rank do not match" + + WARNING_NO_VERSION_FOUND_FOR_MODEL = "No version found for model" + + if not CurrentOvmsType.is_none_type(): + ERROR_COUNT_SHOULD_BE_FROM_1 = f"count should be from 1 to CPU core count : {CurrentOsInfo.get_cpu_amount()}" + ERROR_REST_WORKERS_COUNT_SHOULD_BE_FROM_1 = f"rest workers {ERROR_COUNT_SHOULD_BE_FROM_1}" + ERROR_GRPC_WORKERS_COUNT_SHOULD_BE_FROM_1 = f"grpc_workers {ERROR_COUNT_SHOULD_BE_FROM_1}" + ERROR_WORKERS_COUNT_INFORMATION = "workers count should be from 2 to 10000" + ERROR_SHAPES_INCONSISTENT = "OV does not support reshaping model: {} with provided shape" + ERROR_WRONG_LAYER_NAME = "Config shape - {} not found in model" + ERROR_WRONG_SHAPE_FORMAT = "Error: The provided shape is in wrong format" + ERROR_WRONG_VERSION_POLICY_FORMAT = "Error: Model version policy is in wrong format" + ERROR_WRONG_VERSION_POLICY_PROPERTY = "Error: Model version policy contains unsupported key" + ERROR_CANNOT_PARSE_VERSION_POLICY_PROPERTY = "Couldn't parse model version policy" + ERROR_NOT_REGISTERED_TARGET_DEVICE = 'Device with "{}" name is not registered in the OpenVINO Runtime' + + ERROR_FAILED_TO_LOAD_LIBRARY = "error: Cannot load library" + ERROR_FAILED_TO_CREATE_PLUGIN = "error: Failed to create plugin" + + ERROR_WRONG_PLUGIN_CONFIG_FORMAT = "Error: Plugin config is in wrong format" + ERROR_PLUGIN_CONFIG_UNSUPPORTED_PROPERTY = "Plugin config key: {} not found in supported config keys for device:" + ERROR_INVALID_LOGGING_LEVEL = "log_level should be one of: TRACE, DEBUG, INFO, WARNING, ERROR" + ERROR_INVALID_JSON = "The file is not valid json" + ERROR_INVALID_JSON_STRUCTURE = "Invalid JSON structure. Missing instances in row format" + ERROR_MODEL_DOES_NOT_SUPPORT_BATCH_SIZE = "Model {} does not support setting batch size" + S3_WRONG_AUTHORIZATION = "Invalid or missing S3 credentials" + GS_WRONG_AUTHORIZATION = "Invalid or missing GCS credentials" + AZURE_WRONG_AUTHORIZATION = ( + "Unable to access path: Server failed to authenticate the request. Make sure the " + "value of Authorization header is formed correctly including the signature." + ) + ERROR_DIRECTORY_DOES_NOT_EXISTS = "Directory does not exist: {}" + ERROR_CANNOT_SERVE_ALL_MODELS = "Cannot serve all models" + ERROR_CANNOT_ANONYMOUS_RESHAPE_MULTIPLE_INPUTS = "Anonymous fixed shape is invalid for models with multiple inputs" + ERROR_COULD_NOT_PERFORM_RESHAPE = "Model reshape failed" + ERROR_WRONG_HTTP_METHOD = "Unsupported method" + ERROR_UNABLE_TO_ACCESS_PATH = "Unable to access path:" + INVALID_MODEL_PATH = "The provided base path is invalid or doesn't exists" + MODEL_CANNOT_BE_PARSED = "Model cannot be parsed" + NETWORK_NOT_READ_MODEL = "[ NETWORK_NOT_READ ] Unable to read the model" + UNABLE_TO_READ_THE_MODEL = "Unable to read the model: {}" + CANNOT_OPEN_LIBRARY = "Cannot open library:" + NOT_FOUND_MODEL_IN_PATH = "File not found or cannot open" + ERROR_INVALID_URL = "Invalid request URL" + WRONG_ENDPOINT = "Wrong endpoint" + BAD_REQUEST = "Bad Request" + NOT_IMPLEMENTED = "Not Implemented" + ERROR_NOT_FOUND = "Not Found" + ERROR_INPUT_DATA_TOO_BIG = "Input data is too big" + RESOURCE_EXHAUSTED = {"grpc": "Received message larger than max", "rest": "Request-URI Too Large"} + ABORTED = "Connection aborted" + OMVS_SERVICE_UNAVAILABLE = "Service unavailable" + ERROR_MAX_SEQUENCE_REACHED = "Max sequence number has been reached. Could not create new sequence." + ERROR_COULDNT_CHECK_DIRECTORY = "Couldn't check directory: {}" + NO_VERSION_FOUND_FOR_MODEL = "No version found for model in path: {}" + ERROR_FAILED_TO_CONNECT_TO_ANY_PROXY_ENDPOINT = "Failed to connect to any resolved proxy endpoint" + ERROR_TERMINATE_CALLED = "terminate called after throwing an instance of" + STD_SYSTEM_ERROR = "std::system_error" + + ERROR_CFG_JSON_SCHEMA = "JSON schema parse error:#{}" + ERROR_CFG_NOT_VALID = "Configuration file is not a valid JSON file" + ERROR_CFG_NOT_FOUND = "Configuration file is invalid {}" + ERROR_CFG_NOT_VALID_FORMAT = "Configuration file is not in valid configuration format" + ERROR_CFG_KEY_ERROR = "Keyword:{} Key: #{}" + USE_CONFIG_PATH_OR_MODEL_PATH_WITH_SPARE_MODEL = "Use either config_path or model_path with model_name" + USE_CONFIG_PATH_WITHOUT_MODEL = "Use config_path or model_path with model_name" + ERROR_LOADING_MODEL = "Error occurred while loading model: {}" + ERROR_LOADING_MODEL_INTERNAL_SERVER_ERROR = ( + "Error occurred while loading model: {}; version: {}; error: Internal server error" + ) + ERROR_RELOADING_MODEL_INTERNAL_SERVER_ERROR = ( + "Error occurred while reloading model: {}; versions; error: Internal server error" + ) + ERROR_CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE = "Cannot compile model into target device" + ERROR_CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE_OUT_OF_MEMORY = ( + "Cannot compile model into target device; error: Failed to allocate " + "graph: NC_OUT_OF_MEMORY; model: {}; version: {}; device: {}" + ) + ERROR_EXCEPTION_FROM_SRC_INFERENCE = "Error: Exception from src/inference/src" + + CUSTOM_LOADER_READING_CONFIGURATION = "Reading Custom Loader: {} configuration" + CUSTOM_LOADER_NOT_FOUND = "Specified custom loader {} not found." + CUSTOM_LOADER_TO_BE_USED = "Custom Loader to be used : {}" + CUSTOM_LOADER_LOOKING_FOR_LOADER = "Looking for loader {} in loaders list" + CUSTOM_LOADER_INVALID_CUSTOM_LOADER_OPTIONS = "Invalid custom loader options" + + STATEFUL_MODEL_REGISTERED = "Model: {}, version: {}, has been successfully registered in sequence cleaner" + ERROR_SEQUENCE_ID_NOT_EXIST = "Sequence with provided ID does not exist" + DEFAULT_MODEL_VERSION_CHANGED = "Updating default version for model: {}, from: {}" + + ERROR_LOADING_PRECONDITION_FAILED = "LOADING_PRECONDITION_FAILED after handling: ValidationFailedEvent" + ERROR_INVALID_LAYOUT_NHWC = "Received binary image input but resource not configured to accept NHWC layout" + ERROR_BINARY_INPUTS_CONVERSION_FAILED = "Binary inputs conversion failed." + ERROR_BINARY_INPUTS_NATIVE_FILE_FORMAT_CONVERSION = "Input native file format conversion failed" + ERROR_BINARY_INPUTS_CORRUPT_JPEG_DATA = "Corrupt JPEG data" + + ERROR_INTERPOLATE_NODE_IS_NOT_SUPPORTED = "Interpolate node is not supported:" + + ERROR_EXCEPTION_CATCH = "Exception catch:" + + CPU_EXTENSION_LOADING_CUSTOM_CPU_EXT = "Loading custom CPU extension from {}" + CPU_EXTENSION_LOADED = "Custom CPU extention loaded. Adding it." + CPU_EXTENSION_ADDED = "Extension added." + + ERROR_CPU_EXTENSION_WILL_NOW_TERMINATE = "- will now terminate." + ERROR_CPU_EXTENSION_LOADING_FAILED = "Custom CPU extension loading has failed! Reason: {}" + + ERROR_CPU_EXTENSION_FILE_NOT_EXISTS = ( + "File path provided as an --cpu_extension parameter does not exist in the filesystem: {}" + ) + + ERROR_COULDNT_START_MODEL_MANAGER = "Couldn't start model manager" + + NGINX_STARTED_WITH_PID = "Nginx PID: " + + ERROR_METRICS_REST_PORT_MISSING_CFG = { + False: "rest_port setting is missing, metrics are enabled on rest port", + True: "Error: Missing rest_port parameter in server CLI", + } + ERROR_METRICS_NOT_ENABLED = "metrics_enable setting is missing, required when metrics_list is provided" + ERROR_METRICS_UNABLE_TO_LOAD_SETTINGS = "Couldn't load metrics settings" + ERROR_METRICS_INVALID_METRICS = "Error: Invalid name in metrics_list" + ERROR_METRICS_NOT_SUPPORTED_METRICS = "Metrics family name not supported: {}" + + OPERATOR_UNKNOWN_FIELD = 'strict decoding error: unknown field "{}"' + OPERATOR_WARNING_UNKNOWN_FIELD = 'Warning: unknown field "spec.models_settings.model_config"' + + TRACE_CHECK_CHANGE = "Checking if something changed with model versions" + TRACE_CURRENTLY_REGISTERED_MODEL = "Currently registered model: {}" + OVMS_LOG_LEVEL_TRACE = "log level: TRACE" + OVMS_LOG_LEVEL = "logLevel={}" + + OVMS_IGNORED_BATCH = "Both shape and batch size have been defined. Batch size parameter will be ignored" + + UNKNOWN_ERROR = "Unknown error" + FUNCTIONALITY_NOT_IMPLEMENTED = "Functionality not implemented" + DEMULTIPLICATION_OF_STRINGS_UNSUPPORTED = "Demultiplication of strings in unsupported" + + # CAPI: + SERVABLE_MANAGER_MODULE_STARTED = "ServableManagerModule started" + CLEANER_THREAD_STARTED = "Started cleaner thread" + MODEL_MANAGER_STARTED = "Started model manager thread" + SERVABLE_MANAGER_MODULE_SHUTDOWN = "ServableManagerModule shutdown" + CLEANER_THREAD_SHUTDOWN = "Shutdown cleaner thread" + MODEL_MANAGER_SHUTDOWN = "Shutdown model manager" + + CAPI_STARTING_OVMS_SERVER = "Starting OVMS CAPI server" + CAPI_STARTED_OVMS_SERVER = "Started OVMS Server:" + + CYTHON_LIBRARY_INIT_MSG = "[1/1] Cythonizing {}" + CAPI_VERSION = "C-API version: {}.{}" + + CAPI_MODEL_REQUEST = "Processing C-API inference request for servable: {}; version: {}" + CAPI_TENSOR_ADDED = "Successfully added tensor: {}" + CAPI_TOTAL_REQUEST_PROCESSING_TIME = "Total C-API req processing time:" + + MEDIAPIPE_GRAPH_DEFINITION_FAILED = "Trying to parse mediapipe graph definition: {} failed" + MEDIAPIPE_ERROR_PARSING_GRAPH_NO_FIELD = ( + 'Message type "mediapipe.CalculatorGraphConfig.Node" has no field named "{}"' + ) + MEDIAPIPE_FAILED_TO_OPEN_GRAPH_SHORT = "Failed to open mediapipe graph definition:" + MEDIAPIPE_FAILED_TO_OPEN_GRAPH = "Failed to open mediapipe graph definition: {}, file: {}" + MEDIAPIPE_PIPELINE_VALIDATION_PASS_MSG = ( + "Mediapipe: {} state changed to: AVAILABLE after handling: ValidationPassedEvent" + ) + MEDIAPIPE_PIPELINE_VALIDATION_PASS_MSG_SHORT = "state changed to: AVAILABLE after handling: ValidationPassedEvent" + MEDIAPIPE_UNLOADED = "Mediapipe: {} state changed to: RETIRED" + MEDIAPIPE_LOADING_PRECONDITION_FAILED = ( + "Mediapipe: {} state changed to: LOADING_PRECONDITION_FAILED after handling: ValidationFailedEvent:" + ) + LOADING_MEDIAPIPE_SUBCONFIG_FAILED = "Loading Mediapipe {} models from subconfig {} failed." + MEDIAPIPE_OCCUPIED = "Mediapipe graph name: {} is already occupied by model or pipeline" + MEDIAPIPE_NO_SUPPORT_FOR_TFLITETENSOR_SERIALIZATION = ( + "There is no support for TfLiteTensor deserialization & serialization" + ) + MEDIAPIPE_BASE_PATH_NOT_DEFINED = ( + "base_path not defined in config so it will be set to default based on main config directory" + ) + MEDIAPIPE_GRAPH_PATH_NOT_DEFINED = ( + "graph_path not defined in config so it will be set to default based on base_path and graph name" + ) + MEDIAPIPE_SUBCONFIG_DEFAULT = "subconfig.json provided for graph: {}" + MEDIAPIPE_INTERNAL_ERROR_FAILED = ( + "Mediapipe execution failed. MP status - INTERNAL: CalculatorGraph::Run() failed:" + ) + MEDIAPIPE_NOT_SUPPORTED_PRECISION_FOR_MP = "Not supported precision for Mediapipe tensor deserialization" + MEDIAPIPE_IS_NOT_LOADED_YET = "Mediapipe is not loaded yet" + MEDIAPIPE_GRAPH_NOT_FOUND = "Mediapipe graph definition with requested name is not found" + MEDIAPIPE_ERROR_PARSING_TEXT_FORMAT = "Error parsing text-format mediapipe.CalculatorGraphConfig:" + MEDIAPIPE_EMPTY_GRAPH = "Trying to parse empty mediapipe graph definition: {} failed" + MEDIAPIPE_FAILED_TO_ADD_PACKET = "Failed to add packet to mediapipe graph input stream" + MEDIAPIPE_UNABLE_TO_ATTACH_OBSERVER_TO_OUTPUT_STREAM = 'Unable to attach observer to output stream "{}"' + MEDIAPIPE_UNEXPECTED_INPUT = "Unexpected input - {} is unexpected" + MEDIAPIPE_FAILED_TO_LOAD_MEDIAPIPE_GRAPH_DEFINITION = "Failed to open mediapipe graph definition" + MEDIAPIPE_UNEXPECTED_INPUT_NAME = "Unexpected input name" + MEDIAPIPE_CALCULATOR_FAILED_TO_LOAD_MODEL = "OpenVINOModelServerSessionCalculator failed to load the model" + MEDIAPIPE_MESSAGE_CONTAINS_OV_GENAI_IMAGE_TAG = "Message contains restricted tag" + MEDIAPIPE_REQUEST_PROCESSING_FAILED = "Request processing failed, check its correctness" + MEDIAPIPE_URL_IMAGE_INVALID_ARGUMENT_FILESYSTEM_DISABLED = "Loading images from local filesystem is disabled" + MEDIAPIPE_URL_IMAGE_PARSING_FAILED = "Image parsing failed" + MEDIAPIPE_CANT_OPEN_URL_IMAGE = "parsing failed: can\\'t fopen" + MEDIAPIPE_UNKNOWN_IMAGE_TYPE = "parsing failed: unknown image type" + MEDIAPIPE_URL_DOES_NOT_MATCH_ALLOWED_DOMAIN = \ + "Given url does not match any allowed domain from allowed_media_domains" + + PYTHON_NODE_HANDLER_PATH_NOT_EXISTS = "Python node handler_path: {} does not exist." + PYTHON_NODE_FAILED_TO_PROCESS_GRAPH = "Failed to process python node graph python_model" + PYTHON_NODE_FAILED_TO_EXECUTE_GRAPH = "Failed to execute mediapipe graph: python_model since it is not available" + PYTHON_EXECUTOR_MISSING_REQUIRED_FIELDS = ( + 'Value of type "mediapipe.PythonExecutorCalculatorOptions" stored in google.protobuf.Any has missing ' + "required fields" + ) + PYTHON_NODE_FAILED_TO_PROCESS_CUSTOM_NODE_FILE = "Failed to process python node file {} : {}" + + PYTHON_NODE_ERROR_DURING_GRAPH_EXECUTION = ( + 'Calculator::Process() for node "{}" failed: Error occurred during graph execution' + ) + PYTHON_NODE_ERROR_DURING_NODE_EXECUTION = "Error occurred during node {} execution: {}" + PYTHON_NODE_ERROR_DURING_INITIALIZATION = "Error during python node initialization for handler_path: {} - {}" + PYTHON_NODE_ERROR_DURING_INITIALIZATION_EMPTY_MODULE_NAME = ( + "Error during python node initialization for handler_path: - ValueError: Empty module name" + ) + PYTHON_NODE_ERROR_DURING_INITIALIZATION_EXECUTE_METHOD = ( + "Error during python node initialization. " + "OvmsPythonModel class defined in {} does not implement execute method." + ) + PYTHON_NODE_FAILED_TO_PROCESS_FINALIZE_METHOD = "Failed to process python node finalize method. {}" + PYTHON_NODE_NAME_MISSING_IN_GRAPH = "Python node name is missing in graph: {}" + PYTHON_MEDIAPIPE_FAILED = "Mediapipe graph: {} initialization failed with message: {}" + PYTHON_INPUT_SIDE_PACKET_FAILED = "For input side packets ValidatePacketTypeSet failed" + PYTHON_FAILED_TO_GET_TAG = 'Failed to get tag "PYTHON_NODE_RESOURCES"' + PYTHON_INPUTS_EMPTY = "!cc->Inputs().GetTags().empty()" + PYTHON_OUTPUTS_EMPTY = "!cc->Outputs().GetTags().empty()" + PYTHON_SETTING_INPUT_STREAM = "setting input stream: {} packet type: OVMS_PY_TENSOR from: {}:{}" + PYTHON_SETTING_OUTPUT_STREAM = "setting output stream: {} packet type: OVMS_PY_TENSOR from: {}:{}" + PYTHON_INVALID_INPUT_STREAM = ( + 'Input Stream "{}" for node with sorted index 0 name upper_text does not have a corresponding output stream' + ) + PYTHON_INVALID_OUTPUT_STREAM = ( + 'Output Stream "{}" for node with sorted index 0 name upper_text does not have a corresponding output stream' + ) + PYTHON_OVMS_PY_TENSOR_REASSIGNED = 'tag "{}" index 0 already had a name "{}" but is being reassigned a name "{}"' + PYTHON_DONE_EXECUTION = "Graph {}: Done execution" + PACKET_TIMESTAMP_MISMATCH = 'Packet timestamp mismatch on a calculator receiving from stream "{}".' + PACKET_TIMESTAMP_MISMATCH_SHORT = "Packet timestamp mismatch on a calculator" + PACKET_TIMESTAMP_EXPECTED = "Current minimum expected timestamp is {} but received {}." + PACKET_TIMESTAMP_INPUT_STREAM_HANDLER = ( + "Are you using a custom InputStreamHandler? Note that some InputStreamHandlers allow timestamps that are not " + "strictly monotonically increasing. See for example the ImmediateInputStreamHandler class comment." + ) + PYTHON_NODE_RESOURCE_EXHAUSTED = "RESOURCE_EXHAUSTED: CalculatorGraph::Run() failed:" + PYTHON_NODE_ERROR_ALREADY_PROCESSING_DATA = ( + 'Calculator::Process() for node "{}" failed: Node is already processing data. ' + 'Create new stream for another request.' + ) + + CHAT_TEMPLATE_NOT_LOADED = ( + "Warning: Chat template has not been loaded properly. Servable will not respond to /chat/completions endpoint" + ) + CHAT_TEMPLATE_LOADING_FAILED = "Chat template loading failed with an unexpected error" # only in DEBUG mode + CHAT_TEMPLATE_SYNTAX_ERROR = "Chat template loading failed with error: TemplateSyntaxError" + CHAT_TEMPLATE_TYPE_ERROR = "Chat template loading failed with error: TypeError" + CHAT_TEMPLATE_CALCULATOR_ERROR = ( + 'Calculator::Process() for node "LLMExecutor" failed: ' + 'Error: Chat template not loaded correctly, so it cannot be applied' + ) + CHAT_TEMPLATE_CALCULATOR_ERROR_SHORT = 'Calculator::Process() for node "LLMExecutor" failed' + LLM_REQUEST_PROCESSING_FAILED = "Request processing failed, check its correctness." + LLM_MAX_LENGTH_MUST_BE_GREATER_THAN_PROMPT_TOKENS = \ + r"\'max_length\' must be greater than the number of prompt tokens" + LLM_ALL_REQUESTS_SCHEDULED_REQUESTS = "All requests: {}; Scheduled requests: {};" + LLM_ALL_REQUESTS_SCHEDULED_REQUESTS_NPU = "All requests: {};" + LLM_ALL_REQUESTS_SCHEDULED_REQUESTS_CACHE_USAGE = \ + "All requests: {}; Scheduled requests: {}; Cache type: static, cache usage: {}%" + LLM_CACHE_USAGE = "Cache usage {}%;" + LLM_CALCULATOR_CLOSE = "LLMCalculator [Node: LLMExecutor ] Close" + LLM_STREAM_GENERATION_CANCELLED = "graph wait until done CANCELLED" + LLM_UNARY_GENERATION_CANCELLED = "Mediapipe execution failed. MP status - CANCELLED" + LLM_STREAM_GENERATION_DONE = "data: [DONE]" + LLM_UNARY_GENERATION_DONE = "Complete unary response" + MEDIAPIPE_SINGLE_MESSAGE_ONLY = "This servable accepts only single message requests" + PROMPT_IS_NOT_A_STRING = "prompt is not a string" + ERROR_TYPE_MUST_BE_STRING_BUT_ITS_OBJECT = "type must be string, but is object" + ERROR_TYPE_MUST_BE_STRING_BUT_ITS_NUMBER = "type must be string, but is number" + FINAL_PROMPT_EMPTY = "Final prompt after applying chat template is empty" + ERROR_UNABLE_TO_CAST_PYTHON_INSTANCE = "Chat template loading failed with error: Unable to cast Python instance" + ERROR_UNSUPPORTED_CHAT_TEMPLATE_FORMAT = "Unsupported chat_template format in file" + + EMPTY_RESPONSE_CONTENT = "Empty response content" + FINISH_REASON_NULL = '"finish_reason":null' + + GENERAL_ERROR = "[error]" + HF_IMPORT_MODEL_DOWNLOADED = "Model: {} downloaded" + HF_IMPORT_GRAPH_CREATED = "Graph: graph.pbtxt created" + + CONFIG_MANIPULATION_ERASING_MODEL = "Erasing model from config: {}" + CONFIG_MANIPULATION_MODEL_TO_BE_ADDED = "Model to be added to configuration file" + CONFIG_MANIPULATION_MODEL_TO_BE_REMOVED_FOUND = "Model to be removed found in configuration file" + CONFIG_MANIPULATION_CONFIG_UPDATED = "Config updated: {}" + + MODELS_ENDPOINT_MODEL_NOT_FOUND = "Model not found" + + PULL_ERROR_FAILURE_WHEN_RECEIVING_DATA_FROM_PEER = \ + "[ERROR] curl_easy_perform() failed: Failure when receiving data from the peer" + PULL_ERROR_COULD_NOT_RESOLVE_PROXY_NAME = "[ERROR] curl_easy_perform() failed: Could not resolve proxy name" + PULL_ERROR_TIMEOUT_WAS_REACHED = "[ERROR] curl_easy_perform() failed: Timeout was reached" + PULL_ERROR_COULD_NOT_CONNECT_TO_SERVER = "[ERROR] curl_easy_perform() failed: Couldn't connect to server" + PULL_ERROR_LFS_DOWNLOAD_FAILED = "[ERROR] LFS download failed" + PULL_STATUS_UNCLEAN = "Status: Unclean status detected in libgit2 repository path" + PULL_STATUS_FAILED = "Status: Failed in libgit2 execution of status method" + PULL_INFO_RESUME_DOWNLOAD = "[INFO] curl_easy_perform() trying to resume file download" + + # Audio endpoints + AUDIO_NOT_VALID_WAV_NOR_MP3 = "Received input file is not valid wav nor mp3 audio file" + AUDIO_FILE_PARSING_FAILS = "File parsing fails" + AUDIO_INVALID_LANGUAGE_CODE = "Invalid language code." + AUDIO_TEMPERATURE_OUT_OF_RANGE = "Temperature out of range(0.0, 2.0)" + AUDIO_INVALID_TIMESTAMP_GRANULARITIES = 'Invalid timestamp_granularities type. Allowed types: "segment", "word"' + AUDIO_VOICE_NOT_AVAILABLE = "Requested voice not available" + AUDIO_STREAMING_NOT_SUPPORTED = "streaming is not supported" + AUDIO_WORD_TIMESTAMPS_NOT_SUPPORTED = "Word timestamps not supported for this model" + + +class OvmsMessagesRegex: + # Regex for capturing OVMS log message timestamp in format "[%Y-%m-%d %H:%M:%S.%f]" + # ie.: [2021-11-23 12:10:20.155][1][serving][info][server.cpp:106] OpenVINO Model Server 2022.1.e866798 + OVMV_LOG_TIMESTAMP_RE = re.compile(r"\[(\d{4}\-\d+?\-\d+? \d+?:\d+?:\d+?.\d+?)\]") + + STATUS_CHANGE_RE = re.compile("STATUS CHANGE:") + + +class OvmsTestCapiCythonMessages: + OVMS_CAPI_STARTED_SERVER = "Started OVMS Server:" + OVMS_CAPI_STARTED_STOPPED = "Server stopped" diff --git a/tests/functional/constants/ovms_openai.py b/tests/functional/constants/ovms_openai.py new file mode 100644 index 0000000000..cc32158571 --- /dev/null +++ b/tests/functional/constants/ovms_openai.py @@ -0,0 +1,307 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from dataclasses import dataclass +from typing import Union + +from tests.functional.utils.inference.serving.openai import ( + OpenAIChatCompletionsRequestParams, + OpenAICommonCompletionsRequestParams, + OpenAICommonImagesRequestParams, + OpenAICompletionsRequestParams, + OpenAIEmbeddingsRequestParams, + OpenAIImagesEditsRequestParams, + OpenAIImagesGenerationsRequestParams, + OpenAIResponsesRequestParams, + OpenAIRequestParams, + OpenAIWrapper, + OpenAIAudioTranscriptionsRequestParams, + OpenAIAudioTranslationsRequestParams, + OpenAIAudioSpeechRequestParams, +) + + +@dataclass +class MaxTokensValues: + DEFAULT = 30 + DEFAULT_COMPARE_JINJA = 50 + VLM_COMPARE = 50 + COMPARE = 100 + COMPARE_LONG = 1000 + CACHE_OVERLOAD = 100000 + SHORT = 10 + LONG = 10000 + TOOLS = 2048 + + +class TemperatureValues: + TEST_DEFAULT = 0 + OVMS_DEFAULT = 1 + + +class EncodingFormatValues: + FLOAT = "float" + BASE64 = "base64" + + @classmethod + def values(cls): + values = [ + value for key, value in vars(cls).items() if not key.startswith("__") and not isinstance(value, classmethod) + ] + return values + + +class MaxPromptLenValues: + NPU_DEFAULT = 10240 + + +class ImagesRequestParamsValues: + N_DEFAULT = 1 + NUM_INFERENCE_STEPS_DEFAULT = 50 + NUM_INFERENCE_STEPS_FLUX = 3 + NUM_INFERENCE_STEPS_SDXL = 25 + NUM_INFERENCE_STEPS_DREAMLIKE_INPAINTING = 100 + NUM_INFERENCE_STEPS_NPU = 1 + RNG_SEED_DEFAULT = 42 + SIZE_DEFAULT = "512x512" + SIZE_EDITS_DEFAULT = "336x224" + STRENGTH_DEFAULT = 0.7 + MIXED_NPU_DEVICE = "NPU NPU GPU" + + +class ResponseFormatValues: + DEFAULT = { + "type": "json_schema", + "json_schema": { + "description": "city and country sch", + "schema": { + "properties": { + "city": { + "title": "City", + "type": "string", + }, + "country": { + "title": "Country", + "type": "string", + }, + }, + "required": ["city", "country"], + "additionalProperties": False, + }, + "name": "schema_name", + "strict": False, + }, + } + + +@dataclass +class OvmsCommonRequestParams: + + def prepare_dict_with_extra_body(self, openai_base_classes: list): + request_params_dict = {"extra_body": {}} + for key, value in vars(self).items(): + if value is not None: + if any(key in vars(openai_base_class) for openai_base_class in openai_base_classes): + request_params_dict[key] = value + else: + request_params_dict["extra_body"][key] = value + return request_params_dict + + +@dataclass +class OvmsCommonCompletionsRequestParams(OpenAICommonCompletionsRequestParams, OvmsCommonRequestParams): + ignore_eos: bool = None + length_penalty: float = None + include_stop_str_in_output: bool = None + top_k: int = None + repetition_penalty: float = None + num_assistant_tokens: int = None + assistant_confidence_threshold: float = None + max_ngram_size: int = None + + def set_default_values(self, **kwargs): + super().set_default_values(**kwargs) + self.ignore_eos = False + self.length_penalty = 1.0 + self.repetition_penalty = 1.0 + + +@dataclass +class OvmsChatCompletionsRequestParams(OvmsCommonCompletionsRequestParams, OpenAIChatCompletionsRequestParams): + # Some request parameters are supported by OVMS but not by OpenAI API. For full list go to: + # https://github.com/openvinotoolkit/model_server/blob/main/docs/model_server_rest_api_chat.md + best_of: int = None + tools: list = None + tool_choice: Union[dict, str] = None + chat_template_kwargs: dict = None + + def prepare_dict(self, set_null_values=False, use_extra_body=True): + if use_extra_body: + return self.prepare_dict_with_extra_body( + [OpenAICommonCompletionsRequestParams, OpenAIChatCompletionsRequestParams], + ) + else: + return super().prepare_dict(set_null_values=set_null_values) + + def set_default_values(self, **kwargs): + super().set_default_values(**kwargs) + if not self.stream: + self.best_of = 1 + + +@dataclass +class OvmsCompletionsRequestParams(OvmsCommonCompletionsRequestParams, OpenAICompletionsRequestParams): + # Some of the request parameters are supported by OVMS but not by OpenAI API. For full list go to: + # https://github.com/openvinotoolkit/model_server/blob/main/docs/model_server_rest_api_completions.md + + def prepare_dict(self, set_null_values=False, use_extra_body=True): + if use_extra_body: + return self.prepare_dict_with_extra_body( + [OpenAICommonCompletionsRequestParams, OpenAICompletionsRequestParams], + ) + else: + return super().prepare_dict(set_null_values=set_null_values) + + +@dataclass +class OvmsResponsesRequestParams(OpenAIResponsesRequestParams, OvmsCommonRequestParams): + ignore_eos: bool = None + stop: Union[str, list] = None + top_k: int = None + include_stop_str_in_output: bool = None + repetition_penalty: float = None + frequency_penalty: float = None + presence_penalty: float = None + seed: int = None + best_of: int = None + length_penalty: float = None + n: int = None + num_assistant_tokens: int = None + assistant_confidence_threshold: float = None + max_ngram_size: int = None + + def set_default_values(self, **kwargs): + super().set_default_values(**kwargs) + self.ignore_eos = False + self.stop = "," + + def prepare_dict(self, set_null_values=False, use_extra_body=True): + if use_extra_body: + return self.prepare_dict_with_extra_body([OpenAIResponsesRequestParams]) + else: + return super().prepare_dict(set_null_values=set_null_values) + + +@dataclass +class OvmsCommonImagesRequestParams(OpenAICommonImagesRequestParams, OvmsCommonRequestParams): + prompt_2: str = None + prompt_3: str = None + negative_prompt: str = None + negative_prompt_2: str = None + negative_prompt_3: str = None + num_images_per_prompt: int = None + num_inference_steps: int = None + guidance_scale: float = None + rng_seed: int = None + max_sequence_length: int = None + height: int = None + width: int = None + + def set_default_values(self, **kwargs): + super().set_default_values(**kwargs) + self.num_inference_steps = 50 + self.rng_seed = 42 + + +@dataclass +class OvmsImagesGenerationsRequestParams(OvmsCommonImagesRequestParams, OpenAIImagesGenerationsRequestParams): + + def prepare_dict(self, set_null_values=False, use_extra_body=True): + if use_extra_body: + return self.prepare_dict_with_extra_body( + [OpenAICommonImagesRequestParams, OpenAIImagesGenerationsRequestParams], + ) + else: + return super().prepare_dict(set_null_values=set_null_values) + + +@dataclass +class OvmsImagesEditsRequestParams(OvmsCommonImagesRequestParams, OpenAIImagesEditsRequestParams): + strength: float = None + + def prepare_dict(self, set_null_values=False, use_extra_body=True): + if use_extra_body: + return self.prepare_dict_with_extra_body( + [OpenAICommonImagesRequestParams, OpenAIImagesEditsRequestParams], + ) + else: + return super().prepare_dict(set_null_values=set_null_values) + + def set_default_values(self, **kwargs): + super().set_default_values(**kwargs) + self.strength = 0.7 + + +@dataclass +class OvmsEmbeddingsRequestParams(OpenAIEmbeddingsRequestParams): + pass + + +@dataclass +class OvmsAudioTranscriptionsRequestParams(OpenAIAudioTranscriptionsRequestParams): + pass + + +@dataclass +class OvmsAudioTranslationsRequestParams(OpenAIAudioTranslationsRequestParams): + pass + + +@dataclass +class OvmsAudioSpeechRequestParams(OpenAIAudioSpeechRequestParams): + pass + + +class OvmsOpenAIRequestParamsBuilder: + + def __init__(self, endpoint, **kwargs): + self.endpoint = endpoint + if self.endpoint == OpenAIWrapper.CHAT_COMPLETIONS: + self.request_params = OvmsChatCompletionsRequestParams(**kwargs) + elif self.endpoint == OpenAIWrapper.COMPLETIONS: + self.request_params = OvmsCompletionsRequestParams(**kwargs) + elif self.endpoint == OpenAIWrapper.RESPONSES: + # translate chat/completions parameters + kwargs["max_output_tokens"] = kwargs.pop("max_tokens", None) + self.request_params = OvmsResponsesRequestParams(**kwargs) + elif self.endpoint == OpenAIWrapper.EMBEDDINGS: + self.request_params = OvmsEmbeddingsRequestParams() + elif self.endpoint == OpenAIWrapper.IMAGES_GENERATIONS: + self.request_params = OvmsImagesGenerationsRequestParams(**kwargs) + elif self.endpoint == OpenAIWrapper.IMAGES_EDITS: + self.request_params = OvmsImagesEditsRequestParams(**kwargs) + elif self.endpoint == OpenAIWrapper.MODELS_LIST: + self.request_params = OpenAIRequestParams() # no request parameters available + elif self.endpoint == OpenAIWrapper.MODELS_RETRIEVE: + self.request_params = OpenAIRequestParams() # no request parameters available + elif self.endpoint == OpenAIWrapper.AUDIO_TRANSCRIPTIONS: + self.request_params = OvmsAudioTranscriptionsRequestParams() + elif self.endpoint == OpenAIWrapper.AUDIO_TRANSLATIONS: + self.request_params = OvmsAudioTranslationsRequestParams() + elif self.endpoint == OpenAIWrapper.AUDIO_SPEECH: + self.request_params = OvmsAudioSpeechRequestParams() + else: + raise NotImplementedError diff --git a/tests/functional/constants/ovms_type.py b/tests/functional/constants/ovms_type.py new file mode 100644 index 0000000000..02a1ea9873 --- /dev/null +++ b/tests/functional/constants/ovms_type.py @@ -0,0 +1,71 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.constants.os_type import OsType +from tests.functional.constants.target_device import TargetDevice + + +class OvmsType: + NONE = "NONE" + DOCKER = "DOCKER" + DOCKER_CMD_LINE = "DOCKER_CMD_LINE" + BINARY = "BINARY" + BINARY_DOCKER = "BINARY_DOCKER" + CAPI = "CAPI" + CAPI_DOCKER = "CAPI_DOCKER" + KUBERNETES = "KUBERNETES" # legacy + + +# https://github.com/openvinotoolkit/model_server/blob/main/docs/deploying_server.md#deploying-model-server-on-baremetal-without-container +OVMS_BINARY_DEPENDENCIES = { + OsType.Ubuntu22: "libcurl4-openssl-dev libpugixml1v5 libtbb12 libxml2", + OsType.Ubuntu24: "libcurl4-openssl-dev libpugixml1v5 libtbb12 libxml2", + OsType.Redhat: "https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/tbb-2020.3-8.el9.x86_64.rpm", +} + + +OVMS_BINARY_PACKAGE_NAME = "ovms" +OVMS_BINARY_PACKAGE_EXTENSIONS = (".tar.gz", ".zip") + +OVMS_CAPI_DEPENDENCIES = OVMS_BINARY_DEPENDENCIES + +OVMS_CAPI_UBUNTU_CPU_TOOLS = "build-essential" +OVMS_CAPI_UBUNTU_GPU_TOOLS = "clinfo curl" +OVMS_CAPI_UBUNTU_OPENCL_TOOLS = "opencl-clhpp-headers opencl-c-headers intel-opencl-icd" +OVMS_CAPI_UBUNTU_VA_TOOLS = "gpg" + +OVMS_CAPI_TOOLS_DEPENDENCIES = { + TargetDevice.CPU: { + OsType.Ubuntu24: OVMS_CAPI_UBUNTU_CPU_TOOLS, + OsType.Ubuntu22: OVMS_CAPI_UBUNTU_CPU_TOOLS, + OsType.Redhat: "", + }, + TargetDevice.GPU: { + OsType.Ubuntu24: " ".join([ + OVMS_CAPI_UBUNTU_CPU_TOOLS, + OVMS_CAPI_UBUNTU_GPU_TOOLS, + OVMS_CAPI_UBUNTU_OPENCL_TOOLS, + OVMS_CAPI_UBUNTU_VA_TOOLS, + ]), + OsType.Ubuntu22: " ".join([ + OVMS_CAPI_UBUNTU_CPU_TOOLS, + OVMS_CAPI_UBUNTU_GPU_TOOLS, + OVMS_CAPI_UBUNTU_OPENCL_TOOLS, + OVMS_CAPI_UBUNTU_VA_TOOLS, + ]), + OsType.Redhat: "", + } +} diff --git a/tests/functional/constants/ovsa.py b/tests/functional/constants/ovsa.py new file mode 100644 index 0000000000..eaf279c004 --- /dev/null +++ b/tests/functional/constants/ovsa.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +from tests.functional.config import nginx_certs_dir, ovms_c_repo_path + + +class OVSA: + OVMS_C_NGINX_MTLS_PATH = os.path.join(ovms_c_repo_path, "extras", "nginx-mtls-auth") + NGINX_TMP_DIR_PATH = os.path.join(nginx_certs_dir, "nginx-mtls-auth") + GENERATE_CERTS_CONFIG_NAME = "openssl_ca.conf" + GENERATE_CERTS_CONFIG_PATH = os.path.join(NGINX_TMP_DIR_PATH, GENERATE_CERTS_CONFIG_NAME) + GENERATE_CERTS_SCRIPT_NAME = "generate_certs.sh" + GENERATE_CERTS_SCRIPT_PATH = os.path.join(NGINX_TMP_DIR_PATH, GENERATE_CERTS_SCRIPT_NAME) + + CLIENT_CERT_CA_CRL_NAME = "client_cert_ca.crl" + CLIENT_CERT_CA_NAME = "client_cert_ca.pem" + CLIENT_CERT_NAME = "client.pem" + CLIENT_KEY_NAME = "client.key" + DHPARAMS_NAME = "dhparam.pem" + SERVER_CERT_NAME = "server.pem" + SERVER_KEY_NAME = "server.key" + + CERTS_CONTAINER_PATH = "/certs" + CLIENT_CERT_CA_CONTAINER_PATH = os.path.join(CERTS_CONTAINER_PATH, CLIENT_CERT_CA_NAME) + CLIENT_CERT_CA_CRL_CONTAINER_PATH = os.path.join(CERTS_CONTAINER_PATH, CLIENT_CERT_CA_CRL_NAME) + DHPARAMS_CONTAINER_PATH = os.path.join(CERTS_CONTAINER_PATH, DHPARAMS_NAME) + SERVER_CERT_CONTAINER_PATH = os.path.join(CERTS_CONTAINER_PATH, SERVER_CERT_NAME) + SERVER_KEY_CONTAINER_PATH = os.path.join(CERTS_CONTAINER_PATH, SERVER_KEY_NAME) diff --git a/tests/functional/constants/paths.py b/tests/functional/constants/paths.py new file mode 100644 index 0000000000..c38b76add2 --- /dev/null +++ b/tests/functional/constants/paths.py @@ -0,0 +1,76 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +from tests.functional.constants.os_type import OsType +from tests.functional import config +from tests.functional.constants.target_device import TargetDevice + + +class Paths: + + MODELS_PATH_NAME = "models" + CUSTOM_NODE_PATH_NAME = "custom_nodes" + CUSTOM_LOADER_PATH_NAME = "custom_loader" + CPU_EXTENSIONS = "cpu_extensions" + CONFIG_FILE_NAME = "config.json" + SUBCONFIG_FILE_NAME = "subconfig.json" + GRAPH_NAME = "graph.pbtxt" + IMAGES = "images" + OVMS_PATH_INTERNAL = os.path.join("/") if OsType.Windows not in config.base_os else os.path.join("\\") + MODELS_PATH_INTERNAL = os.path.join(OVMS_PATH_INTERNAL, MODELS_PATH_NAME) + CONFIG_PATH_INTERNAL = os.path.join(MODELS_PATH_INTERNAL, CONFIG_FILE_NAME) + CUSTOM_NODE_LIBRARIES_PATH_INTERNAL = os.path.join(OVMS_PATH_INTERNAL, CUSTOM_NODE_PATH_NAME) + CUSTOM_LOADER_LIBRARIES_PATH_INTERNAL = os.path.join(OVMS_PATH_INTERNAL, CUSTOM_LOADER_PATH_NAME) + ROOT_PATH_CPU_EXTENSIONS = os.path.join(OVMS_PATH_INTERNAL, CPU_EXTENSIONS) + IMAGES_PATH_INTERNAL = os.path.join(OVMS_PATH_INTERNAL, IMAGES) + ZEBRA_PATH_INTERNAL = os.path.join(IMAGES_PATH_INTERNAL, "zebra.jpeg") + CACHE_INTERNAL = os.path.join(OVMS_PATH_INTERNAL, "opt", "cache") + + # DATASET + DATASET_MAIN_PATH = os.path.join("/", "opt", "test_data") + + # CAPI OVMS-TEST + OVMS_TEST_CAPI_WRAPPER_DIR = os.path.join( + config.ovms_c_repo_path, "tests", "functional", "data", "ovms_capi_wrapper" + ) + OVMS_TEST_CAPI_PXD = os.path.join(OVMS_TEST_CAPI_WRAPPER_DIR, "ovms_capi.pxd") + OVMS_TEST_CAPI_AUTOPXD_PY = os.path.join(OVMS_TEST_CAPI_WRAPPER_DIR, "ovms_autopxd.py") + OVMS_TEST_CAPI_WRAPPER_PYX = os.path.join(OVMS_TEST_CAPI_WRAPPER_DIR, "ovms_capi_wrapper.pyx") + OVMS_TEST_CAPI_WRAPPER_MAKEFILE = os.path.join(OVMS_TEST_CAPI_WRAPPER_DIR, "Makefile") + OVMS_TEST_CAPI_WRAPPER_SETUP = os.path.join(OVMS_TEST_CAPI_WRAPPER_DIR, "setup.py") + + @staticmethod + def CAPI_WRAPPER_PACKAGE_CONTENT_PATH(base_os): + return os.path.join(config.c_api_wrapper_dir, base_os, "ovms") + + @staticmethod + def get_target_device_lock_file(target_device, i): + if isinstance(target_device, str): + assert not all(x in target_device for x in [TargetDevice.GPU, TargetDevice.NPU]) + + # generalize HETERO/AUTO/MUTLI:X => `X` + if TargetDevice.GPU in target_device: + return os.path.join(config.ovms_file_locks_dir, f"target_device_{TargetDevice.GPU}_{i}.lock") + if TargetDevice.NPU in target_device: + return os.path.join(config.ovms_file_locks_dir, f"target_device_{TargetDevice.NPU}_{i}.lock") + + return os.path.join(config.ovms_file_locks_dir, f"target_device_{target_device}_{i}.lock") + + +def any_is_relative_to(paths, subpath): + return any([_path in subpath for _path in paths]) diff --git a/tests/functional/constants/pipelines.py b/tests/functional/constants/pipelines.py new file mode 100644 index 0000000000..d466d67f4a --- /dev/null +++ b/tests/functional/constants/pipelines.py @@ -0,0 +1,1860 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from abc import abstractmethod +from copy import deepcopy +from enum import Enum +from pathlib import Path + +import numpy as np + +from tests.functional.config import datasets_path +from tests.functional.models.models_datasets import RandomDataset +from tests.functional.models import ModelInfo +from tests.functional.models.models_static import ( + AgeGender, + ArgMax, + CrnnTf, + Dummy, + DummyAdd2Inputs, + DummyIncrement, + DummyIncrementDecrement, + EastFp32, + Emotion, + FaceDetectionRetail, + GoogleNetV2Fp32, + Increment4d, + Resnet, + ResnetWrongInputShapeDim, + ResnetWrongInputShapes, + VehicleAttributesRecognition, + VehicleDetection, +) +from tests.functional.constants.ovms import Config +from tests.functional.constants.paths import Paths +from tests.functional.object_model.custom_node import ( + CustomNode, + CustomNodeAddSub, + CustomNodeChooseMaximum, + CustomNodeDemultiply, + CustomNodeDemultiplyGather, + CustomNodeDifferentOperations, + CustomNodeDynamicDemultiplex, + CustomNodeEastOcr, + CustomNodeElastic1T, + CustomNodeFaces, + CustomNodeImageTransformation, + CustomNodeVehicles, +) +from tests.functional.object_model.mediapipe_calculators import ( + CorruptedFileCalculator, + MediaPipeCalculator, + OpenVINOInferenceCalculator, + OpenVINOModelServerSessionCalculator, + OVMSOVCalculator, + PythonCalculator, +) + + +class NodesConnection: + + def __init__(self, target_node, target_node_input_id, source_node, source_output_id): + self.target_node = target_node + self.target_node_input_id = target_node_input_id + self.source_node = source_node + self.source_node_output_id = source_output_id + + def __str__(self): + return f"{self.target_node.name}[{self.target_node_input_id}]<-{self.source_node}[{self.source_node_output_id}]" + + def get_source_data_item_name(self): + return self.source_node.get_output_name(self.source_node_output_id) + + def get_target_data_item_name(self): + return self.target_node.get_input_name(self.target_node_input_id) + + def get_target_input(self): + model_input_name = self.target_node.model.input_names[self.target_node_input_id] + return model_input_name, self.target_node.model.inputs[model_input_name] + + def get_source_output(self): + model_output_name = self.source_node.model.output_names[self.source_node_output_id] + return model_output_name, self.source_node.model.outputs[model_output_name] + + @classmethod + def connect(cls, target_node, target_node_input_id, source_node, source_output_id): + connection = NodesConnection(target_node, target_node_input_id, source_node, source_output_id) + connection.target_node.input_connections.append(connection) + connection.source_node.output_connections.append(connection) + + +class NodeType(Enum): + Input = "INPUT" + DL_MODEL = "DL model" + Output = "OUTPUT" + Custom = "custom" + + +class Node: + + def __init__( + self, + name, + model=None, + node_type=None, + input_names=None, + output_names=None, + demultiply_count=None, + gather_from_node=None, + ): + + if node_type is None: + if isinstance(model, CustomNode): + node_type = NodeType.Custom + else: + node_type = NodeType.DL_MODEL + + self.name = name + self.model = model + self.input_connections = [] + self.output_connections = [] + self.node_type = node_type + self.input_names = input_names + self.output_names = output_names + self.demultiply_count = demultiply_count + self.gather_from_node = gather_from_node + + def __str__(self): + return self.name + + def get_input_name(self, id): + if self.input_names: + return self.input_names[id] + else: + if self.node_type == NodeType.Output: + prefix = "output" + else: + prefix = "input" + return f"{prefix}_{id}" + + def get_output_name(self, id): + if self.output_names: + return self.output_names[id] + else: + if self.node_type == NodeType.Input: + prefix = "input" + else: + prefix = self.model.name + return f"{prefix}_{id}" + + def _change_name(self, names, old_name, new_name): + for index, name in enumerate(names): + if name == old_name: + names[index] = new_name + return + + def change_input_name(self, old_name, new_name): + self._change_name(self.input_names, old_name, new_name) + + def change_output_name(self, old_name, new_name): + self._change_name(self.output_names, old_name, new_name) + + def get_expected_output(self, input_data, client_type: str = None): + mapped_input_data = {} + for input_connection in self.input_connections: + input_name = self.model.input_names[input_connection.target_node_input_id] + mapped_input_data[input_name] = input_connection.source_node[input_connection.source_node_output_id] + + return self.model.get_expected_output(mapped_input_data) + + def dump_config(self): + """ + "nodes": [ + { + "name": "node_1", + "type": "custom", + "inputs": [ + { + "input_numbers": { + "node_name": "request", + "data_item": "input" + } + } + ], + "outputs": [ + { + "data_item": "output_numbers", + "alias": "node_1_output_0" + } + ], + "library_name": "lib_node_add_sub", + "params": { + "add_value": "5", + "sub_value": "4" + } + } + """ + config = {"name": self.name, "type": f"{self.node_type.value}", "inputs": [], "outputs": []} + + if self.demultiply_count is not None: + config["demultiply_count"] = self.demultiply_count + + if self.gather_from_node is not None: + config["gather_from_node"] = self.gather_from_node + + if isinstance(self.model, CustomNode): + config["library_name"] = self.model.name + node_parameters = self.model.get_parameters() + if node_parameters: + config["params"] = node_parameters + else: + config["model_name"] = self.model.name + + for input_connection in self.input_connections: # a single model input can be connected only to a single source + input_name = self.model.input_names[input_connection.target_node_input_id] + input_mapping = { + "node_name": input_connection.source_node.name, + "data_item": input_connection.get_source_data_item_name(), + } + config["inputs"].append({input_name: input_mapping}) + + for id, model_output_name in enumerate( + self.model.output_names + ): # a single model output can be connected to multiple targets + config["outputs"].append({"data_item": model_output_name, "alias": self.get_output_name(id)}) + + return config + + +class MediaPipeGraphNode(Node): + def __init__( + self, + name, + model=None, + node_type=None, + input_names=None, + output_names=None, + demultiply_count=None, + gather_from_node=None, + calculator=None, + servable_name=None, + servable_version=None, + input_stream=None, + output_stream=None, + tag_to_input_tensor_names=None, + tag_to_output_tensor_names=None, + ): + + super().__init__(name, model, node_type, input_names, output_names, demultiply_count, gather_from_node) + + self.calculator = calculator + self.input_stream = input_stream + self.output_stream = output_stream + self.tag_to_input_tensor_names = tag_to_input_tensor_names + self.tag_to_output_tensor_names = tag_to_output_tensor_names + + if self.model is not None: + self.servable_name = servable_name if servable_name is not None else self.model.name + self.servable_version = servable_version if servable_version is not None else str(self.model.version) + else: + self.servable_name = servable_name + self.servable_version = servable_version + + +class PythonGraphNode(MediaPipeGraphNode): + def __init__( + self, + name, + calculator=None, + model=None, + input_side_packet=None, + input_stream=None, + output_stream=None, + handler_path=None, + node_options=None, + node_type=None, + input_names=None, + output_names=None, + ): + + super().__init__(name, model, node_type, input_names, output_names) + + self.calculator = calculator + self.input_side_packet = input_side_packet + self.input_stream = input_stream + self.output_stream = output_stream + self.handler_path = handler_path + self.node_options = node_options + + +class Pipeline(ModelInfo): + + def __init__(self, name=None, **kwargs): + self.name = name + self.child_nodes = [] + self.config = {} + self.inputs = {} + self.outputs = {} + self.demultiply_count = None # demultiply_count could be dynamic (value: 0, -1) + self.default_demultiply_count_value = ( + 7 # real demultiply count value used in validation mechanism - generate output shape + ) + assert kwargs.get("use_mapping", None) is not True + self.is_mediapipe = False + + def set_expected_demultiply(self, expected_value, dynamic_mode=False): + self.demultiply_count = -1 if dynamic_mode else expected_value + self.default_demultiply_count_value = expected_value + + def get_demultiply_count(self): + return self.demultiply_count + + @property + def is_on_cloud(self): + return False + + @abstractmethod + def _create_nodes(self, models=None): + raise NotImplementedError() + + def _initialize(self, models=None): + self.child_nodes.extend(self._create_nodes(models)) + self.initialize_inputs_outputs() + self.config_refresh() + + def initialize_inputs_outputs(self): + input_node = self.get_input_node() + output_names = [] + for connection in input_node.output_connections: + _, value = connection.get_target_input() + input_name = connection.get_source_data_item_name() + self.inputs[input_name] = deepcopy(value) + output_names.append(input_name) + + if input_node.output_names is None: + input_node.output_names = output_names + + output_node = self.get_output_node() + input_names = [] + for connection in output_node.input_connections: + _, value = connection.get_source_output() + input_name = connection.get_target_data_item_name() + self.outputs[input_name] = deepcopy(value) + input_names.append(input_name) + + if output_node.input_names is None: + output_node.input_names = input_names + + def prepare_resources(self, base_location): + resource_locations = [] + models = self.get_models() + for model in models: + resource_location_list = model.prepare_resources(base_location) + if resource_location_list is not None: + for location in resource_location_list: + if location not in resource_locations: + resource_locations.append(location) + return resource_locations + + def get_resources(self): + return [self] + + def get_input_node(self): + return [node for node in self.child_nodes if node.node_type == NodeType.Input][0] + + def get_middle_nodes(self): + return [ + node for node in self.child_nodes if node.node_type != NodeType.Input and node.node_type != NodeType.Output + ] + + def get_output_node(self): + return [node for node in self.child_nodes if node.node_type == NodeType.Output][0] + + def get_input_models(self): + input_node = [node for node in self.child_nodes if node.node_type == NodeType.Input][0] + input_models = [] + for connection in input_node.output_connections: + if connection.target_node.model not in input_models: + input_models.append(connection.target_node.model) + return input_models + + def prepare_pipeline_input_data(self, batch_size=None, random_data=False): + input_data = {} + demultiply_count = ( + self.demultiply_count if self.demultiply_count is not None else self.get_input_node().demultiply_count + ) + if demultiply_count is not None: + number_of_batches_in_request = demultiply_count + if demultiply_count <= 0: + number_of_batches_in_request = ( + self.default_demultiply_count_value + ) # we need to set a non zero number here for data generation purpose + + if batch_size is None: + batch_size = self.get_expected_batch_size() + for input_model_type in self.get_input_models(): + for input_name, data in input_model_type.inputs.items(): + if batch_size is not None and data["shape"][0] == -1: + data["shape"][0] = batch_size + + if "dataset" in data: + layout = data.get("layout", None) + if layout is not None and ":" in layout: + layout_str = layout.partition(":")[0] + else: + layout_str = None + input_data[input_name] = data["dataset"].get_data( + shape=data["shape"], + batch_size=batch_size, + transpose_axes=input_model_type.transpose_axes, + layout=layout_str, + ) + if demultiply_count is not None: + dumultipy_content = [] + for i in range(number_of_batches_in_request): + dumultipy_content.append(input_data[input_name]) + input_data[input_name] = np.array(dumultipy_content) + else: + if demultiply_count is not None: + new_data = deepcopy(data["shape"]) + new_data.insert(0, number_of_batches_in_request) + input_data[input_name] = np.ones(new_data, dtype=data["dtype"]) + else: + input_data[input_name] = np.ones(data["shape"], dtype=data["dtype"]) + + return self.map_inputs(input_data) + + def prepare_input_data(self, batch_size=None, input_key=None): + data = self.prepare_pipeline_input_data(batch_size) + return data + + def prepare_model_input_data(self, batch_size=None): + return super(Pipeline, self).prepare_input_data(batch_size) + + def prepare_model_resources(self, base_location): + return super(Pipeline, self).prepare_resources(base_location) + + def map_inputs(self, prepare_inputs: dict): + result_dict = {} + for key, value in self.get_pipeline_inputs_to_model_dataset_map().items(): + result_dict[key] = prepare_inputs[value] + + return result_dict + + @staticmethod + def is_pipeline(): + return True + + def get_custom_nodes(self): + return [node.model for node in self.child_nodes if isinstance(node.model, CustomNode)] + + def get_models(self): + models = [] + for node in self.child_nodes: + if node.node_type not in (NodeType.Input, NodeType.Output): + if any(added_model.name == node.model.name for added_model in models): + continue + + models.append(node.model) + + return models + + def get_regular_models(self): + return [model for model in self.get_models() if not isinstance(model, CustomNode)] + + def get_pipeline_inputs_to_model_dataset_map(self): + inputs_to_models_map = {} + for pipeline_input_name, model_input_name in zip(self.input_names, self.get_input_models()[0].input_names): + inputs_to_models_map[pipeline_input_name] = model_input_name + return inputs_to_models_map + + def config_refresh(self): + refreshed_config = {"name": self.name} + if self.demultiply_count is not None: + refreshed_config["demultiply_count"] = self.demultiply_count + + refreshed_config.update({"inputs": self.input_names, "nodes": [], "outputs": []}) + + nodes = self.child_nodes + regular_nodes = [ + node for node in nodes if node.node_type != NodeType.Input and node.node_type != NodeType.Output + ] + for node in regular_nodes: + refreshed_config["nodes"].append(node.dump_config()) + + output_node = [node for node in nodes if node.node_type == NodeType.Output][0] + for input_connection_of_output_node in output_node.input_connections: + output_map = { + "node_name": input_connection_of_output_node.source_node.name, + "data_item": input_connection_of_output_node.get_source_data_item_name(), + } + output_name = input_connection_of_output_node.get_target_data_item_name() + refreshed_config["outputs"].append({output_name: output_map}) + self.config = refreshed_config + return self.config + + def build_pipeline_config( + self, + config, + custom_nodes, + config_custom_nodes, + models=None, + use_custom_graphs=False, + mediapipe_models=None, + use_subconfig=False, + custom_graph_paths=None, + ): + config[Config.PIPELINE_CONFIG_LIST].append(self.config) + config_custom_nodes = self.build_config_custom_nodes(custom_nodes, config_custom_nodes) + return config, config_custom_nodes + + def build_config_custom_nodes(self, custom_nodes, config_custom_nodes): + if custom_nodes is None: + custom_node_list = self.get_unique_custom_node_list() + for custom_node in custom_node_list: + if type(custom_node) not in [type(x) for x in config_custom_nodes]: + config_custom_nodes.append(custom_node) + return config_custom_nodes + + def map_model_output_to_pipeline_output(self, model_output): + result = {} + for node in self.child_nodes: + if node.node_type == NodeType.Output: + for connection in node.input_connections: + target_name = self.output_names[connection.target_node_input_id] + model = connection.source_node.model + source_name = model.output_names[connection.source_node_output_id] + result[target_name] = model_output[source_name] + return result + + assert False, "Output node not found" + + def get_unique_custom_node_list(self): + result = [] + for custom_node in self.get_custom_nodes(): + if type(custom_node) not in [type(x) for x in result]: + result.append(custom_node) + return result + + def has_custom_nodes(self): + return len(self.get_custom_nodes()) > 0 + + def change_input_name(self, old_name, new_name): + super().change_input_name(old_name, new_name) + self.get_input_node().change_output_name(old_name, new_name) + + def change_output_name(self, old_name, new_name): + super().change_output_name(old_name, new_name) + self.get_output_node().change_input_name(old_name, new_name) + + +class SimplePipeline(Pipeline): + + def __init__(self, model=None, demultiply_count=None, name=None, **kwargs): + name = "single_model_pipeline" if name is None else f"single_model_pipeline_{name}" + super().__init__(name=name, **kwargs) + self.demultiply_count = demultiply_count + + if model is None: + model = Resnet() + + self._initialize([model]) + + def _create_nodes(self, models): + model = models[0] + node1 = Node("node_1", model) + + request = Node("request", node_type=NodeType.Input, output_names=["input"]) + output = Node("output", node_type=NodeType.Output, input_names=["output"]) + + NodesConnection.connect(node1, 0, request, 0) + NodesConnection.connect(output, 0, node1, 0) + + return [request, node1, output] + + +class MultipleInputsOutputsPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("multiple_inputs_outputs_pipeline", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + node1 = Node("node_1", DummyIncrementDecrement()) + + request = Node("request", node_type=NodeType.Input) + output = Node("output", node_type=NodeType.Output) + + NodesConnection.connect(node1, 0, request, 0) + NodesConnection.connect(node1, 1, request, 1) + NodesConnection.connect(output, 0, node1, 0) + NodesConnection.connect(output, 1, node1, 1) + + nodes = [request, node1, output] + return nodes + + def get_expected_output(self, input_data: dict, client_type: str = None): + model_output = self.get_models()[0].get_expected_output(input_data) + return self.map_model_output_to_pipeline_output(model_output) + + +class InputNotConnectedPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("single_input_not_nonnected_pipeline", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + node1 = Node("node_1", DummyIncrementDecrement()) + + request = Node("request", node_type=NodeType.Input) + output = Node("output", node_type=NodeType.Output) + + NodesConnection.connect(node1, 0, request, 0) + NodesConnection.connect(output, 0, node1, 0) + NodesConnection.connect(output, 1, node1, 1) + + nodes = [request, node1, output] + return nodes + + +class ImageClassificationPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("image_classification_pipeline", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + resnet_node = Node("resnet_node", Resnet()) + googlenet_node = Node("googlenet_node", GoogleNetV2Fp32()) + argmax_node = Node("argmax_node", ArgMax()) + + request = Node("request", node_type=NodeType.Input) + output = Node("output", node_type=NodeType.Output) + + NodesConnection.connect(googlenet_node, 0, request, 0) + NodesConnection.connect(resnet_node, 0, request, 0) + NodesConnection.connect(argmax_node, 0, googlenet_node, 0) + NodesConnection.connect(argmax_node, 1, resnet_node, 0) + NodesConnection.connect(output, 0, argmax_node, 0) + + nodes = [request, googlenet_node, resnet_node, argmax_node, output] + return nodes + + +class ComplexDummyPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("complex_pipeline", **kwargs) + self._initialize() + + def get_pipeline_transpose_axes(self): + return None + + def get_pipeline_datasets(self): + return { + "input1": os.path.join(datasets_path, RandomDataset.name), + "input2": os.path.join(datasets_path, RandomDataset.name), + } + + def _create_nodes(self, models=None): + node1 = Node("node_1", DummyIncrementDecrement()) + node2 = Node("node_2", DummyIncrement()) + node3 = Node("node_3", DummyAdd2Inputs()) + node4 = Node("node_4", DummyAdd2Inputs()) + node5 = Node("node_5", DummyIncrement()) + node6 = Node("node_6", DummyIncrement()) + + request = Node("request", node_type=NodeType.Input) + output = Node("output", node_type=NodeType.Output) + + NodesConnection.connect(node1, 0, request, 0) + NodesConnection.connect(node1, 1, request, 1) + NodesConnection.connect(node2, 0, node1, 0) + NodesConnection.connect(node3, 0, node1, 1) + NodesConnection.connect(node3, 1, node6, 0) + NodesConnection.connect(node4, 0, node1, 0) + NodesConnection.connect(node4, 1, node2, 0) + NodesConnection.connect(node5, 0, request, 1) + NodesConnection.connect(node6, 0, request, 1) + NodesConnection.connect(output, 0, node4, 0) + NodesConnection.connect(output, 1, node3, 0) + NodesConnection.connect(output, 2, node5, 0) + NodesConnection.connect(output, 3, node1, 0) + + nodes = [request, node1, node2, node3, node4, node5, node6, output] + return nodes + + +class SameModelsPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("multiple_versions_of_the_same_model_pipeline", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + node1 = Node("node_1", DummyIncrement()) + node2 = Node("node_2", DummyIncrement()) + + request = Node("request", node_type=NodeType.Input) + output = Node("output", node_type=NodeType.Output) + + NodesConnection.connect(node1, 0, request, 0) + NodesConnection.connect(node2, 0, node1, 0) + NodesConnection.connect(output, 0, node2, 0) + + nodes = [request, node1, node2, output] + return nodes + + +class NodeReferringMultipleOutputsFromPreviousNodePipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("pipeline_multiple_inputs_from_the_same_model", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + node1 = Node("node_1", DummyIncrementDecrement()) + node2 = Node("node_2", DummyAdd2Inputs()) + + request = Node("request", node_type=NodeType.Input) + output = Node("output", node_type=NodeType.Output) + + NodesConnection.connect(node1, 0, request, 0) + NodesConnection.connect(node1, 1, request, 1) + NodesConnection.connect(node2, 0, node1, 0) + NodesConnection.connect(node2, 1, node1, 1) + NodesConnection.connect(output, 0, node2, 0) + + nodes = [request, node1, node2, output] + return nodes + + def get_expected_output(self, input_data, client_type: str = None): + models = self.get_regular_models() + input_model = models[0] + output_model = models[1] + input_model_output = input_model.get_expected_output(input_data) + node2 = { + output_model.input_names[0]: input_model_output[input_model.output_names[0]], + output_model.input_names[1]: input_model_output[input_model.output_names[1]], + } + output_model_output = output_model.get_expected_output(node2) + pipeline_output = self.map_model_output_to_pipeline_output(output_model_output) + return pipeline_output + + +class EastAndOcrPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("east_and_ocr_pipeline", **kwargs) + self._initialize() + self.set_expected_output_shape() + + def set_expected_output_shape(self): + self.outputs["texts"]["shape"].insert(0, 0) + + def _create_nodes(self, models=None): + east_node = Node("east_node", EastFp32(), output_names=["scores", "geometry"]) + extract_node = Node("extract_node", CustomNodeEastOcr(), NodeType.Custom, demultiply_count=0) + crnn_model = CrnnTf() + crnn_model.inputs["input"]["layout"] = "NHWC:NCHW" + crnn_node = Node("crnn_node", crnn_model) + + request = Node("request", node_type=NodeType.Input) + output = Node( + "output", + node_type=NodeType.Output, + input_names=["text_images", "text_coordinates", "confidence_levels", "texts"], + ) + NodesConnection.connect(east_node, 0, request, 0) + NodesConnection.connect(extract_node, 0, request, 0) + NodesConnection.connect(extract_node, 1, east_node, 0) + NodesConnection.connect(extract_node, 2, east_node, 1) + NodesConnection.connect(crnn_node, 0, extract_node, 0) + NodesConnection.connect(output, 0, extract_node, 0) + NodesConnection.connect(output, 1, extract_node, 1) + NodesConnection.connect(output, 2, extract_node, 2) + NodesConnection.connect(output, 3, crnn_node, 0) + + nodes = [request, east_node, extract_node, crnn_node, output] + return nodes + + +class DemultiplyPipeline(Pipeline): + + def __init__(self, demultiply_value, **kwargs): + super().__init__("demultiply_pipeline", **kwargs) + self.demultiply_node = Node( + "demultiply", CustomNodeDemultiply(demultiply_value), NodeType.Custom, demultiply_count=-1 + ) + self.resnet_node = Node("resnet", Resnet()) + self._initialize() + self.update_demultiply_value(demultiply_value) + + def update_demultiply_value(self, new_demultiply_value): + self.demultiply_node.model.demultiply_size = new_demultiply_value + self.set_expected_output_shape() + + def set_expected_output_shape(self): + self.outputs["result"]["shape"] = [self.demultiply_node.model.demultiply_size] + self.resnet_node.model.outputs[ + "softmax_tensor" + ]["shape"] + + def _create_nodes(self, models=None): + request = Node("request", node_type=NodeType.Input) + output = Node("output", node_type=NodeType.Output, input_names=["result"]) + + NodesConnection.connect(self.demultiply_node, 0, request, 0) + NodesConnection.connect(self.resnet_node, 0, self.demultiply_node, 0) + NodesConnection.connect(output, 0, self.resnet_node, 0) + + return [request, self.demultiply_node, self.resnet_node, output] + + +class ElasticPipeline(Pipeline): + + def __init__(self, input_shape, output_shape, demultiply_count=None, **kwargs): + super().__init__("elastic_pipeline", **kwargs) + self.custom_node = Node( + "elastic_node", + CustomNodeElastic1T(input_shape, output_shape), + NodeType.Custom, + demultiply_count=demultiply_count, + ) + self._request = Node("request", node_type=NodeType.Input) + self.model_node = Node("resnet", Resnet()) + for key in self.model_node.model.inputs: + self.model_node.model.inputs[key]["shape"] = None + self._initialize() + self.set_expected_output_shape() + + def set_expected_output_shape(self): + demultiply_value = self.custom_node.model.outputs["tensor_out"]["shape"][0] + self.outputs["result"]["shape"] = [demultiply_value] + self.model_node.model.outputs["softmax_tensor"]["shape"] + + def _create_nodes(self, models): + output = Node("output", node_type=NodeType.Output, input_names=["result"]) + + NodesConnection.connect(self.custom_node, 0, self._request, 0) + NodesConnection.connect(self.model_node, 0, self.custom_node, 0) + NodesConnection.connect(output, 0, self.model_node, 0) + return [self._request, self.custom_node, self.model_node, output] + + +class ElasticBatchSizePipeline(Pipeline): + def __init__(self, node_batch_configuration_list, **kwargs): + super().__init__(name="misconfigurated_pipeline", **kwargs) + self.node_batch_cfg_list = node_batch_configuration_list + self._initialize() + + def _create_nodes(self, models): + request = Node("request", node_type=NodeType.Input, output_names=["input"]) + output = Node("output", node_type=NodeType.Output, input_names=["output"]) + + nodes = [request] + for node_name, batch_size in self.node_batch_cfg_list: + model = Dummy(batch_size=batch_size) + model.name = f"{model.name}_{node_name}" + nodes.append(Node(node_name, model)) + nodes.append(output) + + for i in range(1, len(nodes)): + NodesConnection.connect(nodes[i], 0, nodes[i - 1], 0) + return nodes + + +class CustomNodesConnectedToEachOtherPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("custom_nodes_connected_to_each_other_pipeline", **kwargs) + self.custom_node_a = Node("node_1", CustomNodeAddSub(1.5, 0.7), NodeType.Custom) + self.custom_node_b = Node("node_2", CustomNodeAddSub(2.4, 1.2), NodeType.Custom) + self._initialize() + + def set_expected_output_shape(self): + arg_name = list(self.custom_node_a.model.outputs.keys())[0] + self.outputs["output_0"]["shape"] = self.custom_node_a.model.outputs[arg_name]["shape"] + + def _create_nodes(self, models=None): + request = Node("request", node_type=NodeType.Input) + output = Node("output", node_type=NodeType.Output) + nodes = [request, self.custom_node_a, self.custom_node_b, output] + + for i in range(1, len(nodes)): + NodesConnection.connect(nodes[i], 0, nodes[i - 1], 0) + return nodes + + def get_expected_output(self, input_data, client_type: str = None): + custom_nodes = self.get_custom_nodes() + node1_out = custom_nodes[0].get_expected_output(input_data) + node2_out = custom_nodes[1].get_expected_output(node1_out) + return self.map_model_output_to_pipeline_output(node2_out) + + +class CustomNodeNotAllOutputsConnectedPipeline(Pipeline): + def __init__(self, **kwargs): + super().__init__("custom_node_not_all_outputs_connected_pipeline", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + node1 = Node("node_1", CustomNodeDifferentOperations(), NodeType.Custom) + + request = Node("request", node_type=NodeType.Input, output_names=self.input_names) + output = Node("output", node_type=NodeType.Output, input_names=self.output_names) + + NodesConnection.connect(node1, 0, request, 0) + NodesConnection.connect(node1, 1, request, 1) + NodesConnection.connect(output, 0, node1, 0) + + nodes = [request, node1, output] + return nodes + + +class CustomNodeNotAllInputsConnectedPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("custom_node_not_all_inputs_connected_pipeline", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + node1 = Node("node_1", CustomNodeDifferentOperations(), NodeType.Custom) + + request = Node("request", node_type=NodeType.Input, output_names=self.input_names) + output = Node("output", node_type=NodeType.Output, input_names=self.output_names) + + NodesConnection.connect(node1, 1, request, 1) + NodesConnection.connect(output, 0, node1, 0) + NodesConnection.connect(output, 1, node1, 1) + + nodes = [request, node1, output] + return nodes + + +class CyclicGraphPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("cyclic_graph_pipeline", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + model_two_inputs_two_outputs = DummyIncrementDecrement() + model_two_inputs_one_output = DummyAdd2Inputs() + + node1 = Node("node_1", model_two_inputs_one_output) + node2 = Node("node_2", model_two_inputs_two_outputs) + + request = Node("request", node_type=NodeType.Input, output_names=self.input_names) + output = Node("output", node_type=NodeType.Output, input_names=self.output_names) + + NodesConnection.connect(node1, 0, request, 0) + NodesConnection.connect(node1, 1, node2, 1) + NodesConnection.connect(output, 0, node2, 0) + + nodes = [request, node1, node2, output] + return nodes + + +class AgeGenderAndEmotionPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("combined-recognition", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + model_gender = AgeGender() + model_gender.set_input_shape_for_ovms([1, 3, 64, 64]) + + model_emotion = Emotion() + + age_gender_node = Node("age_gender", model_gender, output_names=["age", "gender"]) + emotion_node = Node("emotion_node", model_emotion, output_names=["emotion"]) + + request = Node("request", node_type=NodeType.Input, output_names=["image"]) + output = Node("output", node_type=NodeType.Output, input_names=["age", "gender", "emotion"]) + + NodesConnection.connect(age_gender_node, 0, request, 0) + NodesConnection.connect(emotion_node, 0, request, 0) + + NodesConnection.connect(output, 0, age_gender_node, 0) + NodesConnection.connect(output, 1, age_gender_node, 1) + NodesConnection.connect(output, 2, emotion_node, 0) + + nodes = [request, age_gender_node, emotion_node, output] + return nodes + + +class VehiclesAnalysisPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("multiple_vehicle_recognition", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + detection_model = VehicleDetection() + recognition_model = VehicleAttributesRecognition() + vehicles_custom_node = CustomNodeVehicles() + + vehicle_detection_node = Node("vehicle_detection_node", detection_model, output_names=["detection_out"]) + extract_node = Node( + "extract_node", + vehicles_custom_node, + NodeType.Custom, + demultiply_count=0, + output_names=["vehicle_images", "vehicle_coordinates", "confidence_levels"], + ) + vehicle_recognition_node = Node("vehicle_recognition_node", recognition_model, output_names=["color", "type"]) + + request = Node("request", node_type=NodeType.Input, output_names=["image"]) + output = Node( + "output", + node_type=NodeType.Output, + input_names=["vehicle_images", "vehicle_coordinates", "confidence_levels", "colors", "types"], + ) + + NodesConnection.connect(vehicle_detection_node, 0, request, 0) + + NodesConnection.connect(extract_node, 0, request, 0) + NodesConnection.connect(extract_node, 1, vehicle_detection_node, 0) + + NodesConnection.connect(vehicle_recognition_node, 0, extract_node, 0) + + NodesConnection.connect(output, 0, extract_node, 0) + NodesConnection.connect(output, 1, extract_node, 1) + NodesConnection.connect(output, 2, extract_node, 2) + + NodesConnection.connect(output, 3, vehicle_recognition_node, 0) + NodesConnection.connect(output, 4, vehicle_recognition_node, 1) + + nodes = [request, vehicle_detection_node, extract_node, vehicle_recognition_node, output] + return nodes + + +class FacesAnalysisPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("find_face_images", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + model_face = FaceDetectionRetail() + model_face.input_shapes = [1, 3, 400, 600] + model_face.set_input_shape_for_ovms([1, 3, 400, 600]) + + model_gender = AgeGender() + model_gender.input_shapes = [1, 3, 64, 64] + model_gender.set_input_shape_for_ovms([1, 3, 64, 64]) + + model_emotion = Emotion() + model_emotion.input_shapes = [1, 3, 64, 64] + model_emotion.set_input_shape_for_ovms([1, 3, 64, 64]) + + faces_custom_node = CustomNodeFaces() + + model_gender.inputs["data"]["shape"] = deepcopy(model_emotion.inputs["data"]["shape"]) + model_face.inputs["data"]["shape"] = [1, 3, 400, 600] + + face_detection_node = Node("face_detection_node", model_face, output_names=["detection_out"]) + extract_node = Node( + "extract_node", + faces_custom_node, + NodeType.Custom, + demultiply_count=0, + output_names=["face_images", "face_coordinates", "confidence_levels"], + ) + age_gender_recognition_node = Node("age_gender_recognition_node", model_gender, output_names=["age", "gender"]) + emotion_recognition_node = Node("emotion_recognition_node", model_emotion, output_names=["emotion"]) + + request = Node("request", node_type=NodeType.Input, output_names=["image"]) + output = Node( + "output", + node_type=NodeType.Output, + input_names=["face_images", "face_coordinates", "confidence_levels", "ages", "genders", "emotions"], + ) + + NodesConnection.connect(face_detection_node, 0, request, 0) + + NodesConnection.connect(extract_node, 0, request, 0) + NodesConnection.connect(extract_node, 1, face_detection_node, 0) + + NodesConnection.connect(age_gender_recognition_node, 0, extract_node, 0) + + NodesConnection.connect(emotion_recognition_node, 0, extract_node, 0) + + NodesConnection.connect(output, 0, extract_node, 0) + NodesConnection.connect(output, 1, extract_node, 1) + NodesConnection.connect(output, 2, extract_node, 2) + + NodesConnection.connect(output, 3, age_gender_recognition_node, 0) + NodesConnection.connect(output, 4, age_gender_recognition_node, 1) + NodesConnection.connect(output, 5, emotion_recognition_node, 0) + + nodes = [ + request, + face_detection_node, + extract_node, + age_gender_recognition_node, + emotion_recognition_node, + output, + ] + return nodes + + +class TenDummySerialPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("ten_dummy_serial", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + dummy = Dummy() + dummy.inputs["b"]["shape"] = [1, 150528] + + node1 = Node("node_1", dummy) + node2 = Node("node_2", dummy) + node3 = Node("node_3", dummy) + node4 = Node("node_4", dummy) + node5 = Node("node_5", dummy) + node6 = Node("node_6", dummy) + node7 = Node("node_7", dummy) + node8 = Node("node_8", dummy) + node9 = Node("node_9", dummy) + node10 = Node("node_10", dummy) + + request = Node("request", node_type=NodeType.Input) + output = Node("output", node_type=NodeType.Output) + + NodesConnection.connect(node1, 0, request, 0) + NodesConnection.connect(node2, 0, node1, 0) + NodesConnection.connect(node3, 0, node2, 0) + NodesConnection.connect(node4, 0, node3, 0) + NodesConnection.connect(node5, 0, node4, 0) + NodesConnection.connect(node6, 0, node5, 0) + NodesConnection.connect(node7, 0, node6, 0) + NodesConnection.connect(node8, 0, node7, 0) + NodesConnection.connect(node9, 0, node8, 0) + NodesConnection.connect(node10, 0, node9, 0) + NodesConnection.connect(output, 0, node10, 0) + + nodes = [request, node1, node2, node3, node4, node5, node6, node7, node8, node9, node10, output] + return nodes + + +class DummyDiffOpsMaxPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("dummy_diff_ops_max_pipeline", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + request = Node("request", node_type=NodeType.Input, output_names=self.input_names) + dummy = Dummy() + node_diff = Node("node_diff", CustomNodeDifferentOperations(), NodeType.Custom, demultiply_count=4) + node_dummy_1 = Node("node_d1", dummy) + node_max = Node("node_max", CustomNodeChooseMaximum(), NodeType.Custom, gather_from_node="node_diff") + node_max.model.selection_criteria = CustomNodeChooseMaximum.Method.MAXIMUM_MAXIMUM + node_dummy_2 = Node("node_d2", dummy, output_names=self.input_names) + output = Node("output", node_type=NodeType.Output, input_names=self.output_names) + + NodesConnection.connect(node_diff, 0, request, 0) + NodesConnection.connect(node_diff, 1, request, 1) + NodesConnection.connect(node_dummy_1, 0, node_diff, 0) + NodesConnection.connect(node_max, 0, node_dummy_1, 0) + NodesConnection.connect(node_dummy_2, 0, node_max, 0) + NodesConnection.connect(output, 0, node_dummy_2, 0) + nodes = [request, node_dummy_1, node_dummy_2, node_max, node_diff, output] + return nodes + + +class DummyDynamicDemuxPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("dummy_dag_pipeline", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + self.demultiply_count = -1 + + request = Node("request", node_type=NodeType.Input, output_names=self.input_names) + output = Node("output", node_type=NodeType.Output, input_names=self.output_names) + + dummy = Dummy() + node_dummy = Node("node_d", dummy) + + NodesConnection.connect(node_dummy, 0, request, 0) + NodesConnection.connect(output, 0, node_dummy, 0) + nodes = [request, node_dummy, output] + + return nodes + + +class DifferentDemultiplyValuesPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("dynamic_demultiplex_pipeline", **kwargs) + self.demux_node = Node( + "diff_node", CustomNodeDynamicDemultiplex(), node_type=NodeType.Custom, demultiply_count=-1 + ) + self._initialize() + self.set_expected_output_shape() + + def set_expected_output_shape(self): + self.outputs["output_0"]["shape"] = self.demux_node.model.outputs["dynamic_demultiplex_results"]["shape"] + + def _create_nodes(self, models=None): + request = Node("request", node_type=NodeType.Input, input_names=self.input_names) + dummy_node = Node("dummy_node", Dummy()) + output = Node("output", node_type=NodeType.Output, output_names=self.output_names) + + NodesConnection.connect(self.demux_node, 0, request, 0) + NodesConnection.connect(dummy_node, 0, self.demux_node, 0) + NodesConnection.connect(output, 0, dummy_node, 0) + + nodes = [request, self.demux_node, dummy_node, output] + return nodes + + +class DemultiplexerAndGatherPipeline(Pipeline): + + def __init__(self, demultiply_count, **kwargs): + super().__init__("pipeline_with_demultiplexer_and_gather", **kwargs) + self._demultiply_count = demultiply_count + self._initialize() + + def _create_nodes(self, models=None): + diff_node = Node( + "diff_node", CustomNodeDifferentOperations(), NodeType.Custom, demultiply_count=self._demultiply_count + ) + dummy_node = Node("dummy_node", Dummy()) + demultiply_gather_node = Node( + "demultiply_gather_node", + CustomNodeDemultiplyGather(), + NodeType.Custom, + demultiply_count=self._demultiply_count, + gather_from_node="diff_node", + ) + + request = Node("request", node_type=NodeType.Input, output_names=self.input_names) + output = Node("output", node_type=NodeType.Output, input_names=self.output_names) + + NodesConnection.connect(diff_node, 0, request, 0) + NodesConnection.connect(diff_node, 1, request, 1) + NodesConnection.connect(dummy_node, 0, diff_node, 0) + NodesConnection.connect(demultiply_gather_node, 0, dummy_node, 0) + NodesConnection.connect(output, 0, demultiply_gather_node, 0) + + nodes = [request, dummy_node, demultiply_gather_node, diff_node, output] + return nodes + + +class ImageTransformationPipeline(Pipeline): + + def __init__(self, **kwargs): + super().__init__("image_transformation_test", **kwargs) + self._initialize() + + def _create_nodes(self, models=None): + self.demultiply_count = 0 + image_transformation_node = Node( + "image_transformation_node", CustomNodeImageTransformation(), NodeType.Custom, output_names=["image"] + ) + resnet_node = Node("resnet_node", Resnet()) + + request = Node("request", node_type=NodeType.Input, output_names=["image"]) + output = Node("output", node_type=NodeType.Output, input_names=["image_0", "image_1"]) + + NodesConnection.connect(image_transformation_node, 0, request, 0) + NodesConnection.connect(resnet_node, 0, image_transformation_node, 0) + NodesConnection.connect(output, 0, resnet_node, 0) + + NodesConnection.connect(output, 1, image_transformation_node, 0) + + nodes = [request, image_transformation_node, resnet_node, output] + return nodes + + +class SingleLevelPipeline(Pipeline): + def __init__(self, list_of_models, predict_shape, **kwargs): + super().__init__("single_level_pipeline", **kwargs) + self.predict_shape = predict_shape + self.models = list_of_models + self._initialize() + + def _create_nodes(self, model=None): + names = [f"img_{i}" for i in range(len(self.models))] + request = Node("request", node_type=NodeType.Input, output_names=["img"]) + output = Node("output", node_type=NodeType.Output, input_names=names) + model_nodes = [] + for idx, model in enumerate(self.models): + model_node = Node(f"model_{idx}", model) + NodesConnection.connect(model_node, 0, request, 0) + NodesConnection.connect(output, idx, model_node, 0) + model_nodes.append(model_node) + return [request] + model_nodes + [output] + + def prepare_input_data(self, batch_size=None, random_data=False, input_key=None): + result = {} + for in_name, in_data in self.models[0].inputs.items(): + shape = self.predict_shape.copy() + if batch_size is not None: + shape[0] = batch_size + result[in_name] = np.ones(shape, dtype=in_data["dtype"]) + + return self.map_inputs(result) + + +class MultiLevelPipeline(Pipeline): + def __init__(self, shape_model_list, **kwargs): + super().__init__("multi_level_pipeline", **kwargs) + self._vertical_shape_list = shape_model_list + self._initialize() + + def _create_nodes(self, model=None): + request = Node("request", node_type=NodeType.Input, output_names=["img"]) + + model_nodes = [] + for idx, shape in enumerate(self._vertical_shape_list): + model = Increment4d() + model.name = f"{model.name}_{idx}" + model.update_shapes(shape) + model.set_input_shape_for_ovms(shape) + model_nodes.append(Node(f"model_{idx}", model)) + + output = Node("output", node_type=NodeType.Output, input_names=[model_nodes[-1].name]) + node_list = [request] + model_nodes + [output] + + for i in range(1, len(node_list)): + NodesConnection.connect(node_list[i], 0, node_list[i - 1], 0) + + return node_list + + +class MediaPipe(Pipeline): + name = "MediaPipe" + is_mediapipe = True + is_python_custom_node = False + pbtxt_name = None + + def __init__(self, model=None, pipeline=None, demultiply_count=None, **kwargs): + if pipeline is not None: + pipeline = pipeline(model, demultiply_count, **kwargs) + self.__dict__.update(pipeline.__dict__) + self.is_mediapipe = True + self.calculators = [] + self.graphs = [] + self.regular_models = [] + self.create_header = True + + def _initialize(self, models=None): + self.child_nodes = [] + self.child_nodes.extend(self._create_nodes(models)) + self.initialize_inputs_outputs() + self.graph_refresh() + + @staticmethod + def get_mediapipe_names(config): + return [elem["name"] for elem in config[Config.MEDIAPIPE_CONFIG_LIST]] + + def prepare_input_data(self, batch_size=None, input_key=None): + data = self.prepare_pipeline_input_data(batch_size) + new_data = {} + for i, key in enumerate(list(data.keys()), start=0): + new_input_key = input_key if input_key is not None else "input" + new_data.update({new_input_key: data[key]}) + return new_data + + def build_pipeline_config( + self, + config, + custom_nodes, + config_custom_nodes, + models, + use_custom_graphs=False, + mediapipe_models=None, + use_subconfig=False, + custom_graph_paths=None, + ): + # Mediapipe config.json example: + # { + # "model_config_list": [...], + # "pipeline_config_list": [...], + # "custom_loader_config_list": [...], + # "mediapipe_config_list": [ + # { + # "name": "pipe1", + # "base_path": "/models/pipe1", + # "graph_path": "/models/pipe1/graphdummy.pbtxt" + # } + # ] + # } + if self.config: + config[Config.PIPELINE_CONFIG_LIST].append(self.config) + config_custom_nodes = self.build_config_custom_nodes(custom_nodes, config_custom_nodes) + if use_custom_graphs: + config = self.prepare_custom_graphs_mediapipe_config_list(config, use_subconfig, custom_graph_paths) + else: + mediapipe_models = [self] if mediapipe_models is None else mediapipe_models + config = self.add_mediapipe_graphs_to_config(config, use_subconfig, mediapipe_models) + + return config, config_custom_nodes + + def prepare_custom_graphs_mediapipe_config_list(self, config, use_subconfig=False, custom_graph_paths=None): + # Mediapipe config.json example: + # { + # "model_config_list": [...], + # "pipeline_config_list": [...], + # "custom_loader_config_list": [...], + # "mediapipe_config_list": [ + # { + # "name": "pipe1", + # "base_path": "/models/pipe1", + # "graph_path": "/models/pipe1/graphdummy.pbtxt", + # "subconfig": "/models/pipe1/subconfig.json" + # } + # ] + # } + config[Config.MEDIAPIPE_CONFIG_LIST] = [] + mediapipe_base_path = str(Path(Paths.MODELS_PATH_INTERNAL, self.name)) + for i, calculator in enumerate(custom_graph_paths): + proto_dict = { + "name": self.name, + "base_path": mediapipe_base_path, + "graph_path": os.path.join(mediapipe_base_path, os.path.basename(calculator)), + } + config[Config.MEDIAPIPE_CONFIG_LIST].append(proto_dict) + if use_subconfig and "subconfig" not in str(proto_dict): + proto_dict["subconfig"] = os.path.join(mediapipe_base_path, Paths.SUBCONFIG_FILE_NAME) + config[Config.MEDIAPIPE_CONFIG_LIST][i].update(proto_dict) + return config + + def add_mediapipe_graphs_to_config(self, config, use_subconfig=False, mediapipe_models=None): + # Mediapipe config.json example: + # { + # "model_config_list": [...], + # "pipeline_config_list": [...], + # "custom_loader_config_list": [...], + # "mediapipe_config_list": [ + # { + # "name": "pipe1", + # "base_path": "/models/pipe1/", + # "graph_path": "/models/pipe1/graphdummy.pbtxt", + # "subconfig": "/models/pipe1/subconfig.json" + # } + # ] + # } + + config[Config.MEDIAPIPE_CONFIG_LIST] = ( + [] if config.get(Config.MEDIAPIPE_CONFIG_LIST) is None else config[Config.MEDIAPIPE_CONFIG_LIST] + ) + for i, model in enumerate(mediapipe_models): + model_name = model.name + graph_name = model_name if (not model.is_llm and model.pbtxt_name is None) \ + else getattr(model, "pbtxt_name", None) + graph_filename = f"{graph_name}.pbtxt" + mediapipe_base_path = str(Path(Paths.MODELS_PATH_INTERNAL, model_name)) + graph_path = str(Path(mediapipe_base_path, graph_filename)) if not model.use_relative_paths \ + else graph_filename + proto_dict = { + "name": model_name, + "base_path": mediapipe_base_path, + "graph_path": graph_path, + } + if proto_dict not in config[Config.MEDIAPIPE_CONFIG_LIST]: + config[Config.MEDIAPIPE_CONFIG_LIST].append(proto_dict) + + for regular_model in model.regular_models: + if use_subconfig and "subconfig" not in str(proto_dict): + subconfig_filename = f"subconfig_{regular_model.name}.json" + proto_dict["subconfig"] = ( + os.path.join(mediapipe_base_path, subconfig_filename) + if not model.use_relative_paths + else subconfig_filename + ) + config[Config.MEDIAPIPE_CONFIG_LIST][i].update(proto_dict) + return config + + def graph_refresh(self): + nodes = [] + for child_node in self.child_nodes: + model = child_node.model + if getattr(child_node, "calculator", None) is not None: + content = child_node.calculator.create_proto_content( + model=model, + input_stream=child_node.input_stream, + output_stream=child_node.output_stream, + create_header=self.create_header, + ) + nodes.append(content) + + calculator_class = PythonCalculator if self.is_python_custom_node else MediaPipeCalculator + header = calculator_class.create_proto_header( + model=None, + input_stream=self.get_input_node().output_names, + output_stream=self.get_output_node().input_names, + ) + full_content = header + " \n\n".join(nodes) + self.graphs = [full_content] + + +class SimpleMediaPipe(MediaPipe): + def __init__(self, model=None, demultiply_count=None, **kwargs): + pipeline = SimplePipeline + if model is None: + model = Resnet() + super().__init__(model, pipeline, demultiply_count, **kwargs) + self.calculators = [OpenVINOModelServerSessionCalculator(model=self), OpenVINOInferenceCalculator(model=self)] + self._initialize([model]) + self.regular_models = self.get_regular_models() + assert not self.name.endswith("_mediapipe") + if kwargs.get("name") is None: + self.name += "_mediapipe" + else: + self.name = kwargs.get("name") + + def _create_nodes(self, models=None): + session_calculator = self.calculators[0] + inference_calculator = self.calculators[1] + + model = models[0] + session_node = MediaPipeGraphNode("node1", model, calculator=session_calculator) + inference_node = MediaPipeGraphNode( + "node2", model, calculator=inference_calculator, input_stream="input", output_stream="output" + ) + + request = MediaPipeGraphNode("request", node_type=NodeType.Input, output_names=["input"]) + output = MediaPipeGraphNode("output", node_type=NodeType.Output, input_names=["output"]) + + NodesConnection.connect(session_node, 0, inference_node, 0) + NodesConnection.connect(inference_node, 0, request, 0) + NodesConnection.connect(output, 0, inference_node, 0) + + return [request, session_node, inference_node, output] + + +class FailedToLoadModelMediaPipe(SimpleMediaPipe): + def __init__(self, model=None, demultiply_count=None, **kwargs): + super().__init__(model, demultiply_count, **kwargs) + self.regular_models = self.get_regular_models() + side_feed_calculator = OpenVINOInferenceCalculator() + api_session_calculator = OpenVINOModelServerSessionCalculator( + model=self, session=MediaPipeCalculator.get_valid_model_name(self.regular_models[0]), model_name=self.name + ) + self.calculators = [side_feed_calculator, api_session_calculator] + + +class ImageClassificationMediaPipe(MediaPipe): + def __init__(self, **kwargs): + Pipeline.__init__(self, "image_classification_pipeline", **kwargs) + super().__init__() + self._initialize() + self.regular_models = self.get_regular_models() + + def _create_nodes(self, models=None): + session_calculator = OpenVINOModelServerSessionCalculator() + inference_calculator = OpenVINOInferenceCalculator() + + googlenet_session_node = MediaPipeGraphNode( + "googlenet_session_node", GoogleNetV2Fp32(), calculator=session_calculator + ) + resnet_session_node = MediaPipeGraphNode("resnet_session_node", Resnet(), calculator=session_calculator) + argmax_session_node = MediaPipeGraphNode("argmax_session_node", ArgMax(), calculator=session_calculator) + + googlenet_inference_node = MediaPipeGraphNode( + "googlenet_inference_node", + GoogleNetV2Fp32(), + calculator=inference_calculator, + input_stream="GOOGLE_INPUT:input_0", + output_stream="GOOGLE_OUTPUT:google_output", + ) + resnet_inference_node = MediaPipeGraphNode( + "resnet_inference_node", + Resnet(), + calculator=inference_calculator, + input_stream="RESNET_INPUT:input_0", + output_stream="RESNET_OUTPUT:resnet_output", + ) + argmax_inference_node = MediaPipeGraphNode( + "argmax_inference_node", + ArgMax(), + calculator=inference_calculator, + input_stream=["ARGMAX_INPUT1:google_output", "ARGMAX_INPUT2:resnet_output"], + output_stream="ARGMAX_OUTPUT:argmax_0", + ) + + request = MediaPipeGraphNode("request", node_type=NodeType.Input, output_names=["input_0"]) + output = MediaPipeGraphNode("output", node_type=NodeType.Output, input_names=["argmax_0"]) + + NodesConnection.connect(googlenet_session_node, 0, googlenet_inference_node, 0) + NodesConnection.connect(resnet_session_node, 0, resnet_inference_node, 0) + NodesConnection.connect(argmax_session_node, 0, argmax_inference_node, 0) + + NodesConnection.connect(googlenet_inference_node, 0, request, 0) + NodesConnection.connect(resnet_inference_node, 0, request, 0) + NodesConnection.connect(argmax_inference_node, 0, googlenet_inference_node, 0) + NodesConnection.connect(argmax_inference_node, 1, resnet_inference_node, 0) + NodesConnection.connect(output, 0, argmax_inference_node, 0) + + nodes = [ + request, + googlenet_session_node, + resnet_session_node, + argmax_session_node, + googlenet_inference_node, + resnet_inference_node, + argmax_inference_node, + output, + ] + return nodes + + +class SimpleModelMediaPipe(MediaPipe): + def __init__(self, model=None, use_mapping=False, batch_size=None, single_mediapipe_model_mode=False, + pbtxt_name=None): + model = Resnet(batch_size=batch_size) if model is None else model + self.__dict__.update(model.__dict__) + super().__init__(model) + self.calculators = [OpenVINOInferenceCalculator(), OpenVINOModelServerSessionCalculator()] + self.regular_models = model.get_regular_models() + assert not self.name.endswith("_mediapipe") + self.name += "_mediapipe" + self.pbtxt_name = pbtxt_name + self.single_mediapipe_model_mode = single_mediapipe_model_mode + if self.single_mediapipe_model_mode: + self.base_path = os.path.join(Paths.MODELS_PATH_INTERNAL, self.name) + + def replace_input_data_names(self, input_data, input_key=None): + new_data = {} + for i, key in enumerate(list(input_data.keys()), start=0): + new_input_key = input_key if input_key is not None else f"in_{i}" + new_data.update({new_input_key: input_data[key]}) + return new_data + + def prepare_input_data(self, batch_size=None, input_key=None): + data = super().prepare_model_input_data(batch_size) + new_data = self.replace_input_data_names(data, input_key) + return new_data + + def prepare_input_data_from_model_datasets(self, batch_size=None, input_key=None): + result = ModelInfo.prepare_input_data_from_model_datasets(self, batch_size) + new_data = self.replace_input_data_names(result, input_key) + return new_data + + def get_expected_model_output_data(self): + expected_model_output_data = {} + for i, key in enumerate(list(self.outputs.keys()), start=0): + new_output_key = f"out_{i}" + expected_model_output_data.update({new_output_key: self.outputs[key]}) + return expected_model_output_data + + def validate_outputs(self, outputs, expected_output_shapes=None, provided_input=None): + assert outputs, "Prediction returned no output" + if expected_output_shapes is None: + expected_output_shapes = list(self.output_shapes.values()) + for i, shape in enumerate(expected_output_shapes): # Check for dynamic shape + for j, val in enumerate(shape): + if val == -1: + expected_output_shapes[i][j] = 1 + + expected_outputs = self.get_expected_model_output_data() + for output_name in expected_outputs: + assert ( + output_name in outputs + ), f"Incorrect output name, expected: {output_name}, found: {', '.join(outputs.keys())}" + output_shapes = [list(o.shape) for o in outputs.values()] + assert any( + shape in expected_output_shapes for shape in output_shapes + ), f"Incorrect output shape, expected: {expected_output_shapes}, found: {output_shapes}." + + @staticmethod + def is_pipeline(): + return False + + def get_models(self): + return [self] + + def prepare_resources(self, base_location): + return super().prepare_model_resources(base_location) + + def get_demultiply_count(self): + return None + + +class SimpleDynamicModelMediaPipe(SimpleModelMediaPipe): + def __init__(self, model, **kwargs): + super().__init__(model=model, **kwargs) + self._initialize(models=[model]) + + def _create_nodes(self, models=None): + nodes = [] + if models is not None: + assert len(models) == 1, f"Currently only single model in {self.__class__.__name__} is supported." + model = models[0] + session_calculator = OpenVINOModelServerSessionCalculator() + inference_calculator = OpenVINOInferenceCalculator() + valid_model_name = MediaPipeCalculator.get_upper_model_name(model) + dummy_node = MediaPipeGraphNode(model.name, model, calculator=session_calculator) + dummy_inference_node = MediaPipeGraphNode( + f"{model.name}_inference_node", + model, + calculator=inference_calculator, + input_stream=f"{valid_model_name}:in_0", + output_stream=f"{valid_model_name}:out_0", + ) + + request = MediaPipeGraphNode("request", node_type=NodeType.Input, output_names=[f"{valid_model_name}:in_0"]) + output = MediaPipeGraphNode("output", node_type=NodeType.Output, input_names=[f"{valid_model_name}:out_0"]) + + NodesConnection.connect(dummy_node, 0, dummy_inference_node, 0) + + nodes = [request, dummy_node, dummy_inference_node, output] + return nodes + + +class SimpleModelMediaPipeResnetWrongInputShapes(SimpleModelMediaPipe): + def __init__(self, model=None, use_mapping=False, batch_size=None): + model = ResnetWrongInputShapes() + super().__init__(model, use_mapping, batch_size) + + +class SimpleModelMediaPipeResnetWrongInputShapeDim(SimpleModelMediaPipe): + def __init__(self, model=None, use_mapping=False, batch_size=None): + model = ResnetWrongInputShapeDim() + super().__init__(model, use_mapping, batch_size) + + +class CorruptedFileModelMediaPipe(SimpleModelMediaPipe): + def __init__(self, model=None): + super().__init__(model) + self.calculators = [CorruptedFileCalculator()] + + +class SimpleOneCalculatorMediaPipe(SimpleModelMediaPipe): + def __init__(self, model=None): + super().__init__(model) + self.calculators = [OVMSOVCalculator()] + assert not self.name.endswith("_mediapipe") + self.name += "_mediapipe" + + +class SameModelsMediaPipe(MediaPipe): + def __init__(self, **kwargs): + Pipeline.__init__(self, "same_models_mediapipe", **kwargs) + super().__init__() + self._initialize() + self.regular_models = self.get_regular_models() + + def _create_nodes(self, models=None): + model = Dummy() + dummy_session_name = f"{model.name}_session" + session1_calculator = OpenVINOModelServerSessionCalculator(session=dummy_session_name) + inference1_calculator = OpenVINOInferenceCalculator(session=dummy_session_name) + inference2_calculator = OpenVINOInferenceCalculator(session=dummy_session_name) + + dummy_session_node = MediaPipeGraphNode("dummy_session_node", model, calculator=session1_calculator) + + dummy1_inference_node = MediaPipeGraphNode( + "dummy1_inference_node", + model, + calculator=inference1_calculator, + input_stream="DUMMY1_INPUT:input", + output_stream="DUMMY1_OUTPUT:dummy1_output", + ) + dummy2_inference_node = MediaPipeGraphNode( + "dummy2_inference_node", + model, + calculator=inference2_calculator, + input_stream="DUMMY2_INPUT:dummy1_output", + output_stream="DUMMY2_OUTPUT:output", + ) + + request = MediaPipeGraphNode("request", node_type=NodeType.Input, output_names=["input"]) + output = MediaPipeGraphNode("output", node_type=NodeType.Output, input_names=["output"]) + + NodesConnection.connect(dummy_session_node, 0, dummy1_inference_node, 0) + NodesConnection.connect(dummy_session_node, 0, dummy2_inference_node, 0) + + NodesConnection.connect(dummy1_inference_node, 0, request, 0) + NodesConnection.connect(dummy2_inference_node, 0, dummy1_inference_node, 0) + NodesConnection.connect(output, 0, dummy2_inference_node, 0) + + nodes = [request, dummy_session_node, dummy1_inference_node, dummy2_inference_node, output] + return nodes + + +class ModelsChainMediaPipe(MediaPipe): + def __init__(self, models=None, demultiply_count=None, **kwargs): + Pipeline.__init__(self, "models_chain_mediapipe", **kwargs) + super().__init__() + if models is None: + models = [Dummy()] + self.chain_length = len(models) + self.demultiply_count = demultiply_count + self._initialize(models) + self.regular_models = self.get_regular_models() + + def _create_nodes(self, models=None): + model = models[0] + inference_nodes = [] + + model_name = MediaPipeCalculator.get_valid_model_name(model) + final_input_name = "input" + final_output_name = "output" + session_calculator = OpenVINOModelServerSessionCalculator(session=model_name) + session_node = MediaPipeGraphNode(f"{model_name}_session_node", model, calculator=session_calculator) + request = MediaPipeGraphNode("request", node_type=NodeType.Input, output_names=[final_input_name]) + output = MediaPipeGraphNode("output", node_type=NodeType.Output, input_names=[final_output_name]) + inference_calculators = [OpenVINOInferenceCalculator(session=model_name) for i in range(self.chain_length)] + + for i, inf_calc in enumerate(inference_calculators): + output_stream = f"{model_name}_{i}_output" + inf_node_name = f"{model_name}_{i}_inference_node" + if i == 0: + inf_node = MediaPipeGraphNode( + inf_node_name, + model, + calculator=inf_calc, + input_stream=final_input_name, + output_stream=output_stream, + ) + elif i == (len(inference_calculators) - 1): + inf_node = MediaPipeGraphNode( + inf_node_name, + model, + calculator=inf_calc, + input_stream=inference_nodes[i - 1].output_stream, + output_stream=final_output_name, + ) + elif 0 < i < len(inference_calculators): + inf_node = MediaPipeGraphNode( + inf_node_name, + model, + calculator=inf_calc, + input_stream=inference_nodes[i - 1].output_stream, + output_stream=output_stream, + ) + inference_nodes.append(inf_node) + + for i, inf_node in enumerate(inference_nodes): + NodesConnection.connect(session_node, 0, inf_node, 0) + if i == 0: + NodesConnection.connect(inf_node, 0, request, 0) + elif i == (len(inference_nodes) - 1): + NodesConnection.connect(output, 0, inference_nodes[i - 1], 0) + elif 0 < i < len(inference_nodes): + NodesConnection.connect(inf_node, 0, inference_nodes[i - 1], 0) + + nodes = [request, output, session_node] + inference_nodes + return nodes diff --git a/tests/functional/constants/requirements.py b/tests/functional/constants/requirements.py new file mode 100644 index 0000000000..ecad4df3f0 --- /dev/null +++ b/tests/functional/constants/requirements.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +class Requirements: + + # model types + onnx = "CVS-29667 ONNX" + + # model operations + status = "CVS-35266_CVS-29674_CVS-30289 get model status" + metadata = "CVS-35266_CVS-29674_CVS-30289 get model metadata" + + # features + parity = "CVS-35266 parity" + model_control_api = "CVS-40334 model control API" + online_modification = "CVS-33847 online model modification" + custom_loader = "CVS-40416 custom loader" + nginx = "CVS-40416 nginx" + dag = "CVS-31796_CVS-36434_CVS-41115 DAG" + model_cache = "CVS-62829 model_cache" + cloud = "CVS-31243 cloud" + stateful = "CVS-33882 stateful" + reshape = "CVS-35266 reshape" + dynamic_shape = "CVS-56655 dynamic shapes" + auto_plugin = "CVS-73689 Auto plugin support" + kfservin_api = "CVS-81053 KFServing api" + metrics = "CVS-43549 metrics" + custom_nodes = "CVS-44359 custom nodes" + audio_endpoint = "CVS-174282 audio endpoint" + + # test types + sdl = "CVS-59335 SDL" + benchmarks = "CVS-35094 benchmarks" + example_client = "CVS-35266 example client apps" + documentation = "CVS-35266 documentation" + binary_input = "CVS-30320 binary input format" + cpu_extension = "CVS-68750 cpu extension" + + operator = "CVS-56873 operator" + + models_enabling = "CVS-105320 models enabling" + triton_async = "CVS-114801 triton async" + + scalar_inputs = "CVS-118200 scalar inputs" + + valgrind = "CVS-125781 valgrind" + cliloader = "CVS-130322 opencl traces" + + streaming_api = "CVS-118064 streaming API extension" + mediapipe = "CVS-103194 mediapipe" + python_custom_node = "CVS-117210 python support" + llm = "CVS-129298 LLM execution in ovms based on c++ code only" + openai_api = "CVS-138033 OpenAI API in OVMS" + hf_imports = "CVS-162541 Direct models import from HF Hub in OVMS" + tools = "CVS-166514 Structured response with tools support in chat/completions" diff --git a/tests/functional/constants/target_device.py b/tests/functional/constants/target_device.py index 3865c69bff..8d06e0f0df 100644 --- a/tests/functional/constants/target_device.py +++ b/tests/functional/constants/target_device.py @@ -14,6 +14,8 @@ # limitations under the License. # +from collections import defaultdict + class TargetDevice: CPU = "CPU" @@ -22,3 +24,15 @@ class TargetDevice: AUTO = "AUTO:GPU,CPU" HETERO = "HETERO:GPU,CPU" AUTO_CPU_GPU = "AUTO:CPU,GPU" + + +MAX_WORKERS_PER_TARGET_DEVICE = defaultdict( + lambda: 1, + { # Quite conservative for any non-listed device + TargetDevice.CPU: 0, # no limits ! + TargetDevice.GPU: 4, + TargetDevice.NPU: 4, + TargetDevice.HETERO: 4, # keep in sync with `TARGET_DEVICE_GPU` + TargetDevice.AUTO: 4, + }, +) diff --git a/tests/functional/constants/target_device_configuration.py b/tests/functional/constants/target_device_configuration.py new file mode 100644 index 0000000000..c4eb95328a --- /dev/null +++ b/tests/functional/constants/target_device_configuration.py @@ -0,0 +1,112 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +try: + from grp import getgrnam +except ImportError: + getgrnam = None + +try: + from os import getuid +except ImportError: + getuid = None + +from tests.functional.constants.os_type import get_host_os +from tests.functional.utils.test_framework import FrameworkMessages, get_parameter_from_item, skip_if + +from tests.functional.config import skip_nginx_test +from tests.functional.constants.target_device import TargetDevice +from tests.functional.constants.ovms import BASE_OS_PARAM_NAME + +NETWORK = "network" +PRIVILEGED = "privileged" +VOLUMES = "volumes" +DEVICES = "devices" +HOST = "host" +DOCKER_PARAMS = "docker_params" +""" TARGET_DEVICE_CONFIGURATION: VOLUMES - this map stores a list of devices that should be + mounted for given target device. + String representing device should be in form of: + - :: + + Another representations are possible + - : + cgroup_permissions will be set to `mrw` by default + + - + path_in_container will be the same as path_on_host + cgroup_permissions will be set to `mrw` + """ +""" TARGET_DEVICE_CONFIGURATION: NETWORK - Name of the network this container will be connected to at creation time. + :type str + """ +""" TARGET_DEVICE_CONFIGURATION: USER - Username or UID to run commands as inside the container. + :type int or str + """ +""" TARGET_DEVICE_CONFIGURATION: PRIVILEGED - Give extended privileges to this container. + :type bool + """ +# Currently docker imports are mandatory (even for non-docker types) and this enforce getuid() & getgrnam(...) syscalls +# for non-docker testruns. +TARGET_DEVICE_CONFIGURATION = { + TargetDevice.CPU: lambda: { + VOLUMES: [], + DEVICES: [], + NETWORK: None, + PRIVILEGED: False, + }, + TargetDevice.GPU: lambda: { + VOLUMES: [], + DEVICES: ["/dev/dri:/dev/dri:mrw"], + NETWORK: None, + PRIVILEGED: False, + DOCKER_PARAMS: {"group_add": [getgrnam("render").gr_gid]}, + }, + TargetDevice.NPU: lambda: { + VOLUMES: [], + DEVICES: ["/dev/accel:/dev/accel:mrw", "/dev/dri:/dev/dri:mrw"], + NETWORK: None, + PRIVILEGED: False, + DOCKER_PARAMS: {"group_add": [getgrnam("render").gr_gid]}, + }, + TargetDevice.AUTO: lambda: { + VOLUMES: [], + DEVICES: ["/dev/dri"], + NETWORK: None, + PRIVILEGED: False, + DOCKER_PARAMS: {"group_add": [getgrnam("render").gr_gid]}, + }, + TargetDevice.HETERO: lambda: { + VOLUMES: [], + DEVICES: ["/dev/dri:/dev/dri:mrw"], + NETWORK: None, + PRIVILEGED: False, + DOCKER_PARAMS: {"group_add": [getgrnam("render").gr_gid]}, + }, +} + + +def nginx_mtls_not_supported_for_test(): + """ + Test or test class should be skipped from execution since is not supported by + nginx_mtls images. + """ + return skip_if(skip_nginx_test, msg=FrameworkMessages.NGINX_IMAGE_NOT_SUPPORTED) + + +def deselect_if_host_os_not_match_docker_base_image_runtime(item): + should_skip = get_host_os() != get_parameter_from_item(item, BASE_OS_PARAM_NAME) + return should_skip diff --git a/tests/functional/common_libs/__init__.py b/tests/functional/data/__init__.py similarity index 100% rename from tests/functional/common_libs/__init__.py rename to tests/functional/data/__init__.py diff --git a/tests/functional/data/ovms_capi_wrapper/Makefile b/tests/functional/data/ovms_capi_wrapper/Makefile new file mode 100644 index 0000000000..d5cbad3afe --- /dev/null +++ b/tests/functional/data/ovms_capi_wrapper/Makefile @@ -0,0 +1,26 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ACTIVATE := $(PYVENV)/bin/activate + +default: ovms_capi_wrapper + +ovms_capi_wrapper: setup.py + . $(ACTIVATE) && python3 include/ovms_autopxd.py -i include/ovms.h -o ovms_capi.pxd + . $(ACTIVATE) && python3 setup.py build_ext --inplace + +clean: + -rm *.c* *.html *.pxd *.h build __pycache__ diff --git a/tests/functional/command_wrappers/__init__.py b/tests/functional/data/ovms_capi_wrapper/__init__.py similarity index 93% rename from tests/functional/command_wrappers/__init__.py rename to tests/functional/data/ovms_capi_wrapper/__init__.py index 3e8e2f6775..84cfbcf566 100644 --- a/tests/functional/command_wrappers/__init__.py +++ b/tests/functional/data/ovms_capi_wrapper/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 Intel Corporation +# Copyright (c) 2026 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/functional/data/ovms_capi_wrapper/ovms_autopxd.py b/tests/functional/data/ovms_capi_wrapper/ovms_autopxd.py new file mode 100644 index 0000000000..2380f0211b --- /dev/null +++ b/tests/functional/data/ovms_capi_wrapper/ovms_autopxd.py @@ -0,0 +1,77 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import sys +from argparse import ArgumentParser +from pathlib import Path + +from autopxd import parse +from autopxd.nodes import Block +from autopxd.writer import AutoPxd, escape +from pycparser import c_ast + + +class OvmsAutoPxd(AutoPxd): + apply_cimport_bool_wa = False # apply WA if any 'bool' type was used in header file. + + def visit_IdentifierType(self, node): + super().visit_IdentifierType(node) + if node.names[0] == 'bool': + self.apply_cimport_bool_wa = True + + def visit_Struct(self, node): + kind = "struct" + if "OVMS" in node.name and node.decls is None: + name = node.name + fields = [] + type_decl = self.child_of(c_ast.TypeDecl, -2) + # add the struct definition to the top level + self.decl_stack[0].append(Block(escape(name, True), fields, kind, "cdef")) + if type_decl: + # inline struct, add a reference to whatever name it was defined on the top level + self.append(escape(name)) + else: + return self.visit_Block(node, kind) + + def translate(self, code): + self.visit(parse(code=code)) + pxd_string = "" + if self.stdint_declarations: + cimports = ", ".join(self.stdint_declarations) + pxd_string += f"from libc.stdint cimport {cimports}\n\n" + if self.apply_cimport_bool_wa: + # Workaround for cython issue: 'bool' is not a type identifier + # https://stackoverflow.com/questions/24659723/cython-issue-bool-is-not-a-type-identifier + pxd_string += "from libcpp cimport bool\n\n" + pxd_string += str(self) + return pxd_string + + +if __name__ == "__main__": + parser = ArgumentParser(description="Script translates OVMS header file to .pxd file") + parser.add_argument("-i", "--input_file", help="OVMS header file path") + parser.add_argument("-o", "--output_file", help=".pxd output file path") + + args = parser.parse_args() + + if len(sys.argv) !=5: + args = parser.parse_args(["-h"]) + + input_file_path = Path(args.input_file) + output_file_path = Path(args.output_file) + + with open(output_file_path, "w") as fo: + fo.write(OvmsAutoPxd(input_file_path.name).translate(input_file_path.read_text())) diff --git a/tests/functional/command_wrappers/server.py b/tests/functional/data/ovms_capi_wrapper/ovms_capi_shared.py similarity index 66% rename from tests/functional/command_wrappers/server.py rename to tests/functional/data/ovms_capi_wrapper/ovms_capi_shared.py index b6234833c3..b8c8ba5178 100644 --- a/tests/functional/command_wrappers/server.py +++ b/tests/functional/data/ovms_capi_wrapper/ovms_capi_shared.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 Intel Corporation +# Copyright (c) 2026 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,9 +14,9 @@ # limitations under the License. # +class OvmsModelNotFound(Exception): + pass -def start_ovms_container_command(start_container_command, command_args): - command = start_container_command - for key, value in command_args.items(): - command += " --{key} {value}".format(key=key, value=value) - return command + +class OvmsInferenceFailed(Exception): + pass diff --git a/tests/functional/data/ovms_capi_wrapper/ovms_capi_wrapper.pyx b/tests/functional/data/ovms_capi_wrapper/ovms_capi_wrapper.pyx new file mode 100644 index 0000000000..ff0278b8c6 --- /dev/null +++ b/tests/functional/data/ovms_capi_wrapper/ovms_capi_wrapper.pyx @@ -0,0 +1,364 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pickle +from functools import reduce +from pathlib import Path + +import numpy as np + +from tests.functional.utils.assertions import CapiException +from tests.functional.data.ovms_capi_wrapper import ovms_capi_shared + +cimport ovms_capi as capi # `ovms_capi.pxd` +from libc.stdint cimport int64_t, uint8_t, uint32_t, uint64_t, uintptr_t +from libc.stdio cimport printf +from libc.stdlib cimport malloc + + +cdef capi.OVMS_Server * srv = NULL +cdef capi.OVMS_ServerSettings * serverSettings = NULL +cdef capi.OVMS_ModelsSettings * modelsSettings = NULL + +LOG_LEVELS = { + "TRACE": capi.OVMS_LogLevel_enum.OVMS_LOG_TRACE, + "DEBUG": capi.OVMS_LogLevel_enum.OVMS_LOG_DEBUG, + "INFO": capi.OVMS_LogLevel_enum.OVMS_LOG_INFO, + "WARNING": capi.OVMS_LogLevel_enum.OVMS_LOG_WARNING, + "ERROR": capi.OVMS_LogLevel_enum.OVMS_LOG_ERROR, +} + +OVMS_DATATYPE_TO_NUMPY = { + capi.OVMS_DATATYPE_FP32: np.float32, + capi.OVMS_DATATYPE_I32: np.int32, + capi.OVMS_DATATYPE_I64: np.int64, + capi.OVMS_DATATYPE_U8: np.uint8, +} + + +def pickled_function_parameters(func, **kwargs): + def wrapper(**kwargs): + global tmp_infile + global tmp_outfile + # Get input parameters as kwargs + _raw_kwargs = Path(tmp_infile).read_bytes() + kwargs = pickle.loads(_raw_kwargs) + print(f"kwargs={kwargs}") + try: + result = func(**kwargs) + except Exception as exc: + result = str(exc) # Store exception + + # Store resulting object + Path(tmp_outfile).write_bytes(pickle.dumps(result)) + return wrapper + + +def ovms_start_server(parameters): + cdef uint32_t grpc_port = parameters["grpc_port"] if parameters["grpc_port"] else 0 + cdef uint32_t rest_port = parameters["rest_port"] if parameters["rest_port"] else 0 + log_level = parameters["log_level"] + + config_path = parameters["config_path"].encode() + cdef char *config_file = config_path + + ovms_get_capi_version() + + capi.OVMS_ServerSettingsNew(&serverSettings) + capi.OVMS_ModelsSettingsNew(&modelsSettings) + capi.OVMS_ServerNew(&srv) + + printf("Starting OVMS CAPI server:\n") + printf("config_path %s\n", config_file) + printf("grpc_port %d\n", grpc_port) + printf("rest_port %d\n", rest_port) + #print(f"log_level {log_level}") + + cdef char *cpu_ext_file = NULL + if "cpu_extension_path" in parameters: + cpu_ext_path = parameters["cpu_extension_path"].encode() + cpu_ext_file = cpu_ext_path + printf("cpu_extension_path %s\n", cpu_ext_file) + capi.OVMS_ServerSettingsSetCpuExtensionPath(serverSettings, cpu_ext_file) + + _poll_wait_seconds = parameters.get("file_system_poll_wait_seconds", None) + if _poll_wait_seconds is not None: + printf("file_system_poll_wait_seconds %d\n", int(_poll_wait_seconds)) + capi.OVMS_ServerSettingsSetFileSystemPollWaitSeconds(serverSettings, int(_poll_wait_seconds)) + + capi.OVMS_ServerSettingsSetGrpcPort(serverSettings, grpc_port) + capi.OVMS_ServerSettingsSetRestPort(serverSettings, rest_port) + capi.OVMS_ServerSettingsSetLogLevel(serverSettings, LOG_LEVELS[log_level]) + capi.OVMS_ModelsSettingsSetConfigPath(modelsSettings, config_file) + + cdef capi.OVMS_Status * res = capi.OVMS_ServerStartFromConfigurationFile(srv, serverSettings, modelsSettings) + if res: + # analyze error from OVMS_Status + printf("OVMS_Status %x\n", res) + else: + printf("Started OVMS Server: %x\n", res) + return int( srv) + +def parse_error(_res): + cdef uint32_t code = 0 + cdef const char * details = NULL + cdef capi.OVMS_Status* res = _res + capi.OVMS_StatusCode(res, &code) + capi.OVMS_StatusDetails(res, &details) + + print(f"Error during inference: code {int(code)}, details: {str(details)}\n") + capi.OVMS_StatusDelete(res) + +def server_stop(): + global srv, modelsSettings, serverSettings + printf("Stopping server: %lX\n", srv) + capi.OVMS_ServerDelete(srv) + srv = NULL + capi.OVMS_ModelsSettingsDelete(modelsSettings) + modelsSettings = NULL + capi.OVMS_ServerSettingsDelete(serverSettings) + serverSettings = NULL + print("Server stopped") + +@pickled_function_parameters +def send_inference(model_name, inputs): + print("Prepare inference") + cdef capi.OVMS_InferenceRequest* request = NULL + py_byte_string = model_name.encode("UTF-8") + cdef const char * _name = py_byte_string + + capi.OVMS_InferenceRequestNew(&request, + srv, + _name, # Model name + 1) + + cdef capi.OVMS_DataType _dataType = capi.OVMS_DATATYPE_UNDEFINED + cdef int64_t * _shape_struct = NULL + cdef float *_inputData = NULL + cdef uint32_t dimensions = 0 + cdef size_t _byteSize = 0 + + for input_name, _input in inputs.items(): + print(f"Processing input name {input_name}") + dimensions = len(_input.shape) + _shape_struct = malloc(dimensions * sizeof(int64_t)) + _byteSize = reduce(lambda x,y: x*y, _input.shape) * _input.dtype.itemsize + _inputData = malloc(_byteSize) + + if _input.dtype == np.float32: + _dataType = capi.OVMS_DATATYPE_FP32 + elif _input.dtype == np.uint8: + _dataType = capi.OVMS_DATATYPE_U8 + elif _input.dtype == np.int32: + _dataType = capi.OVMS_DATATYPE_I32 + elif _input.dtype == np.int64: + _dataType = capi.OVMS_DATATYPE_I64 + elif _input.dtype == np.object_: + _dataType = capi.OVMS_DATATYPE_STRING + else: + raise CapiException(f"Unsupported data type: {_input.dtype}") + + py_byte_string = input_name.encode('UTF-8') + _name = py_byte_string + + for i in range(dimensions): + _shape_struct[i] = _input.shape[i] + + printf("Input name: %s\n", _name) + printf("Input dimensions: %u\n", dimensions) + printf("Input datatype: %u\n", _dataType) + printf("Input bytesize: %u\n", _byteSize) + printf("Input shape: %lu %lu %lu %lu\n", _shape_struct[0], _shape_struct[1], _shape_struct[2], _shape_struct[3]) + + + capi.OVMS_InferenceRequestAddInput(request, + _name, + _dataType, + _shape_struct, + dimensions) + + capi.OVMS_InferenceRequestInputSetData(request, + _name, + _inputData, + _byteSize, + capi.OVMS_BUFFERTYPE_CPU, + 0) + + cdef capi.OVMS_InferenceResponse* response = NULL + cdef capi.OVMS_Status* res = NULL + + print("Sending inference to server: {:x}".format( srv)) + res = capi.OVMS_Inference(srv, request, &response) + print("Received inference from server: {:x}".format( res)) + + cdef uint32_t code = 0 + cdef const char * details = NULL + cdef bytes py_string + + if res: + capi.OVMS_StatusCode(res, &code) + capi.OVMS_StatusDetails(res, &details) + py_string = details + print(f"Inference failed:\ncode={code}\ndetails={details}") + capi.OVMS_StatusDelete(res) + exception = ovms_capi_shared.OvmsInferenceFailed(py_string.decode("UTF-8")) + exception.code = int(code) + return exception + + cdef uint32_t outputCount = 0 + capi.OVMS_InferenceResponseOutputCount(response, &outputCount) + print(f"Inference succeeded, read={outputCount}") + result = {} + + cdef void* voutputData = NULL + cdef size_t bytesize = 0 + cdef uint32_t outputId = - 1 + cdef capi.OVMS_DataType datatype = capi.OVMS_DATATYPE_FP32 + cdef int64_t* out_shape = NULL + cdef size_t dimCount = 0 + cdef capi.OVMS_BufferType bufferType = capi.OVMS_BUFFERTYPE_CPU + cdef uint32_t deviceId = 42 + cdef const char* outputName = NULL + + for i in range(outputCount): + outputId = i + capi.OVMS_InferenceResponseOutput(response, + outputId, + &outputName, + &datatype, + &out_shape, + &dimCount, + &voutputData, + &bytesize, + &bufferType, + &deviceId) + + _shape = [int(out_shape[i]) for i in range(dimCount)] + _size_to_read = 4 * reduce(lambda x,y: x*y, _shape) + _output_bytes = bytes([( voutputData)[i] for i in range(_size_to_read)]) + + print(f"_size_to_read = {_size_to_read}") + _np_array = np.ndarray(_shape, dtype=np.float32, buffer=_output_bytes) + + printf("OutputId %u\n", outputId) + printf("DimCount %u\n", dimCount) + printf("Datatype %u\n", datatype) + printf("BufferType %u\n", bufferType) + printf("Bytesize %u\n", bytesize) + print(f"Shape: {_shape}" ) + printf("Output name %s\n", outputName) + + result[outputName.decode("UTF-8")] = _np_array.tolist() + + return result + + +def ovms_get_capi_version(): + cdef uint32_t major = 0 + cdef uint32_t minor = 0 + + capi.OVMS_ApiVersion(&major, &minor) + printf("C-API version: %x.%x\n", major, minor) + + return int( major), int( minor) + +@pickled_function_parameters +def get_model_meta(servableName, servableVersion): + global tmp_outfile + cdef capi.OVMS_ServableMetadata * servableMetadata = NULL + cdef char * _servableName = NULL + cdef int64_t _servableVersion = 0 + + py_byte_string = servableName.encode('UTF-8') + _servableName = py_byte_string + _servableVersion = 0 if not servableVersion else int(servableVersion) + + printf("Get Model Meta: %s\t%ld", _servableName, _servableVersion) + capi.OVMS_GetServableMetadata(srv, + _servableName, + _servableVersion, + &servableMetadata) + printf("ServableMetadata: %x\n", servableMetadata) + if servableMetadata == NULL: + return ovms_capi_shared.OvmsModelNotFound("OVMS_GetServableMetadata failed") + + result = { + "inputs": [], + "outputs": [] + } + + cdef char *info = NULL + cdef char *name = NULL + cdef capi.OVMS_DataType datatype = 0 + cdef uint32_t id = 0 + cdef size_t dimCount = 0 + cdef uint32_t count = 0 + cdef int64_t *shapeMinArray = NULL + cdef int64_t *shapeMaxArray = NULL + capi.OVMS_ServableMetadataInfo(servableMetadata, + &info) + result["info"] = str(info) + + capi.OVMS_ServableMetadataInputCount(servableMetadata, &count) + result["input_count"] = int(count) + for i in range(result["input_count"]): + id = i + capi.OVMS_ServableMetadataInput(servableMetadata, + id, + &name, + &datatype, + &dimCount, + &shapeMinArray, + &shapeMaxArray) + _shape_min = [int(shapeMinArray[i]) for i in range(dimCount)] + _shape_max = [int(shapeMaxArray[i]) for i in range(dimCount)] + result["inputs"].append({ + "name": name.decode("UTF-8"), + "datatype": OVMS_DATATYPE_TO_NUMPY[datatype], # int(datatype), + "dimCount": int(dimCount), + "shapeMinArray": _shape_min, + "shapeMaxArray": _shape_max, + "shape": _shape_min + }) + + capi.OVMS_ServableMetadataOutputCount(servableMetadata, &count) + result["output_count"] = int(count) + for i in range(result["output_count"]): + id = i + capi.OVMS_ServableMetadataOutput(servableMetadata, + id, + &name, + &datatype, + &dimCount, + &shapeMinArray, + &shapeMaxArray) + _shape_min = [int(shapeMinArray[i]) for i in range(dimCount)] + _shape_max = [int(shapeMaxArray[i]) for i in range(dimCount)] + result["outputs"].append({ + "name": name.decode("UTF-8"), + "datatype": OVMS_DATATYPE_TO_NUMPY[datatype], #int(datatype), + "dimCount": int(dimCount), + "shapeMinArray": _shape_min, + "shapeMaxArray": _shape_max, + "shape": _shape_min + }) + + print(f"Meta: {result}") + + Path(tmp_outfile).write_bytes(pickle.dumps(result)) + + return result + diff --git a/tests/functional/data/ovms_capi_wrapper/setup.py b/tests/functional/data/ovms_capi_wrapper/setup.py new file mode 100644 index 0000000000..2ad71fee96 --- /dev/null +++ b/tests/functional/data/ovms_capi_wrapper/setup.py @@ -0,0 +1,72 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from distutils.core import setup +from distutils.extension import Extension +from pathlib import Path + +from Cython.Build import cythonize +from pyximport import pyximport + + +def build_ovms_capi_wrapper(ovms_capi_wrapper_path, + capi_package_content_path): + includes_dir = str(Path(capi_package_content_path, "../include/")) + lib_dir = str(Path(capi_package_content_path, "lib")) + + # Wrap parameters into distutils generic object + ovms_capi_ext = Extension( + name="lib.ovms_capi_wrapper", + sources=["include/ovms_capi_wrapper.pyx"], + libraries=["ovms_shared"], # libovms_shared.so + language="c", + runtime_library_dirs=[lib_dir], + library_dirs=[lib_dir], + include_dirs=[includes_dir] + ) + + # During this stage pyx+pxd file syntax should be verified + extensions = cythonize( + [ovms_capi_ext], + depfile=True, + annotate=True, + compiler_directives={'language_level': "3"} + ) + return extensions + + +def prepare_dynamic_load(ovms_capi_wrapper_path, capi_package_content_path): + extensions = build_ovms_capi_wrapper(ovms_capi_wrapper_path, capi_package_content_path) + + # Little trick to set up automatic ovms_capi Cython compilation on import. + pyximport.install( + language_level=3, + build_in_temp=False, + build_dir=capi_package_content_path, + setup_args={ + "ext_modules": extensions + } + ) + return + + +if __name__ == "__main__": + # Expect valid extracted capi package in `capi_package_content_path` + capi_cython_extensions = build_ovms_capi_wrapper(ovms_capi_wrapper_path=f"{os.getcwd()}/include/ovms_capi_wrapper.pyx", + capi_package_content_path=os.getcwd()) + extension = cythonize(capi_cython_extensions) + setup(ext_modules=extension) diff --git a/tests/functional/model/__init__.py b/tests/functional/data/python_custom_nodes/__init__.py similarity index 91% rename from tests/functional/model/__init__.py rename to tests/functional/data/python_custom_nodes/__init__.py index ef209eab59..84cfbcf566 100644 --- a/tests/functional/model/__init__.py +++ b/tests/functional/data/python_custom_nodes/__init__.py @@ -1,15 +1,15 @@ -# -# Copyright (c) 2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/reservation_manager_single_port.yml b/tests/functional/data/python_custom_nodes/incrementer/__init__.py similarity index 64% rename from tests/reservation_manager_single_port.yml rename to tests/functional/data/python_custom_nodes/incrementer/__init__.py index 47d1313f94..84cfbcf566 100644 --- a/tests/reservation_manager_single_port.yml +++ b/tests/functional/data/python_custom_nodes/incrementer/__init__.py @@ -1,8 +1,5 @@ ---- -# This file is used in tests on commit for tests using single docker container -# via cli with Makefiles. # -# Copyright (c) 2020 Intel Corporation +# Copyright (c) 2026 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,12 +13,3 @@ # See the License for the specific language governing permissions and # limitations under the License. # -config: - pool_range: - start: 10000 - stop: 30000 - pool_part_size: 1 - locks_dir: /tmp - envs: - slices: - - start: OVMS_CPP_CONTAINTER_PORT diff --git a/tests/functional/data/python_custom_nodes/incrementer/incrementer.py b/tests/functional/data/python_custom_nodes/incrementer/incrementer.py new file mode 100644 index 0000000000..9d126a6995 --- /dev/null +++ b/tests/functional/data/python_custom_nodes/incrementer/incrementer.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import List + +from pyovms import Tensor + +# Fetched from: +# https://docs.openvino.ai/nightly/ovms_docs_python_support_reference.html#basic-example + +class OvmsPythonModel: + # Assuming this code is used with nodes + # that have single input and single output + + def initialize(self, kwargs: dict): + self.node_name = kwargs["node_name"] + self.input_names = kwargs["input_names"] + self.output_names = kwargs["output_names"] + + def execute(self, inputs: list): + text = [bytes(input).decode() for input in inputs] + incremented_text = self.increment(text) + outputs_list = [Tensor(output_name, incremented_text[i]) for i, output_name in enumerate(self.output_names)] + return outputs_list + + @staticmethod + def increment(text_input_data: List[bytes]): + data = [(text * 2).encode() for text in text_input_data] + return data diff --git a/tests/functional/data/python_custom_nodes/ovms_basic/__init__.py b/tests/functional/data/python_custom_nodes/ovms_basic/__init__.py new file mode 100644 index 0000000000..84cfbcf566 --- /dev/null +++ b/tests/functional/data/python_custom_nodes/ovms_basic/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/functional/data/python_custom_nodes/ovms_basic/python_model.py b/tests/functional/data/python_custom_nodes/ovms_basic/python_model.py new file mode 100644 index 0000000000..0bf0259837 --- /dev/null +++ b/tests/functional/data/python_custom_nodes/ovms_basic/python_model.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import inspect +from typing import List + +# Fetched from: +# https://raw.githubusercontent.com/openvinotoolkit/model_server/main/docs/python_support/quickstart.md + +class OvmsPythonModel: + def initialize(self, kwargs: dict): + self.node_name = kwargs["node_name"] + self.input_names = kwargs["input_names"] + self.output_names = kwargs["output_names"] + self.class_methods = { + name: func for name, func in inspect.getmembers(OvmsPythonModel, predicate=inspect.isfunction) + } + + def execute(self, inputs: list): + text_input_data = [bytes(input).decode() for input in inputs] + + # Expected method should have the same name as self.node_name. + func = self.class_methods.get(self.node_name, None) + assert func is not None, f"Function for {self.node_name} is None: {func}" + output_data = func(self, text_input_data=text_input_data) + + # Create Tensor with encoded text. + # Should be consistent with the value set in PythonCalculator. + # A list of Tensors is expected, even if there's only one output. + from pyovms import Tensor + outputs_list = [Tensor(output_name, output_data[i]) for i, output_name in enumerate(self.output_names)] + return outputs_list + + def upper_text(self, text_input_data: List[bytes]): + data = [text.upper().encode() for text in text_input_data] + return data diff --git a/tests/functional/data/python_custom_nodes/ovms_basic/python_model_loopback.py b/tests/functional/data/python_custom_nodes/ovms_basic/python_model_loopback.py new file mode 100644 index 0000000000..db9e70096f --- /dev/null +++ b/tests/functional/data/python_custom_nodes/ovms_basic/python_model_loopback.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import inspect + + +class OvmsPythonModel: + def initialize(self, kwargs: dict): + self.node_name = kwargs["node_name"] + self.input_names = kwargs["input_names"] + self.output_names = [output_name for output_name in kwargs["output_names"] if output_name != "loopback"] + self.class_methods = {name: func for name, func in + inspect.getmembers(OvmsPythonModel, predicate=inspect.isfunction)} + + def execute(self, inputs: list): + input_data = inputs[0] + text = bytes(input_data).decode() + + # Expected method should have the same name as self.node_name. + func = self.class_methods.get(self.node_name, None) + assert func is not None, f"Function for {self.node_name} is None: {func}" + + # Create Tensor with encoded text. + # Should be consistent with the value set in PythonCalculator. + # A list of Tensors is expected, even if there's only one output. + from pyovms import Tensor + for i in range(len(text)): + output_data = func(self, text=text, counter=i) + outputs_list = [Tensor(output, output_data) for output in self.output_names] + yield outputs_list + + def loopback_upper_text(self, text: bytes, counter): + data = text[:counter] + text[counter].upper() + text[counter + 1:] + return data.encode() diff --git a/tests/functional/data/python_custom_nodes/ovms_corrupted/__init__.py b/tests/functional/data/python_custom_nodes/ovms_corrupted/__init__.py new file mode 100644 index 0000000000..84cfbcf566 --- /dev/null +++ b/tests/functional/data/python_custom_nodes/ovms_corrupted/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_corrupted_import.py b/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_corrupted_import.py new file mode 100644 index 0000000000..a094fd6fee --- /dev/null +++ b/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_corrupted_import.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import corrupted + + +class OvmsPythonModel: + + def execute(self, inputs: list): + return inputs diff --git a/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_exceptions.py b/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_exceptions.py new file mode 100644 index 0000000000..8099a29ace --- /dev/null +++ b/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_exceptions.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +class OvmsPythonModel: + + EXCEPTION_INITIALIZE = "exception_initialize" + EXCEPTION_EXECUTE = "exception_execute" + EXCEPTION_FINALIZE = "exception_finalize" + + def initialize(self, kwargs: dict): + self.node_name = kwargs["node_name"] + self.input_names = kwargs["input_names"] + self.output_names = kwargs["output_names"] + if self.node_name == self.EXCEPTION_INITIALIZE: + raise Exception(self.EXCEPTION_INITIALIZE) + + def execute(self, inputs: list): + if self.node_name == self.EXCEPTION_EXECUTE: + raise Exception(self.EXCEPTION_EXECUTE) + input_data = inputs[0] + text = bytes(input_data).decode() + output_data = text.upper().encode() + # Create Tensor with encoded text. + # Should be consistent with the value set in PythonCalculator. + # A list of Tensors is expected, even if there's only one output. + from pyovms import Tensor + outputs_list = [Tensor(output, output_data) for output in self.output_names] + return outputs_list + + def finalize(self): + if self.node_name == self.EXCEPTION_FINALIZE: + raise Exception(self.EXCEPTION_FINALIZE) diff --git a/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_loopback_multiple_use_of_valid_outputs.py b/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_loopback_multiple_use_of_valid_outputs.py new file mode 100644 index 0000000000..653e83b0c3 --- /dev/null +++ b/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_loopback_multiple_use_of_valid_outputs.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import inspect + + +class OvmsPythonModel: + def initialize(self, kwargs: dict): + self.node_name = kwargs["node_name"] + self.input_names = kwargs["input_names"] + self.output_names = [output_name for output_name in kwargs["output_names"] if output_name != "loopback"] + self.class_methods = { + name: func for name, func in inspect.getmembers(OvmsPythonModel, predicate=inspect.isfunction) + } + self.alternative_input_text = "Alternative input text here" + + def execute(self, inputs: list): + input_data = inputs[0] + text = bytes(input_data).decode() + + # Expected method should have the same name as self.node_name. + func = self.class_methods.get(self.node_name, None) + assert func is not None, f"Function for {self.node_name} is None: {func}" + + # Create Tensor with encoded text. + # Should be consistent with the value set in PythonCalculator. + # A list of Tensors is expected, even if there's only one output. + from pyovms import Tensor + + for i in range(len(text)): + output_data = func(self, text=text, counter=i) + alternative_output_data = func(self, text=self.alternative_input_text, counter=i) + outputs_list = [Tensor(output, output_data) for output in self.output_names] + outputs_list += [Tensor(output, output_data) for output in self.output_names] + outputs_list += [Tensor(output, alternative_output_data) for output in self.output_names] + yield outputs_list + + def loopback_upper_text(self, text: bytes, counter): + data = text[:counter] + text[counter].upper() + text[counter + 1 :] + return data.encode() diff --git a/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_loopback_return_instead_of_yield.py b/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_loopback_return_instead_of_yield.py new file mode 100644 index 0000000000..de6be9c70c --- /dev/null +++ b/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_loopback_return_instead_of_yield.py @@ -0,0 +1,50 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import inspect + + +class OvmsPythonModel: + def initialize(self, kwargs: dict): + self.node_name = kwargs["node_name"] + self.input_names = kwargs["input_names"] + self.output_names = [output_name for output_name in kwargs["output_names"] if output_name != "loopback"] + self.class_methods = { + name: func for name, func in inspect.getmembers(OvmsPythonModel, predicate=inspect.isfunction) + } + + def execute(self, inputs: list): + input_data = inputs[0] + text = bytes(input_data).decode() + + # Expected method should have the same name as self.node_name. + func = self.class_methods.get(self.node_name, None) + assert func is not None, f"Function for {self.node_name} is None: {func}" + + # Create Tensor with encoded text. + # Should be consistent with the value set in PythonCalculator. + # A list of Tensors is expected, even if there's only one output. + from pyovms import Tensor + + outputs_list = [] + for i in range(len(text)): + output_data = func(self, text=text, counter=i) + outputs_list += [Tensor(output, output_data) for output in self.output_names] + return outputs_list + + def loopback_upper_text(self, text: bytes, counter): + data = text[:counter] + text[counter].upper() + text[counter + 1 :] + return data.encode() diff --git a/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_missing_execute.py b/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_missing_execute.py new file mode 100644 index 0000000000..11a5d8c3a7 --- /dev/null +++ b/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_missing_execute.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +class OvmsPythonModel: + pass diff --git a/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_writing_to_loopback_output_in_execute.py b/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_writing_to_loopback_output_in_execute.py new file mode 100644 index 0000000000..8cd3db94be --- /dev/null +++ b/tests/functional/data/python_custom_nodes/ovms_corrupted/python_model_writing_to_loopback_output_in_execute.py @@ -0,0 +1,50 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import inspect + + +class OvmsPythonModel: + def initialize(self, kwargs: dict): + self.node_name = kwargs["node_name"] + self.input_names = kwargs["input_names"] + self.output_names = ["loopback"] + self.class_methods = { + name: func for name, func in inspect.getmembers(OvmsPythonModel, predicate=inspect.isfunction) + } + print(f'kwargs in writing to loopback: {kwargs["output_names"]}') + + def execute(self, inputs: list): + input_data = inputs[0] + text = bytes(input_data).decode() + + # Expected method should have the same name as self.node_name. + func = self.class_methods.get(self.node_name, None) + assert func is not None, f"Function for {self.node_name} is None: {func}" + + # Create Tensor with encoded text. + # Should be consistent with the value set in PythonCalculator. + # A list of Tensors is expected, even if there's only one output. + from pyovms import Tensor + + for i in range(len(text)): + output_data = func(self, text=text, counter=i) + outputs_list = [Tensor(output, output_data) for output in self.output_names] + yield outputs_list + + def loopback_upper_text(self, text: bytes, counter): + data = text[:counter] + text[counter].upper() + text[counter + 1 :] + return data.encode() diff --git a/tests/functional/fixtures/api_type.py b/tests/functional/fixtures/api_type.py new file mode 100644 index 0000000000..916f3da242 --- /dev/null +++ b/tests/functional/fixtures/api_type.py @@ -0,0 +1,106 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import itertools +import pytest + +from tests.functional.config import ovms_types +from tests.functional.constants.ovms_type import OvmsType +from tests.functional.utils.inference.communication import GRPC, REST +from tests.functional.utils.inference.inference_client_factory import InferenceClientFactory +from tests.functional.utils.inference.serving import KFS, OPENAI, TFS, TRITON, COHERE + + +def api_type_non_fixture(serving, communication, ovms_type=None): + return InferenceClientFactory.get_client(serving=serving, communication=communication, ovms_type=ovms_type) + + +_possible_api_types = list(itertools.product([TFS, KFS], [GRPC, REST])) +if OvmsType.CAPI in ovms_types: + _possible_api_types += [OvmsType.CAPI] + + +@pytest.fixture(scope="session", params=_possible_api_types, ids=lambda x: f":".join(x).upper() if len(x) == 2 else x) +def api_type(request): + if request.param == OvmsType.CAPI: + return api_type_non_fixture(serving=None, communication=None, ovms_type=request.param) + else: + return api_type_non_fixture(*request.param, ovms_type=None) + + +@pytest.fixture(scope="session", params=itertools.product([TFS], [GRPC, REST]), ids=lambda x: f":".join(x).upper()) +def tfs_api_type(request): + return api_type_non_fixture(*request.param) + + +@pytest.fixture(scope="session", params=[(TFS, REST)], ids=lambda x: f":".join(x).upper()) +def tfs_rest_api_type(request): + return api_type_non_fixture(*request.param) + + +@pytest.fixture(scope="session", params=[(TFS, GRPC)], ids=lambda x: f":".join(x).upper()) +def tfs_grpc_api_type(request): + return api_type_non_fixture(*request.param) + + +@pytest.fixture(scope="session", params=[(KFS, GRPC)], ids=lambda x: f":".join(x).upper()) +def kfs_grpc_api_type(request): + return api_type_non_fixture(*request.param) + + +@pytest.fixture(scope="session", params=[(KFS, REST)], ids=lambda x: f":".join(x).upper()) +def kfs_rest_api_type(request): + return api_type_non_fixture(*request.param) + + +@pytest.fixture(scope="session", params=itertools.product([KFS], [GRPC, REST]), ids=lambda x: f":".join(x).upper()) +def kfs_api_type(request): + return api_type_non_fixture(*request.param) + + +@pytest.fixture(scope="session", params=[(OPENAI, REST)], ids=lambda x: f":".join(x).upper()) +def openai_rest_api_type(request): + return api_type_non_fixture(*request.param) + + +@pytest.fixture(scope="session", params=[(COHERE, REST)], ids=lambda x: f":".join(x).upper()) +def cohere_rest_api_type(request): + return api_type_non_fixture(*request.param) + + +@pytest.fixture(scope="session", params=itertools.product([TRITON], [GRPC, REST]), ids=lambda x: f":".join(x).upper()) +def triton_api_type(request): + return api_type_non_fixture(*request.param) + + +@pytest.fixture(scope="session", params=[(TRITON, GRPC)], ids=lambda x: f":".join(x).upper()) +def triton_grpc_api_type(request): + return api_type_non_fixture(*request.param) + + +@pytest.fixture(scope="session", params=[(TRITON, REST)], ids=lambda x: f":".join(x).upper()) +def triton_rest_api_type(request): + return api_type_non_fixture(*request.param) + + +@pytest.fixture(scope="session", params=itertools.product([KFS, TFS], [REST]), ids=lambda x: f":".join(x).upper()) +def rest_api_type(request): + return api_type_non_fixture(*request.param) + + +@pytest.fixture(scope="session", params=itertools.product([KFS, TFS], [GRPC]), ids=lambda x: f":".join(x).upper()) +def grpc_api_type(request): + return api_type_non_fixture(*request.param) diff --git a/tests/functional/fixtures/common_fixtures.py b/tests/functional/fixtures/common_fixtures.py deleted file mode 100644 index 325e8ca642..0000000000 --- a/tests/functional/fixtures/common_fixtures.py +++ /dev/null @@ -1,79 +0,0 @@ -# -# Copyright (c) 2026 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import docker -import grpc # noqa -import logging -import re - -import pytest - -from tensorflow_serving.apis import prediction_service_pb2_grpc, model_service_pb2_grpc # noqa - -from tests.functional.config import image -from tests.functional.constants.constants import MODEL_SERVICE, PREDICTION_SERVICE -from tests.functional.utils.cleanup import get_docker_client - -logger = logging.getLogger(__name__) - - -@pytest.fixture(scope="session") -def get_docker_context(request): - client = get_docker_client() - request.addfinalizer(client.close) - return client - - -@pytest.fixture() -def create_grpc_channel(): - def _create_channel(address: str, service: int): - channel = grpc.insecure_channel(address) - if service == MODEL_SERVICE: - return model_service_pb2_grpc.ModelServiceStub(channel) - elif service == PREDICTION_SERVICE: - return prediction_service_pb2_grpc.PredictionServiceStub(channel) - return None - - return _create_channel - - -def get_docker_image_os_version_from_container(): - client = docker.from_env() - cmd = 'cat /etc/os-release' - os_distname = "__invalid__" - try: - output = client.containers.run(image=image, entrypoint=cmd) - output = output.decode("utf-8") - os_distname = re.search('^PRETTY_NAME="(.+)"\n', output, re.MULTILINE).group(1) - except AttributeError as e: - logger.error(f"Cannot find complete os version information.\n{cmd}\n{output}") - - return os_distname - - -def get_ov_and_ovms_version_from_container(): - client = docker.from_env() - cmd = "/ovms/bin/ovms --version" - _ov_version, _ovms_version = ["__invalid__"] * 2 - try: - output = client.containers.run(image=image, entrypoint=cmd) - output = output.decode("utf-8") - _ovms_version = re.search('OpenVINO Model Server (.+)\n', output, re.MULTILINE).group(1) - _ov_version = re.search('OpenVINO backend (.+)\n', output, re.MULTILINE).group(1) - except AttributeError as e: - logger.error(f"Cannot find complete ovms version information.\n{cmd}\n{output}") - - return _ov_version, _ovms_version diff --git a/tests/functional/fixtures/model_conversion_fixtures.py b/tests/functional/fixtures/model_conversion_fixtures.py deleted file mode 100644 index d883410bf3..0000000000 --- a/tests/functional/fixtures/model_conversion_fixtures.py +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright (c) 2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import pytest -import shutil - -import tests.functional.config as config -from tests.functional.fixtures.model_download_fixtures import download_file -from tests.functional.model.models_information import Resnet, ResnetBS4, ResnetBS8 -import logging -from tests.functional.utils.model_management import convert_model - -logger = logging.getLogger(__name__) - - -@pytest.fixture(scope="session") -def resnet_multiple_batch_sizes(get_docker_context, prepare_json): - resnet_to_convert = [Resnet, ResnetBS4, ResnetBS8] - converted_models = [] - tensorflow_model_path = download_file(model_url_base=Resnet.url, model_name=Resnet.name, - directory=os.path.join(config.test_dir_cache, Resnet.local_conversion_dir), - extension=Resnet.download_extensions[0], - full_path=True) - - for resnet in resnet_to_convert: - logger.info("Converting model {}".format(resnet.name)) - input_shape = list(resnet.input_shape) - - converted_model = convert_model(get_docker_context, tensorflow_model_path, - config.path_to_mount_cache + '/{}/{}'.format(resnet.name, resnet.version), - resnet.name, input_shape) - converted_models.append(converted_model) - - return converted_models - - -@pytest.fixture(scope="session") -def copy_cached_resnet_models(resnet_multiple_batch_sizes, prepare_json): - cached_resnet_models = resnet_multiple_batch_sizes - - for cached_model_path_bin, _ in cached_resnet_models: - cached_model_dir = os.path.dirname(cached_model_path_bin) - dest_model_dir = cached_model_dir.replace(config.path_to_mount_cache, config.path_to_mount) - - logger.info("Copying resnet model from cache to {}".format(dest_model_dir)) - if not os.path.exists(dest_model_dir): - shutil.copytree(cached_model_dir, dest_model_dir) diff --git a/tests/functional/fixtures/model_download_fixtures.py b/tests/functional/fixtures/model_download_fixtures.py deleted file mode 100644 index cdda326657..0000000000 --- a/tests/functional/fixtures/model_download_fixtures.py +++ /dev/null @@ -1,70 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os - -import pytest -import shutil - -import tests.functional.config as config -from tests.functional.model.models_information import AgeGender, PVBDetection, PVBFaceDetectionV2, FaceDetection, PVBFaceDetectionV1, ResnetONNX -from tests.functional.utils.model_management import download_missing_file -import logging - -logger = logging.getLogger(__name__) - -models_to_download = [AgeGender, FaceDetection, PVBDetection, PVBFaceDetectionV1, PVBFaceDetectionV2, ResnetONNX] - - -def download_file(model_url_base, model_name, directory, extension, model_version = None, full_path = False): - if model_version: - local_model_path = os.path.join(directory, model_name, model_version) - else: - local_model_path = os.path.join(directory, model_name) - - if not os.path.exists(local_model_path): - os.makedirs(local_model_path) - - local_model_full_path = os.path.join(local_model_path, model_name + extension) - - download_missing_file(model_url_base + extension, local_model_full_path) - - if full_path: - return local_model_full_path - - return os.path.join(directory, model_name) - - -@pytest.fixture(scope="session") -def models_downloader(): - models_paths = {} - for model in models_to_download: - for extension in model.download_extensions: - models_paths[model.name] = download_file(model.url, model.name, config.path_to_mount_cache, extension, - str(model.version)) - return models_paths - - -@pytest.fixture(scope="session") -def copy_cached_models_to_test_dir(models_downloader): - cached_models_paths = models_downloader - - for model_name, cached_model_dir in cached_models_paths.items(): - dest_model_dir = os.path.join(config.path_to_mount, model_name) - - logger.info("Copying model {} from cache to {}".format(model_name, dest_model_dir)) - if not os.path.exists(dest_model_dir): - shutil.copytree(cached_model_dir, dest_model_dir) diff --git a/tests/functional/fixtures/ovms.py b/tests/functional/fixtures/ovms.py new file mode 100644 index 0000000000..5c86077615 --- /dev/null +++ b/tests/functional/fixtures/ovms.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import platform +import time +from typing import Callable + +import pytest +from _pytest.fixtures import FixtureRequest + +from tests.functional.utils.context import Context +from tests.functional.utils.environment_info import EnvironmentInfo +from tests.functional.utils.inference.communication import GRPC, REST +from tests.functional.utils.logger import get_logger +from tests.functional.utils.marks import MarkGeneral +from tests.functional.constants.os_type import OsType +from tests.functional.utils.port_manager import PortManager +from tests.functional.utils.test_framework import generate_test_object_name, skip_if_runtime + +from tests.functional import config +from tests.functional.config import ( + build_test_image, + delay_between_test, + pytest_global_session_timeout, + run_ovms_with_opencl_trace, + run_ovms_with_valgrind, +) +from tests.functional.models import ModelInfo +from tests.functional.constants.ovms import CurrentOvmsType, CurrentTarget +from tests.functional.constants.ovms_binaries import calculate_ovms_binary_name +from tests.functional.constants.ovms_images import ( + calculate_ovms_binary_image_name, + calculate_ovms_image_name, + calculate_ovms_test_image_name, +) +from tests.functional.constants.ovms_type import OvmsType +from tests.functional.constants.requirements import Requirements +from tests.functional.object_model.cpu_extension import CpuExtension +from tests.functional.object_model.test_environment import TestEnvironment +from tests.functional.object_model.ovms_info import OvmsInfo + +logger = get_logger(__name__) + + +@pytest.fixture(scope="function") +def add_test_finalizer(request: FixtureRequest) -> Callable[..., None]: + return request.addfinalizer + + +@pytest.fixture(scope="function", autouse=True) +def context(request, sigterm_cleaner, target_device, ovms_type, base_os): + context = Context(request.scope, request.node.nodeid, request.node) + request.addfinalizer(context.cleanup) + sigterm_cleaner.test_objects.append(context) + + context.target_device = target_device + context.ovms_type = ovms_type + context.base_os = base_os + logger.info(f"Running test on platform: {platform.node()}") + if base_os == OsType.Windows: + context.ovms_image = None + else: + context.ovms_image = calculate_ovms_image_name(context.target_device, context.base_os) + if ovms_type == OvmsType.BINARY_DOCKER: + context.ovms_image = calculate_ovms_binary_image_name(context.ovms_image) + if OvmsType.DOCKER not in ovms_type: + context.ovms_binary = calculate_ovms_binary_name(context.base_os) + context.ovms_sessions = [] + + # Check if tests is marked by @pytest.mark.reqids(...) with libs build in ovms-testing-image + reqids_node = [x for x in request.node.own_markers if x.name == MarkGeneral.REQIDS.value] + reqids_parent = [x for x in request.node.parent.own_markers if x.name == MarkGeneral.REQIDS.value] + requirements_with_external_libraries = [ + Requirements.custom_loader, + Requirements.cpu_extension, + Requirements.custom_nodes, + Requirements.valgrind, + ] + classes_with_external_libraries_used = ["TestByXCli2"] + use_ovms_testing_image = any([ + reqids_node + and any([x in requirements_with_external_libraries for x in reqids_node[0].args]), # By requirement id + reqids_parent and any([x in requirements_with_external_libraries for x in reqids_parent[0].args]), + request.node.parent.name in classes_with_external_libraries_used, + ]) + # Currently we enable testing image only for test that require custom build binaries: + # (custom nodes, cpu_extensions, etc.). In near future we wish to use only ovms-testing-image. + if use_ovms_testing_image or run_ovms_with_valgrind or run_ovms_with_opencl_trace: + if not build_test_image: + logger.warning(f"Skipping test {request.node.name} because lack of built ovms-testing-image") + skip_if_runtime(True, msg="ovms-testing-image was not built") + context.ovms_test_image = calculate_ovms_test_image_name(context.ovms_image) + + if request.cls.__name__ == "TestStraceOvmsMonitor": + context.ovms_image = f"{context.ovms_image}-strace" + + context.env_info = EnvironmentInfo.get_instance(class_info=OvmsInfo, image=context.ovms_image) + + CpuExtension.base_os = context.base_os + ModelInfo.base_os = context.base_os + ModelInfo.target_device = context.target_device + # Setup helper class for is_*_target + CurrentTarget.target_device = context.target_device + # Setup helper class for is_*_type + CurrentOvmsType.ovms_type = context.ovms_type + + context.port_manager_grpc = PortManager( + GRPC, starting_port=config.grpc_ovms_starting_port, pool_size=config.ports_pool_size + ) + context.port_manager_rest = PortManager( + REST, starting_port=config.rest_ovms_starting_port, pool_size=config.ports_pool_size + ) + + context.test_object_name = generate_test_object_name() + return context + + +@pytest.fixture(scope="function", autouse=bool(delay_between_test)) +def pause_after_test(): + yield + time.sleep(delay_between_test) + + +@pytest.fixture(scope="function", autouse=True) +def set_test_environment(tmpdir): + TestEnvironment.current = TestEnvironment(tmpdir) + + +@pytest.fixture(autouse=True) +def check_session_time(request): + elapsed = time.time() - request.session.start_time + if elapsed > (pytest_global_session_timeout * 60 * 60): + msg = f"Pytest exited due to session timeout: {elapsed}s elapsed" + logger.error(msg) + pytest.exit(msg, returncode=-1) diff --git a/tests/functional/fixtures/params.py b/tests/functional/fixtures/params.py new file mode 100644 index 0000000000..569300460b --- /dev/null +++ b/tests/functional/fixtures/params.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import pytest + +from tests.functional.utils.logger import get_logger +from tests.functional.utils.test_framework import create_venv_and_install_packages + +from tests.functional.config import tmp_dir +from tests.functional.constants.ovms import MediapipeIntermediatePacket, Ovms +from tests.functional.constants.paths import Paths + +logger = get_logger(__name__) + + +@pytest.fixture(scope="session", params=[True, False], ids=["delete_enable_file", "erase_enable_file"]) +def delete_enable_file(request): + return request.param + + +@pytest.fixture(scope="session", params=[Ovms.JPG_IMAGE_FORMAT, Ovms.PNG_IMAGE_FORMAT]) +def image_format(request): + return request.param + + +@pytest.fixture(scope="session", params=[Ovms.LAYOUT_NHWC, Ovms.LAYOUT_NCHW]) +def layout(request): + return request.param + + +@pytest.fixture(scope="function", params=[True, False], ids=["with_config", "without_config"]) +def use_config(request): + return request.param + + +@pytest.fixture(scope="function", params=[True, False], ids=["with_subconfig", "without_subconfig"]) +def use_subconfig(request): + return request.param + + +@pytest.fixture(scope="function", params=[True, False], ids=["relative_paths", "absolute_paths"]) +def use_relative_paths(request): + return request.param + + +@pytest.fixture(scope="function", params=["basic", "full"]) +def valgrind_mode(request): + return request.param + + +@pytest.fixture(scope="function", params=Ovms.V2_OPERATIONS) +def operation(request): + return request.param + + +@pytest.fixture(scope="function", params=[x.name for x in list(MediapipeIntermediatePacket)]) +def mediapipe_intermediate_type_graph(request): + return request.param + + +@pytest.fixture(scope="session") +def optimum_cli_activate_path(): + # prepare venv with optimum-cli needed for downloading models that require conversion + venv_activate_path = create_venv_and_install_packages( + os.path.join(tmp_dir, "optimum_cli_requirements"), + requirements_file_path=Paths.LLM_EXPORT_MODELS_REQUIREMENTS, + ) + return venv_activate_path diff --git a/tests/functional/fixtures/server.py b/tests/functional/fixtures/server.py new file mode 100644 index 0000000000..8fc012a572 --- /dev/null +++ b/tests/functional/fixtures/server.py @@ -0,0 +1,184 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import signal +import sys + +import pytest + +from tests.functional.utils.assertions import ( + OvmsTestException, + UnexpectedResponseError, +) +from tests.functional.utils.context import Context +from tests.functional.utils.logger import get_logger, log_fixture +from tests.functional.utils.test_framework import skip_if_runtime + +from tests.functional.config import ( + resource_monitor_enabled, + run_ovms_with_opencl_trace, + run_ovms_with_valgrind, +) +from tests.functional.constants.ovms_binaries import get_binaries +from tests.functional.constants.ovms_type import OvmsType +from tests.functional.object_model.tools import Cliloader, Valgrind +from tests.functional.object_model.ovms_binary import start_binary_ovms +from tests.functional.object_model.ovms_capi import OvmsCapiParams, start_capi_ovms +from tests.functional.object_model.ovms_docker import OvmsDockerLauncher, OvmsDockerParams +from tests.functional.object_model.ovms_instance import OvmsRunContext +from tests.functional.object_model.test_environment import TestEnvironment + +logger = get_logger(__name__) + + +class SigtermCleaner: + def __init__(self): + self.test_objects = [] + signal.signal(signal.SIGTERM, self.sigterm_handle) + + def sigterm_handle(self, _signo, _stack_frame): + logger.warning(f"Received signal={_signo} from:\n{_stack_frame}") + while len(self.test_objects) > 0: + item = self.test_objects.pop() + name = getattr(item, "name", str(type(item))) + logger.warning(f"Cleanup test object: {name}") + try: + item.cleanup() + except (UnexpectedResponseError, AssertionError) as exc: + logger.exception(str(exc)) + pass + sys.exit(1) + + +@pytest.fixture(scope="session") +def sigterm_cleaner(): + return SigtermCleaner() + + +def start_ovms( + context: Context, + parameters, + environment: dict = None, + ensure_started: bool = True, + entrypoint=None, + entrypoint_params=None, + ovms_type_to_start=None, + ensure_nodeport: bool = True, + ovms_instance_params=None, + valgrind_mode=None, + terminate_signal_type=None, + terminate_method=None, + timeout=None, + **kwargs, +): + if ovms_type_to_start is None: + ovms_type_to_start = context.ovms_type + + context.terminate_signal_type = terminate_signal_type + context.terminate_method = terminate_method + + if run_ovms_with_valgrind or valgrind_mode is not None: + entrypoint = Valgrind.name + valgrind_mode = "basic" if valgrind_mode is None else valgrind_mode + entrypoint_params = Valgrind.get_valgrind_params(valgrind_mode) + if run_ovms_with_opencl_trace: + entrypoint = Cliloader.path + env = Cliloader.env + if environment is not None: + environment.update(env) + else: + environment = env + + if ovms_type_to_start in (OvmsType.DOCKER, OvmsType.DOCKER_CMD_LINE, OvmsType.BINARY_DOCKER): + ovms_docker_params = rewrite_parameters(inherit_class=OvmsDockerParams, base_class_object=parameters) + result = start_docker_ovms( + context, + ovms_docker_params, + environment, + ovms_type_to_start, + ovms_instance_params, + entrypoint, + entrypoint_params, + ) + elif ovms_type_to_start == OvmsType.BINARY: + ovms_binary_path = kwargs.get("ovms_binary_path", None) + ovms_binary_name = kwargs.get("ovms_binary_name", None) + if ovms_binary_path is None or ovms_binary_name is None: + ovms_binary_path, ovms_binary_name = get_binaries( + context.base_os, + context.test_object_name, + TestEnvironment.current.base_dir, + ) + parameters.name = parameters.name if parameters.name is not None else ovms_binary_name + result = start_binary_ovms(context, parameters, ovms_binary_path, environment, **kwargs) + elif ovms_type_to_start == OvmsType.CAPI: + ovms_capi_params = rewrite_parameters(inherit_class=OvmsCapiParams, base_class_object=parameters) + result = start_capi_ovms(context=context, parameters=ovms_capi_params, environment=environment) + elif ovms_type_to_start == OvmsType.NONE: + logger.warning("SKIP: Executing Ovms tests with ovms_type == 'NONE'") + skip_if_runtime(True, msg="'NONE' type not supported") + else: + raise OvmsTestException(f"Unrecognized ovms_type_to_start={ovms_type_to_start}") + + result.attach_context(context) + + result.ovms._dmesg_log.ovms_pid = result.ovms.fetch_and_store_ovms_pid() + + if ensure_started: + assert not parameters.check_version, "OVMS container will not start if --version argument was given." + log_fixture( + "Ensure ovms is running with model(s): {}".format(", ".join([model.name for model in result.models])) + ) + result.ovms.ensure_started(result.models, timeout=timeout, os_type=context.base_os) + + return result + + +def rewrite_parameters(inherit_class, base_class_object): + inherit_object = inherit_class() + for key, value in base_class_object.__dict__.items(): + if base_class_object.__dict__[key] is not None: + inherit_object.__dict__[key] = value + return inherit_object + + +def start_docker_ovms( + context: Context, + parameters, + environment: dict, + ovms_docker_type, + ovms_instance_params=None, + entrypoint=None, + entrypoint_params=None, +) -> OvmsRunContext: + if getattr(context, "ovms_test_image", None) is not None and context.ovms_type != OvmsType.BINARY_DOCKER: + # If testing image was build & set in context: + # replace default image with testing image. In near future all tests should use `ovms_test_image` + parameters.image = context.ovms_test_image + + if parameters.ports_enabled(): + if context.port_manager_grpc is not None and parameters.grpc_port is None: + parameters.grpc_port = context.port_manager_grpc.get_port() + if context.port_manager_rest is not None and parameters.rest_port is None: + parameters.rest_port = context.port_manager_rest.get_port() + + ovms_instance = OvmsDockerLauncher.create( + context, parameters, ovms_docker_type, environment, entrypoint, entrypoint_params, ovms_instance_params + ) + ovms_run_context = OvmsRunContext(ovms_instance, parameters.models) + if resource_monitor_enabled: + ovms_run_context.attach_resource_monitor(context) + return ovms_run_context diff --git a/tests/functional/fixtures/server_detection_model_fixtures.py b/tests/functional/fixtures/server_detection_model_fixtures.py deleted file mode 100644 index 44585d1289..0000000000 --- a/tests/functional/fixtures/server_detection_model_fixtures.py +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import pytest - -import tests.functional.config as config -from tests.functional.model.models_information import FaceDetection -from tests.functional.object_model.server import Server - - -@pytest.fixture(scope="session") -def start_server_face_detection_model_auto_shape(request): - - start_server_command_args = {"model_name": FaceDetection.name, - "model_path": FaceDetection.model_path, - "shape": "auto", - "grpc_workers": 4, - "nireq": 4} - container_name_infix = "test-auto-shape" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - target_device=config.target_device) - return server.start() - - -@pytest.fixture(scope="session") -def start_server_face_detection_model_named_shape(request, copy_cached_models_to_test_dir): - - start_server_command_args = {"model_name": FaceDetection.name, - "model_path": FaceDetection.model_path, - "shape": "\"{\\\"data\\\": \\\"(1, 3, 600, 600)\\\"}\"", - "grpc_workers": 4, - "rest_workers": 2, - "nireq": 2} - container_name_infix = "test-named-shape" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - target_device=config.target_device) - return server.start() - - -@pytest.fixture(scope="session") -def start_server_face_detection_model_nonamed_shape(request, prepare_json): - - start_server_command_args = {"model_name": FaceDetection.name, - "model_path": FaceDetection.model_path, - "shape": "\"(1, 3, 600, 600)\"", - "rest_workers": 4, - "nireq": 2} - container_name_infix = "test-nonamed-shape" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - target_device=config.target_device) - return server.start() diff --git a/tests/functional/fixtures/server_for_update_fixtures.py b/tests/functional/fixtures/server_for_update_fixtures.py deleted file mode 100644 index 2791c3fa06..0000000000 --- a/tests/functional/fixtures/server_for_update_fixtures.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import pytest -import shutil - -import tests.functional.config as config -from tests.functional.model.models_information import Resnet -from tests.functional.utils.parametrization import get_tests_suffix -from tests.functional.object_model.server import Server - - -@pytest.fixture(scope="session") -def start_server_update_flow_latest(request): - - update_test_dir = config.path_to_mount + '/update-{}/'.format(get_tests_suffix()) - # ensure model dir is empty before starting OVMS - shutil.rmtree(update_test_dir, ignore_errors=True) - - start_server_command_args = {"model_name": Resnet.name, - "model_path": "/opt/ml/update-{}".format(get_tests_suffix()), - "grpc_workers": 1, - "nireq": 1} - container_name_infix = "test-update-latest" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - target_device=config.target_device) - return server.start() - - -@pytest.fixture(scope="session") -def start_server_update_flow_specific(request): - - update_test_dir = config.path_to_mount + '/update-{}/'.format(get_tests_suffix()) - # ensure model dir is empty before starting OVMS - shutil.rmtree(update_test_dir, ignore_errors=True) - - start_server_command_args = {"model_name": Resnet.name, - "model_path": "/opt/ml/update-{}".format(get_tests_suffix()), - "model_version_policy": '\'{"specific": { "versions":[1, 3, 4] }}\''} - container_name_infix = "test-update-specific" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - target_device=config.target_device) - return server.start() diff --git a/tests/functional/fixtures/server_local_models_fixtures.py b/tests/functional/fixtures/server_local_models_fixtures.py deleted file mode 100644 index 8abc9412ef..0000000000 --- a/tests/functional/fixtures/server_local_models_fixtures.py +++ /dev/null @@ -1,76 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import shutil -import os -import pytest - -import tests.functional.config as config -from tests.functional.model.models_information import Resnet, ResnetONNX, AgeGender -from tests.functional.object_model.server import Server - - -@pytest.fixture(scope="session") -def start_server_single_model(request): - - start_server_command_args = {"model_name": Resnet.name, - "model_path": Resnet.model_path, - "plugin_config": "\"{\\\"CPU_THROUGHPUT_STREAMS\\\": \\\"CPU_THROUGHPUT_AUTO\\\"}\""} - container_name_infix = "test-single" - - # In this case, slower, non-default serialization method is used - env_variables = ['SERIALIZATON=_prepare_output_as_AppendArrayToTensorProto'] - - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - env_variables, target_device=config.target_device) - return server.start() - -@pytest.fixture(scope="session") -def start_server_single_model_onnx(request): - - start_server_command_args = {"model_name": ResnetONNX.name, - "model_path": ResnetONNX.model_path, - "plugin_config": "\"{\\\"CPU_THROUGHPUT_STREAMS\\\": \\\"CPU_THROUGHPUT_AUTO\\\"}\""} - container_name_infix = "test-single-onnx" - - # In this case, slower, non-default serialization method is used - env_variables = ['SERIALIZATON=_prepare_output_as_AppendArrayToTensorProto'] - - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - env_variables, target_device=config.target_device) - return server.start() - -@pytest.fixture(scope="session") -def start_server_with_mapping(request): - - def delete_mapping_file(): - if os.path.exists(file_dst_path): - os.remove(file_dst_path) - - request.addfinalizer(delete_mapping_file) - - file_dst_path = config.path_to_mount + '/age_gender/1/mapping_config.json' - shutil.copyfile(os.path.join(config.ovms_c_repo_path, 'tests/functional/mapping_config.json'), file_dst_path) - - start_server_command_args = {"model_name": AgeGender.name, - "model_path": AgeGender.model_path} - container_name_infix = "test-2-out" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - target_device=config.target_device) - return server.start() diff --git a/tests/functional/fixtures/server_multi_model_fixtures.py b/tests/functional/fixtures/server_multi_model_fixtures.py deleted file mode 100644 index 72c569ea3a..0000000000 --- a/tests/functional/fixtures/server_multi_model_fixtures.py +++ /dev/null @@ -1,55 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import pytest - -import tests.functional.config as config -from tests.functional.object_model.minio_docker import MinioDocker -from tests.functional.object_model.server import Server - - -@pytest.fixture(scope="session") -def start_server_multi_model( - request, start_minio_server, get_minio_server_s3, model_version_policy_models, resnet_multiple_batch_sizes -): - - aws_access_key_id = os.getenv('MINIO_ACCESS_KEY') - aws_secret_access_key = os.getenv('MINIO_SECRET_KEY') - aws_region = os.getenv('AWS_REGION') - - minio_container, ports = start_minio_server - grpc_port, rest_port = ports["grpc_port"], ports["rest_port"] - - if config.ovms_binary_path: - minio_endpoint = "http://localhost:{}".format(grpc_port) - else: - minio_endpoint = "{}:{}".format(MinioDocker.get_ip(minio_container), grpc_port) - - envs = ['MINIO_ACCESS_KEY=' + aws_access_key_id, - 'MINIO_SECRET_KEY=' + aws_secret_access_key, - 'AWS_ACCESS_KEY_ID=' + aws_access_key_id, - 'AWS_SECRET_ACCESS_KEY=' + aws_secret_access_key, - 'AWS_REGION=' + aws_region, - 'S3_ENDPOINT=' + minio_endpoint] - - start_server_command_args = {"config_path": "{}/config.json".format(config.models_path), - "grpc_workers": 2, - "rest_workers": 2} - container_name_infix = "test-multi" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, envs) - return server.start() diff --git a/tests/functional/fixtures/server_remote_models_fixtures.py b/tests/functional/fixtures/server_remote_models_fixtures.py deleted file mode 100644 index 965dc041ee..0000000000 --- a/tests/functional/fixtures/server_remote_models_fixtures.py +++ /dev/null @@ -1,167 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os - -import boto3 -import pytest -from botocore.client import Config - -import tests.functional.config as config -from tests.functional.model.models_information import Resnet, ResnetS3, ResnetGS -from tests.functional.object_model.minio_docker import MinioDocker -from tests.functional.object_model.server import Server -from tests.functional.utils.parametrization import get_tests_suffix - - -@pytest.fixture(scope="session") -def start_server_single_model_from_gc(request): - - start_server_command_args = {"model_name": Resnet.name, - "model_path": ResnetGS.model_path #, - #"target_device": "CPU", - #"nireq": 4, - #"plugin_config": "\"{\\\"CPU_THROUGHPUT_STREAMS\\\": \\\"2\\\", " - # "\\\"CPU_THREADS_NUM\\\": \\\"4\\\"}\"" - } - container_name_infix = "test-single-gs" - envs = ['https_proxy=' + os.getenv('https_proxy', "")] - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, envs, - target_device=config.target_device) - return server.start() - - -@pytest.fixture(scope="session") -def get_docker_network(request, get_docker_context): - - client = get_docker_context - existing = None - - try: - existing = client.networks.get("minio-network-{}".format( - get_tests_suffix())) - except Exception as e: - pass - - if existing is not None: - existing.remove() - - network = client.networks.create("minio-network-{}".format( - get_tests_suffix())) - - request.addfinalizer(network.remove) - - return network - - -@pytest.fixture(scope="session") -def start_minio_server(request, get_docker_context): - - """sudo docker run -d -p 9099:9000 minio/minio server /data""" - client = get_docker_context - client.images.pull(config.minio_image) - container_name = "minio.locals3-{}.com".format(get_tests_suffix()) - - minio_access_key = os.getenv('MINIO_ACCESS_KEY') - minio_secret_key = os.getenv('MINIO_SECRET_KEY') - - if minio_access_key is None or minio_secret_key is None: - minio_access_key = "MINIO_A_KEY" - minio_secret_key = "MINIO_S_KEY" - os.environ["MINIO_ACCESS_KEY"] = "MINIO_A_KEY" - os.environ["MINIO_SECRET_KEY"] = "MINIO_S_KEY" - - envs = ['MINIO_ACCESS_KEY=' + minio_access_key, - 'MINIO_SECRET_KEY=' + minio_secret_key] - - minio_docker = MinioDocker(request, container_name, config.start_minio_container_command, - envs) - - return minio_docker.start() - - -@pytest.fixture(scope="session") -def get_minio_server_s3(start_minio_server, copy_cached_resnet_models): - - path_to_mount = config.path_to_mount + '/{}/{}'.format(Resnet.name, Resnet.version) - input_bin = os.path.join(path_to_mount, '{}.bin'.format(Resnet.name)) - input_xml = os.path.join(path_to_mount, '{}.xml'.format(Resnet.name)) - - minio_access_key = os.getenv('MINIO_ACCESS_KEY') - minio_secret_key = os.getenv('MINIO_SECRET_KEY') - aws_region = os.getenv('AWS_REGION') - - if aws_region is None: - aws_region = "eu-central-1" - os.environ["AWS_REGION"] = aws_region - - if minio_access_key is None or minio_secret_key is None: - minio_access_key = "MINIO_A_KEY" - minio_secret_key = "MINIO_S_KEY" - os.environ["MINIO_ACCESS_KEY"] = minio_access_key - os.environ["MINIO_SECRET_KEY"] = minio_secret_key - - minio_container, ports = start_minio_server - s3 = boto3.resource('s3', - endpoint_url='http://localhost:{}'.format( - ports["grpc_port"]), - aws_access_key_id=os.getenv('MINIO_ACCESS_KEY'), - aws_secret_access_key=os.getenv('MINIO_SECRET_KEY'), - config=Config(signature_version='s3v4'), - region_name=aws_region) - - bucket = s3.Bucket('inference') - if not bucket.creation_date: - bucket_conf = {'LocationConstraint': aws_region} - s3.create_bucket(Bucket='inference', - CreateBucketConfiguration=bucket_conf) - - bucket.upload_file(input_bin, '{name}/{version}/{name}.bin'.format(name=Resnet.name, version=Resnet.version)) - bucket.upload_file(input_xml, '{name}/{version}/{name}.xml'.format(name=Resnet.name, version=Resnet.version)) - - return s3, ports, minio_container - - -@pytest.fixture(scope="session") -def start_server_single_model_from_minio(request, get_minio_server_s3): - - aws_access_key_id = os.getenv('MINIO_ACCESS_KEY') - aws_secret_access_key = os.getenv('MINIO_SECRET_KEY') - aws_region = os.getenv('AWS_REGION') - - _, ports, minio_container = get_minio_server_s3 - grpc_port = ports["grpc_port"] - - if config.ovms_binary_path: - minio_endpoint = "http://localhost:{}".format(grpc_port) - else: - minio_endpoint = "{}:{}".format(MinioDocker.get_ip(minio_container), grpc_port) - - envs = ['MINIO_ACCESS_KEY=' + aws_access_key_id, - 'MINIO_SECRET_KEY=' + aws_secret_access_key, - 'AWS_ACCESS_KEY_ID=' + aws_access_key_id, - 'AWS_SECRET_ACCESS_KEY=' + aws_secret_access_key, - 'AWS_REGION=' + aws_region, - 'S3_ENDPOINT=' + minio_endpoint] - - start_server_command_args = {"model_name": Resnet.name, - "model_path": ResnetS3.model_path} - container_name_infix = "test-single-minio" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, envs, - target_device=config.target_device) - return server.start() diff --git a/tests/functional/fixtures/server_with_batching_fixtures.py b/tests/functional/fixtures/server_with_batching_fixtures.py deleted file mode 100644 index d18ea2cfe6..0000000000 --- a/tests/functional/fixtures/server_with_batching_fixtures.py +++ /dev/null @@ -1,95 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import pytest - -import tests.functional.config as config -from tests.functional.model.models_information import ResnetBS8, AgeGender -from tests.functional.object_model.server import Server - - -@pytest.fixture(scope="session") -def start_server_batch_model(request): - start_server_command_args = {"model_name": ResnetBS8.name, - "model_path": ResnetBS8.model_path} - container_name_infix = "test-batch" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - target_device=config.target_device) - return server.start() - - -@pytest.fixture(scope="session") -def start_server_batch_model_2out(request): - - start_server_command_args = {"model_name": AgeGender.name, - "model_path": AgeGender.model_path} - container_name_infix = "test-batch-2out" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - target_device=config.target_device) - return server.start() - - -@pytest.fixture(scope="session") -def start_server_batch_model_auto(request): - - start_server_command_args = {"model_name": ResnetBS8.name, - "model_path": ResnetBS8.model_path, - "batch_size": "auto"} - container_name_infix = "test-autobatch" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - target_device=config.target_device) - return server.start() - - -@pytest.fixture(scope="session") -def start_server_batch_model_auto_2out(request): - - start_server_command_args = {"model_name": AgeGender.name, - "model_path": AgeGender.model_path, - "batch_size": "auto"} - container_name_infix = "test-autobatch-2out" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - target_device=config.target_device) - return server.start() - - -@pytest.fixture(scope="session") -def start_server_batch_model_bs4(request): - - start_server_command_args = {"model_name": ResnetBS8.name, - "model_path": ResnetBS8.model_path, - "batch_size": 4} - container_name_infix = "test-batch4" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command, - target_device=config.target_device) - return server.start() - - -@pytest.fixture(scope="session") -def start_server_batch_model_auto_bs4_2out(request): - - start_server_command_args = {"model_name": AgeGender.name, - "model_path": AgeGender.model_path, - "batch_size": 4} - container_name_infix = "test-batch4-2out" - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command) - return server.start() diff --git a/tests/functional/fixtures/server_with_version_policy_fixtures.py b/tests/functional/fixtures/server_with_version_policy_fixtures.py deleted file mode 100644 index 83d7f3fd68..0000000000 --- a/tests/functional/fixtures/server_with_version_policy_fixtures.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import pytest -import shutil -from distutils.dir_util import copy_tree - -import tests.functional.config as config -from tests.functional.model.models_information import FaceDetection, PVBFaceDetectionV2, AgeGender -from tests.functional.object_model.server import Server - - -@pytest.fixture(scope="session") -def start_server_model_ver_policy(request, resnet_multiple_batch_sizes): - - shutil.copyfile(os.path.join(config.ovms_c_repo_path, 'tests/functional/mapping_config.json'), - config.path_to_mount + '/model_ver/3/mapping_config.json') - - start_server_command_args = {"config_path": "{}/model_version_policy_config.json".format(config.models_path)} - container_name_infix = "test-batch4-2out" - - server = Server(request, start_server_command_args, - container_name_infix, config.start_container_command) - return server.start() - - -@pytest.fixture(scope="session") -def model_version_policy_models(models_downloader): - model_ver_dir = os.path.join(config.path_to_mount, 'model_ver') - - face_detection = os.path.join(models_downloader[FaceDetection.name], str(FaceDetection.version)) - face_detection_dir = os.path.join(model_ver_dir, '1') - face_detection_bin = os.path.join(face_detection_dir, FaceDetection.name + ".bin") - - pvb_detection = os.path.join(models_downloader[PVBFaceDetectionV2.name], str(PVBFaceDetectionV2.version)) - pvb_detection_dir = os.path.join(model_ver_dir, '2') - pvb_detection_bin = os.path.join(pvb_detection_dir, PVBFaceDetectionV2.name + ".bin") - - age_gender = os.path.join(models_downloader[AgeGender.name], str(AgeGender.version)) - age_gender_dir = os.path.join(model_ver_dir, '3') - age_gender_bin = os.path.join(age_gender_dir, AgeGender.name + ".bin") - - if not (os.path.exists(model_ver_dir) - and os.path.exists(face_detection_bin) - and os.path.exists(pvb_detection_bin) - and os.path.exists(age_gender_bin)): - os.makedirs(model_ver_dir, exist_ok=True) - copy_tree(face_detection, face_detection_dir) - copy_tree(pvb_detection, pvb_detection_dir) - copy_tree(age_gender, age_gender_dir) - - return face_detection_dir, pvb_detection_dir, age_gender_dir diff --git a/tests/functional/fixtures/test_files_fixtures.py b/tests/functional/fixtures/test_files_fixtures.py deleted file mode 100644 index 69b6fae6b3..0000000000 --- a/tests/functional/fixtures/test_files_fixtures.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright (c) 2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import pytest - -import tests.functional.config as config - - -def new_file_name(file): - return file.replace("_template", "") - - -@pytest.fixture(scope="session") -def prepare_json(request, copy_cached_models_to_test_dir): - files_to_prepare = ["config_template.json", "model_version_policy_config_template.json"] - path_to_config = os.path.join(config.ovms_c_repo_path, "tests/functional/") - - def finalizer(): - for file in files_to_prepare: - os.remove(os.path.join(config.path_to_mount, new_file_name(file))) - - request.addfinalizer(finalizer) - - for file_to_prepare in files_to_prepare: - file_to_prepare_path = file_to_prepare if path_to_config.strip(os.path.sep) in os.getcwd() \ - else os.path.join(path_to_config, file_to_prepare) - with open(file_to_prepare_path, "r") as template: - new_file_path = os.path.join(config.path_to_mount, new_file_name(file_to_prepare)) - with open(new_file_path, "w+") as config_file: - for line in template: - if "{path}" in line: - line = line.replace("{path}", config.models_path) - elif "{target_device}" in line: - line = line.replace("{target_device}", config.target_device) - - config_file.write(line) diff --git a/tests/functional/mapping_config.json b/tests/functional/mapping_config.json deleted file mode 100644 index 0964250afc..0000000000 --- a/tests/functional/mapping_config.json +++ /dev/null @@ -1,8 +0,0 @@ - { - "inputs": - { "data":"new_key"}, - "outputs": - { "age_conv3":"age", - "prob":"gender"} - } - diff --git a/tests/functional/model/models_information.py b/tests/functional/model/models_information.py deleted file mode 100644 index 42f15db99d..0000000000 --- a/tests/functional/model/models_information.py +++ /dev/null @@ -1,166 +0,0 @@ -# -# Copyright (c) 2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -import os -import numpy as np - -import tests.functional.config as config - -MODEL_REPOSITORY_SERVER = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/2022.1/models_bin" -BUILD_DIR = "1" -BUILD_012020 = "012020" -PRECISION = "FP32" -FACE_DETECTION_MODEL = "face-detection-retail-0004" -AGE_GENDER_RECOGNITION_MODEL = "age-gender-recognition-retail-0013" -PERSON_VEHICLE_BIKE_DETECTION_MODEL = "person-vehicle-bike-detection-crossroad-0078" -RESNET_50 = "resnet-50-tf" -RESNET_V1_50 = "resnet_v1-50" -OPEN_MODEL_ZOO_MODELS_LOCATION = "{repo}/{build}".format(repo=MODEL_REPOSITORY_SERVER, - build=BUILD_DIR) -URL_OPEN_MODEL_ZOO_FORMAT = "{model_location}/{model}/{precision}/{model}" - - -class AgeGender: - name = "age_gender" - dtype = np.float32 - input_name = "new_key" - input_shape = (1, 3, 62, 62) - output_name = ['age', 'gender'] - output_shape = {'age': (1, 1, 1, 1), - 'gender': (1, 2, 1, 1)} - rest_request_format = 'column_name' - url = URL_OPEN_MODEL_ZOO_FORMAT.format(model_location=OPEN_MODEL_ZOO_MODELS_LOCATION, - model=AGE_GENDER_RECOGNITION_MODEL, precision=PRECISION) # noqa - version = 1 - download_extensions = [".xml", ".bin"] - model_path = os.path.join(config.models_path, name) - - -class PVBDetection: - name = "pvb_detection" - dtype = np.float32 - input_name = "data" - input_shape = (1, 3, 300, 300) - output_name = "detection_out" - output_shape = (1, 1, 200, 7) - rest_request_format = 'column_name' - url = URL_OPEN_MODEL_ZOO_FORMAT.format(model_location=OPEN_MODEL_ZOO_MODELS_LOCATION, - model=PERSON_VEHICLE_BIKE_DETECTION_MODEL, precision=PRECISION) # noqa - version = 1 - download_extensions = [".xml", ".bin"] - - -class FaceDetection: - name = "face_detection" - dtype = np.float32 - input_name = "data" - input_shape = (1, 3, 300, 300) - output_name = "detection_out" - output_shape = (1, 1, 200, 7) - rest_request_format = 'column_name' - url = URL_OPEN_MODEL_ZOO_FORMAT.format(model_location=OPEN_MODEL_ZOO_MODELS_LOCATION, - model=FACE_DETECTION_MODEL, precision=PRECISION) # noqa - version = 1 - download_extensions = [".xml", ".bin"] - model_path = os.path.join(config.models_path, name) - - -class PVBFaceDetectionV1(FaceDetection): - name = "pvb_face_multi_version" - - -class PVBFaceDetectionV2(PVBDetection): - name = "pvb_face_multi_version" - input_shape = (1, 3, 1024, 1024) - version = 2 - - -PVBFaceDetection = [PVBFaceDetectionV1, PVBFaceDetectionV2] - - -class Resnet: - name = "resnet" - dtype = np.float32 - input_name = "map/TensorArrayStack/TensorArrayGatherV3" - input_shape = (1, 224, 224, 3) - output_name = "softmax_tensor:0" - output_shape = (1, 1001) - rest_request_format = 'column_name' - model_path = os.path.join(config.models_path, name) - url = "https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/resnet-50-tf/" + RESNET_V1_50 - local_conversion_dir = "tensorflow_format" - download_extensions = [".pb"] - version = 1 - - -class ResnetBS4: - name = "resnet_bs4" - dtype = np.float32 - input_name = "map/TensorArrayStack/TensorArrayGatherV3" - input_shape = (4, 224, 224, 3) - output_name = "softmax_tensor:0" - output_shape = (4, 1001) - rest_request_format = 'row_noname' - version = 1 - - -class ResnetBS8: - name = "resnet_bs8" - dtype = np.float32 - input_name = "map/TensorArrayStack/TensorArrayGatherV3" - input_shape = (8, 224, 224, 3) - output_name = "softmax_tensor:0" - output_shape = (8, 1001) - rest_request_format = 'row_noname' - model_path = os.path.join(config.models_path, name) - version = 1 - - -class ResnetS3: - name = "resnet_s3" - dtype = np.float32 - input_name = "map/TensorArrayStack/TensorArrayGatherV3" - input_shape = (1, 224, 224, 3) - output_name = "softmax_tensor:0" - output_shape = (1, 1001) - rest_request_format = 'row_name' - model_path = "s3://inference/resnet" - - -class ResnetGS: - name = "resnet_gs" - dtype = np.float32 - input_name = "0" - input_shape = (1, 3, 224, 224) - output_name = "1463" - output_shape = (1, 1000) - rest_request_format = 'row_name' - model_path = "gs://ovms-public-eu/resnet50-binary" - - -class ResnetONNX: - name = "resnet_onnx" - dtype = np.float32 - input_name = "gpu_0/data_0" - input_shape = (1, 3, 224, 224) - output_name = "gpu_0/softmax_1" - output_shape = (1, 1000) - rest_request_format = 'row_name' - url = "https://github.com/onnx/models/raw/cf382db7781fc8193249386d6b50a4753659d058/vision/classification/resnet/model/resnet50-caffe2-v1-9" - download_extensions = [".onnx"] - version = 1 - model_path = os.path.join(config.models_path, name) diff --git a/tests/functional/model_version_policy_config_template.json b/tests/functional/model_version_policy_config_template.json deleted file mode 100644 index b6be66830c..0000000000 --- a/tests/functional/model_version_policy_config_template.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "model_config_list":[ - { - "config":{ - "name":"latest", - "base_path":"{path}/model_ver", - "model_version_policy": {"latest": { "num_versions":2 }}, - "target_device": "{target_device}" - } - }, - { - "config": { - "name": "specific", - "base_path": "{path}/model_ver", - "model_version_policy": {"specific": { "versions":[1, 3] }}, - "target_device": "{target_device}" - } - }, - { - "config": { - "name": "all", - "base_path": "{path}/model_ver", - "model_version_policy": {"all": {}}, - "target_device": "{target_device}" - } - } - ] -} From e3dfa72ad39c160395042707e317c296b556bd13 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Fri, 8 May 2026 12:52:33 +0200 Subject: [PATCH 03/12] object_model + tests --- tests/functional/object_model/__init__.py | 2 +- .../functional/object_model/cpu_extension.py | 109 ++ .../functional/object_model/custom_loader.py | 228 +++ tests/functional/object_model/custom_node.py | 407 +++++ .../object_model/dmesg_log_monitor.py | 185 +++ tests/functional/object_model/docker.py | 185 --- .../object_model/inference_helpers.py | 1457 +++++++++++++++++ .../object_model/mediapipe_calculators.py | 986 +++++++++++ tests/functional/object_model/minio_docker.py | 46 - tests/functional/object_model/ovms_binary.py | 435 ++++- tests/functional/object_model/ovms_capi.py | 318 ++++ tests/functional/object_model/ovms_command.py | 342 ++++ tests/functional/object_model/ovms_config.py | 349 ++++ tests/functional/object_model/ovms_docker.py | 751 ++++++++- tests/functional/object_model/ovms_info.py | 223 +++ .../functional/object_model/ovms_instance.py | 544 ++++++ .../object_model/ovms_log_monitor.py | 437 +++++ .../object_model/ovms_mapping_config.py | 126 ++ tests/functional/object_model/ovms_params.py | 189 +++ tests/functional/object_model/ovsa.py | 149 ++ .../object_model/package_manager.py | 330 ++++ .../python_custom_nodes/__init__.py | 15 + .../python_custom_nodes/common.py | 51 + .../python_custom_nodes.py | 450 +++++ .../object_model/resource_monitor.py | 144 ++ tests/functional/object_model/server.py | 75 - tests/functional/object_model/shape.py | 120 ++ .../object_model/test_environment.py | 90 + tests/functional/object_model/test_helpers.py | 333 ++++ tests/functional/object_model/tools.py | 37 + tests/functional/test_llm_json.py | 360 ---- tests/functional/test_model_version_policy.py | 253 --- .../test_model_versions_handling.py | 170 -- tests/functional/test_reshaping.py | 181 -- 34 files changed, 8692 insertions(+), 1385 deletions(-) create mode 100644 tests/functional/object_model/cpu_extension.py create mode 100644 tests/functional/object_model/custom_loader.py create mode 100644 tests/functional/object_model/custom_node.py create mode 100644 tests/functional/object_model/dmesg_log_monitor.py delete mode 100644 tests/functional/object_model/docker.py create mode 100644 tests/functional/object_model/inference_helpers.py create mode 100644 tests/functional/object_model/mediapipe_calculators.py delete mode 100644 tests/functional/object_model/minio_docker.py create mode 100644 tests/functional/object_model/ovms_capi.py create mode 100644 tests/functional/object_model/ovms_command.py create mode 100644 tests/functional/object_model/ovms_config.py create mode 100644 tests/functional/object_model/ovms_info.py create mode 100644 tests/functional/object_model/ovms_instance.py create mode 100644 tests/functional/object_model/ovms_log_monitor.py create mode 100644 tests/functional/object_model/ovms_mapping_config.py create mode 100644 tests/functional/object_model/ovms_params.py create mode 100644 tests/functional/object_model/ovsa.py create mode 100644 tests/functional/object_model/package_manager.py create mode 100644 tests/functional/object_model/python_custom_nodes/__init__.py create mode 100644 tests/functional/object_model/python_custom_nodes/common.py create mode 100644 tests/functional/object_model/python_custom_nodes/python_custom_nodes.py create mode 100644 tests/functional/object_model/resource_monitor.py delete mode 100644 tests/functional/object_model/server.py create mode 100644 tests/functional/object_model/shape.py create mode 100644 tests/functional/object_model/test_environment.py create mode 100644 tests/functional/object_model/test_helpers.py create mode 100644 tests/functional/object_model/tools.py delete mode 100644 tests/functional/test_llm_json.py delete mode 100644 tests/functional/test_model_version_policy.py delete mode 100644 tests/functional/test_model_versions_handling.py delete mode 100644 tests/functional/test_reshaping.py diff --git a/tests/functional/object_model/__init__.py b/tests/functional/object_model/__init__.py index 3e8e2f6775..84cfbcf566 100644 --- a/tests/functional/object_model/__init__.py +++ b/tests/functional/object_model/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 Intel Corporation +# Copyright (c) 2026 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/functional/object_model/cpu_extension.py b/tests/functional/object_model/cpu_extension.py new file mode 100644 index 0000000000..86647fd1f9 --- /dev/null +++ b/tests/functional/object_model/cpu_extension.py @@ -0,0 +1,109 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from dataclasses import dataclass +from pathlib import Path + +from tests.functional.config import ovms_c_repo_path, tmp_dir +from tests.functional.constants.paths import Paths + + +@dataclass +class CpuExtension: + base_os: str = "" + host_lib_path: str = "" + lib_path: str = "" + HELLO_WORLD_MESSAGE: str = "" + + def __post_init__(self): + if not self.base_os: + self.base_os = CpuExtension.base_os + + +@dataclass +class MuseModelExtension(CpuExtension): + lib_name: str = "libopenvino_tokenizers.so" + lib_path: str = f"/ovms/lib/{lib_name}" + + +@dataclass +class InvalidCpuExtension(CpuExtension): + lib_name: str = "cpu_extension.lib" + lib_path: str = "/path/to/non/existent/cpu_extension.lib" + + +@dataclass +class BuildableCpuExtension(CpuExtension): + make_dir: str = "" + host_lib_path: str = "" + + +@dataclass +class OvmsCBuildableCpuExtension(BuildableCpuExtension): + make_dir: str = Path(ovms_c_repo_path) + + +@dataclass +class SimpleReluCpuExtension(OvmsCBuildableCpuExtension): + extension_name: str = "SampleCpuExtension" + lib_name: str = "libcustom_relu_cpu_extension.so" + HELLO_WORLD_MESSAGE: str = "Running Relu custom kernel for the first time (next messages won't be printed)" + + def __post_init__(self): + super().__post_init__() + self.cpu_extension_src_dir = Path(self.make_dir, "src", "example", self.extension_name) + self.cpu_extension_dst_host_path = os.path.join(tmp_dir, self.extension_name) + self.lib_path = os.path.join(Paths.ROOT_PATH_CPU_EXTENSIONS, self.extension_name, self.lib_name) + self.host_lib_path = os.path.join(tmp_dir, self.extension_name, self.lib_name) + + +@dataclass +class OvmsTestBuildableCpuExtension(BuildableCpuExtension): + make_dir: str = "" + lib_name: str = "" + extension_name: str = "" + cpp_file: str = "" + + def __post_init__(self): + self.ovms_c_cpu_ext = SimpleReluCpuExtension() + super().__post_init__() + self.make_dir = Path(tmp_dir, self.extension_name) + + +@dataclass +class CorruptedLibCpuExtension(OvmsTestBuildableCpuExtension): + extension_name: str = "corrupted_lib" + cpp_file: str = "CorruptedLib.cpp" + lib_name: str = "libcorrupted_lib_cpu_extension.so" + + def __post_init__(self): + super().__post_init__() + self.host_lib_path = os.path.join(tmp_dir, self.extension_name, self.lib_name) + self.lib_path = os.path.join(Paths.ROOT_PATH_CPU_EXTENSIONS, self.extension_name, self.lib_name) + + +@dataclass +class ThrowExceptionCpuExtension(OvmsTestBuildableCpuExtension): + extension_name: str = "throw_exceptions" + cpp_file: str = "ThrowExceptions.cpp" + lib_name: str = "libthrow_exception_cpu_extension.so" + HELLO_WORLD_MESSAGE: str = "Executing ThrowExceptions evaluate()" + + def __post_init__(self): + super().__post_init__() + self.host_lib_path = os.path.join(tmp_dir, self.extension_name, self.lib_name) + self.lib_path = os.path.join(Paths.ROOT_PATH_CPU_EXTENSIONS, self.extension_name, self.lib_name) diff --git a/tests/functional/object_model/custom_loader.py b/tests/functional/object_model/custom_loader.py new file mode 100644 index 0000000000..8ca05ceb99 --- /dev/null +++ b/tests/functional/object_model/custom_loader.py @@ -0,0 +1,228 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import shutil +from pathlib import Path + +from tests.functional.utils.assertions import OvmsTestException +from tests.functional.utils.logger import get_logger +from tests.functional.utils.process import Process +from tests.functional.utils.test_framework import skip_if_runtime +from tests.functional.config import disable_custom_loader, ovms_c_repo_path +from tests.functional.constants.custom_loader import CustomLoaderConsts +from tests.functional.constants.ovms import Config, CurrentOvmsType +from tests.functional.constants.paths import Paths +from tests.functional.object_model.test_environment import TestEnvironment + +logger = get_logger(__name__) + + +class CustomLoader: + PARENT_KEY = Config.CUSTOM_LOADER_CONFIG_LIST + ENABLE_FILE_DISABLE_PHRASE = "DISABLED" + + DEFAULT_LOADER_NAME = CustomLoaderConsts.DEFAULT_LOADER_NAME + + def __init__( + self, + model, + name: str = DEFAULT_LOADER_NAME, + loader_config_file: str = None, + enable_file: str = None, + prepare_custom_loader_resources: bool = False, + ): + skip_if_runtime(disable_custom_loader, "Custom loader disabled") + + file_name = CustomLoaderConsts.SAMPLE_CUSTOM_LOADER_LIB_NAME + self.ovms_type = CurrentOvmsType.ovms_type + internal_path = os.path.join( + Paths.CUSTOM_LOADER_LIBRARIES_PATH_INTERNAL, CustomLoaderConsts.SAMPLE_CUSTOM_LOADER_NAME, file_name + ) + self._loader_config = self.LoaderConfig(name, internal_path, loader_config_file) + + self._model = model + self._model_options = self.ModelOptions(name, enable_file) + self._model_options.update_from_model(model) + self.prepare_custom_loader_resources = prepare_custom_loader_resources + + @staticmethod + def get_custom_loader_path(image): + cmd = f"docker cp $(docker create --rm {image}):{CustomLoaderConsts.SAMPLE_CUSTOM_LOADER_DOCKER_LIB} ." + proc = Process() + cwd = os.path.join(ovms_c_repo_path, "tests", "functional", "utils", "ovms_testing_image") + proc.run_and_check(cmd, cwd=cwd) + dst_file_path = os.path.join(cwd, CustomLoaderConsts.SAMPLE_CUSTOM_LOADER_LIB_NAME) + return dst_file_path + + def prepare_resources(self, base_location): + resource_path = Path(base_location + Paths.MODELS_PATH_INTERNAL, Paths.CUSTOM_LOADER_PATH_NAME) + resource_path.mkdir(parents=True, exist_ok=True) + src_file_path = CustomLoaderConsts.SAMPLE_CUSTOM_LOADER_LIB + dst_file_path = os.path.join(resource_path, os.path.basename(src_file_path)) + if not os.path.exists(dst_file_path): + shutil.copy(src_file_path, dst_file_path) + return str(resource_path) + + @property + def loader_config(self): + return self._loader_config + + @property + def model_options(self): + return self._model_options + + def get_volume_mount(self): + loader_container_path = self.loader_config.get_loader_container_path() + return {self.loader_host_path: {"bind": loader_container_path, "mode": "ro"}} + + def get_enable_file_name(self): + return self._model_options["custom_loader_options"].get(CustomLoader.ModelOptions.ENABLE_FILE_KEY, "") + + @property + def name(self): + return self.loader_config.name + + @staticmethod + def create_model_with_custom_loader(model_type, use_enable_file=False): + model = model_type() + enable_file = f"{model.name}.status" if use_enable_file else None + custom_loader = CustomLoader(model, enable_file=enable_file) + model.custom_loader = custom_loader + return model + + @staticmethod + def attach_custom_loader_to_models( + models, name: str = DEFAULT_LOADER_NAME, loader_host_path: str = None, loader_config_file: str = None + ): + if loader_host_path is None: + loader_host_path = CustomLoaderConsts.SAMPLE_CUSTOM_LOADER_LIB + + for model in models: + model.custom_loader = CustomLoader(model, name, loader_host_path, loader_config_file) + + def get_model_enable_file_path(self, container_name): + model_path_on_host = os.path.join( + TestEnvironment.current.base_dir, container_name, "models", self._model.name, str(self._model.version) + ) + model_enable_file_path = os.path.join(model_path_on_host, self.get_enable_file_name()) + return model_enable_file_path + + def enable_model(self, container_name, delete_enable_file=False, ovms_run=None): + logger.debug(f"Enable model {self.name} in container {container_name}") + model_enable_file_path = self.get_model_enable_file_path(container_name) + if delete_enable_file: + Path(model_enable_file_path).unlink("") + else: + Path(model_enable_file_path).write_text("") + + def disable_model(self, container_name, ovms_run=None): + logger.debug(f"Disable model {self.name} in container {container_name}") + model_enable_file_path = self.get_model_enable_file_path(container_name) + Path(model_enable_file_path).write_text(CustomLoader.ENABLE_FILE_DISABLE_PHRASE) + + def add_enable_file_entry(self, enable_file_value): + self.model_options["custom_loader_options"]["enable_file"] = enable_file_value + + def remove_enable_file_entry(self): + del self.model_options["custom_loader_options"]["enable_file"] + + class LoaderConfig(dict): + PARENT_KEY = "config" + LOADER_NAME_KEY = "loader_name" + LIBRARY_PATH_KEY = "library_path" + LOADER_CONFIG_FILE_KEY = "loader_config_file" + + def __init__(self, name: str, loader_container_path: str, loader_config_file: str = None): + super().__init__() + config = dict() + config.update({self.LOADER_NAME_KEY: name, self.LIBRARY_PATH_KEY: loader_container_path}) + if loader_config_file: + config.update({self.LOADER_CONFIG_FILE_KEY: loader_config_file}) + self.update({self.PARENT_KEY: config}) + + def __hash__(self): + return f"{self}".__hash__() + + def get_loader_container_path(self): + return self[self.PARENT_KEY][self.LIBRARY_PATH_KEY] + + @property + def name(self): + return self[self.PARENT_KEY][self.LOADER_NAME_KEY] + + class ModelOptions(dict): + PARENT_KEY = "custom_loader_options" + LOADER_NAME_KEY = "loader_name" + MODEL_FILE_KEY = "model_file" + BIN_FILE_KEY = "bin_file" + ENABLE_FILE_KEY = "enable_file" + + def __init__(self, loader_name: str = None, enable_file: str = None): + super().__init__() + config = dict() + config.update({self.LOADER_NAME_KEY: loader_name}) + if enable_file: + config.update({self.ENABLE_FILE_KEY: enable_file}) + + self.update({self.PARENT_KEY: config}) + + @classmethod + def create_from_model(cls, model): + model_options = cls(f"{model.name}_custom_loader") + model_options.update_from_model(model) + return model_options + + def update_from_model(self, model): + file_list = model.get_model_files() + + if model.model_type.value == "ONNX": # ModelType.ONNX: # Unable to use symbol: circular import + onnx_model_file = list(filter(lambda x: x.endswith(".onnx"), file_list)) + assert len(onnx_model_file) == 1, f"Expected single .onnx file, got: {len(onnx_model_file)}" + + self[self.PARENT_KEY].update({ + self.MODEL_FILE_KEY: onnx_model_file[0], + }) + elif model.model_type.value == "IR": # ModelType.IR: # Unable to use symbol: circular import + xml_file = list(filter(lambda x: x.endswith(".xml"), file_list)) + bin_file = list(filter(lambda x: x.endswith(".bin"), file_list)) + assert len(xml_file) == 1, f"Expected single .xml file, got: {len(xml_file)}" + assert len(bin_file) == 1, f"Expected single .bin file, got: {len(bin_file)}" + + self[self.PARENT_KEY].update({ + self.MODEL_FILE_KEY: xml_file[0], + self.BIN_FILE_KEY: bin_file[0], + }) + elif model.model_type.value == "PDPD": # ModelType.PDPD # Unable to use symbol: circular import + pdiparams_file = list(filter(lambda x: x.endswith(".pdiparams"), file_list)) + pdmodel_file = list(filter(lambda x: x.endswith(".pdmodel"), file_list)) + + assert len(pdmodel_file) == 1, f"Expected single .pdmodel file, got: {len(pdmodel_file)}" + assert len(pdiparams_file) == 1, f"Expected single .pdiparams file, got: {len(pdiparams_file)}" + + self[self.PARENT_KEY].update({ + self.MODEL_FILE_KEY: pdmodel_file[0], + self.BIN_FILE_KEY: pdiparams_file[0], + }) + elif model.model_type.value == "TFSM": + pb_file = list(filter(lambda x: x.endswith(".pb"), file_list)) + assert len(pb_file) == 1, f"Expected single .pb file, got: {len(pb_file)}" + + self[self.PARENT_KEY].update({ + self.MODEL_FILE_KEY: pb_file[0], + }) + else: + raise OvmsTestException(f"Unexpected model type=={model.model_type.value}") diff --git a/tests/functional/object_model/custom_node.py b/tests/functional/object_model/custom_node.py new file mode 100644 index 0000000000..88dad25660 --- /dev/null +++ b/tests/functional/object_model/custom_node.py @@ -0,0 +1,407 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import shutil +from dataclasses import dataclass +from enum import Enum +from pathlib import Path + +import numpy as np + +from tests.functional.utils.logger import get_logger +from tests.functional.utils.process import Process +from tests.functional.config import custom_nodes_path, ovms_c_repo_path +from tests.functional.models import ModelInfo +from tests.functional.constants.ovms import CurrentOvmsType +from tests.functional.constants.ovms_type import OvmsType +from tests.functional.constants.paths import Paths + +logger = get_logger(__name__) + + +@dataclass +class CustomNode(ModelInfo): + path: str = None + filename: str = None + + def __post_init__(self): + super().__post_init__() + self.ovms_type = CurrentOvmsType.ovms_type + + if self.filename is None: + self.filename = f"libcustom_node_{self.name}.so" + + if self.path is None: + if self.ovms_type == OvmsType.KUBERNETES: + self.path = os.path.join("/config", self.filename) + else: + self.path = os.path.join(Paths.CUSTOM_NODE_LIBRARIES_PATH_INTERNAL, self.name, self.filename) + + def get_config(self): + config = {"name": self.name, "base_path": self.path} + return config + + def get_parameters(self): + return None + + def prepare_resources(self, resource_location): + raise NotImplementedError() + + @classmethod + def get_volume_mount(cls): + return {custom_nodes_path: {"bind": Paths.CUSTOM_NODE_LIBRARIES_PATH_INTERNAL, "mode": "ro"}} + + def get_parameters(self): + return {} + + @staticmethod + def copy_model_server_src_directory(destination_dir): + src_dst_path = os.path.join(destination_dir, "src") + if not os.path.exists(src_dst_path): + shutil.copytree(os.path.join(ovms_c_repo_path, "src"), src_dst_path) + third_party_dst_path = os.path.join(destination_dir, "third_party", "opencv") + if not os.path.exists(third_party_dst_path): + shutil.copytree(os.path.join(ovms_c_repo_path, "third_party", "opencv"), third_party_dst_path) + return src_dst_path + + @staticmethod + def get_custom_nodes_path(image): + cmd = f"docker cp $(docker create --rm {image}):/{Paths.CUSTOM_NODE_PATH_NAME} ." + proc = Process() + cwd = os.path.join(ovms_c_repo_path, "tests", "functional", "utils", "ovms_testing_image") + proc.run_and_check(cmd, cwd=cwd) + dst_file_path = os.path.join(cwd, Paths.CUSTOM_NODE_PATH_NAME) + return dst_file_path + + +@dataclass +class DevCustomNode(CustomNode): + src_type: str = "cpp" + src_dir: str = None + src_file_path: str = None + build_successfully: bool = None + + def __post_init__(self): + super().__post_init__() + if self.filename is None: + self.filename = f"libcustom_{self.name}.so" + + if self.src_file_path is None: + self.src_file_path = os.path.join(self.src_dir, self.name, f"{self.name}.{self.src_type}") + + def prepare_resources(self, resource_location): + pass + # src_dst_path = self.copy_model_server_src_directory(resource_location) + # lib_path = self.get_output_lib_path(src_dst_path) + # assert lib_path.exists() + # dst_lib_path = Path(resource_location, Paths.CUSTOM_NODE_PATH_NAME, lib_path.name) + # os.makedirs(dst_lib_path.parent, exist_ok=True) + # shutil.copyfile(lib_path, dst_lib_path) + # return [dst_lib_path.parent] + + def get_output_lib_path(self, tmp_ovms_source_files_path): + make_cwd = Path(tmp_ovms_source_files_path, Paths.CUSTOM_NODE_PATH_NAME) + output_lib_path = Path(make_cwd, "lib", self.base_os, self.filename) + return output_lib_path + + +@dataclass +class OvmsCCustomNode(DevCustomNode): + def __post_init__(self): + self.src_dir = os.path.join(ovms_c_repo_path, "src", Paths.CUSTOM_NODE_PATH_NAME) + self.src_file_path = os.path.join(self.src_dir, self.name, f"{self.name}.{self.src_type}") + super().__post_init__() + + +@dataclass +class OvmsCUnitTestCustomNode(DevCustomNode): + def __post_init__(self): + self.src_dir = os.path.join(ovms_c_repo_path, "src", "test", Paths.CUSTOM_NODE_PATH_NAME) + self.src_file_path = os.path.join(self.src_dir, f"{self.name}.{self.src_type}") + self.path = os.path.join(Paths.CUSTOM_NODE_LIBRARIES_PATH_INTERNAL, f"libcustom_node_{self.name}.so") + super().__post_init__() + + +# Custom nodes located in ovms-tests repo data/custom_nodes/ +@dataclass +class OvmsTestDevCustomNode(DevCustomNode): + def __post_init__(self): + self.src_dir = os.path.join(ovms_test_repo_path, "data", "ovms_testing_image", Paths.CUSTOM_NODE_PATH_NAME) + self.src_file_path = os.path.join(self.src_dir, self.name, f"{self.name}.{self.src_type}") + super().__post_init__() + + +@dataclass +class CustomNodeEastOcr(OvmsCCustomNode): + + def __init__(self, **kwargs): + super().__init__( + name="east_ocr", + inputs={ + "image": {"shape": [1, 3, 1024, 100], "dtype": np.float32}, + "scores": {"shape": [1, 256, 480, 1], "dtype": np.float32}, + "geometry": {"shape": [1, 256, 480, 5], "dtype": np.float32}, + }, + outputs={ + "text_images": {"shape": [0, 1, 3, 32, 100], "dtype": np.float32}, + "text_coordinates": {"shape": [0, 1, 4], "dtype": np.int32}, + "confidence_levels": {"shape": [0, 1, 1], "dtype": np.float32}, + }, + **kwargs, + ) + + def get_parameters(self): + return { + "original_image_width": "1920", + "original_image_height": "1024", + "original_image_layout": "NHWC", + "target_image_layout": "NHWC", + "target_image_width": "100", + "target_image_height": "32", + "confidence_threshold": "0.9", + "debug": "true", + } + + +@dataclass +class CustomNodeVehicles(OvmsCCustomNode): + + def __init__(self, **kwargs): + super().__init__( + name="model_zoo_intel_object_detection", + inputs={ + "image": {"shape": [1, 3, 512, 512], "dtype": np.float32}, + "detection": {"shape": [1, 1, 200, 7], "dtype": np.float32}, + }, + outputs={ + "images": {"shape": [0, 1, 3, 512, 512], "dtype": np.float32}, + "coordinates": {"shape": [0, 1, 4], "dtype": np.int32}, + "confidences": {"shape": [0, 1, 1], "dtype": np.float32}, + }, + **kwargs, + ) + + def get_parameters(self): + return { + "original_image_width": "512", + "original_image_height": "512", + "target_image_width": "72", + "target_image_height": "72", + "original_image_layout": "NHWC", + "target_image_layout": "NHWC", + "convert_to_gray_scale": "false", + "max_output_batch": "100", + "confidence_threshold": "0.7", + "debug": "false", + } + + +@dataclass +class CustomNodeFaces(OvmsCCustomNode): + + def __init__(self, **kwargs): + super().__init__( + name="model_zoo_intel_object_detection", + inputs={ + "image": {"shape": [1, 3, 600, 400], "dtype": np.float32}, + "detection": {"shape": [1, 1, 200, 7], "dtype": np.float32}, + }, + outputs={ + "images": {"shape": [0, 1, 3, 600, 400], "dtype": np.float32}, + "coordinates": {"shape": [0, 1, 4], "dtype": np.int32}, + "confidences": {"shape": [0, 1, 1], "dtype": np.float32}, + }, + **kwargs, + ) + + def get_parameters(self): + return { + "original_image_width": "600", + "original_image_height": "400", + "target_image_width": "64", + "target_image_height": "64", + "original_image_layout": "NHWC", + "target_image_layout": "NHWC", + "convert_to_gray_scale": "false", + "max_output_batch": "100", + "confidence_threshold": "0.7", + "debug": "true", + } + + +@dataclass +class CustomNodeImageTransformation(OvmsCCustomNode): + + def __init__(self, original_image_layout="NCHW", target_image_layout="NCHW", **kwargs): + super().__init__( + name="image_transformation", + inputs={"image": {"shape": [1, 3, 224, 224], "dtype": np.float32}}, + outputs={"image": {"shape": [1, 3, 224, 224], "dtype": np.float32}}, + **kwargs, + ) + + self.original_image_layout = original_image_layout + self.target_image_layout = target_image_layout + + def get_parameters(self): + return { + "target_image_width": "224", + "target_image_height": "224", + "original_image_color_order": "RGB", + "target_image_color_order": "RGB", + "original_image_layout": self.original_image_layout, + "target_image_layout": self.target_image_layout, + "scale_values": "[0.003921568627451,0.003921568627451,0.003921568627451]", + "mean_values": "[-2,-2,-2]", + "debug": "true", + } + + +@dataclass +class CustomNodeDemultiply(OvmsTestDevCustomNode): + ORIGINAL_DEMULTIPLY_COUNT = 3 + + def __init__(self, demultiply_size=None, **kwargs): + super().__init__( + name="demultiply", + inputs={"tensor": {"shape": [1, 3, 224, 224], "dtype": np.float32}}, + outputs={"tensor_out": {"shape": [demultiply_size, 1, 3, 224, 224], "dtype": np.float32}}, + **kwargs, + ) + + self.demultiply_size = demultiply_size + + def get_parameters(self): + return {"demultiply_size": str(self.demultiply_size)} + + +@dataclass +class CustomNodeElastic1T(OvmsTestDevCustomNode): + + def __init__(self, input_shape=None, output_shape=None, **kwargs): + super().__init__( + name="elastic_in_1t_out_1t", + inputs={"tensor_in": {"shape": input_shape, "dtype": np.float32}}, + outputs={"tensor_out": {"shape": output_shape, "dtype": np.float32}}, + **kwargs, + ) + self.input_shape = input_shape + self.output_shape = output_shape + + def get_parameters(self): + return {"input_shape": str(self.input_shape), "output_shape": str(self.output_shape)} + + +@dataclass +class CustomNodeDemultiplyGather(OvmsTestDevCustomNode): + + def __init__(self, **kwargs): + super().__init__( + name="demultiply_gather", + inputs={"tensor": {"shape": [4, 1, 10], "dtype": np.float32}}, + outputs={"tensor_out": {"shape": [4, 4, 1, 10], "dtype": np.float32}}, + **kwargs, + ) + + def get_parameters(self): + return { + "demultiply_count": "4", + } + + +@dataclass +class CustomNodeDifferentOperations(OvmsCUnitTestCustomNode): + + def __init__(self, **kwargs): + super().__init__( + name="node_perform_different_operations", + inputs={ + "input_numbers": {"shape": [1, 10], "dtype": np.float32}, + "op_factors": {"shape": [1, 4], "dtype": np.float32}, + }, + outputs={ + "different_ops_results": {"shape": [0, 1, 10], "dtype": np.float32}, + "factors_results": {"shape": [0, 1, 0], "dtype": np.float32}, + }, + **kwargs, + ) + + +@dataclass +class CustomNodeChooseMaximum(OvmsCUnitTestCustomNode): + + class Method(Enum): + MAXIMUM_MINIMUM = "MAXIMUM_MINIMUM" + MAXIMUM_AVERAGE = "MAXIMUM_AVERAGE" + MAXIMUM_MAXIMUM = "MAXIMUM_MAXIMUM" + + selection_criteria: Method = None + + def __init__(self, **kwargs): + super().__init__( + name="node_choose_maximum", + inputs={"input_tensors": {"shape": [4, 1, 10], "dtype": np.float32}}, + outputs={"maximum_tensor": {"shape": [1, 10], "dtype": np.float32}}, + **kwargs, + ) + + def __post_init__(self): + self.selection_criteria = CustomNodeChooseMaximum.Method.MAXIMUM_MINIMUM + super().__post_init__() + + def get_parameters(self): + return {"selection_criteria": str(self.selection_criteria.value)} + + +@dataclass +class CustomNodeDynamicDemultiplex(OvmsCUnitTestCustomNode): + + def __init__(self, **kwargs): + super().__init__( + name="node_dynamic_demultiplex", + inputs={"input_numbers": {"shape": [1, 10], "dtype": np.float32}}, + outputs={"dynamic_demultiplex_results": {"shape": [0, 1, 10], "dtype": np.float32}}, + **kwargs, + ) + + +@dataclass +class CustomNodeAddSub(OvmsCUnitTestCustomNode): + def __init__(self, add_value=None, sub_value=None, **kwargs): + super().__init__( + name="node_add_sub", + inputs={"input_numbers": {"shape": [1, 2], "dtype": np.float32}}, + outputs={"output_numbers": {"shape": [1, 2], "dtype": np.float32}}, + **kwargs, + ) + self.add_value = np.float32(add_value) + self.sub_value = np.float32(sub_value) + + def __post_init__(self): + self.src_type = "c" + super().__post_init__() + + def get_expected_output(self, input_data: dict, client_type: str = None): + if input_data is None: + return None + + input_value = list(input_data.values())[0] + result = input_value + self.add_value - self.sub_value + return {self.output_names[0]: result} + + def get_parameters(self): + return {"add_value": str(self.add_value), "sub_value": str(self.sub_value)} diff --git a/tests/functional/object_model/dmesg_log_monitor.py b/tests/functional/object_model/dmesg_log_monitor.py new file mode 100644 index 0000000000..84c4c76ecf --- /dev/null +++ b/tests/functional/object_model/dmesg_log_monitor.py @@ -0,0 +1,185 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import re + +import requests + +import tests.functional.utils.assertions as assertions_module +from tests.functional.utils.assertions import ( + BadRIPValue, + DmesgBpFilterFail, + DmesgError, + GeneralProtectionFault, + GPUHangError, + OOMKillError, + OvmsCrashed, + SegfaultError, + TrapDivideError, +) +from tests.functional.utils.core import get_children_from_module +from tests.functional.utils.logger import get_logger +from tests.functional.utils.process import Process +from tests.functional.utils.test_framework import generate_test_object_name +from tests.functional.config import artifacts_dir +from tests.functional.utils.log_monitor import LogMonitor + +logger = get_logger(__name__) +MAX_DMESG_LINES = 1000 + +DMESG_UNEXPECTED_MESSAGES = set() + + +class DummyLogMonitor(LogMonitor): + def _get_unexpected_messages_regex(self): + return [] + + def _get_unexpected_messages(self): + pass + + def _refresh(self): + pass + + def get_all_logs(self): + return [] + + +class DmesgLogMonitor(LogMonitor): + + def _get_unexpected_messages_regex(self): + return [DmesgBpFilterFail.regex] + + def _get_unexpected_messages(self): + messages = [ + BadRIPValue.msg, + SegfaultError.msg, + GPUHangError.msg, + GeneralProtectionFault.msg, + TrapDivideError.msg, + OOMKillError.msg, + ] + if self.ovms_pid: + messages.append(f"ovms[{self.ovms_pid}]") + return messages + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._proc = Process() + self._proc.set_log_silence() + self._proc.disable_check_stderr() + self._start_test_marker = "" + self.ovms_pid = kwargs.get("ovms_pid", None) + + def _calculate_current_offset(self, start_log_marker): + log_cnt = len(self._read_lines) + if start_log_marker: + while log_cnt > 0: + start_log_entry = self._read_lines[log_cnt - 1] + if start_log_entry == start_log_marker: + self.current_offset = log_cnt + break + log_cnt -= 1 + + if not self._start_test_marker and self._read_lines: + assert len(self._read_lines) > self.current_offset + self._start_test_marker = self._read_lines[self.current_offset - 1] + + return log_cnt + + def _refresh(self, start_position=None): + if start_position is None: + start_position = self._read_lines[self.current_offset - 1] if self._read_lines else "" + # skip non-fatal log levels ("notice,info,debug") + level = ",".join(["emerg", "alert", "crit", "err", "warn", "info"]) + stdout = self._proc.run_and_check(f"dmesg -T --level={level}", exception_type=DmesgError) + self._read_lines = stdout.splitlines() + self.current_offset = self._calculate_current_offset(start_position) + + def _generate_smoke_dmesg_log_name(self): + name = generate_test_object_name(separator="_") + filename = f"dmesg_smoke_{name}.log" + return filename + + def get_all_logs(self): + try: + self._refresh(self._start_test_marker) + except requests.exceptions.HTTPError as e: + raise OvmsCrashed(msg=str(e), dmesg_log=self.get_logs_as_txt()) + return self._read_lines[self.current_offset:] + + def reset_to_logger_creation(self): + if self._start_test_marker: + self.current_offset = self._calculate_current_offset(self._start_test_marker) + + def filter_unexpected_messages( + self, log_entry, found_unexpected_messages, unexpected_messages, unexpected_messages_re + ): + super().filter_unexpected_messages( + log_entry, found_unexpected_messages, unexpected_messages, unexpected_messages_re + ) + if log_entry.split("] ")[-1].startswith(" in lib"): + logger.info("Get the next dmesg line to catch lib") + found_unexpected_messages.append(log_entry) + + def raise_on_unexpected_messages(self, logs=None, filter_known_messages=False): + unexpected_messages = self.check_for_unexpected_messages(logs=logs) + DMESG_UNEXPECTED_MESSAGES.update(unexpected_messages) + if filter_known_messages: + unexpected_messages = [ + unexpected_message + for unexpected_message in unexpected_messages + if unexpected_message not in DMESG_UNEXPECTED_MESSAGES + ] + logger.info(f"Dmesg OVMS process ID: {self.ovms_pid}") + if unexpected_messages: + dmesg_exceptions = get_children_from_module(DmesgError, assertions_module) # [(name, class_def), ...] + for name, exception_class in dmesg_exceptions: + msg = getattr(exception_class, "msg", None) + regex = getattr(exception_class, "regex", None) + if (msg and any(filter(lambda x: msg in x, unexpected_messages))) or ( + regex and any(filter(lambda x: regex.match(x), unexpected_messages)) + ): + logger.error(f"Found unexpected message in dmesg logs: {msg}") + raise exception_class("\n".join(unexpected_messages), dmesg_log=logs) + + def dump_dmesg_logs_into_file(self, level=None): + sysctl_stdout = self._proc.run_and_check("sysctl kernel.dmesg_restrict") + if re.match(r"kernel.dmesg_restrict = (\d)", sysctl_stdout).group(1) != "0": + message = ( + "Missing permissions to use dmesg without sudo. " "Please set 'sudo sysctl -w kernel.dmesg_restrict=0'." + ) + raise DmesgError(message) + filename = self._generate_smoke_dmesg_log_name() + if level is None: + level = ["emerg", "alert", "crit", "err", "warn"] + level = ",".join(level) + os.makedirs(artifacts_dir, exist_ok=True) + file_path = os.path.join(artifacts_dir, filename) + stdout = self._proc.run_and_check( + f"dmesg --level={level} -T | tail -n {MAX_DMESG_LINES} | tee {file_path}", exception_type=DmesgError + ) + logger.info(f"Logs saved {file_path}") + return file_path, stdout + + def search_for_error_in_dmesg_file(self, stdout): + if stdout is not None: + logs = stdout.splitlines() + self.raise_on_unexpected_messages(logs=logs) + + def clear_dmesg_buffer(self): + self._proc.run_and_check("sudo dmesg -C") + logger.info("Dmesg cleared") diff --git a/tests/functional/object_model/docker.py b/tests/functional/object_model/docker.py deleted file mode 100644 index d9df91a917..0000000000 --- a/tests/functional/object_model/docker.py +++ /dev/null @@ -1,185 +0,0 @@ -# -# Copyright (c) 2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -try: - from grp import getgrnam -except ImportError: - getgrnam = None - -import os -import time -from typing import List -from datetime import datetime - -import docker -from docker.types import DeviceRequest -import logging -from retry.api import retry_call -from tests.functional.utils.files_operation import get_path_friendly_test_name - -import tests.functional.config as config -from tests.functional.utils.grpc import port_manager_grpc -from tests.functional.utils.rest import port_manager_rest -from tests.functional.constants.target_device import TargetDevice - - -logger = logging.getLogger(__name__) -CONTAINER_STATUS_RUNNING = "running" -TERMINAL_STATUSES = ["exited"] - -TARGET_DEVICE_CONFIGURATION = { - TargetDevice.CPU: lambda: { - 'volumes': {}, - "privileged": False, - }, - - TargetDevice.GPU: lambda: { - 'volumes': {}, - "devices": ["/dev/dri:/dev/dri:mrw"], - "privileged": False, - "user": None, - "group_add": [getgrnam('render').gr_gid, getgrnam('video').gr_gid] if getgrnam is not None else [] - }, - -} - - -class Docker: - - COMMON_RETRY = {"tries": 360, "delay": 0.5} - GETTING_LOGS_RETRY = COMMON_RETRY - GETTING_STATUS_RETRY = COMMON_RETRY - - def __init__(self, request, container_name, start_container_command, - env_vars_container=None, image=config.image, container_log_line=config.container_log_line, - server=None): - self.server = server - self.client = docker.from_env() - self.grpc_port = port_manager_grpc.get_port() - self.rest_port = port_manager_rest.get_port() - self.image = image - self.container = None - self.request = request - self.container_name = container_name - self.start_container_command = start_container_command - self.env_vars_container = env_vars_container if env_vars_container else [] - self.container_log_line = container_log_line - self.logs = "" - - def start(self): - start_result = None - try: - start_result = self._start() - finally: - if start_result is None: - self.stop() # Failed to start container so clean it up - return start_result - - def _start(self): - logger.info(f"Starting container: {self.container_name}") - - ports = {'{}/tcp'.format(self.grpc_port): self.grpc_port, '{}/tcp'.format(self.rest_port): self.rest_port} - device_cfg = TARGET_DEVICE_CONFIGURATION[config.target_device]() - volumes_dict = {config.path_to_mount: {'bind': '/opt/ml', 'mode': 'ro'}} - device_cfg['volumes'].update(volumes_dict) - - self.container = self.client.containers.run(image=self.image, detach=True, - name=self.container_name, - ports=ports, - command=self.start_container_command, - environment=self.env_vars_container, - **device_cfg) - self.ensure_container_status(status=CONTAINER_STATUS_RUNNING, terminal_statuses=TERMINAL_STATUSES) - self.ensure_logs_contains() - logger.info(f"Container started grpc_port:{self.grpc_port}\trest_port:{self.rest_port}") - logger.debug(f"Container starting command args: {self.start_container_command}") - return self.container, {"grpc_port": self.grpc_port, "rest_port": self.rest_port} - - def stop(self): - if self.container is not None: - logger.info(f"Stopping container: {self.container_name}") - self.container.stop(timeout=10) - self.save_container_logs() - self.container.remove(v=True) - self.container = None - port_manager_grpc.release_port(self.grpc_port) - port_manager_rest.release_port(self.rest_port) - logger.info(f"Container successfully closed and removed: {self.container_name}") - - def save_container_logs(self): - logs = self.get_logs() - if config.log_level == "DEBUG": - logger.info(logs) - if config.artifacts_dir != "": - location = getattr(self.request.node, "location", None) - self.save_container_logs_to_file(logs=logs, location=location) - - def get_logs(self): - self.logs = self.container.logs().decode() - return self.logs - - def ensure_logs(self): - logs = self.get_logs() - for log_line in self.container_log_line: - if log_line not in logs: - assert False, f"Not found required phrase {log_line}" - - - def ensure_logs_contains(self): - result = None - try: - result = retry_call(self.ensure_logs, exceptions=AssertionError, **Docker.GETTING_LOGS_RETRY) - except: - if config.log_level == "DEBUG": - logger.info(str(self.get_logs())) - return result - - def get_container_status(self): - container = self.client.containers.get(self.container.id) - return container.status - - def ensure_status(self, status, terminal_statuses=None): - current_status = self.get_container_status() - logger.debug(f"Ensure container status, expected_status={status}\t current_status={current_status}") - ovms_logs = self.container.logs().decode('ascii') - if terminal_statuses is not None and current_status in terminal_statuses: - raise RuntimeError("Received terminal status '{}' for container {} - \nOVMS LOGS:\n===\n{}\n===".format( - current_status, self.container_name, ovms_logs)) - assert current_status == status, \ - "Not expected status for container {} found. \n" \ - "Expected: {}, \n" \ - "received: {}\n" \ - "Ovms logs: {}\n".format(self.container.name, status, self.container.status, ovms_logs) - - def ensure_container_status(self, status: str = CONTAINER_STATUS_RUNNING, - terminal_statuses: List[str] = None): - container_statuses = {"status": status} - if terminal_statuses: - container_statuses["terminal_statuses"] = terminal_statuses - time.sleep(1) - return retry_call(self.ensure_status, fkwargs=container_statuses, - exceptions=AssertionError, **Docker.GETTING_STATUS_RETRY) - - def save_container_logs_to_file(self, logs, dir_path: str = config.artifacts_dir, location=None): - time_stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - if location: - file_name = f"ovms_{get_path_friendly_test_name(location)}_{time_stamp}.log" - else: - file_name = f"ovms_{self.server.started_by_fixture.lstrip('start_')}_{time_stamp}.log" - os.makedirs(dir_path, exist_ok=True) - file_path = os.path.join(dir_path, file_name) - with open(file_path, "w+") as text_file: - text_file.write(logs) diff --git a/tests/functional/object_model/inference_helpers.py b/tests/functional/object_model/inference_helpers.py new file mode 100644 index 0000000000..ed69824b1f --- /dev/null +++ b/tests/functional/object_model/inference_helpers.py @@ -0,0 +1,1457 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import base64 +import json +import os +import struct +import time +import cohere +from collections import defaultdict +from copy import deepcopy +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from functools import reduce +from inspect import isclass +from pathlib import Path +from threading import Event +from typing import List, Union + +import grpc +import numpy as np +import requests +import tritonclient +from google.protobuf.json_format import MessageToJson +from grpc import RpcError +from grpc._channel import _InactiveRpcError +from openai import OpenAI +from pydantic import BaseModel +from retry.api import retry_call +from tensorflow import make_tensor_proto +from tensorflow_serving.apis import get_model_status_pb2 +from tensorflow_serving.apis.predict_pb2 import PredictRequest +from tritonclient.grpc import service_pb2, service_pb2_grpc +from tritonclient.grpc.service_pb2 import ModelInferRequest +from tritonclient.utils import InferenceServerException, deserialize_bytes_tensor, serialize_byte_tensor + +from tests.functional.utils.assertions import ModelNotReadyException, StreamingApiException, UnexpectedResponseError +from tests.functional.utils.inference.communication.grpc import GRPC, GrpcCommunicationInterface, channel_options +from tests.functional.utils.inference.communication.rest import REST, RestCommunicationInterface +from tests.functional.utils.inference.serving.cohere import RerankApi, CohereRequestParams, CohereWrapper +from tests.functional.utils.inference.serving.kf import KFS, KserveWrapper +from tests.functional.utils.inference.serving.openai import ( + ChatCompletionsApi, + CompletionsApi, + EmbeddingsApi, + OpenAIRequestParams, + OpenAIWrapper, + ImagesApi, + AudioApi, + ResponsesApi, +) +from tests.functional.utils.inference.serving.tf import TensorFlowServingWrapper +from tests.functional.utils.logger import get_logger +from tests.functional.utils.test_framework import FrameworkMessages, skip_if_runtime +from tests.functional.utils.generative_ai.validation_utils import GenerativeAIValidationUtils +from tests.functional.config import binary_io_images_path, wait_for_messages_timeout +from tests.functional.models.models_datasets import ( + BinaryDummyModelDataset, + DefaultBinaryDataset, + ExactShapeBinaryDataset, + LanguageModelDataset, + ModelDataset, +) +from tests.functional.models import ModelInfo +from tests.functional.constants.ovms import CurrentTarget as ct +from tests.functional.constants.ovms import MediaPipeConstants, Ovms +from tests.functional.constants.pipelines import SimpleMediaPipe +from tests.functional.object_model.ovms_instance import OvmsInstance +from tests.functional.object_model.ovsa import OvsaCerts +from tests.functional.object_model.python_custom_nodes.common import STREAMING_CHANNEL_ARGS +from tests.functional.object_model.python_custom_nodes.python_custom_nodes import SimplePythonCustomNodeMediaPipe +from tests.functional.object_model.test_environment import TestEnvironment +from tests.functional.object_model.test_helpers import run_all_actions + +logger = get_logger(__name__) + + +class InferenceBuilder(object): + + def __init__(self, model): + self.model = model + + def create_client( + self, api_type, port, batch_size=Ovms.BATCHSIZE, ovsa_certs=None, model_version=None, client_type=None + ): + ovsa_certs = ovsa_certs if ovsa_certs is not None else OvsaCerts.default_certs + if client_type == KFS: + assert 0, "Please check flow for KFS client" + kfs_api_type = InferenceClientKFS if api_type == InferenceClientTFS else InferenceRestClientKFS + inference_client = self.create_kfs_client(kfs_api_type, port) + else: + inference_client = api_type( + port=port, + model_name=self.model.name, + batch_size=batch_size, + input_names=list(self.model.inputs.keys()), + output_names=list(self.model.outputs.keys()), + model_meta_from_serving=False, + ssl_certificates=ovsa_certs, + model_version=model_version, + ) + inference_client._model = self.model + return inference_client + + def create_client_and_data(self, inference_request, random_data=False): + if inference_request.client_type == KFS: + # This should be included into KserveWrapper, please correct calling test not to use `create_client_and_data` + assert False, "Please correct it" + kfs_api_type = ( + InferenceClientKFS if inference_request.api_type == InferenceClientTFS else InferenceRestClientKFS + ) + port = inference_request.get_port() + inference_client = self.create_kfs_client(kfs_api_type, port) + else: + inference_client = self.create_client( + inference_request.api_type, + inference_request.get_port(), + inference_request.batch_size, + client_type=inference_request.client_type, + model_version=inference_request.model_version, + ) + if inference_request is not None and inference_request.dataset: + input_data = inference_request.load_data() + else: + input_data = self.model.prepare_input_data(inference_request.batch_size, random_data=random_data) + return inference_client, input_data + + def create_kfs_client(self, api_type, port): + kfs_api_client = api_type(port, model_name=self.model.name, batch_size=self.model.batch_size) + kfs_api_client.model = self.model + kfs_api_client.port = port + kfs_api_client.model_name = self.model.name + return kfs_api_client + + +@dataclass(frozen=False) +class InferenceRequest(object): + ovms: OvmsInstance = None + model: ModelInfo = None + api_type: object = None + batch_size: int = None + dataset: ModelDataset = None + client_type: str = None + model_version: str = None + + def get_port(self): + return self.ovms.ovms_ports[self.api_type.type] + + def prepare_request_to_send(self, client, input_data, mediapipe_name=None): + request = client.prepare_request(input_objects=input_data, mediapipe_name=mediapipe_name) + logger.debug(f"Request: {request}") + return request + + def validate(self, response): + pass + + def get_expected_output_shape(self): + expected_shape = {} + for output_name, output_data in self.model.outputs.items(): + expected_shape[output_name] = deepcopy(output_data["shape"]) + if self.batch_size: + expected_shape[output_name][0] = self.batch_size + if self.model.get_demultiply_count() is not None: + expected_shape[output_name].insert(0, self.model.default_demultiply_count_value) + + return expected_shape + + +@dataclass(frozen=False) +class BinaryInferenceRequest(InferenceRequest): + layout: str = Ovms.BINARY_IO_LAYOUT_ROW_NAME + dataset: ModelDataset = field(default_factory=lambda: DefaultBinaryDataset()) + format: str = None + validate_match: bool = True + batch_size: int = 1 + + def _create_post_request(self, input_names, image_data, request_format=Ovms.BINARY_IO_LAYOUT_ROW_NAME): + signature = "serving_default" + instances = [] + for input_name in input_names: + data = image_data[input_name] + for single_data in data: + image_bytes_encoded = base64.b64encode(single_data).decode("utf-8") + if request_format == Ovms.BINARY_IO_LAYOUT_ROW_NAME: + instances.append({input_name: {"b64": image_bytes_encoded}}) + else: + instances.append({"b64": image_bytes_encoded}) + + if request_format == Ovms.BINARY_IO_LAYOUT_ROW_NAME: + data_obj = {"signature_name": signature, "instances": instances} + elif request_format == Ovms.BINARY_IO_LAYOUT_ROW_NONAME: + data_obj = {"signature_name": signature, "instances": instances} + elif request_format == Ovms.BINARY_IO_LAYOUT_COLUMN_NAME: + data_obj = {"signature_name": signature, "inputs": {input_name: instances}} + elif request_format == Ovms.BINARY_IO_LAYOUT_COLUMN_NONAME: + data_obj = {"signature_name": signature, "inputs": instances} + else: + print("invalid request format defined") + exit(1) + data_json = json.dumps(data_obj) + return {"request": data_json} + + def prepare_request_to_send(self, client, input_data): + if isinstance(client, KserveWrapper) and isinstance(client, GrpcCommunicationInterface): + request = ModelInferRequest() + request.model_name = self.model.name + inputs, outputs = self.prepare_binary_inputs_outputs(input_data) + request.inputs.extend(inputs) + request.outputs.extend(outputs) + request = {"request": request} + elif isinstance(client, KserveWrapper) and isinstance(client, RestCommunicationInterface): + request = self._create_kfs_post_request(input_data) + elif isinstance(client, TensorFlowServingWrapper) and isinstance(client, RestCommunicationInterface): + request = self._create_post_request(self.model.input_names, input_data, request_format=self.layout) + elif isinstance(client, TensorFlowServingWrapper) and isinstance(client, GrpcCommunicationInterface): + request = PredictRequest() + request.model_spec.name = self.model.name + for input_name, input_object in input_data.items(): + request.inputs[input_name].CopyFrom(make_tensor_proto(input_object, shape=[len(input_object)])) + request = {"request": request} + else: + raise NotImplementedError + return request + + def _create_kfs_post_request(self, input_data): + batch_i = 0 + image_binary_size = [] + shape = [] + inputs = [] + input_object = None + + for input_name, input_object in input_data.items(): + for obj in input_object: + if batch_i < len(input_object): + image_binary_size.append(len(obj)) + batch_i += 1 + + shape.extend([len(input_object)]) + _summarized_size = 4 * len(image_binary_size) + reduce(lambda x, y: x + y, image_binary_size) + + _input = { + "name": input_name, + "shape": shape, + "datatype": "BYTES", + "parameters": {"binary_data_size": _summarized_size}, + } + inputs.append(_input) + + request_header = json.dumps({"inputs": inputs}, separators=(",", ":")) + + # https://wiki.ith.intel.com/display/OVMS/Changes+in+KFS+BYTES+format + # https://github.com/openvinotoolkit/model_server/blob/main/docs/binary_input_kfs.md + if input_object is not None: + _ovms_formatted_binary_objects = b"".join([len(x).to_bytes(4, "little") + x for x in input_object]) + binary_data = _ovms_formatted_binary_objects + else: + binary_data = b"" + + request_body = struct.pack( + "{}s{}s".format(len(request_header), len(binary_data)), request_header.encode(), binary_data + ) + return { + "request": request_body, + "inference_header": {"Inference-Header-Content-Length": str(len(request_header))}, + } + + def load_data(self): + result = dict() + for param_name, param_data in self.model.inputs.items(): + result[param_name] = self.dataset.get_data( + param_data["shape"], self.batch_size, self.model.transpose_axes, None + ) + return result + + def validate(self, response): + if self.validate_match: + assert self.dataset.verify_match(response) + + def prepare_binary_inputs_outputs(self, input_data): + dtype = "BYTES" + inputs = [] + outputs = [] + + for input_name, input_object in input_data.items(): + input = service_pb2.ModelInferRequest().InferInputTensor() + input.name = input_name + input.datatype = dtype + input.shape.extend([len(input_object)]) + batch_i = 0 + while batch_i < len(input_object): + input.contents.bytes_contents.append(input_object[batch_i]) + batch_i += 1 + inputs.append(input) + + for output_name in self.model.outputs: + output = service_pb2.ModelInferRequest().InferRequestedOutputTensor() + output.name = output_name + outputs.append(output) + + return inputs, outputs + + +@dataclass(frozen=False) +class BinaryInferenceStaticShapeRequest(BinaryInferenceRequest): + layout: str = Ovms.BINARY_IO_LAYOUT_ROW_NAME + dataset: ModelDataset = None + format: str = None + validate_match: bool = True + batch_size: int = 1 + + def __post_init__(self): + self.dataset = ExactShapeBinaryDataset(shape=[224, 224, 3]) + + def validate(self, response): + return True + + +@dataclass(frozen=False) +class BinaryInferenceHeteroShapeRequest(BinaryInferenceRequest): + batch_config: list = field(default_factory=lambda: []) + tmp_data_file_location: str = None + format: str = Ovms.PNG_IMAGE_FORMAT + + def __post_init__(self): + self.batch_size = len(self.batch_config) + for _, in_data in self.model.inputs.items(): + in_data["dataset"] = BinaryDummyModelDataset() + + def load_data(self): + result = defaultdict(lambda: []) + tmp_binary_dir = Path(os.path.join(self.tmp_data_file_location, Path(binary_io_images_path).name)) + for param_name, param_data in self.model.inputs.items(): + for shape in self.batch_config: + result[param_name].append(param_data["dataset"].create_data(tmp_binary_dir, shape, self.format)) + return result + + def validate(self, response): + return True + + +@dataclass(frozen=False) +class LLMInferenceRequest(InferenceRequest): + request_parameters: OpenAIRequestParams = None + set_null_values: bool = False + use_extra_body: bool = True + + def __post_init__(self): + self.api_type.create_base_url() + self.openai_client = OpenAI(base_url=self.api_type.base_url, api_key=self.api_type.API_KEY_UNUSED) + self.request_parameters_dict = self.prepare_request_parameters_dict( + set_null_values=self.set_null_values, + use_extra_body=self.use_extra_body, + ) + if hasattr(self.request_parameters, "stream"): + self.stream = self.request_parameters.stream if self.request_parameters.stream is not None else False + + def prepare_request_parameters_dict(self, set_null_values=False, use_extra_body=True): + if self.request_parameters: + assert isinstance(self.request_parameters, (OpenAIRequestParams, CohereRequestParams)), ( + f"Wrong type of request_parameters expected: {OpenAIRequestParams} or {CohereRequestParams}" + f"actual: {type(self.request_parameters)}" + ) + return self.request_parameters.prepare_dict( + set_null_values=set_null_values, + use_extra_body=use_extra_body, + ) + else: + return {} + + def create_chat_completions(self, messages, model_name=None, timeout=None): + model = model_name if model_name is not None else self.api_type.model.name + response_format = self.request_parameters_dict.get("response_format", None) + if response_format is not None and isclass(response_format) and issubclass(response_format, BaseModel): + chat_completions = self.openai_client.beta.chat.completions.parse( + model=model, + messages=messages, + timeout=timeout, + **self.request_parameters_dict, + ) + else: + chat_completions = self.openai_client.chat.completions.create( + model=model, + messages=messages, + timeout=timeout, + **self.request_parameters_dict, + ) + outputs = [chat_completions] if not self.stream else chat_completions + return outputs + + def create_completions(self, prompt, model_name=None, timeout=None): + model = model_name if model_name is not None else self.api_type.model.name + completions = self.openai_client.completions.create( + model=model, + prompt=prompt, + **self.request_parameters_dict, + timeout=timeout, + ) + outputs = [completions] if not self.stream else completions + return outputs + + def create_responses(self, input_content, model_name=None, timeout=None): + model = model_name if model_name is not None else self.api_type.model.name + response = self.openai_client.responses.create( + model=model, + input=input_content, + **self.request_parameters_dict, + timeout=timeout, + ) + outputs = [response] if not self.stream else response + return outputs + + def create_embeddings(self, embeddings_input, model_name=None, timeout=None): + model = model_name if model_name is not None else self.api_type.model.name + embeddings = self.openai_client.embeddings.create( + model=model, + input=embeddings_input, + **self.request_parameters_dict, + timeout=timeout, + ) + return embeddings + + def create_image_generation(self, prompt, model_name=None, timeout=None): + model = model_name if model_name is not None else self.api_type.model.name + image = self.openai_client.images.generate( + model=model, + prompt=prompt, + **self.request_parameters_dict, + timeout=timeout, + ) + return image + + def create_image_edit(self, prompt, image_path=None, mask_path=None, model_name=None, timeout=None): + model = model_name if model_name is not None else self.api_type.model.name + if mask_path is not None: + self.request_parameters_dict["mask"] = open(mask_path, "rb") + image = self.openai_client.images.edit( + model=model, + prompt=prompt, + image=open(image_path, "rb"), + **self.request_parameters_dict, + timeout=timeout, + ) + return image + + def create_models_list(self): + models_list = self.openai_client.models.list() + return models_list + + def create_models_retrieve(self, model_name=None): + model = model_name if model_name is not None else self.api_type.model.name + retrieve = self.openai_client.models.retrieve(model=model) + return retrieve + + def create_audio_speech(self, input_text, speech_file_path, model_name=None, timeout=None): + model = model_name if model_name is not None else self.api_type.model.name + voice = self.request_parameters_dict.pop("voice", None) + with self.openai_client.audio.speech.with_streaming_response.create( + model=model, + voice=voice, # voice is a required parameter in OpenAI API; OVMS accepts None for default + input=input_text, + **self.request_parameters_dict, + timeout=timeout, + ) as response: + response.stream_to_file(speech_file_path) + return response + + def create_audio_transcription(self, audio_file_path, model_name=None, timeout=None): + model = model_name if model_name is not None else self.api_type.model.name + with open(audio_file_path, "rb") as audio_file: + transcript = self.openai_client.audio.transcriptions.create( + model=model, + file=audio_file, + **self.request_parameters_dict, + timeout=timeout, + ) + return transcript.text + + def create_audio_translation(self, audio_file_path, model_name=None, timeout=None): + model = model_name if model_name is not None else self.api_type.model.name + with open(audio_file_path, "rb") as audio_file: + translation = self.openai_client.audio.translations.create( + model=model, + file=audio_file, + **self.request_parameters_dict, + timeout=timeout, + ) + return translation.text + + +@dataclass(frozen=False) +class RerankLLMInferenceRequest(LLMInferenceRequest): + request_parameters: OpenAIRequestParams = None + + def __post_init__(self): + self.api_type.create_base_url() + self.cohere_client = cohere.Client(base_url=self.api_type.base_url, api_key=self.api_type.API_KEY_NOT_USED) + self.request_parameters_dict = self.prepare_request_parameters_dict() + + def create_rerank(self, rerank_input, model_name=None): + model = model_name if model_name is not None else self.api_type.model.name + rerank = self.cohere_client.rerank( + model=model, + query=rerank_input["query"], + documents=rerank_input["documents"], + **self.request_parameters_dict, + ) + return rerank + + +class InferenceResponse(object): + + def __init__(self, inference_info, response): + self.inference_info = inference_info + self.response = response + + @classmethod + def create(cls, inference_info, response): + return InferenceResponse(inference_info, response) + + def ensure_outputs_exist(self): + for output_name in self.inference_info.model.outputs: + assert output_name in self.response, "Incorrect output name, expected: {}, found: {}.".format( + output_name, ", ".join(self.response.keys()) + ) + + def validate(self, input_data): + self.ensure_outputs_exist() + expected_output = self.inference_info.model.get_expected_output(input_data) + if expected_output is not None: + self.validate_expected_output(expected_output, self.response) + + self.validate_expected_shape(self.response) + self.inference_info.inference_request.validate(self.response) + + def validate_expected_output(self, expected_output, response, equal=False): + for key in expected_output.keys(): + if equal: + result = np.array_equal(expected_output[key], self.response[key]) + else: + # Note: rtol=1.e-3 was changed from default value (rtol=1.e-5) after a bug was found: CVS-107839 + rtol = ( + 1.0e-3 + if any([ct.is_gpu_target(), ct.is_auto_target(), ct.is_hetero_target()]) + else 1.0e-5 + ) + result = np.allclose(expected_output[key], response[key], rtol=rtol) + if not result: + logger.info(f"Received output for key '{key}': {response[key]}") + logger.info(f"Expected output for key '{key}': {expected_output[key]}") + + assert result, "Received output is different than expected" + + def validate_expected_shape(self, response): + output_shape = {name: list(data.shape) for name, data in response.items()} + expected_shape = self.inference_info.inference_request.get_expected_output_shape() + + validation_pass = True + for name, expected_data in expected_shape.items(): + for dim, expected_dim_value in enumerate(expected_data): + if expected_dim_value > 0: + validation_pass = expected_dim_value == output_shape[name][dim] + + assert validation_pass, "Incorrect output shape, expected: {}, found: {}.".format(expected_shape, output_shape) + logger.debug(f"Output shape: {output_shape} (expected: {expected_shape})") + + +class MediaPipeInferenceResponse(InferenceResponse): + + def __init__(self, inference_info, response): + super().__init__(inference_info, response) + + @classmethod + def create(cls, inference_info, response): + return MediaPipeInferenceResponse(inference_info, response) + + def ensure_outputs_exist(self, output_key=None): + for elem in self.response: + if output_key is not None: + elem == output_key + else: + assert MediaPipeConstants.DEFAULT_OUTPUT_STREAM in elem + + def validate(self, input_data, output_key=None): + logger.debug(f"MediaPipeInferenceResponse: {self.response}") + self.ensure_outputs_exist(output_key) + expected_output = self.inference_info.model.get_expected_output(input_data, self.inference_info.client.type) + if expected_output is not None: + if isinstance(self.inference_info.model, SimplePythonCustomNodeMediaPipe): + self.validate_expected_output(expected_output, self.response, equal=True) + else: + self.validate_expected_output(expected_output, self.response) + self.validate_expected_shape(self.response, output_key=output_key) + self.inference_info.inference_request.validate(self.response) + + def validate_expected_shape(self, response, output_key=None): + output_shape = {name: list(data.shape) for name, data in response.items()} + expected_shape = self.inference_info.inference_request.get_expected_output_shape() + + validation_pass = True + for name, expected_data in expected_shape.items(): + for dim, expected_dim_value in enumerate(expected_data): + if expected_dim_value > 0: + if output_key is not None: + output_shape_key = output_key + else: + expected_shape_idx = list(expected_shape.keys()).index(name) + 1 + output_shape_key = f"out_{expected_shape_idx}" + validation_pass = expected_dim_value == output_shape[output_shape_key][dim] + + assert validation_pass, "Incorrect output shape, expected: {}, found: {}.".format(expected_shape, output_shape) + logger.debug(f"Output shape: {output_shape} (expected: {expected_shape})") + + +class LLMInferenceResponse(InferenceResponse): + + def __init__(self, inference_info, response): + super().__init__(inference_info, response) + + @classmethod + def create(cls, inference_info, response): + return LLMInferenceResponse(inference_info, response) + + def validate(self): + logger.info(self.response) + for choice in self.response["choices"]: + assert len(choice["message"]["content"]) > 1, f"Empty message: {choice}" + assert self.response["model"] == self.inference_info.model.name, f"Invalid model name: {self.response['model']}" + + +class InferenceInfo(object): + + @classmethod + def create(cls, client, model, timeout=wait_for_messages_timeout, input_data=None, inference_request=None): + if model.is_stateful: + return StatefulInferenceInfo(client, model, timeout, input_data, inference_request) + else: + return InferenceInfo(client, model, timeout, input_data, inference_request) + + def __init__( + self, client, model, timeout=wait_for_messages_timeout, input_data=None, inference_request=None + ): + self.client = client + self.input_data = input_data + self.model = model + self.timeout = timeout + self.inference_request = inference_request + + def clear_input_data(self): + for name in self.model.input_names: + data = self.input_data[name] + self.input_data[name] = np.zeros(data.shape, dtype=data.dtype) + + def predict(self): + request = self.inference_request.prepare_request_to_send(self.client, self.input_data) + result = self.client.predict(request, self.timeout) + return result + + def predict_sequence_step(self, tensor_data, sequence_ctrl, sequence_id): + request = self.client.prepare_stateful_request(tensor_data, sequence_ctrl, sequence_id) + return self.client.predict_stateful_request(request, self.timeout) + + def get_metadata(self): + meta = self.client.get_model_meta() + json_data = json.loads(MessageToJson(meta)) + return json_data + + def get_and_validate_metadata(self, expected_shape_dict): + metadata = self.get_metadata() + assert self.model.name == metadata["modelSpec"]["name"] + assert self.model.version == int(metadata["modelSpec"]["version"]) + + metadata_inputs = metadata["metadata"]["signature_def"]["signatureDef"]["serving_default"]["inputs"] + for in_name in self.model.inputs: + assert in_name in metadata_inputs + + for idx, expected_dim_value in enumerate(expected_shape_dict[in_name]): + dim_value = metadata_inputs[in_name]["tensorShape"]["dim"][idx]["size"] + if isinstance(expected_dim_value, str): + assert expected_dim_value == dim_value + else: + assert expected_dim_value == int(dim_value) + + +class StatefulInferenceInfo(InferenceInfo): + SEQUENCE_START = 1 + SEQUENCE_END = 2 + + def __init__(self, client, model, timeout=30, input_data=None, inference_request=None): + super().__init__(client, model, timeout, input_data, inference_request) + + def _get_sequence_control_data(self, data_length, iteration_index): + result = None + if iteration_index == 0: + result = sequence_ctrl = StatefulInferenceInfo.SEQUENCE_START + elif iteration_index == data_length + self.model.context_window_left + self.model.context_window_right - 1: + result = sequence_ctrl = StatefulInferenceInfo.SEQUENCE_END + return result + + def get_utterance_name_list(self): + input_param_name = self.model.input_names[0] + return list(self.input_data[input_param_name].keys()) + + def predict(self, sequence_id=None): + result = {} + for utterance in self.get_utterance_name_list(): + result[utterance] = self.predict_utterance(utterance, sequence_id) + return result + + def predict_utterance(self, utterance_name, sequence_id=None): + logger.info(f"Model ({self.model.name}) predict [{utterance_name}]") + result = [] + utterance_length = self.get_utterance_length(utterance_name) + offset = self.model.context_window_left + self.model.context_window_right + + for idx in range(utterance_length): + sequence_ctrl = self._get_sequence_control_data(utterance_length, idx) + tensor_data = self.get_utterance_data(utterance_name, idx) + sequence_id, output = self.predict_sequence_step(tensor_data, sequence_ctrl, sequence_id) + if idx >= offset: + result.append(output) # collect data for idx: = data_length: + data_idx = data_length - 1 # fill last data with tensor[-1] + result = {} + for name in self.model.input_names: + result[name] = self.input_data[name][utterance_name][data_idx] + return result + + def clear_input_data(self): + step = 10 + for name in self.model.input_names: + for utterance in self.input_data[name]: + for idx, data in enumerate(self.input_data[name][utterance]): + if idx % step == 0: + data = self.input_data[name][utterance][idx] + self.input_data[name][utterance][idx] = np.zeros(data.shape, dtype=data.dtype) + + +def prepare_requests( + inference_requests: List[InferenceRequest], timeout=wait_for_messages_timeout, random_data=False +): + inference_infos = [] + for inference_request in inference_requests: + port = inference_request.ovms.get_port(inference_request.api_type) + inference_client = inference_request.api_type(port=port, model=inference_request.model) + input_data = inference_client.create_client_data(inference_request) + inference_info = InferenceInfo.create( + inference_client, + inference_request.model, + input_data=input_data, + inference_request=inference_request, + timeout=timeout, + ) + inference_infos.append(inference_info) + return inference_infos + + +def predict_and_assert(inference_infos: List[InferenceInfo], validate_results=True, output_key=None): + for i, inference_info in enumerate(inference_infos): + logger.debug(f"Running predict request with {i} index for model '{inference_info.client.model_name}'") + outputs = inference_info.predict() + assert outputs, "Prediction returned no output" + if validate_results: + if inference_info.model.is_mediapipe: + if isinstance(inference_info.model, SimpleMediaPipe) and output_key is None: + output_key = "output" + else: + output_key = output_key + MediaPipeInferenceResponse.create(inference_info, outputs).validate( + inference_info.input_data, output_key=output_key + ) + elif inference_info.model.is_llm: + LLMInferenceResponse.create(inference_info, outputs).validate() + else: + InferenceResponse.create(inference_info, outputs).validate(inference_info.input_data) + logger.info("Predict finished.") + + +def predict_request(inference_request_list, parallel=False): + result = {} + if parallel: + + def execute_predict(infer_request): + infer_result = infer_request.predict() + return (infer_request.model.name, infer_result) + + arguments_list = [[x] for x in inference_request_list] + result_list = run_all_actions(execute_predict, arguments_list) + + for model_name, infer_result in result_list: + if model_name not in result: + result[model_name] = [] + result[model_name].append(infer_result) + else: + for i, inference_info in enumerate(inference_request_list): + logger.info(f"Running {i} predict request for model '{inference_info.client.model_name}'") + outputs = inference_info.predict() + assert outputs, "Prediction returned no output" + if inference_info.model.name not in result: + result[inference_info.model.name] = [] + result[inference_info.model.name].append(outputs) + return result + + +def ensure_predict(inference_info): + + return retry_call( + predict_and_assert, + fargs=[[inference_info]], + exceptions=(AssertionError, UnexpectedResponseError, RpcError), + tries=60, + delay=0.1, + ) + + +def healthy_check(models, port, api_type): + for model in models: + client = InferenceBuilder(model).create_client(api_type, port) + if not issubclass(api_type, KserveWrapper): + get_model_status(client, [Ovms.ModelStatus.AVAILABLE]) + + data = model.prepare_input_data() + request = client.prepare_request(input_objects=data) + outputs = client.predict(request) + model.validate_outputs(outputs) + + +def prepare_requests_and_run_predict( + inference_requests: List[InferenceRequest], repeat: int = 10, validate_results=True, timeout=30, random_data=False +): + inference_requests_total = inference_requests * repeat + inference_infos = prepare_requests(inference_requests_total, timeout=timeout, random_data=random_data) + predict_and_assert(inference_infos, validate_results=validate_results) + + +def prepare_and_run_set_of_predict_requests(ovms: OvmsInstance, models, api_type, number_of_infer_request_dict=None): + inference_request_list = [] + for model in models: + infer_requests_multiplier = 1 + if number_of_infer_request_dict is not None: + infer_requests_multiplier = number_of_infer_request_dict[model.name] + inference_request_list.extend( + prepare_requests([InferenceRequest(model=model, api_type=api_type, ovms=ovms)] * infer_requests_multiplier) + ) + + return predict_request(inference_request_list) + + +def validate_accuracy_for_stateful_models(models, results, accuracy_level=0.1): + for model in models: + error_report_dict_list = model.calculate_error(results[model.name]) + for error_report_dict in error_report_dict_list: + for utterance_name, error_result in error_report_dict.items(): + for output_name in model.output_names: + error_msg = ( + f"Detect unexpected error level for model: {model.name} (utternace: " + f"{utterance_name} output: {output_name})! Expected error level < " + f"{accuracy_level} (detected: {error_result[output_name]})" + ) + assert error_result[output_name] < accuracy_level, error_msg + logger.info(f"Validate accuracy for stateful models - PASSED") + + +def get_model_status(client, accepted_model_states=None, model_version=None, port=None): + model_state = None + port = port if port is not None else client.port + if client.serving == KFS: + if accepted_model_states is not None: + for elem in accepted_model_states: + is_ready = True if elem == Ovms.ModelStatus.AVAILABLE else False + try: + model_state = check_model_readiness(client.model, port, type(client), timeout=30, is_ready=is_ready) + except ModelNotReadyException: + logger.info(f"Model state not in accepted state: {elem}") + finally: + break + else: + raise ModelNotReadyException(f"Failed to check model: {client.model}") + else: + model_state = check_model_readiness(client.model, port, type(client)) + else: + status = client.get_model_status() + logger.debug(f"status: {status}") + if model_version is None: + model_state = Ovms.ModelStatus(status.model_version_status[0].state) + else: + for model_version_status in status.model_version_status: + if model_version_status.version == model_version: + model_state = Ovms.ModelStatus(model_version_status.state) + break + if accepted_model_states: + if model_state not in accepted_model_states: + model_str_name = client.model_name + raise ValueError(f"Incorrect state of {model_str_name}: {model_state}") + return model_state + + +def get_and_validate_model_status(inference, expected_models_status): + status = inference.get_model_status() + if expected_models_status is not None: + assert len(expected_models_status) == len(status.model_version_status) + + for i, model_version_status in enumerate(status.model_version_status): + model_state = model_version_status.state + error_message = model_version_status.status.error_message + version = model_version_status.version + + if expected_models_status is None or expected_models_status[i].get("accepted_states", None) is None: + model_accepted_states = [ + get_model_status_pb2.ModelVersionStatus.START, + get_model_status_pb2.ModelVersionStatus.AVAILABLE, + get_model_status_pb2.ModelVersionStatus.UNLOADING, + get_model_status_pb2.ModelVersionStatus.LOADING, + get_model_status_pb2.ModelVersionStatus.END, + ] + else: + model_accepted_states = expected_models_status[i]["accepted_states"] + + if model_state not in model_accepted_states: + raise ValueError(f"Incorrect model state: {model_state}") + + if expected_models_status is None or expected_models_status[i].get("accepted_error_messages", None) is None: + if model_state == get_model_status_pb2.ModelVersionStatus.LOADING: + model_accepted_error_messages = ["OK", "UNKNOWN"] + else: + model_accepted_error_messages = ["OK"] + else: + model_accepted_error_messages = expected_models_status[i]["accepted_error_messages"] + + if error_message not in model_accepted_error_messages: + raise ValueError(f"Incorrect error message: {model_state}") + + if expected_models_status is not None: + assert version == expected_models_status[i]["version"] + + return status + + +def get_multiple_model_status(models_and_expected_state): + for client, state in models_and_expected_state: + try: + model_state = get_model_status(client, accepted_model_states=[state]) + except (_InactiveRpcError, UnexpectedResponseError) as e: + if state in [Ovms.ModelStatus.UNKNOWN, Ovms.ModelStatus.UNDEFINED]: + pass # It is expected exceptions for given ModelStatus so proceed. + else: + assert False, f"Unexpected exception {e} for fetching given model state: {client.model_name}" + + +def wait_for_model_status(client, accepted_model_states, model_version=None, timeout=None): + skip_if_runtime(client.serving == KFS, FrameworkMessages.KFS_GET_MODEL_STATUS_NOT_SUPPORTED) + if not timeout: + timeout = client._model.get_ovms_loading_time() + + expected_exceptions = ( + UnexpectedResponseError, + ConnectionError, + _InactiveRpcError, + IndexError, + requests.exceptions.ConnectionError, + ) + last_exception = "" + received_status = None + end_timeout = time.time() + timeout + while time.time() <= end_timeout: + try: + model_status_response = client.get_model_status(version=model_version) + received_status = model_status_response.model_version_status[0].state + if Ovms.ModelStatus(received_status) in accepted_model_states: + break + except expected_exceptions as exception: + last_exception = str(exception) + else: + raise TimeoutError() + + time.sleep(1) + + assert received_status, f"Failed to obtain model version status: {last_exception}" + + model_status_str = ", ".join(map(str, accepted_model_states)) + msg = f"Unexpected model status, current: {received_status} expected: {model_status_str}" + assert Ovms.ModelStatus(received_status) in accepted_model_states, msg + + +def wait_for_model_meta(client, model, wait_time=1): + end_time = datetime.now() + timedelta(seconds=60) + validation_passed = False + received_meta = "" + logger.info(f"Waiting for receiving proper metadata for model {model.name}") + while datetime.now() < end_time: + try: + received_meta = client.get_model_meta() + client.validate_meta(model, received_meta) + validation_passed = True + if client.communication == REST: + response = received_meta.text + else: + response = MessageToJson(received_meta) + received_meta_str = json.loads(response) + logger.info(f"Expected metadata received for model {model.name}:\r\n{received_meta_str}") + break + except (RpcError, AssertionError) as ex: + time.sleep(wait_time) + + assert validation_passed, f"Unexpected model metadata, current: {received_meta} for model: {model}" + + +def prepare_v2_grpc_stub(port): + final_server_address = TestEnvironment.get_server_address() + url = f"{final_server_address}:{port}" + channel = grpc.insecure_channel(url, options=channel_options) + grpc_stub = service_pb2_grpc.GRPCInferenceServiceStub(channel) + + return grpc_stub + + +def prepare_v2_model_infer_request(port, api_type, input_data=None): + if api_type.communication == GRPC: + grpc_stub = prepare_v2_grpc_stub(port) + request = api_type.get_predict_grpc_request(input_data) + return request, grpc_stub + else: + raise NotImplementedError() + + +def check_model_readiness(model, port, kfs_api_type, is_ready=True, timeout=None): + if timeout is None: + timeout = model.get_ovms_loading_time() + end_time = datetime.now() + timedelta(seconds=timeout) + last_exception = "" + success = False + kfs_api = kfs_api_type(model=model, port=port) + while datetime.now() < end_time: + try: + response = kfs_api.is_model_ready(model.name, model.version) + if response and is_ready: + logger.info(f"Model {model.name} Ready:\n{response}") + success = True + break + elif not response and not is_ready: + logger.info(f"Model {model.name} is not Ready: {response}") + success = True + break + except (UnexpectedResponseError, _InactiveRpcError, InferenceServerException) as e: + last_exception = str(e) + if not is_ready and kfs_api.type == REST and (e.status == 404 or e.status == 503): + success = True + response = None + break + + time.sleep(0.5) + + assert success, ModelNotReadyException(f"Failed to check model: {last_exception}") + return response + + +def execute_kfs_live_ready(model, api_type, port, model_name=None, model_version=None): + model_name = model_name if model_name is not None else model.name + model_version = model_version if model_version is not None else model.version + kfs_client = api_type(model=model, port=port) + assert kfs_client.is_server_ready(), "Server is not ready" + assert kfs_client.is_model_ready(model_name, model_version), f"Model {model_name} is not ready" + + +def prepare_mediapipe_requests(ovms, model, api_type, port, input_key): + request = InferenceRequest(ovms=ovms, model=model, api_type=api_type) + inference_client = api_type(port=port, model=model) + input_data = model.prepare_input_data(input_key=input_key) + inference_info = InferenceInfo.create( + inference_client, request.model, input_data=input_data, inference_request=request + ) + return [inference_info] + + +def run_mediapipe_inference( + ovms, + model, + api_type, + port, + input_key=MediaPipeConstants.DEFAULT_INPUT_STREAM, + output_key=MediaPipeConstants.DEFAULT_OUTPUT_STREAM, +): + logger.info(f"Run inference for {model.name}") + inference_infos = prepare_mediapipe_requests(ovms, model, api_type, port, input_key) + predict_and_assert(inference_infos, output_key=output_key) + + +def prepare_mediapipe_binary_requests(ovms, model, api_type, input_key, dataset=None): + dataset = DefaultBinaryDataset(image_format=Ovms.JPG_IMAGE_FORMAT, offset=0) if dataset is None else dataset + request = BinaryInferenceRequest(ovms=ovms, model=model, api_type=api_type, dataset=dataset) + + inference_infos = prepare_requests([request]) + for inference_info in inference_infos: + for key in inference_info.input_data.keys(): + inference_info.input_data[input_key] = inference_info.input_data[key] + del inference_info.input_data[key] + break + return inference_infos + + +def run_streaming_inference(model, api_type, port): + if api_type.type == GRPC: + client = api_type(model=model, port=port) + prompts = [] + for i in range(len(model.inputs)): + prompts.extend(LanguageModelDataset(i).get_str_input_data()) + batch_size = model.batch_size + if batch_size is not None and batch_size > 1: + # Generate various random text + prompts_data = [ + prompts if i == 0 else LanguageModelDataset.generate_random_text_list(model.inputs_number) + for i in range(batch_size) + ] + provided_input = prompts_data + else: + prompts_data = [prompts] + provided_input = prompts + + logger.debug(f"Provided input: {provided_input}") + results = streaming_api_inference_language_models(model, client, prompts=prompts_data) + model.validate_outputs(outputs=results, provided_input=provided_input) + else: + raise NotImplementedError + + +def log_request_info(request_params, model_name, prompt): + logger.info(f"Run request with parameters: '{request_params}' for model: '{model_name}' with prompt: '{prompt}'.") + + +def validate_ttr(data, reference=0.5): + # Calculate Type-Token Ratio (TTR) + assert data != "", f"Empty data: {data}. Please verify model's response." + words = data.split() + unique_words = set(words) + ttr = len(unique_words) / len(words) + assert ttr > reference, f"Output TTR is too low: {ttr}" + + +def run_llm_inference( + model: Union[ModelInfo, List[ModelInfo]], + api_type, + port, + endpoint, + dataset=None, + input_data_type=None, + validate_outputs=True, + validate_outputs_ttr=True, + allow_empty_response=False, + timeout=None, + log_request=True, + ttr_reference=0.5, + **kwargs, +): + if api_type.type == REST: + api_client = api_type(model=model, port=port) + infer_request = kwargs.get("infer_request", None) + request_parameters = kwargs.get("request_parameters", {}) + if isinstance(model, list): + assert endpoint == OpenAIWrapper.MODELS_LIST, f"model parameter cannot be list when used with {endpoint}" + input_objects = None + model_name = None + else: + assert endpoint != OpenAIWrapper.MODELS_LIST, f"model parameter has to be list when used with {endpoint}" + input_objects = model.prepare_input_data(dataset=dataset, input_data_type=input_data_type) + model_base_path = kwargs.get("model_base_path", None) + model_name = model_base_path if model_base_path is not None else model.name + if endpoint != CohereWrapper.RERANK: + infer_request = infer_request if infer_request is not None else \ + LLMInferenceRequest(api_type=api_client, request_parameters=request_parameters) + else: + infer_request = infer_request if infer_request is not None else \ + RerankLLMInferenceRequest(api_type=api_client, request_parameters=request_parameters) + outputs = None + if endpoint == OpenAIWrapper.CHAT_COMPLETIONS: + messages = ChatCompletionsApi.prepare_chat_completions_input_content(input_objects) + if log_request: + log_request_info(request_parameters, model_name, messages) + raw_outputs = infer_request.create_chat_completions(messages, model_name=model_name, timeout=timeout) + raw_outputs = list(raw_outputs) if infer_request.stream and raw_outputs else raw_outputs + if validate_outputs: + outputs = GenerativeAIValidationUtils.validate_chat_completions_outputs( + model_name=model_name, + outputs=raw_outputs, + stream=infer_request.stream, + allow_empty_response=allow_empty_response, + ) + if validate_outputs_ttr: + validate_ttr(outputs[0], reference=ttr_reference) + elif endpoint == OpenAIWrapper.COMPLETIONS: + prompt = CompletionsApi.prepare_completions_input_content(input_objects) + if log_request: + log_request_info(request_parameters, model_name, prompt) + raw_outputs = infer_request.create_completions(prompt, model_name=model_name, timeout=timeout) + raw_outputs = list(raw_outputs) if infer_request.stream and raw_outputs else raw_outputs + if validate_outputs: + outputs = GenerativeAIValidationUtils.validate_completions_outputs( + model_name=model_name, + outputs=raw_outputs, + stream=infer_request.stream, + allow_empty_response=allow_empty_response, + model_instance=model, + ) + if validate_outputs_ttr: + validate_ttr(outputs[0], reference=ttr_reference) + elif endpoint == OpenAIWrapper.RESPONSES: + input_content = ResponsesApi.prepare_responses_input_content(input_objects) + if log_request: + log_request_info(request_parameters, model_name, input_content) + raw_outputs = infer_request.create_responses(input_content, model_name=model_name, timeout=timeout) + raw_outputs = list(raw_outputs) if infer_request.stream and raw_outputs else raw_outputs + if validate_outputs: + outputs = GenerativeAIValidationUtils.validate_responses_outputs( + model_name=model_name, + outputs=raw_outputs, + stream=infer_request.stream, + allow_empty_response=allow_empty_response, + ) + if validate_outputs_ttr: + validate_ttr(outputs[0], reference=ttr_reference) + elif endpoint == OpenAIWrapper.EMBEDDINGS: + embeddings_input = EmbeddingsApi.prepare_embeddings_input_content(input_objects) + if log_request: + log_request_info(request_parameters, model_name, embeddings_input) + raw_outputs = infer_request.create_embeddings(embeddings_input, model_name=model_name, timeout=timeout) + if validate_outputs: + outputs = GenerativeAIValidationUtils.validate_embeddings_outputs( + model_name=model_name, + outputs=raw_outputs, + allow_empty_response=allow_empty_response, + ) + elif endpoint == CohereWrapper.RERANK: + rerank_input = RerankApi.prepare_rerank_input_content(input_objects) + if log_request: + log_request_info(request_parameters, model_name, rerank_input) + raw_outputs = infer_request.create_rerank(rerank_input, model_name=model_name) + if validate_outputs: + outputs = GenerativeAIValidationUtils.validate_rerank_outputs( + model_name=model_name, + outputs=raw_outputs, + allow_empty_response=allow_empty_response, + ) + elif endpoint == OpenAIWrapper.IMAGES_GENERATIONS: + prompt = ImagesApi.prepare_image_generation_input_content(input_objects) + if log_request: + log_request_info(request_parameters, model_name, prompt) + raw_outputs = infer_request.create_image_generation(prompt, model_name=model_name, timeout=timeout) + if validate_outputs: + outputs = GenerativeAIValidationUtils.validate_image_outputs( + model_name=model_name, + outputs=raw_outputs, + image_path=kwargs.get("image_path", None), + request_parameters=request_parameters, + ) + elif endpoint == OpenAIWrapper.IMAGES_EDITS: + prompt, image_path = ImagesApi.prepare_image_edit_input_content(input_objects) + mask_path = dataset.mask_path if hasattr(dataset, "mask_path") else None + if log_request: + message = f"Run request with parameters: '{request_parameters}' for model: '{model_name}' " \ + f"with prompt: '{prompt}', image_path: '{image_path}'" + if mask_path is not None: + message += f", mask_path: '{mask_path}'" + logger.info(message) + raw_outputs = infer_request.create_image_edit( + prompt, image_path, mask_path=mask_path, model_name=model_name, timeout=timeout) + if validate_outputs: + outputs = GenerativeAIValidationUtils.validate_image_outputs( + model_name=model_name, + outputs=raw_outputs, + image_path=kwargs.get("image_path", None), + request_parameters=request_parameters, + ) + elif endpoint == OpenAIWrapper.MODELS_LIST: + raw_outputs = infer_request.create_models_list() + if validate_outputs: + outputs = GenerativeAIValidationUtils.validate_models_list_outputs(models=model, outputs=raw_outputs) + elif endpoint == OpenAIWrapper.MODELS_RETRIEVE: + raw_outputs = infer_request.create_models_retrieve() + if validate_outputs: + outputs = GenerativeAIValidationUtils.validate_models_retrieve_outputs(model_name=model_name, outputs=raw_outputs) + else: + raise NotImplementedError + else: + raise NotImplementedError + return outputs, raw_outputs + + +def run_audio_inference( + model: ModelInfo, + api_type, + port, + endpoint, + dataset=None, + validate_outputs=True, + validate_output_wer=False, + allow_empty_response=False, + timeout=None, + log_request=True, + wer_threshold=0.4, + **kwargs, +): + if api_type.type == REST: + api_client = api_type(model=model, port=port) + request_parameters = kwargs.get("request_parameters", {}) + infer_request = kwargs.get("infer_request") or LLMInferenceRequest( + api_type=api_client, request_parameters=request_parameters + ) + + model_name = kwargs.get("model_base_path") or model.name + reference_text, reference_audio_file = AudioApi.prepare_audio_input_content( + model.prepare_input_data(dataset=dataset) + ) + outputs = None + raw_outputs = None + + if endpoint == OpenAIWrapper.AUDIO_SPEECH: + speech_file_path = kwargs.get("speech_file_path") + assert speech_file_path is not None, "speech_file_path is required for audio speech endpoint" + if log_request: + log_request_info(request_parameters, model_name, reference_text) + raw_outputs = infer_request.create_audio_speech( + reference_text, speech_file_path, model_name=model_name, timeout=timeout + ) + if validate_outputs: + outputs = GenerativeAIValidationUtils.validate_audio_speech_outputs( + speech_file_path=speech_file_path, + allow_empty_response=allow_empty_response, + ) + elif endpoint in OpenAIWrapper.AVAILABLE_AUDIO_ASR_ENDPOINTS: + if log_request: + log_request_info(request_parameters, model_name, reference_audio_file) + create_fn = ( + infer_request.create_audio_transcription + if endpoint == OpenAIWrapper.AUDIO_TRANSCRIPTIONS + else infer_request.create_audio_translation + ) + raw_outputs = create_fn(reference_audio_file, model_name=model_name, timeout=timeout) + if validate_outputs: + outputs = GenerativeAIValidationUtils.validate_audio_asr_outputs( + outputs=raw_outputs, + allow_empty_response=allow_empty_response, + ) + if validate_output_wer and reference_text: + GenerativeAIValidationUtils.validate_wer(reference_text, raw_outputs, threshold=wer_threshold) + else: + raise NotImplementedError + + return outputs, raw_outputs + + raise NotImplementedError + +def run_llm_inference_and_validate_against_reference( + model, + api_type, + port, + endpoint, + request_parameters, + reference_outputs, +): + outputs = run_llm_inference( + model, + api_type, + port, + endpoint, + request_parameters=request_parameters, + ) + assert ( + outputs == reference_outputs + ), f"Output messages:\n'{outputs}'\ndo not match reference:\n'{reference_outputs}'" + + +def streaming_api_inference_language_models(mediapipe_model, kfs_client, prompts): + # Fetched from: https://github.com/openvinotoolkit/model_server/blob/main/demos/python_demos/llm_text_generation/client_stream.py + results_decoded = [] + event, client, results = prepare_streaming_api_inference(mediapipe_model, kfs_client, prompts, results_decoded) + infer_inputs = streaming_api_inference_prepare_infer_inputs(mediapipe_model, prompts) + client.async_stream_infer(model_name=mediapipe_model.name, inputs=infer_inputs) + event.wait(timeout=10) + client.stop_stream() + logger.info("Stream stopped") + return results + + +def prepare_streaming_api_inference(mediapipe_model, kfs_client, prompts, results_decoded): + + def callback(result, error): + + def decode_result(result, output_name, results_decoded): + logger.debug(f"Result as numpy for output_name {output_name}: {result.as_numpy(output_name)}") + if len(prompts) == 1: + results_decoded.append(result.as_numpy(output_name).tobytes().decode()) + else: + deserialized_results = deserialize_bytes_tensor(result._result.raw_output_contents[0]) + decoded_list = [content.decode() for content in deserialized_results] + results_decoded.append(decoded_list) + logger.debug(f"Results decoded for output_name {output_name}: {results_decoded}") + return results_decoded + + if error: + raise error + elif any(result.as_numpy(output_name) is not None for output_name in mediapipe_model.output_names): + for output_name in mediapipe_model.output_names: + if result.as_numpy(output_name) is not None: + decode_result(result, output_name, results_decoded) + break + else: + raise StreamingApiException(f"Unexpected output: {result}") + + client = tritonclient.grpc.InferenceServerClient(kfs_client.url, channel_args=STREAMING_CHANNEL_ARGS, verbose=True) + + event = Event() + client.start_stream(callback=callback) + return event, client, results_decoded + + +def streaming_api_inference_prepare_infer_inputs(mediapipe_model, prompts): + _infer_inputs = [] + for i, _input in enumerate(mediapipe_model.inputs.keys()): + infer_input = tritonclient.grpc.InferInput(mediapipe_model.input_names[i], [len(prompts)], "BYTES") + if len(prompts) == 1: + if isinstance(prompts[0], list): + infer_input._raw_content = prompts[0][i].encode() + else: + infer_input._raw_content = prompts[0].encode() + else: + if mediapipe_model.inputs_number == 1: + infer_input._raw_content = serialize_byte_tensor(np.array(prompts, dtype=np.object_)).item() + else: + infer_input._raw_content = serialize_byte_tensor(np.array(prompts[i], dtype=np.object_)).item() + _infer_inputs.append(infer_input) + + logger.debug(f"Infer inputs raw content: {[inp._raw_content for inp in _infer_inputs]}") + return _infer_inputs diff --git a/tests/functional/object_model/mediapipe_calculators.py b/tests/functional/object_model/mediapipe_calculators.py new file mode 100644 index 0000000000..d318b31b1a --- /dev/null +++ b/tests/functional/object_model/mediapipe_calculators.py @@ -0,0 +1,986 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import os +import shutil +from dataclasses import dataclass, field +from pathlib import Path +from typing import List + +from tests.functional.utils.assertions import InvalidReturnCodeException +from tests.functional.utils.logger import get_logger +from tests.functional.utils.process import Process + +from tests.functional.constants.generative_ai import GenerativeAIPluginConfig +from tests.functional.config import ( + enable_prefix_caching_config, + kv_cache_size_value, + kv_cache_precision_value, + max_num_batched_tokens, + pipeline_type as config_pipeline_type, + mediapipe_repo_branch, + ovms_c_repo_path, +) +from tests.functional.models import ModelInfo +from tests.functional.constants.target_device import TargetDevice +from tests.functional.constants.ovms import Config, MediaPipeConstants +from tests.functional.constants.paths import Paths +from tests.functional.object_model.test_environment import TestEnvironment + +logger = get_logger(__name__) + +dummy_mediapipe_calculators = [os.path.join(ovms_c_repo_path, "src/test/mediapipe/graphdummyadapterfull.pbtxt")] + + +@dataclass +class MediaPipeCalculator: + src_type: str = "cc" + src_dir: str = None + src_file_path: str = None + src_filename: str = None + name: str = None + + @classmethod + def prepare_proto_calculator(cls, parameters, config_path_on_host, config_file=None): + # Create new graph files content from MediaPipeCalculator. + + calculators = [] + mediapipe_models = [model for model in parameters.models if model.is_mediapipe] + config_data = ( + parameters.custom_config + if parameters.custom_config is not None + else json.loads(Path(config_file).read_text()) if config_file is not None else {} + ) + for mediapipe_model in mediapipe_models: + dst_path = os.path.join(config_path_on_host, mediapipe_model.name) if config_path_on_host is not None \ + else os.path.join(TestEnvironment.current.base_dir, parameters.name, Paths.MODELS_PATH_NAME, + mediapipe_model.name) + # Scenario 1. With any graph path + if parameters.use_custom_graphs and parameters.custom_graph_paths: + # Copy existing graphs from custom_graph_paths. + for calc in parameters.custom_graph_paths: + calculators.append(calc) + real_path = os.path.expanduser(calc) + real_path = os.path.realpath(real_path) + logger.info( + "Copy custom calculator file to {}, content:\n{}".format(dst_path, Path(real_path).read_text()) + ) + Path(dst_path).mkdir(parents=True, exist_ok=True) + shutil.copy(real_path, dst_path) + # Scenario 2. With mediapipe pipelines and python custom nodes + elif mediapipe_model.graphs: + calculator_class = PythonCalculator if mediapipe_model.is_python_custom_node else cls + model = mediapipe_model if mediapipe_model.is_python_custom_node else None + for graph in mediapipe_model.graphs: + calculators.append(graph) + mediapipe_model_graph_paths = [ + elem.get("graph_path", "") + for elem in config_data[Config.MEDIAPIPE_CONFIG_LIST] + if elem["name"] == mediapipe_model.name + ] + for graph_path in mediapipe_model_graph_paths: + calculator_class.save( + model=model, + content=graph, + dst_path=dst_path, + filename=os.path.basename(graph_path), + save_only=True, + ) + # Scenario 3. With SimpleModelMediapipe + else: + # Get graph paths from mediapipe_model + if ((not config_data and mediapipe_model.single_mediapipe_model_mode) + or mediapipe_model.pbtxt_name is not None): + mediapipe_model_graph_paths = [f"{mediapipe_model.pbtxt_name}.pbtxt"] + elif not Config.MEDIAPIPE_CONFIG_LIST in config_data: + mediapipe_model_graph_paths = [ + elem["config"].get("graph_path", "") + for elem in config_data[Config.MODEL_CONFIG_LIST] + if elem["config"]["name"] == mediapipe_model.name + ] + else: + mediapipe_model_graph_paths = [ + elem.get("graph_path", "") + for elem in config_data[Config.MEDIAPIPE_CONFIG_LIST] + if elem["name"] == mediapipe_model.name + ] + + contents = {} + for regular_model in mediapipe_model.regular_models: + for calc in mediapipe_model.calculators: + calculators.append(calc) + model = calc.model if calc.model is not None else regular_model + contents.update({calc.name: calc.create_proto_content(model=model)}) + + content_to_save = " ".join(value for key, value in contents.items()) + if all(["pbtxt" in elem for elem in mediapipe_model_graph_paths]): + for path in mediapipe_model_graph_paths: + filename = os.path.basename(path) + cls.save(mediapipe_model, content_to_save, dst_path=dst_path, filename=filename) + else: + filename = Paths.GRAPH_NAME + cls.save(mediapipe_model, content_to_save, dst_path=dst_path, filename=filename) + + mediapipe_model.graphs = [content_to_save] + + return calculators + + @classmethod + def create_proto_header( + cls, + model, + input_stream=MediaPipeConstants.DEFAULT_INPUT_STREAM, + output_stream=MediaPipeConstants.DEFAULT_OUTPUT_STREAM, + ): + input_streams = "" + output_streams = "" + if model is not None: + for i, model_input in enumerate(model.inputs, start=0): + input_streams += f'input_stream: "{input_stream}_{i}" \n' + + for i, model_output in enumerate(model.outputs, start=0): + output_streams += f'output_stream: "{output_stream}_{i}" \n' + else: + if isinstance(input_stream, List): + for inp in input_stream: + input_streams += f'input_stream: "{inp}" \n' + if isinstance(output_stream, List): + for out in output_stream: + output_streams += f'output_stream: "{out}" \n' + return f"{input_streams}\n{output_streams}\n" + + def create_input_output_streams(self, model, input_stream, output_stream): + # Note: Key name must be capitalized: + # https://github.com/google/mediapipe/blob/master/mediapipe/framework/tool/validate_name.cc#L35 + + input_streams = [] + output_streams = [] + if isinstance(input_stream, List): + for elem in input_stream: + input_streams.append(elem.split(":")[-1]) + else: + input_streams = [input_stream.split(":")[-1] for i in range(len(model.inputs))] + + if isinstance(output_stream, List): + for elem in output_stream: + output_streams.append(elem.split(":")[-1]) + else: + output_streams = [output_stream.split(":")[-1] for i in range(len(model.outputs))] + + inputs = "" + model_name = self.get_upper_model_name(model) + for (i, model_input), inp_stream in zip(enumerate(model.inputs, start=0), input_streams): + inp = f"{model_name}_INPUT_{i}" + inputs += ( + f'input_stream: "{inp}:{inp_stream}_{i}" \n' + if inp_stream == MediaPipeConstants.DEFAULT_INPUT_STREAM + else f'input_stream: "{inp}:{inp_stream}" \n' + ) + + outputs = "" + for (i, model_output), out_stream in zip(enumerate(model.outputs, start=0), output_streams): + out = f"{model_name}_OUTPUT_{i}" + outputs += ( + f'output_stream: "{out}:{out_stream}_{i}" \n' + if out_stream == MediaPipeConstants.DEFAULT_OUTPUT_STREAM + else f'output_stream: "{out}:{out_stream}" \n' + ) + + return inputs, outputs + + @classmethod + def get_full_content(cls, content, model, input_stream, output_stream): + header = cls.create_proto_header(model, input_stream, output_stream) + content = header + content + return content + + @classmethod + def save( + cls, + model, + content, + dst_path, + filename=Paths.GRAPH_NAME, + input_stream=MediaPipeConstants.DEFAULT_INPUT_STREAM, + output_stream=MediaPipeConstants.DEFAULT_OUTPUT_STREAM, + save_only=False, + ): + """Save .pbtxt file to config_path_on_host location with given filename""" + if not save_only: + content = cls.get_full_content(content, model, input_stream, output_stream) + Path(dst_path).mkdir(parents=True, exist_ok=True) + file_path = os.path.join(dst_path, filename) + with open(file_path, "w+") as f: + f.write(content) + logger.info(f"Saving calculator file to {file_path}, content:\n{content}") + return file_path + + @staticmethod + def load(filepath): + with open(filepath, "r") as f: + data = f.read() + return data + + @classmethod + def get_upper_model_name(cls, model): + return cls.get_valid_model_name(model).upper() + + @staticmethod + def get_valid_model_name(model): + model_name = ("model_" + model.name) if model.name[0].isdigit() else model.name + return model_name.replace("-", "_").lower() + + +@dataclass +class OvmsCMediaPipeCalculator(MediaPipeCalculator): + + def __post_init__(self): + self.src_dir = ( + f"https://github.com/openvinotoolkit/mediapipe/blob/{mediapipe_repo_branch}/mediapipe/calculators/ovms" + ) + if self.src_filename is not None: + self.src_file_path = os.path.join(self.src_dir, f"{self.src_filename}.{self.src_type}") + + def create_input_output_tensor_names(self, model): + # Note: Key name must be capitalized: + # https://github.com/google/mediapipe/blob/master/mediapipe/framework/tool/validate_name.cc#L35 + + model_inputs_keys = list(model.inputs.keys()) + model_outputs_keys = list(model.outputs.keys()) + + input_tensors = "" + model_name = self.get_upper_model_name(model) + for i, model_input in enumerate(model_inputs_keys, start=0): + inp = f"{model_name}_INPUT_{i}" + input_tensor_names = f'tag_to_input_tensor_names {{key: "{inp}" value: "{model_input}"}}' + input_tensors += f"{input_tensor_names} \n" + + output_tensors = "" + for i, model_output in enumerate(model_outputs_keys, start=0): + out = f"{model_name}_OUTPUT_{i}" + output_tensor_names = f'tag_to_output_tensor_names {{key: "{out}" value: "{model_output}"}}' + output_tensors += f"{output_tensor_names} \n" + + return input_tensors, output_tensors + + +@dataclass +class OvmsCUnitTestMediaPipeCalculator(MediaPipeCalculator): + + def __post_init__(self): + self.src_dir = os.path.join(ovms_c_repo_path, "src", "test", "mediapipe") + self.src_file_path = os.path.join(self.src_dir, f"{self.src_filename}.{self.src_type}") + + +@dataclass +class OVMSOVCalculator(OvmsCMediaPipeCalculator): + name: str = "OVMSOVCalculator" + src_filename: str = "ovms_calculator" + model: ModelInfo = None + src_dir: str = os.path.join(ovms_c_repo_path, "src", "mediapipe_calculators") + + def create_proto_content( + self, + model, + input_stream=MediaPipeConstants.DEFAULT_INPUT_STREAM, + output_stream=MediaPipeConstants.DEFAULT_OUTPUT_STREAM, + create_header=True, + ): + model = self.model if self.model is not None else model + input_streams, output_streams = self.create_input_output_streams(model, input_stream, output_stream) + input_tensor_names, output_tensor_names = self.create_input_output_tensor_names(model) + + content = ( + "node {\n" + f'calculator: "{self.name}"\n' + f"{input_streams}\n" + f"{output_streams}\n" + "node_options: {\n" + "[type.googleapis.com / mediapipe.OVMSCalculatorOptions]: {\n" + f'servable_name: "{model.name}"\n' + f'servable_version: "{model.version}"\n' + f"{input_tensor_names}\n" + f"{output_tensor_names}" + "}}}" + ) + + return content + + +@dataclass +class OpenVINOInferenceCalculator(OvmsCMediaPipeCalculator): + name: str = "OpenVINOInferenceCalculator" + src_filename: str = "openvinoinferencecalculator" + model: ModelInfo = None + session: str = None + input_stream: str = None + output_stream: str = None + + def create_proto_content( + self, + model, + input_stream=MediaPipeConstants.DEFAULT_INPUT_STREAM, + output_stream=MediaPipeConstants.DEFAULT_OUTPUT_STREAM, + create_header=True, + ): + model = self.model if self.model is not None else model + input_stream = self.input_stream if self.input_stream is not None else input_stream + output_stream = self.output_stream if self.output_stream is not None else output_stream + input_streams, output_streams = self.create_input_output_streams(model, input_stream, output_stream) + input_tensor_names, output_tensor_names = self.create_input_output_tensor_names(model) + session = MediaPipeCalculator.get_valid_model_name(model) if self.session is None else self.session + + content = ( + "node: {\n" + f'calculator: "{self.name}"\n' + f'input_side_packet: "SESSION:{session}"\n' + f"{input_streams}\n" + f"{output_streams}\n" + "node_options: {\n" + "[type.googleapis.com / mediapipe.OpenVINOInferenceCalculatorOptions]: {\n" + f"{input_tensor_names}\n" + f"{output_tensor_names}" + "}}}" + ) + + return content + + +@dataclass +class OpenVINOModelServerSessionCalculator(OvmsCMediaPipeCalculator): + name: str = "OpenVINOModelServerSessionCalculator" + src_filename: str = "openvinomodelserversessioncalculator" + model: ModelInfo = None + session: str = None + model_name: str = None + + def create_proto_content(self, model, input_stream=None, output_stream=None, create_header=True): + model = self.model if self.model is not None else model + model_name = self.model_name if self.model_name is not None else model.name + model.name = model_name + session = MediaPipeCalculator.get_valid_model_name(model) if self.session is None else self.session + content = ( + "node: {\n" + f'calculator: "{self.name}"\n' + f'output_side_packet: "SESSION:{session}" \n' + "node_options: {\n" + "[type.googleapis.com / mediapipe.OpenVINOModelServerSessionCalculatorOptions]: {\n" + f'servable_name: "{model_name}"\n' + f'servable_version: "{model.version}"\n' + "}}}" + ) + + return content + + +@dataclass +class InputSidePacketUserTestCalc(OvmsCUnitTestMediaPipeCalculator): + name: str = "InputSidePacketUserTestCalc" + src_filename: str = "inputsidepacketusertestcalc" + + +@dataclass +class OVMSTestKFSPassCalculator(OvmsCUnitTestMediaPipeCalculator): + name: str = "OVMSTestKFSPassCalculator" + src_filename: str = "ovms_kfs_calculator" + + +@dataclass +class CorruptedFileCalculator(OVMSOVCalculator): + name: str = "CorruptedFileCalculator" + src_filename: str = None + model: ModelInfo = None + + def create_proto_content( + self, + model, + input_stream=MediaPipeConstants.DEFAULT_INPUT_STREAM, + output_stream=MediaPipeConstants.DEFAULT_OUTPUT_STREAM, + create_header=True, + ): + model = self.model if self.model is not None else model + ovms_ov_content = super(CorruptedFileCalculator, self).create_proto_content(model) + content = ovms_ov_content.replace("input_stream", self.name) + return content + + +@dataclass +class PythonCalculator(MediaPipeCalculator): + name: str = "PythonExecutorCalculator" + src_filename: str = None + input_side_packet: str = "PYTHON_NODE_RESOURCES:py" + model: ModelInfo = None + handler_path: str = None + handler_path_internal: str = "/models/python_model/model.py" + input_streams: str = None + output_streams: str = None + node_name: str = None + loopback: bool = False + + def __post_init__(self): + if self.model is not None: + self.input_streams = "\n".join( + f'input_stream: "PYTHON_MODEL_INPUT_{i}:{model_input}"' + for i, model_input in enumerate(self.model.input_names) + ) + self.output_streams = "\n".join( + f'output_stream: "PYTHON_MODEL_OUTPUT_{i}:{model_output}"' + for i, model_output in enumerate(self.model.output_names) + ) + + def create_proto_content(self, model, input_stream=None, output_stream=None, create_header=True): + model = self.model if self.model is not None else model + if input_stream is not None and output_stream is not None: + # Scenario, when input_stream and output_stream have unique values, e.g. PythonCustomNodeChainMediaPipe + input_streams, output_streams = self.create_input_output_streams(model, input_stream, output_stream) + else: + # Default scenario, when self.input_streams are defined in __post_init__ + input_streams = self.input_streams + output_streams = self.output_streams + + header = ( + self.create_proto_header(model) if create_header else "" + ) # If create_header=False, header will be create in graph_refresh() + if not self.loopback: + content = f"""{header} +node: {{ + name: "{self.node_name}" + calculator: "{self.name}" + input_side_packet: "{self.input_side_packet}" + {input_streams} + {output_streams} + node_options: {{ + [type.googleapis.com / mediapipe.PythonExecutorCalculatorOptions]: {{ + handler_path: "{self.handler_path_internal}" + }} + }} +}}""" + else: + content = f"""{header} +node: {{ + name: "{self.node_name}" + calculator: "{self.name}" + input_side_packet: "{self.input_side_packet}" + {input_streams} + input_stream: "LOOPBACK:loopback" + input_stream_info: {{ + tag_index: "LOOPBACK:0", + back_edge: true + }} + input_stream_handler {{ + input_stream_handler: "SyncSetInputStreamHandler", + options {{ + [mediapipe.SyncSetInputStreamHandlerOptions.ext] {{ + sync_set {{ + tag_index: "LOOPBACK:0" + }} + }} + }} + }} + {output_streams} + output_stream: "LOOPBACK:loopback" + node_options: {{ + [type.googleapis.com / mediapipe.PythonExecutorCalculatorOptions]: {{ + handler_path: "{self.handler_path_internal}" + }} + }} +}}""" + return content + + def prepare_resources(self, base_location): + dst = Path(base_location, f"./{self.handler_path_internal}") + dst.parent.mkdir(exist_ok=True, parents=True) + shutil.copy(self.handler_path, dst) + return str(dst) + + @classmethod + def create_proto_header(cls, model, input_stream="text_input", output_stream="text_output"): + input_streams = "" + output_streams = "" + if model is not None: + input_streams = "\n".join( + f'input_stream: "OVMS_PY_TENSOR{i}:{model_input}"' for i, model_input in enumerate(model.input_names) + ) + output_streams = "\n".join( + f'output_stream: "OVMS_PY_TENSOR{i}:{model_output}"' + for i, model_output in enumerate(model.output_names) + ) + else: + if isinstance(input_stream, List): + for i, inp in enumerate(input_stream): + input_streams += f'input_stream: "OVMS_PY_TENSOR{i}:{inp}" \n' + if isinstance(output_stream, List): + for i, out in enumerate(output_stream): + output_streams += f'output_stream: "OVMS_PY_TENSOR{i}:{out}" \n' + return f"{input_streams}\n{output_streams}\n" + + +@dataclass +class LLMCalculator(PythonCalculator): + name: str = "LLMExecutor" + src_filename: str = "llm_calculator" + input_side_packet: str = "LLM_NODE_RESOURCES:llm" + model: ModelInfo = None + models_path: str = None + models_path_internal: str = "./" + input_streams: str = None + output_streams: str = None + node_name: str = None + loopback: bool = False + kv_cache_size: int = kv_cache_size_value + plugin_config: dict = field(default_factory={GenerativeAIPluginConfig.KV_CACHE_PRECISION: kv_cache_precision_value}) + best_of_limit: int = None + max_tokens_limit: int = None + device: str = None + enable_tool_guided_generation: bool = False + + def __post_init__(self): + self.input_streams = 'input_stream: "REQUEST:in"' + self.output_streams = 'output_stream: "RESPONSE:out"' + self.src_dir = os.path.join(ovms_c_repo_path, "src", "llm") + self.src_file_path = os.path.join(self.src_dir, f"{self.src_filename}.{self.src_type}") + + def create_node_content(self, header, input_streams, output_streams): + best_of_limit_str = f"\nbest_of_limit: {self.best_of_limit}," if self.best_of_limit is not None else "" + max_tokens_limit_str = ( + f"\nmax_tokens_limit: {self.max_tokens_limit}," if self.max_tokens_limit is not None else "" + ) + device = f'device: "{self.device}",' if self.device is not None else "" + content = f"""{header} +node: {{ + name: "{self.node_name}" + calculator: "{self.name}" + input_side_packet: "{self.input_side_packet}" + {input_streams} + {output_streams} + node_options: {{ + [type.googleapis.com / mediapipe.LLMCalculatorOptions]: {{ + models_path: "{self.models_path_internal}",{best_of_limit_str}{max_tokens_limit_str} + cache_size: {self.kv_cache_size}, + {device} + }} + }} +}}""" + return content + + def create_proto_content(self, model, input_stream=None, output_stream=None, create_header=True): + input_streams = self.input_streams + output_streams = self.output_streams + header = self.create_proto_header(self.input_streams, self.output_streams) + content = self.create_node_content(header, input_streams, output_streams) + return content + + @staticmethod + def _copy_model_tree(proc, src, dst): + if "C:\\" in src: + proc.run_and_check( + f"robocopy /J /E /NP /NFL /NJH \"{src}\" \"{dst}\"", + env=os.environ.copy(), + exit_code_check=1, + exception_type=InvalidReturnCodeException, + timeout=1800, + ) + else: + shutil.copytree(src, dst) + + def prepare_resources(self, base_location): + dst_base = Path(base_location, "models") + dst = Path(dst_base, f"./{self.model.name}") + dst.parent.mkdir(exist_ok=True, parents=True) + if not Path.exists(dst): + proc = Process() + proc.disable_check_stderr() + try: + self._copy_model_tree(proc, self.models_path, dst) + except Exception as e: # pylint: disable=broad-exception-caught + if dst.exists(): + shutil.rmtree(dst, ignore_errors=True) + raise e + return str(dst_base) + + @classmethod + def create_proto_header(cls, input_stream="text_input", output_stream="text_output"): + return "\n".join([input_stream, output_stream]) + + +@dataclass +class HttpLLMCalculator(LLMCalculator): + name: str = "HttpLLMCalculator" + src_filename: str = "http_llm_calculator" + input_side_packet: str = "LLM_NODE_RESOURCES:llm" + model: ModelInfo = None + models_path: str = None + models_path_internal: str = "." + input_streams: str = None + output_streams: str = None + node_name: str = None + loopback: bool = True + kv_cache_size: int = kv_cache_size_value + plugin_config: dict = field(default_factory={GenerativeAIPluginConfig.KV_CACHE_PRECISION: kv_cache_precision_value}) + best_of_limit: int = None + max_tokens_limit: int = None + device: str = None + + def __post_init__(self): + super().__post_init__() + self.input_streams = 'input_stream: "HTTP_REQUEST_PAYLOAD:input"' + self.output_streams = 'output_stream: "HTTP_RESPONSE_PAYLOAD:output"' + + def create_node_content(self, header, input_streams, output_streams): + best_of_limit_str = f"\nbest_of_limit: {self.best_of_limit}," if self.best_of_limit is not None else "" + max_tokens_limit_str = ( + f"\nmax_tokens_limit: {self.max_tokens_limit}," if self.max_tokens_limit is not None else "" + ) + + def get_plugin_config_params_list(plugin_config_dict): + plugin_config_params = [] + for plugin_config_key, plugin_config_value in plugin_config_dict.items(): + if plugin_config_value is not None: + if type(plugin_config_value) == int: + plugin_config_params.append(f'"{plugin_config_key}": {plugin_config_value}') + elif type(plugin_config_value) == bool: + plugin_config_value = "true" if plugin_config_value else "false" + plugin_config_params.append(f'"{plugin_config_key}": {plugin_config_value}') + elif type(plugin_config_value) == str: + plugin_config_params.append(f'"{plugin_config_key}": "{plugin_config_value}"') + elif type(plugin_config_value) == dict: + plugin_config_params_dict = get_plugin_config_params_list(plugin_config_value) + plugin_config_params_dict_str = ', '.join(plugin_config_params_dict) + plugin_config_params.append(f"\"{plugin_config_key}\": {{ {plugin_config_params_dict_str} }}") + else: + raise NotImplementedError() + return plugin_config_params + + plugin_config_str = "" + plugin_config_params = get_plugin_config_params_list(self.plugin_config) + if plugin_config_params: + plugin_config_str = f"plugin_config: '{{ {', '.join(plugin_config_params)} }}'," + + tool_parser_str = "" + if any([ + self.model.tool_parser is not None and self.model.tools_enabled, + self.model.tool_parser is not None and self.model.is_agentic, + ]): + tool_parser_str = f'tool_parser: "{self.model.tool_parser}",' + if self.model.reasoning_parser is not None: + tool_parser_str += f' reasoning_parser: "{self.model.reasoning_parser}",' + + pipeline_type_str = "" + model_pipeline_type = getattr(self.model, "pipeline_type", None) + + if model_pipeline_type is not None: + pipeline_type_str = f'pipeline_type: {model_pipeline_type},' + elif config_pipeline_type is not None: + pipeline_type_str = f'pipeline_type: {config_pipeline_type},' + + max_num_batched_tokens_str = "" + model_max_num_batched_tokens = self.model.max_num_batched_tokens + max_num_batched_tokens_value = model_max_num_batched_tokens if model_max_num_batched_tokens is not None \ + else max_num_batched_tokens + if max_num_batched_tokens_value is not None: + max_num_batched_tokens_str = f"max_num_batched_tokens: {max_num_batched_tokens_value}" + + enable_prefix_caching_str = "" + if enable_prefix_caching_config: + enable_prefix_caching_str = f"enable_prefix_caching: true" + + tool_guided_str = "" + if self.model.enable_tool_guided_generation and self.enable_tool_guided_generation: + tool_guided_str = "enable_tool_guided_generation: true" + + device = f'device: "{self.device}",' if self.device is not None else "" + + content = f"""{header} +node: {{ + name: "{self.node_name}" + calculator: "{self.name}" + input_side_packet: "{self.input_side_packet}" + {input_streams} + input_stream: "LOOPBACK:loopback" + input_stream_info: {{ + tag_index: "LOOPBACK:0", + back_edge: true + }} + node_options: {{ + [type.googleapis.com / mediapipe.LLMCalculatorOptions]: {{ + models_path: "{self.models_path_internal}",{best_of_limit_str}{max_tokens_limit_str} + cache_size: {self.kv_cache_size}, + {device} + {tool_parser_str} + {plugin_config_str} + {pipeline_type_str} + {tool_guided_str} + {max_num_batched_tokens_str} + {enable_prefix_caching_str} + }} + }} + input_stream_handler {{ + input_stream_handler: "SyncSetInputStreamHandler", + options {{ + [mediapipe.SyncSetInputStreamHandlerOptions.ext] {{ + sync_set {{ + tag_index: "LOOPBACK:0" + }} + }} + }} + }} + {output_streams} + output_stream: "LOOPBACK:loopback" +}}""" + return content + + +@dataclass +class ImageGenCalculator(LLMCalculator): + name: str = "ImageGenCalculator" + src_filename: str = "image_gen_calculator" + input_side_packet: str = "IMAGE_GEN_NODE_RESOURCES:pipes" + model: ModelInfo = None + models_path: str = None + models_path_internal: str = "." + input_streams: str = None + output_streams: str = None + node_name: str = None + loopback: bool = True + kv_cache_size: int = kv_cache_size_value + plugin_config: dict = None + best_of_limit: int = None + max_tokens_limit: int = None + device: str = None + resolution: str = None + + def __post_init__(self): + self.input_streams = 'input_stream: "HTTP_REQUEST_PAYLOAD:input"' + self.output_streams = 'output_stream: "HTTP_RESPONSE_PAYLOAD:output"' + + def create_node_content(self, header, input_streams, output_streams): + resolution = f'resolution: "{self.resolution}"' if self.resolution is not None else "" + device = f'device: "{self.device}",' if self.device is not None else "" + + content = f"""{header} +node: {{ + name: "{self.node_name}" + calculator: "{self.name}" + input_side_packet: "{self.input_side_packet}" + {input_streams} + node_options: {{ + [type.googleapis.com / mediapipe.ImageGenCalculatorOptions]: {{ + models_path: "{self.models_path_internal}", + {device} + {resolution} + }} + }} + {output_streams} +}}""" + return content + + +@dataclass +class EmbeddingsCalculatorOV(LLMCalculator): + name: str = "EmbeddingsCalculatorOV" + src_filename: str = "embeddings_calculator_ov" + model: ModelInfo = None + models_path: str = None + models_path_internal: str = "." + input_streams: str = None + output_streams: str = None + node_name: str = None + loopback: bool = False + kv_cache_size: int = kv_cache_size_value + plugin_config: dict = None + best_of_limit: int = None + max_tokens_limit: int = None + device: str = None + normalize_embeddings: str = "true" + truncate: str = "false" + + def __post_init__(self): + super().__post_init__() + self.input_streams = 'input_stream: "REQUEST_PAYLOAD:input"' + self.output_streams = 'output_stream: "RESPONSE_PAYLOAD:output"' + + def create_node_content(self, header, input_streams, output_streams): + limits = [] + if self.best_of_limit is not None: limits.append(f"best_of_limit: {self.best_of_limit}") + if self.max_tokens_limit is not None: limits.append(f"max_tokens_limit: {self.max_tokens_limit}") + + models_path_line = f'models_path: "{self.models_path_internal}"' + if limits: models_path_line += f", {', '.join(limits)}" + + pooling_str = "" + model_obj = getattr(self, "model", None) + if model_obj is not None and getattr(model_obj, "pooling", None) is not None: + pooling_str = f'pooling: {model_obj.pooling},' + + target_device_str = f'target_device: "{self.device}",' if self.device is not None else "" + num_streams = 2 if self.device == TargetDevice.GPU else 1 + plugin_config_str = f'plugin_config: \'{{ "NUM_STREAMS": "{num_streams}" }}\',' + + content =f"""{header} +node {{ + name: "{self.node_name}", + calculator: "{self.name}" + input_side_packet: "EMBEDDINGS_NODE_RESOURCES:embeddings_servable" + {input_streams} + {output_streams} + node_options: {{ + [type.googleapis.com / mediapipe.EmbeddingsCalculatorOVOptions]: {{ + {models_path_line} + normalize_embeddings: {self.normalize_embeddings}, + truncate: {self.truncate}, + {pooling_str} + {target_device_str} + {plugin_config_str} + }} + }} +}} +""" + return content + + +@dataclass +class RerankCalculatorOV(LLMCalculator): + name: str = "RerankCalculatorOV" + src_filename: str = "rerank_calculator_ov" + model: ModelInfo = None + models_path: str = None + models_path_internal: str = "." + input_streams: str = None + output_streams: str = None + node_name: str = None + loopback: bool = False + kv_cache_size: int = kv_cache_size_value + plugin_config: dict = None + best_of_limit: int = None + max_tokens_limit: int = None + device: str = None + max_allowed_chunks: int = 10000 + + def __post_init__(self): + super().__post_init__() + self.input_streams = 'input_stream: "REQUEST_PAYLOAD:input"' + self.output_streams = 'output_stream: "RESPONSE_PAYLOAD:output"' + + def create_node_content(self, header, input_streams, output_streams): + target_device_str = f'target_device: "{self.device}",' if self.device is not None else "" + num_streams = 2 if self.device == TargetDevice.GPU else 1 + plugin_config_str = f'plugin_config: \'{{ "NUM_STREAMS": "{num_streams}" }}\',' + content = f"""{header} +node {{ + name: "{self.node_name}", + calculator: "{self.name}" + input_side_packet: "RERANK_NODE_RESOURCES:rerank_servable" + {input_streams} + {output_streams} + node_options: {{ + [type.googleapis.com / mediapipe.RerankCalculatorOVOptions]: {{ + models_path: "{self.models_path_internal}", + max_allowed_chunks: {self.max_allowed_chunks}, + {target_device_str} + {plugin_config_str} + }} + }} +}}""" + return content + + +@dataclass +class S2tCalculator(LLMCalculator): + name: str = "S2tCalculator" + src_filename: str = "s2t_calculator" + model: ModelInfo = None + models_path: str = None + models_path_internal: str = "." + input_streams: str = None + output_streams: str = None + node_name: str = None + loopback: bool = False + kv_cache_size: int = kv_cache_size_value + plugin_config: dict = None + best_of_limit: int = None + max_tokens_limit: int = None + device: str = None + + def __post_init__(self): + super().__post_init__() + self.input_streams = 'input_stream: "HTTP_REQUEST_PAYLOAD:input"' + self.output_streams = 'output_stream: "HTTP_RESPONSE_PAYLOAD:output"' + + def create_node_content(self, header, input_streams, output_streams): + target_device_str = f'target_device: "{self.device}",' if self.device is not None else "" + num_streams = 2 if self.device == TargetDevice.GPU else 1 + plugin_config_str = f'plugin_config: \'{{ "NUM_STREAMS": "{num_streams}" }}\',' + content = f"""{header} +node {{ + name: "{self.node_name}", + calculator: "{self.name}" + input_side_packet: "STT_NODE_RESOURCES:s2t_servable" + {input_streams} + {output_streams} + node_options: {{ + [type.googleapis.com / mediapipe.S2tCalculatorOptions]: {{ + models_path: "{self.models_path_internal}", + {target_device_str} + {plugin_config_str} + }} + }} +}}""" + return content + + +@dataclass +class T2sCalculator(LLMCalculator): + name: str = "T2sCalculator" + src_filename: str = "t2s_calculator" + model: ModelInfo = None + models_path: str = None + models_path_internal: str = "." + input_streams: str = None + output_streams: str = None + node_name: str = None + loopback: bool = False + kv_cache_size: int = kv_cache_size_value + plugin_config: dict = None + best_of_limit: int = None + max_tokens_limit: int = None + device: str = None + + def __post_init__(self): + super().__post_init__() + self.input_streams = 'input_stream: "HTTP_REQUEST_PAYLOAD:input"' + self.output_streams = 'output_stream: "HTTP_RESPONSE_PAYLOAD:output"' + + def create_node_content(self, header, input_streams, output_streams): + target_device_str = f'target_device: "{self.device}",' if self.device is not None else "" + num_streams = 2 if self.device == TargetDevice.GPU else 1 + plugin_config_str = f'plugin_config: \'{{ "NUM_STREAMS": "{num_streams}" }}\',' + content = f"""{header} +node {{ + name: "{self.node_name}", + calculator: "{self.name}" + input_side_packet: "TTS_NODE_RESOURCES:t2s_servable" + {input_streams} + {output_streams} + node_options: {{ + [type.googleapis.com / mediapipe.T2sCalculatorOptions]: {{ + models_path: "{self.models_path_internal}", + {target_device_str} + {plugin_config_str} + }} + }} +}}""" + return content diff --git a/tests/functional/object_model/minio_docker.py b/tests/functional/object_model/minio_docker.py deleted file mode 100644 index d7c9b2722c..0000000000 --- a/tests/functional/object_model/minio_docker.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from docker import DockerClient - -import tests.functional.config as config -from tests.functional.object_model.docker import Docker -from tests.functional.utils.parametrization import generate_test_object_name -from tests.functional.utils.helpers import SingletonMeta - - -class MinioDocker(Docker, metaclass=SingletonMeta): - def __init__(self, request, container_name, start_container_command=config.start_minio_container_command, - env_vars_container=None, image=config.minio_image, - container_log_line=config.container_minio_log_line): - container_name = generate_test_object_name(prefix=container_name) - super().__init__(request, container_name, start_container_command, - env_vars_container, image, container_log_line) - self.start_container_command = start_container_command.format(self.grpc_port) - self.start_result = None - - def start(self): - if not self.start_result: - self.start_container_command = self.start_container_command.format(self.grpc_port) - try: - self.start_result = super().start() - finally: - if self.start_result is None: - self.stop() - return self.start_result - - @staticmethod - def get_ip(container): - return DockerClient().containers.get(container.id).attrs["NetworkSettings"]["IPAddress"] diff --git a/tests/functional/object_model/ovms_binary.py b/tests/functional/object_model/ovms_binary.py index 83697e1df5..7e474adae9 100644 --- a/tests/functional/object_model/ovms_binary.py +++ b/tests/functional/object_model/ovms_binary.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 Intel Corporation +# Copyright (c) 2026 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,94 +13,355 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +import json import os -import queue -import shlex -import subprocess -import threading -import time - -from retry.api import retry_call - -import tests.functional.config as config -from tests.functional.command_wrappers.server import start_ovms_container_command - -from tests.functional.utils.grpc import port_manager_grpc -from tests.functional.utils.rest import port_manager_rest - - -class OvmsBinary: - - COMMON_RETRY = {"tries": 90, "delay": 2} - GETTING_LOGS_RETRY = COMMON_RETRY - GETTING_STATUS_RETRY = COMMON_RETRY - - def __init__(self, request, command_args, start_command, env_vars=None, cwd=None): - self.request = request - self.command_args = command_args - self.grpc_port = port_manager_grpc.get_port() - self.rest_port = port_manager_rest.get_port() - self.command_args["port"] = self.grpc_port - self.command_args["rest_port"] = self.rest_port - self.command = shlex.split("./" + os.path.basename(config.ovms_binary_path) + - start_ovms_container_command(start_command, self.command_args)) - self.cwd = cwd if cwd else os.path.dirname(config.ovms_binary_path) - self.env_vars = env_vars - self.process = None - self.logs_flag = True - - def output_reader(self, logs_queue): - for line in iter(self.process.stdout.readline, b''): - if self.logs_flag: - logs_queue.put(line) +import psutil +from datetime import datetime +from pathlib import Path + +from tests.functional.utils.context import Context +from tests.functional.utils.logger import get_logger +from tests.functional.constants.os_type import OsType +from tests.functional.utils.process import Process + +from tests.functional.constants.core import CONTAINER_STATUS_EXITED, CONTAINER_STATUS_RUNNING +from tests.functional.models.models_static import Muse +from tests.functional.constants.ovms_binaries import get_ovms_binary_cmd_setup +from tests.functional.constants.ovms_type import OvmsType +from tests.functional.constants.paths import Paths +from tests.functional.utils.log_monitor import LogMonitor +from tests.functional.object_model.cpu_extension import MuseModelExtension +from tests.functional.object_model.mediapipe_calculators import MediaPipeCalculator +from tests.functional.object_model.ovms_command import create_ovms_command +from tests.functional.object_model.ovms_config import OvmsConfig +from tests.functional.object_model.ovms_docker import OvmsDockerLauncher, OvmsDockerParams +from tests.functional.object_model.ovms_instance import OvmsInstance, OvmsRunContext +from tests.functional.object_model.ovms_log_monitor import BinaryOvmsLogMonitor +from tests.functional.object_model.test_environment import TestEnvironment +from tests.functional.utils.remote_test_environment import copy_custom_lib_to_host + +logger = get_logger(__name__) + + +def start_binary_ovms( + context: Context, parameters: OvmsDockerParams, path_to_binary_ovms, environment: dict = None, **kwargs +): + resources_dir, _ = TestEnvironment.current.prepare_container_folders(parameters.name, parameters.get_models()) + + if parameters.target_device is None: + parameters.target_device = context.target_device + + if environment is not None: + for key, value in environment.items(): + os.environ[key] = value + + # Automatically set use_config if the following conditions are met + use_config = ( + bool(parameters.use_config) + or bool(parameters.custom_config) + or (parameters.models is not None + and (any([model.is_pipeline() for model in parameters.models]) + or len(parameters.models) > 1 + or any(model.is_mediapipe and not model.single_mediapipe_model_mode for model in parameters.models))) + ) + + config_path_on_host = None + config_dir_path_on_host = None + if use_config: + config_dir_path_on_host, _ = OvmsDockerLauncher.create_config(parameters, parameters.name) + config_path_on_host = os.path.join(config_dir_path_on_host, Paths.CONFIG_FILE_NAME) + OvmsConfig.replace_config_models_paths_for_binary( + context, + config_path_on_host, + resources_dir, + parameters.name, + **kwargs, + ) + + if parameters.use_subconfig: + if use_config: + config_dict = OvmsConfig.load(config_path_on_host) + for mediapipe_model in config_dict['mediapipe_config_list']: + subconfig_path = os.path.join( + config_dir_path_on_host, mediapipe_model["name"], + os.path.basename(mediapipe_model["subconfig"]) + ) if "subconfig" in mediapipe_model else \ + os.path.join(config_dir_path_on_host, mediapipe_model["name"], Paths.SUBCONFIG_FILE_NAME) + OvmsConfig.replace_subconfig_paths(parameters.name, subconfig_path, resources_dir) + else: + config_dir_path_on_host = os.path.join(TestEnvironment.current.base_dir, parameters.name, Paths.MODELS_PATH_NAME) + subconfig_dict, subconfig_path = OvmsConfig.create_subconfig(parameters.name, parameters, config_dir_path_on_host) + OvmsConfig.replace_subconfig_paths(parameters.name, subconfig_path, resources_dir) + + if parameters.models is not None and any(model.is_mediapipe for model in parameters.models): + MediaPipeCalculator.prepare_proto_calculator(parameters, config_dir_path_on_host, config_path_on_host) + + cpu_extension_path = None + if parameters.models is not None and any(isinstance(model, Muse) for model in parameters.models): + cpu_extension = MuseModelExtension() + cpu_extension_path = cpu_extension.lib_path[1:] + elif parameters.cpu_extension: + if kwargs.get("replace_cpu_extension_params_for_binary", True): + host_dir = os.path.join(resources_dir, Paths.CPU_EXTENSIONS) + host_lib_path = os.path.join(host_dir, parameters.cpu_extension.lib_name) + cpu_extension_path = host_lib_path + if context.base_os == OsType.Windows: + raise NotImplementedError("Custom resources are not implemented for Windows") + copy_custom_lib_to_host(context.ovms_test_image, parameters.cpu_extension.lib_path, host_lib_path) + else: + cpu_extension_path = parameters.cpu_extension.lib_path + + if parameters.ports_enabled(): + if context.port_manager_grpc is not None and parameters.grpc_port is None: + parameters.grpc_port = context.port_manager_grpc.get_port() + if context.port_manager_rest is not None and parameters.rest_port is None: + parameters.rest_port = context.port_manager_rest.get_port() + + if parameters.check_version: + cmd = create_ovms_command( + config_path=None, + model_path=None, + model_name=None, + parameters=parameters, + cpu_extension_path=cpu_extension_path, + batch_size=None, + shape=None, + ovms_type=OvmsType.BINARY, + base_os=context.base_os, + ) + elif use_config: + cmd = create_ovms_command( + config_path=config_path_on_host, + model_path=None, + model_name=None, + parameters=parameters, + cpu_extension_path=cpu_extension_path, + batch_size=None, + shape=None, + ovms_type=OvmsType.BINARY, + base_os=context.base_os, + resolution=parameters.resolution, + ) + else: + model = parameters.models[0] if parameters.models is not None else None + pull = parameters.pull + task = parameters.task + task_params = parameters.task_params + list_models = parameters.list_models + overwrite_models = parameters.overwrite_models + add_to_config = parameters.add_to_config + remove_from_config = parameters.remove_from_config + batch_size, shape, single_mediapipe_model_mode, gguf_filename = None, None, None, None + if model is not None and model.is_hf_direct_load: + source_model = parameters.source_model if parameters.source_model is not None else model.name + model_repository_path = get_model_repository_path(context, parameters) + model_name = model.name + model_path = None + if model.gguf_filename: + gguf_filename = model.gguf_filename + elif list_models is not None: + model_repository_path = get_model_repository_path(context, parameters) + source_model, model_name, model_path, gguf_filename = None, None, None, None + elif add_to_config: + source_model = None + model_name = parameters.model_name + model_path = parameters.model_path + if model_path is None: + model_repository_path = get_model_repository_path(context, parameters) else: - break + model_repository_path = None + config_path_on_host = os.path.join(TestEnvironment.current.base_dir, context.test_object_name, + Paths.MODELS_PATH_NAME, Paths.CONFIG_FILE_NAME) + elif remove_from_config: + source_model, model_repository_path, model_path, gguf_filename = None, None, None, None + model_name = parameters.model_name + config_path_on_host = os.path.join(TestEnvironment.current.base_dir, context.test_object_name, + Paths.MODELS_PATH_NAME, Paths.CONFIG_FILE_NAME) + else: + model_name = parameters.model_name if parameters.model_name is not None else model.name + model_path = parameters.model_path if parameters.model_path is not None else model.get_model_path() + model_path = model_path.replace(Paths.MODELS_PATH_INTERNAL, f"{resources_dir}{Paths.MODELS_PATH_INTERNAL}") + source_model, model_repository_path, gguf_filename = None, None, None + batch_size = model.batch_size + shape = model.input_shape_for_ovms + single_mediapipe_model_mode = model.single_mediapipe_model_mode + cmd = create_ovms_command( + config_path=config_path_on_host, + model_path=model_path, + model_name=model_name, + parameters=parameters, + cpu_extension_path=cpu_extension_path, + batch_size=batch_size, + shape=shape, + ovms_type=OvmsType.BINARY, + base_os=context.base_os, + single_mediapipe_model_mode=single_mediapipe_model_mode, + pull=pull, + source_model=source_model, + gguf_filename=gguf_filename, + model_repository_path=model_repository_path, + task=task, + task_params=task_params, + list_models=list_models, + overwrite_models=overwrite_models, + add_to_config=add_to_config, + remove_from_config=remove_from_config, + resolution=parameters.resolution, + cache_size=parameters.cache_size, + pooling=model.pooling if model is not None else None, + ) + + ovms_binary = OvmsBinary( + name=parameters.name if parameters.name is not None else context.test_object_name, + parameters=parameters, + cmd=cmd, + path_to_binary=path_to_binary_ovms, + container_folder=resources_dir, + ) + ovms_binary.start( + base_os=context.base_os, + environment=environment, + venv_activate_path=kwargs.get("venv_activate_path", None), + ) + context.test_objects.append(ovms_binary) + + return OvmsRunContext(ovms_binary, parameters.models) - def start(self): - - def finalizer(): - self.process.kill() - self.logs_flag = False - time.sleep(2) - get_logs_thread.join() - port_manager_grpc.release_port(self.grpc_port) - port_manager_rest.release_port(self.rest_port) - - self.request.addfinalizer(finalizer) - - logs_queue = queue.Queue() - get_logs_thread = threading.Thread(target=self.output_reader, args=(logs_queue,)) - - env_vars_dict = {} - if self.env_vars is not None: - for env in self.env_vars: - env_vars_dict[env.split("=")[0]] = env.split("=")[1] - - self.process = subprocess.Popen(self.command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - stdin=subprocess.PIPE, - universal_newlines=True, - cwd=self.cwd, env=env_vars_dict) - get_logs_thread.start() - self.ensure_logs_contains(logs_queue) - - return self.process, {"grpc_port": self.grpc_port, "rest_port": self.rest_port} - - def stop(self): - pass - - @staticmethod - def ensure_logs(logs_queue): - logs = "" - while True: - try: - line = logs_queue.get(block=False) - except queue.Empty: + +def get_model_repository_path(context, parameters): + if parameters.model_repository_path is not None: + model_repository_path = parameters.model_repository_path + else: + model_repository_path = os.path.join( + TestEnvironment.current.base_dir, + context.test_object_name, + Paths.MODELS_PATH_NAME, + ) + if not os.path.exists(model_repository_path): + Path(model_repository_path).mkdir(parents=True, exist_ok=True) + return model_repository_path + + +class OvmsBinary(OvmsInstance): + + def __init__(self, name, parameters, cmd, path_to_binary, container_folder=None, **kwargs): + self.process = Process() if kwargs.get("process", None) is None else kwargs.pop("process") + self.cmd = cmd + self.path_to_binary = path_to_binary + super().__init__( + name=name, + container_folder=container_folder, + default_logger=BinaryOvmsLogMonitor(self.process), + rest_port=parameters.rest_port, + grpc_port=parameters.grpc_port, + target_device=parameters.target_device, + **kwargs, + ) + + def fetch_and_store_ovms_pid(self, timeout=60): + ovms_pid = None + parent_proc_pid = self.process._proc.pid + parent_proc = psutil.Process(parent_proc_pid) + start = datetime.now() + while (datetime.now() - start).total_seconds() <= timeout: + for child in parent_proc.children(): + if "ovms" in child.name(): + ovms_pid = child.pid + break + if ovms_pid is not None: break - logs += line - assert config.container_log_line in logs + self._dmesg_log.ovms_pid = ovms_pid + return ovms_pid + + def execute_command(self, cmd, cwd=None, stream=False): + cwd = TestEnvironment.current.base_dir.strpath if cwd is None else cwd + proc = Process() + returncode, stdout, stderr = proc.run(cmd, cwd=cwd) + assert returncode in [ + 0, + None, + ], f"Unexpected error code detected: {returncode} (expect 0) during executing command: {cmd}; Error {stderr}" + return returncode, stdout + + def start(self, ensure_started=False, environment=None, *args, **kwargs): + self._dmesg_log.get_all_logs() + base_os = kwargs.get("base_os", None) + # In order to correctly handle OVMS logs it is required to redirect stderr stream to stdout: 2>&1 + if base_os == OsType.Windows: + resource_dir = os.path.join(*Path(self.path_to_binary).parts[:-2]) + venv_activate_path = kwargs.get("venv_activate_path", None) + # full path required for Windows to find the binary + pre_cmd, env = get_ovms_binary_cmd_setup( + base_os=base_os, + resources_dir_path=os.path.join(resource_dir, "ovms"), + environment=environment, + venv_activate_path=venv_activate_path, + ) + cmd = f"{pre_cmd}{resource_dir}\\{self.cmd} 2>&1" + else: + resource_dir = os.path.join(*Path(self.path_to_binary).parts[:-3]) + pre_cmd, env = get_ovms_binary_cmd_setup( + base_os=base_os, + resources_dir_path=os.path.join(resource_dir, "ovms"), + environment=environment, + ) + cmd = f"{pre_cmd} ./{self.cmd} 2>&1" + self.process.async_run(cmd, cwd=resource_dir, env=env) + + def ensure_status(self, status: str = CONTAINER_STATUS_RUNNING): + self.process.is_alive() + + def get_status(self, status=None, timeout=None): + return CONTAINER_STATUS_RUNNING if self.process.is_alive() else CONTAINER_STATUS_EXITED + + def _create_logger(self) -> LogMonitor: + return BinaryOvmsLogMonitor(self.process) + + def cleanup(self): + self.kill() + super().cleanup() + self.release_ports() + self.get_dmesg_log_monitor().raise_on_unexpected_messages(filter_known_messages=True) + + def kill(self): + self.process.kill(force=True) if self.process.is_alive() else True + + def update_model_list_and_config( + self, + name, + models, + models_to_verify=None, + resources_paths=None, + context=None, + params=None, + **kwargs + ): + resources_dir, models_dir_on_host = TestEnvironment.current.prepare_container_folders(name, models) + + if models_to_verify: + ovms_log = self.create_log(False) + + config_path_on_host = os.path.join(resources_dir, os.path.join(Paths.MODELS_PATH_NAME, Paths.CONFIG_FILE_NAME)) + if models: + if models_to_verify is not None and any(model.is_mediapipe for model in models_to_verify): + assert params is not None, "Params should be provided to create MediaPipe calculators" + self.prepare_mediapipe_config_and_graph(name, params, models) + OvmsConfig.replace_config_models_paths_for_binary( + context, config_path_on_host, resources_dir, name, **kwargs + ) + else: + OvmsConfig.generate(name, models) + OvmsConfig.replace_config_models_paths_for_binary( + context, config_path_on_host, resources_dir, name, **kwargs + ) + else: + OvmsConfig.generate(name, models) + + config_dict = json.loads(Path(config_path_on_host).read_text()) + + if models_to_verify: + break_msg_list = self.get_break_msg_list(models_to_verify) + ovms_log.models_loaded(models_to_verify, break_msg_list=break_msg_list) - def ensure_logs_contains(self, logs_queue): - return retry_call(self.ensure_logs, exceptions=AssertionError, **OvmsBinary.GETTING_LOGS_RETRY, - fargs=(logs_queue,)) + return config_dict diff --git a/tests/functional/object_model/ovms_capi.py b/tests/functional/object_model/ovms_capi.py new file mode 100644 index 0000000000..67e934d364 --- /dev/null +++ b/tests/functional/object_model/ovms_capi.py @@ -0,0 +1,318 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import pickle +import re +import sys +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from select import select +from typing import List + +import numpy as np + +from tests.functional.utils.assertions import CapiException +from tests.functional.utils.context import Context +from tests.functional.utils.logger import get_logger +from tests.functional.utils.test_framework import generate_test_object_name, skip_if_runtime +from tests.functional.config import ovms_c_repo_path +from tests.functional.constants.core import CONTAINER_STATUS_RUNNING +from tests.functional.models import ModelInfo +from tests.functional.constants.ovms import Config +from tests.functional.constants.ovms_messages import OvmsMessages +from tests.functional.constants.ovms_type import OvmsType +from tests.functional.constants.paths import Paths +from tests.functional.utils.log_monitor import LogMonitor +from tests.functional.object_model.cpu_extension import MuseModelExtension +from tests.functional.object_model.ovms_binary import OvmsBinary +from tests.functional.object_model.ovms_config import OvmsConfig +from tests.functional.object_model.ovms_docker import OvmsDockerParams +from tests.functional.object_model.ovms_instance import OvmsRunContext +from tests.functional.object_model.ovms_log_monitor import BinaryOvmsLogMonitor +from tests.functional.object_model.ovms_params import OvmsParams +from tests.functional.object_model.test_environment import TestEnvironment +from tests.functional.utils.remote_test_environment import copy_custom_lib_to_host + +logger = get_logger(__name__) + + +@dataclass(frozen=False) +class OvmsCapiParams(OvmsParams): + config_path: str = "" + cpu_extension_path: str = "" + + +class OvmsCapiLogMonitor(BinaryOvmsLogMonitor): + def is_running(self, ovms_ports, timeout, os_type=None): + return True + + +@dataclass +class OvmsCapiRunContext(OvmsRunContext): + addr: int = 0 + + +class OvmsCapiInstance(OvmsBinary): + PYTHON_SNIPPET = "'from tests.functional.object_model.ovms_capi import OvmsCapiInstance; OvmsCapiInstance({},{}).main_loop({})'" + + def __init__(self, parameters, base_os, **kwargs): + if isinstance(parameters, dict): + # spawned instance + logger.info(parameters) + parameters = OvmsCapiParams(**parameters) + self.base_os = base_os + self.parameters = parameters + self.pickle_friendly_parameters = self.parameters.to_str() + capi_package_content = Paths.CAPI_WRAPPER_PACKAGE_CONTENT_PATH(self.base_os) + cmd = self.PYTHON_SNIPPET.format( + f"parameters={self.pickle_friendly_parameters}", + f'base_os="{self.base_os}"', + f'capi_package_content="{capi_package_content}"', + ) + + super().__init__( + name=parameters.name, + cmd=cmd, + parameters=parameters, + path_to_binary=None, + container_folder=getattr(parameters, "resources_dir", None), + **kwargs, + ) + + def get_port(self, api_type): + if not isinstance(api_type, str) and api_type.communication == OvmsType.CAPI: + return self + else: + return super().get_port(api_type) + + def get_status(self, status=None, timeout=None): + status = Path(f"/proc/{self.process._proc.pid}/status").read_text() + status = [line for line in status.splitlines() if line.startswith("State:")] + if "sleeping" in status[0]: + result = CONTAINER_STATUS_RUNNING + else: + result = None + return result + + def start(self, ensure_started=False, use_valgrind=False, *args, **kwargs): + cmd = [sys.executable, "-c", f"{self.cmd}", "2>&1"] + if use_valgrind: + cmd = cmd.insert(0, "valgrind") + cmd = " ".join(cmd) + + self.process.async_run(cmd, cwd=os.getcwd(), daemon_mode=True, use_stdin=True) + self.ovms_pid = self.process._proc.pid + self._stdin_proc_fd = f"/proc/{self.ovms_pid}/fd/0" + + def cleanup(self): + try: + self.send_terminate_command() + except Exception as e: + logger.error(e) + time.sleep(1) # give some time for terminate + logger.debug("Cleaning OVMS CAPI instance") + super().cleanup() + + def _create_logger(self) -> LogMonitor: + return OvmsCapiLogMonitor(self.process) + + def execute_command(self, cmd, cwd=None): + # Execute command from host. + try: + # Execute python api command without any validation (sic!). + # Expect valid command sent by host process. + print(cmd) + result = exec(cmd) + except Exception as e: + logger.exception(e) + + def ensure_started( + self, + expected_loaded_models: List[ModelInfo] = None, + custom_messages: list = None, + expected_unloaded_models: List[ModelInfo] = None, + timeout: int = None, + os_type: str = None, + ): + if not custom_messages: + custom_messages = [] + custom_messages.append(OvmsMessages.CAPI_STARTED_OVMS_SERVER) + self._default_log.ensure_contains_messages(custom_messages, timeout=timeout, ovms_instance=self) + + @staticmethod + def ensure_newline(cmd): + return cmd if cmd.endswith("\n") else f"{cmd}\n" + + def send_command_to_process(self, cmd): + # Write directly to stdin file descriptor in /proc/ filesystem. + # It should mitigate interprocess communication issues in 'pure pythonic' approach. + logger.debug(self._stdin_proc_fd) + with open(self._stdin_proc_fd, "w") as fd: + fd.write(self.ensure_newline(cmd)) + + def send_command_to_process_with_output(self, cmd, cmd_kwargs): + with tempfile.NamedTemporaryFile() as tmp_infile: + with tempfile.NamedTemporaryFile() as tmp_outfile: + cmd = f"self.capi.tmp_infile='{tmp_infile.name}'; self.capi.tmp_outfile='{tmp_outfile.name}'; {cmd}" + Path(tmp_infile.name).write_bytes(pickle.dumps(cmd_kwargs)) + + self.send_command_to_process(cmd) + output_path = Path(tmp_outfile.name) + + t_end = time.time() + 20 + while time.time() < t_end and output_path.stat().st_size == 0: + time.sleep(0.5) + assert output_path.stat().st_size + raw_data = output_path.read_bytes() + result = pickle.loads(raw_data) + if issubclass(type(result), Exception): + raise result + return result + + def send_start_server_command(self): + cmd = f"self.srv = self.capi.ovms_start_server({self.pickle_friendly_parameters})" + return self.send_command_to_process(cmd) + + def send_stop_server_command(self): + cmd = f"self.srv = self.capi.server_stop()" + return self.send_command_to_process(cmd) + + def send_terminate_command(self): + cmd = f"self.running = False" + return self.send_command_to_process(cmd) + + def send_terminate_command(self): + cmd = f"self.running = False" + return self.send_command_to_process(cmd) + + def send_get_model_meta_command(self, model_name, model_version): + cmd = f"self.capi.get_model_meta()" + cmd_kwargs = {"servableName": model_name, "servableVersion": model_version} + return self.send_command_to_process_with_output(cmd, cmd_kwargs) + + def send_inference(self, model, input_data): + if not input_data: + skip_if_runtime( + any(-1 in shape for input_name, shape in model.input_shapes.items()), + msg="Dynamic shapes not supported for now", + ) + + _in_data = {key: value.shape for key, value in input_data.items()} + # Write directly to stdin file descriptor in /proc/ filesystem. + # It should mitigate interprocess communication issues in 'pure pythonic' approach. + cmd = f"self.result = self.capi.send_inference()" + cmd_kwargs = { + "model_name": model.name, + "inputs": input_data, + } + result = self.send_command_to_process_with_output(cmd, cmd_kwargs) + if "Unsupported data type" in result: + raise CapiException(result) + result = {key: np.asarray(val) for key, val in result.items()} + return result + + def send_get_capi_api_version(self): + cmd = "self.major, self.minor = self.capi.ovms_get_capi_version()" + self.send_command_to_process(cmd) + + def get_major_minor_version(self): + filepath = os.path.join(ovms_c_repo_path, "src/ovms.h") + with open(filepath, "r") as f: + data = f.read() + + major = re.search(r"OVMS_API_VERSION_MAJOR (\d+)", data).group(1) + minor = re.search(r"OVMS_API_VERSION_MINOR (\d+)", data).group(1) + + return major, minor + + def fetch_command(self): + # Expect commands submitted by host process via OvmsCapiInstance.send_command_to_process + cmd = input() # Simple blocking read should suffice for now. + return cmd.strip() + + def main_loop(self, capi_package_content): + """ + This method should be called in spawned thread. + """ + # Prepare PYTHONPATHs prior imports + sys.path.append(os.path.join(capi_package_content, "lib")) + + import ovms_capi_wrapper as capi # pylint: disable=import-outside-toplevel, import-error + + self.capi = capi + self.running = True + while self.running: + rlist, _, _ = select([sys.stdin], [], [], 1.0) + if rlist: + cmd = self.fetch_command() + self.execute_command(cmd) + else: + print("Sleeping ...") + time.sleep(0.5) + + +def start_capi_ovms(context: Context, parameters: OvmsDockerParams, environment: dict = None, resources_dir=None): + if parameters.name is None: + parameters.name = ( + context.test_object_name if context.test_object_name is not None else generate_test_object_name() + ) + if not resources_dir: + parameters.resources_dir, _ = TestEnvironment.current.prepare_container_folders( + parameters.name, parameters.get_models() + ) + else: + parameters.resources_dir = resources_dir + + if parameters.cpu_extension: + if isinstance(parameters.cpu_extension, MuseModelExtension): + capi_package = Path(Paths.CAPI_WRAPPER_PACKAGE_CONTENT_PATH(context.base_os)) + parameters.cpu_extension_path = str(Path(capi_package.parent, f"./{parameters.cpu_extension.lib_path}")) + else: + host_dir = os.path.join(parameters.resources_dir, Paths.CPU_EXTENSIONS) + host_lib_path = os.path.join(host_dir, parameters.cpu_extension.lib_name) + copy_custom_lib_to_host(context.ovms_test_image, parameters.cpu_extension.lib_path, host_lib_path) + parameters.cpu_extension_path = str(host_lib_path) + + if context.port_manager_grpc is not None and parameters.grpc_port is None: + parameters.grpc_port = context.port_manager_grpc.get_port() + if context.port_manager_rest is not None and parameters.rest_port is None: + parameters.rest_port = context.port_manager_rest.get_port() + + # https://github.com/openvinotoolkit/model_server/blob/main/src/ovms.h#L394 + # CAPI is supported only with config.json file. + if parameters.custom_config is None: + indirect_config_path, config_dict = OvmsConfig.generate_from_parameters( + parameters.name, parameters, parameters.resources_dir + ) + else: + config_dict = parameters.custom_config + for model in config_dict[Config.MODEL_CONFIG_LIST]: + if not model["config"]["base_path"].startswith(parameters.resources_dir): + model_config_base_path = model["config"]["base_path"] + model["config"]["base_path"] = os.path.join(parameters.resources_dir, model_config_base_path.strip("/")) + indirect_config_path = OvmsConfig.save(parameters.name, config_dict) + parameters.config_path = str(Path(parameters.resources_dir, f"./{indirect_config_path}")) + + capi_instance = OvmsCapiInstance(parameters, context.base_os) + capi_instance.start() # Execute main loop + context.test_objects.append(capi_instance) + + capi_instance.send_start_server_command() + + return OvmsCapiRunContext(capi_instance, parameters.models) diff --git a/tests/functional/object_model/ovms_command.py b/tests/functional/object_model/ovms_command.py new file mode 100644 index 0000000000..c073f5e02f --- /dev/null +++ b/tests/functional/object_model/ovms_command.py @@ -0,0 +1,342 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from dataclasses import dataclass +from typing import Union + +from tests.functional.utils.logger import get_logger +from tests.functional.constants.os_type import OsType +from tests.functional.config import enable_plugin_config_target_device +from tests.functional.constants.metrics import MetricsPolicy +from tests.functional.constants.ovms import Ovms, set_plugin_config_boolean_value +from tests.functional.constants.ovms_openai import ImagesRequestParamsValues +from tests.functional.constants.paths import Paths + +logger = get_logger(__name__) + + +def create_ovms_command( + config_path, + model_path, + model_name, + parameters, + cpu_extension_path, + batch_size=None, + shape=None, + ovms_type=None, + base_os=None, + pull=None, + source_model=None, + gguf_filename=None, + model_repository_path=None, + task=None, + task_params=None, + list_models=None, + overwrite_models=None, + add_to_config=False, + remove_from_config=False, + single_mediapipe_model_mode=False, + resolution=None, + cache_size=None, + pooling=None, +): + is_stateful = parameters.is_stateful_model_present() if parameters.is_stateful is None else parameters.is_stateful + layout = parameters.get_layout_from_regular_models() + + common_parameters = { + "grpc_port": parameters.grpc_port, + "rest_port": parameters.rest_port, + "logging_level": parameters.log_level, + "layout": layout, + "stateful": is_stateful, + "rest_workers": parameters.rest_workers, + "grpc_workers": parameters.grpc_workers, + "check_version": parameters.check_version, + "file_system_poll_wait_seconds": parameters.file_system_poll_wait_seconds, + "cpu_extension": cpu_extension_path, + "metrics_enable": parameters.metrics_enable, + "metrics_list": parameters.metrics_list, + "sequence_cleaner_poll_wait_minutes": parameters.sequence_cleaner_poll_wait_minutes, + "ovms_type": ovms_type, + "base_os": base_os, + "allowed_local_media_path": parameters.allowed_local_media_path, + "allowed_media_domains": parameters.allowed_media_domains, + "pooling": pooling, + } + pull_parameters = { + "pull": pull, + "source_model": source_model, + "gguf_filename": gguf_filename, + "model_repository_path": model_repository_path, + "task": task, + "task_params": task_params, + "list_models": list_models, + "overwrite_models": overwrite_models, + "add_to_config": add_to_config, + "remove_from_config": remove_from_config, + "resolution": resolution, + "cache_size": cache_size, + } + if config_path is not None: + if add_to_config or remove_from_config: + return OvmsCommand(config_path=config_path, model_name=model_name, **common_parameters, **pull_parameters) + return OvmsCommand(config_path=config_path, **common_parameters) + else: + plugin_config = parameters.get_plugin_config_from_regular_models() + if enable_plugin_config_target_device: + plugin_config_target_device = Ovms.PLUGIN_CONFIG[parameters.target_device] + plugin_config = ( + {**plugin_config, **plugin_config_target_device} + if plugin_config is not None + else {**plugin_config_target_device} + ) + use_parameter = not any([single_mediapipe_model_mode, list_models, add_to_config, remove_from_config]) + return OvmsCommand( + model_path=model_path, + model_name=model_name, + plugin_config=plugin_config if use_parameter else None, + batchsize=batch_size if use_parameter else None, + nireq=parameters.nireq if use_parameter else None, + target_device=parameters.target_device if use_parameter else None, + shape=shape if use_parameter else None, + model_version_policy=parameters.model_version_policy if use_parameter else None, + max_sequence_number=parameters.max_sequence_number if use_parameter else None, + idle_sequence_cleanup=parameters.idle_sequence_cleanup if use_parameter else None, + low_latency_transformation=parameters.low_latency_transformation if use_parameter else None, + **common_parameters, + **pull_parameters, + ) + + +@dataclass +class OvmsCommand(object): + logging_level: str = None + model_path: str = None + model_name: str = None + plugin_config: Union[dict, str] = None + grpc_port: int = None + grpc_workers: int = None + nireq: int = None + target_device: str = None + batchsize: Union[int, str] = None + config_path: str = None + rest_port: int = None + rest_workers: int = None + shape: str = None + model_version_policy: str = None + file_system_poll_wait_seconds: str = None + max_sequence_number: int = None + sequence_cleaner_poll_wait_minutes: str = None + low_latency_transformation: bool = None + stateful: bool = False + check_version: bool = False + layout: str = None + cpu_extension: str = None + idle_sequence_cleanup: bool = None + metrics_enable: MetricsPolicy = MetricsPolicy.NotDefined + metrics_list: list = None + ovms_type: str = None + base_os: str = None + allowed_local_media_path: str = None + allowed_media_domains: str = None + pull: bool = False + source_model: str = None + gguf_filename: str = None + model_repository_path: str = None + task: str = None + task_params: str = None + list_models: bool = None + overwrite_models: bool = None + add_to_config: bool = False + remove_from_config: bool = False + resolution: str = None + cache_size: int = None + pooling: str = None + + def to_list(self): + ovms_command, _ = Ovms.get_ovms_binary_paths(self.ovms_type, self.base_os) + command_parts = [ovms_command] + + if self.check_version: + command_parts.append("--version") + else: + if self.pull: + command_parts.append("--pull") + + if self.source_model is not None: + command_parts.append("--source_model") + command_parts.append(self.source_model) + if self.gguf_filename is not None: + command_parts.append("--gguf_filename") + command_parts.append(self.gguf_filename) + + if self.model_repository_path is not None: + command_parts.append("--model_repository_path") + command_parts.append(self.model_repository_path) + + if self.task is not None: + command_parts.append("--task") + command_parts.append(self.task) + + if self.task_params is not None: + command_parts.append("--task_params") + command_parts.append(self.task_params) + + if self.list_models: + command_parts.append("--list_models") + + if self.overwrite_models: + command_parts.append("--overwrite_models") + + if self.add_to_config: + command_parts.append("--add_to_config") + + if self.remove_from_config: + command_parts.append("--remove_from_config") + + if self.resolution is not None: + command_parts.append("--resolution") + command_parts.append(self.resolution) + + if self.cache_size is not None: + command_parts.append("--cache_size") + command_parts.append(str(self.cache_size)) + + if self.model_path and self.config_path: + logger.debug("Both config_path and model_path with model_name not set!!!") + + if self.logging_level is not None: + command_parts.append("--log_level") + command_parts.append(str(self.logging_level)) + + if self.grpc_port is not None: + command_parts.append("--port") + command_parts.append(str(self.grpc_port)) + + if self.grpc_workers is not None: + command_parts.append("--grpc_workers") + command_parts.append(str(self.grpc_workers)) + + if self.nireq is not None: + command_parts.append("--nireq") + command_parts.append(str(self.nireq)) + + if self.rest_workers is not None: + command_parts.append("--rest_workers") + command_parts.append(str(self.rest_workers)) + + if self.rest_port is not None: + command_parts.append("--rest_port") + command_parts.append(str(self.rest_port)) + + if self.config_path is not None: + command_parts.append("--config_path") + command_parts.append(self.config_path) + + if self.model_path is not None: + command_parts.append("--model_path") + command_parts.append(self.model_path) + + if self.model_name is not None: + command_parts.append("--model_name") + command_parts.append(self.model_name) + + if self.target_device is not None: + command_parts.append("--target_device") + command_parts.append(f'"{str(self.target_device)}"' if + self.target_device == ImagesRequestParamsValues.MIXED_NPU_DEVICE + else str(self.target_device)) + + if self.batchsize is not None: + command_parts.append("--batch_size") + command_parts.append(str(self.batchsize)) + + if self.allowed_local_media_path is not None: + command_parts.append("--allowed_local_media_path") + images_path = Paths.IMAGES_PATH_INTERNAL if self.base_os != OsType.Windows \ + else self.allowed_local_media_path + command_parts.append(images_path) + + if self.allowed_media_domains is not None: + command_parts.append("--allowed_media_domains") + command_parts.append(self.allowed_media_domains) + + if self.plugin_config is not None: + command_parts.append("--plugin_config") + plugin_config_str = str(self.plugin_config).replace('"', '\\"').replace("'", '\\"') + plugin_config_str = set_plugin_config_boolean_value(plugin_config_str) + command_parts.append(f"\"{plugin_config_str}\"") + + if self.shape is not None: + command_parts.append("--shape") + shape_str = str(self.shape).replace('"', '\\"').replace("'", '\\"') + if isinstance(self.shape, dict): + shape_str = shape_str.replace("(", '\\"(').replace(")", ')\\"') + command_parts.append(f"\"{shape_str}\"") + + if self.model_version_policy is not None: + command_parts.append("--model_version_policy") + model_version_policy_str = str(self.model_version_policy).replace('"', '\\"').replace("'", '\\"') + command_parts.append(f"\"{model_version_policy_str}\"") + + if self.file_system_poll_wait_seconds is not None: + command_parts.append("--file_system_poll_wait_seconds") + command_parts.append(str(self.file_system_poll_wait_seconds)) + + if self.sequence_cleaner_poll_wait_minutes is not None: + command_parts.append("--sequence_cleaner_poll_wait_minutes") + command_parts.append(str(self.sequence_cleaner_poll_wait_minutes)) + + if self.idle_sequence_cleanup is not None: + command_parts.append("--idle_sequence_cleanup") + command_parts.append(str(self.idle_sequence_cleanup)) + + if self.max_sequence_number is not None: + command_parts.append("--max_sequence_number") + command_parts.append(str(self.max_sequence_number)) + + if self.low_latency_transformation is not None: + command_parts.append("--low_latency_transformation") + command_parts.append(str(self.low_latency_transformation)) + + if self.layout: + command_parts.append("--layout") + command_parts.append(str(self.layout)) + + if self.cpu_extension: + command_parts.append("--cpu_extension") + command_parts.append(str(self.cpu_extension)) + + if self.stateful: + command_parts.append("--stateful") + + if self.metrics_enable == MetricsPolicy.EnabledInCli: + command_parts.append("--metrics_enable") + + if self.metrics_list: + command_parts.append(f'--metrics_list {",".join(self.metrics_list)}') + + if self.metrics_enable == MetricsPolicy.EnabledMetricsList and self.metrics_list is not None: + command_parts.append(f'--metrics_list {",".join(self.metrics_list)}') + + if self.pooling is not None: + command_parts.append("--pooling") + command_parts.append(str(self.pooling)) + + return command_parts + + def __str__(self): + return " ".join(self.to_list()).replace(" False", "=False").replace(" True", "=True") diff --git a/tests/functional/object_model/ovms_config.py b/tests/functional/object_model/ovms_config.py new file mode 100644 index 0000000000..0e77953e1d --- /dev/null +++ b/tests/functional/object_model/ovms_config.py @@ -0,0 +1,349 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import os +from pathlib import Path +from typing import List + +from tests.functional.utils.logger import get_logger +from tests.functional.constants.os_type import OsType +from tests.functional.config import enable_plugin_config_target_device +from tests.functional.constants.custom_loader import CustomLoaderConsts +from tests.functional.models import ModelInfo +from tests.functional.constants.ovms import Config, CurrentOvmsType, Ovms, set_plugin_config_boolean_value +from tests.functional.constants.ovms_type import OvmsType +from tests.functional.constants.paths import Paths +from tests.functional.constants.pipelines import Pipeline +from tests.functional.object_model.custom_loader import CustomLoader +from tests.functional.object_model.custom_node import CustomNode +from tests.functional.object_model.ovms_params import MetricsPolicy, OvmsParams +from tests.functional.object_model.test_environment import TestEnvironment +from tests.functional.utils.remote_test_environment import copy_custom_lib_to_host + +logger = get_logger(__name__) + + +class OvmsConfig(object): + + @staticmethod + def generate(name, models, **kwargs): + params = OvmsParams(models=models) + return OvmsConfig.generate_from_parameters(name, params, **kwargs) + + @staticmethod + def generate_from_parameters(name, parameters, resource_dir=None): + models = parameters.models + pipelines = [] + regular_models = [] + regular_models_used_in_pipelines = [] + for model in models or []: + if isinstance(model, Pipeline) and model.is_pipeline(): + pipelines.append(model) + regular_models_used_in_pipelines.extend(model.get_regular_models()) + else: + regular_models.append(model) + + for model in regular_models_used_in_pipelines: + model_already_exists = False + for regular_model in regular_models: + if regular_model.name == model.name: + model_already_exists = True + if not model_already_exists: + regular_models.append(model) + + config_dict = OvmsConfig.build( + models=regular_models, + pipelines=pipelines, + resource_dir=resource_dir, + use_subconfig=parameters.use_subconfig, + ) + + if parameters.metrics_enable in [MetricsPolicy.EnabledInConfig, MetricsPolicy.Disabled, + MetricsPolicy.EnabledMetricsList]: + enable = True if parameters.metrics_enable == MetricsPolicy.EnabledInConfig else False + config_dict[Config.MONITORING] = {"metrics": {"enable": enable}} + + if parameters.metrics_list is not None: + metrics = config_dict[Config.MONITORING].get("metrics", {}) + metrics["metrics_list"] = parameters.metrics_list + + if enable_plugin_config_target_device: + plugin_config_target_device = Ovms.PLUGIN_CONFIG[parameters.target_device] + if plugin_config_target_device: + for model_config in config_dict[Config.MODEL_CONFIG_LIST]: + plugin_config = model_config[Config.CONFIG].get(Config.PLUGIN_CONFIG, None) + if plugin_config is not None: + model_config[Config.CONFIG][Config.PLUGIN_CONFIG] = { + **plugin_config, + **plugin_config_target_device, + } + else: + model_config[Config.CONFIG][Config.PLUGIN_CONFIG] = plugin_config_target_device + return OvmsConfig.save(name, config_dict), config_dict + + @staticmethod + def save(name, config_dict: dict, config_path: str = None): + config_json = json.dumps(config_dict, indent=2) + config_json = set_plugin_config_boolean_value(config_json, config_file=True) + config_path = ( + os.path.join(TestEnvironment.current.base_dir, name, Paths.MODELS_PATH_NAME, Paths.CONFIG_FILE_NAME) + if config_path is None + else config_path + ) + logger.info("Saving config file to {}, content:\n{}".format(config_path, config_json)) + + os.makedirs(os.path.dirname(config_path), exist_ok=True) + with open(os.path.join(config_path), "w") as outfile: + outfile.write(config_json) + + return Paths.CONFIG_PATH_INTERNAL + + @staticmethod + def save_without_encoding(config_path, config_dict: dict): + logger.info("Saving config file to {}, content:\n{}".format(config_path, str(config_dict))) + os.makedirs(os.path.dirname(config_path), exist_ok=True) + with open(os.path.join(config_path), "w") as outfile: + outfile.write(str(config_dict)) + + return Paths.CONFIG_PATH_INTERNAL + + @staticmethod + def build( + models: List[ModelInfo] = [], + pipelines: List[Pipeline] = None, + custom_nodes: List[CustomNode] = None, + metrics_enable=MetricsPolicy.NotDefined, + resource_dir=None, + mediapipe_models=None, + use_custom_graphs=False, + use_subconfig=False, + custom_graph_paths=None, + ) -> dict: + config = OvmsConfig.build_ovms_config( + models, + pipelines, + custom_nodes, + metrics_enable, + resource_dir, + mediapipe_models, + use_custom_graphs, + use_subconfig, + custom_graph_paths, + ) + return config + + @staticmethod + def build_ovms_config( + models: List[ModelInfo] = [], + pipelines: List[Pipeline] = None, + custom_nodes: List[CustomNode] = None, + metrics_enable=MetricsPolicy.NotDefined, + resource_dir=None, + mediapipe_models=None, + use_custom_graphs=False, + use_subconfig=False, + custom_graph_paths=None, + ) -> dict: + if use_subconfig: + config = {Config.MODEL_CONFIG_LIST: []} + else: + config = {Config.MODEL_CONFIG_LIST: [model.get_config() for model in models]} + if all([c is None for c in config[Config.MODEL_CONFIG_LIST]]): + config = {Config.MODEL_CONFIG_LIST: []} + if resource_dir and CurrentOvmsType.ovms_type in [ + OvmsType.CAPI, + OvmsType.BINARY, + ]: # Add `resource_dir` prefix to default `base_path` + for model in config[Config.MODEL_CONFIG_LIST]: + if not model["config"]["base_path"].startswith(resource_dir): + model_config_base_path = model["config"]["base_path"] + model["config"]["base_path"] = os.path.join( + resource_dir, + *model_config_base_path.split(os.path.sep)[1:] + ) + config_custom_nodes = [] + if pipelines is not None: + config[Config.PIPELINE_CONFIG_LIST] = [] + for pipeline in pipelines: + config, config_custom_nodes = pipeline.build_pipeline_config( + config, + custom_nodes, + config_custom_nodes, + models, + use_custom_graphs, + mediapipe_models, + use_subconfig, + custom_graph_paths, + ) + + if config_custom_nodes: + config[Config.CUSTOM_NODE_LIBRARY_CONFIG_LIST] = [ + custom_node.get_config() for custom_node in config_custom_nodes + ] + for model in config[Config.CUSTOM_NODE_LIBRARY_CONFIG_LIST]: + if ( + CurrentOvmsType.ovms_type in [OvmsType.CAPI, OvmsType.BINARY] + and resource_dir + and CurrentOvmsType.ovms_type in [OvmsType.CAPI, OvmsType.BINARY] + and not model["base_path"].startswith(resource_dir) + ): + model["base_path"] = os.path.join(resource_dir, f'./{model["base_path"]}') + + loader_configs = set([model.custom_loader.loader_config for model in models if model.custom_loader is not None]) + for loader in loader_configs: + if ( + resource_dir + and CurrentOvmsType.ovms_type in [OvmsType.CAPI, OvmsType.BINARY] + and not loader["config"]["library_path"].startswith(resource_dir) + ): + loader["config"]["library_path"] = os.path.join(resource_dir, f"./{loader['config']['library_path']}") + config[CustomLoader.PARENT_KEY] = list(loader_configs) + + if metrics_enable in [MetricsPolicy.EnabledInConfig, MetricsPolicy.Disabled]: + enabled = True if metrics_enable == MetricsPolicy.EnabledInConfig else False + config[Config.MONITORING] = {"metrics": {"enable": enabled}} + + # If MediaPipe pipelines (e.g. SimpleMediaPipe, ImageClassificationMediaPipe) are not defined, we use SimpleModelMediaPipe models + mediapipe_models = [model for model in models if model.is_mediapipe] + if not use_custom_graphs and len(mediapipe_models) > 0: + # Scenario for any model without specifying custom_graph_paths (e.g. test_mediapipe_various_models) + config = mediapipe_models[0].add_mediapipe_graphs_to_config(config, use_subconfig, mediapipe_models) + + if use_custom_graphs: + for model in mediapipe_models: + # Scenario with any graph path (custom_graph_paths) for model (e.g. test_mediapipe_dummy_basic__) + config = model.prepare_custom_graphs_mediapipe_config_list(config, use_subconfig, custom_graph_paths) + + return config + + @staticmethod + def load(config_path): + with open(config_path, "r") as f: + try: + config_json = f.read() + config_dict = json.loads(config_json) + except ValueError as e: + logger.error("Error while loading json: {}".format(config_json)) + raise e + return config_dict + + @staticmethod + def replace_config_models_paths_for_binary(context, config_path, resources_dir, name, **kwargs): + config_dict = OvmsConfig.load(config_path) + if config_dict is not None: + if kwargs.get("replace_config_models_paths_for_binary", True): + for model in config_dict[Config.MODEL_CONFIG_LIST]: + new_base_path = model["config"]["base_path"].replace( + Paths.MODELS_PATH_INTERNAL, os.path.join(resources_dir, Paths.MODELS_PATH_NAME), 1 + ) + model["config"]["base_path"] = new_base_path + if "graph_path" in model["config"]: + model["config"]["graph_path"] = model["config"]["graph_path"].replace( + Paths.MODELS_PATH_INTERNAL, os.path.join(resources_dir, Paths.MODELS_PATH_NAME), 1 + ) + + if kwargs.get("replace_config_custom_loader_paths_for_binary", True): + for custom_loader in config_dict.get(Config.CUSTOM_LOADER_CONFIG_LIST, ""): + custom_loader_library_path = custom_loader["config"]["library_path"] + new_library_path = os.path.join(resources_dir, Paths.CUSTOM_LOADER_PATH_NAME, + CustomLoaderConsts.SAMPLE_CUSTOM_LOADER_NAME, + CustomLoaderConsts.SAMPLE_CUSTOM_LOADER_LIB_NAME) + custom_loader["config"]["library_path"] = new_library_path + if context.base_os == OsType.Windows: + raise NotImplementedError("Custom resources are not implemented for Windows") + copy_custom_lib_to_host(context.ovms_test_image, custom_loader_library_path, new_library_path) + + if kwargs.get("replace_config_custom_nodes_paths_for_binary", True): + for custom_node in config_dict.get(Config.CUSTOM_NODE_LIBRARY_CONFIG_LIST, ""): + custom_node_library_path = custom_node["base_path"] + new_library_path = os.path.join(resources_dir, custom_node_library_path) + custom_node["base_path"] = new_library_path + if context.base_os == OsType.Windows: + raise NotImplementedError("Custom resources are not implemented for Windows") + copy_custom_lib_to_host(context.ovms_test_image, custom_node_library_path, new_library_path) + + if kwargs.get("replace_config_mediapipe_paths_for_binary", True): + for mediapipe_config in config_dict.get(Config.MEDIAPIPE_CONFIG_LIST, []): + if "graph_path" in mediapipe_config: + mediapipe_config["graph_path"] = mediapipe_config["graph_path"].replace( + Paths.MODELS_PATH_INTERNAL, os.path.join(resources_dir, Paths.MODELS_PATH_NAME), 1 + ) + if "base_path" in mediapipe_config: + mediapipe_config["base_path"] = mediapipe_config["base_path"].replace( + Paths.MODELS_PATH_INTERNAL, os.path.join(resources_dir, Paths.MODELS_PATH_NAME), 1 + ) + if "subconfig" in mediapipe_config: + mediapipe_config["subconfig"] = mediapipe_config["subconfig"].replace( + Paths.MODELS_PATH_INTERNAL, os.path.join(resources_dir, Paths.MODELS_PATH_NAME), 1 + ) + + OvmsConfig.save(name, config_dict) + + @staticmethod + def replace_subconfig_paths(name, subconfig_path, resources_dir): + subconfig_dict = json.loads(Path(subconfig_path).read_text()) + for i, model in enumerate(subconfig_dict["model_config_list"]): + subconfig_dict["model_config_list"][i]["config"]["base_path"] = model["config"]["base_path"].replace( + Paths.MODELS_PATH_INTERNAL, os.path.join(resources_dir, Paths.MODELS_PATH_NAME) + ) + OvmsConfig.save(name, subconfig_dict, subconfig_path) + logger.info("Subconfig paths replaced") + + @staticmethod + def create_subconfig(name, parameters, config_path_on_host): + config_dict = None + if parameters.custom_config is not None: + config_dict = parameters.custom_config + else: + config_path = Path(os.path.join(config_path_on_host, Paths.CONFIG_FILE_NAME)) + if config_path.exists(): + config_dict = json.loads(config_path.read_text()) + + subconfig_dict = {Config.MODEL_CONFIG_LIST: []} + mediapipe_model = [model for model in parameters.models if model.is_mediapipe][0] + feature_extraction_models = \ + [ + model for model in parameters.models + if hasattr(model, "is_feature_extraction") and model.is_feature_extraction + ] + rerank_models = \ + [ + model for model in parameters.models + if hasattr(model, "is_rerank") and model.is_rerank + ] + regular_models = [model for model in mediapipe_model.regular_models] + + filename = Paths.SUBCONFIG_FILE_NAME + subconfigs = [os.path.basename(elem.get("subconfig", "")) + for elem in config_dict[Config.MEDIAPIPE_CONFIG_LIST]] if config_dict is not None else [] + if Paths.SUBCONFIG_FILE_NAME in subconfigs: + for model in regular_models: + subconfig_dict[Config.MODEL_CONFIG_LIST].append(model.get_config()) + else: + for model in regular_models: + subconfig_dict[Config.MODEL_CONFIG_LIST].append(model.get_config()) + filename = ( + f"subconfig_{model.name}.json" + if config_dict is not None and all(["subconfig" in elem + for elem in config_dict[Config.MEDIAPIPE_CONFIG_LIST]]) + else Paths.SUBCONFIG_FILE_NAME + ) + mediapipe_resources_path = os.path.join(config_path_on_host, mediapipe_model.name) + Path(mediapipe_resources_path).mkdir(parents=True, exist_ok=True) + subconfig_path = os.path.join(mediapipe_resources_path, filename) + OvmsConfig.save(name, subconfig_dict, subconfig_path) + return subconfig_dict, subconfig_path diff --git a/tests/functional/object_model/ovms_docker.py b/tests/functional/object_model/ovms_docker.py index a72bcb4da9..2796e0b115 100644 --- a/tests/functional/object_model/ovms_docker.py +++ b/tests/functional/object_model/ovms_docker.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 Intel Corporation +# Copyright (c) 2026 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,29 +14,726 @@ # limitations under the License. # -import tests.functional.config as config -from tests.functional.command_wrappers.server import start_ovms_container_command -from tests.functional.object_model.docker import Docker -from tests.functional.utils.parametrization import generate_test_object_name - - -class OvmsDocker(Docker): - def __init__(self, request, command_args, container_name_infix, start_container_command, - env_vars=None, image=config.image, container_log_line=config.container_log_line, - server_log_level=config.log_level, target_device=None, server=None): - self.command_args = command_args - self.container_name_infix = container_name_infix - self.server_log_level = server_log_level - container_name_prefix = image.split(":")[0].split("/")[-1] - container_name = generate_test_object_name(prefix="{}-{}".format(container_name_prefix, container_name_infix)) - super().__init__(request, container_name, start_container_command, - env_vars, image, container_log_line, server) - self.command_args["port"] = self.grpc_port - self.command_args["rest_port"] = self.rest_port - self.command_args["log_level"] = self.server_log_level - if target_device: - self.command_args["target_device"] = target_device - self.start_container_command = start_ovms_container_command(self.start_container_command, self.command_args) - - def start(self): - return super().start() +import os +import re +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +import docker + +from tests.functional.utils.context import Context +from tests.functional.utils.inference.communication import REST +from tests.functional.utils.logger import get_logger +from tests.functional.utils.process import Process +from tests.functional.utils.test_framework import generate_test_object_name +from tests.functional.object_model.ovms_command import OvmsCommand, create_ovms_command +from tests.functional.constants.core import CONTAINER_STATUS_DEAD, CONTAINER_STATUS_EXITED, CONTAINER_STATUS_RUNNING +from tests.functional.constants.ovms import Config, Ovms +from tests.functional.constants.ovms_type import OvmsType +from tests.functional.constants.paths import Paths +from tests.functional.constants.pipelines import MediaPipe +from tests.functional.constants.target_device_configuration import ( + DEVICES, + DOCKER_PARAMS, + HOST, + NETWORK, + PRIVILEGED, + TARGET_DEVICE_CONFIGURATION, + VOLUMES, +) +from tests.functional.utils.docker import DockerContainer, Limits +from tests.functional.object_model.mediapipe_calculators import MediaPipeCalculator +from tests.functional.object_model.ovms_config import OvmsConfig +from tests.functional.object_model.ovms_instance import OvmsInstance +from tests.functional.object_model.ovms_log_monitor import OvmsCmdLineDockerLogMonitor, OvmsDockerLogMonitor +from tests.functional.object_model.ovms_mapping_config import OvmsMappingConfig +from tests.functional.object_model.ovms_params import OvmsParams +from tests.functional.object_model.ovsa import OvsaCerts +from tests.functional.object_model.test_environment import TestEnvironment +from tests.functional.object_model.ovms_info import OvmsInfo + +logger = get_logger(__name__) + + +@dataclass(frozen=False) +class OvmsDockerParams(OvmsParams): + detach: bool = True + limits: Limits = None + privileged: bool = False + ovsa_certs: OvsaCerts = None + process_params: Callable[[dict], None] = None + volumes: dict = None + network: str = None + + +class OvmsDockerLauncher(object): + + @classmethod + def _update_ports(cls, context, port, parameters, ovms_instance_params, full_cmd): + if port == parameters.grpc_port: + parameters.grpc_port = context.port_manager_grpc.get_port() + updated_port = parameters.grpc_port + context.port_manager_grpc.release_port(port) + ovms_instance_params["instance_kwargs"]["grpc_port"] = updated_port + elif port == parameters.rest_port: + parameters.rest_port = context.port_manager_rest.get_port() + updated_port = parameters.rest_port + context.port_manager_rest.release_port(port) + ovms_instance_params["instance_kwargs"]["rest_port"] = updated_port + logger.info(f"Updating ports for OVMS docker. Port {port} was reserved. Trying port {updated_port}") + ovms_instance_params["command"] = re.sub(rf"{port}", f"{updated_port}", ovms_instance_params["command"]) + del ovms_instance_params["docker_kwargs"]["ports"][f"{port}/tcp"] + ovms_instance_params["docker_kwargs"]["ports"][f"{updated_port}/tcp"] = updated_port + ovms_instance_params["name"] = f"{ovms_instance_params['name']}_{updated_port}" + full_cmd = re.sub(rf"{port}", f"{updated_port}", full_cmd) + cls._log_create_docker(ovms_instance_params, full_cmd) + + @staticmethod + def _log_create_docker(ovms_instance_params, full_cmd): + logger.info( + f"Docker info\nContainer {ovms_instance_params['name']}\nimage: {ovms_instance_params['image']}\n" + f"command: {ovms_instance_params['command']}" + ) + logger.info(f"Docker kwargs: {ovms_instance_params['docker_kwargs']}") + logger.info(f"Instance kwargs: {ovms_instance_params['instance_kwargs']}") + logger.info(f"Running cmd: {full_cmd}") + + @classmethod + def create( + cls, + context: Context, + parameters: OvmsDockerParams, + ovms_docker_type, + environment, + entrypoint=None, + entrypoint_params=None, + ovms_instance_params=None, + ): + if parameters.models is not None: + logger.info( + "Creating ovms with model(s): {}".format(", ".join([model.name for model in parameters.models])) + ) + if parameters.name is None: + parameters.name = ( + context.test_object_name if context.test_object_name is not None else generate_test_object_name() + ) + if parameters.target_device is None and not parameters.single_mediapipe_model_mode: + # Batch_size and target_device are not supported for single model mode in MediaPipe + parameters.target_device = context.target_device + if parameters.image is None: + parameters.image = context.ovms_image + + if ovms_instance_params is None: + ovms_instance_params = cls.build_ovms_instance_params(context, parameters) + if parameters.process_params is not None: + parameters.process_params(ovms_instance_params) + + if isinstance(ovms_instance_params["command"], OvmsCommand): + ovms_instance_params["command"] = str(ovms_instance_params["command"]) + + full_cmd = parse_cmd(ovms_instance_params, environment, entrypoint, entrypoint_params) + + cls._log_create_docker(ovms_instance_params, full_cmd) + + target_device_lock = OvmsInstance.acquire_target_device_lock(parameters.target_device) + + if ovms_docker_type == OvmsType.DOCKER_CMD_LINE: + process = Process() + docker_id = process.run_and_check(full_cmd) + ovms_docker = OvmsCmdLineDockerInstance( + docker_id.strip(), + ovms_instance_params["name"], + ovms_instance_params["instance_kwargs"]["container_folder"], + ovms_instance_params["instance_kwargs"]["rest_port"], + ovms_instance_params["instance_kwargs"]["grpc_port"], + target_device=ovms_instance_params["instance_kwargs"]["target_device"], + lock_file=target_device_lock, + ) + else: + kwargs = {"environment": environment, "entrypoint": entrypoint} + if context.terminate_signal_type is not None: + kwargs["stop_signal"] = context.terminate_signal_type + container = None + while container is None: + try: + container = DockerContainer(None).client.run( + ovms_instance_params["image"], + ovms_instance_params["command"], + stdout=True, + stderr=False, + remove=False, + detach=parameters.detach, + name=ovms_instance_params["name"], + **ovms_instance_params["docker_kwargs"], + **kwargs, + ) + except docker.errors.APIError as e: + match = re.search(r"\d\.\d\.\d\.\d\:(\d{4,5})", e.explanation) + if match: + # in case port is already allocated choose new port + port = int(match.group(1)) + cls._update_ports(context, port, parameters, ovms_instance_params, full_cmd) + else: + raise e + docker_container = DockerContainer( + container, + ovms_instance_params["command"], + parameters.detach, + **ovms_instance_params["docker_kwargs"], + **kwargs, + ) + ovms_docker = OvmsDockerInstance( + context=context, + container=docker_container, + lock_file=target_device_lock, + **ovms_instance_params["instance_kwargs"], + ) + + ovms_docker._dmesg_log.get_all_logs() + context.test_objects.append(ovms_docker) + return ovms_docker + + @classmethod + def _prepare_ports(cls, grpc_port: int = None, rest_port: int = None) -> dict: + ports = dict() + if grpc_port is not None: + ports.update({f"{grpc_port}/tcp": grpc_port}) + if rest_port is not None: + ports.update({f"{rest_port}/tcp": rest_port}) + return ports + + @classmethod + def prepare_models_mapping(cls, context, ovms_container, models): + if not models: + return + for model in models: + mapping_exists = OvmsMappingConfig.mapping_exists(ovms_container, model) + if model.use_mapping is None: + continue # Use default mapping.json (if exists) do not touch nor modify. + if model.use_mapping is True: + if mapping_exists: + # Delete original mapping since it is tested in case: `default_model_mapping` + OvmsMappingConfig.delete_mapping(model) + OvmsMappingConfig.generate(model, context) # create generic mapping + if model.use_mapping is False: + if mapping_exists: + OvmsMappingConfig.delete_mapping(model) # just delete mapping + + @classmethod + def build_ovms_instance_params(cls, context: Context, parameters: OvmsDockerParams): + name = parameters.name + regular_models = parameters.get_regular_models() + container_folder, models_dir_on_host = TestEnvironment.current.prepare_container_folders( + name, parameters.get_models() + ) + volumes = parameters.volumes if parameters.volumes is not None else \ + cls.prepare_new_volumes_for_container(models_dir_on_host) + if parameters.use_cache: + if parameters.cache_dir_path is None: + cache_dir_on_host = os.path.join(container_folder, "../cache") + else: + cache_dir_on_host = parameters.cache_dir_path + cls.prepare_volume_for_cache(cache_dir_on_host, volumes) + + if parameters.allowed_local_media_path is not None: + cls.prepare_volume_for_images(parameters.allowed_local_media_path, volumes) + + cls.prepare_models_mapping(context, container_folder, parameters.models) + + config_path_on_host, config_path = cls.create_config(parameters, name) + + config_file = None + config_data = "" + if config_path_on_host is not None: + if parameters.volumes is None: + volumes.update(cls.prepare_new_volumes_for_container([config_path_on_host])) + config_file = os.path.join(config_path_on_host, Paths.CONFIG_FILE_NAME) + config_data = Path(config_file).read_text() + + # Create and save .pbtxt file for each model in pipeline + if ( + parameters.custom_config is not None + and parameters.custom_config.get(Config.MEDIAPIPE_CONFIG_LIST) is not None + or Config.MEDIAPIPE_CONFIG_LIST in config_data + or "mediapipe" in config_data + or (not parameters.use_config and parameters.single_mediapipe_model_mode) + ): + MediaPipeCalculator.prepare_proto_calculator(parameters, config_path_on_host, config_file) + + assert not ( + parameters.shape and len(regular_models) > 1 + ), "Forbidden to pass more than one model and `shape` or `batch_size` parameters." + + cpu_extension_path = None + if parameters.cpu_extension: + cpu_extension_path = parameters.cpu_extension.lib_path + + OvmsInfo.get_local_image(parameters.image) + image = OvmsInfo.IMAGES[parameters.image] + + target_device_is_valid = parameters.target_device in TARGET_DEVICE_CONFIGURATION + + extra_docker_params = {} + batch_size, shape = None, None + model_name, model_path = parameters.model_name, parameters.model_path + pull = parameters.pull + task = parameters.task + task_params = parameters.task_params + list_models = parameters.list_models + overwrite_models = parameters.overwrite_models + add_to_config = parameters.add_to_config + remove_from_config = parameters.remove_from_config + resolution = parameters.resolution + cache_size = parameters.cache_size + source_model, model_repository_path, gguf_filename = None, None, None + extra_docker_params.update({"user": f"{os.getuid()}:{os.getgid()}"}) + model = None + if config_path is None and regular_models: + model = regular_models[0] + if model.is_hf_direct_load: + source_model = parameters.source_model if parameters.source_model is not None else model.name + model_repository_path = parameters.model_repository_path \ + if parameters.model_repository_path is not None else Paths.MODELS_PATH_INTERNAL + if model.gguf_filename: + gguf_filename = model.gguf_filename + model_name = model.name + model_path = None + volumes = parameters.volumes if parameters.volumes is not None else \ + cls.prepare_new_volumes_for_container(models_dir_on_host, mode="rw") + else: + # Batch_size and target_device are not supported for single model mode in MediaPipe + batch_size = model.batch_size if not parameters.single_mediapipe_model_mode else None + shape = model.input_shape_for_ovms + if model_name is None: + model_name = model.name + if model_path is None: + model_path = model.get_model_path() + elif list_models is not None: + model_repository_path = parameters.model_repository_path \ + if parameters.model_repository_path is not None else Paths.MODELS_PATH_INTERNAL + source_model, model_name, model_path = None, None, None + elif add_to_config: + source_model = None + model_name = parameters.model_name + model_path = parameters.model_path + if model_path is None: + model_repository_path = parameters.model_repository_path \ + if parameters.model_repository_path is not None else Paths.MODELS_PATH_INTERNAL + else: + model_repository_path = None + config_path = Paths.CONFIG_PATH_INTERNAL + elif remove_from_config: + source_model, model_repository_path, model_path = None, None, None + model_name = parameters.model_name + config_path = Paths.CONFIG_PATH_INTERNAL + + ports = cls._prepare_ports(parameters.grpc_port, parameters.rest_port) + if target_device_is_valid and not parameters.check_version: + # Note: currently we use lambda: expression for performing 'lazy init' of syscalls: + # getuid() & getgrnam('users')/getgrnam('render') + target_device_conf = TARGET_DEVICE_CONFIGURATION[parameters.target_device]() + devices = target_device_conf[DEVICES] + network = parameters.network if parameters.network is not None else target_device_conf[NETWORK] + if DOCKER_PARAMS in target_device_conf: + extra_docker_params.update(target_device_conf[DOCKER_PARAMS]) + privileged = parameters.privileged or target_device_conf[PRIVILEGED] + volumes = cls.update_volume_for_mounts(target_device_conf[VOLUMES], volumes) + else: # required for negative test cases + devices = [] + network = parameters.network + privileged = parameters.privileged + + ovsa_certs = parameters.ovsa_certs if parameters.ovsa_certs is not None else OvsaCerts.default_certs + if ovsa_certs is not None: + volumes.update(ovsa_certs.create_ovsa_volume_bindings()) + if parameters.custom_command is None: + command = create_ovms_command( + config_path=config_path, + model_path=model_path, + model_name=model_name, + parameters=parameters, + cpu_extension_path=cpu_extension_path, + batch_size=batch_size, + shape=shape, + ovms_type=OvmsType.DOCKER, + base_os=context.base_os, + pull=pull, + source_model=source_model, + gguf_filename=gguf_filename, + model_repository_path=model_repository_path, + task=task, + task_params=task_params, + list_models=list_models, + overwrite_models=overwrite_models, + add_to_config=add_to_config, + remove_from_config=remove_from_config, + resolution=resolution, + cache_size=cache_size, + pooling=model.pooling if model is not None else None, + ) + else: + command = parameters.custom_command + + docker_kwargs = dict(volumes=volumes, devices=devices, network=network, privileged=privileged) + docker_kwargs.update(extra_docker_params) + + instance_kwargs = dict( + container_folder=container_folder, + rest_port=parameters.rest_port, + grpc_port=parameters.grpc_port, + target_device=parameters.target_device, + ) + if parameters.limits: + docker_kwargs.update(parameters.limits) + + if network != HOST: + # this sends ports to docker api create/run methods + docker_kwargs["ports"] = ports + + result = { + "image": image, + "command": command, + "name": name, + "ports": "ports", + "docker_kwargs": docker_kwargs, + "instance_kwargs": instance_kwargs, + } + return result + + @classmethod + def create_config(cls, parameters, name): + regular_models = parameters.get_regular_models() + using_custom_loader = any([(x.custom_loader is not None) for x in regular_models]) + if using_custom_loader and not parameters.use_config: + msg = f"Custom loader is supported only with config file passed with --config_path." + logger.error(msg) + raise Exception(msg) + + config_dir_path_on_host = os.path.join(TestEnvironment.current.base_dir, name, Paths.MODELS_PATH_NAME) + if parameters.create_config_method is not None: + parameters.create_config_method(os.path.join(config_dir_path_on_host, Paths.CONFIG_FILE_NAME)) + use_config = True + elif parameters.custom_config is not None: + OvmsConfig.save(name, parameters.custom_config) + use_config = True + else: + # Automatically set use_config if the following conditions are met + use_config = ( + len(regular_models) > 1 + or parameters.use_config + or any((isinstance(model, MediaPipe) and not model.single_mediapipe_model_mode) + for model in regular_models) + ) + if use_config: + OvmsConfig.generate_from_parameters(name, parameters) + + if parameters.use_subconfig: + OvmsConfig.create_subconfig(name, parameters, config_dir_path_on_host) + + if use_config: + return config_dir_path_on_host, Paths.CONFIG_PATH_INTERNAL + else: + return None, None + + @staticmethod + def prepare_new_volumes_for_container(container_folders, mode="ro"): + """ + Prepare dictionary with instruction how docker should map those locations. + + Parameters: + container_folders (set(str)): Set of path located on host that should be mapped to docker. + + Returns: + dict: Instruction how to map host directories to docker instance. + """ + result = {} + for resource in container_folders: + base_dir = os.path.basename(resource) + if "_mediapipe" in base_dir: + # Single model mode in MediaPipe + result[resource] = { + "bind": os.path.join(Paths.OVMS_PATH_INTERNAL, Paths.MODELS_PATH_NAME, base_dir), "mode": mode, + } + else: + result[resource] = {"bind": os.path.join(Paths.OVMS_PATH_INTERNAL, base_dir), "mode": mode} + + return result + + @staticmethod + def prepare_volume_for_cache(cache_dir_on_host, volumes): + oldmask = os.umask(0) + try: + os.makedirs(cache_dir_on_host, mode=0o777, exist_ok=True) + finally: + os.umask(oldmask) + volumes.update({cache_dir_on_host: {"bind": Paths.CACHE_INTERNAL, "mode": "rw"}}) + + @staticmethod + def prepare_volume_for_images(images_path, volumes): + volumes.update({images_path: {"bind": Paths.IMAGES_PATH_INTERNAL, "mode": "rw"}}) + + @staticmethod + def update_volume_for_mounts(mounts: list, volumes: dict): + for mount in mounts: + volumes.update(mount) + return volumes + + +def parse_cmd(result, environment, entrypoint, entrypoint_params): + + full_cmd = f"docker run -d" + if result["docker_kwargs"]["privileged"]: + full_cmd += " --privileged" + + if result["docker_kwargs"]["network"]: + full_cmd += f" --network {result['docker_kwargs']['network']}" + + if result["docker_kwargs"].get("user", None): + full_cmd += f" --user {result['docker_kwargs']['user']}" + + for cgroup_rule in result["docker_kwargs"].get("device_cgroup_rules", []): + full_cmd += f" --device-cgroup-rule '{cgroup_rule}'" + + for device in result["docker_kwargs"]["devices"]: + full_cmd += f" --device {device}" + + for group_add in result["docker_kwargs"].get("group_add", []): + full_cmd += f" --group-add {group_add}" + + for container_port, host_port in result["docker_kwargs"]["ports"].items(): + port = container_port.partition("/")[0] + full_cmd += f" -p {host_port}:{port}" + + env_str = "" + if environment: + for key, value in environment.items(): + env_str += f" -e {key}='{value}'" + + full_cmd += env_str + + for volume, bind_info in result["docker_kwargs"]["volumes"].items(): + full_cmd += f" -v {volume}:{bind_info['bind']}:{bind_info['mode']}" + + if entrypoint is not None: + full_cmd += f" --entrypoint {entrypoint}" + + full_cmd += " " + result["image"].tags[0] + + if entrypoint_params is not None: + result["command"] = f"{entrypoint_params} {result['command']}" + + # '/ovms/bin/ovms --log_level INFO --port 9007 --rest_port 8005 --config_path /models/config.json' + if entrypoint is not None: + full_cmd += " " + result["command"] if type(result["command"]) == str else " " + " ".join(result["command"]) + elif "/ovms/bin/ovms" in result["command"]: + result["command"] = result["command"].partition("/ovms/bin/ovms")[2].strip() + full_cmd += " " + result["command"] + else: + full_cmd += " " + " ".join(result["command"]) + return full_cmd + + +class OvmsDockerInstance(OvmsInstance): + + def __init__(self, container, container_folder, rest_port, grpc_port, target_device, **kwargs): + if container is not None: + self.container = container + self.name = container.container.name + super().__init__( + name=self.name, + container_folder=container_folder, + default_logger=OvmsDockerLogMonitor(container.container), + rest_port=rest_port, + grpc_port=grpc_port, + target_device=target_device, + **kwargs, + ) + + def fetch_and_store_ovms_pid(self, timeout=60): + docker_client = docker.APIClient() + docker_inspect_output = docker_client.inspect_container(self.get_short_id()) + self.ovms_pid = docker_inspect_output["State"]["Pid"] + super().fetch_and_store_ovms_pid() + logger.info(f"OVMS Process ID = [{self.ovms_pid}]") + return self.ovms_pid + + def is_ovms_running(self): + return self.get_status() not in [CONTAINER_STATUS_EXITED, CONTAINER_STATUS_DEAD] + + def get_ip(self): + return self.container.container.attrs["NetworkSettings"]["IPAddress"] + + def get_rest_port(self): + return self.ovms_ports[REST] + + def get_status(self, status=None, timeout=None): + return self.container.get_status() + + def _create_logger(self): + return OvmsDockerLogMonitor(self.container.container) + + def execute_command(self, cmd, stream=False, cwd=None, workdir=None): + exit_code, output = self.container.container.exec_run(user="root", cmd=cmd, stream=stream, workdir=workdir) + stdout = output if stream else output.decode() + return exit_code, stdout + + def get_short_id(self): + return self.container.container.short_id + + def cleanup(self): + if not self.container.deleted: + try: + super().cleanup() + except ValueError as e: + print(f"Error occurred during cleanup:\n{e}") + finally: + try: + logger.info(f"Removing container {self.container.container.short_id}") + self.container.cleanup() + logger.info("Container removed") + self.release_ports() + self._dmesg_log.raise_on_unexpected_messages(filter_known_messages=True) + except Exception as e: + logger.info(f"Cleanup triggered exception: {str(e)}") + raise e + + def start(self, ensure_started=False, *args, **kwargs): + self.container.start_container() + if ensure_started: + self.ensure_started(*args, **kwargs) + + def stop_ovms( + self, context, ensure_deleted=False, terminate_signal_type=Ovms.SIGTERM_SIGNAL, remove_container=True + ): + context.terminate_signal_type = ( + terminate_signal_type if context.terminate_signal_type is None else context.terminate_signal_type + ) + signal = self.get_signal_type(context.terminate_signal_type) + if context.terminate_method == Ovms.STOP_METHOD: + self.container.stop_container() + else: + self.container.kill_container(signal=signal) + if remove_container: + self.container.remove_container(ensure_deleted) + self.container._set_deleted(True) + + @staticmethod + def from_docker_id(docker_id, **kwargs): + client = docker.from_env() + container = client.containers.get(docker_id) + docker_container = DockerContainer(container) + return OvmsDockerInstance(docker_container, "", None, None, None, **kwargs) + + +class OvmsCmdLineDockerInstance(OvmsInstance): + + def __init__(self, docker_id, name, container_folder, rest_port, grpc_port, target_device, **kwargs): + self.docker_id = docker_id + self._disposed = False + super().__init__( + name=name, + container_folder=container_folder, + default_logger=OvmsCmdLineDockerLogMonitor(self.docker_id), + rest_port=rest_port, + grpc_port=grpc_port, + target_device=target_device, + **kwargs, + ) + + def fetch_and_store_ovms_pid(self, timeout=60): + self._dmesg_log.ovms_pid = None # Not implemetned yet + + def _create_logger(self): + return OvmsCmdLineDockerLogMonitor(self.docker_id) + + def ensure_status(self, status: str = CONTAINER_STATUS_RUNNING, timeout: int = 60): + process = Process() + short_id = self.get_short_id() + timeout = time.time() + timeout + current_status = None + while True: + try: + _, _stdout, _ = process.run(f"docker ps --filter id={short_id}") + if short_id in _stdout: + current_status = CONTAINER_STATUS_RUNNING + else: + current_status = CONTAINER_STATUS_EXITED + if current_status == status: + break + except AssertionError: + pass + if time.time() > timeout: + raise TimeoutError(f"Current status: {current_status}") + return current_status + + def get_status(self, status=None, timeout=60): + return self.ensure_status(status, timeout=timeout) + + def stop_container(self, signal=Ovms.SIGTERM_SIGNAL): + process = Process() + process.run(f"docker stop --signal {signal} {self.docker_id}") + + def kill_container(self, signal=Ovms.SIGKILL_SIGNAL): + process = Process() + process.run(f"docker kill --signal {signal} {self.docker_id}") # SIGTERM, SIGINT; default: SIGKILL + + def remove_container(self, ensure_deleted: bool = False): + process = Process() + process.run(f"docker rm {self.docker_id}") + if ensure_deleted: + timeout = time.time() + 60 + while True: + short_id = self.get_short_id() + _, stdout, _ = process.run(f"docker ps --filter id={self.get_short_id()}") + if short_id not in stdout: + break + elif time.time() > timeout: + raise TimeoutError(f"Container {short_id} is not removed") + + def execute_command(self, cmd, stream=False, cwd=None): + process = Process() + if stream: + detach = "-d" + else: + detach = "" + exit_code, stdout, stderr = process.run(f"docker exec {detach} -u root {self.docker_id} {cmd}", cwd=cwd) + return exit_code, stdout + + def get_env_variables(self): + _, printenv_output = self.execute_command("env") + output = {} + if printenv_output: + for line in printenv_output.split(): + env_variable = line.split("=") + output[env_variable[0]] = env_variable[1] + return output + + def get_short_id(self): + return self.docker_id[0:11] + + def cleanup(self): + if not self._disposed: + try: + super().cleanup() + finally: + try: + process = Process() + process.run_and_check(f"docker kill {self.docker_id}") + self.release_ports() + except Exception as e: + logger.info(str(e)) + + self._disposed = True + + def start(self, ensure_started=False, *args, **kwargs): + pass + + def stop_ovms(self, context, ensure_deleted=False): + signal = self.get_signal_type(context.terminate_signal_type) + if context.terminate_method == Ovms.STOP_METHOD: + self.stop_container(signal=signal) + else: + self.kill_container(signal=signal) + self.remove_container(ensure_deleted) diff --git a/tests/functional/object_model/ovms_info.py b/tests/functional/object_model/ovms_info.py new file mode 100644 index 0000000000..237223118e --- /dev/null +++ b/tests/functional/object_model/ovms_info.py @@ -0,0 +1,223 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +import docker +import re + +from tests.functional.utils.environment_info import DEFAULT_FULL_VERSION_NUMBER, BaseInfo +from tests.functional.utils.logger import get_logger +from tests.functional.constants.os_type import get_host_os_details +from tests.functional.utils.process import Process + +from tests.functional.config import airplane_mode, base_os, ovms_image_local +from tests.functional.config import tmp_dir +from tests.functional.constants.ovms_binaries import get_binaries, get_ovms_binary_cmd_setup +from tests.functional.constants.ovms_type import OvmsType, OVMS_BINARY_PACKAGE_EXTENSIONS + +logger = get_logger(__name__) + + +class OvmsInfo(BaseInfo): + """Retrieves OVMS version and os distname from container.""" + + _os_distname = None + _ovms_version = None + _ov_version = None + _ov_genai_version = None + _info_read = False + _docker_ovms_types = [OvmsType.DOCKER, OvmsType.DOCKER_CMD_LINE] + + IMAGES = {} + + @property + def os_distname(self): + """This method gets operating system distribution name from given container/binary name.""" + try: + if self.image is not None and self.image.endswith(OVMS_BINARY_PACKAGE_EXTENSIONS): + self._get_info_from_binary() + else: + self._get_info_from_container() + return self._os_distname + except Exception as exc: + logger.error(f"Couldn't retrieve OVMS os distname from.\nException: {exc}") + raise exc + + @property + def version(self): + """This method gets OVMS version from given container/binary name.""" + try: + if self.image is not None and self.image.endswith(OVMS_BINARY_PACKAGE_EXTENSIONS): + self._get_info_from_binary() + else: + self._get_info_from_container() + return f"{self._ovms_version}_OV{self._ov_version}_genAI{self._ov_genai_version}" + except Exception as exc: + logger.error(f"Couldn't retrieve OVMS version from.\nException: {exc}") + raise exc + + def get_ovms_version(self, image=None): + try: + if image is not None and image.endswith(OVMS_BINARY_PACKAGE_EXTENSIONS): + self._get_info_from_binary() + else: + self._get_info_from_container(image=image) + return self._ovms_version + except Exception as exc: + logger.error(f"Couldn't retrieve OVMS version from.\nException: {exc}") + raise exc + + def get_ovms_ov_version(self, image=None): + try: + if image is not None and image.endswith(OVMS_BINARY_PACKAGE_EXTENSIONS): + self._get_info_from_binary() + else: + self._get_info_from_container(image=image) + return self._ovms_version, self._ov_version, self._ov_genai_version + except Exception as exc: + logger.error(f"Couldn't retrieve OVMS version from.\nException: {exc}") + raise exc + + @classmethod + def get(cls): + """This method gets OVMS version in dict format.""" + + try: + cls._get_info_from_container() + return {"version": cls.version()} + except Exception as exc: + logger.warning(f"Couldn't retrieve OVMS version from.\nException: {exc}") + return {"version": DEFAULT_FULL_VERSION_NUMBER} + + def _get_info_from_container(self, image=None): + """Run container and get all desired information.""" + from tests.functional.utils.docker import DockerClient # pylint: disable=import-outside-toplevel + + if self._info_read: + return + + if airplane_mode or ovms_image_local or image is not None: + image = image if image is not None else self.image + self.get_local_image(image) + else: + image = self.image + self.docker_pull_image_cli(image) + + client = DockerClient() + ovms_container = client.create( + image=image, + entrypoint="sleep", + command="inf", + ) + + try: + ovms_container.start() + cmd = "/ovms/bin/ovms --version" + exit_code, version_output = ovms_container.exec_run(cmd=cmd) + logger.info(f"Container version output: {version_output}") + assert all([exit_code == 0, "OpenVINO Model Server" in str(version_output)]), ( + f"Failed to run cmd: {cmd}; " f"exit_code: {exit_code}; " f"output: {version_output}" + ) + _ovms_version_string = version_output.decode().strip() + + self._ovms_version = re.search(r"Model Server\s+(\d[^\r\n]+)", _ovms_version_string).group(1).strip() + self._ov_version = re.search(r"OpenVINO backend\s+([^\r\n]+)", _ovms_version_string).group(1).strip() + match_ov_genai_version = re.search(r"OpenVINO GenAI backend\s+([^\r\n]+)", _ovms_version_string) + self._ov_genai_version = match_ov_genai_version.group(1).strip() if match_ov_genai_version else \ + "Not specified" + self._ovms_build_flags = re.search(r"Bazel build flags:\s*([^\r\n]+)", _ovms_version_string).group(1).strip() + + # Get ovms container os distname + exit_code, os_release = ovms_container.exec_run(cmd=["bash", "-c", "cat /etc/*-release"]) + os_release = os_release.decode("utf-8") + os_distname_regex = re.compile("^PRETTY_NAME=") + + os_distname = None + for line in os_release.splitlines(): + os_distname_match = os_distname_regex.match(line) + if os_distname_match: + os_distname = line[os_distname_match.end() :].strip("\"'") + break + self._os_distname = os_distname + + self._info_read = True + except Exception as exc: + err_msg = str(getattr(exc, "args", [""])) + logger.error( + "Couldn't retrieve OVMS info by running container: " f"\nException: {exc}; message: {err_msg};" + ) + raise exc + finally: + ovms_container.remove(force=True) + + def _get_info_from_binary(self): + self._os_distname = get_host_os_details() + path_to_binary_ovms, _ = get_binaries(base_os[0], "OvmsInfo_ovms_version", tmp_dir) + proc = Process() + proc.disable_check_stderr() + pre_cmd, env = get_ovms_binary_cmd_setup(base_os[0], resources_dir_path=os.path.dirname(path_to_binary_ovms)) + cmd = f"{pre_cmd}{path_to_binary_ovms} --version" + _, stdout, _ = proc.run_and_check_return_all(cmd, env=env) + match_ovms_version = re.search(r"Model Server\s+(\d[^\r\n]+)", stdout) + if match_ovms_version is not None: + self._ovms_version = match_ovms_version.group(1).strip() + else: + self._ovms_version = "Not specified" + match_ov_version = re.search(r"OpenVINO backend\s+([^\r\n]+)", stdout) + if match_ov_version is not None: + self._ov_version = match_ov_version.group(1).strip() + else: + self._ov_version = "Not specified" + match_ov_genai_version = re.search(r"OpenVINO GenAI backend\s+([^\r\n]+)", stdout) + if match_ov_genai_version is not None: + self._ov_genai_version = match_ov_genai_version.group(1).strip() + else: + self._ov_genai_version = "Not specified" + match_ovms_build_flags = re.search(r"Bazel build flags:\s*([^\r\n]+)", stdout) + if match_ovms_build_flags is not None: + self._ovms_build_flags = match_ovms_build_flags.group(1).strip() + else: + self._ovms_build_flags = "Not specified" + + @classmethod + def get_local_image(cls, local_image): + image = docker.from_env().images.get(local_image) + cls.IMAGES[local_image] = image + return cls.IMAGES[local_image] + + @staticmethod + def docker_pull_image_cli(image_to_pool): + """ + Execute command line command for docker image downloading. + This method is required to be run prior `pull_latest_image` for newly used images (unique tag). + """ + logger.info(f"Pulling image: {image_to_pool}") + proc = Process() + proc.run_and_check(f"docker pull {image_to_pool}") + + @classmethod + def pull_latest_image(cls, image_to_pull, force_pull=False): + cls.docker_pull_image_cli(image_to_pull) # ensure image is available on host. + + from tests.functional.utils.docker import DockerClient # pylint: disable=import-outside-toplevel + + if image_to_pull not in cls.IMAGES or force_pull: + repository, tag = image_to_pull.split(":") + logger.info("Pulling image: {} tag: {}".format(repository, tag)) + image = DockerClient().pull(repository=repository, tag=tag) + cls.IMAGES[image_to_pull] = image + return cls.IMAGES[image_to_pull] diff --git a/tests/functional/object_model/ovms_instance.py b/tests/functional/object_model/ovms_instance.py new file mode 100644 index 0000000000..eecf2ea112 --- /dev/null +++ b/tests/functional/object_model/ovms_instance.py @@ -0,0 +1,544 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import random +import re +import shutil +import signal +from abc import ABC, abstractmethod +from dataclasses import dataclass +from datetime import datetime, timedelta +from time import sleep +from typing import List + +from docker.errors import APIError + +import tests.functional.utils.assertions as assertions_module +from tests.functional.utils.assertions import ( + CPP_STD_EXCEPTION, + DmesgError, + DockerCannotCloseProperly, + OvmsTestException, + UnwantedMessageError, + get_exception_by_ovms_log, +) +from tests.functional.utils.context import Context +from tests.functional.utils.core import SelfDeletingFileLock +from tests.functional.utils.core import get_children_from_module +from tests.functional.utils.inference.communication import GRPC, REST +from tests.functional.utils.logger import get_logger +from tests.functional.constants.os_type import OsType +from tests.functional.utils.port_manager import PortManager +from tests.functional.utils.process import Process +from tests.functional.utils.test_framework import change_dir_permissions, is_single_threaded +from tests.functional.config import ( + artifacts_dir, + container_proxy, + disable_dmesg_log_monitor, + machine_is_reserved_for_test_session, + wait_for_messages_timeout, +) +from tests.functional.constants.core import CONTAINER_STATUS_EXITED, CONTAINER_STATUS_RUNNING +from tests.functional.models import ModelInfo +from tests.functional.constants.target_device import MAX_WORKERS_PER_TARGET_DEVICE +from tests.functional.constants.ovms import CurrentTarget as ct +from tests.functional.constants.ovms import Ovms +from tests.functional.constants.ovms_messages import OvmsMessages +from tests.functional.constants.paths import Paths +from tests.functional.constants.pipelines import Pipeline +from tests.functional.utils.log_monitor import LogMonitor +from tests.functional.object_model.dmesg_log_monitor import DmesgLogMonitor, DummyLogMonitor +from tests.functional.object_model.mediapipe_calculators import MediaPipeCalculator +from tests.functional.object_model.ovms_config import OvmsConfig +from tests.functional.object_model.package_manager import PackageManager +from tests.functional.object_model.resource_monitor import DockerResourceMonitor +from tests.functional.object_model.test_environment import TestEnvironment + +logger = get_logger(__name__) + + +class OvmsInstance(ABC): + + def __init__( + self, + name, + container_folder, + default_logger, + rest_port, + grpc_port, + target_device=None, + remote_ip=None, + lock_file=None, + context=None, + full_command=None, + ): + self.name = name + self.container_folder = container_folder + self.context = context + self.ovms_pid = None + self._default_log = default_logger + self._default_log.context = context + if disable_dmesg_log_monitor: + self._dmesg_log = DummyLogMonitor() + else: + self._dmesg_log = DmesgLogMonitor() + + self.ovms_ports = {} + self.remote_ip = remote_ip + if grpc_port: + self.ovms_ports[GRPC] = grpc_port + if rest_port: + self.ovms_ports[REST] = rest_port + self.target_device = target_device + self.target_device_lock_file = lock_file + self.full_command = full_command + + def get_break_msg_list(self, models): + break_msg_list = [ + CPP_STD_EXCEPTION, + OvmsMessages.ERROR_CANNOT_COMPILE_MODEL_INTO_TARGET_DEVICE, + OvmsMessages.CANNOT_OPEN_LIBRARY, + OvmsMessages.ERROR_FAILED_TO_PARSE_SHAPE_SHORT, + OvmsMessages.ERROR_LOADING_PRECONDITION_FAILED, + OvmsMessages.ERROR_DURING_LOADING_INPUT_TENSORS, + ] + + custom_node_present = any(map(lambda x: getattr(x, "child_nodes", None), models)) + if custom_node_present: + break_msg_list.extend([ + OvmsMessages.PIPELINE_REFERS_TO_INCORRECT_LIBRARY, + ]) + + if any([x.is_mediapipe for x in models]): + break_msg_list.extend([ + OvmsMessages.MEDIAPIPE_FAILED_TO_OPEN_GRAPH_SHORT, + ]) + + custom_loader_present = any(map(lambda x: x.custom_loader is not None, models)) + if custom_loader_present: + break_msg_list.extend([ + OvmsMessages.CUSTOM_LOADER_INVALID_CUSTOM_LOADER_OPTIONS, + ]) + + if any(map(lambda x: x.is_on_cloud, models)): + break_msg_list.extend([ + OvmsMessages.ERROR_FAILED_TO_CONNECT_TO_ANY_PROXY_ENDPOINT, + OvmsMessages.S3_WRONG_AUTHORIZATION, + OvmsMessages.GS_WRONG_AUTHORIZATION, + ]) + + if ct.is_plugin_target(): + break_msg_list.extend( + [OvmsMessages.ERROR_FAILED_TO_CREATE_PLUGIN, OvmsMessages.ERROR_FAILED_TO_LOAD_LIBRARY] + ) + + if all([x.is_hf_direct_load and not x.is_local for x in models]): + break_msg_list.extend([ + OvmsMessages.WARNING_NO_VERSION_FOUND_FOR_MODEL, + ]) + + return break_msg_list + + def ensure_models_loaded(self, models: List[ModelInfo] = None, timeout=None): + callbacks = [] + if machine_is_reserved_for_test_session and is_single_threaded(): + callbacks += [self._dmesg_log.raise_on_unexpected_messages] + custom_msg_list = [] + + # Prepare messages for mapping. + for model in models: + if model.use_mapping is True: + mapping_dict = model.get_mapping_dict(self.container_folder) + msg = self._default_log.get_log_models_mapping_messages(model, mapping_dict, shapeless=True) + custom_msg_list.extend(msg) + + self._default_log.models_loaded( + models, + break_msg_list=self.get_break_msg_list(models), + custom_msg_list=custom_msg_list, + timeout=timeout, + callbacks=callbacks, + ovms_instance=self, + ) + + def ensure_models_unloaded(self, models: List[ModelInfo] = None, timeout=None): + self._default_log.models_unloaded(models, timeout=timeout) + + def ensure_started( + self, + expected_loaded_models: List[ModelInfo] = None, + custom_messages: list = None, + expected_unloaded_models: List[ModelInfo] = None, + timeout: int = None, + os_type: str = None, + ): + self.wait_for_status(CONTAINER_STATUS_RUNNING, break_status=CONTAINER_STATUS_EXITED) + + if expected_loaded_models is not None: + expected_models = [] + for model in expected_loaded_models: + if isinstance(model, Pipeline): + if not model.is_mediapipe: + # To avoid double model addition (get_models() returns [self] for MediaPipe) + expected_models.extend(model.get_regular_models()) + expected_models.append(model) + else: + expected_models.append(model) + self.ensure_models_loaded(expected_models, timeout=timeout) + + self._default_log.models_unloaded(expected_unloaded_models, timeout=timeout, ovms_instance=self) + self._default_log.ensure_contains_messages(custom_messages, timeout=timeout, ovms_instance=self) + self._default_log.raise_on_unexpected_messages() + self._default_log.reset_to_ovms_creation() + self._default_log.is_running(self.ovms_ports, timeout, os_type) + self._default_log.reset_to_ovms_creation() + self._default_log.flush() + + self._dmesg_log.raise_on_unexpected_messages() + + def ensure_reloading(self, models: List[ModelInfo], timeout=None): + self._default_log.reloading(models, timeout) + + def prepare_mediapipe_config_and_graph(self, name, params, models): + config_dict = OvmsConfig.build(models=models) + params.custom_config = config_dict + params.models = models + config_path_on_host = os.path.join(self.container_folder, Paths.MODELS_PATH_NAME) + MediaPipeCalculator.prepare_proto_calculator(params, config_path_on_host) + OvmsConfig.save(name, config_dict) + return config_dict + + def update_model_list_and_config( + self, + name, + models, + models_to_verify=None, + resources_paths=None, + context=None, + params=None, + **kwargs + ): + self.prepare_resources(models) + + if models_to_verify: + ovms_log = self.create_log(False) + + if models_to_verify is not None and any(model.is_mediapipe for model in models_to_verify): + assert params is not None, "Params should be provided to create MediaPipe calculators" + config_dict = self.prepare_mediapipe_config_and_graph(name, params, models) + else: + _, config_dict = OvmsConfig.generate(name, models) + + if models_to_verify: + break_msg_list = self.get_break_msg_list(models_to_verify) + timeout = kwargs.get("timeout", wait_for_messages_timeout) + ovms_log.models_loaded(models_to_verify, break_msg_list=break_msg_list, ovms_instance=self, timeout=timeout) + + return config_dict + + def unload_all_models(self): + self.update_model_list_and_config(self.name, []) + + def get_port(self, api_type): + if isinstance(api_type, str): + return self.ovms_ports[api_type] + else: + return self.ovms_ports[api_type.type] + + def execute_and_check(self, cmd, verbose=False, cwd=None): + exit_code, stdout = self.execute_command(cmd, cwd) + assert exit_code == 0, f"Unexpected return code: {exit_code} during executing cmd: {cmd}\n\tOutput: {stdout}" + if verbose: + logger.info(stdout) + return stdout + + def get_env_variables(self): + return self.container.kwargs.get("environment", []) + + def install_package(self, base_os, tool_name): + environment = self.get_env_variables() + tool_installed = False + + for i in range(3): + try: + if i != 0: + logger.info(f"Running retry {i}") + sleep(10) + + cmd = PackageManager.create(base_os).get_install_cmd(tool_name) + if environment is None or "http_proxy" not in environment: + self.execute_and_check( + f"/bin/bash -c " + f"'export http_proxy={container_proxy} https_proxy={container_proxy}" + f" && {cmd}'" + ) + else: + self.execute_and_check(f"/bin/bash -c '{cmd}'") + + tool_installed = True + break + except Exception as e: + logger.info(f"Exception received. {e}") + sleep(5) + + assert tool_installed, f"Failed to install {tool_name} in container" + + def get_logs(self): + return self._default_log.get_all_logs() + + def get_logs_as_txt(self): + return self._default_log.get_logs_as_txt() + + def change_log_monitor(self, log_monitor): + self._default_log = log_monitor + + def wait_for_container_status_exited(self): + try: + self.wait_for_status(status=CONTAINER_STATUS_EXITED) + except Exception as e: + logger.error("Container cannot close properly") + logger.exception(str(e)) + raise DockerCannotCloseProperly(str(e)) + + def wait_for_status(self, status: str = CONTAINER_STATUS_RUNNING, break_status: str = None, timeout=60): + ovms_logs_lines = None + end_time = datetime.now() + timedelta(seconds=timeout) + while datetime.now() < end_time: + current_status = self.get_status(status) + if break_status and current_status == break_status: + ovms_logs_lines = self.get_logs() + result = get_exception_by_ovms_log(ovms_logs_lines) + if result: + exception, line = result + raise exception(line) + else: + raise OvmsTestException(f"Received break status: {current_status}", ovms_log=ovms_logs_lines) + if current_status == status: + break + + def ensure_status(self, status: str = CONTAINER_STATUS_RUNNING): + assert self.container.ensure_status(status) + + @abstractmethod + def execute_command(self, cmd, stream=False, cwd=None): + pass + + def prepare_resources(self, models): + """ + Execute on each model method prepare_resources (copy models, prepare custom_nodes and custom libraries if required) + + Parameters: + models (List[ModelInfo]): list of objects that needs to be prepared (ModelInfo, Pipeline, CustomNode, CustomLibrary) + """ + TestEnvironment.current.prepare_container_folders(self.name, models) + + def create_log(self, reset_to_ovms_creation, wait_for_log_timeout=None, use_default_logger=False) -> LogMonitor: + log = self._default_log if use_default_logger else self._create_logger() + log.get_all_logs() + if wait_for_log_timeout is not None: + start = datetime.now() + while not log._read_lines and (datetime.now() - start).total_seconds() <= wait_for_log_timeout: + log.get_all_logs() + sleep(1) + if not reset_to_ovms_creation: + log.flush() + log.logger_creation_start_offset = log.current_offset + if not log._read_lines: + log._read_lines = self._default_log._read_lines[:] # fix for CVS-126060 + return log + + def get_dmesg_log_monitor(self): + return self._dmesg_log + + def fetch_and_store_ovms_pid(self, timeout=60): + """ + Fetch and save OVMS process id. + Expect that child class object will provide `self.ovms_pid` + """ + self._dmesg_log.ovms_pid = self.ovms_pid + + @abstractmethod + def fetch_and_store_ovms_pid(self, timeout=10): + """ + Fetch and save OVMS process id. + """ + pass + + @abstractmethod + def start(self, ensure_started=False, *args, **kwargs): + raise NotImplementedError() + + @abstractmethod + def _create_logger(self) -> LogMonitor: + raise NotImplementedError() + + @staticmethod + def get_signal_type(terminate_signal_type): + if terminate_signal_type == Ovms.SIGKILL_SIGNAL: + return signal.SIGKILL + elif terminate_signal_type == Ovms.SIGINT_SIGNAL: + return signal.SIGINT + elif terminate_signal_type == Ovms.SIGTERM_SIGNAL: + return signal.SIGTERM + else: + raise NotImplementedError(f"Unknown signal: {terminate_signal_type}") + + def filter_unexpected_messages(self, unexpected_messages): + for msg in unexpected_messages: + # 1) This message is expected in OV tests with batch_size=0 + # 2) This message can happen in Cython instances that use threading # CVS-165321 + if all([ + OvmsMessages.ERROR_TERMINATE_CALLED in msg, + self.context is not None and "test_ov_app_cpp_batch_size_0" in self.context.name, + ]) or all([ + OvmsMessages.ERROR_TERMINATE_CALLED in msg, + OvmsMessages.STD_SYSTEM_ERROR in msg, + hasattr(self, "cmd") and self.cmd is not None and "OvmsCapiInstance" in self.cmd.__str__, + ]): + unexpected_messages.remove(msg) + return unexpected_messages + + def cleanup(self, timeout=30): + try: + if self.target_device_lock_file and self.target_device_lock_file.is_locked: + self.target_device_lock_file.release() + except Exception as e: + logger.exception(e) + + if artifacts_dir != "": + unexpected_messages = [] + for log_type, log in [("ovms", self._default_log), ("dmesg", self._dmesg_log)]: + log_name = f"{log_type}_{self.name}.log" + _, log_messages = log.cleanup(log_name) + if log_messages is not None: + unexpected_messages.extend(log_messages) + unexpected_messages = self.filter_unexpected_messages(unexpected_messages) + if unexpected_messages: + logger.error(f"Errors found in {log_type}: {unexpected_messages}") + dmesg_exceptions = get_children_from_module(DmesgError, assertions_module) + error_message = ( + f"Found unexpected messages in log files after OVMS instance cleanup: {unexpected_messages}" + ) + ovms_log = self._default_log.get_all_logs() + dmesg_log = self._dmesg_log.get_all_logs() + for name, exception_class in dmesg_exceptions: + msg = getattr(exception_class, "msg", None) + for m in unexpected_messages: + if m in msg: + raise exception_class(error_message, ovms_log, dmesg_log) + raise UnwantedMessageError(error_message, ovms_log, dmesg_log) + + if self.container_folder is not None and os.path.exists(self.container_folder): + error = None + start = datetime.now() + while (datetime.now() - start).total_seconds() <= timeout: + try: + shutil.rmtree(self.container_folder) + except PermissionError as e: + error = e + sleep(1) + else: + break + else: + if ( + hasattr(self, "cmd") and + self.cmd is not None and + self.cmd.base_os == OsType.Windows and + type(error) == PermissionError + ): + change_dir_permissions(self.container_folder) + shutil.rmtree(self.container_folder) + else: + raise error + + def release_ports(self): + for api in [REST, GRPC]: + port = self.ovms_ports.pop(api, None) + if port: + logger.info(f"Releasing {api} port") + PortManager(api).release_port(port) + + @staticmethod + def acquire_target_device_lock(target_device): + target_device = target_device.strip("'").split(" ")[0] if type(target_device) == str else target_device + max_locks = MAX_WORKERS_PER_TARGET_DEVICE[target_device] + if max_locks == 0: # No lock required + return None + if max_locks == 1: + timeout = None # just wait patiently for your device + else: + timeout = 1.0 # do not wait too long, try another lock + lock_files = [ + SelfDeletingFileLock(Paths.get_target_device_lock_file(target_device, i)) for i in range(max_locks) + ] + while True: + idx = random.randint(0, max_locks - 1) + acquired = lock_files[idx].acquire_no_raise(timeout) + if acquired: + return lock_files[idx] + + def stop_ovms_inside_kill(self, context, terminate_signal_type=Ovms.TERM_SIGNAL, ensure_ovms_killed=True): + exit_code, _ = self.execute_command("ps --version") + if exit_code != 0: + self.install_package(base_os=context.base_os, tool_name="procps-ng") + + output = self.execute_and_check("ps aux") + msg_list = [x for x in output.splitlines() if "strace" not in x and "xargs" not in x and "ovms/bin" in x] + assert len(msg_list) == 1, f"Unexpected number of OVMS processes: {msg_list}" + + ovms_pid = re.search(r"^\w+\s+(\d+)", msg_list[0]).group(1) + self.execute_and_check(f"/bin/bash -c 'kill {ovms_pid} -s {terminate_signal_type}'") + + if ensure_ovms_killed: + try: + cmd = f"ps -p {ovms_pid}" + exit_code, _ = self.execute_command(cmd) + i = 0 + ovms_log_monitor = self.create_log(True) + while exit_code == 0: + logger.warning( + f"OVMS is still running. Tail of OVMS output: {ovms_log_monitor.get_all_logs()[:-2]}" + ) + assert i < 60, f"Unable to stop OVMS process (PID: {ovms_pid})" + i += 1 + sleep(1) + exit_code, _ = self.execute_command(cmd) + except APIError as e: + logger.warning(e) + proc = Process() + short_id = self.get_short_id() + code, stdout, stderr = proc.run_and_check_return_all(f"docker ps -a --filter id={short_id}") + assert short_id in stdout, f"OVMS is still running. Docker id: {short_id}, OVMS id: {ovms_pid}." + + +@dataclass +class OvmsRunContext: + ovms: OvmsInstance = None + models: List[ModelInfo] = None + context: Context = None + + def attach_context(self, context): + self.context = context + context.ovms_sessions.append(self) + + def attach_resource_monitor(self, context, start=True): + if hasattr(self.ovms, "container"): + self.resource_monitor = DockerResourceMonitor(self.ovms.container) + if start: + self.resource_monitor.start() + context.test_objects.append(self.resource_monitor) + return self.resource_monitor diff --git a/tests/functional/object_model/ovms_log_monitor.py b/tests/functional/object_model/ovms_log_monitor.py new file mode 100644 index 0000000000..bbd6e3791d --- /dev/null +++ b/tests/functional/object_model/ovms_log_monitor.py @@ -0,0 +1,437 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import datetime +import re +import time + +import requests +from docker import from_env + +from tests.functional.utils.assertions import OvmsCrashed +from tests.functional.utils.logger import get_logger +from tests.functional.utils.process import PID_STATE_SLEEPING, PID_STATE_ZOMBIE, Process, get_pid_status +from tests.functional.config import is_nginx_mtls, wait_for_messages_timeout +from tests.functional.constants.core import CONTAINER_STATUS_DEAD, CONTAINER_STATUS_EXITED +from tests.functional.constants.custom_loader import CustomLoaderConsts +from tests.functional.constants.ovms_messages import OvmsMessages, OvmsMessagesRegex +from tests.functional.utils.log_monitor import LogMonitor + +logger = get_logger(__name__) + + +class OvmsLogMonitor(LogMonitor): + + def _get_unexpected_messages_regex(self): + return [] + + def _get_unexpected_messages(self): + return ["terminate called", "Exception caught in REST request handler"] + + def ensure_contains_messages( + self, + str_set_to_find, + break_msg_list=None, + timeout=None, + callbacks=[], + ovms_instance=None, + all_messages=True, + ): + self.wait_for_messages( + str_set_to_find, + break_msg_list, + raise_exception_if_not_found=True, + timeout=timeout, + callbacks=callbacks, + ovms_instance=ovms_instance, + all_messages=all_messages, + ) + + def reset_to_ovms_creation(self): + self.current_offset = 0 + + def reset_to_logger_creation(self): + self.current_offset = self.logger_creation_start_offset + + @staticmethod + def _calculate_batch_size_str(model): + batch_size = model.batch_size + if model.input_shape_for_ovms is not None: + input_shape = model.input_shape_for_ovms + if isinstance(input_shape, dict): + input_shape = [x for x in input_shape.values()][0] + + if isinstance(input_shape, str): + match = re.findall(r"([-\d:]+)", input_shape) + if match: + batch_size = match[0] + elif input_shape == "auto": + batch_size = 1 + elif isinstance(input_shape, list) or isinstance(input_shape, tuple): + batch_size = int(input_shape[0]) + elif batch_size is not None and isinstance(batch_size, str) and ":" in batch_size: + batch_size = ( + f"[{'~'.join(batch_size.split(':'))}]" # if batch_size = "1:3" -> expected entry in OVMS log: "[1~3]" + ) + elif model.input_shape_for_ovms is not None and isinstance(model.input_shape_for_ovms, str): + match = re.findall(r"([-\d:]+)", model.input_shape_for_ovms) + if match: + batch_size = match[0] + elif model.input_shape_for_ovms == "auto": + batch_size = 1 + elif batch_size == "auto": + batch_size = 1 + else: + batch_size = model.get_expected_batch_size() + return batch_size + + def _get_log_models_started_messages(self, models): + result = [] + for model in models or []: + batch_size_str = self._calculate_batch_size_str(model) + result.append(OvmsMessages.OVMS_SERVER_RUNNING_MSG.format(model.version, model.name)) + result.append(OvmsMessages.OVMS_MODEL_LOADED.format(model.name, model.version, batch_size_str)) + return result + self._get_log_custom_loader_loaded_messages(models) + + def _get_log_models_loaded(self, models): + result = [] + for model in models or []: + if model.is_hf_direct_load: + result.append(OvmsMessages.MEDIAPIPE_PIPELINE_VALIDATION_PASS_MSG.format(model.name)) + elif model.is_mediapipe: + result.append(OvmsMessages.MEDIAPIPE_PIPELINE_VALIDATION_PASS_MSG.format(model.name)) + for reg_model in model.regular_models: + batch_size_str = self._calculate_batch_size_str(reg_model) + result.append( + OvmsMessages.OVMS_MODEL_LOADED.format(reg_model.name, reg_model.version, batch_size_str) + ) + elif model.is_pipeline(): + result.append(OvmsMessages.PIPELINE_STARTED.format(model.name)) + else: + batch_size_str = self._calculate_batch_size_str(model) + result.append(OvmsMessages.OVMS_MODEL_LOADED.format(model.name, model.version, batch_size_str)) + return result + + @staticmethod + def _get_log_models_failed_to_load(models): + phrases = [ + OvmsMessages.ERROR_EXCEPTION_CATCH, + OvmsMessages.ERROR_UNABLE_TO_ACCESS_PATH, + OvmsMessages.OVMS_ERROR_OCCURED_WHILE_LOADING_MODEL_GENERIC, + OvmsMessages.OVMS_ERROR_INCORRECT_WEIGHTS_IN_BIN_FILE, + ] + for model in models or []: + phrases.append(OvmsMessages.OVMS_MODEL_FAILED_TO_LOAD.format(model.name, model.version)) + return phrases + + @staticmethod + def _get_log_models_reloading_messages(models): + def _calculate_shape_str(shape): + result = shape + if ":" in in_shape: + match_range = re.findall(r"\d+:\d+", in_shape) + for range in match_range: + new_value = f"[{range.replace(':', '~')}]" + result = in_shape.replace(range, new_value) + return result + + result = [] + for model in models or []: + result.append(OvmsMessages.MODEL_RELOADING.format(model.name)) + if model.input_shape_for_ovms is not None: + for in_name, in_shape in model.input_shape_for_ovms.items(): + log_shape_str = _calculate_shape_str(in_shape) + result.append(OvmsMessages.MODEL_INPUT_SHAPE_RELOADING.format(in_name, in_name, log_shape_str)) + return result + + @staticmethod + def get_log_models_mapping_messages(model, mapping_dict, reload=False, shapeless=False): + def _calculate_shape_str(shape): + return str(tuple(shape["shape"])).replace(" ", "") # [1, 1, 1, 1] -> (1,1,1,1) + + result = [] + if reload: + result.append(OvmsMessages.MODEL_RELOADING.format(model.name)) + for (_, in_shape), (in_key, in_value) in zip(model.inputs.items(), mapping_dict["inputs"].items()): + if shapeless: + result.append(OvmsMessages.MODEL_INPUT_NAME_MAPPING_NAME.format(in_key, in_value)) + else: + log_input_shape_str = _calculate_shape_str(in_shape) + result.append(OvmsMessages.MODEL_INPUT_SHAPE_RELOADING.format(in_key, in_value, log_input_shape_str)) + for (_, out_shape), (out_key, out_value) in zip(model.outputs.items(), mapping_dict["outputs"].items()): + if shapeless: + result.append(OvmsMessages.MODEL_OUTPUT_NAME_MAPPING_NAME.format(out_key, out_value)) + else: + log_output_shape_str = _calculate_shape_str(out_shape) + result.append( + OvmsMessages.MODEL_OUTPUT_SHAPE_RELOADING.format(out_key, out_value, log_output_shape_str) + ) + + return result + + @staticmethod + def _get_log_models_unloaded_messages(models): + result = [] + for model in models or []: + if model.is_mediapipe: + result.append(OvmsMessages.MEDIAPIPE_UNLOADED.format(model.name)) + for reg_model in model.regular_models: + result.append(OvmsMessages.OVMS_SERVER_UNLOADED_MSG.format(reg_model.version, reg_model.name)) + else: + result.append(OvmsMessages.OVMS_SERVER_UNLOADED_MSG.format(model.version, model.name)) + return result + + @staticmethod + def _get_log_pipelines_started_messages(pipelines): + result = [] + for pipeline in pipelines: + result.append(OvmsMessages.PIPELINE_STARTED.format(pipeline.name)) + return result + + @staticmethod + def _get_log_pipelines_unloaded_messages(pipelines): + result = [] + for pipeline in pipelines: + if pipeline.is_mediapipe: + result.append(OvmsMessages.MEDIAPIPE_UNLOADED.format(pipeline.name)) + else: + result.append(OvmsMessages.PIPELINE_UNLOADED.format(pipeline.name)) + return result + + @staticmethod + def _get_log_custom_loader_loaded_messages(models): + result = [] + custom_loaders = set() # using set instead list will help avoid redundancy + + for model in models or []: + if model.custom_loader is not None: + custom_loaders.add(model.custom_loader) + + for cs in custom_loaders: + loader_loaded_logs = CustomLoaderConsts.get_logs_for_loaded_loader(cs.name) + result.extend(loader_loaded_logs) + return result + + def started(self, models, pipelines=None, timeout=None): + msg_list = self._get_log_models_started_messages(models) + break_msg_list = self._get_log_models_failed_to_load(models) + if pipelines is not None: + msg_list.extend(self._get_log_pipelines_started_messages(pipelines)) + if timeout is None: + timeout = 60 + if models: + timeout += sum([model.get_ovms_loading_time() for model in models]) + self.ensure_contains_messages(msg_list, break_msg_list, timeout=timeout) + + def reloading(self, models, timeout=30): + msg_list = self._get_log_models_reloading_messages(models) + self.ensure_contains_messages(msg_list, timeout=timeout) + + def model_mapping_loaded(self, model, mapping_dict, timeout=30, reload=False): + msg_list = self.get_log_models_mapping_messages(model, mapping_dict, reload=reload) + start_time = time.time() + # model mapping is loading: should be fast since txt data is loaded. + self.ensure_contains_messages(msg_list, timeout=timeout) + timeout -= time.time() - start_time + + # model binaries is loading: could be time-consuming. + self.models_loaded([model], timeout=timeout) + + def models_unloaded(self, models, pipelines=None, timeout=None, ovms_instance=None): + msg_list = self._get_log_models_unloaded_messages(models) + if pipelines is not None: + msg_list.extend(self._get_log_pipelines_unloaded_messages(pipelines)) + if timeout is None: + timeout = 60 + if models: + timeout += sum([model.get_ovms_loading_time() for model in models]) + self.ensure_contains_messages(msg_list, timeout=timeout, ovms_instance=ovms_instance) + + def models_loaded( + self, models, custom_msg_list=None, break_msg_list=None, timeout=None, callbacks=[], ovms_instance=None + ): + if timeout is None: + timeout = wait_for_messages_timeout + if models: + timeout += sum([model.get_ovms_loading_time() for model in models]) + msg_list = self._get_log_models_loaded(models) + if custom_msg_list is not None: + msg_list.extend(custom_msg_list) + self.ensure_contains_messages( + msg_list, break_msg_list, timeout=timeout, callbacks=callbacks, ovms_instance=ovms_instance + ) + + def custom_loader_loaded(self, models, timeout=30): + msg_list = self._get_log_custom_loader_loaded_messages(models) + self.ensure_contains_messages(msg_list, timeout=timeout) + + def action_on_models( + self, models_loaded=None, models_unloaded=None, models_reloaded=None, custom_msg_list=None, timeout=30 + ): + msg_list = self._get_log_models_started_messages(models_loaded) + msg_list += self._get_log_models_unloaded_messages(models_unloaded) + msg_list += self._get_log_models_reloading_messages(models_reloaded) + if custom_msg_list: + msg_list += custom_msg_list + self.ensure_contains_messages(msg_list, timeout=timeout) + + def is_running(self, ovms_ports, timeout, os_type=None): + msg_list = [OvmsMessages.OVMS_SERVICES_RUNNING_MSG[key] for key in ovms_ports] + if is_nginx_mtls: + msg_list += [OvmsMessages.NGINX_STARTED_WITH_PID] + break_msg_list = [OvmsMessages.OVMS_STOPPING] + self.ensure_contains_messages(msg_list, break_msg_list, timeout=timeout) + + @staticmethod + def get_message_time(msg): + # message example '[2022-01-20 16:45:47.495][1][serving][info][modelinstance.cpp:718] Loaded model' + msg_time_str = msg.split("[")[1].strip("]") + msg_time = datetime.datetime.strptime(msg_time_str, "%Y-%m-%d %H:%M:%S.%f") + return msg_time + + def get_models_loading_time(self, models, is_reload): + result = [] + for model in models: + if is_reload: + found_messages, messages_to_find_vs_results_map = self.find_messages( + [OvmsMessages.MODEL_RELOADING.format(model.name)], raise_exception_if_not_found=True + ) + else: + found_messages, messages_to_find_vs_results_map = self.find_messages( + [OvmsMessages.MODEL_LOADING.format(model.name, model.version, model.base_path)], + raise_exception_if_not_found=True, + ) + model_loading_msg = list(messages_to_find_vs_results_map.values())[0] + start_loading_time = self.get_message_time(model_loading_msg) + + start_msg_list = self._get_log_models_loaded([model]) + found_messages, messages_to_find_vs_results_map = self.find_messages( + start_msg_list, raise_exception_if_not_found=True + ) + model_loaded_msg = list(messages_to_find_vs_results_map.values())[0] + finished_loading_time = self.get_message_time(model_loaded_msg) + loading_time = finished_loading_time - start_loading_time + result.append((model, loading_time)) + logger.info(f"{model} model loading time: {loading_time}") + + return result + + @staticmethod + def find_no_of_infer_requests(log_lines): + string_to_find = "] Loaded model" + for i, ll in enumerate(log_lines): + if string_to_find in ll: + noir = int(log_lines[i].split()[-1]) + return noir + + @staticmethod + def get_status_change_from_logs(logs, expected_state=""): + status_change_messages = list(filter(lambda x: OvmsMessagesRegex.STATUS_CHANGE_RE.search(x), logs)) + status_change_messages = list(filter(lambda x: expected_state in x, status_change_messages)) + return status_change_messages + + def get_log_value(self, msg_to_found): + all_messages_found, messages_to_find_vs_results_map = self.find_messages( + [msg_to_found], raise_exception_if_not_found=True + ) + assert all_messages_found + # Searching for log phrase, e.g.: + # "Number of OpenVINO streams: 11" or "Number of OpenVINO streams: 4" + # "No of InferRequests: 11" "No of InferRequests: 4" + # from the whole log line, e.g.: + # "["2022-12-08 12:09:17.212][1][serving][info][modelinstance.cpp:790] Loaded model resnet-50-tf; version: 1; batch size: 1; No of InferRequests: 4" + phrase = re.search(rf"{msg_to_found} \d+", messages_to_find_vs_results_map[msg_to_found]).group() + value = re.search(r"\d+", phrase).group() # searching for single value + return value + + +class BinaryOvmsLogMonitor(OvmsLogMonitor): + def __init__(self, ovms_process, **kwargs): + super().__init__(**kwargs) + self._proc = ovms_process + + def is_ovms_running(self): + status = get_pid_status(self._proc._proc.pid) + if status in [PID_STATE_ZOMBIE]: + return False + if status in [PID_STATE_SLEEPING]: + return True + return True + + def get_all_logs(self): + stdout, stderr = self._proc.get_output() + if stderr: + logger.error( + f"Detect non-empty stderr! It is recommended to redirect stderr to stdout: 2>&1. STDERR: {stderr}" + ) + self._read_lines += stdout.splitlines() + return self._read_lines + + +class OvmsDockerLogMonitor(OvmsLogMonitor): + @staticmethod + def create(container_id): + client = from_env() + container = client.containers.get(container_id) + return OvmsDockerLogMonitor(container) + + def __init__(self, container, **kwargs): + super().__init__(**kwargs) + self._container = container + + def get_all_logs(self): + try: + self._read_lines = self._container.logs().decode().splitlines() + except requests.exceptions.HTTPError as e: + raise OvmsCrashed(msg=str(e), ovms_log=self.get_logs_as_txt()) + return self._read_lines + + def get_logs_as_txt(self): + process = Process() + process.disable_check_stderr() + exit_code, stdout, _ = process.run(f"docker logs {self._container.id} 2>&1") + return stdout + + def is_ovms_running(self): + self._container.reload() + return self._container.status not in [CONTAINER_STATUS_EXITED, CONTAINER_STATUS_DEAD] + + +class OvmsDockerStreamLogMonitor(OvmsDockerLogMonitor): + def __init__(self, container, output_steam, **kwargs): + super().__init__(container, **kwargs) + self._output_steam = output_steam + + def get_all_logs(self): + if self._container.container.get_status() != CONTAINER_STATUS_EXITED: + logger.warning("Attempt for getting logs from unclosed docker, reading stream could lead to deadlock") + return [] + self._read_lines = list(map(lambda x: x.decode().strip(), self._output_steam)) + return self._read_lines + + +class OvmsCmdLineDockerLogMonitor(OvmsLogMonitor): + + def __init__(self, docker_id, **kwargs): + super().__init__(**kwargs) + self.docker_id = docker_id + self.process = Process() + self.process.set_log_silence() + + def get_all_logs(self): + exit_code, stdout, _ = self.process.run(f"docker logs {self.docker_id} 2>&1") + self._read_lines = stdout.splitlines() + return self._read_lines diff --git a/tests/functional/object_model/ovms_mapping_config.py b/tests/functional/object_model/ovms_mapping_config.py new file mode 100644 index 0000000000..9e7e26cdc0 --- /dev/null +++ b/tests/functional/object_model/ovms_mapping_config.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +from pathlib import Path + +from tests.functional.utils.logger import get_logger +from tests.functional.object_model.test_environment import TestEnvironment + +logger = get_logger(__name__) + + +class OvmsMappingConfig(object): + FILE_NAME = "mapping_config.json" + + @staticmethod + def generate(model, context, mapped_input_names: list = None, mapped_output_names: list = None): + + container_folder = TestEnvironment.current.prepare_container_folders(context.test_object_name, [model])[0] + + config_dict = {"inputs": {}, "outputs": {}} + inputs_pairs = {} + outputs_pairs = {} + + model_inputs_keys = list(model.inputs.keys()) + model_outputs_keys = list(model.outputs.keys()) + + if mapped_input_names is not None and mapped_output_names is not None: + assert len(mapped_input_names) <= len( + model_inputs_keys + ), f"Incorrect format of mapped input names: {mapped_input_names} - too many elements in list" + assert len(mapped_output_names) <= len( + model_outputs_keys + ), f"Incorrect format of mapped output names: {mapped_output_names} - too many elements in list" + + for model_input_key, mapped_input_name in zip(model_inputs_keys, mapped_input_names): + inputs_pair = {model_input_key: mapped_input_name} + inputs_pairs.update(inputs_pair) + + for model_output_key, mapped_output_name in zip(model_outputs_keys, mapped_output_names): + outputs_pair = {model_output_key: mapped_output_name} + outputs_pairs.update(outputs_pair) + else: + for i, model_input_key in enumerate(model_inputs_keys): + inputs_pair = {model_input_key: f"{model.name}_input_{i}"} + inputs_pairs.update(inputs_pair) + + outputs_pairs = {} + for i, model_output_key in enumerate(model_outputs_keys): + outputs_pair = {model_output_key: f"{model.name}_output_{i}"} + outputs_pairs.update(outputs_pair) + + config_dict["inputs"].update(inputs_pairs) + config_dict["outputs"].update(outputs_pairs) + + mapping_dict, model = OvmsMappingConfig.prepare_model_inputs_outputs(config_dict, model) + mapping_config_path = OvmsMappingConfig.save(mapping_dict, container_folder, model) + + return mapping_dict, mapping_config_path, container_folder + + @staticmethod + def mapping_config_purepath(ovms_container, model): + return Path(ovms_container, f".{model.base_path}", str(model.version), OvmsMappingConfig.FILE_NAME) + + @staticmethod + def mapping_config_path(ovms_container, model): + return str(OvmsMappingConfig.mapping_config_purepath(ovms_container, model)) + + @staticmethod + def mapping_exists(ovms_container, model): + return OvmsMappingConfig.mapping_config_purepath(ovms_container, model).exists() + + @staticmethod + def delete_mapping(ovms_container, model): + mapping_file = OvmsMappingConfig.mapping_config_purepath(ovms_container, model) + assert mapping_file.exists(), f"Trying to delete unexisting mapping in {mapping_file}" + + # NOTE: + # If OVMS container is running, please reload config.json. For details please take a peek: + # OvmsInstance.update_model_list_and_config(...) + mapping_file.unlink() + + @staticmethod + def save(config_dict: dict, ovms_container, model): + file_dst_path = OvmsMappingConfig.mapping_config_path(ovms_container, model) + + logger.info("Saving config file to {}, content:\n{}".format(file_dst_path, config_dict)) + + with open(file_dst_path, "w") as fp: + json.dump(config_dict, fp, indent=2) + + return file_dst_path + + @staticmethod + def load_config(config_path): + with open(config_path, "r") as f: + config_json = f.read() + try: + config_dict = json.loads(config_json) + except ValueError as e: + logger.error("Error while loading json: {}".format(config_json)) + raise e + return config_dict + + @staticmethod + def prepare_model_inputs_outputs(mapping_dict: dict, model): + for config_inputs_key, config_inputs_value in mapping_dict["inputs"].items(): + model.inputs[config_inputs_value] = model.inputs.pop(config_inputs_key) + + for config_outputs_key, config_outputs_value in mapping_dict["outputs"].items(): + model.outputs[config_outputs_value] = model.outputs.pop(config_outputs_key) + + return mapping_dict, model diff --git a/tests/functional/object_model/ovms_params.py b/tests/functional/object_model/ovms_params.py new file mode 100644 index 0000000000..137083390f --- /dev/null +++ b/tests/functional/object_model/ovms_params.py @@ -0,0 +1,189 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +from dataclasses import dataclass +from typing import Any, Callable, List + +from dataclasses_json import dataclass_json + +from tests.functional.utils.core import ComplexEncoder +from tests.functional.utils.logger import get_logger +from tests.functional.object_model.ovms_command import OvmsCommand +from tests.functional.config import logging_level_ovms +from tests.functional.constants.metrics import MetricsPolicy +from tests.functional.models.models_static import ModelInfo, Muse +from tests.functional.models.models_library import ModelsLib +from tests.functional.object_model.cpu_extension import MuseModelExtension +from tests.functional.object_model.custom_loader import CustomLoader + +logger = get_logger(__name__) + + +@dataclass_json +@dataclass(frozen=False) +class OvmsParams(object): + name: str = None + grpc_port: int = None + log_level: str = logging_level_ovms + models: List[ModelInfo] = None + model_name: str = None + model_path: str = None + nireq: int = None + rest_port: int = None + target_device: str = None + check_version: bool = False + use_config: bool = False + custom_config: dict = None + create_config_method: Callable[[str], None] = None + image: str = None + shape: Any = None + is_stateful: bool = None + sequence_cleaner_poll_wait_minutes: int = None + max_sequence_number: int = None + low_latency_transformation: bool = None + idle_sequence_cleanup: bool = None + model_version_policy: Any = None + file_system_poll_wait_seconds: int = None + cpu_extension: str = None + custom_loaders: List[CustomLoader] = None + rest_workers: int = None + grpc_workers: int = None + use_cache: bool = False + cache_dir_path: str = None + custom_command: OvmsCommand = None + metrics_enable: MetricsPolicy = MetricsPolicy.NotDefined + metrics_list: List[str] = None + custom_graph_paths: List[str] = None + use_custom_graphs: bool = False + use_subconfig: bool = False + single_mediapipe_model_mode: bool = False + allowed_local_media_path: str = None + allowed_media_domains: str = None + pull: bool = None + source_model: str = None + gguf_filename: str = None + model_repository_path: str = None + task: str = None + task_params: str = None + list_models: bool = None + overwrite_models: bool = None + add_to_config: bool = False + remove_from_config: bool = False + resolution: str = None + cache_size: int = None + + def __post_init__(self): + if self.models is not None and any(isinstance(model, Muse) for model in self.models): + self.cpu_extension = MuseModelExtension() + + def get_shape_param(self): + result = self.shape + if isinstance(self.shape, list): + result = str(tuple(self.shape)) + return result + + def get_regular_models(self): + result = [] + if self.list_models or self.add_to_config or self.remove_from_config: + return result + elif self.model_name is not None: + result.append(ModelsLib.create_model(self.model_name)) + elif self.models is None: + result.append(ModelsLib.get_default_model(self.target_device)()) + + for model in self.models or []: + result += model.get_regular_models() + return result + + def get_layout_from_regular_models(self, regular_models_list=None): + result = None + if regular_models_list is None: + regular_models_list = self.get_regular_models() + for model in regular_models_list: + for input_layout in model.input_layouts.values(): + if input_layout is not None: + result = input_layout + break + return result + + def get_plugin_config_from_regular_models(self, regular_models_list=None): + plugin_config = None + if regular_models_list is None: + regular_models_list = self.get_regular_models() + for model in regular_models_list: + plugin_config = model.plugin_config + if plugin_config is not None: + break + return plugin_config + + def get_models(self): + result = [] + if self.list_models or self.add_to_config or self.remove_from_config: + return result + if self.models is not None: + result += self.models + else: + if self.model_name is not None: + result.append(ModelsLib.create_model(self.model_name)) + elif self.models is None: + result.append(ModelsLib.get_default_model(self.target_device)()) + return result + + def is_stateful_model_present(self): + is_stateful = False + if self.models is not None: + is_stateful = any([x.is_stateful for x in self.models]) + else: + if self.models: + model = self.models[0] + is_stateful = model.is_stateful + return is_stateful + + def to_str(self): + dict_params = self.to_dict() + if "cpu_extension" in dict_params: + dict_params["cpu_extension_path"] = self.cpu_extension_path + del dict_params["cpu_extension"] + + if "custom_loader" in dict_params: + del dict_params["custom_loader"] + + for model in dict_params.get("models", []): + if "custom_loader" in model: + # dict_params["custom_loader_path"] = self.cpu_extension_path + del model["custom_loader"] + + serialized_params = json.dumps(dict_params, cls=ComplexEncoder) + to_replace = {"null": "None", "false": "False", "true": "True"} + for key, value in to_replace.items(): + serialized_params = serialized_params.replace(key, value) + return serialized_params + + @classmethod + def from_str(cls, params): + deserialized = cls.from_json(params) + return deserialized + + def ports_enabled(self): + ports_enabled = not any([ + self.pull, + self.list_models, + self.overwrite_models, + self.add_to_config, + self.remove_from_config, + ]) + return ports_enabled diff --git a/tests/functional/object_model/ovsa.py b/tests/functional/object_model/ovsa.py new file mode 100644 index 0000000000..36c1880e28 --- /dev/null +++ b/tests/functional/object_model/ovsa.py @@ -0,0 +1,149 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import shutil +import stat +from datetime import datetime +from pathlib import Path + +from cryptography import x509 + +from tests.functional.utils.core import SelfDeletingFileLock +from tests.functional.utils.inference.communication.base import AbstractCommunicationInterface +from tests.functional.utils.logger import get_logger +from tests.functional.utils.process import Process +from tests.functional.utils.ssl import SslCertificates +from tests.functional.constants.ovsa import OVSA + +logger = get_logger(__name__) + + +class OvsaCerts(SslCertificates): + default_certs = None + + def __init__( + self, + nginx_mtls_path: str = OVSA.NGINX_TMP_DIR_PATH, + server_cert_name: str = OVSA.SERVER_CERT_NAME, + server_key_name: str = OVSA.SERVER_KEY_NAME, + client_cert_name: str = OVSA.CLIENT_CERT_NAME, + client_key_name: str = OVSA.CLIENT_KEY_NAME, + client_cert_ca_name: str = OVSA.CLIENT_CERT_CA_NAME, + client_cert_ca_crl_name: str = OVSA.CLIENT_CERT_CA_CRL_NAME, + dhparam_name: str = OVSA.DHPARAMS_NAME, + mount_a_dir: bool = False, + ): + def get_absolute_path(target_value): + if not os.path.isabs(target_value): + target_value = os.path.join(nginx_mtls_path, target_value) + return target_value + + super().__init__() + + self.certs_loaded = False + self.nginx_mtls_path = nginx_mtls_path + self.client_cert_ca_crl_path = get_absolute_path(client_cert_ca_crl_name) + self.client_cert_ca_path = get_absolute_path(client_cert_ca_name) + self.client_cert_path = get_absolute_path(client_cert_name) + self.client_key_path = get_absolute_path(client_key_name) + self.dhparam_path = get_absolute_path(dhparam_name) + self.server_cert_path = get_absolute_path(server_cert_name) + self.server_key_path = get_absolute_path(server_key_name) + + self.mount_a_dir = mount_a_dir + + self.initialize_certificates() + + def initialize_certificates(self): + try: + self.load_client_cert_ca_crl(self.client_cert_ca_crl_path) + self.load_client_cert_ca(self.client_cert_ca_path) + self.load_client_cert(self.client_cert_path) + self.load_client_key(self.client_key_path) + self.load_dhparam(self.dhparam_path) + self.load_server_cert(self.server_cert_path) + self.load_server_key(self.server_key_path) + except (FileNotFoundError, IsADirectoryError) as e: + logger.info(f"Some certificates missing, expecting to generate new: {e}") + self.certs_loaded = False + return + self.certs_loaded = True + + def create_ovsa_volume_bindings(self) -> dict: + if self.mount_a_dir: + return {self.nginx_mtls_path: {"bind": OVSA.CERTS_CONTAINER_PATH, "mode": "ro"}} + return { + self.client_cert_ca_crl_path: {"bind": OVSA.CLIENT_CERT_CA_CRL_CONTAINER_PATH, "mode": "ro"}, + self.client_cert_ca_path: {"bind": OVSA.CLIENT_CERT_CA_CONTAINER_PATH, "mode": "ro"}, + self.dhparam_path: {"bind": OVSA.DHPARAMS_CONTAINER_PATH, "mode": "ro"}, + self.server_cert_path: {"bind": OVSA.SERVER_CERT_CONTAINER_PATH, "mode": "ro"}, + self.server_key_path: {"bind": OVSA.SERVER_KEY_CONTAINER_PATH, "mode": "ro"}, + } + + def are_valid(self) -> bool: + if not self.certs_loaded: + return False + + for file in [self.server_cert_path, self.client_cert_ca_path, self.client_cert_path, self.server_key_path]: + file = Path(file) + if not file.exists(): + logger.warning(f"Certificate file={str(file)} do not exist") + return False + if file.stat().st_mode & stat.S_IRGRP == 0: + logger.warning(f"Certificate file={str(file)} got insufficient reading rights") + return False + + _cert = x509.load_pem_x509_certificate(self.client_cert) + _cert_ca = x509.load_pem_x509_certificate(self.client_cert_ca) + + timezone = _cert.not_valid_before_utc.tzinfo + valid = _cert.not_valid_before_utc < datetime.now(timezone) < _cert.not_valid_after_utc + valid = valid and (_cert_ca.not_valid_before_utc < datetime.now(timezone) < _cert_ca.not_valid_after_utc) + return valid + + @staticmethod + def generate_ovsa_certs(mount_a_dir: bool = False, destination_path=OVSA.NGINX_TMP_DIR_PATH, skip_if_valid=False): + destination_path = Path(destination_path) + if not destination_path.exists(): + destination_path.mkdir(parents=True) + + logger.info("Generate Certs") + with SelfDeletingFileLock(f"{Path(destination_path, '.dir.lock')}") as fl: + certs = OvsaCerts(mount_a_dir=mount_a_dir, nginx_mtls_path=destination_path) + if certs.are_valid() and skip_if_valid: + logger.info("Certificates are still valid and do not require generation") + return certs + + gen_certs_process = Process() + path_to_generate_script = os.path.join(destination_path, OVSA.GENERATE_CERTS_SCRIPT_NAME) + shutil.copytree(OVSA.OVMS_C_NGINX_MTLS_PATH, destination_path, dirs_exist_ok=True) + gen_certs_process.run_and_check(f"chmod +x {path_to_generate_script}") + exit_code, stdout, stderr = gen_certs_process.run( + cmd=path_to_generate_script, cwd=destination_path, timeout=900 + ) + assert exit_code == 0, f"Error generating OVSA certs:\nstdout: {stdout}\nstderr: {stderr}" + certs = OvsaCerts(mount_a_dir=mount_a_dir, nginx_mtls_path=destination_path) + assert certs.are_valid() + return certs + + @staticmethod + def init_ovsa_certs(_certs=None): + certs = _certs if _certs else OvsaCerts() + assert certs.are_valid() + OvsaCerts.default_certs = certs + AbstractCommunicationInterface._default_certs = certs + return certs diff --git a/tests/functional/object_model/package_manager.py b/tests/functional/object_model/package_manager.py new file mode 100644 index 0000000000..14a3442d80 --- /dev/null +++ b/tests/functional/object_model/package_manager.py @@ -0,0 +1,330 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +from abc import ABC, abstractmethod + +import pytest + +from tests.functional.utils.assertions import ( + AptInstallException, + InstallPkgVersionException, + OvmsTestException, + UpgradePkgException, +) +from tests.functional.utils.logger import get_logger +from tests.functional.constants.os_type import OsType +from tests.functional.utils.process import Process +from tests.functional.constants.target_device import TargetDevice + +logger = get_logger(__name__) + +GPU_LIBS_TO_SKIP = ["intel-level-zero-gpu", "level-zero", "libigdgmm12"] + + +class PackageManager(ABC): + + def __init__(self): + pass + + @staticmethod + def create(base_os=OsType.Ubuntu24): + if OsType.Redhat in base_os: + return MicrodnfPackageManager() + elif OsType.Ubuntu22 in base_os or OsType.Ubuntu24 in base_os: + return AptPackageManager() + + raise NotImplementedError() + + @abstractmethod + def get_install_cmd(self, package_name): + raise NotImplementedError() + + def get_upgrade_pkg_cmd(self, package_name): + raise NotImplementedError() + + @abstractmethod + def get_install_version_cmd(self, package_name, version): + return NotImplementedError() + + @abstractmethod + def get_list_of_installed_packages(self, container_id): + raise NotImplementedError() + + @abstractmethod + def get_dependencies(self, pkg_name_list, container_id): + raise NotImplementedError() + + def run_process(self, cmd, exception_type=OvmsTestException): + proc = Process() + proc.disable_check_stderr() + stdout = proc.run_and_check(cmd, exception_type=exception_type) + return stdout + + def get_missing_packages(self, container_pkg_list, host_pkg_list): + missing_packages = {c: container_pkg_list[c] for c in container_pkg_list if c not in host_pkg_list} + return missing_packages + + def get_packages_to_upgrade(self, host_pkg_list, container_pkg_list): + packages_to_upgrade = { + c: container_pkg_list[c] + for c in container_pkg_list + if c in host_pkg_list and host_pkg_list[c]["version"] < container_pkg_list[c]["version"] + } + return packages_to_upgrade + + def install_missing_packages_on_host(self, container_pkg_list, host_pkg_list, missing_pkg_list): + pkgs_to_install = {} + for pkg, version in missing_pkg_list.items(): + if self.context.target_device == TargetDevice.GPU and pkg in GPU_LIBS_TO_SKIP: + logger.debug(f"Skip package {pkg} - will be downloaded in GpuDriverInstaller.") + continue + logger.debug(f"Installing {pkg} ...") + pkgs_to_install.update({pkg: version}) + + # Map each element to string: =: + pkgs_to_install = list(map(lambda x: f"{x[0]}={x[1]['version']}", pkgs_to_install.items())) + + # It is important to install packages all at once with explicit version: + # it would help to minimalize risk of version mismatch of some low level packages + if len(pkgs_to_install) != 0: + cmd = self.install_cmd.format(" ".join(pkgs_to_install)) # install packages all at once + _ = self.run_process(cmd, exception_type=AptInstallException) + after_install_host_pkg_list = self.get_list_of_installed_packages(container_id=None) + assert not self.get_missing_packages(container_pkg_list, after_install_host_pkg_list) + return host_pkg_list + else: # Return if there are no more pkgs_to_install except those defined in GPU_LIBS_TO_SKIP + return host_pkg_list + + def upgrade_packages(self, packages_to_upgrade, container_pkg_list): + for key, value in packages_to_upgrade.items(): + logger.debug(f"Upgrading pkg '{key}' to version '{value['version']}' ...") + cmd = self.get_install_version_cmd(key, value["version"]) + try: + self.run_process(cmd, exception_type=InstallPkgVersionException) + except InstallPkgVersionException as e: + cmd, retcode, stdout, stderr = e.get_process_details() + if "The following packages have unmet dependencies" in stdout: + logger.debug(f"Upgrading all system packages ...") + self.run_process(self.upgrade_cmd, exception_type=UpgradePkgException) + host_packages = self.get_list_of_installed_packages(container_id=None) + if not self.get_packages_to_upgrade(host_packages, container_pkg_list): + break + elif f"Version '{value['version']}' for '{key}' was not found" in stderr: + logger.debug(f"Upgrading package {key}") + upgrade_pkg_cmd = self.get_upgrade_pkg_cmd(key) + self.run_process(upgrade_pkg_cmd, exception_type=UpgradePkgException) + else: + raise InstallPkgVersionException(f"Found exception in executing cmd: {cmd}; stdout: {stdout}") + host_packages = self.get_list_of_installed_packages(container_id=None) + pkgs = self.get_packages_to_upgrade(host_packages, container_pkg_list) + if not pkgs: + return + else: + for key, value in pkgs.items(): + logger.warning( + f"Failed to upgrade package {key}. Continue with the current version: {value['version']}" + ) + + +class DnfPackageManager(PackageManager): + + def __init__(self): + self.update_cmd = "dnf update -y" + self.install_cmd = "dnf install -y" + self.upgrade_cmd = "dnf upgrade -y" + + def get_install_cmd(self, package_name): + return f"{self.install_cmd} {package_name}" + + def get_install_version_cmd(self, package_name, version): + return f"{self.install_cmd} {package_name}-{version}" + + def get_upgrade_pkg_cmd(self, package_name): + return f"{self.upgrade_cmd} {package_name}" + + def get_list_of_installed_packages(self, container_id): + process = Process() + error_code, stdout, stderr = process.run(f"docker exec {container_id} rpm --query --all") + assert error_code == 0, f"Detected unexpected return code: {error_code} (stderr: {stderr})" + + package_list = stdout.splitlines() + detected_packages = {} + + # general regex e.g. publicsuffix-list-dafsa-20180723-1.el8.noarch + # also handles dots in package name e.g. python3.12-pip-wheel-23.2.1-5.el9.noarch + rpm_list_pkg_regexp = re.compile(r"^([\w\.\-\+]+)\-([\w\.]+)\-([\w\.\+]+)\.(\w+)$") + # regex for packages that have no architecture info in name e.g. gpg-pubkey-fd431d51-4ae0493b + rpm_list_pkg_no_arch_regexp = re.compile(r"^([\w\.\-]+)\-(\w+)\-(\w+)$") + + for pkg in package_list: + match = rpm_list_pkg_regexp.match(pkg) + if match: + name, version, release, arch = match.groups() + else: + match = rpm_list_pkg_no_arch_regexp.match(pkg) + assert match, f"Unable to parse package info: {pkg}" + name, version, release = match.groups() + arch = "noarch" + detected_packages[name] = {"arch": arch, "version": version} + + return detected_packages + + def get_dependencies(self, pkg_name_list, container_id): + result = set() + process = Process() + cmd = f"docker exec {container_id} dnf" + error_code, stdout, stderr = process.run(cmd) + assert error_code == 0, ( + f"Please install dnf package. " f"Detected unexpected return code: {error_code} (stderr: {stderr})" + ) + # Example of output for curl tool: + # glibc-0:2.28-225.el8.i686 + # glibc-0:2.28-225.el8.x86_64 + # libcurl-0:7.61.1-30.el8_8.2.x86_64 + # openssl-libs-1:1.1.1k-9.el8_7.x86_64 + # zlib-0:1.2.11-21.el8_7.x86_64 + dnf_repoquery_regexp = re.compile(r"^([\w\-\+]+)\-\d\:([\w\.]+)\-([\w\.\+]+)\.(\w+)$") + for pkg_name in pkg_name_list: + cmd = f"docker exec {container_id} dnf repoquery --requires --resolve {pkg_name}" + error_code, stdout, stderr = process.run(cmd) + assert error_code == 0, f"Detected unexpected return code: {error_code} (stderr: {stderr})" + for line in stdout.splitlines(): + match = dnf_repoquery_regexp.match(line) + assert match, f"Unable to parse package info: {line}" + name = match.group(1) + result.add(name) + return list(result) + + +class MicrodnfPackageManager(DnfPackageManager): + + def __init__(self): + self.update_cmd = "microdnf update -y" + self.install_cmd = "microdnf install -y" + self.upgrade_cmd = "microdnf upgrade -y" + + +class YumPackageManager(PackageManager): + + def __init__(self): + self.update_cmd = "sudo yum update -y" + self.install_cmd = "sudo yum install {} -y" + self.upgrade_cmd = "sudo yum upgrade -y" + + def get_install_cmd(self, package_name): + return f"yum clean all; yum install {package_name} -y" + + def get_install_version_cmd(self, package_name, version): + return f"sudo yum install {package_name}-{version} -y" + + def get_upgrade_pkg_cmd(self, package_name): + return f"sudo yum upgrade {package_name} -y" + + def get_list_of_installed_packages(self, container_id): + process = Process() + error_code, stdout, stderr = process.run(f"docker exec {container_id} yum list installed | grep installed") + assert error_code == 0, f"Detect unexpected return code: {error_code} (stderr: {stderr})" + + package_list = stdout.splitlines() + detected_packages = {} + for pkg in package_list: + pkg_name, pkg_version, _ = pkg.split() + name, arch = pkg_name.split(".") + detected_packages[name] = {"arch": arch, "version": pkg_version} + + return detected_packages + + def get_dependencies(self, pkg_name_list, container_id): + for pkg_name in pkg_name_list: + if pkg_name != "curl": + pytest.skip(reason=f"Unable to collect dependencies for {pkg_name}!") + return [] + + +class AptPackageManager(PackageManager): + + def __init__(self): + self.update_cmd = "sudo apt update -y" + self.install_cmd = "sudo apt install {} -y" + self.upgrade_cmd = "sudo apt upgrade -y" + + def get_install_cmd(self, package_name): + return f"apt-get update -y && apt-get install -y {package_name}" + + def get_install_version_cmd(self, package_name, version): + return f"sudo apt install {package_name}={version} -y" + + def get_upgrade_pkg_cmd(self, package_name): + return f"sudo apt upgrade {package_name} -y" + + def get_list_of_installed_packages(self, container_id): + list_installed_cmd = "apt list --installed | grep installed" + if container_id is None: # get installed packages on host + cmd = list_installed_cmd + else: # get installed packages in container + cmd = f"docker exec {container_id} {list_installed_cmd}" + apt_list_pkg_regexp = re.compile(r"^([^\/]+)\/[^\s]+\s([^\s]+)\s(\w+)") + process = Process() + error_code, stdout, stderr = process.run(cmd) + assert error_code == 0, f"Detect unexpected return code: {error_code} (stderr: {stderr})" + + package_list = stdout.splitlines() + detected_packages = {} + for pkg in package_list: + # ppp/focal-updates,focal-security,now 2.4.7-2+4.1ubuntu5.1 amd64 [installed,automatic] + # pptp-linux/focal,now 1.10.0-1build1 amd64 [installed,automatic] + # procps/focal,now 2:3.3.16-1ubuntu2 amd64 [installed,upgradable to: 2:3.3.16-1ubuntu2.2] + # psmisc/focal,now 23.3-1 amd64 [installed,automatic] + # libgnutls30/now 3.6.13-2ubuntu1.3 amd64 [installed,upgradable to: 3.6.13-2ubuntu1.6] + # login/now 1:4.8.1-1ubuntu5.20.04 amd64 [installed,upgradable to: 1:4.8.1-1ubuntu5.20.04.1] + # passwd/now 1:4.8.1-1ubuntu5.20.04 amd64 [installed,upgradable to: 1:4.8.1-1ubuntu5.20.04.1] + match = apt_list_pkg_regexp.search(pkg) + assert match, f"Unable to parse package info: {pkg}" + name, version, arch = match.groups() + detected_packages[name] = {"arch": arch, "version": version} + + return detected_packages + + def get_dependencies(self, pkg_name_list, container_id): + result = [] + process = Process() + # Example of output for curl tool: + # WARNING: apt does not have a stable CLI interface. Use with caution in scripts. + # + # Reading package lists... + # Building dependency tree... + # Reading state information... + # The following packages were automatically installed and are no longer required: + # (...) + # libsasl2-modules-db libsqlite3-0 libssh-4 libwind0-heimdal publicsuffix + # Use 'apt autoremove' to remove them. + # The following packages will be REMOVED: + # curl + for pkg_name in pkg_name_list: + error_code, stdout, stderr = process.run(f"docker exec {container_id} apt -s remove {pkg_name}") + assert error_code == 0, f"Detect unexpected return code: {error_code} (stderr: {stderr})" + header = "The following packages were automatically installed and are no longer required:" + footer = "Use 'apt autoremove' to remove them." + start = stdout.find(header) + len(header) + end = stdout.find(footer) + lib_list_str = stdout[start:end] + for line in lib_list_str.splitlines(): + libs = line.strip().split() + for lib in libs: + if lib not in result: + result.append(lib) + return result diff --git a/tests/functional/object_model/python_custom_nodes/__init__.py b/tests/functional/object_model/python_custom_nodes/__init__.py new file mode 100644 index 0000000000..84cfbcf566 --- /dev/null +++ b/tests/functional/object_model/python_custom_nodes/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/functional/object_model/python_custom_nodes/common.py b/tests/functional/object_model/python_custom_nodes/common.py new file mode 100644 index 0000000000..bfb5500ab6 --- /dev/null +++ b/tests/functional/object_model/python_custom_nodes/common.py @@ -0,0 +1,51 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from pathlib import Path + +from tests.functional.config import ovms_c_repo_path + +PYTHON_CUSTOM_NODE_EXPECTED_CLASS_NAME = "OvmsPythonModel" +PYTHON_CUSTOM_NODES_DIR = Path(ovms_c_repo_path, "tests", "functional", "data", "python_custom_nodes") + + +class OvmsPythonModelFiles: + PYTHON_MODEL_FILEPATH = Path(PYTHON_CUSTOM_NODES_DIR, "ovms_basic/python_model.py") + PYTHON_MODEL_LOOPBACK_FILEPATH = Path(PYTHON_CUSTOM_NODES_DIR, "ovms_basic/python_model_loopback.py") + # corrupted model.py files + PYTHON_MODEL_EXCEPTIONS_FILEPATH = Path(PYTHON_CUSTOM_NODES_DIR, "ovms_corrupted/python_model_exceptions.py") + PYTHON_MODEL_CORRUPTED_IMPORT_FILEPATH = Path( + PYTHON_CUSTOM_NODES_DIR, "ovms_corrupted/python_model_corrupted_import.py" + ) + PYTHON_MODEL_MISSING_EXECUTE_FILEPATH = Path( + PYTHON_CUSTOM_NODES_DIR, "ovms_corrupted/python_model_missing_execute.py" + ) + PYTHON_INCREMENTER_FILEPATH = Path(PYTHON_CUSTOM_NODES_DIR, "incrementer/incrementer.py") + PYTHON_MODEL_LOOPBACK_RETURN_INSTEAD_OF_YIELD_FILEPATH = Path( + PYTHON_CUSTOM_NODES_DIR, "ovms_corrupted/python_model_loopback_return_instead_of_yield.py" + ) + PYTHON_MODEL_WRITING_TO_LOOPBACK_OUTPUT_IN_EXECUTE_FILEPATH = Path( + PYTHON_CUSTOM_NODES_DIR, "ovms_corrupted/python_model_writing_to_loopback_output_in_execute.py" + ) + PYTHON_MODEL_MULTIPLE_USE_OF_VALID_OUTPUTS_FILEPATH = Path( + PYTHON_CUSTOM_NODES_DIR, "ovms_corrupted/python_model_loopback_multiple_use_of_valid_outputs.py" + ) + + +STREAMING_CHANNEL_ARGS = [ + # Do not drop the connection for long workloads + ("grpc.http2.max_pings_without_data", 0), +] diff --git a/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py b/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py new file mode 100644 index 0000000000..43bcda8315 --- /dev/null +++ b/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py @@ -0,0 +1,450 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import numpy as np + +from tests.functional.utils.inference.communication import GRPC +from tests.functional.utils.logger import get_logger +from tests.functional.constants.generative_ai import GenerativeAIPluginConfig +from tests.functional.models.models_datasets import LanguageModelDataset +from tests.functional.constants.ovms import Ovms +from tests.functional.constants.pipelines import MediaPipe, NodesConnection, NodeType, PythonGraphNode +from tests.functional.object_model.mediapipe_calculators import HttpLLMCalculator, PythonCalculator, \ + ImageGenCalculator, EmbeddingsCalculatorOV, RerankCalculatorOV, S2tCalculator, T2sCalculator +from tests.functional import config + +logger = get_logger(__name__) + + +class SimplePythonCustomNodeMediaPipe(MediaPipe): + inputs_number = 1 + input_name: str = "text_input" + outputs_number = 1 + output_name: str = "text_output" + input_names: list = None + output_names: list = None + child_nodes: list = None + inputs: dict = None + outputs: list = None + base_path: str = "" + name: str = "python_model" + is_python_custom_node: bool = True + batch_size: int = 1 + + def __init__(self, handler_path, node_name="upper_text", loopback=False, initialize_graphs=True, **kwargs): + self.node_name = node_name + self.config = {} + self.prepare_llm_model_inputs_outputs(model=self, dataset=LanguageModelDataset, kwargs=kwargs) + + super().__init__(**kwargs) + self.calculators = [ + PythonCalculator(handler_path=handler_path, model=self, node_name=node_name, loopback=loopback) + ] + self.graphs = [calc.create_proto_content(model=self) for calc in self.calculators] if initialize_graphs else [] + + if self.child_nodes is None: + self.child_nodes = [] + if self.inputs is None: + self.inputs = {} + if self.outputs is None: + self.outputs = [] + + self.handler_path = handler_path + + @staticmethod + def prepare_llm_model_inputs_outputs(model, dataset, **kwargs): + inputs_number = kwargs.get("inputs_number", None) + model.inputs_number = inputs_number if inputs_number is not None else model.inputs_number + model.inputs = { + f"{model.input_name}{i}": {'shape': [-1, -1], 'dtype': str, 'dataset': dataset(data_sample=i)} + for i in range(model.inputs_number) + } + outputs_number = kwargs.get("outputs_number", None) + model.outputs_number = outputs_number if outputs_number is not None else model.outputs_number + model.outputs = { + f"{model.output_name}{i}": {'shape': [-1, 512], 'dtype': str} for i in range(model.outputs_number) + } + return model + + @property + def input_names(self): + return list(self.inputs.keys()) + + @property + def output_names(self): + return list(self.outputs.keys()) + + def get_config(self): + return None + + def prepare_resources(self, base_location): + resource_locations = [] + for calc in self.calculators: + resource_locations.append(calc.prepare_resources(base_location)) + return resource_locations + + def prepare_input_data(self, batch_size=None, input_key=None, dataset=None, input_data_type=None): + if dataset is not None: + if not isinstance(dataset, type): + dataset_obj = dataset + else: + dataset_obj = dataset() + input_data = {input_name: dataset_obj.get_data(None, None, None) for input_name in self.input_names} + elif input_data_type == "string": + input_data = { + input_name: self.inputs[input_name]["dataset"].get_string_data() + for input_name in self.input_names + } + else: + input_data = { + input_name: self.inputs[input_name]["dataset"].get_data(None, None, None) + for input_name in self.input_names + } + return input_data + + def get_expected_output(self, input_data: dict, client_type: str = None): + output_data = {} + + for i, input_name in enumerate(self.inputs.keys()): + if self.node_name == "incrementer": + multiply_value = 8 + decoded_text = "".join(item.decode() for i in range(multiply_value) for item in input_data[input_name]) + else: + multiply_value = 1 + decoded_text = "".join(item.decode() for item in input_data[input_name]).upper() + + if client_type == GRPC: + filler = Ovms.OUTPUT_FILLER + input_text = filler.join(decoded_text) + filler + char_array = [ord(char) for char in input_text] + output_data[self.output_names[i]] = np.array(char_array, dtype=np.object_) + else: + input_text = decoded_text + char_array = [ord(char) for char in input_text] + # For REST API and BYTES type, every batch is always preceding by the 4 bytes, that contains its size + # [42, 0, 0, 0] is constant value for "Lorem ipsum dolor sit amet" + # https://github.com/openvinotoolkit/model_server/blob/main/docs/model_server_rest_api_kfs.md + splitted = np.array_split(np.array(char_array, dtype=np.object_), multiply_value) + extended_with_length = [np.insert(elem, 0, [42, 0, 0, 0]) for elem in splitted] + output_data[self.output_names[i]] = np.concatenate(extended_with_length, dtype=np.object_) + + return output_data + + def validate_outputs(self, outputs, expected_output_shapes=None, provided_input=None): + assert outputs, f"No output collected for node {self.node_name}" + output_mismatch_error = f"Output mismatch for node {self.node_name} outputs: {outputs}" + if self.node_name == "upper_text": + assert len(outputs) == self.outputs_number, ( + f"Invalid outputs length: {len(outputs)}; " f"Expected: {self.outputs_number}" + ) + if self.batch_size is not None and self.batch_size > 1: + for _input_elem, _output_elem in zip(provided_input, outputs): + for i, j in zip(_input_elem, _output_elem): + assert i.upper() == j, output_mismatch_error + else: + for i, _input in enumerate(provided_input): + assert _input.upper() == outputs[i], output_mismatch_error + elif self.node_name == "loopback_upper_text": + # Expected output: [ + # "Lorem ipsum dolor sit amet", "LOrem ipsum dolor sit amet", "LoRem ipsum dolor sit amet", ... + # ] + _input = provided_input[0] + expected_outputs = [_input[:j] + _input[j].upper() + _input[j + 1:] for j in range(len(_input))] + assert expected_outputs == outputs, output_mismatch_error + elif self.node_name == "incrementer": + for i, _input in enumerate(provided_input): + # Incrementer value (2) defined in: incrementer.py + assert _input * 2 ** self.chain_length == outputs[i], output_mismatch_error + else: + raise NotImplementedError + + +class PythonCustomNodeChainMediaPipe(SimplePythonCustomNodeMediaPipe): + def __init__(self, models=None, handler_path=None, **kwargs): + self.node_name = "incrementer" + self.input_name = "first_number" + self.output_name = "last_number" + self.inputs = {self.input_name: {"shape": [-1, -1], "dtype": str, "dataset": LanguageModelDataset()}} + self.outputs = {self.output_name: {"shape": [-1, 512], "dtype": str}} + self.chain_length = len(models) + self.calculators = [] + self.models = models + self.handler_path = handler_path + self.config = {} + self.regular_models = [] + self.is_mediapipe = True + self.is_python_custom_node = True + + for model in self.models: + calc = PythonCalculator(handler_path=self.handler_path, model=model, node_name=model.node_name) + self.calculators.append(calc) + + if self.child_nodes is None: + self.child_nodes = [] + + self.create_header = False + self._initialize(models) + + def _create_nodes(self, models=None): + final_input_name = "first_number" + final_output_name = "last_number" + + request = PythonGraphNode("request", node_type=NodeType.Input, output_names=[final_input_name]) + output = PythonGraphNode("output", node_type=NodeType.Output, input_names=[final_output_name]) + + model_nodes = [] + for i, model in enumerate(models): + output_stream = f"{model.node_name}_{i}_output" + calculator = self.calculators[i] + if i == 0: + python_node = PythonGraphNode( + name=model.node_name, + model=model, + calculator=calculator, + input_stream=final_input_name, + output_stream=output_stream, + ) + model_nodes.append(python_node) + elif i == (len(models) - 1): + python_node = PythonGraphNode( + name=model.node_name, + model=model, + calculator=calculator, + input_stream=model_nodes[i - 1].output_stream, + output_stream=final_output_name, + ) + model_nodes.append(python_node) + elif 0 < i < len(models): + python_node = PythonGraphNode( + name=model.node_name, + model=model, + calculator=calculator, + input_stream=model_nodes[i - 1].output_stream, + output_stream=output_stream, + ) + model_nodes.append(python_node) + + for i, node in enumerate(model_nodes): + if i == 0: + NodesConnection.connect(node, 0, request, 0) + elif i == (len(model_nodes) - 1): + NodesConnection.connect(output, 0, model_nodes[i - 1], 0) + elif 0 < i < len(model_nodes): + NodesConnection.connect(node, 0, model_nodes[i - 1], 0) + + nodes = model_nodes + [request, output] + return nodes + + @staticmethod + def is_pipeline(): + return True + + +class SimpleLLM(SimplePythonCustomNodeMediaPipe): + inputs_number = 1 + input_name: str = "input" + outputs_number = 1 + output_name: str = "output" + child_nodes: list = None + inputs: dict = None + outputs: list = None + base_path: str = "" + name: str = "" + pbtxt_name: str = "simple_llm" + is_python_custom_node: bool = True + is_llm: bool = True + calculator_class = HttpLLMCalculator + precision: str = None + allows_reasoning: bool = False + + def __init__(self, models_path, node_name="LLMExecutor", loopback=True, initialize_graphs=True, **kwargs): + model = kwargs["model"] + self.node_name = node_name + self.config = {} + dataset = model.get_default_dataset() + self.prepare_llm_model_inputs_outputs(model=self, dataset=dataset, kwargs=kwargs) + + self.regular_models = [] + self.is_mediapipe = True + self.is_python_custom_node = True + best_of_limit = kwargs.get("best_of_limit", None) + max_tokens_limit = kwargs.get("max_tokens_limit", None) + kv_cache_size = kwargs.get("kv_cache_size", config.kv_cache_size_value) + plugin_config = kwargs.get( + "plugin_config", {GenerativeAIPluginConfig.KV_CACHE_PRECISION: config.kv_cache_precision_value} + ) + enable_tool_guided_generation = kwargs.get("enable_tool_guided_generation", False) + self.calculators = [ + self.calculator_class( + models_path=models_path, + model=model, + node_name=node_name, + loopback=loopback, + best_of_limit=best_of_limit, + max_tokens_limit=max_tokens_limit, + kv_cache_size=kv_cache_size, + plugin_config=plugin_config, + device=self.target_device, + enable_tool_guided_generation=enable_tool_guided_generation, + ) + ] + self.name = self.calculators[0].model.name + self.graphs = [calc.create_proto_content(model=self) for calc in self.calculators] if initialize_graphs else [] + + if self.child_nodes is None: + self.child_nodes = [] + if self.inputs is None: + self.inputs = {} + if self.outputs is None: + self.outputs = [] + + self.models_path = models_path + self.model_timeout = getattr(model, "model_timeout", None) + + +class SimpleImageGenerationLLM(SimpleLLM): + inputs_number = 1 + input_name: str = "input" + outputs_number = 1 + output_name: str = "output" + child_nodes: list = None + inputs: dict = None + outputs: list = None + base_path: str = "" + name: str = "" + pbtxt_name: str = "simple_llm" + is_python_custom_node: bool = True + is_llm: bool = True + calculator_class = ImageGenCalculator + precision: str = None + + def __init__(self, models_path, node_name="ImageGenExecutor", initialize_graphs=True, **kwargs): + model = kwargs["model"] + self.node_name = node_name + self.config = {} + dataset = model.get_default_dataset() + self.prepare_llm_model_inputs_outputs(model=self, dataset=dataset, kwargs=kwargs) + + self.regular_models = [] + self.is_mediapipe = True + self.is_python_custom_node = True + kv_cache_size = kwargs.get("kv_cache_size", config.kv_cache_size_value) + target_device = kwargs.get("target_device", self.target_device) + resolution = kwargs.get("resolution", None) + self.calculators = [ + self.calculator_class( + models_path=models_path, + model=model, + node_name=node_name, + kv_cache_size=kv_cache_size, + device=target_device, + resolution=resolution, + ) + ] + self.name = self.calculators[0].model.name + self.graphs = [calc.create_proto_content(model=self) for calc in self.calculators] if initialize_graphs else [] + + if self.child_nodes is None: + self.child_nodes = [] + if self.inputs is None: + self.inputs = {} + if self.outputs is None: + self.outputs = [] + + self.models_path = models_path + self.model_timeout = getattr(model, "model_timeout", None) + + +class SimpleFeatureExtractionLLM(SimpleLLM): + inputs_number: int = 1 + input_name: str = "input" + outputs_number: int = 1 + output_name: str = "output" + child_nodes: list = None + inputs: dict = None + outputs: list = None + base_path: str = "" + name: str = "" + pbtxt_name: str = "simple_llm" + is_python_custom_node: bool = True + is_llm: bool = True + use_subconfig: bool = True + is_feature_extraction: bool = True + calculator_class = EmbeddingsCalculatorOV + + def __init__(self, models_path, node_name="LLMExecutor", loopback=False, initialize_graphs=True, **kwargs): + super().__init__(models_path, node_name, loopback, initialize_graphs, **kwargs) + + +class SimpleRerankLLM(SimpleLLM): + inputs_number: int = 1 + input_name: str = "input" + outputs_number: int = 1 + output_name: str = "output" + child_nodes: list = None + inputs: dict = None + outputs: list = None + base_path: str = "" + name: str = "" + pbtxt_name: str = "simple_llm" + is_python_custom_node: bool = True + is_llm: bool = True + use_subconfig: bool = True + is_rerank: bool = True + calculator_class = RerankCalculatorOV + + def __init__(self, models_path, node_name="LLMExecutor", loopback=False, initialize_graphs=True, **kwargs): + super().__init__(models_path, node_name, loopback, initialize_graphs, **kwargs) + + +class SimpleAsrModel(SimpleLLM): + inputs_number: int = 1 + input_name: str = "input" + outputs_number: int = 1 + output_name: str = "output" + child_nodes: list = None + inputs: dict = None + outputs: list = None + base_path: str = "" + name: str = "" + pbtxt_name: str = "simple_llm" + is_python_custom_node: bool = True + is_llm: bool = False + is_asr_model: bool = True + calculator_class = S2tCalculator + + def __init__(self, models_path, node_name="S2tExecutor", loopback=False, initialize_graphs=True, **kwargs): + super().__init__(models_path, node_name, loopback, initialize_graphs, **kwargs) + + +class SimpleTtsModel(SimpleLLM): + inputs_number: int = 1 + input_name: str = "input" + outputs_number: int = 1 + output_name: str = "output" + child_nodes: list = None + inputs: dict = None + outputs: list = None + base_path: str = "" + name: str = "" + pbtxt_name: str = "simple_llm" + is_python_custom_node: bool = True + is_llm: bool = False + is_tts_model: bool = True + calculator_class = T2sCalculator + + def __init__(self, models_path, node_name="T2sExecutor", loopback=False, initialize_graphs=True, **kwargs): + super().__init__(models_path, node_name, loopback, initialize_graphs, **kwargs) diff --git a/tests/functional/object_model/resource_monitor.py b/tests/functional/object_model/resource_monitor.py new file mode 100644 index 0000000000..45f48ddc44 --- /dev/null +++ b/tests/functional/object_model/resource_monitor.py @@ -0,0 +1,144 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import csv +import threading +from abc import ABC, abstractmethod +from pathlib import Path + +import numpy as np +from dateutil import parser + +from tests.functional.utils.logger import get_logger +from tests.functional.config import artifacts_dir + +logger = get_logger(__name__) + + +class ResourceMonitor(threading.Thread, ABC): + def __init__(self): + threading.Thread.__init__(self) + self._stop_event = threading.Event() + + def stop(self): + self._stop_event.set() + self.join() + + def run(self): + while not self._stop_event.is_set(): + try: + self.check_resources() + except StopIteration as e: + self._stop_event.set() + break + self.save_data() + + @abstractmethod + def check_resources(self): + pass + + @abstractmethod + def save_data(self): + pass + + +class DockerResourceMonitor(ResourceMonitor): + MEMORY_USAGE = "MEMORY_USAGE" + FIELDS = ["DATE", "PIDS_COUNT", MEMORY_USAGE] # + ["CPU_USAGE"] # Enable in further releases + FIELDS_TO_STATS = { + "DATE": lambda x: x["read"], + "PIDS_COUNT": lambda x: int(x["pids_stats"].get("current", "0")), + MEMORY_USAGE: lambda x: "{:.2f}M".format(float(x["memory_stats"].get("usage", "0.0")) / (2**20)), + # Enable after debug & fixing + # "CPU_USAGE": lambda x: + # [cpu / x['cpu_stats']['cpu_usage']['total_usage'] for cpu in x['cpu_stats']['cpu_usage']['percpu_usage']], + } + + def __init__(self, container): + super().__init__() + self.container = container.container + self._docker_stats_stream = self.container.stats(stream=False, decode=False) + self._docker_stats_data_raw = [] + + def cleanup(self): + if not self._stop_event.is_set(): + if self.is_alive(): + self.stop() + self.save_data() + + def get_field_data(self, field, stats): + value = DockerResourceMonitor.FIELDS_TO_STATS[field](stats) + return value + + def save_data(self): + self.rows = [] + # Parse raw stats + for stats in self._docker_stats_data_raw: + row = {} + for field in DockerResourceMonitor.FIELDS: + row[field] = self.get_field_data(field, stats) + self.rows.append(row) + log_path = Path(artifacts_dir, f"docker_stats_{self.container.name}.log") + with log_path.open("w") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=DockerResourceMonitor.FIELDS) + writer.writeheader() + writer.writerows(self.rows) + return log_path + + def plot_fo_file(self, x, y, field, filename): + pass # Uncomment when fixed regressions + # plt.clf() # clear trash if any + # plt.plot(x, y) + # plt.xlabel('Time (s)') + # plt.title(field) + # plt.savefig(os.path.join(artifacts_dir, filename)) + # plt.clf() # clear after drawing + + def save_diagrams(self): + started = parser.parse(self.rows[0]["DATE"]) + x = np.array([(parser.parse(x["DATE"]) - started).seconds for x in self.rows]) + for field in DockerResourceMonitor.FIELDS[1:]: + y = np.array([x[field] for x in self.rows]) + filename = "diagram_{}_{}.png".format(field, self.container.name) + self.plot_fo_file(x, y, field, filename) + + def check_resources(self): + result = self._get_resource_data() + self._docker_stats_data_raw.append(result) + + def _get_resource_data(self): + """ + Example stats: + 'read' = {str} '2022-01-12T12:49:43.4286187Z' + 'preread' = {str} '0001-01-01T00:00:00Z' + 'pids_stats' = {dict: 2} {'current': 20, 'limit': 9384} + 'blkio_stats' = {dict: 8} {'io_service_bytes_recursive': ... + 'num_procs' = {int} 0 + 'storage_stats' = {dict: 0} {} + 'cpu_stats' = {dict: 4} {'cpu_usage': {'total_usage': 94376000, 'percpu_usage': ... + 'precpu_stats' = {dict: 2} {'cpu_usage': {'total_usage': 0, 'usage_in_kernelmode': 0, ... + 'memory_stats' = {dict: 4} {'usage': 7974912, 'max_usage': 8007680, 'stats': {'active_anon': 3448832 ... + 'name' = {str} '/testrjasinsk_12_134942_101685' + 'id' = {str} 'b18a5c94499d60f6bb98f844883e283cb3e154b269bbc89874f112d788dc050c' + 'networks' = {dict: 1} {'eth0': {'rx_bytes': 90, 'rx_packets': 1, 'rx_errors': 0, ... + """ + stats = self.container.stats(stream=False, decode=False) + return stats + + def get_stats_by_field(self, field): + result = self._get_resource_data() + self._docker_stats_data_raw.append(result) + return self.get_field_data(field, result) diff --git a/tests/functional/object_model/server.py b/tests/functional/object_model/server.py deleted file mode 100644 index fbe817e5e0..0000000000 --- a/tests/functional/object_model/server.py +++ /dev/null @@ -1,75 +0,0 @@ -# -# Copyright (c) 2019-2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import tests.functional.config as config -from tests.functional.object_model.ovms_binary import OvmsBinary -from tests.functional.object_model.ovms_docker import OvmsDocker -import logging - -logger = logging.getLogger(__name__) - - -class Server: - running_instances = [] - - def __init__(self, request, command_args, container_name_infix, start_container_command, - env_vars=None, image=config.image, container_log_line=config.container_log_line, - server_log_level=config.log_level, target_device=None): - self.request = request - self.command_args = command_args - self.container_name_infix = container_name_infix - self.start_container_command = start_container_command - self.env_vars = env_vars - self.image = image - self.container_log_line = container_log_line - self.server_log_level = server_log_level - self.target_device = target_device - self.started_by_fixture = request.fixturename - - def start(self): - assert self not in Server.running_instances - - if config.ovms_binary_path is not None: - self.ovms = OvmsBinary(self.request, self.command_args, self.start_container_command, self.env_vars) - else: - self.ovms = OvmsDocker(self.request, self.command_args, self.container_name_infix, - self.start_container_command, self.env_vars, self.image, - self.container_log_line, self.server_log_level, self.target_device, server=self) - start_result = None - try: - start_result = self.ovms.start() - finally: - if start_result is None: - self.stop() - else: - Server.running_instances.append(self) - return start_result - - def stop(self): - self.ovms.stop() - if self in Server.running_instances: - Server.running_instances.remove(self) - - @classmethod - def stop_all_instances(cls): - for instance in cls.running_instances: - instance.stop() - - @classmethod - def stop_by_fixture_name(cls, fixture_name): - instance = list(filter(lambda x: x.started_by_fixture == fixture_name, cls.running_instances)) - if instance: - logger.debug(f"Stopping server instance spawned by {fixture_name}") - instance[0].stop() diff --git a/tests/functional/object_model/shape.py b/tests/functional/object_model/shape.py new file mode 100644 index 0000000000..31b2c8397d --- /dev/null +++ b/tests/functional/object_model/shape.py @@ -0,0 +1,120 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.constants.ovms import Ovms + +SHAPE_AND_LAYOUT_EFFECTS_TABLE = """ + Shape and layout effects table. + +Native Model ; Native Model Layout ; Model Optimizer ;OVMS layout parameter ;Expected OVMS metadata ; Is ; Is binary resize +Shape ; (Inherited from ; layout parameter ; ; ; binary ; auto alignment + ; training ; ; ; ; supported ; supported + ; framework) ; ; ; ; ; + ; ; ; ; ; ; +(1,224,224,3) ; NHWC ; ; ;(1,224,224,3) N... ;yes ; no +(1,224,224,3) ; NHWC ; ;--layout NCHW ;(1,224,224,3) NCHW ;no ; n/a +(1,224,224,3) ; NHWC ; ;--layout NHWC ;(1,224,224,3) NHWC ;yes ; yes +(1,224,224,3) ; NHWC ; ;--layout NCHW:NHWC ;(1,3,224,224) NCHW ;no ; n/a +(1,224,224,3) ; NHWC ; ;--layout NHWC:NCHW ;(1,224,3,224) NHWC ;yes ; yes +(1,224,224,3) ; NHWC ; --layout NHWC ; ;(1,224,224,3) NHWC ;yes ; yes +(1,224,224,3) ; NHWC ; --layout NHWC ;--layout NCHW ;(1,224,224,3) NCHW ;no ; n/a +(1,224,224,3) ; NHWC ; --layout NHWC ;--layout NHWC ;(1,224,224,3) NHWC ;yes ; yes +(1,224,224,3) ; NHWC ; --layout NHWC ;--layout NCHW:NHWC ;(1,3,224,224) NCHW ;no ; n/a +(1,224,224,3) ; NHWC ; --layout NHWC ;--layout NHWC:NCHW ;(1,224,3,224) NHWC ;yes ; yes +(1,224,224,3) ; NHWC ; --layout NCHW ; ;(1,224,224,3) NCHW ;no ; n/a +(1,224,224,3) ; NHWC ; --layout NCHW ;--layout NCHW ;(1,224,224,3) NCHW ;no ; n/a +(1,224,224,3) ; NHWC ; --layout NCHW ;--layout NHWC ;(1,224,224,3) NHWC ;yes ; yes +(1,224,224,3) ; NHWC ; --layout NCHW ;--layout NCHW:NHWC ;(1,3,224,224) NCHW ;no ; n/a +(1,224,224,3) ; NHWC ; --layout NCHW ;--layout NHWC:NCHW ;(1,224,3,224) NHWC ;yes ; yes +(1,224,224,3) ; NHWC ; --layout NHWC->NCHW ; ;(1,3,224,224) NCHW ;no ; n/a +(1,224,224,3) ; NHWC ; --layout NHWC->NCHW ;--layout NCHW ;(1,3,224,224) NCHW ;no ; n/a +(1,224,224,3) ; NHWC ; --layout NHWC->NCHW ;--layout NHWC ;(1,3,224,224) NHWC ;yes ; yes +(1,224,224,3) ; NHWC ; --layout NHWC->NCHW ;--layout NCHW:NHWC ;(1,224,3,224) NCHW ;no ; n/a +(1,224,224,3) ; NHWC ; --layout NHWC->NCHW ;--layout NHWC:NCHW ;(1,224,224,3) NHWC ;yes ; yes +(1,224,224,3) ; NHWC ; --layout NCHW->NHWC ; ;(1,224,3,224) NHWC ;yes ; yes +(1,224,224,3) ; NHWC ; --layout NCHW->NHWC ;--layout NCHW ;(1,224,3,224) NCHW ;no; n ./a +(1,224,224,3) ; NHWC ; --layout NCHW->NHWC ;--layout NHWC ;(1,224,3,224) NHWC ;yes ; yes +(1,224,224,3) ; NHWC ; --layout NCHW->NHWC ;--layout NCHW:NHWC ;(1,224,224,3) NCHW ;no ; n/a +(1,224,224,3) ; NHWC ; --layout NCHW->NHWC ;--layout NHWC:NCHW ;(1,224,224,3) NHWC ;yes ; yes +(1,3,224,224) ; NCHW ; ; ;(1,3,224,224) N... ;yes ; no +(1,3,224,224) ; NCHW ; ;--layout NCHW ;(1,3,224,224) NCHW ;no ; n/a +(1,3,224,224) ; NCHW ; ;--layout NHWC ;(1,3,224,224) NHWC ;yes ; yes +(1,3,224,224) ; NCHW ; ;--layout NCHW:NHWC ;(1,224,3,224) NCHW ;no ; n/a +(1,3,224,224) ; NCHW ; ;--layout NHWC:NCHW ;(1,224,224,3) NHWC ;yes ; yes +(1,3,224,224) ; NCHW ; --layout NHWC ; ;(1,3,224,224) NHWC ;yes ; yes +(1,3,224,224) ; NCHW ; --layout NHWC ;--layout NCHW ;(1,3,224,224) NCHW ;no ; n/a +(1,3,224,224) ; NCHW ; --layout NHWC ;--layout NHWC ;(1,3,224,224) NHWC ;yes ; yes +(1,3,224,224) ; NCHW ; --layout NHWC ;--layout NCHW:NHWC ;(1,224,3,224) NCHW ;no ; n/a +(1,3,224,224) ; NCHW ; --layout NHWC ;--layout NHWC:NCHW ;(1,224,224,3) NHWC ;yes ; yes +(1,3,224,224) ; NCHW ; --layout NCHW ; ;(1,3,224,224) NCHW ;no ; n/a +(1,3,224,224) ; NCHW ; --layout NCHW ;--layout NCHW ;(1,3,224,224) NCHW ;no ; n/a +(1,3,224,224) ; NCHW ; --layout NCHW ;--layout NHWC ;(1,3,224,224) NHWC ;yes ; yes +(1,3,224,224) ; NCHW ; --layout NCHW ;--layout NCHW:NHWC ;(1,224,3,224) NCHW ;no ; n/a +(1,3,224,224) ; NCHW ; --layout NCHW ;--layout NHWC:NCHW ;(1,224,224,3) NHWC ;yes ; yes +(1,3,224,224) ; NCHW ; --layout NHWC->NCHW ; ;(1,224,3,224) NCHW ;no ; n/a +(1,3,224,224) ; NCHW ; --layout NHWC->NCHW ;--layout NCHW ;(1,224,3,224) NCHW ;no ; n/a +(1,3,224,224) ; NCHW ; --layout NHWC->NCHW ;--layout NHWC ;(1,224,3,224) NHWC ;yes ; yes +(1,3,224,224) ; NCHW ; --layout NHWC->NCHW ;--layout NCHW:NHWC ;(1,224,224,3) NCHW ;no ; n/a +(1,3,224,224) ; NCHW ; --layout NHWC->NCHW ;--layout NHWC:NCHW ;(1,3,224,224) NHWC ;yes ; yes +(1,3,224,224) ; NCHW ; --layout NCHW->NHWC ; ;(1,224,224,3) NHWC ;yes ; yes +(1,3,224,224) ; NCHW ; --layout NCHW->NHWC ;--layout NCHW ;(1,224,224,3) NCHW ;no ; n/a +(1,3,224,224) ; NCHW ; --layout NCHW->NHWC ;--layout NHWC ;(1,224,224,3) NHWC ;yes ; yes +(1,3,224,224) ; NCHW ; --layout NCHW->NHWC ;--layout NCHW:NHWC ;(1,3,224,224) NCHW ;no ; n/a +(1,3,224,224) ; NCHW ; --layout NCHW->NHWC ;--layout NHWC:NCHW ;(1,224,3,224) NHWC ;yes ; yes + +""" + + +class Shape(list): + layout = None + mappings = [] + + def __init__(self, _list, _layout=None): + self.init_by_list(_list, _layout) + + def set_layout(self, _list, _layout=None): + if not _layout: + # Set default layout + if len(_list) == 2: + self.layout = "NC" + elif len(_list) == 3: + self.layout = "NCW" + elif len(_list) == 4: + self.layout = "NCHW" + elif len(_list) == 5: + self.layout = "NCHWD" + else: + assert "Unrecognized shape" + else: + if _layout in [Ovms.LAYOUT_NHWC, Ovms.LAYOUT_NCHW]: + _layout = _layout.split(":")[0] + else: + assert 0 + self.layout = _layout + assert len(_list) == len(_layout) + + def init_by_list(self, _list, _layout=None): + self.set_layout(_list, _layout) + for i in range(len(_list)): + setattr(self, self.layout[i], _list[i]) + self[:] = _list[:] + + def get_shape_by_layout(self, layout=None): + if not layout: + layout = self.layout + if layout: + return [getattr(self, letter) for letter in layout] + return self[:] diff --git a/tests/functional/object_model/test_environment.py b/tests/functional/object_model/test_environment.py new file mode 100644 index 0000000000..3c10356943 --- /dev/null +++ b/tests/functional/object_model/test_environment.py @@ -0,0 +1,90 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import os +import socket +from pathlib import Path + +from tests.functional.utils.logger import get_logger +from tests.functional.config import server_address + +logger = get_logger(__name__) + + +class TestEnvironment(object): + __test__ = False + current = None + + def __init__(self, base_dir): + self.base_dir = base_dir + + @staticmethod + def update_model_files(model, models_dir): + if hasattr(model, "max_position_embeddings") and model.max_position_embeddings is not None: + config_file_path = os.path.join(models_dir[0], model.name, "config.json") + if os.path.exists(config_file_path): + with open(config_file_path, "r") as fo: + config_data = json.load(fo) + config_data["max_position_embeddings"] = model.max_position_embeddings + with open(config_file_path, "w") as fo: + json.dump(config_data, fo) + logger.info( + f"max_position_embeddings value was updated to {model.max_position_embeddings} " + f"in model's config file: {config_file_path}." + ) + + def prepare_container_folders(self, dir_name, models): + """ + Method execute prepare_resources on each model. + + Parameters: + name (str): Tmp name of a container + models (List[ModelInfo]): List of resources. + + Returns: + str: location of resources directory path on host (container folder) + Set(str): set of models directory path on host + """ + resources_dir = os.path.join(self.base_dir, dir_name) + Path(resources_dir).mkdir(parents=True, exist_ok=True) + models_dir_on_host = set() + for model in models or []: + models_dir = model.prepare_resources(resources_dir) + models_dir_on_host.update(models_dir) + self.update_model_files(model, models_dir) + return resources_dir, models_dir_on_host + + @staticmethod + def get_server_address(): + final_server_address = ( + os.environ.get("REMOTE_SERVER_ADDRESS") + if os.environ.get("REMOTE_SERVER_ADDRESS") is not None + else server_address + ) + return final_server_address + + @staticmethod + def get_local_ip(): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + s.connect(("", 0)) + return s.getsockname()[0] + + @staticmethod + def get_ip_from_hostname(hostname): + ip = socket.gethostbyname(hostname) + return ip diff --git a/tests/functional/object_model/test_helpers.py b/tests/functional/object_model/test_helpers.py new file mode 100644 index 0000000000..00cd7938de --- /dev/null +++ b/tests/functional/object_model/test_helpers.py @@ -0,0 +1,333 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import concurrent.futures +import enum +import requests + +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from http import HTTPStatus +from math import prod +from retry.api import retry_call +from statistics import mean + +from tests.functional.utils.assertions import InvalidReturnCodeException +from tests.functional.utils.logger import get_logger, step + +from tests.functional.object_model.test_environment import TestEnvironment + +logger = get_logger(__name__) + + +def run_in_loop_during(action_to_run_in_loop, parallel_action, runs): + for run in range(runs): + logger.info(f"Run: {run}") + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(parallel_action) + + counter = 0 + while not future.done(): + action_to_run_in_loop() + counter += 1 + + future.result() + logger.info(f"Main thread action executed {counter} times.") + + +def run_all_actions_in_loop(actions, runs, max_workers=None): + logger.info(f"Starting n={runs} parallel actions={','.join(map(lambda x: str(x), actions))}") + with ThreadPoolExecutor(max_workers) as executor: + futures = [] + for run in range(runs): + logger.debug(f"Run: {run}") + futures.extend([executor.submit(action) for action in actions]) + for future in concurrent.futures.as_completed(futures): + future.result() + logger.info("All actions finished") + + +def run_all_actions(action, arguments_list, max_workers=None): + result = [] + with ThreadPoolExecutor(max_workers=max_workers) as executor: + thread_list = [] + for arguments in arguments_list: + thread = executor.submit(action, *arguments) + thread_list.append(thread) + + for thread in concurrent.futures.as_completed(thread_list): + thread_result = thread.result() + result.append(thread_result) + return result + + +# Model Control API Support +class Endpoints(enum.Enum): + GET_CONFIG = "/v1/config" + RELOAD_CONFIG = "/v1/config/reload" + + +def send_request_to_endpoint(port, address=None, endpoint=None, expected_code=None, retry=1, timeout=60): + address = TestEnvironment.get_server_address() if address is None else address + url_with_endpoint = f"http://{address}:{port}{endpoint}" + logger.info(f"Try to send request to endpoint: {url_with_endpoint}") + + if endpoint == Endpoints.RELOAD_CONFIG.value: + func = requests.post + elif endpoint == Endpoints.GET_CONFIG.value: + func = requests.get + else: + msg = f"Not supported endpoint: {endpoint}" + raise ValueError(msg) + retry_setup = {"tries": int(retry), "delay": 1} + kwargs = {"url": url_with_endpoint, "params": {}, "timeout": timeout} + ret = retry_call(func, fkwargs=kwargs, **retry_setup) + if expected_code is None: + return ret + + if ret.status_code != expected_code: + logger.warning(f"Response text:\n{ret.text}") + msg1 = f"Received status code is {ret.status_code}." + msg2 = f"Expected return code is {expected_code}." + if all([ret.status_code == HTTPStatus.OK.value, + expected_code == HTTPStatus.CREATED.value, + endpoint == Endpoints.RELOAD_CONFIG.value]): + logger.warning(f"{msg1} {msg2} Both of those codes are accepted.") + return ret + elif not ret.status_code == expected_code: + raise InvalidReturnCodeException(f"{msg1} {msg2}") + logger.info(msg1) + return ret + + +def send_reload_request(port, address=None, expected_code=None, retry=1, timeout=60): + address = TestEnvironment.get_server_address() if address is None else address + endpoint = Endpoints.RELOAD_CONFIG.value + return send_request_to_endpoint(port, address, endpoint, expected_code, retry, timeout=timeout) + + +def get_config_request(port, address=None, expected_code=None, retry=1): + address = TestEnvironment.get_server_address() if address is None else address + endpoint = Endpoints.GET_CONFIG.value + return send_request_to_endpoint(port, address, endpoint, expected_code, retry) + + +def _generate_permutations(input_shape, shape_results, example_input_data_for_predict, skip_first_items=0): + result = [] + result_predict_shape = [] + shape_count = [len(v) for _, v in shape_results.items()] + for i in range(prod(shape_count)): + item = {k: None for k in input_shape} + example_array = [] + + tmp_shape_count = shape_count.copy() + tmp_shape_count.reverse() + + input_array = {} + for in_name in input_shape: + current_cnt = tmp_shape_count.pop() + idx = (i // prod(tmp_shape_count)) % current_cnt + item[in_name] = shape_results[in_name][idx] + input_array[in_name] = example_input_data_for_predict[in_name][idx] + + result.append(item) + + example_cnt = [len(v) for _, v in input_array.items()] + for j in range(prod(example_cnt)): + item = defaultdict(None) + tmp_example_cnt = example_cnt.copy() + tmp_example_cnt.reverse() + for in_name in input_array: + current_cnt = tmp_example_cnt.pop() + idx = (j // prod(tmp_example_cnt)) % current_cnt + item[in_name] = input_array[in_name][idx] + example_array.append(item) + result_predict_shape.append(example_array) + return result[skip_first_items:], result_predict_shape[skip_first_items:] + + +def generate_dynamic_shape_permutation(model): + """ + Generate possible shape with -1 and example input data shape for predict operation for testing purposes (tuple) + For example input_shape = {'in': [1, 3, 224]}. + This example contains 3 dim so we have 2^3 -1 (shape without -1 is not dynamic) + Output: + Output (note len of first and second output arguments are the same): + first output argument: {: } + {'in': '(-1,3,224)'} + {'in': '(1,-1,224)'} + {'in': '(-1,-1,224)'} + {'in': '(1,3,-1)'} + {'in': '(-1,3,-1)'} + {'in': '(1,-1,-1)'} + {'in': '(-1,-1,-1)'} + second output argument: [{: }] + [{'in': [1, 3, 224]}, {'in': [2, 3, 224]}], + [{'in': [1, 1, 224]}, {'in': [1, 6, 224]}], + [{'in': [1, 1, 224]}, {'in': [2, 1, 224]}, {'in': [1, 6, 224]}, {'in': [2, 6, 224]}], + [{'in': [1, 3, 112]}, {'in': [1, 3, 448]}], + [{'in': [1, 3, 224]}, {'in': [2, 3, 224]}, {'in': [1, 3, 448]}, {'in': [2, 3, 448]}], + [{'in': [1, 1, 224]}, {'in': [1, 6, 224]}, {'in': [1, 1, 448]}, {'in': [1, 6, 448]}], + [{'in': [1, 1, 224]}, {'in': [2, 1, 224]}, {'in': [1, 6, 224]}, {'in': [2, 6, 224]}, + {'in': [1, 1, 448]}, {'in': [2, 1, 448]}, {'in': [1, 6, 448]}, {'in': [2, 6, 448]}] + + {'': (, )} + {'in': ('(-1,3,224)', [[1, 3, 224], [2, 3, 224]]) + {'in': ('(1,-1,224), [[1, 1, 224], [1, 6, 224]]) + {'in': ('(-1,-1,224), [[1, 1, 224], [2, 1, 224], [1, 6, 224], [2, 6, 224]]) + {'in': ('(1,3,-1), [[1, 3, 112], [1, 3, 448]]) + {'in': ('(-1,3,-1), [[1, 3, 224], [2, 3, 224], [1, 3, 448], [2, 3, 448]]) + {'in': ('(1,-1,-1), [[1, 1, 224], [1, 6, 224], [1, 1, 448], [1, 6, 448]]) + {'in': ('(-1,-1,-1)', [[1, 1, 224], [2, 1, 224], [1, 6, 224], [2, 6, 224], + [1, 1, 448], [2, 1, 448], [1, 6, 448], [2, 6, 448]]) + """ + input_shape = model.input_shapes.copy() + shape_results = defaultdict(lambda: []) + example_input_shape_for_predict_operation = defaultdict(lambda: []) + for in_name, shape in input_shape.items(): + layout = None + if len(shape) >= 4: + layout = model.inputs[in_name].get("layout", "NCHW") + + for i in range(2 ** len(shape)): + new_shape = shape.copy() + shape_for_predict_list = [shape.copy()] + for dim in range(len(shape)): + if (i >> dim) % 2 == 1: + new_shape[dim] = -1 + + copy_shape_for_predict_list = deepcopy(shape_for_predict_list) + for for_predict, copy_for_predict in zip(shape_for_predict_list, copy_shape_for_predict_list): + for_predict[dim] = max(1, shape[dim] // 2) + if layout is not None and layout.index("C") == dim: + copy_for_predict[dim] = shape[dim] + else: + copy_for_predict[dim] = shape[dim] * 2 + shape_for_predict_list += copy_shape_for_predict_list + + shape_results[in_name].append(f"({','.join([str(x) for x in new_shape])})") + example_input_shape_for_predict_operation[in_name].append(shape_for_predict_list) + return _generate_permutations(input_shape, shape_results, example_input_shape_for_predict_operation, 1) + + +def generate_range_shape_permutation(model, skip_dims=2, generate_low_range=None, generate_high_range=None): + """ + Generate possible shape with range and example of input data shape for predict operation: + for example input_shape = [1, 2, 3, 4]. + For dim = 4 - function generate different sequence for dim 3, 4 (skip_dims = 2 -> skip dim 1, 2) + Function generate possible permutation for dim(input_shape) - skip_dims - in + or example perm(2) = 2^2 - 1 (option without range is not dynamic). + Function create range based on value from input shape: + {generate_low_range(input_shape[dim])}:{generate_high_range(input_shape[dim])} + Default values are: + generate_low_range(x) = x // 2 + generate_high_range(x) = x * 2 + For example input: {'in': [1, 3, 224, 224]} + Output (note len of first and second output arguments are the same): + first output argument: {: } + {'in': '(1,3,112:448,224)'}, + {'in': '(1,3,224,112:448)'}, + {'in': '(1,3,112:448,112:448)' + second output argument: [{: }] + [{'in': [1, 3, 112, 224]}, {'in': [1, 3, 448, 224]}], + [{'in': [1, 3, 224, 112]}, {'in': [1, 3, 224, 448]}], + [{'in': [1, 3, 112, 112]}, {'in': [1, 3, 448, 112]}, {'in': [1, 3, 112, 448]}, {'in': [1, 3, 448, 448]}] + """ + input_shape = model.input_shapes.copy() + if generate_low_range is None: + generate_low_range = lambda x: x // 2 + if generate_high_range is None: + generate_high_range = lambda x: x * 2 + + shape_results = defaultdict(lambda: []) + example_input_shape_for_predict_operation = defaultdict(lambda: []) + for in_name, shape in input_shape.items(): + layout = None + if len(shape) >= 4: + layout = model.inputs[in_name].get("layout", "NCHW") + + for i in range(2 ** len(shape[skip_dims:])): + new_shape = shape[skip_dims:] + shape_for_predict_list = [shape.copy()] + for dim in range(len(new_shape)): + if (i >> dim) % 2 == 1: + high_value = generate_high_range(new_shape[dim]) + if layout is not None and layout.index("C") == (dim + skip_dims): + copy_for_predict[dim] = shape[dim] + + new_shape[dim] = f"{generate_low_range(new_shape[dim])}:{high_value}" + + copy_shape_for_predict_list = deepcopy(shape_for_predict_list) + for for_predict, copy_for_predict in zip(shape_for_predict_list, copy_shape_for_predict_list): + for_predict[dim + skip_dims] = max(1, generate_low_range(shape[dim + skip_dims])) + copy_for_predict[dim + skip_dims] = generate_high_range(shape[dim + skip_dims]) + shape_for_predict_list += copy_shape_for_predict_list + + new_item = [str(x) for x in shape[:skip_dims]] + [str(x) for x in new_shape] + shape_results[in_name].append(f"({','.join(new_item)})") + example_input_shape_for_predict_operation[in_name].append(shape_for_predict_list) + return _generate_permutations(input_shape, shape_results, example_input_shape_for_predict_operation, 1) + + +def check_initial_memory_usage(context, result, model): + step("Log initial memory usage") + memory_stats = [] + errors_logged = [] + res_monitor = result.attach_resource_monitor(context, start=False) + memory_stats.append(float(res_monitor.get_stats_by_field(res_monitor.MEMORY_USAGE).replace("M", ""))) + logger.info(f"{model.name} initial memory usage:\t{memory_stats[0]}M") + return memory_stats, errors_logged, res_monitor + + +def validate_memory_usage( + iteration, + memory_stats, + errors_logged, + validate_initial_memory_usage=True, + check_memory_usage_initial_multiplier=2, + check_memory_usage_multiplier=1.5, +): + if len(memory_stats) > 2: + current_memory_usage = memory_stats[-1] + initial_memory_usage = memory_stats[0] + first_iter_memory_usage = memory_stats[1] + + check_memory_usage_initial = current_memory_usage < check_memory_usage_initial_multiplier * initial_memory_usage + if validate_initial_memory_usage and not check_memory_usage_initial: + errors_logged.append( + f"Too much memory usage! Initial: {initial_memory_usage}M; " + f"{iteration} iteration: {current_memory_usage}M" + ) + check_memory_usage = current_memory_usage < check_memory_usage_multiplier * first_iter_memory_usage + if not check_memory_usage: + errors_logged.append( + f"Too much memory usage! 1st iteration: {first_iter_memory_usage}M; " + f"{iteration} iteration: {current_memory_usage}M" + ) + + +def log_memory_usage_stats(memory_stats, errors_logged): + initial_memory_usage = memory_stats[0] + average_memory_usage = mean(memory_stats[1:]) + minimum_memory_usage = min(memory_stats[1:]) + maximum_memory_usage = max(memory_stats[1:]) + + logger.info(f"Initial memory usage:\t{initial_memory_usage}") + logger.info(f"Average inference memory usage:\t{average_memory_usage}") + logger.info(f"Minimum inference memory usage:\t{minimum_memory_usage}") + logger.info(f"Maximum inference memory usage:\t{maximum_memory_usage}") + assert not errors_logged, f"Too much memory usage! Errors logged: {errors_logged}" diff --git a/tests/functional/object_model/tools.py b/tests/functional/object_model/tools.py new file mode 100644 index 0000000000..233a82163b --- /dev/null +++ b/tests/functional/object_model/tools.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from tests.functional.constants.ovms import Ovms + + +class Valgrind: + name = "valgrind" + basic_params_set = f"--quiet --max-threads={Ovms.MAX_THREADS_VALGRIND}" + full_params_set = f"--leak-check=full --show-leak-kinds=all --track-origins=yes --verbose " \ + f"--max-threads={Ovms.MAX_THREADS_VALGRIND}" + + @classmethod + def get_valgrind_params(cls, valgrind_mode="basic"): + params = cls.basic_params_set if valgrind_mode == "basic" else cls.full_params_set + return params + + +class Cliloader: + # https://wiki.ith.intel.com/display/OVMS/How+to+capture+openCL+traces + name = "cliloader" + path = os.path.join("/opencl-intercept-layer/install/bin", name) + env = {"CLI_CallLogging": "1", "CLI_DevicePerformanceTiming": "1"} diff --git a/tests/functional/test_llm_json.py b/tests/functional/test_llm_json.py deleted file mode 100644 index c20a93f249..0000000000 --- a/tests/functional/test_llm_json.py +++ /dev/null @@ -1,360 +0,0 @@ -# -# Copyright (c) 2025 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -from jsonschema import validate, ValidationError -import logging -from openai import OpenAI -import os -from pydantic import BaseModel -import pytest -import requests - -model_name = os.getenv("MODEL_NAME", "meta-llama/Llama-3.1-8B-Instruct") -base_url = os.getenv("BASE_URL", "http://localhost:8000/v3") - -logger = logging.getLogger(__name__) -xfail = pytest.mark.xfail -skip = pytest.mark.skip - - -@pytest.mark.priority_low -class TestSingleModelInference: - - @skip(reason="not implemented yet") - @pytest.mark.api_enabling - def test_chat_with_tool_definition(self): - """ - Description - sending a content with user question with tools definition. Response should be the json file compatible with tool schema. - - input data - - OpenAI chat with tools definition - - Expected results - - json file compatible with tool schema - - """ - - client = OpenAI(base_url=base_url, api_key="unused") - - tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. Bogotá, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } - }] - - messages = [ - {"role": "user", "content": "What is the weather like in Paris today?"} - ] - completion = client.chat.completions.create( - model=model_name, - messages=messages, - tools=tools, - tool_choice={"type": "function", "function": {"name": "get_weather"}}, - ) - - print("COMPLETION:",completion) - - body = { - "model": model_name, - "messages": messages, - "tools":tools, - "tool_choice": {"type": "function", "function": {"name": "get_weather"}}, - "max_tokens": 1, - } - - response = requests.post( - url=f"{base_url}/chat/completions", - json=body, - headers={"Authorization": f"Bearer unused"} - ) - - if response.status_code == 200: - print("API Response:", response.json()) - else: - print(f"Failed to get response: {response.status_code}, {response.text}") - - tool_args = completion.choices[0].message.tool_calls[0].function.arguments - - # Assert that tool_call is a valid JSON and matches the schema - try: - tool_call_json = json.loads(tool_args) - schema = tools[0]["function"]["parameters"] - validate(instance=tool_call_json, schema=schema) - assert True, "tool_call is a valid JSON and matches the schema" - except json.JSONDecodeError as e: - assert False, f"tool_call is not a valid JSON: {e}" - except ValidationError as e: - assert False, f"tool_call does not match the schema: {e}" - - assert completion.choices[0].message.tool_calls[0].id != "" - - messages.append({'role': 'assistant', 'reasoning_content': None, 'content': '', 'tool_calls': [{'id': completion.choices[0].message.tool_calls[0].id, 'type': 'function', 'function': {'name': 'get_weather', 'arguments': '{"location": "Paris, France"}'}}]}) - messages.append({"role": "tool", "tool_call_id": completion.choices[0].message.tool_calls[0].id, "content": "15 degrees Celsius"}) - - print("Messages after tool call:", messages) - - completion = client.chat.completions.create( - model=model_name, - messages=messages, - tools=tools - ) - print(completion.choices[0].message) - - assert "Paris" in completion.choices[0].message.content - assert "15 degrees" in completion.choices[0].message.content - assert completion.choices[0].message.tool_calls is None or completion.choices[0].message.tool_calls == [] - - @skip(reason="not implemented yet") - @pytest.mark.api_enabling - def test_chat_with_dual_tools_definition(self): - """ - Description - sending a content with user question with tools definition. Response should be the json file compatible with tool schema. - - input data - - OpenAI chat with tools definition - - Expected results - - json file compatible with tool schema - - """ - client = OpenAI(base_url=base_url, api_key="unused") - - tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. Bogotá, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } - }, - { - "type": "function", - "function": { - "name": "get_pollutions", - "description": "Get current level of air pollutions for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. Bogotá, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } - }] - - messages = [ - {"role": "user", "content": "What is the temperature and pollution level in New York?"} - ] - completion = client.chat.completions.create( - model=model_name, - messages=messages, - tools=tools, - tool_choice="auto" - ) - - print("COMPLETION:",completion) - - tool_args0 = completion.choices[0].message.tool_calls[0].function.arguments - tool_args1 = completion.choices[0].message.tool_calls[1].function.arguments - - # Assert that tool_call is a valid JSON and matches the schema - - try: - tool_call_json = json.loads(tool_args0) - schema = tools[0]["function"]["parameters"] - validate(instance=tool_call_json, schema=schema) - tool_call_json = json.loads(tool_args1) - schema = tools[1]["function"]["parameters"] - validate(instance=tool_call_json, schema=schema) - assert True, "tool_call is a valid JSON and matches the schema" - except json.JSONDecodeError as e: - assert False, f"tool_call is not a valid JSON: {e}" - except ValidationError as e: - assert False, f"tool_call does not match the schema: {e}" - - messages.append(completion.choices[0].message) - messages.append({"role": "tool", "tool_call_id": completion.choices[0].message.tool_calls[0].id, "content": "15 degrees Celsius"}) - messages.append({"role": "tool", "tool_call_id": completion.choices[0].message.tool_calls[1].id, "content": "pm10 28µg/m3"}) - - print("Messages after tool call:", messages) - - # Llama3 does not support multiple tools in a single chat completion call - if "Llama3" in model_name: - with pytest.raises(Exception): - # This should raise an exception because we cannot use multiple tools in a single chat completion call - client.chat.completions.create( - model=model_name, - messages=messages, - tools=tools - ) - else: - completion = client.chat.completions.create( - model=model_name, - messages=messages, - tools=tools - ) - content = completion.choices[0].message.content - assert "New York" in content - assert "15 degrees Celsius" in content or "15°C" in content or "15 °C" in content - assert "pm10" in content or "PM10" in content - assert "28 µg/m" in content or "28µg/m" in content - - @skip(reason="not implemented yet") - @pytest.mark.api_enabling - def test_chat_with_tool_definition_stream(self): - """ - Description - sending a content with user question with tools definition. Response should be the json file compatible with tool schema. - - input data - - OpenAI chat with tools definition - - Expected results - - json file compatible with tool schema - - """ - client = OpenAI(base_url=base_url, api_key="unused") - - tools = [{ - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. Bogotá, Colombia" - } - }, - "required": [ - "location" - ], - "additionalProperties": False - }, - "strict": True - } - }] - - messages = [ - {"role": "user", "content": "What is the weather like in Paris today?"} - ] - completion = client.chat.completions.create( - model=model_name, - messages=messages, - tools=tools, - tool_choice="auto", - stream=True, - ) - arguments = "" - function_name = "" - for chunk in completion: - if chunk.choices[0].delta.tool_calls is not None: - if chunk.choices[0].delta.tool_calls[0].function.name is not None: - function_name = chunk.choices[0].delta.tool_calls[0].function.name - print("Function name:", function_name) - assert chunk.choices[0].delta.tool_calls[0].function.name == "get_weather" - if chunk.choices[0].delta.tool_calls[0].function.arguments is not None: - arguments += chunk.choices[0].delta.tool_calls[0].function.arguments - assert arguments == '{"location": "Paris, France"}' - - @skip(reason="not implemented yet") - @pytest.mark.api_enabling - def test_chat_with_structured_output(self): - """ - Description - sending a content with user question with tools definition. Response should be the json file compatible with tool schema. - - input data - - OpenAI chat with tools definition - - Expected results - - json file compatible with tool schema - - """ - client = OpenAI(base_url=base_url, api_key="unused") - class CalendarEvent(BaseModel): - event_name: str - date: str - participants: list[str] - - completion = client.beta.chat.completions.parse( - model=model_name, - messages=[ - {"role": "system", "content": "Extract the event information and place them in json format."}, - {"role": "user", "content": "Alice and Bob are going to a Science Fair on Friday."}, - ], - temperature=0.0, - max_tokens=100, - response_format=CalendarEvent, - ) - - print("CalendarEvent as JSON:", json.dumps(CalendarEvent.schema(), indent=2)) - print("COMPLETION CONTENT:",completion.choices[0].message.content) - json_str = completion.choices[0].message.content - try: - schema = CalendarEvent.schema() - validate(instance=json.loads(json_str), schema=schema) - print("json_str is compatible with the schema defined in CalendarEvent.") - except ValidationError as e: - assert False, f"json_str does not match the schema: {e}" - except json.JSONDecodeError as e: - assert False, f"json_str is not a valid JSON: {e}" - event = completion.choices[0].message.parsed - print("Parsed event:", event) - assert event.event_name.lower() == "science fair".lower() - assert event.date == "Friday" - assert event.participants == ["Alice", "Bob"] diff --git a/tests/functional/test_model_version_policy.py b/tests/functional/test_model_version_policy.py deleted file mode 100644 index 098c4c9f80..0000000000 --- a/tests/functional/test_model_version_policy.py +++ /dev/null @@ -1,253 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import pytest -import requests -from google.protobuf.json_format import Parse -from tensorflow_serving.apis import get_model_metadata_pb2, get_model_status_pb2 # noqa - -from tests.functional.constants.constants import MODEL_SERVICE, NOT_TO_BE_REPORTED_IF_SKIPPED -from tests.functional.constants.target_device import TargetDevice -from tests.functional.config import skip_nginx_test -from tests.functional.conftest import devices_not_supported_for_test -from tests.functional.model.models_information import AgeGender, PVBDetection, PVBFaceDetectionV2 -from tests.functional.utils.grpc import create_channel, get_model_metadata_request, get_model_metadata, \ - model_metadata_response, get_model_status -import logging -from tests.functional.utils.models_utils import ModelVersionState, ErrorCode, ERROR_MESSAGE # noqa -from tests.functional.utils.rest import get_metadata_url, get_status_url, get_model_status_response_rest - -logger = logging.getLogger(__name__) - - -@pytest.mark.priority_low -@pytest.mark.skipif(skip_nginx_test, reason=NOT_TO_BE_REPORTED_IF_SKIPPED) -@devices_not_supported_for_test([TargetDevice.NPU]) -class TestModelVerPolicy: - - @pytest.mark.parametrize("model_name, throw_error", [ - ('all', [False, False, False]), - ('specific', [False, True, False]), - ('latest', [True, False, False]), - ]) - @pytest.mark.api_enabling - def test_get_model_metadata(self, model_version_policy_models, - start_server_model_ver_policy, - model_name, throw_error): - - _, ports = start_server_model_ver_policy - logger.info("Downloaded model files: {}".format(model_version_policy_models)) - - # Connect to grpc service - stub = create_channel(port=ports["grpc_port"]) - - versions = [1, 2, 3] - expected_outputs_metadata = [ - {PVBDetection.output_name: {'dtype': 1, 'shape': list(PVBDetection.output_shape)}}, - {PVBFaceDetectionV2.output_name: {'dtype': 1, 'shape': list(PVBFaceDetectionV2.output_shape)}}] - expected_output_metadata = {} - for output_name, shape in AgeGender.output_shape.items(): - expected_output_metadata[output_name] = {'dtype': 1, 'shape': list(shape)} - expected_outputs_metadata.append(expected_output_metadata) - expected_inputs_metadata = [ - {PVBDetection.input_name: {'dtype': 1, 'shape': list(PVBDetection.input_shape)}}, - {PVBFaceDetectionV2.input_name: {'dtype': 1, 'shape': list(PVBFaceDetectionV2.input_shape)}}, - {AgeGender.input_name: {'dtype': 1, 'shape': list(AgeGender.input_shape)}}] - - for x in range(len(versions)): - logger.info("Getting info about model version: {}".format(versions[x])) - expected_input_metadata = expected_inputs_metadata[x] - expected_output_metadata = expected_outputs_metadata[x] - request = get_model_metadata_request(model_name=model_name, - version=versions[x]) - if not throw_error[x]: - response = get_model_metadata(stub, request) - input_metadata, output_metadata = model_metadata_response( - response=response) - - logger.info("Input metadata: {}".format(input_metadata)) - logger.info("Output metadata: {}".format(output_metadata)) - - assert model_name == response.model_spec.name - assert expected_input_metadata == input_metadata - assert expected_output_metadata == output_metadata - else: - with pytest.raises(Exception) as e: - get_model_metadata(stub, request) - assert "Model with requested version is not found" in str(e.value) - - @pytest.mark.parametrize("model_name, throw_error", [ - ('all', [False, False, False]), - ('specific', [False, True, False]), - ('latest', [True, False, False]), - ]) - @pytest.mark.api_enabling - def test_get_model_status(self, model_version_policy_models, - start_server_model_ver_policy, - model_name, throw_error): - - _, ports = start_server_model_ver_policy - logger.info("Downloaded model files: {}".format(model_version_policy_models)) - - # Connect to grpc service - stub = create_channel(port=ports["grpc_port"], service=MODEL_SERVICE) - - versions = [1, 2, 3] - for x in range(len(versions)): - request = get_model_status(model_name=model_name, - version=versions[x]) - if not throw_error[x]: - response = stub.GetModelStatus(request, 60) - versions_statuses = response.model_version_status - version_status = versions_statuses[0] - assert version_status.version == versions[x] - assert version_status.state == ModelVersionState.AVAILABLE - assert version_status.status.error_code == ErrorCode.OK - assert version_status.status.error_message == ERROR_MESSAGE[ - ModelVersionState.AVAILABLE][ErrorCode.OK] - else: - with pytest.raises(Exception) as e: - stub.GetModelStatus(request, 60) - assert "Model with requested version is not found" in str(e.value) - - # aggregated results check - if model_name == 'all': - request = get_model_status(model_name=model_name) - response = stub.GetModelStatus(request, 60) - versions_statuses = response.model_version_status - assert len(versions_statuses) == 3 - for version_status in versions_statuses: - assert version_status.state == ModelVersionState.AVAILABLE - assert version_status.status.error_code == ErrorCode.OK - assert version_status.status.error_message == ERROR_MESSAGE[ - ModelVersionState.AVAILABLE][ErrorCode.OK] - - @pytest.mark.parametrize("model_name, throw_error", [ - ('all', [False, False, False]), - ('specific', [False, True, False]), - ('latest', [True, False, False]), - ]) - @pytest.mark.api_enabling - def test_get_model_metadata_rest(self, model_version_policy_models, - start_server_model_ver_policy, - model_name, throw_error): - """ - Description - Execute GetModelMetadata request using REST API interface - hosting multiple models - - input data - - directory with 2 models in IR format - - docker image - - fixtures used - - model downloader - - input data downloader - - service launching - - Expected results - - response contains proper response about model metadata for both - models set in config file: - model resnet_v1_50, pnasnet_large - - both served models handles appropriate input formats - - """ - - _, ports = start_server_model_ver_policy - logger.info("Downloaded model files: {}".format(model_version_policy_models)) - - logger.info("Getting info about models") - versions = [1, 2, 3] - expected_outputs_metadata = [ - {PVBDetection.output_name: {'dtype': 1, 'shape': list(PVBDetection.output_shape)}}, - {PVBFaceDetectionV2.output_name: {'dtype': 1, 'shape': list(PVBFaceDetectionV2.output_shape)}}] - expected_output_metadata = {} - for output_name, shape in AgeGender.output_shape.items(): - expected_output_metadata[output_name] = {'dtype': 1, 'shape': list(shape)} - expected_outputs_metadata.append(expected_output_metadata) - expected_inputs_metadata = [ - {PVBDetection.input_name: {'dtype': 1, 'shape': list(PVBDetection.input_shape)}}, - {PVBFaceDetectionV2.input_name: {'dtype': 1, 'shape': list(PVBFaceDetectionV2.input_shape)}}, - {AgeGender.input_name: {'dtype': 1, 'shape': list(AgeGender.input_shape)}}] - - for x in range(len(versions)): - logger.info("Getting info about model version: {}".format(versions[x])) - expected_input_metadata = expected_inputs_metadata[x] - expected_output_metadata = expected_outputs_metadata[x] - rest_url = get_metadata_url(model=model_name, port=ports["rest_port"], version=str(versions[x])) - result = requests.get(rest_url) - logger.info("Result: {}".format(result.text)) - if not throw_error[x]: - output_json = result.text - metadata_pb = get_model_metadata_pb2. \ - GetModelMetadataResponse() - response = Parse(output_json, metadata_pb, - ignore_unknown_fields=True) - input_metadata, output_metadata = model_metadata_response( - response=response) - - logger.info("Input metadata: {}".format(input_metadata)) - logger.info("Output metadata: {}".format(output_metadata)) - - assert model_name == response.model_spec.name - assert expected_input_metadata == input_metadata - assert expected_output_metadata == output_metadata - else: - assert 404 == result.status_code - - @pytest.mark.parametrize("model_name, throw_error", [ - ('all', [False, False, False]), - ('specific', [False, True, False]), - ('latest', [True, False, False]), - ]) - @pytest.mark.api_enabling - def test_get_model_status_rest(self, model_version_policy_models, - start_server_model_ver_policy, - model_name, throw_error): - - _, ports = start_server_model_ver_policy - logger.info("Downloaded model files: {}".format(model_version_policy_models)) - - versions = [1, 2, 3] - for x in range(len(versions)): - rest_url = get_status_url(model=model_name, port=ports["rest_port"], version=str(versions[x])) - result = requests.get(rest_url) - if not throw_error[x]: - output_json = result.text - status_pb = get_model_status_pb2.GetModelStatusResponse() - response = Parse(output_json, status_pb, - ignore_unknown_fields=False) - versions_statuses = response.model_version_status - version_status = versions_statuses[0] - assert version_status.version == versions[x] - assert version_status.state == ModelVersionState.AVAILABLE - assert version_status.status.error_code == ErrorCode.OK - assert version_status.status.error_message == ERROR_MESSAGE[ - ModelVersionState.AVAILABLE][ErrorCode.OK] - else: - assert 404 == result.status_code - - # aggregated results check - if model_name == 'all': - rest_url = get_status_url(model=model_name, port=ports["rest_port"]) - response = get_model_status_response_rest(rest_url) - versions_statuses = response.model_version_status - assert len(versions_statuses) == 3 - for version_status in versions_statuses: - assert version_status.state == ModelVersionState.AVAILABLE - assert version_status.status.error_code == ErrorCode.OK - assert version_status.status.error_message == ERROR_MESSAGE[ - ModelVersionState.AVAILABLE][ErrorCode.OK] diff --git a/tests/functional/test_model_versions_handling.py b/tests/functional/test_model_versions_handling.py deleted file mode 100644 index 918bd6a8d3..0000000000 --- a/tests/functional/test_model_versions_handling.py +++ /dev/null @@ -1,170 +0,0 @@ -# -# Copyright (c) 2018-2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import pytest -import numpy as np - -from tests.functional.config import skip_nginx_test -from tests.functional.constants.constants import MODEL_SERVICE, NOT_TO_BE_REPORTED_IF_SKIPPED -from tests.functional.constants.target_device import TargetDevice -from tests.functional.conftest import devices_not_supported_for_test -from tests.functional.model.models_information import PVBFaceDetectionV2, PVBFaceDetection -from tests.functional.utils.grpc import create_channel, infer, get_model_metadata_request, get_model_metadata, model_metadata_response, \ - get_model_status -import logging -from tests.functional.utils.models_utils import ModelVersionState, ErrorCode, \ - ERROR_MESSAGE # noqa -from tests.functional.utils.rest import get_predict_url, get_metadata_url, get_status_url, infer_rest, \ - get_model_metadata_response_rest, get_model_status_response_rest - -logger = logging.getLogger(__name__) - - -@pytest.mark.priority_low -@pytest.mark.skipif(skip_nginx_test, reason=NOT_TO_BE_REPORTED_IF_SKIPPED) -@devices_not_supported_for_test([TargetDevice.GPU, TargetDevice.NPU]) -class TestModelVersionHandling: - model_name = "pvb_face_multi_version" - - @pytest.mark.parametrize("version", [1, 2, None], ids=("version 1", "version 2", "no version specified")) - @pytest.mark.api_enabling - def test_run_inference(self, start_server_multi_model, version): - - _, ports = start_server_multi_model - - # Connect to grpc service - stub = create_channel(port=ports["grpc_port"]) - model_info = PVBFaceDetectionV2 if version is None else PVBFaceDetection[version - 1] - - img = np.ones(model_info.input_shape, dtype=model_info.dtype) - - output = infer(img, input_tensor=model_info.input_name, - grpc_stub=stub, model_spec_name=self.model_name, - model_spec_version=version, # face detection - output_tensors=[model_info.output_name]) - logger.info("Output shape: {}".format(output[model_info.output_name].shape)) - assert output[model_info.output_name].shape == model_info.output_shape, \ - '{} with version 1 has invalid output'.format(self.model_name) - - @pytest.mark.parametrize("version", [1, 2, None], ids=("version 1", "version 2", "no version specified")) - @pytest.mark.api_enabling - def test_get_model_metadata(self, start_server_multi_model, version): - - _, ports = start_server_multi_model - - # Connect to grpc service - stub = create_channel(port=ports["grpc_port"]) - model_info = PVBFaceDetectionV2 if version is None else PVBFaceDetection[version - 1] - - logger.info("Getting info about pvb_face_detection model " - "version: {}".format("no_version" if version is None else version)) - expected_input_metadata = {model_info.input_name: {'dtype': 1, 'shape': list(model_info.input_shape)}} - expected_output_metadata = {model_info.output_name: {'dtype': 1, 'shape': list(model_info.output_shape)}} - - request = get_model_metadata_request(model_name=self.model_name, - version=version) - response = get_model_metadata(stub, request) - input_metadata, output_metadata = model_metadata_response( - response=response) - logger.info("Input metadata: {}".format(input_metadata)) - logger.info("Output metadata: {}".format(output_metadata)) - - assert response.model_spec.name == self.model_name - assert expected_input_metadata == input_metadata - assert expected_output_metadata == output_metadata - - @pytest.mark.parametrize("version", [1, 2, None], ids=("version 1", "version 2", "no version specified")) - @pytest.mark.api_enabling - def test_get_model_status(self, start_server_multi_model, version): - - _, ports = start_server_multi_model - - # Connect to grpc service - stub = create_channel(port=ports["grpc_port"], service=MODEL_SERVICE) - request = get_model_status(model_name=self.model_name, - version=version) - response = stub.GetModelStatus(request, 60) - - versions_statuses = response.model_version_status - version_status = versions_statuses[0] - if version is None: - assert len(versions_statuses) == 2 - else: - assert version_status.version == version - assert version_status.state == ModelVersionState.AVAILABLE - assert version_status.status.error_code == ErrorCode.OK - assert version_status.status.error_message == ERROR_MESSAGE[ - ModelVersionState.AVAILABLE][ErrorCode.OK] - - @pytest.mark.parametrize("version", [1, 2, None], ids=("version 1", "version 2", "no version specified")) - @pytest.mark.api_enabling - def test_run_inference_rest(self, start_server_multi_model, version): - - _, ports = start_server_multi_model - - model_info = PVBFaceDetectionV2 if version is None else PVBFaceDetection[version - 1] - - img = np.ones(model_info.input_shape, dtype=model_info.dtype) - rest_url = get_predict_url(model=self.model_name, port=ports["rest_port"], version=version) - output = infer_rest(img, - input_tensor=model_info.input_name, rest_url=rest_url, - output_tensors=[model_info.output_name], - request_format='column_name') - logger.info("Output shape: {}".format(output[model_info.output_name].shape)) - assert output[model_info.output_name].shape == model_info.output_shape, \ - '{} with version 1 has invalid output'.format(self.model_name) - - @pytest.mark.parametrize("version", [1, 2, None], ids=("version 1", "version 2", "no version specified")) - @pytest.mark.api_enabling - def test_get_model_metadata_rest(self, start_server_multi_model, version): - - _, ports = start_server_multi_model - - model_info = PVBFaceDetectionV2 if version is None else PVBFaceDetection[version - 1] - - rest_url = get_metadata_url(model=self.model_name, port=ports["rest_port"], version=version) - - expected_input_metadata = {model_info.input_name: {'dtype': 1, 'shape': list(model_info.input_shape)}} - expected_output_metadata = {model_info.output_name: {'dtype': 1, 'shape': list(model_info.output_shape)}} - logger.info("Getting info about resnet model version: {}".format(rest_url)) - response = get_model_metadata_response_rest(rest_url) - input_metadata, output_metadata = model_metadata_response(response=response) - logger.info("Input metadata: {}".format(input_metadata)) - logger.info("Output metadata: {}".format(output_metadata)) - - assert response.model_spec.name == self.model_name - assert expected_input_metadata == input_metadata - assert expected_output_metadata == output_metadata - - @pytest.mark.parametrize("version", [1, 2, None], ids=("version 1", "version 2", "no version specified")) - @pytest.mark.api_enabling - def test_get_model_status_rest(self, start_server_multi_model, version): - - _, ports = start_server_multi_model - - rest_url = get_status_url(model=self.model_name, port=ports["rest_port"], version=version) - - response = get_model_status_response_rest(rest_url) - versions_statuses = response.model_version_status - version_status = versions_statuses[0] - if version is None: - assert len(versions_statuses) == 2 - else: - assert version_status.version == version - assert version_status.state == ModelVersionState.AVAILABLE - assert version_status.status.error_code == ErrorCode.OK - assert version_status.status.error_message == ERROR_MESSAGE[ - ModelVersionState.AVAILABLE][ErrorCode.OK] diff --git a/tests/functional/test_reshaping.py b/tests/functional/test_reshaping.py deleted file mode 100644 index b39b0a298d..0000000000 --- a/tests/functional/test_reshaping.py +++ /dev/null @@ -1,181 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -import pytest -from tests.functional.constants.constants import ERROR_SHAPE, NOT_TO_BE_REPORTED_IF_SKIPPED -from tests.functional.constants.target_device import TargetDevice -from tests.functional.config import skip_nginx_test -from tests.functional.conftest import devices_not_supported_for_test -from tests.functional.model.models_information import FaceDetection -from tests.functional.utils.grpc import create_channel, infer -import logging -from tests.functional.utils.rest import get_predict_url, infer_rest - -logger = logging.getLogger(__name__) - -auto_shapes = [ - {'in': (1, 3, 300, 300), 'out': (1, 1, 200, 7)}, - {'in': (1, 3, 500, 500), 'out': (1, 1, 200, 7)}, - {'in': (1, 3, 224, 224), 'out': (1, 1, 200, 7)}, - {'in': (4, 3, 224, 224), 'out': (1, 1, 800, 7)}, - {'in': (8, 3, 312, 142), 'out': (1, 1, 1600, 7)}, - {'in': (1, 3, 1024, 1024), 'out': (1, 1, 200, 7)}, -] - -fixed_shape = {'in': (1, 3, 600, 600), 'out': (1, 1, 200, 7)} - - -@pytest.mark.priority_low -@pytest.mark.skipif(skip_nginx_test, reason=NOT_TO_BE_REPORTED_IF_SKIPPED) -@devices_not_supported_for_test([TargetDevice.GPU, TargetDevice.NPU]) -class TestModelReshaping: - - @pytest.mark.parametrize( - "shape, is_correct", [(fixed_shape['in'], True), (FaceDetection.input_shape, False)] - ) - @pytest.mark.api_enabling - def test_single_local_model_reshaping_fixed(self, start_server_face_detection_model_named_shape, - start_server_face_detection_model_nonamed_shape, shape, is_correct): - - _, ports_named = start_server_face_detection_model_named_shape - _, ports_nonamed = start_server_face_detection_model_nonamed_shape - - # Connect to grpc service - stubs = [create_channel(port=ports_named["grpc_port"]), create_channel(port=ports_nonamed["grpc_port"])] - imgs = np.zeros(shape, FaceDetection.dtype) - - for stub in stubs: - self.run_inference_grpc(imgs, FaceDetection.output_name, fixed_shape['out'], - is_correct, FaceDetection.name, stub) - - @pytest.mark.parametrize("shape, is_correct", - [(fixed_shape['in'], True), (FaceDetection.input_shape, - False)]) - @pytest.mark.parametrize("request_format", - ['row_name', 'row_noname', - 'column_name', 'column_noname']) - @pytest.mark.api_enabling - def test_single_local_model_reshaping_fixed_rest(self, start_server_face_detection_model_named_shape, - start_server_face_detection_model_nonamed_shape, shape, is_correct, - request_format): - - _, ports_named = start_server_face_detection_model_named_shape - _, ports_nonamed = start_server_face_detection_model_nonamed_shape - - imgs = np.zeros(shape, FaceDetection.dtype) - rest_ports = [ports_named["rest_port"], ports_nonamed["rest_port"]] - for rest_port in rest_ports: - rest_url = get_predict_url(model="face_detection", port=rest_port) - self.run_inference_rest(imgs, FaceDetection.output_name, fixed_shape['out'], - is_correct, request_format, rest_url) - - @pytest.mark.api_enabling - def test_multi_local_model_reshaping_auto(self, start_server_multi_model): - - _, ports = start_server_multi_model - - # Connect to grpc service - stub = create_channel(port=ports["grpc_port"]) - - for shape in auto_shapes: - imgs = np.zeros(shape['in'], FaceDetection.dtype) - self.run_inference_grpc(imgs, FaceDetection.output_name, shape['out'], True, - "face_detection_auto", stub) - - @pytest.mark.parametrize("shape, is_correct", - [(fixed_shape['in'], True), (FaceDetection.input_shape, - False)]) - @pytest.mark.api_enabling - def test_multi_local_model_reshaping_fixed(self, start_server_multi_model, shape, is_correct): - - _, ports = start_server_multi_model - - # Connect to grpc service - stub = create_channel(port=ports["grpc_port"]) - - models_names = ["face_detection_fixed_nonamed", - "face_detection_fixed_named"] - - imgs = np.zeros(shape, FaceDetection.dtype) - for model_name in models_names: - self.run_inference_grpc(imgs, FaceDetection.output_name, fixed_shape['out'], - is_correct, model_name, stub) - - @pytest.mark.parametrize("request_format", - ['row_name', 'row_noname', - 'column_name', 'column_noname']) - @pytest.mark.api_enabling - def test_multi_local_model_reshaping_auto_rest(self, start_server_multi_model, request_format): - - _, ports = start_server_multi_model - - for shape in auto_shapes: - imgs = np.zeros(shape['in'], FaceDetection.dtype) - rest_url = get_predict_url(model="face_detection_auto", port=ports["rest_port"]) - self.run_inference_rest(imgs, FaceDetection.output_name, shape['out'], True, - request_format, rest_url) - - @pytest.mark.parametrize("shape, is_correct", - [(fixed_shape['in'], True), (FaceDetection.input_shape, - False)]) - @pytest.mark.parametrize("request_format", - ['row_name', 'row_noname', - 'column_name', 'column_noname']) - @pytest.mark.api_enabling - def test_multi_local_model_reshaping_fixed_rest(self, start_server_multi_model, shape, is_correct, request_format): - - _, ports = start_server_multi_model - - models_names = ["face_detection_fixed_nonamed", - "face_detection_fixed_named"] - imgs = np.zeros(shape, FaceDetection.dtype) - for model_name in models_names: - rest_url = get_predict_url(model=model_name, port=ports["rest_port"]) - self.run_inference_rest(imgs, FaceDetection.output_name, fixed_shape['out'], - is_correct, request_format, rest_url) - - @staticmethod - def run_inference_rest(imgs, out_name, out_shape, is_correct, - request_format, rest_url): - logger.info("Running rest inference call") - output = infer_rest(imgs, input_tensor='data', - rest_url=rest_url, - output_tensors=[out_name], - request_format=request_format, - raise_error=is_correct) - if is_correct: - logger.info("Output shape: {}".format(output[out_name].shape)) - assert output[out_name].shape == out_shape, ERROR_SHAPE - else: - assert not output - - @staticmethod - def run_inference_grpc(imgs, out_name, out_shape, is_correct, model_name, stub): - logger.info(f"Running grpc inference call") - if is_correct: - output = infer(imgs, input_tensor=FaceDetection.input_name, grpc_stub=stub, - model_spec_name=model_name, - model_spec_version=None, - output_tensors=[out_name]) - logger.info("Output shape: {}".format(output[out_name].shape)) - assert output[out_name].shape == out_shape, ERROR_SHAPE - else: - with pytest.raises(Exception): - infer(imgs, input_tensor=FaceDetection.input_name, grpc_stub=stub, - model_spec_name=model_name, - model_spec_version=None, - output_tensors=[out_name]) From dabcb553a7804caa853760dea73c14e9b1b2a00e Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Fri, 8 May 2026 13:13:39 +0200 Subject: [PATCH 04/12] utils --- tests/functional/constants/generative_ai.py | 49 ++ tests/functional/utils/cleanup.py | 93 --- tests/functional/utils/context.py | 87 +++ tests/functional/utils/core.py | 166 +++++ tests/functional/utils/docker.py | 444 ++++++++++++ tests/functional/utils/environment_info.py | 160 +++++ .../utils/generative_ai/__init__.py | 15 + .../utils/generative_ai/validation_utils.py | 569 ++++++++++++++++ tests/functional/utils/git_operations.py | 78 +++ tests/functional/utils/grpc.py | 114 ---- tests/functional/utils/helpers.py | 38 +- tests/functional/utils/hooks.py | 412 +++++++++++ tests/functional/utils/http/__init__.py | 15 + tests/functional/utils/http/base.py | 78 +++ .../utils/http/client_auth/__init__.py | 15 + .../functional/utils/http/client_auth/auth.py | 566 ++++++++++++++++ .../functional/utils/http/client_auth/base.py | 81 +++ .../utils/http/client_auth/exceptions.py | 26 + tests/functional/utils/http/exceptions.py | 29 + tests/functional/utils/http/http_client.py | 97 +++ .../utils/http/http_client_configuration.py | 131 ++++ .../utils/http/http_client_factory.py | 83 +++ tests/functional/utils/http/http_session.py | 207 ++++++ .../utils/http/http_socket_wrapper.py | 74 ++ tests/functional/utils/inference/__init__.py | 15 + tests/functional/utils/inference/capi.py | 111 +++ .../utils/inference/communication/__init__.py | 18 + .../utils/inference/communication/base.py | 53 ++ .../inference/communication/constants.py | 19 + .../utils/inference/communication/grpc.py | 194 ++++++ .../utils/inference/communication/rest.py | 210 ++++++ .../inference/inference_client_factory.py | 139 ++++ .../utils/inference/serving/__init__.py | 21 + .../utils/inference/serving/base.py | 183 +++++ .../utils/inference/serving/cohere.py | 92 +++ .../utils/inference/serving/common.py | 81 +++ .../functional/utils/inference/serving/kf.py | 632 +++++++++++++++++ .../utils/inference/serving/openai.py | 401 +++++++++++ .../functional/utils/inference/serving/tf.py | 473 +++++++++++++ .../utils/inference/serving/triton.py | 211 ++++++ tests/functional/utils/log_monitor.py | 244 +++++++ tests/functional/utils/logger.py | 326 ++++++++- tests/functional/utils/marks.py | 323 +++++++++ tests/functional/utils/model_management.py | 118 ---- tests/functional/utils/models_utils.py | 96 --- tests/functional/utils/numpy_loader.py | 153 +++++ tests/functional/utils/other.py | 73 -- .../ovms_testing_image/Dockerfile.redhat | 188 ++++++ .../ovms_testing_image/Dockerfile.ubuntu | 186 +++++ .../corrupted_lib/CorruptedLib.cpp | 41 ++ .../throw_exceptions/ThrowExceptions.cpp | 85 +++ .../custom_nodes/demultiply/demultiply.cpp | 169 +++++ .../demultiply_gather/demultiply_gather.cpp | 155 +++++ .../elastic_in_1t_out_1t.cpp | 202 ++++++ .../ov/ovms_basic/ovms/CMakeLists.txt} | 17 +- .../ov/ovms_basic/ovms/src/ov.cpp | 51 ++ .../ov/ovms_reshape_model/ovms/CMakeLists.txt | 24 + .../ovms/src/ov_reshape_model.cpp | 50 ++ tests/functional/utils/parametrization.py | 77 --- tests/functional/utils/port_manager.py | 124 ++-- tests/functional/utils/process.py | 329 +++++++-- .../utils/remote_test_environment.py | 28 + .../utils/reservation_manager/__init__.py | 15 + .../utils/reservation_manager/__main__.py | 70 ++ .../utils/reservation_manager/args.py | 166 +++++ .../utils/reservation_manager/env_manager.py | 148 ++++ .../utils/reservation_manager/exceptions.py | 50 ++ .../utils/reservation_manager/locker.py | 43 ++ .../utils/reservation_manager/manager.py | 639 ++++++++++++++++++ .../reservation_manager/manager_config.py | 103 +++ .../reservation_manager.yml | 32 + .../utils/reservation_manager/runner.py | 86 +++ .../reservation_manager/unittests/__init__.py | 15 + .../unittests/test_manager.py | 163 +++++ tests/functional/utils/rest.py | 147 ---- tests/functional/utils/ssl.py | 98 +++ tests/functional/utils/test_framework.py | 267 ++++++++ tests/reservation_manager.yml | 15 +- 78 files changed, 10729 insertions(+), 867 deletions(-) delete mode 100644 tests/functional/utils/cleanup.py create mode 100644 tests/functional/utils/context.py create mode 100644 tests/functional/utils/core.py create mode 100644 tests/functional/utils/docker.py create mode 100644 tests/functional/utils/environment_info.py create mode 100644 tests/functional/utils/generative_ai/__init__.py create mode 100644 tests/functional/utils/generative_ai/validation_utils.py create mode 100644 tests/functional/utils/git_operations.py delete mode 100644 tests/functional/utils/grpc.py create mode 100644 tests/functional/utils/hooks.py create mode 100644 tests/functional/utils/http/__init__.py create mode 100644 tests/functional/utils/http/base.py create mode 100644 tests/functional/utils/http/client_auth/__init__.py create mode 100644 tests/functional/utils/http/client_auth/auth.py create mode 100644 tests/functional/utils/http/client_auth/base.py create mode 100644 tests/functional/utils/http/client_auth/exceptions.py create mode 100644 tests/functional/utils/http/exceptions.py create mode 100644 tests/functional/utils/http/http_client.py create mode 100644 tests/functional/utils/http/http_client_configuration.py create mode 100644 tests/functional/utils/http/http_client_factory.py create mode 100644 tests/functional/utils/http/http_session.py create mode 100644 tests/functional/utils/http/http_socket_wrapper.py create mode 100644 tests/functional/utils/inference/__init__.py create mode 100644 tests/functional/utils/inference/capi.py create mode 100644 tests/functional/utils/inference/communication/__init__.py create mode 100644 tests/functional/utils/inference/communication/base.py create mode 100644 tests/functional/utils/inference/communication/constants.py create mode 100644 tests/functional/utils/inference/communication/grpc.py create mode 100644 tests/functional/utils/inference/communication/rest.py create mode 100644 tests/functional/utils/inference/inference_client_factory.py create mode 100644 tests/functional/utils/inference/serving/__init__.py create mode 100644 tests/functional/utils/inference/serving/base.py create mode 100644 tests/functional/utils/inference/serving/cohere.py create mode 100644 tests/functional/utils/inference/serving/common.py create mode 100644 tests/functional/utils/inference/serving/kf.py create mode 100644 tests/functional/utils/inference/serving/openai.py create mode 100644 tests/functional/utils/inference/serving/tf.py create mode 100644 tests/functional/utils/inference/serving/triton.py create mode 100644 tests/functional/utils/log_monitor.py create mode 100644 tests/functional/utils/marks.py delete mode 100644 tests/functional/utils/model_management.py delete mode 100644 tests/functional/utils/models_utils.py create mode 100644 tests/functional/utils/numpy_loader.py delete mode 100644 tests/functional/utils/other.py create mode 100644 tests/functional/utils/ovms_testing_image/Dockerfile.redhat create mode 100644 tests/functional/utils/ovms_testing_image/Dockerfile.ubuntu create mode 100644 tests/functional/utils/ovms_testing_image/cpu_extensions/corrupted_lib/CorruptedLib.cpp create mode 100644 tests/functional/utils/ovms_testing_image/cpu_extensions/throw_exceptions/ThrowExceptions.cpp create mode 100644 tests/functional/utils/ovms_testing_image/custom_nodes/demultiply/demultiply.cpp create mode 100644 tests/functional/utils/ovms_testing_image/custom_nodes/demultiply_gather/demultiply_gather.cpp create mode 100644 tests/functional/utils/ovms_testing_image/custom_nodes/elastic_in_1t_out_1t/elastic_in_1t_out_1t.cpp rename tests/functional/utils/{files_operation.py => ovms_testing_image/ov/ovms_basic/ovms/CMakeLists.txt} (58%) create mode 100644 tests/functional/utils/ovms_testing_image/ov/ovms_basic/ovms/src/ov.cpp create mode 100644 tests/functional/utils/ovms_testing_image/ov/ovms_reshape_model/ovms/CMakeLists.txt create mode 100644 tests/functional/utils/ovms_testing_image/ov/ovms_reshape_model/ovms/src/ov_reshape_model.cpp delete mode 100644 tests/functional/utils/parametrization.py create mode 100644 tests/functional/utils/remote_test_environment.py create mode 100644 tests/functional/utils/reservation_manager/__init__.py create mode 100644 tests/functional/utils/reservation_manager/__main__.py create mode 100644 tests/functional/utils/reservation_manager/args.py create mode 100644 tests/functional/utils/reservation_manager/env_manager.py create mode 100644 tests/functional/utils/reservation_manager/exceptions.py create mode 100644 tests/functional/utils/reservation_manager/locker.py create mode 100644 tests/functional/utils/reservation_manager/manager.py create mode 100644 tests/functional/utils/reservation_manager/manager_config.py create mode 100644 tests/functional/utils/reservation_manager/reservation_manager.yml create mode 100644 tests/functional/utils/reservation_manager/runner.py create mode 100644 tests/functional/utils/reservation_manager/unittests/__init__.py create mode 100644 tests/functional/utils/reservation_manager/unittests/test_manager.py delete mode 100644 tests/functional/utils/rest.py create mode 100644 tests/functional/utils/ssl.py create mode 100644 tests/functional/utils/test_framework.py diff --git a/tests/functional/constants/generative_ai.py b/tests/functional/constants/generative_ai.py index f630aa81f0..063b35c09d 100644 --- a/tests/functional/constants/generative_ai.py +++ b/tests/functional/constants/generative_ai.py @@ -28,3 +28,52 @@ class GenerativeAIPluginConfig: NPUW_ONLINE_PIPELINE = "NPUW_ONLINE_PIPELINE" MAX_PROMPT_LEN = "MAX_PROMPT_LEN" PROMPT_LOOKUP = "prompt_lookup" + + +class Tools: + GET_WEATHER_TOOLS = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current temperature for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City and country e.g. Bogotá, Colombia" + } + }, + "required": [ + "location" + ], + "additionalProperties": False + }, + "strict": True + } + }] + GET_WEATHER_TOOL_CHOICE = {"type": "function", "function": {"name": "get_weather"}} + + GET_POLLUTIONS_TOOLS = [{ + "type": "function", + "function": { + "name": "get_pollutions", + "description": "Get current level of air pollutions for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City and country e.g. Bogotá, Colombia" + } + }, + "required": [ + "location" + ], + "additionalProperties": False + }, + "strict": True + } + }] + + WEATHER_AND_POLLUTIONS_TOOLS = [GET_WEATHER_TOOLS[0], GET_POLLUTIONS_TOOLS[0]] diff --git a/tests/functional/utils/cleanup.py b/tests/functional/utils/cleanup.py deleted file mode 100644 index d6f2117d10..0000000000 --- a/tests/functional/utils/cleanup.py +++ /dev/null @@ -1,93 +0,0 @@ -# -# Copyright (c) 2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import docker -from docker.errors import APIError -import os -import shutil - -from tests.functional.utils.parametrization import get_tests_suffix -import tests.functional.config as config - - -def get_docker_client(): - return docker.from_env() - - -def get_containers_with_tests_suffix(): - tests_suffix = get_tests_suffix() - - client = get_docker_client() - containers = client.containers.list(all=True, ignore_removed=True) - - detected_container_names = [] - for container in containers: - if tests_suffix in container.name: - detected_container_names.append(container.name) - client.close() - return detected_container_names - - -def clean_hanging_docker_resources(): - client = get_docker_client() - containers = client.containers.list(all=True, ignore_removed=True) - networks = client.networks.list() - tests_suffix = get_tests_suffix() - clean_hanging_containers(containers, tests_suffix) - clean_hanging_networks(networks, tests_suffix) - client.close() - - -def clean_hanging_containers(containers, tests_suffix): - for container in containers: - if tests_suffix in container.name: - kill_container(container) - remove_resource(container) - - -def clean_hanging_networks(networks, tests_suffix): - for network in networks: - if tests_suffix in network.name: - remove_resource(network) - - -def kill_container(container): - try: - container.kill() - except APIError as e: - handle_cleanup_exception(e) - - -def remove_resource(resource): - try: - resource.remove() - except APIError as e: - handle_cleanup_exception(e) - - -def handle_cleanup_exception(docker_error): - # It is okay to have these errors as - # it means resource not exist or being removed or killed already - allowed_errors = [404, 409] - if docker_error.status_code in allowed_errors: - pass - else: - raise - - -def delete_test_directory(): - if os.path.exists(config.test_dir): - shutil.rmtree(config.test_dir) diff --git a/tests/functional/utils/context.py b/tests/functional/utils/context.py new file mode 100644 index 0000000000..5af8b4a680 --- /dev/null +++ b/tests/functional/utils/context.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +from docker.errors import APIError as DockerAPIError +from docker.errors import NotFound as DockerObjectNotFound + +import tests.functional.utils.assertions as assertions_module +from tests.functional.utils.assertions import DmesgError, UnexpectedResponseError +from tests.functional.utils.core import get_children_from_module +from tests.functional.utils.logger import get_logger + + +class Context(object): + logger = get_logger("context") + EXCEPTIONS_TO_CATCH = [ + UnexpectedResponseError, + AssertionError, + DockerObjectNotFound, + DockerAPIError, + PermissionError, + ] + + def __init__(self, scope: str, name: str = None, function_item=None): + self.name = name + self.scope = scope + self.cleaning = False + self.transfers = [] + self.test_objects = [] + self.setup_process_pid = os.getpid() + self.function_item = function_item + + def _cleanup_test_objects(self, object_list: list): + self.logger.debug(f"Cleanup of {self.scope} context {' for ' + self.name if self.name else ''}.") + while len(object_list) > 0: + item = object_list.pop() + if callable(item): + try: + self.logger.info(f"calling {item!s} to /get object to/ clean.") + item = item() + except BaseException as exc: + self.logger.exception(f"Cannot call on callable item {item!r}", exc_info=exc) + continue + if item is None: + self.logger.debug("After calling callable object result is None. " + "Assuming object is already cleaned.") + else: + exceptions_to_catch = self.EXCEPTIONS_TO_CATCH.copy() + dmesg_exceptions = get_children_from_module(DmesgError, assertions_module) + dmesg_exceptions = [exc[1] for exc in dmesg_exceptions] + exceptions_to_catch.extend(dmesg_exceptions) + if hasattr(item, "cleanup_exceptions_to_catch"): + exceptions_to_catch.extend(item.cleanup_exceptions_to_catch) + try: + self.logger.info(f"cleaning: {item!r}") + if hasattr(item, "cleanup"): + item.cleanup() + else: + self.logger.warning(f"Cannot call cleanup on item {item!r}") + continue + except tuple(exceptions_to_catch) as e: + self.logger.exception(f"Error while deleting {item!r}.", exc_info=e) + if not hasattr(self.function_item, "runtime_bugmarks") or not self.function_item.runtime_bugmarks: + # raise exception only if test is not bugmarked since raising exception here will cause + # whole job to fail + raise e + + def cleanup(self): + """Context cleanup only when cleanup process == setup process""" + if self.setup_process_pid == os.getpid(): + self.cleaning = True + self._cleanup_test_objects(self.test_objects) + self.cleaning = False diff --git a/tests/functional/utils/core.py b/tests/functional/utils/core.py new file mode 100644 index 0000000000..715c7dcbae --- /dev/null +++ b/tests/functional/utils/core.py @@ -0,0 +1,166 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import inspect +import json +import os +import time + +from collections import defaultdict +from datetime import datetime, timedelta +from enum import Enum +from filelock import UnixFileLock, WindowsFileLock +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +from tests.functional.constants.os_type import get_host_os, OsType + + +def is_ancestor(obj, ancestor): + if not (inspect.isclass(obj) and obj.__bases__): + return False + if ancestor in obj.__bases__: # child + return True + return any(filter(lambda x: is_ancestor(x, ancestor), obj.__bases__)) # check if grandchild ... + + +def get_children_from_module(parent, module): + members = inspect.getmembers(module) # [(name, class_def), ...] + children = list(filter(lambda x: is_ancestor(x[1], parent), members)) + return children + + +def get_token_value(token_file_path, fallback_value=None): + if os.path.exists(token_file_path): + token_value = Path(token_file_path).read_text().strip() + return token_value + return fallback_value + + +def get_username(): + try: + user_name = os.getlogin() + except OSError as e: + user = os.environ.get("USER", "not_known_user") + logname = os.environ.get("LOGNAME", user) + user_name = os.environ.get("USERNAME", logname) + return user_name + + +def wait_until_file_exists(path, timeout=60): + deadline = datetime.now() + timedelta(seconds=timeout) + path = Path(path) + while not path.exists() and datetime.now() < deadline: + time.sleep(1) + assert path.exists(), f"File do not exist {str(path)}" + + +class SelfDeletingCommonFileLock: + + def __init__(self, lock_file, self_delete=False, **kwargs): + super().__init__(lock_file, **kwargs) + self.self_delete = self_delete + + def acquire(self, **kwargs): + Path(self.lock_file).parent.mkdir(parents=True, exist_ok=True) # Create dir if not existing + super().acquire(**kwargs) + + def acquire_no_raise(self, timeout): + try: + self.acquire(timeout=timeout) + except TimeoutError as e: + return False + return True + + def __exit__(self, exc_type, exc_value, exc_traceback): + super().__exit__(exc_type, exc_value, exc_traceback) + if self.self_delete: + Path(self.lock_file).unlink(missing_ok=True) + + +class SelfDeletingWindowsFileLock(SelfDeletingCommonFileLock, WindowsFileLock): + pass + + +class SelfDeletingUnixFileLock(SelfDeletingCommonFileLock, UnixFileLock): + pass + + def acquire(self, **kwargs): + Path(self.lock_file).parent.mkdir(parents=True, exist_ok=True) # Create dir if not existing + super().acquire(**kwargs) + + def acquire_no_raise(self, timeout): + try: + self.acquire(timeout=timeout) + except TimeoutError as e: + return False + return True + + +SelfDeletingFileLock = SelfDeletingWindowsFileLock if get_host_os() == OsType.Windows else SelfDeletingUnixFileLock + + +class TmpDir(str): + """Creates safe temp dir""" + TMP_DIR = None + SEPARATOR = "_" + + def __new__(cls, temp_dir: str = None) -> str: + if cls.TMP_DIR is None: + cls.TMP_DIR = TemporaryDirectory( + prefix=datetime.now().strftime('%Y%m%d_%H%M%S_%f') + cls.SEPARATOR, + suffix=cls.SEPARATOR + get_username(), + dir=temp_dir) + return cls.TMP_DIR.name + + +class NamedSingletonMeta(type): + """ + Metaclass for defining Named Singleton Classes (extension for Singleton Classes) + + src: + https://www.datacamp.com/community/tutorials/python-metaclasses + + Singleton Design using a Metaclass + + This is a design pattern that restricts the instantiation of a class to only one object. + This could prove useful for example when designing a class to connect to the database. + One might want to have just one instance of the connection class. + """ + _instances = defaultdict(dict) + + def __call__(cls, name: str, *args: Any, **kwargs: Any) -> Any: + name = name.lower() + if name not in cls._instances[cls]: + cls._instances[cls][name] = super().__call__(name, *args, **kwargs) + return cls._instances[cls][name] + + +class ExtendedEnum(Enum): + @classmethod + def list(cls): + return list(map(lambda c: c.value, cls)) + + +class ComplexEncoder(json.JSONEncoder): + def default(self, obj): + if hasattr(obj, 'to_str'): + return obj.to_str() + elif isinstance(obj, type): + return str(obj) + else: + return json.JSONEncoder.default(self, obj) diff --git a/tests/functional/utils/docker.py b/tests/functional/utils/docker.py new file mode 100644 index 0000000000..1cdbfae08f --- /dev/null +++ b/tests/functional/utils/docker.py @@ -0,0 +1,444 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pprint +import signal +import time +from abc import ABCMeta +from io import BytesIO +from typing import Any, Callable, List, Tuple, Union + +import docker +from docker import Context +from docker.models.containers import Container +from retry.api import retry_call +from typing_extensions import TypedDict + +from tests.functional.utils.logger import get_logger +from tests.functional.utils.process import Process +from tests.functional.utils.test_framework import generate_test_object_name +from tests.functional.config import docker_client_timeout +from tests.functional.constants.core import CONTAINER_STATUS_RUNNING + +logger = get_logger(__name__) + + +class DockerClient(docker.DockerClient): + + def build(self, dockerfile: str, build_args, nocache: bool = True, **kwargs) -> tuple: + logs = [] + with open(dockerfile, "r") as file: + data = file.read() + file_obj = BytesIO(data.encode("utf-8")) + image, generator = self.images.build(fileobj=file_obj, nocache=nocache, buildargs=build_args, **kwargs) + while True: + try: + output = generator.__next__() + logs.append(str(output.values())) + except StopIteration: + break + + return image, logs + + def push(self, repository, tag=None, **kwargs): + logs = self.images.push(repository=repository, tag=tag, **kwargs).split("\n") + + for line in logs: + assert ( + "requested access to the resource is denied" not in line + ), "Unauthorized to push docker image: {}".format(line) + assert "error" not in line, "Failed to push docker image: {}".format(line) + return logs + + def pull(self, repository, tag): + return self.images.pull(repository=repository, tag=tag) + + def remove(self, image_id: str, **kwargs): + return self.images.remove(image=image_id, **kwargs) + + def get(self, container_id_or_name) -> Container: + container = self.containers.get(container_id_or_name) + return container + + def create(self, image, command=None, **kwargs) -> Container: + container = self.containers.create(image, command, **kwargs) + return container + + def run(self, image, command=None, stdout=True, stderr=False, remove=False, **kwargs) -> Union[Container, None]: + container = self.containers.run(image, command, stdout, stderr, remove, **kwargs) + return container + + def list_containers(self, all_containers=False, before=None, filters=None, limit=-1, since=None) -> List[Container]: + return self.containers.list(all_containers, before, filters, limit, since) + + +class Limits(TypedDict): + cpu_period: int + cpu_quota: int + cpu_shares: int + cpuset_cpus: str + kernel_memory: Union[int, str] + mem_limit: Union[int, str] + mem_reservation: Union[int, str] + mem_swappiness: int + memswap_limit: Union[int, str] + oom_kill_disable: bool + + +class DockerContainer(metaclass=ABCMeta): + TITLES_HEADER = "Titles" + PROCESSES_HEADER = "Processes" + PID_HEADER = "PID" + COMMAND_HEADER = "CMD" + COMMON_RETRY = {"tries": 30, "delay": 2} + NOT_ON_LIST_RETRY = {"tries": 10, "delay": 2} + GETTING_LOGS_RETRY = COMMON_RETRY + GETTING_STATUS_RETRY = COMMON_RETRY + + def __init__( + self, + container: Container, + command=None, + detach: bool = True, + ports: dict = None, + volumes: dict = None, + devices: List[type(str)] = None, + limits: Limits = None, + **kwargs, + ): + self.kwargs = kwargs + self.container = container + self.image = None if not container else self.container.image + self.name = None if not container else self.container.name + self.command = command + self.detach = detach + self.ports = ports + self.volumes = volumes + self.limits = limits + self.devices = devices + self.id = self.name + self._is_deleted = False + self.client = DockerClient(timeout=docker_client_timeout) + + @classmethod + def run( + cls, + context: Context, + image, + command=None, + stdout=True, + stderr=False, + remove=False, + name: str = None, + detach: bool = True, + ports: dict = None, + volumes: dict = None, + devices: List[type(str)] = None, + limits: Limits = None, + **kwargs, + ): + logger.info("Running container with:\n image: {}\n command: {}\n volumes: {}".format(image, command, volumes)) + if limits is not None: + kwargs.update(limits) + container = cls.client.run( + image, + command, + stdout=stdout, + stderr=stderr, + remove=remove, + name=cls.container_name(name), + detach=detach, + ports=ports, + volumes=volumes, + devices=devices, + **kwargs, + ) + instance = cls( + container, command, detach=detach, ports=ports, volumes=volumes, devices=devices, limits=limits, **kwargs + ) + context.test_objects.append(instance) + return instance + + @classmethod + def create( + cls, + context: Context, + image, + command=None, + name: str = None, + detach: bool = True, + ports: dict = None, + volumes: dict = None, + devices: List[type(str)] = None, + limits: Limits = None, + **kwargs, + ): + logger.info("Creating container with:\n image: {}\n command: {}\n volumes: {}".format(image, command, volumes)) + if limits is not None: + kwargs.update(limits) + container = cls.client.create( + image, + command, + name=cls.container_name(name), + detach=detach, + ports=ports, + volumes=volumes, + devices=devices, + **kwargs, + ) + instance = cls( + container, command, detach=detach, ports=ports, volumes=volumes, devices=devices, limits=limits, **kwargs + ) + context.test_objects.append(instance) + return instance + + @classmethod + def get(cls, container_id_or_name) -> Union["DockerContainer", None]: + container = cls.client.get(container_id_or_name) # type: Container + return cls.from_response(container) + + @classmethod + def list(cls, all_containers=False, before=None, filters=None, limit=-1, since=None) -> List["DockerContainer"]: + container_list = cls.client.list_containers( + all_containers, before, filters, limit, since + ) # type: List[Container] + return cls.list_from_response(container_list) + + @classmethod + def from_response(cls, rsp: Container): + return cls(container=rsp) + + def restart(self, stdout=True, stderr=False, remove=False, **kwargs): + if len(kwargs): + self.kwargs.update(kwargs) + self.delete() + self.container = self.client.run( + self.image, + self.command, + stdout=stdout, + stderr=stderr, + remove=remove, + name=self.name, + detach=self.detach, + ports=self.ports, + volumes=self.volumes, + **self.kwargs, + ) + + @classmethod + def container_name(cls, container_name: str = None): + return generate_test_object_name() if container_name is None else container_name + + @classmethod + def volume(cls, external_path: str, internal_path: str, mode: str = "ro", volumes: dict = None): + if not isinstance(volumes, dict): + volumes = dict() + volumes[external_path] = {"bind": internal_path, "mode": mode} + return volumes + + def start_container(self): + assert self.container is not None, "Lack of container {} to start (is None)\nContainers found:\n{}".format( + self.name, repr(self.client.list_containers(all_containers=True)) + ) + return self.container.start() + + def stop_container(self, **kwargs): + assert self.container is not None, "Lack of container {} to stop (is None)\nContainers found:\n{}".format( + self.name, repr(self.client.list_containers(all_containers=True)) + ) + return self.container.stop(**kwargs) + + def kill_container(self, signal=signal.SIGTERM): + assert self.container is not None, "Lack of container {} to kill (is None)\nContainers found:\n{}".format( + self.name, repr(self.client.list_containers(all_containers=True)) + ) + return self.container.kill(signal=signal) # SIGKILL (not supported for Windows), SIGINT; default: SIGTERM + + def remove_container(self, ensure_deleted: bool = False): + assert self.container is not None, "Lack of container {} to remove (is None)\nContainers found:\n{}".format( + self.name, repr(self.client.list_containers(all_containers=True)) + ) + removed = self.container.remove() + if ensure_deleted: + self.ensure_not_on_list(self.name) + return removed + + def delete(self, ensure_deleted: bool = False): + self.stop_container() + self.remove_container(ensure_deleted) + + def check_non_empty_logs(self, specific_str: str, acceptable_logs_length_trigger: int = 0, **kwargs): + logs = self.get_logs(**kwargs) + assert len(logs) > acceptable_logs_length_trigger, "Logs list for {} should not be empty".format(self.name) + assert specific_str in logs, "Specific string: {} not found in logs: {}".format(specific_str, logs) + return logs + + def ensure_logs_contain_specific_str( + self, specific_str: str, acceptable_logs_length_trigger: int = 0, retry_kwargs: dict = None, **kwargs + ): + args = [specific_str, acceptable_logs_length_trigger] + getting_logs_retry = self.GETTING_LOGS_RETRY.copy() + if retry_kwargs: + getting_logs_retry.update(retry_kwargs) + return retry_call( + self.check_non_empty_logs, fargs=args, fkwargs=kwargs, exceptions=AssertionError, **getting_logs_retry + ) + + @property + def ports_mapping(self): + self.container.reload() + return self.container.ports + + def get_first_host_port_mapping(self, mapped_port): + port_mapping = self.ports_mapping.get(mapped_port, []) or [] + first_host_port_mapping = next(iter(port_mapping), {}) + return first_host_port_mapping + + def get_host_port_mapping(self, mapped_port: str) -> Tuple[str, str]: + """ + gets first host and port mapping + :param mapped_port: str -> port mapping in format: "3456/tcp" + :return: Tuple[str, str] -> (host_ip, host_port) + """ + host = self.get_first_host_port_mapping(mapped_port) + host_port = host.get("HostPort", None) + host_ip = host.get("HostIP", None) + assert host_port is not None, f"Cannot get mapped port for {mapped_port} in {self.ports}" + return host_ip, host_port + + def update(self): + self.container = self.client.containers.get(self.container.id) # type: Container + return self + + def get_status(self, status=None, timeout=None): + self.update() + return self.container.status + + def assert_status(self, status): + current_status = self.get_status() + assert current_status == status, ( + "Not expected status for container {} found. \n " + "Expected: {}, \n " + "received: {}".format(self.container.name, status, self.container.status) + ) + return True + + def ensure_status(self, status: str = CONTAINER_STATUS_RUNNING): + container_status = {"status": status} + return retry_call( + self.assert_status, fkwargs=container_status, exceptions=AssertionError, **self.GETTING_STATUS_RETRY + ) + + def get_logs(self, **kwargs) -> Union[bool, str]: + assert self.container is not None, "Lack of container to get logs from (is None)" + return self.container.logs(**kwargs).decode() + + @classmethod + def check_not_on_list(cls, container: Union[str, "DockerContainer"], comparator: Callable[[Any, Any], bool] = None): + current_list = cls.list() + logger.debug( + "Searching for container with a name: {name}, among:\n{elem}\n".format( + name=container if isinstance(container, str) else container.name, + elem="\n".join([repr(elem) for elem in current_list]), + ) + ) + if comparator is None: + assert container not in current_list, "{} was found on: {}".format(container, pprint.pformat(current_list)) + else: + for member in current_list: + assert comparator(container, member) is False, "{} was found on: {}".format( + container, pprint.pformat(current_list) + ) + + @classmethod + def ensure_not_on_list( + cls, + container: Union[str, "DockerContainer"], + comparator: Callable[[Any, Any], bool] = None, + ensure_count: int = 1, + ): + retry_call( + cls.check_not_on_list, + fargs=[container, comparator], + exceptions=AssertionError, + tries=cls.NOT_ON_LIST_RETRY["tries"], + delay=cls.NOT_ON_LIST_RETRY["delay"], + ) + for count in range(1, ensure_count): + time.sleep(cls.NOT_ON_LIST_RETRY["delay"]) + cls.check_not_on_list(container, comparator) + + def __repr__(self): + _id = self.container.id if self.container is not None else "" + ports = pprint.pformat(self.container.ports) if self.container is not None else "" + return "<%s: %s%s>@%s" % ( + self.__class__.__name__, + self.id, + " (%s)" % _id, + "ports: %s." % ports, + ) + + @classmethod + def list_from_response(cls, rsp): + """Will create list of object from the response""" + items = [] + for item in rsp: + items.append(cls.from_response(item)) + return items + + @property + def deleted(self): + return self._is_deleted + + def cleanup(self): + """Method called by context after tests have finished""" + self.delete() + self._set_deleted(True) + + def _set_deleted(self, is_deleted): + self._is_deleted = bool(is_deleted) + + def list_containers(self): + containers = self.client.list_containers() + return containers + + def prune(self, filters=None): + return self.client.containers.prune(filters) + + +class DockerNetwork: + + def __init__(self, context, network_name=None): + self.context = context + self.network_name = network_name if network_name is not None else context.test_object_name + self.proc = Process() + self.proc.disable_check_stderr() + self.context.test_objects.append(self.cleanup) + + def create_network(self): + self.proc.run_and_check(f"docker network create {self.network_name}") + + def connect_network(self, container_name): + self.proc.run(f"docker network connect {self.network_name} {container_name}") + + def disconnect_network(self, container_name): + self.proc.run(f"docker network disconnect {self.network_name} {container_name}") + + def remove_network(self): + self.proc.run_and_check(f"docker network rm {self.network_name}") + + def cleanup(self): + self.remove_network() diff --git a/tests/functional/utils/environment_info.py b/tests/functional/utils/environment_info.py new file mode 100644 index 0000000000..24f8bd6bcd --- /dev/null +++ b/tests/functional/utils/environment_info.py @@ -0,0 +1,160 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import platform +import re + +import psutil + +from tests.functional.utils.logger import get_logger +from tests.functional.utils.process import Process +from tests.functional import config + +logger = get_logger(__name__) + +DEFAULT_BUILD_NUMBER = 0 +DEFAULT_SHORT_VERSION_NUMBER = "0.0.0" +DEFAULT_FULL_VERSION_NUMBER = f"{DEFAULT_SHORT_VERSION_NUMBER}-{config.product_version_suffix}-{DEFAULT_BUILD_NUMBER}" + + +class BaseInfo: + """Retrieves environment info""" + glob_version = None + glob_os_distname = None + + def __init__(self, image=None): + self.image = image + + @property + def version(self): + """Retrieves version, but only once. + + If retrieval doesn't work, default version is returned. + """ + if self.glob_version is None: + + self.glob_version = self.get() + self.glob_version = \ + self.glob_version["version"] + + return self.glob_version + + @property + def os_distname(self): + """Retrieves os distname, but only once.""" + if self.glob_os_distname is None: + self.glob_os_distname = CurrentOsInfo.get_os_distname() + + return self.glob_os_distname + + @classmethod + def get(cls): + """ + Returns constant environment info. + """ + logger.info("BASIC INFO WITHOUT ANY API CALL") + return {"version": DEFAULT_FULL_VERSION_NUMBER} + + +class EnvironmentInfo(object): + _instances = {} + + @classmethod + def get_instance(cls, class_info, image): + idx = class_info, image + if idx not in cls._instances: + cls._instances[idx] = cls(class_info, image) + return cls._instances[idx] + + def __init__(self, class_info=BaseInfo, image=None): + """Stores details about environment such as build number, version number + and allows their retrieval""" + self.env_info = class_info(image) + + def get_build_number(self): + """Retrieves build number (OVMS/OV/OV GenAI versions) from the environment info""" + if config.product_build_number_from_env: + return self._retrieve_build_number_from_environment() + if config.product_build_number: + return config.product_build_number + return DEFAULT_BUILD_NUMBER + + def get_version_number(self): + """Retrieves version number (OVMS version) from the environment info""" + if config.product_version_number_from_env: + return self._retrieve_version_number_from_environment() + if config.product_version: + return config.product_version + return DEFAULT_FULL_VERSION_NUMBER + + @classmethod + def get_environment_name(cls): + """Retrieves the environment name that will be reported for a test run""" + return config.environment_name + + def get_os_distname(self): + """Retrieves the operating system distribution name""" + return self.env_info.os_distname + + def _retrieve_version_number_from_environment(self): + return self._version_number_from_environment_version(self.env_info.version) + + def _retrieve_build_number_from_environment(self): + return self._build_number_from_environment_version(self.env_info.version) + + @classmethod + def _build_number_from_environment_version(cls, environment_version): + return environment_version + + @classmethod + def _version_number_from_environment_version(cls, environment_version): + return environment_version.split("_")[0] + + +class CurrentOsInfo: + LINUX = "Linux" + + UBUNTU = "Ubuntu" + REDHAT = "RedHat" + + """Returns the current system/OS name """ + @staticmethod + def get_os_name(): + return platform.system() + + @staticmethod + def get_os_distname(): + if CurrentOsInfo.LINUX == platform.system(): + if CurrentOsInfo.UBUNTU.lower() in platform.version().lower(): + return CurrentOsInfo.UBUNTU + elif CurrentOsInfo.REDHAT.lower() in platform.version().lower(): + return CurrentOsInfo.REDHAT + return platform.system() + + @staticmethod + def get_os_distversion(): + if CurrentOsInfo.LINUX == platform.system(): + proc = Process() + proc.disable_check_stderr() + _, stdout, _ = proc.run_and_check_return_all("cat /etc/os-release") + os_version_match = re.search(r"VERSION_ID=\"(.+)\"", stdout) + assert os_version_match, "Unable to detect OS version" + return os_version_match.group(1) + return platform.version() + + @staticmethod + def get_cpu_amount(): + return psutil.cpu_count() diff --git a/tests/functional/utils/generative_ai/__init__.py b/tests/functional/utils/generative_ai/__init__.py new file mode 100644 index 0000000000..84cfbcf566 --- /dev/null +++ b/tests/functional/utils/generative_ai/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/functional/utils/generative_ai/validation_utils.py b/tests/functional/utils/generative_ai/validation_utils.py new file mode 100644 index 0000000000..0b4ffe60b8 --- /dev/null +++ b/tests/functional/utils/generative_ai/validation_utils.py @@ -0,0 +1,569 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=too-many-nested-blocks +# pylint: disable=unused-argument + +import base64 +import inspect +import os +import shutil +import sys +from io import BytesIO + +import numpy as np +import soundfile as sf +from jiwer import wer, wer_standardize, Compose, RemovePunctuation +from PIL import Image + +from tests.functional.utils.logger import get_logger, step +from tests.functional.utils.inference.serving.openai import OpenAIWrapper, OpenAIFinishReason +from tests.functional.config import save_image_to_artifacts +from tests.functional.config import artifacts_dir, pipeline_type +from tests.functional.models.models_datasets import FeatureExtractionModelDataset + +logger = get_logger(__name__) + + +class GenerativeAIValidationUtils: + + @staticmethod + def validate_llm_outputs( + model_name, + outputs, + stream=False, + validate_func=None, + allow_empty_response=False, + tools_enabled=False, + validate_tools=False, + **kwargs, + ): + logger.info(outputs) + model_instance = kwargs.get("model_instance", None) + outputs_content = [] + assert outputs is not None and len(outputs) > 0, f"No output collected for node with model: {model_name}" + stream_content = [] + for index, output in enumerate(outputs): + assert output.model == model_name, f"Invalid model name: {output.model}; Expected: {model_name}" + for choice in output.choices: + logger.debug(f"Choice: {choice}") + if validate_func is not None: + validate_func( + stream, + choice, + outputs_content, + stream_content, + allow_empty_response, + tools_enabled, + validate_tools, + index=index, + **kwargs, + ) + if tools_enabled: + model_type = None + for _, model_type in inspect.getmembers(sys.modules['ovms.constants.models'], inspect.isclass): + if (model_instance is not None and hasattr(model_type, "name") and + model_instance.name == model_type.name): + if model_instance.allows_reasoning: + if hasattr(choice, "message") and hasattr(choice.message, "model_extra"): + assert "reasoning_content" in choice.message.model_extra, \ + f"Empty reasoning content: {choice}" + logger.info(f"Reasoning content: {choice.message.model_extra['reasoning_content']}") + else: + assert "reasoning_content" not in str(choice), \ + f"Reasoning content is not empty: {choice}" + if stream: + logger.info(stream_content) + if tools_enabled: + assert len(stream_content) > 0, f"Empty tool calls: {stream_content}" + if validate_tools: + return stream_content + outputs_content.append("".join(stream_content)) + else: + assert len(stream_content) > 0, f"Empty stream_content: {stream_content}" + outputs_content.append("".join(stream_content)) + return outputs_content + + @staticmethod + def validate_finish_reason(endpoint, raw_outputs, request_params, finish_reason): + # validate finish reason only for ignore_eos=True (default) - for ignore_eos=False value may vary + if request_params.ignore_eos or request_params.ignore_eos is None: + assert len(raw_outputs) > 0, "No outputs to check!" + stream_finish_reason = None + error_message = "Unexpected finish_reason: {}; expected: {}" + for raw_output in raw_outputs: + if endpoint == OpenAIWrapper.RESPONSES: + if request_params.stream: + if hasattr(raw_output, "response"): + if finish_reason == OpenAIFinishReason.STOP: + stream_finish_reason = OpenAIFinishReason.STOP \ + if raw_output.response.completed_at is not None else \ + raw_output.response.completed_at + else: + stream_finish_reason = raw_output.response.incomplete_details.reason \ + if hasattr(raw_output.response.incomplete_details, "reason") else \ + raw_output.response.incomplete_details + assert stream_finish_reason in [None, finish_reason], \ + error_message.format(stream_finish_reason, finish_reason) + else: + if finish_reason == OpenAIFinishReason.STOP: + assert raw_output.completed_at is not None, \ + error_message.format(raw_output.completed_at, finish_reason) + else: + assert raw_output.incomplete_details is not None and \ + raw_output.incomplete_details.reason == finish_reason, \ + error_message.format(raw_output.incomplete_details, finish_reason) + else: + for choice in raw_output.choices: + if request_params.stream: + stream_finish_reason = choice.finish_reason + assert stream_finish_reason in [None, finish_reason], \ + error_message.format(stream_finish_reason, finish_reason) + else: + assert choice.finish_reason == finish_reason, \ + error_message.format(choice.finish_reason, finish_reason) + if request_params.stream: + assert stream_finish_reason == finish_reason, \ + error_message.format(stream_finish_reason, finish_reason) + + @staticmethod + def validate_stop(outputs, stop, stream, include_stop_str_in_output): + stop = [stop] if isinstance(stop, str) else stop + if stream: + assert any(stop_value in outputs[-1] for stop_value in stop), \ + f"None of the stop values: {stop} were found in the output: {outputs[-1]}" + else: + for output in outputs: + if include_stop_str_in_output: + assert any(stop_value in output for stop_value in stop), \ + f"None of the stop values: {stop} were found in the output: {output}" + else: + assert all(stop_value not in output for stop_value in stop), \ + f"Stop values: {stop} were found in the output: {output}" + + @staticmethod + def validate_usage(endpoint, stream, raw_outputs, max_tokens=None): + assert len(raw_outputs) > 0, "No outputs to check!" + for raw_output in raw_outputs[:-1]: + if endpoint == OpenAIWrapper.RESPONSES: + if not hasattr(raw_output, "response"): + continue + usage = raw_output.response.usage if stream else raw_output.usage + else: + usage = raw_output.usage + assert usage is None, f"Unexpected usage value: {usage}; expected: None" + + last_usage = raw_outputs[-1].response.usage if endpoint == OpenAIWrapper.RESPONSES and stream else \ + raw_outputs[-1].usage + generated_tokens = last_usage.output_tokens if endpoint == OpenAIWrapper.RESPONSES else \ + last_usage.completion_tokens + prompt_tokens = last_usage.input_tokens if endpoint == OpenAIWrapper.RESPONSES else last_usage.prompt_tokens + total_tokens = last_usage.total_tokens + + if max_tokens is not None: + assert max_tokens == generated_tokens, \ + f"Unexpected token count: {generated_tokens}; expected: {max_tokens}" + else: + assert generated_tokens > 0, f"No generated tokens reported: {generated_tokens}" + assert generated_tokens + prompt_tokens == total_tokens, \ + f"Unexpected total_tokens value: {total_tokens}; expected: {generated_tokens + prompt_tokens}" + + return generated_tokens, prompt_tokens, total_tokens + + @classmethod + def validate_chat_completions_outputs( + cls, model_name, outputs, stream=False, allow_empty_response=False, tools_enabled=False, + validate_tools=False, **kwargs + ): + model_instance = kwargs.get("model_instance", None) + model_pipeline_type = getattr(model_instance, "pipeline_type", None) + effective_pipeline_type = model_pipeline_type if model_pipeline_type is not None else pipeline_type + + def validate_choice( + stream, + choice, + outputs_content, + stream_content, + allow_empty_response, + tools_enabled=False, + validate_tools=False, + **kwargs, + ): + if stream: + if kwargs.get("index", None) == 0 and effective_pipeline_type in (None, "CB"): + if choice.delta.content is None: + # check role for empty streaming messages + assert choice.delta.role == "assistant", \ + f"Unexpected role in the first response: {choice.delta.role}" + if tools_enabled and validate_tools: + if choice.delta.tool_calls is not None: + stream_content.append(choice.delta.tool_calls) + else: + if choice.delta.content is not None: + stream_content.append(choice.delta.content) + else: + if not allow_empty_response: + assert len(choice.message.content) > 0, f"Empty response content: {choice}" + if tools_enabled and validate_tools: + # When tools are enabled content might not be empty + assert choice.message.role == "assistant", f"Unexpected role: {choice.message.role}" + assert len(choice.message.tool_calls) > 0, f"Empty tool calls: {choice.message}" + logger.info(choice.message.tool_calls) + outputs_content.append(choice.message.tool_calls) + else: + logger.info(choice.message.content) + outputs_content.append(choice.message.content) + + return cls.validate_llm_outputs( + model_name, outputs, stream, validate_choice, allow_empty_response, tools_enabled, validate_tools, **kwargs + ) + + @classmethod + def validate_completions_outputs( + cls, model_name, outputs, stream=False, allow_empty_response=False, **kwargs + ): + def validate_choice( + stream, + choice, + outputs_content, + stream_content, + allow_empty_response, + tools_enabled=False, + validate_tools=False, + **kwargs, + ): + if stream: + if choice.text is not None: + stream_content.append(choice.text) + else: + if not allow_empty_response: + assert len(choice.text) > 0, f"Empty response content: {choice}" + logger.info(choice.text) + outputs_content.append(choice.text) + + return cls.validate_llm_outputs(model_name, outputs, stream, validate_choice, allow_empty_response, **kwargs) + + @classmethod + def validate_responses_outputs(cls, model_name, outputs, stream=False, allow_empty_response=False, **kwargs): + logger.info(outputs) + outputs_content = [] + assert outputs is not None and len(outputs) > 0, f"No output collected for node with model: {model_name}" + stream_content = [] + for output in outputs: + if stream: + if output.type == "response.created": + assert output.response.model == model_name,\ + f"Invalid model name: {output.response.model}; Expected: {model_name}" + elif output.type == "response.output_text.delta" and output.delta is not None: + stream_content.append(output.delta) + elif output.type in ("response.completed", "response.incomplete"): + if not allow_empty_response: + assert len(stream_content) > 0, f"Empty stream_content: {stream_content}" + assert "".join(stream_content) == output.response.output_text, \ + f"stream_content: {stream_content} does not match output_text: {output.response.output_text}" + logger.info(output.response.output_text) + outputs_content.append(output.response.output_text) + else: + assert output.model == model_name, f"Invalid model name: {output.model}; Expected: {model_name}" + for output_item in output.output: + if output_item.type == "message": + for content_item in output_item.content: + if content_item.type == "output_text": + if not allow_empty_response: + assert content_item.text, f"Empty response content: {content_item}" + logger.info(content_item.text) + outputs_content.append(content_item.text) + return outputs_content + + @classmethod + def validate_embeddings_outputs(cls, model_name, outputs, allow_empty_response=False): + outputs_content = [] + assert outputs is not None and len(outputs.data) > 0, f"No output collected for node with model: {model_name}" + for output in outputs.data: + if not allow_empty_response: + output_embedding = output.embedding + assert len(output_embedding) > 0, f"Empty response content: {output_embedding}" + logger.info(output_embedding) + outputs_content.append(output_embedding) + return outputs_content + + @classmethod + def validate_rerank_outputs(cls, model_name, outputs, allow_empty_response=False): + outputs_content = [] + assert outputs is not None and len(outputs.results) > 0, \ + f"No output collected for node with model: {model_name}" + + for i, output in enumerate(outputs.results): + if not allow_empty_response: + relevance_score = output.relevance_score + assert relevance_score > 0, f"Empty response content: {relevance_score}" + logger.info(f"{[i]}: relevance_score={relevance_score}") + outputs_content.append(relevance_score) + return outputs_content + + @staticmethod + def is_valid_image_pillow(image_path): + try: + with Image.open(image_path) as img: + img.verify() + return True + except (IOError, SyntaxError) as e: + logger.error(e) + return False + + @classmethod + def validate_image_outputs(cls, model_name, outputs, image_path=None, **kwargs): + outputs_content = [] + request_parameters = kwargs.get("request_parameters", None) + + assert outputs is not None and len(outputs.data) == request_parameters.n, \ + f"No output collected for node with model: {model_name}" + + image_base64 = outputs.data[0].b64_json + image_bytes = base64.b64decode(image_base64) + + if image_path is not None: + image = Image.open(BytesIO(image_bytes)) + image.save(image_path) + logger.info(f"Image saved: {image_path}") + if save_image_to_artifacts: + image_dst = os.path.join(artifacts_dir, os.path.basename(image_path)) + shutil.copy(image_path, image_dst) + logger.info(f"Image saved: {image_dst}") + assert cls.is_valid_image_pillow(image_path), f"Image is invalid: {image_bytes}" + width, height = image.size + x, y = request_parameters.size.split("x") + assert width == int(x) and height == int(y), f"Unexpected image size: {image.size}" + + outputs_content.append(image_base64) + return outputs_content + + @staticmethod + def validate_jinja_outputs(jinja_template, outputs): + try: + logger.info("Check if output contains jinja template keyword") + assert jinja_template.lower() in str(outputs).lower() + except AssertionError: + logger.info("Check if output does not contain OpenVINO keyword (default prompt)") + assert "OpenVINO".lower() not in str(outputs).lower(), f"Jinja template was probably not used correctly. " \ + f"Outputs: {outputs}" + + @classmethod + def validate_models_list_outputs(cls, models, outputs): + models_names = [model.name for model in models] + models_list = [] + for model_object in outputs.data: + cls.validate_models_retrieve_outputs(model_object.id, model_object) + models_list.append(model_object.id) + assert set(models_names) == set(models_list), \ + f"v3/models output: {models_list} does not match models loaded to OVMS: {models_names}" + return models_list + + @classmethod + def validate_models_retrieve_outputs(cls, model_name, outputs): + assert outputs.id == model_name, f"Unexpected model id retrieved. Expected: {model_name}. Actual: {outputs.id}" + assert outputs.object == "model", \ + f"Unexpected model object type retrieved. Expected: model. Actual: {outputs.object}" + assert isinstance(outputs.created, int), \ + f"Wrong format for created parameter. Expected type int: Actual output: {outputs.created}" + assert outputs.owned_by == "OVMS", f"Wrong model name retrieved. Expected: OVMS. Actual: {outputs.owned_by}" + return outputs.id + + @staticmethod + def _analyze_audio(file_path): + """Read audio file and compute quality metrics using soundfile + numpy. + + Supports WAV, FLAC, OGG natively. MP3 support requires libmpg123 on the system. + + Returns: + dict with keys: duration_sec, sample_rate, rms, spectral_flatness + """ + data, sample_rate = sf.read(file_path, dtype="float32") + # Convert stereo to mono if needed + if data.ndim > 1: + data = np.mean(data, axis=1) + + n_samples = len(data) + duration_sec = n_samples / sample_rate + + # RMS energy — silence detector + rms = float(np.sqrt(np.mean(data ** 2))) + + # Spectral flatness — noise vs speech detector + # Wiener entropy: geometric_mean(|FFT|) / arithmetic_mean(|FFT|) + # White noise → ~1.0, speech → ~0.05-0.4 + magnitude = np.abs(np.fft.rfft(data)) + magnitude = magnitude[magnitude > 0] # avoid log(0) + if len(magnitude) > 0: + log_mean = np.mean(np.log(magnitude)) + spectral_flatness = float(np.exp(log_mean) / np.mean(magnitude)) + else: + spectral_flatness = 0.0 + + metrics = { + "duration_sec": round(duration_sec, 3), + "sample_rate": sample_rate, + "rms": round(rms, 6), + "spectral_flatness": round(spectral_flatness, 4), + } + logger.info(f"Audio metrics for {os.path.basename(file_path)}: {metrics}") + return metrics + + @classmethod + def validate_audio_speech_outputs( + cls, speech_file_path, allow_empty_response=False, + min_duration_sec=5, max_spectral_flatness=0.85, + ): + """Validate speech audio output file. + + Checks: + - File exists and is not empty + - Audio duration >= min_duration_sec + - Audio is not silence (RMS > 0) + - Audio is not pure noise (spectral_flatness < max_spectral_flatness; + speech typically has flatness 0.05-0.4, white noise ~1.0) + + Args: + speech_file_path: Path to the generated audio file (WAV, MP3, etc.) + allow_empty_response: If True, skip content checks. + min_duration_sec: Minimum expected audio duration in seconds. + max_spectral_flatness: Maximum spectral flatness (above = likely noise, not speech). + """ + assert os.path.exists(speech_file_path), f"Speech output file not found: {speech_file_path}" + file_size = os.path.getsize(speech_file_path) + if not allow_empty_response: + assert file_size > 0, f"Speech output file is empty: {speech_file_path}" + logger.info(f"Audio speech output saved to {speech_file_path} ({file_size} bytes)") + + if not allow_empty_response: + metrics = cls._analyze_audio(speech_file_path) + + assert metrics["duration_sec"] >= min_duration_sec, ( + f"Audio too short: {metrics['duration_sec']}s < {min_duration_sec}s minimum. " + f"File: {speech_file_path}" + ) + assert metrics["rms"] > 0, ( + f"Audio is silent (RMS=0). File: {speech_file_path}" + ) + assert metrics["spectral_flatness"] < max_spectral_flatness, ( + f"Audio appears to be noise, not speech " + f"(spectral_flatness={metrics['spectral_flatness']} >= {max_spectral_flatness}). " + f"File: {speech_file_path}" + ) + + return speech_file_path + + @staticmethod + def validate_audio_asr_outputs(outputs, allow_empty_response=False): + assert outputs is not None, "Audio ASR output is None" + if not allow_empty_response: + assert len(outputs.strip()) > 0, f"Audio ASR output is empty: '{outputs}'" + word_count = len(outputs.strip().split()) + assert word_count >= 1, f"Audio ASR output has no words: '{outputs}'" + logger.info(f"Audio ASR output ({len(outputs.split())} words): {outputs}") + return outputs + + @staticmethod + def validate_wer(reference, hypothesis, threshold=0.4): + """Calculate Word Error Rate (WER) and assert it is below the threshold.""" + normalize_text = Compose([ + RemovePunctuation(), + wer_standardize + ]) + error_rate = wer(reference, hypothesis, reference_transform=normalize_text, hypothesis_transform=normalize_text) + assert error_rate < threshold, ( + f"Output WER is too high. " + f"threshold={threshold}, error_rate={error_rate:.4f}. " + f"Reference: '{str(reference)[:100]}'. " + f"Hypothesis: '{str(hypothesis)[:100]}'." + ) + + @staticmethod + def compute_cosine_similarity(embedding_a, embedding_b): + a = np.array(embedding_a) + b = np.array(embedding_b) + return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))) + + @classmethod + def create_embeddings_getter( # pylint: disable=import-outside-toplevel + cls, embeddings_model, api_type, port, request_parameters=None, inference_fn=None): + """Create a callable that returns an embedding vector for a given text string. + + Uses an OVMS-hosted embeddings model to compute embeddings. The returned callable + can be passed to validate_text_similarity as the embeddings_getter parameter. + + Args: + embeddings_model: Embeddings model instance (e.g. AlibabaNLPGteLargeEnv15). + api_type: OpenAI REST API type. + port: OVMS port where the embeddings model is served. + request_parameters: Optional pre-built request parameters for embeddings endpoint. + If None, will be built automatically via LLMUtils.prepare_request_params. + inference_fn: Callable to run LLM inference (e.g. run_llm_inference). + Injected to avoid circular import between this module and inference_helpers. + + Returns: + Callable[[str], list[float]] that takes text and returns its embedding vector. + """ + assert inference_fn is not None, ( + "inference_fn is required" + ) + + if request_parameters is None: + from llm.utils import LLMUtils + request_parameters = LLMUtils.prepare_request_params(OpenAIWrapper.EMBEDDINGS) + + def getter(text): + class TextDataset(FeatureExtractionModelDataset): + input_data = [text] + + _, raw_outputs, _, _ = inference_fn( + embeddings_model, api_type, port, + OpenAIWrapper.EMBEDDINGS, + dataset=TextDataset, + input_data_type="list", + request_parameters=request_parameters, + ) + return raw_outputs.data[0].embedding + + return getter + + @classmethod + def validate_text_similarity( + cls, + reference_text, + hypothesis_texts, + embeddings_getter, + cos_sim_threshold=0.7, + ): + if isinstance(hypothesis_texts, str): + hypothesis_texts = [hypothesis_texts] + + step(f"Validate text similarity (threshold={cos_sim_threshold})") + reference_embedding = embeddings_getter(reference_text) + + cos_sim_errors = [] + for text in hypothesis_texts: + hypothesis_embedding = embeddings_getter(text) + cos_sim = cls.compute_cosine_similarity(reference_embedding, hypothesis_embedding) + logger.info(f"Text similarity cos_sim={cos_sim:.4f} for: '{text[:80]}...'") + if cos_sim < cos_sim_threshold: + cos_sim_errors.append({"cos_sim": round(cos_sim, 4), "text": text}) + + assert not cos_sim_errors, ( + f"Text similarity below threshold ({cos_sim_threshold}). " + f"Reference: '{reference_text[:100]}'. " + f"Failed comparisons: {cos_sim_errors}" + ) diff --git a/tests/functional/utils/git_operations.py b/tests/functional/utils/git_operations.py new file mode 100644 index 0000000000..9bab7410ae --- /dev/null +++ b/tests/functional/utils/git_operations.py @@ -0,0 +1,78 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from pathlib import Path + +from git import InvalidGitRepositoryError, NoSuchPathError, Repo, cmd + +from tests.functional.utils.assertions import GitCloneException +from tests.functional.utils.logger import get_logger + +logger = get_logger(__name__) + + +def _get_current_git_repo_object(): + current_directory = os.getcwd() + if not Path(current_directory, ".git").exists(): + print(f"{current_directory} isn't git repository") + return None + try: + repo = Repo(current_directory, search_parent_directories=True) + except (NoSuchPathError, InvalidGitRepositoryError) as e: + print(f"Cannot get repo from current directory: {current_directory}") + return None + return repo + + +def get_current_branch(): + repo = _get_current_git_repo_object() + if not repo: + branch = "" + else: + try: + branch = repo.active_branch.name + except TypeError: + branch = 'DETACHED_' + repo.head.object.hexsha + return branch + + +def get_commit_id(): + repo = _get_current_git_repo_object() + if repo: + commit_id = repo.head.object.hexsha + else: + commit_id = "" + return commit_id + + +def git_pull_repository_branch(repo_path, repo_branch): + repo = Repo(repo_path) + g = cmd.Git(repo_path) + g.pull() + repo.git.checkout(repo_branch) + + +def clone_git_repository(repo_url, repo_path, repo_branch, commit_sha=None): + if not os.path.exists(repo_path): + logger.info(f"Clone {repo_url} repository (branch {repo_branch}) to {repo_path}") + try: + repo = Repo.clone_from(repo_url, repo_path, branch=repo_branch) + if commit_sha is not None: + repo.git.checkout(commit_sha) + except Exception as exc: + raise GitCloneException(exc.stderr) + return repo_path diff --git a/tests/functional/utils/grpc.py b/tests/functional/utils/grpc.py deleted file mode 100644 index bf96312ae7..0000000000 --- a/tests/functional/utils/grpc.py +++ /dev/null @@ -1,114 +0,0 @@ -# -# Copyright (c) 2019-2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import grpc # noqa -from retry.api import retry_call -from tensorflow import make_tensor_proto, make_ndarray -from tensorflow_serving.apis import prediction_service_pb2_grpc, model_service_pb2_grpc, predict_pb2, \ - get_model_metadata_pb2, get_model_status_pb2 - -from tests.functional.config import infer_timeout -from tests.functional.config import grpc_ovms_starting_port, ports_pool_size -from tests.functional.utils.port_manager import PortManager -from tests.functional.constants.constants import MODEL_SERVICE, PREDICTION_SERVICE -import logging - -logger = logging.getLogger(__name__) - -DEFAULT_GRPC_PORT = str(grpc_ovms_starting_port) -DEFAULT_ADDRESS = 'localhost' - -port_manager_grpc = PortManager("gRPC", starting_port=grpc_ovms_starting_port, pool_size=ports_pool_size) - - -def create_channel(address: str = DEFAULT_ADDRESS, port: str = DEFAULT_GRPC_PORT, service: int = PREDICTION_SERVICE): - url = '{}:{}'.format(address, port) - channel = grpc.insecure_channel(url) - if service == PREDICTION_SERVICE: - return prediction_service_pb2_grpc.PredictionServiceStub(channel) - elif service == MODEL_SERVICE: - return model_service_pb2_grpc.ModelServiceStub(channel) - return None - - -def infer(img, input_tensor, grpc_stub, model_spec_name, - model_spec_version, output_tensors): - request = predict_pb2.PredictRequest() - request.model_spec.name = model_spec_name - if model_spec_version is not None: - request.model_spec.version.value = model_spec_version - logger.info("Input shape: {}".format(img.shape)) - request.inputs[input_tensor].CopyFrom( - make_tensor_proto(img, shape=list(img.shape))) - result = grpc_stub.Predict(request, infer_timeout) - data = {} - for output_tensor in output_tensors: - data[output_tensor] = make_ndarray(result.outputs[output_tensor]) - return data - - -def get_model_metadata_request(model_name, metadata_field: str = "signature_def", - version=None): - request = get_model_metadata_pb2.GetModelMetadataRequest() - request.model_spec.name = model_name - if version is not None: - request.model_spec.version.value = version - request.metadata_field.append(metadata_field) - return request - - -def get_model_metadata(stub, request, timeout=10): - rargs = (request, int(timeout)) - func = stub.GetModelMetadata - retry_setup = {"tries": 48, "delay": 1} - response = retry_call(func, rargs, **retry_setup) - return response - - -def model_metadata_response(response): - signature_def = response.metadata['signature_def'] - signature_map = get_model_metadata_pb2.SignatureDefMap() - signature_map.ParseFromString(signature_def.value) - serving_default = signature_map.ListFields()[0][1]['serving_default'] - serving_inputs = serving_default.inputs - input_blobs_keys = {key: {} for key in serving_inputs.keys()} - tensor_shape = {key: serving_inputs[key].tensor_shape - for key in serving_inputs.keys()} - for input_blob in input_blobs_keys: - inputs_shape = [d.size for d in tensor_shape[input_blob].dim] - tensor_dtype = serving_inputs[input_blob].dtype - input_blobs_keys[input_blob].update({'shape': inputs_shape}) - input_blobs_keys[input_blob].update({'dtype': tensor_dtype}) - - serving_outputs = serving_default.outputs - output_blobs_keys = {key: {} for key in serving_outputs.keys()} - tensor_shape = {key: serving_outputs[key].tensor_shape - for key in serving_outputs.keys()} - for output_blob in output_blobs_keys: - outputs_shape = [d.size for d in tensor_shape[output_blob].dim] - tensor_dtype = serving_outputs[output_blob].dtype - output_blobs_keys[output_blob].update({'shape': outputs_shape}) - output_blobs_keys[output_blob].update({'dtype': tensor_dtype}) - - return input_blobs_keys, output_blobs_keys - - -def get_model_status(model_name, version=None): - request = get_model_status_pb2.GetModelStatusRequest() - request.model_spec.name = model_name - if version is not None: - request.model_spec.version.value = version - return request diff --git a/tests/functional/utils/helpers.py b/tests/functional/utils/helpers.py index 9cbd9bbed2..1bcc14a7f5 100644 --- a/tests/functional/utils/helpers.py +++ b/tests/functional/utils/helpers.py @@ -13,31 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os -from typing import Any - -from tests.functional.constants.target_device import TargetDevice - - -class SingletonMeta(type): - """ - Metaclass for defining Singleton Classes - - src: - https://www.datacamp.com/community/tutorials/python-metaclasses - Singleton Design using a Metaclass +import os +import random - This is a design pattern that restricts the instantiation of a class to only one object. - This could prove useful for example when designing a class to connect to the database. - One might want to have just one instance of the connection class. - """ - _instances = {} +from time import strftime - def __call__(cls, *args: Any, **kwargs: Any) -> Any: - if cls not in cls._instances: - cls._instances[cls] = super().__call__(*args, **kwargs) - return cls._instances[cls] +from tests.functional.constants.target_device import TargetDevice ALL_AVAILABLE_OPTIONS = "*" @@ -142,3 +124,15 @@ def get_xdist_worker_nr(): else: xdist_current_worker = int(xdist_current_worker.lstrip("gw")) return xdist_current_worker + + +def get_short_date_string(): + date_str = strftime("%Y%m%d%H%M%S") + return date_str + + +def generate_test_object_name(separator="_", prefix=""): + date_str = get_short_date_string() + random_sha = hex(random.getrandbits(128))[2:8] + name = separator.join([item for item in (prefix, date_str, random_sha) if item]) + return name diff --git a/tests/functional/utils/hooks.py b/tests/functional/utils/hooks.py new file mode 100644 index 0000000000..8601bfb333 --- /dev/null +++ b/tests/functional/utils/hooks.py @@ -0,0 +1,412 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import shutil +import warnings + +from collections import defaultdict +from docker import errors as docker_errors +from pathlib import Path + +from tests.functional import config +from tests.functional.models.models_library import ModelsLib +from tests.functional.utils.reservation_manager.args import parse_args +from tests.functional.utils.reservation_manager.manager import Manager as ReservationManager +from tests.functional.config import ( + c_api_wrapper_dir, + cleanup_env_on_startup, + global_tmp_dir_default, + ovms_c_repo_path, + tmp_dir, +) +from tests.functional.constants.os_type import get_host_os, OsType +from tests.functional.constants.ovms import ( + BASE_OS_PARAM_NAME, + OVMS_TYPE_PARAM_NAME, + TARGET_DEVICE_PARAM_NAME, + USES_MAPPING_PARAM_NAME, +) +from tests.functional.constants.ovms_images import calculate_ovms_image_name +from tests.functional.constants.paths import Paths +from tests.functional.constants.target_device import TargetDevice +from tests.functional.object_model.ovms_info import OvmsInfo +from tests.functional.utils.core import TmpDir +from tests.functional.utils.docker import DockerClient, DockerContainer +from tests.functional.utils.environment_info import EnvironmentInfo +from tests.functional.utils.logger import get_logger +from tests.functional.utils.marks import MarkTestParameters +from tests.functional.utils.process import Process +from tests.functional.utils.test_framework import get_test_object_prefix + +logger = get_logger(__name__) + + +CURRENT_TARGET_DEVICE_DICT = {} + +DEVICE_ID_TO_DETAILED_TARGET_DEVICE_NAME_MAP = defaultdict(lambda: ("", []), {}) + + +def init_environment(_config): + init_cleanup() + global CURRENT_TARGET_DEVICE_DICT + # additional constant CURRENT_TARGET_DEVICE_DICT needs to be used due to being unable to read + # current_target_device_dict from config when using xdist=0 + _config.current_target_device_dict = CURRENT_TARGET_DEVICE_DICT + + +def init_cleanup(): + if cleanup_env_on_startup: + if get_host_os() == OsType.Windows: + cleanup_ovms_processes() + else: + cleanup_docker(cleanup_docker_containers) + + +def clean_container(container): + try: + container.stop(timeout=1) + container.remove(force=True) + except docker_errors.NotFound: + logger.warning(f"Container: {container.name} already removed") + except docker_errors.APIError: + logger.warning(f"Removal of container: {container.name} already in progress") + else: + logger.warning(f"Killing running container: {container.name}") + + +def cleanup_docker(cleanup_docker_func): + try: + cleanup_docker_func() + except docker_errors.APIError as error: + logger.warning(f"Error occured during docker cleanup: {error}") + + +def cleanup_docker_containers(): + dc = DockerContainer(None) + for container in dc.list_containers(): + clean_container(container) + logger.warning("Removing all stopped containers") + prune_results = dc.prune() + containers_deleted = prune_results.get("ContainersDeleted", []) + for container in containers_deleted or []: + logger.info(f"Removed container: {str(container)}") + + +def cleanup_docker_images(): + """Remove docker images build during test session""" + if OsType.Windows in config.base_os: + return + docker_client = DockerClient() + test_object_prefix = get_test_object_prefix() + for image in docker_client.images.list(): + for image_tag in image.tags: + if test_object_prefix in image_tag: + docker_client.images.remove(image=image.id, force=True, noprune=False) + logger.info(f"Removed docker image: {image.id}") + + +def teardown_environment(): + if get_host_os() == OsType.Windows: + if config.teardown_ovms_processes: + cleanup_ovms_processes() + else: + if config.teardown_docker_containers: + cleanup_docker(cleanup_docker_containers) + if config.teardown_docker_images: + cleanup_docker(cleanup_docker_images) + + +def cleanup_ovms_processes(): + proc = Process() + proc.disable_check_stderr() + proc.run("taskkill /F /IM ovms.exe /T", print_stdout=False) + + +def clear_ovms_capi_artifacts(): + if not cleanup_env_on_startup: + return + proc = Process() + proc.disable_check_stderr() + if get_host_os() == OsType.Windows: + if os.path.exists(c_api_wrapper_dir): + proc.run_and_check(f"rmdir /S /Q {c_api_wrapper_dir}") + else: + proc.run_and_check("make clean", cwd=Paths.OVMS_TEST_CAPI_WRAPPER_DIR) + proc.run_and_check(f"rm -rf {c_api_wrapper_dir}") + + +def setup_artifacts_dir(): + if not config.artifacts_dir: + return + artifacts_dir_path = Path(config.artifacts_dir) + if not artifacts_dir_path.exists(): + artifacts_dir_path.mkdir(parents=True) + + if config.clean_artifacts_dir: + for file in artifacts_dir_path.glob("*"): + logger.info(f"Deleting old artifacts: {file}") + if file.is_dir(): + shutil.rmtree(file) + else: + file.unlink() + + +def setup_tmp_repos_dir(config): + config.tmp_repos_dir = TmpDir() + + +def get_marker_args(metafunc, marker_name): + _marker = [marker for marker in metafunc.definition.own_markers if marker.name == marker_name] + if _marker: + return _marker[0].args + return + + +def get_ids_with_target_device(parameter, func): + # return id for target_device + if parameter in vars(TargetDevice).values(): + return CURRENT_TARGET_DEVICE_DICT.get(parameter, parameter) + # return custom id + return func(parameter) + + +def parametrize_model_type(metafunc): + args = get_marker_args(metafunc, MarkTestParameters.MODEL_TYPE) + if args is None: + parametrize_target_device(metafunc) + return + if isinstance(args[0], dict): + params_list = [ + (device_type, result) for device_type in config.target_devices for result in args[0][device_type] + ] + else: + params_list = [(device_type, result) for device_type in config.target_devices for result in args[0]] + ids_list = [ + f"{CURRENT_TARGET_DEVICE_DICT.get(device_type, device_type)}-{model_type.__name__}" + for device_type, model_type in params_list + ] + metafunc.parametrize(f"{TARGET_DEVICE_PARAM_NAME}, {MarkTestParameters.MODEL_TYPE}", params_list, ids=ids_list) + + +def parametrize_model_aux_type(metafunc): + """Parametrize auxiliary (second) model for tests that need two models simultaneously. + + Requires model_type to also be present in the test — model_type handles target_device parametrization, + so model_aux_type only parametrizes the model class itself (no device cross-product). + """ + assert MarkTestParameters.MODEL_TYPE in metafunc.fixturenames, ( + f"model_aux_type requires model_type to also be a fixture in test {metafunc.function.__name__}" + ) + args = get_marker_args(metafunc, MarkTestParameters.MODEL_AUX_TYPE) + if args is None: + return + if isinstance(args[0], dict): + params_list = [model for device_type in config.target_devices for model in args[0][device_type]] + else: + params_list = list(args[0]) + ids_list = [model.__name__ for model in params_list] + metafunc.parametrize(MarkTestParameters.MODEL_AUX_TYPE, params_list, ids=ids_list) + + +def parametrize_all_models(metafunc): + args = get_marker_args(metafunc, MarkTestParameters.ALL_MODELS) + if args is None: + parametrize_target_device(metafunc, config.target_devices) + return + params_list = [] + for device_type in config.target_devices: + for _models in args[0]: + if isinstance(_models, dict): + params_list.append((device_type, _models[device_type])) + else: + params_list.append((device_type, _models)) + ids_list = lambda i: get_ids_with_target_device( + i, lambda x: x[-1].name if len(x) > 0 and hasattr(x[-1], "name") else "empty" + ) + metafunc.parametrize(f"{TARGET_DEVICE_PARAM_NAME}, {MarkTestParameters.ALL_MODELS}", params_list, ids=ids_list) + + +def parametrize_many_models(metafunc): + args = get_marker_args(metafunc, MarkTestParameters.MANY_MODELS) + if args is None: + parametrize_target_device(metafunc) + return + # (ModelsLib.get_many_models, 8, 100) -> ("CPU", (ModelsLib.get_many_models(8), 100)) + params_list = [ + (device_type, (arg[0](device_type, arg[1]), arg[2])) for device_type in config.target_devices for arg in args + ] + ids_list = lambda i: get_ids_with_target_device(i, lambda x: f"count={len(x[0])}-iters={x[1]}") + metafunc.parametrize(f"{TARGET_DEVICE_PARAM_NAME}, {MarkTestParameters.MANY_MODELS}", params_list, ids=ids_list) + + +def parametrize_iteration_info(metafunc): + args = get_marker_args(metafunc, MarkTestParameters.ITERATION_INFO) + if args is None: + parametrize_target_device(metafunc) + return + # [(ModelsLib.various_models, 0, False, True)] -> ("CPU", ([(ModelsLib.various_models["CPU"][0], False, True)]) + params_list = [] + for device_type in config.target_devices: + for _iteration_info_item in args: + iteration_info = [] + for _iteration_info in _iteration_info_item: + iteration_info.append( + (_iteration_info[0][device_type][_iteration_info[1]], _iteration_info[2], _iteration_info[3]) + ) + params_list.append((device_type, iteration_info)) + ids_list = lambda i: get_ids_with_target_device(i, ModelsLib.generate_ids_for_iteration_info) + metafunc.parametrize(f"{TARGET_DEVICE_PARAM_NAME}, {MarkTestParameters.ITERATION_INFO}", params_list, ids=ids_list) + + +def parametrize_input_shape(metafunc): + params_list = [] + args = get_marker_args(metafunc, MarkTestParameters.INPUT_SHAPE) + if args is not None: + # (ModelsLib.create_input_shapes_for_auto_reshape_tests, ModelsLib.reshapeable_model, ModelType.ONNX) -> + # ("CPU", ModelsLib.create_input_shapes_for_auto_reshape_tests(ModelsLib.reshapeable_model["CPU"][ModelType.ONNX])) + params_list = [ + (device_type, shape) + for device_type in config.target_devices + for shape in args[0](args[1][device_type][args[2]]) + ] + args = get_marker_args(metafunc, MarkTestParameters.INPUT_SHAPE_NO_AUTO) + if args is not None: + params_list = [(device_type, shape) for device_type in config.target_devices for shape in args[0][device_type]] + if params_list: + ids_list = lambda i: get_ids_with_target_device(i, ModelsLib.generate_model_shape_ids) + metafunc.parametrize(f"{TARGET_DEVICE_PARAM_NAME}, {MarkTestParameters.INPUT_SHAPE}", params_list, ids=ids_list) + else: + parametrize_target_device(metafunc) + + +def parametrize_plugin_config(metafunc): + args = get_marker_args(metafunc, MarkTestParameters.PLUGIN_CONFIG) + if args is None: + parametrize_target_device(metafunc) + return + params_list = [ + (device_type, plugin_config) for device_type in config.target_devices for plugin_config in args[0][device_type] + ] + ids_list = lambda i: get_ids_with_target_device(i, lambda x: "-".join(map(lambda y: "%s=%s" % y, x.items()))) + metafunc.parametrize(f"{TARGET_DEVICE_PARAM_NAME}, {MarkTestParameters.PLUGIN_CONFIG}", params_list, ids=ids_list) + + +def parametrize_target_device(metafunc): + ids = [CURRENT_TARGET_DEVICE_DICT.get(x, x) for x in config.target_devices] + metafunc.parametrize(TARGET_DEVICE_PARAM_NAME, config.target_devices, ids=ids) + + +def parametrize_ovms_type(metafunc): + metafunc.parametrize(OVMS_TYPE_PARAM_NAME, config.ovms_types) + + +def parametrize_uses_mapping(metafunc): + value_to_id = { + True: "use_mapping", + False: "no_mapping", + None: "default_model_mapping", + } + ids = [value_to_id[x] for x in config.uses_mapping] + metafunc.parametrize(USES_MAPPING_PARAM_NAME, config.uses_mapping, ids=ids) + + +def validate_port_pool(_config): + # This function should be called only in master xdist thread. + if any([ + config.ports_pool_size is None, + config.grpc_ovms_starting_port is None, + config.rest_ovms_starting_port is None, + ]): + print("Creating pool configuration:") + args = parse_args([ + "--reservation-file-json", + os.path.join(tmp_dir, "reservation.json"), + "--reservation-file-env", + os.path.join(tmp_dir, "reservation.env"), + "-c", + os.path.join(ovms_c_repo_path, "tests", "reservation_manager.yml"), + "--locks-dir", + global_tmp_dir_default, + ]) + + _config.reservation_manager = ReservationManager.manager_from_args(args=args) + _config.reservation_manager.independent.create() + os.environ.update(_config.reservation_manager.env_mgr.environment) + config.ports_pool_size = config.get_int("TT_PORTS_POOL_SIZE") + config.grpc_ovms_starting_port = config.get_int("TT_GRPC_OVMS_STARTING_PORT") + config.rest_ovms_starting_port = config.get_int("TT_REST_OVMS_STARTING_PORT") + else: + print("Successfully read reservation manager configuration.") + print("Port pool configuration:") + print(f"ports_pool_size={config.ports_pool_size}") + print(f"grpc_ovms_starting_port={config.grpc_ovms_starting_port}") + print(f"rest_ovms_starting_port={config.rest_ovms_starting_port}") + + +def parametrize_base_os(metafunc): + ids = [] + params = config.base_os + for _os in config.base_os: + if OsType.Windows in config.base_os: + os_name = OsType.Windows + if len(config.base_os) > 1: + raise NotImplementedError("Iterating with Windows OS is not supported") + elif any(_os.lower() == value for key, value in vars(OsType).items() if not key.startswith("__")): + image = calculate_ovms_image_name(config.target_devices[0], _os) + env_info = EnvironmentInfo.get_instance(class_info=OvmsInfo, image=image) + dist_name = env_info.get_os_distname() + if dist_name.startswith("Ubuntu 22.04") or dist_name.startswith(OsType.Ubuntu22): + os_name = OsType.Ubuntu22 + elif dist_name.startswith("Ubuntu 24.04") or dist_name.startswith(OsType.Ubuntu24): + os_name = OsType.Ubuntu24 + elif dist_name.startswith("Red Hat") or dist_name.startswith(OsType.Redhat): + os_name = OsType.Redhat + else: + raise Exception("Unexpected OS") + if config.is_nginx_mtls: + os_name = f"{os_name}_NGINX" + ids.append(os_name.upper()) + metafunc.parametrize(BASE_OS_PARAM_NAME, params, ids=ids) + + +def log_configuration_variables(): + logger.info("============== configuration variables ==============") + pt_env_vars = list(filter(lambda x: x[0].startswith("TT_"), os.environ.items())) + pt_env_vars.sort() + for env_var in pt_env_vars: + logger.info("{}={}".format(*env_var)) + + +def mute_warnings(): + # Mute warning: + # ResourceWarning: unclosed None: + self.name = username + self.password = password + self._session = None + + def __repr__(self): + return f"{self.__class__.__name__}(id={self.id!s}, name={self.name})" + + @property + def session(self): + return self._session + + @session.setter + def session(self, session: Session): + self._session = session + + @classmethod + def from_response(cls, rsp): + pass + + def delete(self): + pass + + def is_logged_in(self): + raise NotImplementedError("Must be implemented in subclass " + "to provide appropriate check " + "if is user logged in before use") diff --git a/tests/functional/utils/http/client_auth/__init__.py b/tests/functional/utils/http/client_auth/__init__.py new file mode 100644 index 0000000000..84cfbcf566 --- /dev/null +++ b/tests/functional/utils/http/client_auth/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/functional/utils/http/client_auth/auth.py b/tests/functional/utils/http/client_auth/auth.py new file mode 100644 index 0000000000..63c2128d92 --- /dev/null +++ b/tests/functional/utils/http/client_auth/auth.py @@ -0,0 +1,566 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pprint +import time +from enum import Enum +from typing import Callable, Union +from urllib.parse import urlparse + +import requests +from requests import Response, Session +from requests.auth import AuthBase, HTTPBasicAuth, extract_cookies_to_jar + +from tests.functional.utils.assertions import UnexpectedResponseError +from tests.functional.utils.http.base import HttpClientType, HttpMethod, HttpUser +from tests.functional.utils.http.http_client_configuration import HttpClientConfiguration +from tests.functional.utils.http.http_session import HttpSession +from tests.functional.utils.logger import get_logger +from tests.functional.config import http_proxy, https_proxy + +from tests.functional.utils.http.client_auth.base import ClientAuthBase +from tests.functional.utils.http.client_auth.exceptions import ( + ClientAuthFactoryInvalidAuthTypeException, + ClientAuthSessionMissingResponseSessionHeaderException, +) + +logger = get_logger(__name__) + + +class HTTPSessionAuth(AuthBase): + """Attaches session authentication to the given request object.""" + + def __init__(self, session): + self._session = session + + def __call__(self, request): + request.headers['Session'] = self._session + return request + + +class HTTPTokenAuth(AuthBase): + """Attaches token authentication to the given request object.""" + + def __init__(self, token): + self._token = token + + def __call__(self, request): + request.headers['Authorization'] = self._token + return request + + +class ClientAuthType(Enum): + """Client authentication types.""" + + HTTP_SESSION = "HttpSession" + HTTP_BASIC = "HttpBasic" + NO_AUTH = "NoAuth" + TOKEN_AUTH = "TokenAuth" + TOKEN_NO_AUTH = "TokenNoAuth" + SSL = "SSL" + LOGIN_PAGE = "Login Page" + OAUTH2_PROXY_AUTH = "OAuth2ProxyAuth" + + +class NoAuthConfigurationProvider(object): + """Provide configuration for no client_auth http client.""" + + @classmethod + def get(cls, url: str, proxies=None) -> HttpClientConfiguration: + """Provide http client configuration.""" + return HttpClientConfiguration( + client_type=HttpClientType.NO_AUTH, + url=url, + proxies=proxies + ) + + +class SslAuthConfigurationProvider(object): + """Provide configuration for https client with SSL/TLS.""" + + @classmethod + def get(cls, url: str, cert: tuple, proxies=None) -> HttpClientConfiguration: + """Provide http client configuration.""" + return HttpClientConfiguration( + client_type=HttpClientType.SSL, + url=url, + cert=cert, + proxies=proxies + ) + + +# pylint: disable=too-many-instance-attributes +class ClientAuthToken(ClientAuthBase): + """Base class that all token based http client authentication implementations derive from.""" + request_headers = {"Accept": "application/json"} + token_life_time = None + token_name = None + token_header_format = "Bearer {}" + + def __init__(self, url: str, session: HttpSession, params: dict = None): + self._token = None + self._token_header = None + self._token_timestamp = None + self._response = None + super().__init__(url, session, params) + + @property + def token(self) -> str: + """token is the token retrieved from the response during authentication.""" + return self._token + + @property + def authenticated(self) -> bool: + """Check if current user is authenticated.""" + return self._token and not self._is_token_expired() + + def authenticate(self) -> AuthBase: + """Use session credentials to authenticate.""" + self._response = self.session.request(**self.auth_request_params) + self._set_token() + self._http_auth = HTTPTokenAuth(self._token_header) + return self._http_auth + + def _is_token_expired(self): + """Check if token has been expired.""" + return time.time() - self._token_timestamp > self.token_life_time + + def _set_token(self): + """Set token taken from token request response.""" + if self.token_name not in self._response: + raise ClientAuthTokenMissingResponseTokenKeyException() + self._token_timestamp = time.time() + self._token = self._response[self.token_name] + self._token_header = self.token_header_format.format(self._token) + + def parse_params(self, params: dict): + self.token_name = params.get("token_name", "access_token") + self.token_life_time = params.get("token_life_time", 298) + self.request_data_params = params.get("request_data", {}) + self.request_params = params.get("request_params", {}) + + @property + def auth_request_params(self) -> dict: + """build request params""" + request_params = super().auth_request_params + request_params.update(self.request_params) + return request_params + + @property + def request_data(self) -> dict: + """Token request data.""" + request_data = super().request_data + request_data.update(self.request_data_params) + return request_data + + +class ClientAuthTokenMissingResponseTokenKeyException(Exception): + """Exception that is thrown when no token is found in the response""" + def __init__(self): + super().__init__("Token key is missing in token request response.") + + + +class ClientAuthSession(ClientAuthBase): + """Base class that all session based http client authentication implementations derive from.""" + DATA_OR_BODY = "body" + request_headers = {"Content-Type": "application/json"} + session_life_time = 600 + + def __init__(self, url: str, session: HttpSession, params: dict): + self._session_id = None + self._session_timestamp = None + self._response = None + super().__init__(url, session, params) + + @property + def session_id(self) -> str: + """session is the session retrieved from the response during authentication.""" + return self._session_id + + @property + def authenticated(self) -> bool: + """Check if current user is authenticated.""" + return self._session_id and not self._is_session_expired() + + def authenticate(self) -> AuthBase: + """Use session credentials to authenticate.""" + self._response = self.session.request(**self.auth_request_params) + self._set_session_id() + self._http_auth = HTTPSessionAuth(self._session_id) + return self._http_auth + + def _is_session_expired(self): + """Check if token has been expired.""" + return time.time() - self._session_timestamp > self.session_life_time + + def _set_session_id(self): + """Set token taken from token request response.""" + self._session_id = self._response.headers.get("Session", None) + if self._session_id is None: + raise ClientAuthSessionMissingResponseSessionHeaderException() + self._session_timestamp = time.time() + + @property + def auth_request_params(self) -> dict: + """build request params""" + request_params = super().auth_request_params + request_params.update({ + 'method': HttpMethod.POST, + 'raw_response': True, + 'log_message': "Retrieve session id.", + }) + return request_params + + def parse_params(self, params: dict): + self.session_life_time = params.get("session_life_time", 60 * 10) + self.request_data_params = params.get("request_data", {}) + self.request_params = params.get("request_params", {}) + + +class ClientAuthHttpBasic(ClientAuthBase): + """Http basic based http client authentication.""" + + def authenticate(self) -> AuthBase: + """Use session credentials to authenticate.""" + self._http_auth = HTTPBasicAuth(*self.request_data.values()) + return self._http_auth + + @property + def authenticated(self) -> bool: + """Check if current user is authenticated.""" + return True + + + +class ClientAuthTokenProvided(ClientAuthBase): + """Token based http client authentication.""" + + def __init__(self, url: str, session: HttpSession, params: dict = None): + self._token = params['token'] + super().__init__(url, session, params) + + def authenticate(self) -> AuthBase: + """Use session credentials to authenticate.""" + self._http_auth = HTTPTokenAuth(self._token) + return self._http_auth + + @property + def authenticated(self) -> bool: + """Check if current user is authenticated.""" + return True + + +class ClientAuthNoAuth(ClientAuthBase): + """No authentication.""" + + def authenticate(self) -> AuthBase: + pass + + @property + def authenticated(self) -> bool: + """always authenticated""" + return True + +class OAuth2ProxyAuth(AuthBase): + """Fake authorization, installed hooks will handle authorization""" + def __call__(self, r): + return r + +class ClientAuthSsl(ClientAuthNoAuth): + """ + Class implemented to comply with coding standard. + Authorisation is taken care by SSL certificates, no extra auth is needed. + """ + pass + + +class HTTPCookieAuth(AuthBase): + """Attaches session authentication to the given request object.""" + + def __init__(self, cookie): + self._cookie = cookie + + def __call__(self, request): + for key, value in self._cookie.items(): + if request.headers.get('Cookie', None) is not None: + request.headers['Cookie'] = f"{request.headers['Cookie']};{key}={value}" + else: + request.headers['Cookie'] = f"{key}={value}" + + return request + + @property + def cookie(self): + return self._cookie + + + +class ClientAuthLoginPage(ClientAuthBase): + """Login page based http client authentication.""" + CSRF_NAME = "_oauth2_proxy_csrf" + PROXY_NAME = "_oauth2_proxy" + proxies = { + "http": http_proxy , + "https": https_proxy, + "no_proxy": "" + } if http_proxy != "" else None + + def authenticate(self) -> AuthBase: + """Use session credentials to authenticate.""" + + response = requests.post(url=self._url, + data=self._request_data(), + cookies=self._request_cookies(), + verify=False, + allow_redirects=True, + proxies=self.proxies) + if not response.ok: + raise UnexpectedResponseError(response.status_code, response.text) + previous_response = response.history[-1] + cookie = {self.PROXY_NAME: previous_response.cookies.get(self.PROXY_NAME)} + + return HTTPCookieAuth(cookie=cookie) + + @property + def authenticated(self) -> bool: + return True + + def parse_params(self, params: dict) -> None: + """params parser to pass configuration customizations""" + self.request_params = params + + def _request_headers(self): + """Prepare request data.""" + csrf_name = "_oauth2_proxy_csrf" + return { + "Cookie": f"{csrf_name}={self.request_params[csrf_name]}", + } + + def _request_cookies(self): + """Prepare request data.""" + return {self.CSRF_NAME: self.request_params[self.CSRF_NAME]} + + def _request_data(self): + """Prepare request data.""" + data = { + "login": self.session.username, + "password": self.session.password, + } + return data + + +class ClientAuthOAuth2Proxy(ClientAuthBase): + """Login page based http client authentication.""" + CSRF_NAME = "_oauth2_proxy_csrf" + PROXY_NAME = "_oauth2_proxy" + + def __init__(self, url: str, session: HttpSession, params: dict = None): + self.hooks_added = False + super().__init__(url, session, params) + + def cookies_repr(self, indent: int): + cookies = [] + for name, value in self.session.cookies.iteritems(): + name_ = f"{name}:" + space = 24 - indent + cookies.append(f"{' '*indent}{name_:{space}} {value[:30]}") + return "\n".join(cookies) + "\n" + + def login_hook(self, http_session: HttpSession) -> Callable[[Response], Response]: + state = dict(logging_hook_fn=0, redirects=0, + authorizations=0, authorizations_skipped=0) + + def login_hook_fn(initial_response: Response, *args, **kwargs) -> Response: + logger.verbose(f"\nLogin hook for session: {str(id(http_session))[-4:]}." + f"User: {http_session.username} " + f" state:\n{pprint.pformat(state)}\n" + f" proxy:\n{pprint.pformat(http_session.session.proxies)}") + state["logging_hook_fn"] += 1 + session = http_session.session + location = initial_response.headers.get("Location") + if initial_response.status_code == 302 and location is not None: + state["redirects"] += 1 + next_url = urlparse(location) + initial_request = initial_response.request + started_authorization_path = getattr(session, "__authorization_started", + next_url.path) + if any(path == next_url.path for path in ["/oauth2/start", "/dex/auth"]) and\ + started_authorization_path == next_url.path: + tt_resend_counter = int( + initial_request.headers.get("tt_resend_counter", "1") + ) + if tt_resend_counter < 4 and http_session.user.is_logged_in(): + state["authorizations_skipped"] += 1 + logger.verbose( + f"\nLogin hook for session: {str(id(http_session))[-4:]}.\n" + f"User: {http_session.username} " + f"re-send counter is {tt_resend_counter}, re-using cookies.") + return re_send_initial_request(initial_response, session) + logger.verbose(f"re-send counter is {tt_resend_counter}, re-authorizing.") + state["authorizations"] += 1 + logger.verbose( + f"\nLogin hook for session: {str(id(http_session))[-4:]}.\n" + f"User: {http_session.username} is unauthorized. " + f"Starting authorization:\n" + f" redirecting to {location:.100}\n" + f" from {initial_response.url:.100}\n" + f" cookies:\n" + f"{self.cookies_repr(4)}") + if hasattr(session, "__authorization_started"): + delattr(session, "__authorization_started") + raise RuntimeError(f"Authorization of {http_session.username} not " + f"successful for session " + f"{str(id(http_session))[-4:]}.") + setattr(session, "__authorization_started", next_url.path) + if self.CSRF_NAME not in session.cookies and\ + self.CSRF_NAME in initial_response.cookies: + extract_cookies_to_jar(session.cookies, + initial_response.request, + initial_response.raw) + oauth2_proxy_csrf_response = session.get(location) + oauth2_proxy_csrf_request = oauth2_proxy_csrf_response.request + oauth2_proxy_response = session\ + .post(oauth2_proxy_csrf_request.url, + data=self._authorization_credentials_data(), + cookies=oauth2_proxy_csrf_response.cookies) + if oauth2_proxy_response.ok and self.PROXY_NAME in session.cookies: + delattr(session, "__authorization_started") + logger.verbose( + f"\nLogin hook for session: {str(id(http_session))[-4:]}.\n" + f"User: {http_session.username} is authorized. " + f"Stopping authorization\n" + f" Cookies:\n" + f"{self.cookies_repr(4)}") + initial_url = urlparse(initial_response.request.url) + if "oauth2/sign_in" not in initial_url.path: + return re_send_initial_request(initial_response, session) + return oauth2_proxy_response + else: + logger.verbose( + f"\nLogin hook for session: {str(id(http_session))[-4:]}.\n" + f"User: {http_session.username} is redirected.\n" + f" redirecting to {location:.100}\n" + f" from {initial_response.url:.100}\n" + f" cookies:\n" + f"{self.cookies_repr(4)}") + return initial_response + + def re_send_initial_request(initial_response: Response, session: Session): + re_authorized_request = initial_response.request.copy() + tt_resend_counter = int(re_authorized_request.headers.get("tt_resend_counter", "1")) + logger.verbose( + f"\nLogin hook for session: {str(id(http_session))[-4:]}.\n" + f"User: {http_session.username} is re-sending: {tt_resend_counter}") + if tt_resend_counter > 5: + if hasattr(session, "__authorization_started"): + delattr(session, "__authorization_started") + raise RuntimeError("Resend counter exceeded.") + headers = re_authorized_request.headers + headers.pop('Cookie', None) + headers.update({ + "tt_resend_counter": str(tt_resend_counter + 1) + }) + re_authorized_request.prepare_headers(headers) + re_authorized_request.prepare_cookies(session.cookies) + if tt_resend_counter > 1: + re_send_cookie = re_authorized_request.headers.get("Cookie", None) + + re_send_cookie = re_send_cookie[len("_oauth2_proxy="):][:30] if re_send_cookie \ + else "None" + session_cookie = session.cookies.get(self.PROXY_NAME) + session_cookie = session_cookie[:30] if session_cookie else "None" + logger.verbose( + f"\nLogin hook for session: {str(id(http_session))[-4:]}.\n" + f"User: {http_session.username} is re-sending {tt_resend_counter} with:\n" + f"{'header cookie:':24} {re_send_cookie}\n" + f"{'session cookie:':24} {session_cookie}") + re_authorized_response = session.send(re_authorized_request) + return re_authorized_response + + return login_hook_fn + + def authenticate(self) -> AuthBase: + """Use session credentials to authenticate.""" + session = self.session._session + logger.verbose(f" ********************* " + f"Authenticating {self.session.username} " + f"******************") + if not self.hooks_added: + logger.verbose("adding hooks") + session.hooks["response"].append(self.login_hook(self.session)) + self.hooks_added = True + return OAuth2ProxyAuth() + + @property + def authenticated(self) -> bool: + return True + + def parse_params(self, params: dict) -> None: + """params parser to pass configuration customizations""" + self.request_params = params + + def _authorization_credentials_data(self): + """Prepare request data.""" + data = { + "login": self.session.user.id, + "password": self.session.user.password, + } + return data + + +class ClientAuthFactory(object): + """Client authentication factory.""" + + EMPTY_URL = "" + + @staticmethod + def get(username: Union[HttpUser, str] = None, + password: str = None, + auth_type: ClientAuthType = None, + proxies: dict = None, + cert: tuple = None, + auth_url: str = EMPTY_URL, + params: dict = None) -> ClientAuthBase: + """Create client authentication for given type.""" + session = HttpSession(username, password, proxies, cert) + + if auth_type == ClientAuthType.TOKEN_AUTH: + return ClientAuthToken(auth_url, session, params) + + elif auth_type == ClientAuthType.HTTP_SESSION: + return ClientAuthSession(auth_url, session, params) + + elif auth_type == ClientAuthType.HTTP_BASIC: + return ClientAuthHttpBasic(auth_url, session, params) + + elif auth_type == ClientAuthType.TOKEN_NO_AUTH: + return ClientAuthTokenProvided(auth_url, session, params) + + elif auth_type == ClientAuthType.NO_AUTH: + return ClientAuthNoAuth(auth_url, session) + + elif auth_type == ClientAuthType.SSL: + return ClientAuthSsl(auth_url, session) + + elif auth_type == ClientAuthType.LOGIN_PAGE: + return ClientAuthLoginPage(auth_url, session, params) + + elif auth_type == ClientAuthType.OAUTH2_PROXY_AUTH: + return ClientAuthOAuth2Proxy(auth_url, session, params) + + else: + raise ClientAuthFactoryInvalidAuthTypeException(auth_type) diff --git a/tests/functional/utils/http/client_auth/base.py b/tests/functional/utils/http/client_auth/base.py new file mode 100644 index 0000000000..4b5c4570c9 --- /dev/null +++ b/tests/functional/utils/http/client_auth/base.py @@ -0,0 +1,81 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from abc import ABCMeta, abstractmethod +from collections import OrderedDict + +from requests.auth import AuthBase + +from tests.functional.utils.http.http_session import HttpSession + + +class ClientAuthBase(object, metaclass=ABCMeta): + """Base class that all http client authentication implementations derive from. + + It performs automatic authentication. + """ + DATA_OR_BODY = "data" + _http_auth = None + request_headers = None + request_data_params = None + request_params = None + + def __init__(self, url: str, session: HttpSession, params: dict = None): + """ + Args: + url: url + session: http session + params: additional params + """ + self._url = url + self.session = session + self.parse_params(params if params is not None else {}) + self._http_auth = self.authenticate() + + @property + def http_auth(self) -> AuthBase: + """Http authentication method.""" + return self._http_auth + + @property + @abstractmethod + def authenticated(self) -> bool: + """Is current user already authenticated.""" + + @abstractmethod + def authenticate(self) -> AuthBase: + """Use session credentials to authenticate.""" + + @property + def request_data(self) -> OrderedDict: + """Token request data.""" + return OrderedDict([ + ("username", self.session.username), + ("password", self.session.password), + ]) + + def parse_params(self, params: dict) -> None: + """params parser to pass configuration customizations""" + pass + + @property + def auth_request_params(self) -> dict: + """build request params""" + return { + 'url': self._url, + 'headers': self.request_headers, + self.DATA_OR_BODY: self.request_data, + } diff --git a/tests/functional/utils/http/client_auth/exceptions.py b/tests/functional/utils/http/client_auth/exceptions.py new file mode 100644 index 0000000000..74637d3165 --- /dev/null +++ b/tests/functional/utils/http/client_auth/exceptions.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.utils.assertions import TemplateMessageException + + +class ClientAuthFactoryInvalidAuthTypeException(TemplateMessageException): + TEMPLATE = "Client authentication with type {} is not implemented." + + +class ClientAuthSessionMissingResponseSessionHeaderException(TemplateMessageException): + """Exception that is thrown when no session id is found in the response""" + TEMPLATE = "Session header is missing in session request response." diff --git a/tests/functional/utils/http/exceptions.py b/tests/functional/utils/http/exceptions.py new file mode 100644 index 0000000000..a7ec59bab9 --- /dev/null +++ b/tests/functional/utils/http/exceptions.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.utils.assertions import TemplateMessageException + + +class HttpClientFactoryInvalidClientTypeException(TemplateMessageException): + TEMPLATE = "Http client with type {} is not implemented." + + +class HttpClientConfigurationEmptyPropertyException(TemplateMessageException): + TEMPLATE = "Property '{}' can not be empty." + + +class HttpClientConfigurationInvalidPropertyTypeException(TemplateMessageException): + TEMPLATE = "Property '{}' has invalid type." diff --git a/tests/functional/utils/http/http_client.py b/tests/functional/utils/http/http_client.py new file mode 100644 index 0000000000..a104120ae2 --- /dev/null +++ b/tests/functional/utils/http/http_client.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.utils.http.base import HttpMethod +from tests.functional.utils.http.client_auth.base import ClientAuthBase +from tests.functional.utils.http.http_session import HttpSession + + +class HttpClient(object): + """Http api client.""" + + def __init__(self, url: str, auth: ClientAuthBase): + self.url = url + self._auth = auth + + @property + def auth(self) -> ClientAuthBase: + """Client client_auth.""" + return self._auth + + @property + def cookies(self): + """Session cookies.""" + return self.auth.session.cookies + + @property + def session(self) -> HttpSession: + return self._auth.session + + @session.setter + def session(self, session: HttpSession): + self._auth.session = session + + def get_cookie(self, name: str): + return self.session.get_cookie(name) + + # pylint: disable=too-many-arguments + def request(self, method: HttpMethod, path, path_params=None, url=None, headers=None, files=None, params=None, + data=None, body=None, credentials=None, msg="", raw_response=False, timeout=900, raise_exception=True, + log_response_content=True): + """Perform request and return response.""" + if not self._auth.authenticated: + self._auth.authenticate() + if credentials is None: + credentials = self._auth.http_auth + url = self.url if url is None else url + response = self._auth.session.request( + method=method, + url=url, + path=path, + path_params=path_params, + headers=headers, + files=files, + params=params, + data=data, + body=body, + auth=credentials, + log_message=msg, + raw_response=raw_response, + timeout=timeout, + raise_exception=raise_exception, + log_response_content=log_response_content + ) + + if "session_expired" == format(response).strip(): + self._auth.authenticate() + return self._auth.session.request( + method=method, + url=url, + path=path, + path_params=path_params, + headers=headers, + files=files, + params=params, + data=data, + body=body, + auth=self._auth.http_auth, + log_message=msg, + raw_response=raw_response, + timeout=timeout, + raise_exception=raise_exception + ) + + return response diff --git a/tests/functional/utils/http/http_client_configuration.py b/tests/functional/utils/http/http_client_configuration.py new file mode 100644 index 0000000000..70dcb7e2cf --- /dev/null +++ b/tests/functional/utils/http/http_client_configuration.py @@ -0,0 +1,131 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Union +from urllib.parse import urlparse, urlunparse + +from tests.functional.utils.http.base import HttpClientType, HttpUser +from tests.functional.utils.http.exceptions import ( + HttpClientConfigurationEmptyPropertyException, + HttpClientConfigurationInvalidPropertyTypeException, +) + + +# pylint: disable=too-many-instance-attributes +class HttpClientConfiguration(object): + """Http client configuration.""" + identity_attributes = ("client_type", "url", "username", "password") + + # pylint: disable=too-many-arguments + def __init__(self, client_type: HttpClientType, url: str, + username: Union[HttpUser, str] = None, password: str = None, + proxies: dict = None, cert: tuple = None, auth_uri: str = None, params: dict = None): + """ + Args: + client_type: HttpClientType enum + url: url to connect to. + username: user name or HttpUser instance for authentication + password: user password for authentication + proxies: proxies + cert: certs + auth_uri: authorization uri -> client_auth path or client_auth url for authorization. + If path provided auth_url is build from url and path. + params: additional params if any + """ + self._validate("client_type", HttpClientType, client_type) + self._validate("url", str, url) + self._client_type = client_type + self._url = url + self._auth_url = self._prepare_auth_url(url, auth_uri) if auth_uri is not None else None + self._username = username + self._password = password + self.proxies = proxies + self.cert = cert + self.params = params + + def _prepare_auth_url(self, url: str, auth_uri: str) -> str: + """ + Build client_auth url. If auth_uri is url use it, if not build auth_url based on url and auth_uri + Args: + url: host url + auth_uri: authorization path or authorization url if different from host url + + Returns: str authorization url. + """ + self._auth_uri = urlparse(auth_uri) + if not self._auth_uri.netloc: + url = urlparse(url) + auth_url = urlunparse((url.scheme, url.netloc, + self._auth_uri.path, self._auth_uri.params, + self._auth_uri.query, self._auth_uri.fragment)) + else: + auth_url = auth_uri + return auth_url + + def __eq__(self, other): + return all(getattr(self, a) == getattr(other, a) for a in self.identity_attributes) + + def __hash__(self): + return hash(tuple(getattr(self, a) for a in self.identity_attributes)) + + @property + def client_type(self): + """Client type.""" + return self._client_type + + @property + def url(self): + """Client api url address.""" + return self._url + + @property + def auth_url(self): + """Client authorization path""" + return self._auth_url + + @property + def username(self): + """Client client_auth username.""" + return self._username + + @property + def password(self): + """Client client_auth password.""" + return self._password + + @property + def as_dict(self) -> dict: + kwargs = dict() + self.set_value(kwargs, "username", self.username) + self.set_value(kwargs, "password", self.password) + self.set_value(kwargs, "proxies", self.proxies) + self.set_value(kwargs, "cert", self.cert) + self.set_value(kwargs, "auth_url", self.auth_url) + self.set_value(kwargs, "params", self.params) + return kwargs + + @staticmethod + def set_value(d: dict, key: str, value): + if value is not None: + d[key] = value + + @staticmethod + def _validate(property_name, property_type, property_value): + """Validate if given property has valid type and value.""" + if not property_value: + raise HttpClientConfigurationEmptyPropertyException(property_name) + if not isinstance(property_value, property_type): + raise HttpClientConfigurationInvalidPropertyTypeException(property_name) diff --git a/tests/functional/utils/http/http_client_factory.py b/tests/functional/utils/http/http_client_factory.py new file mode 100644 index 0000000000..b894352239 --- /dev/null +++ b/tests/functional/utils/http/http_client_factory.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.utils.http.base import HttpClientType +from tests.functional.utils.http.client_auth.auth import ClientAuthFactory, ClientAuthType +from tests.functional.utils.http.exceptions import HttpClientFactoryInvalidClientTypeException +from tests.functional.utils.http.http_client import HttpClient +from tests.functional.utils.http.http_client_configuration import HttpClientConfiguration + + +class HttpClientFactory(object): + """Http client factory with implemented singleton behaviour for each generated client.""" + + _INSTANCES = {} + + @classmethod + def get(cls, configuration: HttpClientConfiguration) -> HttpClient: + """Create http client for given configuration.""" + client_type = configuration.client_type + + if client_type == HttpClientType.TOKEN_AUTH: + return cls._get_instance(configuration, ClientAuthType.TOKEN_AUTH) + + elif client_type == HttpClientType.SESSION_AUTH: + return cls._get_instance(configuration, ClientAuthType.HTTP_SESSION) + + elif client_type == HttpClientType.NO_AUTH: + return cls._get_instance(configuration, ClientAuthType.NO_AUTH) + + elif client_type == HttpClientType.K8S: + return cls._get_instance(configuration, ClientAuthType.TOKEN_NO_AUTH) + + elif client_type == HttpClientType.BROKER: + return cls._get_instance(configuration, ClientAuthType.HTTP_BASIC) + + elif client_type == HttpClientType.BASIC_AUTH: + return cls._get_instance(configuration, ClientAuthType.HTTP_BASIC) + + elif client_type == HttpClientType.API: + return cls._get_instance(configuration, ClientAuthType.LOGIN_PAGE) + + elif client_type == HttpClientType.OAUTH2_PROXY_AUTH: + return cls._get_instance(configuration, ClientAuthType.OAUTH2_PROXY_AUTH) + + elif client_type == HttpClientType.SSL: + return cls._get_instance(configuration, ClientAuthType.SSL) + + else: + raise HttpClientFactoryInvalidClientTypeException(client_type) + + @classmethod + def remove(cls, configuration: HttpClientConfiguration): + """Remove client instance from cached instances.""" + if configuration in cls._INSTANCES: + del cls._INSTANCES[configuration] + + @classmethod + def _get_instance(cls, configuration: HttpClientConfiguration, auth_type): + """Check if there is already created requested client type and return it otherwise create new instance.""" + if configuration in cls._INSTANCES: + return cls._INSTANCES[configuration] + return cls._create_instance(configuration, auth_type) + + @classmethod + def _create_instance(cls, configuration: HttpClientConfiguration, auth_type): + """Create new client instance.""" + auth = ClientAuthFactory.get(auth_type=auth_type, **configuration.as_dict) + instance = HttpClient(configuration.url, auth) + cls._INSTANCES[configuration] = instance + return instance diff --git a/tests/functional/utils/http/http_session.py b/tests/functional/utils/http/http_session.py new file mode 100644 index 0000000000..77ec957061 --- /dev/null +++ b/tests/functional/utils/http/http_session.py @@ -0,0 +1,207 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Wrapper for the Session class from the requests library.""" + +import json +from abc import ABCMeta +from http.cookiejar import Cookie, CookieJar +from typing import Optional, Union + +from requests import PreparedRequest, Request, Session, exceptions +from requests.adapters import HTTPAdapter +from requests_toolbelt import MultipartEncoder +from retry import retry + +from tests.functional.utils.assertions import UnexpectedResponseError +from tests.functional.utils.http.base import HttpMethod, HttpUser +from tests.functional.utils.logger import LoggerType, get_logger +from tests.functional.config import http_proxy, https_proxy, logged_response_body_length, no_proxy, ssl_validation + +POOL_SIZE = 100 + +logger = get_logger(__name__) + + +class HttpSession(object, metaclass=ABCMeta): + """HttpSession is wrapper for the Session class from the requests library. + + It stores the information about the username, password, possible proxies and certificates. + It does not do any authentication per se. + For ease of use it has a "request" method that prepares and performs the request. + """ + def __init__(self, username: Union[HttpUser, str] = None, password: str = None, + proxies: dict = None, cert: tuple = None): + self._user = username if isinstance(username, HttpUser) else HttpUser(username, password) + self._session = Session() + if proxies is not None: + self._session.proxies = proxies + elif http_proxy: + self._session.proxies = {"http": http_proxy, + "https": https_proxy, + "no_proxy": no_proxy} + + self._session.verify = ssl_validation + if cert is not None: + self._session.cert = cert + try: + self._session.verify = cert[2] + except IndexError: + pass + + adapter = HTTPAdapter(pool_connections=POOL_SIZE, pool_maxsize=POOL_SIZE) + self._session.mount('http://', adapter) + self._session.mount('https://', adapter) + + @property + def user(self): + return self._user + + @property + def username(self) -> str: + """Session user name.""" + return self.user.name + + @property + def password(self) -> str: + """Session user password.""" + return self.user.password + + @property + def session(self) -> Session: + """Session cookies.""" + return self._session + + @property + def cookies(self): + """Session cookies.""" + return self.session.cookies + + def get_cookie(self, name: str) -> Optional[Cookie]: + return self.get_cookie_from_jar(name, self.cookies) + + @staticmethod + def get_cookie_from_jar(name: str, cookies: CookieJar) -> Optional[Cookie]: + matching_cookies = filter(lambda c: c.name == name, + iter(cookies)) + first_cookie = next(matching_cookies, None) + for cookie in matching_cookies: + logger.warning(f"Additional matching cookie: {cookie}") + return first_cookie + + # pylint: disable=too-many-arguments + def request(self, method: HttpMethod, url, path="", path_params=None, + headers=None, files=None, data=None, params=None, auth=None, + body=None, log_message="", raw_response=False, timeout=None, + raise_exception=True, log_response_content=True): + """Wrapper for the request method from the Session library""" + path_params = {} if path_params is None else path_params + url = f"{url}/{path}".format(**path_params) + logger.debug(f"\nSending rq for session: {str(id(self))[-4:]} " + f"request {method} as a\n" + f"user: {self.username} to {url}") + request = self._request_prepare(method, url, headers, files, + data, params, auth, body, log_message) + return self._request_perform(request, path, path_params, raw_response, timeout=timeout, + raise_exception=raise_exception, + log_response_content=log_response_content) + + # pylint: disable=too-many-arguments + def _request_prepare(self, method, url, headers, files, data, params, auth, body, log_message): + """Prepare request to perform.""" + request = Request(method=method, url=url, headers=headers, + files=files, data=data, params=params, + auth=auth, json=body) + + prepared_request = self._session.prepare_request(request) + logger.debug(f"Prepared request: {prepared_request.__dict__}") + return prepared_request + + def _request_perform(self, request: PreparedRequest, path: str, path_params, raw_response: bool, + timeout: int, raise_exception: bool, log_response_content=True): + """Perform request and return response.""" + response = self._send_request_and_get_raw_response(request, timeout=timeout) + if log_response_content: + # workaround for downloading large files - reading response.text takes too long + HttpSession.log_http_response(response) + else: + HttpSession.log_http_response(response, logged_body_length=0) + + if raise_exception and not response.ok and response.text.strip() != "session_expired": + raise UnexpectedResponseError(response.status_code, response.text) + + if raw_response is True: + return response + + try: + return json.loads(response.text) + except ValueError: + return response.text + + @retry(exceptions=exceptions.ConnectionError, tries=2) + def _send_request_and_get_raw_response(self, request: PreparedRequest, timeout: int): + return self._session.send(request, timeout=timeout) + + @staticmethod + def _format_message_body(msg_body, logged_body_length=None): + limit = int(logged_response_body_length) if logged_body_length is None else logged_body_length + if 0 < limit < len(msg_body): + half = limit // 2 + msg_body = f"{msg_body[:half]}[...]{msg_body[-half:]}" + elif limit == 0: + msg_body = "[...]" + return msg_body + + @staticmethod + def log_http_response(response, logged_body_length=None, history_depth=1): + """If logged_body_length < 0, full response body is logged""" + if history_depth > 0 and response.history: + for history_response in response.history: + HttpSession.log_http_response(response=history_response, + logged_body_length=logged_body_length, + history_depth=history_depth - 1) + + msg_body = HttpSession._format_message_body(response.text, logged_body_length) + msg = [ + "\n----------------Response------------------", + f"Status code: {response.status_code}", + f"Headers: {response.headers}", + f"Content: {msg_body}", + "-----------------------------------------\n" + ] + get_logger(LoggerType.HTTP_RESPONSE).debug("\n".join(msg)) + + @staticmethod + def log_http_request(prepared_request, username, description="", data=None): + if isinstance(prepared_request.body, MultipartEncoder): + body = prepared_request.body + else: + prepared_body = prepared_request.body + body = prepared_body if not data else json.dumps(data) + content_type = prepared_request.headers.get("content-type", "") + msg_body = HttpSession._format_message_body(body) if ( + body is not None and "multipart/form-data" not in content_type + ) else "" + msg = [ + description, + "----------------Request------------------", + f"Client name: {username}", + f"URL: {prepared_request.method} {prepared_request.url}", + f"Headers: {prepared_request.headers}", + f"Body: {msg_body}", + "-----------------------------------------" + ] + get_logger(LoggerType.HTTP_REQUEST).debug("\n".join(msg)) diff --git a/tests/functional/utils/http/http_socket_wrapper.py b/tests/functional/utils/http/http_socket_wrapper.py new file mode 100644 index 0000000000..9f4195acd3 --- /dev/null +++ b/tests/functional/utils/http/http_socket_wrapper.py @@ -0,0 +1,74 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +import socket +from time import time + + +class HttpSocketWrapper: + _EXP_BODY_LEN = re.compile(r'Content-Length: (\d+)\s') + _EXP_BODY_SEPARATOR = re.compile(r'\s{4}') + + def __init__(self, host, port): + self._family = socket.AF_INET + self._type = socket.SOCK_STREAM + self.host = host + self.port = port + + def send(self, method, path, body='', headers={}): + default_headers = {"Host": f"{self.host}:{self.port}", + "Content-Length": str(len(body)), + "Connection": 'close'} + default_headers.update(headers) + header = f"{method} {path} HTTP/1.1\n" + for name, value in default_headers.items(): + header += f"{name}: {value}\n" + header += "\n" + msg = header.encode('ascii') + body.encode('ascii') + + with socket.socket(self._family, self._type) as soc: + soc.connect((self.host, self.port)) + start = time() + soc.sendall(msg) + + header, body = self._receive(soc) + end = time() + + return end - start, header, body + + def _receive(self, soc): + base_len, header, body = 10, '', '' + + while HttpSocketWrapper._EXP_BODY_SEPARATOR.search(header) is None: + header += soc.recv(base_len).decode('ascii') + + match = HttpSocketWrapper._EXP_BODY_SEPARATOR.search(header) + body = header[match.end():] + header = header[:match.start()] + + match = HttpSocketWrapper._EXP_BODY_LEN.search(header) + if match: + body_len = int(match.group(1)) + base_len = body_len - len(body) + body += soc.recv(base_len).decode('ascii') + else: + tmp_msg = soc.recv(base_len).decode('ascii') + while len(tmp_msg) > 0: + body += tmp_msg + tmp_msg = soc.recv(base_len).decode('ascii') + + return header, body diff --git a/tests/functional/utils/inference/__init__.py b/tests/functional/utils/inference/__init__.py new file mode 100644 index 0000000000..84cfbcf566 --- /dev/null +++ b/tests/functional/utils/inference/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/functional/utils/inference/capi.py b/tests/functional/utils/inference/capi.py new file mode 100644 index 0000000000..f7dc4b767f --- /dev/null +++ b/tests/functional/utils/inference/capi.py @@ -0,0 +1,111 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest + +from tests.functional.utils.assertions import NotSupported, _assert_status_code_and_message +from tests.functional.utils.inference.communication.base import AbstractCommunicationInterface +from tests.functional.utils.inference.serving.base import AbstractServingWrapper +from tests.functional.data.ovms_capi_wrapper.ovms_capi_shared import OvmsInferenceFailed, OvmsModelNotFound + + +class CapiServingWrapper(AbstractServingWrapper, AbstractCommunicationInterface): + + NOT_FOUND = OvmsModelNotFound + INVALID_ARGUMENT = OvmsInferenceFailed + INTERNAL = None + ABORTED = None + RESOURCE_EXHAUSTED = None + FAILED_PRECONDITION = None + UNAVAILABLE = None + ALREADY_EXISTS = None + + def __init__(self, ovms_capi_instance, **kwargs): + self.ovms_capi_instance = ovms_capi_instance + + def set_grpc_stubs(self): + raise NotImplementedError() + + def create_inference(self): + pass # no special init required + + def predict(self, request, timeout=60, raw=False): + return self.ovms_capi_instance.send_inference(self.model, request) + + def predict_stream(self): + raise NotSupported("Streaming API is supported only for KFS:GRPC communication") + + def get_rest_path(self, operation, model_version=None, model_name=None): + raise NotImplementedError() + + def get_inputs_outputs_from_response(self, response): + if getattr(self.model, "inputs", None) is None: + self.model.inputs = {} + if getattr(self.model, "outputs", None) is None: + self.model.outputs = {} + + for _input in response['inputs']: + self.model.inputs[_input['name']] = { + 'shape': _input['shape'], + 'dtype': _input['datatype'] + } + + for output in response['outputs']: + self.model.outputs[output['name']] = { + 'shape': output['shape'], + 'dtype': output['datatype'] + } + + return + + def get_model_meta_grpc_request(self, model_name=None): + raise NotImplementedError() + + def get_predict_grpc_request(self): + raise NotImplementedError() + + def prepare_request(self, input_objects: dict, **kwargs): + return input_objects + + def get_next_response_from_stream(self, output_stream): + raise NotSupported("Streaming API is supported only for KFS:GRPC communication") + + def get_model_meta(self, timeout=60, version=None, update_model_info=True, model_name=None): + response = self.ovms_capi_instance.send_get_model_meta_command(self.model.name, self.model.version) + if update_model_info: + self.get_inputs_outputs_from_response(response) + return response + + def validate_meta(self, model, meta): + for shape in model.input_shapes.values(): + if not any(shape == _input['shape'] for _input in meta["inputs"]): + raise Exception(f"Cannot find shape={shape} in meta={meta}") + for shape in model.output_shapes.values(): + if not any(shape == _output['shape'] for _output in meta["outputs"]): + raise Exception(f"Cannot find shape={shape} in meta={meta}") + + def get_model_status(self, version=None): + raise NotSupported("Get model status is not supported in C_API") + + def send_predict_request(self, request, timeout): + raise NotImplementedError() + + @staticmethod + def assert_raises_exception(status, error_message_phrase, callable_obj, *args, **kwargs): + with pytest.raises(status) as e: + callable_obj(*args, **kwargs) + _assert_status_code_and_message(status, error_message_phrase, + e.value.status, e.value.error_message, e) diff --git a/tests/functional/utils/inference/communication/__init__.py b/tests/functional/utils/inference/communication/__init__.py new file mode 100644 index 0000000000..ebc836864b --- /dev/null +++ b/tests/functional/utils/inference/communication/__init__.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.utils.inference.communication.grpc import GRPC +from tests.functional.utils.inference.communication.rest import REST diff --git a/tests/functional/utils/inference/communication/base.py b/tests/functional/utils/inference/communication/base.py new file mode 100644 index 0000000000..3d377ccd43 --- /dev/null +++ b/tests/functional/utils/inference/communication/base.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import abc + +from tests.functional.utils.ssl import SslCertificates +from tests.functional.object_model.test_environment import TestEnvironment + + +class AbstractCommunicationInterface(metaclass=abc.ABCMeta): + def __init__(self, port: int = None, address: str = None, + ssl_certificates: SslCertificates = None, **kwargs): + self.ssl_certificates = ssl_certificates + self.address = TestEnvironment.get_server_address() if address is None else address + self.port = port + self.url = f"{self.address}:{self.port}" + + @abc.abstractmethod + def prepare_request(self, input_objects: dict, **kwargs): + """ + Abstract method for preparing the inference request. + """ + pass + + @abc.abstractmethod + def get_model_meta(self, timeout=60, version=None, update_model_info=True, model_name=None): + pass + + @abc.abstractmethod + def get_model_status(self, model_name=None): + pass + + @abc.abstractmethod + def send_predict_request(self, request, timeout): + pass + + @staticmethod + @abc.abstractmethod + def assert_raises_exception(status, error_message_phrase, callable_obj, *args, **kwargs): + pass diff --git a/tests/functional/utils/inference/communication/constants.py b/tests/functional/utils/inference/communication/constants.py new file mode 100644 index 0000000000..0cbcfbdffc --- /dev/null +++ b/tests/functional/utils/inference/communication/constants.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re + +NOT_A_NUMBER_REGEX = re.compile(r"(NaN|,\s,)", re.IGNORECASE) diff --git a/tests/functional/utils/inference/communication/grpc.py b/tests/functional/utils/inference/communication/grpc.py new file mode 100644 index 0000000000..ce9fadb83b --- /dev/null +++ b/tests/functional/utils/inference/communication/grpc.py @@ -0,0 +1,194 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json + +import grpc +from grpc._channel import _InactiveRpcError + +from tests.functional.constants.ovms import Ovms + +from tests.functional.utils.assertions import AccuracyException, NotSupported, assert_raises_grpc_exception +from tests.functional.utils.inference.communication.base import AbstractCommunicationInterface +from tests.functional.utils.inference.communication.constants import NOT_A_NUMBER_REGEX + +GRPC_TIMEOUT = 60 +GRPC = "grpc" +channel_options = [('grpc.max_message_length', 100 * 1024 * 1024), + ('grpc.max_send_message_length ', 100 * 1024 * 1024), + ('grpc.max_receive_message_length', 100 * 1024 * 1024)] + + +class GrpcErrorCode: + """ + Defines available gRPC status codes. + """ + INVALID_ARGUMENT = grpc.StatusCode.INVALID_ARGUMENT.value[0] + OK = grpc.StatusCode.OK.value[0] + CANCELLED = grpc.StatusCode.CANCELLED.value[0] + UNKNOWN = grpc.StatusCode.UNKNOWN.value[0] + DEADLINE_EXCEEDED = grpc.StatusCode.DEADLINE_EXCEEDED.value[0] + UNAVAILABLE = grpc.StatusCode.UNAVAILABLE.value[0] + UNIMPLEMENTED = grpc.StatusCode.UNIMPLEMENTED.value[0] + ABORTED = grpc.StatusCode.ABORTED.value[0] + RESOURCE_EXHAUSTED = grpc.StatusCode.RESOURCE_EXHAUSTED.value[0] + NOT_FOUND = grpc.StatusCode.NOT_FOUND.value[0] + INTERNAL = grpc.StatusCode.INTERNAL.value[0] + FAILED_PRECONDITION = grpc.StatusCode.FAILED_PRECONDITION.value[0] + ALREADY_EXISTS = grpc.StatusCode.ALREADY_EXISTS.value[0] + + +class GrpcCommunicationInterface(AbstractCommunicationInterface): + type = GRPC + + NOT_FOUND = GrpcErrorCode.NOT_FOUND + INVALID_ARGUMENT = GrpcErrorCode.INVALID_ARGUMENT + INTERNAL = GrpcErrorCode.INTERNAL + ABORTED = GrpcErrorCode.ABORTED + RESOURCE_EXHAUSTED = GrpcErrorCode.RESOURCE_EXHAUSTED + FAILED_PRECONDITION = GrpcErrorCode.FAILED_PRECONDITION + UNAVAILABLE = GrpcErrorCode.UNAVAILABLE + ALREADY_EXISTS = GrpcErrorCode.ALREADY_EXISTS + UNKNOWN = GrpcErrorCode.UNKNOWN + + DEFAULT_EXCEPTION = _InactiveRpcError + + def get_grpc_channel(self): + """ + Creates gRPC channel with given inference input options. + Returns: + channel (Channel) + """ + if self.ssl_certificates is not None: + creds = self.ssl_certificates.get_grpc_ssl_channel_credentials() + channel = grpc.secure_channel(target=self.url, options=channel_options, credentials=creds) + else: + channel = grpc.insecure_channel(self.url, options=channel_options) + return channel + + def create_communication_service(self): + """ + Method for creating GRPC client. + """ + self.channel = self.get_grpc_channel() + self.set_grpc_stubs() + if self.model_meta_from_serving: + self.get_model_meta() + + @staticmethod + def assert_raises_exception(status, error_message_phrase, callable_obj, context=None, *args, **kwargs): + """ + Check if callable_obj returns specific exception. + Returns: + assert_raises_http_exception + """ + return assert_raises_grpc_exception( + status, error_message_phrase, callable_obj, context, *args, **kwargs + ) + + def prepare_request(self, input_objects: dict, raw=False, mediapipe_name=None, **kwargs): + raw = True if self.model.is_mediapipe else raw + request = self.get_predict_grpc_request(input_objects, raw, mediapipe_name) + if "context" in kwargs: + kwargs['context'].request = request + return {'request': request} + + @classmethod + def prepare_body_format(cls, input_objects: dict, request_format=Ovms.BINARY_IO_LAYOUT_ROW_NAME): + """ + Returns request's body dictionary as json data. + """ + data_obj = cls.prepare_body_dict(input_objects, request_format) + data_json = json.dumps(data_obj) + return data_json + + def send_predict_request(self, request: dict, timeout: int, version: int or None=None) -> dict: + """ + Sends request to server and returns the response as a dictionary. + :param dict request: + :param int timeout: + :param int or None version: + :rtype: dict + """ + result = self.send_predict_grpc_request(request['request'], timeout) + return result + + def process_predict_output(self, result, raw=False): + r""" + Example transformation: + result = {PredictResponse} outputs {\n + key: "softmax_tensor"\n + value {\n + dtype: DT_FLOAT\n tensor_shape {\n dim {\n size: 1\n }\n + dim {\n size: 1001\n }\n }\n + tensor_content: "q\25309\233 (...) + DESCRIPTOR = {MessageDescriptor} + OutputsEntry = {GeneratedProtocolMessageType} + + model_spec = {ModelSpec} + outputs = {MessageMapContainer: 1} { + 'softmax_tensor': + dtype: DT_FLOAT\n + tensor_shape {\n dim {\n size: 1\n }\n dim {\n size: 1001\n }\n}\n + tensor_content: "q\25309\233\336\ (...) + + """ + outputs = self.process_predict_grpc_output(result, raw=raw) + # check if there are unexpected values in output + if NOT_A_NUMBER_REGEX.search(str(outputs)): + raise AccuracyException(f"NaN values found in output: {str(outputs)}") + return outputs + + def get_model_meta(self, timeout=60, version=None, update_model_info=True, model_name=None): + """ + Gets information about model metadata. + """ + request = self.get_model_meta_grpc_request(model_name=model_name) + response = self.send_model_meta_grpc_request(request) + if update_model_info: + self.set_serving_inputs_outputs_grpc(response, model_name=model_name) + return response + + def get_model_status(self, timeout=60, version=None, model_name=None): + request = self.get_model_status_grpc_request(model_name=model_name, version=version) + response = self.send_model_status_grpc_request(request) + return response + + def get_metrics(self): + raise NotSupported("Metrics can be loaded only via REST") + + def prepare_stateful_request(self, input_objects: dict, sequence_ctrl=None, sequence_id=None, + ctrl_dtype=None, id_dtype=None): + return self.prepare_stateful_request_grpc(input_objects, sequence_ctrl, sequence_id, ctrl_dtype, id_dtype) + + def predict_stateful_request(self, request, timeout): + return self.predict_stateful_request_grpc(request['request'], timeout) + + def is_server_live(self): + return self.is_server_live_grpc() + + def is_server_ready(self): + return self.is_server_ready_grpc() + + def is_model_ready(self, model_name, model_version=""): + return self.is_model_ready_grpc(model_name, model_version) + + def validate_meta(self, model, meta): + return self.validate_meta_grpc(model, meta) + + def get_server_metadata(self, name=None, version=None): + return self.get_server_metadata_grpc(name, version) diff --git a/tests/functional/utils/inference/communication/rest.py b/tests/functional/utils/inference/communication/rest.py new file mode 100644 index 0000000000..9e5b64b838 --- /dev/null +++ b/tests/functional/utils/inference/communication/rest.py @@ -0,0 +1,210 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +from http import HTTPStatus + +from tests.functional.constants.ovms import Ovms +from tests.functional.utils.assertions import AccuracyException, UnexpectedResponseError, assert_raises_http_exception +from tests.functional.utils.http.base import HttpMethod +from tests.functional.utils.http.client_auth.auth import NoAuthConfigurationProvider, SslAuthConfigurationProvider +from tests.functional.utils.http.http_client_factory import HttpClientFactory +from tests.functional.utils.logger import get_logger +from tests.functional.utils.inference.communication.base import AbstractCommunicationInterface +from tests.functional.utils.inference.communication.constants import NOT_A_NUMBER_REGEX + +logger = get_logger(__name__) + +REST = 'rest' +HTTP_PROTOCOL = "http://" +HTTPS_PROTOCOL = "https://" + + +class RestCommunicationInterface(AbstractCommunicationInterface): + type = REST + + NOT_FOUND = HTTPStatus.NOT_FOUND + INVALID_ARGUMENT = HTTPStatus.BAD_REQUEST + INTERNAL = HTTPStatus.INTERNAL_SERVER_ERROR + FAILED_PRECONDITION = HTTPStatus.PRECONDITION_FAILED + UNAVAILABLE = HTTPStatus.SERVICE_UNAVAILABLE + ALREADY_EXISTS = HTTPStatus.CONFLICT + RESOURCE_EXHAUSTED = HTTPStatus.REQUEST_ENTITY_TOO_LARGE + + DEFAULT_EXCEPTION = UnexpectedResponseError + + METADATA = "metadata" + MODELS = "models" + STATUS = "status" + VERSIONS = "versions" + METRICS = "metrics" + + def __init__(self, **kwargs) -> None: + super().__init__(**kwargs) # init generic params + + def create_communication_service(self): + """ + Method for creating HTTP client. + """ + if self.ssl_certificates is None: + assert HTTPS_PROTOCOL not in self.url, \ + "Using https protocol without ssl certificates not allowed. " \ + f"Provided url invalid: {self.url}" + url = HTTP_PROTOCOL + self.url if HTTP_PROTOCOL not in self.url else self.url + client = HttpClientFactory.get(NoAuthConfigurationProvider.get(url=url, proxies={})) + else: + assert HTTP_PROTOCOL not in self.url, \ + "Using http protocol with ssl certificates not allowed. " \ + f"Provided url invalid: {self.url}" + url = HTTPS_PROTOCOL + self.url if HTTPS_PROTOCOL not in self.url else self.url + cert = self.ssl_certificates.get_https_cert() + client = HttpClientFactory.get( + SslAuthConfigurationProvider.get(url=url, cert=cert, proxies={})) + self.client = client + if self.model_meta_from_serving: + self.get_model_meta() + + def prepare_request(self, input_objects: dict, request_format=Ovms.BINARY_IO_LAYOUT_ROW_NAME, **kwargs): + data_json = self.prepare_body_format(input_objects=input_objects, request_format=request_format, + model=self.model) + if "context" in kwargs: + kwargs['context'].request = data_json + return {'request': data_json} + + @classmethod + def prepare_body_format(cls, input_objects: dict, request_format=Ovms.BINARY_IO_LAYOUT_ROW_NAME, **kwargs): + """ + Returns request's body dictionary as json data. + """ + data_obj = cls.prepare_body_dict(input_objects, request_format=request_format, **kwargs) + data_json = json.dumps(data_obj) + return data_json + + def send_predict_request(self, request, timeout, version=None): + version = self.model.version if not version else version + rest_path = self.get_rest_path(self.PREDICT, model_version=version) + + data = request if type(request) == str else request.get('request', None) + try: + headers = request.get('inference_header', None) + except AttributeError: + headers = None + result = self.client.request(HttpMethod.POST, + path=rest_path, + data=data, + headers=headers, + timeout=timeout, + raw_response=True) + return result + + def get_model_meta(self, timeout=60, version=None, update_model_info=True, model_name=None): + rest_path = self.get_rest_path(self.METADATA, model_version=version) + self.model_meta_response = self.client.request( + HttpMethod.GET, path=rest_path, timeout=timeout, raw_response=True + ) + + if update_model_info: + self.get_inputs_outputs_from_response(self.model_meta_response) + return self.model_meta_response + + def get_metrics(self, raw_response=True): + """ + Gets metrics and returns output as string. + """ + result = self.client.request(HttpMethod.GET, self.METRICS, raw_response=True) + return result.text + + def get_model_status(self, timeout=60, version=None, model_name=None): + if version is None: + version = self.model.version + response = self.get_model_status_rest(timeout, version, model_name) + return response + + def set_serving_inputs_outputs(self, response): + """ + Sets inference response inputs and outputs. + Parameters: + response (GetModelMetadataResponse): inference response + """ + signature_def = response.metadata['signature_def'] + # signature_map = get_model_metadata_pb2.SignatureDefMap() + # signature_map.ParseFromString(signature_def.value) + # serving_default = signature_map.ListFields()[0][1]['serving_default'] + # serving_inputs = serving_default.inputs + # serving_outputs = serving_default.outputs + # + # if self.input_names is None: + # self.input_names = list(serving_inputs.keys()) + # if self.output_names is None: + # self.output_names = list(serving_outputs.keys()) + # + # self.input_dims = {} + # self.input_data_types = {} + # self.output_dims = {} + # self.output_data_types = {} + # + # for input_name in self.input_names: + # serving_input = serving_inputs[input_name] + # input_tensor_shape = serving_input.tensor_shape + # self.input_dims[input_name] = [d.size for d in input_tensor_shape.dim] + # self.input_data_types[input_name] = self.get_data_type(serving_input.dtype) + # + # for output_name in self.output_names: + # serving_output = serving_outputs[output_name] + # output_tensor_shape = serving_output.tensor_shape + # self.output_dims[output_name] = [d.size for d in output_tensor_shape.dim] + # self.output_data_types[output_name] = self.get_data_type(serving_output.dtype) + + def process_predict_output(self, result, **kwargs): + # check if there are unexpected values in output (TFS - NaN/ KFS - empty values) + if NOT_A_NUMBER_REGEX.search(result.text): + raise AccuracyException(f"NaN values found in output: {result.text}") + output_json = json.loads(result.text) + outputs = self.process_json_output(output_json) + return outputs + + @staticmethod + def assert_raises_exception(status, error_message_phrase, callable_obj, context=None, *args, **kwargs): + """ + Check if callable_obj returns specific exception. + Returns: + assert_raises_http_exception + """ + return assert_raises_http_exception( + status, error_message_phrase, callable_obj, context, *args, **kwargs + ) + + def prepare_stateful_request(self, input_objects: dict, sequence_ctrl=None, sequence_id=None, + ctrl_dtype=None, id_dtype=None): + return self.prepare_stateful_request_rest(input_objects, sequence_ctrl, sequence_id, ctrl_dtype, id_dtype) + + def predict_stateful_request(self, request, timeout): + return self.predict_stateful_request_rest(request, timeout) + + def is_server_live(self): + return self.is_server_live_rest() + + def is_server_ready(self): + return self.is_server_ready_rest() + + def is_model_ready(self, model_name, model_version=""): + return self.is_model_ready_rest(model_name, model_version) + + def validate_meta(self, model, meta): + return self.validate_meta_rest(model, meta) + + def get_server_metadata(self, name=None, version=None): + return self.get_server_metadata_rest(name, version) diff --git a/tests/functional/utils/inference/inference_client_factory.py b/tests/functional/utils/inference/inference_client_factory.py new file mode 100644 index 0000000000..89b84d5449 --- /dev/null +++ b/tests/functional/utils/inference/inference_client_factory.py @@ -0,0 +1,139 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import types + +from tests.functional.utils.inference.capi import CapiServingWrapper +from tests.functional.utils.inference.communication.grpc import GRPC, GrpcCommunicationInterface +from tests.functional.utils.inference.communication.rest import REST, RestCommunicationInterface +from tests.functional.utils.inference.serving.cohere import COHERE, CohereWrapper +from tests.functional.utils.inference.serving.kf import KFS, KserveWrapper +from tests.functional.utils.inference.serving.openai import OPENAI, OpenAIWrapper +from tests.functional.utils.inference.serving.tf import TFS, TensorFlowServingWrapper +from tests.functional.utils.inference.serving.triton import TRITON, TritonServingWrapper +from tests.functional.constants.ovms_type import OvmsType +from tests.functional.object_model.ovsa import OvsaCerts + + +class InferenceClientFactory: + @staticmethod + def get_client(serving, communication, ovms_type=None): + serving_class, communication_class = [None] * 2 + if ovms_type == OvmsType.CAPI: + communication_class = CapiServingWrapper + else: + if serving == TFS: + serving_class = TensorFlowServingWrapper + elif serving == KFS: + serving_class = KserveWrapper + elif serving == TRITON: + serving_class = TritonServingWrapper + elif serving == OPENAI: + serving_class = OpenAIWrapper + elif serving == COHERE: + serving_class = CohereWrapper + else: + raise Exception + + if serving in [TFS, KFS, TRITON, OPENAI, COHERE]: + if communication == REST: + communication_class = RestCommunicationInterface + elif communication == GRPC: + communication_class = GrpcCommunicationInterface + else: + raise Exception + + # pylint: disable=too-many-arguments + def common_inference_client_init(self, + model=None, + model_name=None, + model_version=None, + inputs: dict = None, + input_names: list = None, + input_data_types: dict = None, + outputs: dict = None, + output_names: list = None, + output_data_types: dict = None, + model_meta_from_serving: bool = None, + is_mediapipe: bool = None, + ssl_certificates: object = OvsaCerts.default_certs, + context=None, + **kwargs): + """ + Aggregated __init__ method that will properly initialize any pair of communication & serving classes. + """ + + self.context = context + + # Copy simple parameters + if model: + self.model = model + if model_meta_from_serving is None: + # If we got properly filled model description, do not fetch it via get_model_meta + model_meta_from_serving = False + else: + # use generic object as storage for parameters (version, name, inputs, outputs, etc.) + # this step is required for @property methods. + self.model = types.new_class("model_placeholder")() + + self.model.name = model_name + self.model.version = model_version + self.model.inputs = inputs if inputs else {} + self.model.outputs = outputs if outputs else {} + self.model.input_names = input_names if input_names else [] + self.model.output_names = output_names if output_names else [] + self.model.input_data_types = input_data_types if input_data_types else {} + self.model.output_data_types = output_data_types if output_data_types else {} + self.model.input_dims = {} + self.model.output_dims = {} + if is_mediapipe or (getattr(self.model, "name", None) is not None and "mediapipe" in self.model.name): + self.model.is_mediapipe = True + else: + self.model.is_mediapipe = False + + # Init each parent class separate + if serving_class: + serving_class.__init__(self, model_meta_from_serving=model_meta_from_serving, **kwargs) + if communication_class: + if communication_class == CapiServingWrapper and kwargs.get("ovms_capi_instance", None) is None: + # Little trick to easily pass OvmsContext + kwargs["ovms_capi_instance"] = kwargs["port"] + del kwargs["port"] # `port` is applicable only for GRPC/REST communication + communication_class.__init__(self, ssl_certificates=ssl_certificates, **kwargs) + else: + self.type = communication + + # Initialize inference engine: + self.create_inference() + if ovms_type: + name = f"{ovms_type.title()}InferenceClient" + bases = (communication_class,) + else: + name = f"{serving.title()}{communication.title()}InferenceClient" + bases = (serving_class,) + if communication_class: + bases += (communication_class,) + + inference_class_type = type( + name, + bases, + { + "__init__": common_inference_client_init, + } + ) + inference_class_type.serving = serving + inference_class_type.communication = communication if communication else ovms_type + return inference_class_type diff --git a/tests/functional/utils/inference/serving/__init__.py b/tests/functional/utils/inference/serving/__init__.py new file mode 100644 index 0000000000..68183727a5 --- /dev/null +++ b/tests/functional/utils/inference/serving/__init__.py @@ -0,0 +1,21 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.utils.inference.serving.kf import KFS +from tests.functional.utils.inference.serving.openai import OPENAI +from tests.functional.utils.inference.serving.tf import TFS +from tests.functional.utils.inference.serving.triton import TRITON +from tests.functional.utils.inference.serving.cohere import COHERE diff --git a/tests/functional/utils/inference/serving/base.py b/tests/functional/utils/inference/serving/base.py new file mode 100644 index 0000000000..80a4583fdb --- /dev/null +++ b/tests/functional/utils/inference/serving/base.py @@ -0,0 +1,183 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import abc + +import numpy as np + + +class AbstractServingWrapper(metaclass=abc.ABCMeta): + def __init__(self, model_meta_from_serving, **kwargs): + # During inference client init fetch model description via get_model_meta call. + self.model_meta_from_serving = model_meta_from_serving + + @abc.abstractmethod + def set_grpc_stubs(self): + """ + Assigns objects for inference purposes. + """ + pass + + @abc.abstractmethod + def create_inference(self): + """ + Assigns objects for inference purposes. + """ + pass + + @abc.abstractmethod + def predict(self, request): + pass + + @abc.abstractmethod + def get_rest_path(self, operation, model_version=None, model_name=None): + """ + REST path construction is dependent from serving used: (Tensorflow / KServe) + """ + pass + + @abc.abstractmethod + def get_inputs_outputs_from_response(self, response): + pass + + @abc.abstractmethod + def get_model_meta_grpc_request(self, model_name=None): + pass + + @abc.abstractmethod + def get_predict_grpc_request(self): + pass + + def get_and_validate_meta(self, model, model_name=None): + """ + Validates and returns model metadata. + Parameters: + model (ModelInfo): model class object + Returns: + meta (ModelMetadataResponse): model metadata + """ + meta = self.get_model_meta(version=model.version, model_name=model_name) + self.validate_meta(model, meta) + return meta + + def get_and_validate_metadata(self, model, model_name=None): + return self.get_and_validate_meta(model, model_name=model_name) + + @property + def model_name(self): + return self.model.name + + @property + def model_version(self): + return None if not self.model.version else str(self.model.version) + + def set_inputs(self, inputs): + self.model.inputs = inputs + + def set_outputs(self, outputs): + self.model.outputs = outputs + + @property + def inputs(self): + return self.model.inputs + + @property + def outputs(self): + return self.model.outputs + + @property + def input_names(self): + return list(self.model.input_names or self.model.inputs.keys()) + + @property + def output_names(self): + return list(self.model.output_names or self.model.outputs.keys()) + + @property + def input_dims(self): + return {k: v['shape'] for k, v in self.model.inputs.items()} + + @property + def output_dims(self): + return {k: v['shape'] for k, v in self.model.outputs.items()} + + @property + def input_data_types(self): + return {k: v['dtype'] for k, v in self.model.inputs.items()} + + @property + def output_data_types(self): + return {k: v['dtype'] for k, v in self.model.outputs.items()} + + def create_client_data(self, inference_request): + if inference_request is not None and inference_request.dataset: + input_data = inference_request.load_data() + else: + input_data = self.model.prepare_input_data(inference_request.batch_size) + return input_data + + def prepare_and_predict_stateful_request(self, input_objects: dict, sequence_ctrl=None, + sequence_id=None, timeout=900): + """ + Prepares and predicts stateful request. + Parameters: + input_objects (dict): model inputs + sequence_ctrl (int): inference sequence control + sequence_id (int): inference sequence id + timeout (int): timeout + Returns: + result (Predict): prediction result + """ + request = self.prepare_stateful_request(input_objects, sequence_ctrl, sequence_id) + result = self.predict_stateful_request(request, timeout) + return result + + def get_and_validate_status(self, model): + """ + Validates and returns model status. + Parameters: + model (ModelInfo): model class object + Returns: + status (GetModelStatusResponse): model status + + """ + status = self.get_model_status() + self.validate_status(model, status) + return status + + @staticmethod + def validate_v2_model_name_version(name, version, model): + assert name == model.name, f"Expected {model.name} model_name; Actual: {name}" + if version is not None: + assert version == str(model.version), f"Expected {model.version} model_version; Actual: {version}" + + def cast_type_to_string(self, data_type): + # https://github.com/openvinotoolkit/model_server/blob/main/src/kfs_frontend/kfs_utils.cpp + if data_type == np.float32: + result = 'FP32' + elif data_type == np.int32: + result = 'INT32' + elif data_type == np.int64: + result = 'INT64' + elif data_type == str: + result = 'BYTES' + elif data_type == np.uint8: + result = 'UINT8' + elif data_type == np.object_: + result = 'BYTES' + else: + raise NotImplementedError() + return result diff --git a/tests/functional/utils/inference/serving/cohere.py b/tests/functional/utils/inference/serving/cohere.py new file mode 100644 index 0000000000..add171de6b --- /dev/null +++ b/tests/functional/utils/inference/serving/cohere.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=unused-argument + + +from dataclasses import dataclass +from typing import Union + +from tests.functional.utils.inference.serving.common import LLMCommonWrapper +from tests.functional.utils.logger import get_logger + +logger = get_logger(__name__) + +COHERE = "COHERE" + + +class CohereWrapper(LLMCommonWrapper): + API_KEY_NOT_USED = "not_used" + RERANK = "rerank" + PREDICT = RERANK + + @staticmethod + def prepare_body_dict(input_objects: dict, request_format=None, **kwargs): + model = kwargs.get("model", None) + assert model is not None, "No model provided" + model_name = model.name + + endpoint = kwargs.get("endpoint", CohereWrapper.RERANK) + body_dict = {} + if endpoint == CohereWrapper.RERANK: + documents = [f'"{doc}"' for doc in input_objects["input0"]["documents"]] + body_dict = { + "model": model_name, + "query": input_objects["input0"]["query"], + "documents": documents + } + else: + raise NotImplementedError(f"Invalid endpoint: {endpoint}") + return LLMCommonWrapper.prepare_body_dict_from_request_params(CohereRequestParams, body_dict, **kwargs) + + def get_model_meta_grpc_request(self, model_name=None): + raise NotImplementedError + + def get_predict_grpc_request(self): + raise NotImplementedError + + def set_grpc_stubs(self): + raise NotImplementedError + + +class RerankApi: + + @staticmethod + def prepare_rerank_input_content(input_objects): + for input_name in input_objects: + return input_objects[input_name] + + +@dataclass +class CohereRequestParams: + + def prepare_dict(self, **kwargs): + request_params_dict = {key: value for key, value in vars(self).items() if value is not None} + return request_params_dict + +class CohereRerankRequestParams(CohereRequestParams): + documents: Union[str, list] = None + top_n: int = None + return_documents: bool = None + + def set_default_values(self): + self.top_n = 1000 + self.return_documents = False + + +@dataclass +class OvmsRerankRequestParams(CohereRerankRequestParams): + pass diff --git a/tests/functional/utils/inference/serving/common.py b/tests/functional/utils/inference/serving/common.py new file mode 100644 index 0000000000..834de63d06 --- /dev/null +++ b/tests/functional/utils/inference/serving/common.py @@ -0,0 +1,81 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=unused-argument +# pylint: disable=attribute-defined-outside-init +# pylint: disable=no-member + +import json + +from tests.functional.utils.inference.serving.base import AbstractServingWrapper +from tests.functional.utils.logger import get_logger + +logger = get_logger(__name__) + + +class LLMCommonWrapper(AbstractServingWrapper): + REST_VERSION = "v3" + + def create_base_url(self, rest_version=None): + rest_version = rest_version if rest_version is not None else self.REST_VERSION + self.base_url = f"http://{self.url}/{rest_version}" + + def set_grpc_stubs(self): + raise NotImplementedError + + def create_inference(self): + self.communication_service = self.create_communication_service() + return self.communication_service + + def predict(self, request, timeout=300, raw=False): + result = self.send_predict_request(request, timeout) + return result + + def get_rest_path(self, operation, model_version=None, model_name=None): + """ + Expect 1 REST path formats for OpenAI format: + - POST: (CHAT COMPLETIONS) + http://{REST_URL}:{REST_PORT}/v3/chat/completions + """ + rest_path = [self.REST_VERSION, operation] + rest_path = "/".join(rest_path) + return rest_path + + def get_inputs_outputs_from_response(self, response): + pass + + def get_content_from_response(self, response): + model_specification = json.loads(response.text) + self.model.content = model_specification["choices"]["message"]["content"] + + def get_model_meta_grpc_request(self, model_name=None): + # GRPC not supported + raise NotImplementedError + + def get_predict_grpc_request(self): + # GRPC not supported + raise NotImplementedError + + @staticmethod + def prepare_body_dict_from_request_params(request_params_type, body_dict, **kwargs): + request_parameters = kwargs.get("request_parameters", None) + if request_parameters is not None: + assert isinstance(request_parameters, request_params_type), \ + f"Wrong type of request_parameters expected: {request_params_type} " \ + f"actual: {type(request_parameters)}" + body_dict.update(request_parameters.prepare_dict(use_extra_body=False)) + + return body_dict diff --git a/tests/functional/utils/inference/serving/kf.py b/tests/functional/utils/inference/serving/kf.py new file mode 100644 index 0000000000..8d67349e57 --- /dev/null +++ b/tests/functional/utils/inference/serving/kf.py @@ -0,0 +1,632 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import enum +import json +from http import HTTPStatus + +import grpc +import numpy as np +from google.protobuf.json_format import MessageToJson +from tritonclient.grpc import service_pb2, service_pb2_grpc +from tritonclient.utils import np_to_triton_dtype, triton_to_np_dtype + +from tests.functional.utils.assertions import StreamingApiException +from tests.functional.utils.http.base import HttpMethod +from tests.functional.utils.inference.communication.grpc import GRPC_TIMEOUT +from tests.functional.utils.inference.serving.base import AbstractServingWrapper +from tests.functional.utils.logger import get_logger +from tests.functional.utils.test_framework import FrameworkMessages, skip_if_runtime +from tests.functional.constants.metrics import Metric +from tests.functional.constants.ovms import Ovms + +logger = get_logger(__name__) + +KFS = "KFS" + + +class DataType(enum.Enum): + """ + A set of KServe data types bound to auto() values. + """ + INVALID = enum.auto() + BOOL = enum.auto() + UINT8 = enum.auto() + UINT16 = enum.auto() + UINT32 = enum.auto() + UINT64 = enum.auto() + INT8 = enum.auto() + INT16 = enum.auto() + INT32 = enum.auto() + INT64 = enum.auto() + FP16 = enum.auto() + FP32 = enum.auto() + FP64 = enum.auto() + STRING = enum.auto() + BYTES = enum.auto() + + +class KserveWrapper(AbstractServingWrapper): + REST_VERSION = "v2" + PREDICT = "infer" + MODEL_READY = "ready" + + METRICS_PROTOCOL = Metric.KServe + + def __init__(self, model_meta_from_serving: bool = True, **kwargs): + super().__init__(model_meta_from_serving=model_meta_from_serving, **kwargs) + self.model_service_stub = None + + def set_grpc_stubs(self): + """ + Assigns objects for inference purposes. + """ + self.model_service_stub = service_pb2_grpc.GRPCInferenceServiceStub(self.channel) + + def create_inference(self): + """ + Assigns objects for inference purposes. + """ + self.create_communication_service() + + if self.model_meta_from_serving: + self.get_model_meta() + + def send_predict_grpc_request(self, request, timeout=GRPC_TIMEOUT): + return self.model_service_stub.ModelInfer(request) + + @staticmethod + def process_predict_grpc_output(response, raw=False): + outputs = {} + for idx, output in enumerate(response.outputs): + np_array = np.frombuffer(response.raw_output_contents[idx], dtype=triton_to_np_dtype(output.datatype)) + outputs[output.name] = np_array.reshape(output.shape) + return outputs + + def predict(self, request, timeout=60, raw=False): + result = self.send_predict_request(request, timeout) + outputs = self.process_predict_output(result, raw=raw) + return outputs + + def predict_stream(self, streaming_generator, inference_requests): + output_stream = self.model_service_stub.ModelStreamInfer(streaming_generator.main_loop(inference_requests)) + return output_stream + + def get_next_response_from_stream(self, output_stream): + infer_responses_dict = {} + for _ in self.outputs: + try: + response = next(output_stream, None) + except grpc._channel._MultiThreadedRendezvous as exc: + ovms_log = None + if self.context and self.context.ovms_sessions: + ovms = self.context.ovms_sessions[0].ovms + ovms_log = ovms.get_logs_as_txt() + raise StreamingApiException(exc, ovms_log=ovms_log) from exc + if response is None: + logger.warning("Attempt of getting response from empty stream.") + else: + if not len(response.error_message) == 0: + raise StreamingApiException(f"Error message not empty: {response.error_message}") + infer_responses_dict.update(self.process_predict_output(response.infer_response)) + return infer_responses_dict, response + + def get_model_meta_grpc_request(self, model_name=None): + version = str(self.model_version) if self.model_version else None + model_name = model_name if model_name is not None else self.model.name + return service_pb2.ModelMetadataRequest(name=model_name, version=version) + + def get_predict_grpc_request(self, input_objects, raw=False, mediapipe_name=None): + request = service_pb2.ModelInferRequest() + request.model_name = self.model_name if mediapipe_name is None else mediapipe_name + if self.model_version: + request.model_version = self.model_version + + request = self.prepare_v2_grpc_input_tensor(request, input_data=input_objects, raw_input_contents=raw) + return request + + def send_model_meta_grpc_request(self, request): + metadata = None + response = self.model_service_stub.ModelMetadata(request=request, metadata=metadata) + return response + + def get_rest_path(self, operation=None, model_name=None, model_version=None): + """ + Expect 2 REST path formats for KServe format: + - GET: (METADATA) + URL: http://{REST_URL}:{REST_PORT}/v2/models/{model_name} + - POST: (PREDICT) + URL: http://{REST_URL}:{REST_PORT}/v2/models/{model_name}/infer + + """ + model_name = model_name if model_name else self.model.name + assert model_name + rest_path = [self.REST_VERSION, self.MODELS, model_name] + if model_version is not None: + rest_path.append(self.VERSIONS) + rest_path.append(str(model_version)) + if operation not in [self.METADATA, None]: + rest_path.append(operation) + + rest_path = "/".join(rest_path) + return rest_path + + def get_inputs_outputs_from_response(self, response): + model_specification = json.loads(response.text) + + self.model.inputs = {} + self.model.outputs = {} + + for _input in model_specification['inputs']: + self.model.inputs[_input['name']] = { + 'shape': _input['shape'], + 'dtype': triton_to_np_dtype(_input['datatype']) + } + + for output in model_specification['outputs']: + self.model.outputs[output['name']] = { + 'shape': output['shape'], + 'dtype': triton_to_np_dtype(output['datatype']) + } + + @staticmethod + def prepare_body_dict(input_objects: dict, request_format=Ovms.BINARY_IO_LAYOUT_ROW_NAME, **kwargs): + """ + example body dict: + { + 'request': b'{ + "inputs": [{ + "name": "input_name", + "shape": [], + "datatype": "BYTES", + "parameters": {"binary_data_size": "9025"} + }] + }, + 'inference_header': { + 'inputs': [{ + 'name': 'input_name', + 'shape': [1], + 'datatype': 'BYTES', + 'parameters': {'binary_data_size': '9025'} + }] + }, + 'inference_header_binary': b'{ + "inputs": [{ + "name": "input_name", + "shape": [1], + "datatype": "BYTES", + "parameters": {"binary_data_size": "9025"} + }] + }' + } + """ + inputs = [] + for input_name, input_data in input_objects.items(): + if input_data.dtype == np.object_: + _data = [x.decode() for x in input_data] + elif input_data.shape: + _data = input_data.tolist() + else: + _data = [input_data.tolist()] + + inputs.append({ + "name": input_name, + "shape": list(input_data.shape), + "datatype": "BYTES" if str(input_data.dtype) == " Tuple[str, str]: + for input_name in input_objects: + return input_objects[input_name][0] + + +class OpenAIRequestParams: + + def prepare_dict(self, set_null_values=False, **kwargs): + if set_null_values: + request_params_dict = {key: value for key, value in vars(self).items()} + else: + request_params_dict = {key: value for key, value in vars(self).items() if value is not None} + return request_params_dict + + +@dataclass +class OpenAICommonCompletionsRequestParams(OpenAIRequestParams): + stream: bool = None + stream_options: dict = None + max_tokens: int = None + n: int = None + temperature: float = None + top_p: float = None + frequency_penalty: float = None + presence_penalty: float = None + seed: int = None + stop: Union[str, list] = None + + def set_default_values(self, **kwargs): + self.stream = kwargs.get("stream", False) + self.n = 1 + self.temperature = 1.0 + self.top_p = 1.0 + self.seed = 0 + self.stop = "," + + +@dataclass +class OpenAIChatCompletionsRequestParams(OpenAICommonCompletionsRequestParams): + logprobs: bool = None + response_format: dict = None + tools: list = None + tool_choice: Union[dict, str] = None + response_format: BaseModel = None + + def set_default_values(self, **kwargs): + super().set_default_values(**kwargs) + if not self.stream: + self.logprobs = False + + +@dataclass +class OpenAICompletionsRequestParams(OpenAICommonCompletionsRequestParams): + best_of: int = None + logprobs: int = None + + def set_default_values(self, **kwargs): + super().set_default_values(**kwargs) + if not self.stream: + self.best_of = 1 + self.logprobs = 1 + + +@dataclass +class OpenAICommonImagesRequestParams(OpenAIRequestParams): + size: str = None + n: int = None + response_format: str = None + + def set_default_values(self, **kwargs): + self.size = "512x512" + self.n = 1 + self.response_format = "b64_json" + + +@dataclass +class OpenAIImagesGenerationsRequestParams(OpenAICommonImagesRequestParams): + pass + + +@dataclass +class OpenAIImagesEditsRequestParams(OpenAICommonImagesRequestParams): + pass + + +class OpenAIFinishReason: + LENGTH = "length" + STOP = "stop" + TOOL_CALLS = "tool_calls" + MAX_TOKENS = "max_tokens" + + +@dataclass +class OpenAIResponsesRequestParams(OpenAIRequestParams): + stream: bool = None + max_output_tokens: int = None + temperature: float = None + top_p: float = None + tools: list = None + tool_choice: Union[dict, str] = None + + def set_default_values(self, **kwargs): + self.stream = kwargs.get("stream", False) + self.temperature = 1.0 + self.top_p = 1.0 + + +@dataclass +class OpenAIEmbeddingsRequestParams(OpenAIRequestParams): + encoding_format: str = None + + def set_default_values(self): + self.encoding_format = "float" + + +@dataclass +class OpenAIAudioSpeechRequestParams(OpenAIRequestParams): + + def set_default_values(self, **kwargs): + # No defaults needed for TTS; kept for interface compatibility + pass + + +@dataclass +class OpenAIAudioTranscriptionsRequestParams(OpenAIRequestParams): + language: str = None + temperature: float = None + timestamp_granularities: list = None + + def set_default_values(self, **kwargs): + self.language = "en" + self.temperature = 0.0 + self.timestamp_granularities = ["segment"] + + +@dataclass +class OpenAIAudioTranslationsRequestParams(OpenAIRequestParams): + temperature: float = None + + def set_default_values(self, **kwargs): + self.temperature = 0.0 diff --git a/tests/functional/utils/inference/serving/tf.py b/tests/functional/utils/inference/serving/tf.py new file mode 100644 index 0000000000..b2c65d1268 --- /dev/null +++ b/tests/functional/utils/inference/serving/tf.py @@ -0,0 +1,473 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json + +import numpy as np +import tensorflow +from google.protobuf.json_format import MessageToJson, Parse +from tensorboard.util.tensor_util import make_ndarray +from tensorflow import make_tensor_proto +from tensorflow.core.framework import types_pb2 +from tensorflow_serving.apis import ( + get_model_metadata_pb2, get_model_status_pb2, model_service_pb2_grpc, predict_pb2, prediction_service_pb2_grpc,) + +from tests.functional.utils.assertions import InvalidMetadataException, NotSupported +from tests.functional.utils.http.base import HttpMethod +from tests.functional.utils.inference.communication.grpc import GRPC_TIMEOUT +from tests.functional.utils.inference.serving.base import AbstractServingWrapper +from tests.functional.utils.logger import get_logger +from tests.functional.constants.metrics import Metric +from tests.functional.constants.ovms import Ovms + +logger = get_logger(__name__) + +TFS = "TFS" + + +class TensorFlowServingWrapper(AbstractServingWrapper): + REST_VERSION = "v1" + PREDICT = ":predict" + + METRICS_PROTOCOL = Metric.TensorFlowServing + + + def set_grpc_stubs(self): + """ + Assigns objects for inference purposes. + """ + self.predict_stub = prediction_service_pb2_grpc.PredictionServiceStub(self.channel) + self.model_service_stub = model_service_pb2_grpc.ModelServiceStub(self.channel) + + def get_model_status_grpc_request(self, model_name=None, version=None): + request = get_model_status_pb2.GetModelStatusRequest() + request.model_spec.name = self.model_name if model_name is None else model_name + if version is not None: + request.model_spec.version.value = int(version) + + if self.model_version is not None: + request.model_spec.version.value = int(self.model_version) + return request + + def get_model_meta_grpc_request(self, model_name=None): + metadata_field = "signature_def" + request = get_model_metadata_pb2.GetModelMetadataRequest() + request.model_spec.name = self.model_name + if self.model_version is not None: + request.model_spec.version.value = int(self.model_version) + request.metadata_field.append(metadata_field) + return request + + def send_model_status_grpc_request(self, request): + response = self.model_service_stub.GetModelStatus( + request, wait_for_ready=True, timeout=GRPC_TIMEOUT + ) + return response + + def send_model_meta_grpc_request(self, request): + response = self.predict_stub.GetModelMetadata( + request, wait_for_ready=True, timeout=GRPC_TIMEOUT + ) + return response + + def get_predict_grpc_request(self, input_objects, raw=False, mediapipe_name=None): + request = predict_pb2.PredictRequest() + request.model_spec.name = self.model_name + if self.model_version is not None: + request.model_spec.version.value = int(self.model_version) + for input_name, input_object in input_objects.items(): + request.inputs[input_name].CopyFrom( + make_tensor_proto(input_object, shape=input_object.shape) + ) + return request + + @staticmethod + def process_predict_grpc_output(result, **kwargs): + outputs = { + output_name: make_ndarray(output) for output_name, output in result.outputs.items() + } + return outputs + + def create_inference(self): + """ + Assigns objects for inference purposes. + """ + # method from brother class (multiple inheritance) + self.communication_service = self.create_communication_service() + return self.communication_service + + def send_predict_grpc_request(self, request, timeout=GRPC_TIMEOUT): + return self.predict_stub.Predict(request, wait_for_ready=True, timeout=timeout) + + def predict(self, request, timeout=60, raw=False): + result = self.send_predict_request(request, timeout) + outputs = self.process_predict_output(result) + return outputs + + def get_rest_path(self, operation=None, model_version=None, model_name=None): + """ + Expect 2 REST path formats for TF format: + - GET: (METADATA, MODELS) + http://{REST_URL}:{REST_PORT}/v1/models/{MODEL_NAME}/versions/{MODEL_VERSION}/{OPERATION} + - POST: (PREDICT) + http://{REST_URL}:{REST_PORT}/v1/models/{MODEL_NAME}/versions/{MODEL_VERSION}:predict + """ + model_name = model_name if model_name is not None else (self.model.name if self.model else self.model_name) + assert model_name + rest_path = [self.REST_VERSION, self.MODELS, model_name] + if model_version is not None: + rest_path.append(self.VERSIONS) + rest_path.append(str(model_version)) + if operation == self.PREDICT: + rest_path[-1] = "".join([rest_path[-1], operation]) + elif operation not in [self.STATUS, None]: + rest_path.append(operation) + rest_path = "/".join(rest_path) + return rest_path + + + @staticmethod + def prepare_body_dict(input_objects: dict, request_format=Ovms.BINARY_IO_LAYOUT_ROW_NAME, **kwargs): + """ + Prepare HTTP request's body in given format: + - row_name, + - column_name, + - row_noname, + - column_noname + """ + signature = "serving_default" + if request_format == Ovms.BINARY_IO_LAYOUT_ROW_NAME: + instances = [] + for input_name, input_object in input_objects.items(): + if input_object.shape: + for i in range(0, input_object.shape[0], 1): + input_data = input_object[i].decode() if ( + input_object.dtype == np.object_) else input_object[i].tolist() + instances.append({input_name: input_data}) + else: + instances.append({input_name: str(input_object[()])}) + # https://numpy.org/doc/stable/reference/arrays.scalars.html#indexing + data_obj = {"signature_name": signature, "instances": instances} + elif request_format == Ovms.BINARY_IO_LAYOUT_ROW_NONAME: + instances = [] + for input_object in input_objects.values(): + if input_object.shape: + for i in range(0, input_object.shape[0]): + input_data = input_object[i].decode() if ( + input_object.dtype == np.object_) else input_object[i].tolist() + instances.append(input_data) + else: + # https://numpy.org/doc/stable/reference/arrays.scalars.html#indexing + instances.append([str(input_object[()])]) + data_obj = {"signature_name": signature, 'instances': instances} + elif request_format == Ovms.BINARY_IO_LAYOUT_COLUMN_NAME: + inputs = {} + for input_name, input_object in input_objects.items(): + inputs[input_name] = [x.decode() for x in input_object.tolist()] if input_object.dtype == np.object_ \ + else input_object.tolist() + data_obj = {"signature_name": signature, 'inputs': inputs} + elif request_format == Ovms.BINARY_IO_LAYOUT_COLUMN_NONAME: + assert len(input_objects) == 1, \ + f"Only single input is required if {Ovms.BINARY_IO_LAYOUT_COLUMN_NONAME} format is used" + input_object = list(input_objects.items())[0][1] + _input = [x.decode() for x in input_object.tolist()] if input_object.dtype == np.object_ \ + else input_object.tolist() + data_obj = {"signature_name": signature, 'inputs': _input} + else: + raise ValueError(f"Unknown response format: {request_format}") + return data_obj + + def get_inputs_outputs_from_response(self, response): + # expect content to be dictionary encoded as bytes: + # model.content == b'{\n "modelSpec": {\n "name": "resnet-50-tf",\n "signatureName": "" ... + model_specification = json.loads(response.text) + + serving_default = model_specification['metadata']['signature_def']['signatureDef']['serving_default'] + + self.model.inputs = {} + self.model.outputs = {} + + for name, details in serving_default['inputs'].items(): + self.model.inputs[details['name']] = { + 'shape': [int(x['size']) for x in details["tensorShape"]["dim"]], + 'dtype': tensorflow.dtypes.as_dtype(getattr(types_pb2, details['dtype'])) + } + + for name, details in serving_default['outputs'].items(): + self.model.outputs[details['name']] = { + 'shape': [int(x['size']) for x in details["tensorShape"]["dim"]], + 'dtype': tensorflow.dtypes.as_dtype(getattr(types_pb2, details['dtype'])) + } + + def process_json_output(self, result_dict): + """ + Converts predict result to output as a numpy array. + Input: + result_dict = {'predictions': []} + Output: + = {ndarray: (1, 1001)} + """ + output = {} + if "outputs" in result_dict: + key_name = "outputs" + if isinstance(result_dict[key_name], dict): + for output_tensor in self.output_names: + output[output_tensor] = np.asarray(result_dict[key_name][output_tensor]) + else: + output[self.output_names[0]] = np.asarray(result_dict[key_name]) + elif "predictions" in result_dict: + key_name = "predictions" + if isinstance(result_dict[key_name][0], dict): + for row in result_dict[key_name]: + for output_tensor in self.output_names: + if output_tensor not in output: + output[output_tensor] = [] + output[output_tensor].append(row[output_tensor]) + for output_tensor in self.output_names: + output[output_tensor] = np.asarray(output[output_tensor]) + else: + output[self.output_names[0]] = np.asarray(result_dict[key_name]) + else: + logger.error(f"Missing required response in {result_dict}") + return output + + def set_serving_inputs_outputs_grpc(self, response, **kwargs): + """ + Sets inference response inputs and outputs. + Parameters: + response (GetModelMetadataResponse): inference response + """ + signature_def = response.metadata['signature_def'] + signature_map = get_model_metadata_pb2.SignatureDefMap() + signature_map.ParseFromString(signature_def.value) + serving_default = signature_map.ListFields()[0][1]['serving_default'] + + inputs = {} + outputs = {} + + for name, details in serving_default.inputs.items(): + inputs[name] = { + 'shape': [x.size for x in details.tensor_shape.dim], + 'dtype': tensorflow.dtypes.as_dtype(details.dtype) + } + + for name, details in serving_default.outputs.items(): + outputs[name] = { + 'shape': [x.size for x in details.tensor_shape.dim], + 'dtype': tensorflow.dtypes.as_dtype(details.dtype) + } + + self.set_inputs(inputs) + self.set_outputs(outputs) + + + @staticmethod + def get_data_type(data_type): + """ + Converts given data_type to numpy format. + Parameters: + data_type (int) + Returns: + result (np) + """ + result = None + if data_type == 6: + result = np.int8 + elif data_type == 3: + result = np.int32 + elif data_type == 9: + result = np.int64 + elif data_type == 1: + result = np.float32 + else: + raise NotImplementedError() + return result + + def get_model_status_rest(self, timeout=60, version=None, model_name=None): + rest_path = self.get_rest_path(None, model_version=version, model_name=model_name) + response = self.client.request(HttpMethod.GET, path=rest_path, timeout=timeout, raw_response=True) + + # Transform JSON friendly output to protobuf compatible object required by callers. + status_pb = get_model_status_pb2.GetModelStatusResponse() + response = Parse(response.text, status_pb, ignore_unknown_fields=False) + + return response + + def prepare_stateful_request_rest(self, input_objects: dict, sequence_ctrl=None, sequence_id=None, + ctrl_dtype=None, id_dtype=None): + data_obj = self.prepare_body_dict( + input_objects, request_format=Ovms.BINARY_IO_LAYOUT_COLUMN_NAME + ) + if sequence_ctrl is not None: + data_obj['inputs']['sequence_control_input'] = [sequence_ctrl] + if sequence_id is not None: + data_obj['inputs']['sequence_id'] = [sequence_id] + return {'request': json.dumps(data_obj)} + + def predict_stateful_request_rest(self, request, timeout=900): + result = self.send_predict_request(request, timeout) + + output_json = json.loads(result.text) + sequence_id = output_json['outputs'].pop('sequence_id')[0] \ + if 'outputs' in output_json else None + outputs = self.process_json_output(output_json) + return sequence_id, outputs + + def prepare_stateful_request_grpc(self, input_objects: dict, sequence_ctrl=None, sequence_id=None, + ctrl_dtype=None, id_dtype=None): + sequence_id_dtype = id_dtype if id_dtype else 'uint64' + sequence_ctrl_dtype = ctrl_dtype if ctrl_dtype else 'uint32' + request = self.prepare_request(input_objects) + if sequence_ctrl is not None: + request['request'].inputs['sequence_control_input'].CopyFrom( + make_tensor_proto([sequence_ctrl], dtype=sequence_ctrl_dtype) + ) + if sequence_id is not None: + request['request'].inputs['sequence_id'].CopyFrom( + make_tensor_proto([sequence_id], dtype=sequence_id_dtype) + ) + return request + + def predict_stateful_request_grpc(self, request, timeout=900): + result = self.predict_stub.Predict( + request, wait_for_ready=True, timeout=timeout + ) + data = {} + sequence_id = result.outputs.pop('sequence_id').uint64_val[0] + for output_name, output in result.outputs.items(): + data[output_name] = make_ndarray(output) + return sequence_id, data + + + # KFS not supported API calls: + def is_server_live_grpc(self): + raise NotSupported("is_server_live is not available in TFS") + + def is_server_live_rest(self): + raise NotSupported("is_server_live is not available in TFS") + + def is_server_ready_grpc(self): + raise NotSupported("is_server_live is not available in TFS") + + def is_server_ready_rest(self): + raise NotSupported("is_server_live is not available in TFS") + + def is_model_ready_grpc(self, model_name, model_version=""): + """ + Gets information about model readiness (specific only for KFS - gRPC or REST). + GET http://${REST_URL}:${REST_PORT}/v2/models/${MODEL_NAME}[/versions/${MODEL_VERSION}]/ready + Response: True (ready) or False (not ready) + """ + raise NotSupported() + + def is_model_ready_rest(self, model_name, model_version=""): + """ + Gets information about model readiness (specific only for KFS - gRPC or REST). + GET http://${REST_URL}:${REST_PORT}/v2/models/${MODEL_NAME}[/versions/${MODEL_VERSION}]/ready + Response: True (ready) or False (not ready) + """ + raise NotSupported() + + def cast_type_to_string(self, data_type): + # https://github.com/openvinotoolkit/model_server/blob/main/src/tfs_frontend/tfs_utils.cpp + if data_type == np.float32: + result = 'DT_FLOAT' + elif data_type == np.int32: + result = 'DT_INT32' + elif data_type == np.int64: + result = 'DT_INT64' + elif data_type == str: + result = 'DT_STRING' + elif data_type == np.uint8: + result = 'DT_UINT8' + else: + raise NotImplementedError() + return result + + def validate_meta_grpc(self, model, meta): + """ + Validates model metadata. + Parameters: + model (ModelInfo): model class object + meta (ModelMetadataResponse): model metadata + """ + json_data = json.loads(MessageToJson(meta)) + + assert meta.model_spec.name == model.name, \ + f"Unexpected model name (expected: {model.name}, " \ + f"detected: {meta.model_spec.name})" + assert meta.model_spec.version.value == model.version + assert "signature_def" in meta.metadata + assert meta.metadata['signature_def'].type_url == \ + 'type.googleapis.com/tensorflow.serving.SignatureDefMap' + def validate(test_data, val_shapes, val_types): + assert len(test_data) == len(val_shapes), \ + f"Unexpected argument list (shapes; expect: {len(val_shapes)}, " \ + f"detect: {len(test_data)})" + assert len(test_data) == len(val_types), \ + f"Unexpected argument list (shapes; expect: {len(val_types)}, " \ + f"detect: {len(test_data)})" + for arg_name, arg_data in test_data.items(): + for test, val_dim in zip(arg_data['tensorShape']['dim'], val_shapes[arg_name]): + if int(test['size']) != val_dim: + raise InvalidMetadataException( + f"Unexpected shape (expected: {val_shapes[arg_name]}, " \ + f"detected: {arg_data['tensorShape']['dim']})") + val_type = self.cast_type_to_string(val_types[arg_name]) + assert arg_data['dtype'] == val_type, \ + f"Unexpected type (expected: {val_type}, detected: {arg_data['dtype']}" + + data = json_data['metadata']['signature_def']['signatureDef']['serving_default'] + validate( + test_data=data['inputs'], val_shapes=model.input_shapes, val_types=model.input_types + ) + validate( + test_data=data['outputs'], val_shapes=model.output_shapes, val_types=model.output_types + ) + + def validate_meta_rest(self, model, response): + metadata = json.loads(response.text) + assert model.name == metadata['modelSpec']['name'] + assert model.version == int(metadata['modelSpec']['version']) + + metadata_inputs = metadata['metadata']['signature_def']['signatureDef']['serving_default']['inputs'] + metadata_outputs = metadata['metadata']['signature_def']['signatureDef']['serving_default']['outputs'] + + for name, description in model.inputs.items(): + assert name in metadata_inputs + assert model.inputs[name]['shape'] == [ + int(x['size']) for x in metadata_inputs[name]['tensorShape']['dim'] + ] + assert self.cast_type_to_string(model.inputs[name]['dtype']) == metadata_inputs[name]['dtype'] + + for name, description in model.outputs.items(): + assert name in metadata_outputs + assert model.outputs[name]['shape'] == [ + int(x['size']) for x in metadata_outputs[name]['tensorShape']['dim'] + ] + assert self.cast_type_to_string(model.outputs[name]['dtype']) == metadata_outputs[name]['dtype'] + + def validate_status(self, model, status): + to_check = status.model_version_status[0] + assert model.version == to_check.version, f"Unexpected version (detected: {to_check.version}, expected: "\ + f"{model.version})" + expected_res = get_model_status_pb2.ModelVersionStatus.State.AVAILABLE + state_map = get_model_status_pb2.ModelVersionStatus.State.items() + assert to_check.state == expected_res, f"Unexpected state (detected: {to_check.state}, expected "\ + f"{expected_res} - map: {state_map})" + assert to_check.status.error_message == 'OK', f"Unexpected error msg (detected: "\ + f"{to_check.status.error_message}, expected: OK" + diff --git a/tests/functional/utils/inference/serving/triton.py b/tests/functional/utils/inference/serving/triton.py new file mode 100644 index 0000000000..35783f42fd --- /dev/null +++ b/tests/functional/utils/inference/serving/triton.py @@ -0,0 +1,211 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import queue + +import numpy as np +import tritonclient.grpc as triton_grpc +import tritonclient.http as triton_http +from tritonclient.utils import np_to_triton_dtype + +from tests.functional.utils.assertions import InvalidMetadataException +from tests.functional.utils.inference.communication import GRPC +from tests.functional.utils.inference.communication.base import AbstractCommunicationInterface +from tests.functional.utils.inference.serving.base import AbstractServingWrapper + +TRITON = "TRITON" + + +class TritonServingWrapper(AbstractServingWrapper, AbstractCommunicationInterface): + def __init__(self, **kwargs): + self._triton_client = None + self._requests_sent = 0 + self._async_infer_request_results_queue = queue.Queue() + AbstractServingWrapper.__init__(self, **kwargs) + AbstractCommunicationInterface.__init__(self, **kwargs) + + def _async_infer_ready_callback(self, result, error): + assert error is None, f"Async infer call failed: {error}" + self._async_infer_request_results_queue.put(result, block=True) + + def get_single_grpc_async_infer_result(self): + result = self._async_infer_request_results_queue.get(block=True) + response = result.get_response() + outputs = { + output.name: result.as_numpy(output.name) for output in response.outputs + } + return outputs + + def get_single_rest_async_infer_result(self, async_request): + result = async_request.get_result() + response = result.get_response() + outputs = { + output["name"]: result.as_numpy(output["name"]) for output in response['outputs'] + } + return outputs + + def get_async_infer_results(self, number_of_results): + if self.type == GRPC: + result = [self.get_single_grpc_async_infer_result() for i in range(number_of_results)] + else: + result = [self.get_single_rest_async_infer_result(async_request) for async_request in self.async_requests] + return result + + def async_infer(self, request): + self.async_requests = [] + for _data in request['request']: + if self.type == GRPC: + self._triton_client.async_infer( + model_name=self.model.name, + callback=self._async_infer_ready_callback, + inputs=_data) + else: + self.async_requests.append( + self._triton_client.async_infer( + model_name=self.model.name, + inputs=_data + ) + ) + self._requests_sent += 1 + + def infer(self, inputs=None, outputs=None): + results = self._triton_client.infer( + model_name=self.model_name, + inputs=inputs, + outputs=outputs) + return results + + def prepare_request(self, input_objects: dict, **kwargs): + data = [] + for input_name, val in input_objects.items(): + _in = self.api_client.InferInput(input_name, val.shape, np_to_triton_dtype(val.dtype)) + _in.set_data_from_numpy(val) + data.append(_in) + return {"request": [data]} + + def get_model_meta(self, timeout=60, version=None, update_model_info=True, model_name=None): + raise NotImplementedError("Not supported yet") + + def get_model_status(self, model_name=None): + raise NotImplementedError("Not supported yet") + + def send_predict_request(self, request, timeout): + raise NotImplementedError("Not supported yet") + + @staticmethod + def assert_raises_exception(status, error_message_phrase, callable_obj, *args, **kwargs): + raise NotImplementedError("Not supported yet") + + def set_grpc_stubs(self): + raise NotImplementedError("Not supported yet") + + def create_inference(self): + self.api_client = triton_grpc if self.type == GRPC else triton_http + self._triton_client = self.api_client.InferenceServerClient(url=self.url, verbose=True) + + def predict(self, request): + raise NotImplementedError("Not supported yet") + + def get_rest_path(self, operation, model_version=None, model_name=None): + raise NotImplementedError("Not supported yet") + + def get_inputs_outputs_from_response(self, response): + raise NotImplementedError("Not supported yet") + + def get_model_meta_grpc_request(self, model_name=None): + raise NotImplementedError("Not supported yet") + + def get_predict_grpc_request(self): + raise NotImplementedError("Not supported yet") + + def run_triton_infer(self, inputs=None, outputs=None, input_key=None): + if self.model.is_mediapipe: + inputs = self.prepare_triton_mediapipe_inputs(input_key) if inputs is None else inputs + verify_version = False + + else: + inputs = self.prepare_triton_input_data(inputs) if inputs is None else inputs + outputs = self.prepare_triton_output_data() if outputs is None else outputs + verify_version = True + + results = self.infer(inputs=inputs, outputs=outputs) + + response = results.get_response() + self.validate_triton_response(response, verify_version=verify_version) + + def prepare_triton_input_data(self, input_data=None): + model_inputs = self.model.inputs if input_data is None else input_data + inputs = [] + for in_model_name, input_details in model_inputs.items(): + dataset_input = None + triton_dtype = self.cast_type_to_string(input_details['dtype']) if not self.model.is_language else "BYTES" + if input_details.get('dataset', None) is not None: + dataset_input = self.model.prepare_input_data_from_model_datasets() + shape = input_details['shape'] if not getattr(dataset_input[in_model_name], "shape", None) else \ + dataset_input[in_model_name].shape + else: + shape = input_details['shape'] + + _input = self.api_client.InferInput(in_model_name, list(shape), triton_dtype) + + if dataset_input is not None: + input_data = dataset_input[in_model_name] + else: + input_data = np.ones(shape=tuple(shape), dtype=model_inputs[in_model_name]['dtype']) + _input.set_data_from_numpy(input_data) + inputs.append(_input) + + return inputs + + def prepare_triton_output_data(self): + outputs = [] + for i, out_model_name in enumerate(self.model.outputs): + outputs.append(self.api_client.InferRequestedOutput(out_model_name)) + return outputs + + def validate_triton_response(self, response, verify_version=True): + if self.communication == GRPC: + response_outputs = response.outputs + name = response.model_name + version = response.model_version if verify_version else None + raw_outputs = response.raw_output_contents + else: + response_outputs = response["outputs"] + name = response['model_name'] + version = response['model_version'] if verify_version else None + raw_outputs = response_outputs + + error_message = (f"Invalid number of raw_output_contents - Expected: {len(self.model.outputs)}; " + f"Actual: {len(raw_outputs)}") + if not len(response_outputs) == len(self.model.outputs): + raise InvalidMetadataException(error_message) + + self.validate_v2_model_name_version(name, version, self.model) + + def prepare_triton_mediapipe_inputs(self, input_key=None): + inputs = [] + input_keys = [input_key] if input_key is not None else self.model.inputs.keys() + for _input_key in input_keys: + dataset_input = self.model.prepare_input_data(input_key=_input_key)[_input_key] + numpy_dataset_input = np.array(dataset_input) + + triton_dtype = self.cast_type_to_string(numpy_dataset_input.dtype) + shape = numpy_dataset_input.shape + _input = self.api_client.InferInput(_input_key, list(shape), triton_dtype) + _input.set_data_from_numpy(numpy_dataset_input) + inputs.append(_input) + + return inputs diff --git a/tests/functional/utils/log_monitor.py b/tests/functional/utils/log_monitor.py new file mode 100644 index 0000000000..87ff8a5e47 --- /dev/null +++ b/tests/functional/utils/log_monitor.py @@ -0,0 +1,244 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from abc import ABC, abstractmethod +from datetime import datetime +from time import sleep + +from tests.functional.utils.assertions import LogMessageNotFoundException, OvmsCrashed, UnwantedMessageError +from tests.functional.utils.logger import get_logger +from tests.functional.config import artifacts_dir, wait_for_messages_timeout + +logger = get_logger(__name__) + + +class LogMonitor(ABC): + + def __init__(self, **kwargs): + self.context = None + self._read_lines = [] + self.current_offset = 0 + self.logger_creation_start_offset = 0 + + def _read_log_line(self): + log_line = None + if self.current_offset >= len(self._read_lines): + self._refresh() + + if self.current_offset < len(self._read_lines): + log_line = self._read_lines[self.current_offset] + self.current_offset += 1 + return log_line + + def _refresh(self, start_position=None): + return self.get_all_logs() + + @abstractmethod + def get_all_logs(self): + raise NotImplemented() + + def get_logs_as_txt(self): + return "\n".join(self.get_all_logs()) + + @abstractmethod + def _get_unexpected_messages(self): + return [] + + @abstractmethod + def _get_unexpected_messages_regex(self): + return [] + + def raise_on_unexpected_messages(self, **kwargs): + unexpected_messages = self.check_for_unexpected_messages() + if not unexpected_messages: + return + + def filter_unexpected_messages( + self, log_entry, found_unexpected_messages, unexpected_messages, unexpected_messages_re + ): + if any(filter(lambda x: x in log_entry, unexpected_messages)) or any( + filter(lambda x: x.match(log_entry), unexpected_messages_re) + ): + found_unexpected_messages.append(log_entry) + + def check_for_unexpected_messages(self, logs=None): + if logs is None: + logs = self.get_all_logs() + + unexpected_messages = self._get_unexpected_messages() + unexpected_messages_re = self._get_unexpected_messages_regex() + + found_unexpected_messages = [] + for log_entry in logs: + self.filter_unexpected_messages( + log_entry, found_unexpected_messages, unexpected_messages, unexpected_messages_re + ) + return found_unexpected_messages + + def cleanup(self, filename): + logs = self.get_all_logs() + found_unexpected_messages = self.check_for_unexpected_messages(logs) + path = self.save_to_file(filename, logs) + return path, found_unexpected_messages + + @staticmethod + def save_to_file(filename, logs): + os.makedirs(artifacts_dir, exist_ok=True) + file_path = os.path.join(artifacts_dir, filename) + with open(file_path, "w", encoding="utf-8") as fd: + for line in logs: + fd.write(f"{line}\n") + return file_path + + def reset_to_logger_creation(self): + self.current_offset = self.logger_creation_start_offset + + def flush(self): + self.current_offset = len(self._read_lines) + + def wait_for_messages( + self, + messages_to_find, + break_msg_list=None, + raise_exception_if_not_found=True, + timeout=None, + callbacks=[], + ovms_instance=None, + check_ovms_running=True, + all_messages=False, + ): + break_msg_list = [] if break_msg_list is None else break_msg_list + if timeout is None: + timeout = wait_for_messages_timeout + + messages_found = False + found_lines = [] + if messages_to_find: + if isinstance(messages_to_find, str): + messages_to_find = [messages_to_find] + + messages_to_find_vs_results_map = dict.fromkeys(messages_to_find, None) + start = datetime.now() + while not messages_found and (datetime.now() - start).total_seconds() <= timeout: + log_line = self._read_log_line() + if log_line is None: + if not check_ovms_running: + continue + else: + if not self.is_ovms_running(): + break # OVMS is not running, no new output is expected. + # Run callbacks at idle. + for callback in callbacks: + callback() + sleep(1) + continue + + found_lines.append(log_line) + + for specific_msg in messages_to_find_vs_results_map: + if messages_to_find_vs_results_map[specific_msg] is None: + if isinstance(specific_msg, str): + messages_to_find_vs_results_map[specific_msg] = log_line if specific_msg in log_line \ + else None + elif isinstance(specific_msg, tuple): + for msg in specific_msg: + if msg in log_line: + messages_to_find_vs_results_map[specific_msg] = log_line + break + else: + raise NotImplementedError() + + if any(map(lambda break_msg: break_msg in log_line, break_msg_list)): + ovms_log = self.get_logs_as_txt() + logger.info(ovms_log) + raise UnwantedMessageError( + f"Found message: '{log_line}'", ovms_log=ovms_log, context=self.context + ) + + if all_messages: + messages_found = all(x for x in messages_to_find_vs_results_map.values()) + else: + messages_found = any(x for x in messages_to_find_vs_results_map.values()) + self._log_search_info( + raise_exception_if_not_found, + messages_found, + found_lines, + messages_to_find_vs_results_map, + ovms_instance, + ) + + # Run all callbacks at end. + # Callbacks can be used to validate proper ovms load at many sources (ie: dmesg) + for callback in callbacks: + callback() + return messages_found + + def find_messages(self, messages_to_find, raise_exception_if_not_found=False): + all_messages_found = False + found_lines = [] + + if messages_to_find: + if messages_to_find is str: + messages_to_find = [messages_to_find] + + messages_to_find_vs_results_map = dict.fromkeys(messages_to_find, None) + while not all_messages_found: + log_line = self._read_log_line() + if log_line is None: + break + else: + found_lines.append(log_line) + + for specific_msg in messages_to_find_vs_results_map: + if messages_to_find_vs_results_map[specific_msg] is None: + messages_to_find_vs_results_map[specific_msg] = log_line if specific_msg in log_line else None + + all_messages_found = all([x for x in messages_to_find_vs_results_map.values()]) + self._log_search_info( + raise_exception_if_not_found, all_messages_found, found_lines, messages_to_find_vs_results_map + ) + return all_messages_found, messages_to_find_vs_results_map + + def _log_search_info( + self, + raise_exception_if_not_found, + all_messages_found, + found_lines, + messages_to_find_vs_results_map, + ovms_instance=None, + ): + if all_messages_found: + for msg, log_entry in messages_to_find_vs_results_map.items(): + logger.debug(f"Find log procedure result: \n\tMessage: {msg} \n\tLog entry: {log_entry}") + else: + if raise_exception_if_not_found: + missing_messages = [k for k, v in messages_to_find_vs_results_map.items() if v is None] + found_lines_str = "\n".join(found_lines) + error_msg = ( + f"Unable to find following messages in logs: {missing_messages}\n= Ovms logs {18 * '='}\n" + f"{found_lines_str}\n{30 * '='}" + ) + if len(missing_messages) != 0: + exception = OvmsCrashed if not self.is_ovms_running() else LogMessageNotFoundException + dmesg_log = None + if ovms_instance is not None: + ovms_instance._dmesg_log.raise_on_unexpected_messages() + dmesg_log = ovms_instance._dmesg_log.get_logs_as_txt() + raise exception(error_msg, ovms_log=self.get_logs_as_txt(), dmesg_log=dmesg_log) + + def is_ovms_running(self): + return True diff --git a/tests/functional/utils/logger.py b/tests/functional/utils/logger.py index d54f8c3ea9..f2ee64a46e 100644 --- a/tests/functional/utils/logger.py +++ b/tests/functional/utils/logger.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 Intel Corporation +# Copyright (c) 2026 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,17 +14,323 @@ # limitations under the License. # +import base64 +import hashlib +import inspect import logging +import os +import re +import weakref +from datetime import datetime +from typing import Generator, List, Tuple, Union, cast -import tests.functional.config as config +from tests.functional.constants.os_type import OsType, get_host_os +from tests.functional.utils.helpers import get_xdist_worker_count -LOGGER_LEVEL = "INFO" +from tests.functional import config -def init_logger(): - logger = logging.getLogger(None) - log_formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s") - logger.setLevel(config.log_level) - console_handler = logging.StreamHandler() - console_handler.setFormatter(log_formatter) - logger.addHandler(console_handler) +SEPARATOR = "=" * 20 +FIXTURE_SEPARATOR = "*" * 20 +UNDEFINED = "" +UNDEFINED_BASE64 = base64.b64encode(UNDEFINED.encode('utf-8')) +API = "api" +LOCALHOST = "localhost" + +ONE_K = 1024 +ONE_M = ONE_K * ONE_K + + +log_username = f"- [{config.host_os_user}] " if config.log_username and config.host_os_user is not None else "" +worker_count = get_xdist_worker_count() +worker_id = os.environ.get("PYTEST_XDIST_WORKER", "") +worker_string = f"[{worker_id}] " if worker_count > 0 else "" +logger_format = config.logger_format(worker_string, log_username) + +logging.basicConfig( + level=config.logging_level, + format=logger_format, +) + + +class Chunks(Generator): + """ + generator yielding tuple: no of part, number of parts, and part of the input list + """ + def __init__(self, seq: List[str], max_number_of_elements: int = 1000) -> None: + super().__init__() + self.seq = tuple(seq) + assert max_number_of_elements > 0, "Incorrect number of elements, should be more than zero" + self.chunk_len = max_number_of_elements + self.no_of_chunks = (len(self.seq) // self.chunk_len) + 1 + self.current_chunk = 0 + self.index_iterator = iter(range(0, len(self.seq), self.chunk_len)) + + def __next__(self) -> Tuple[int, int, list]: + return self.send(None) + + def __iter__(self) -> 'Chunks': + return self + + def send(self, ignored_value) -> Tuple[int, int, list]: + index = next(self.index_iterator) + return_chunk = self.current_chunk, self.no_of_chunks, list(self.seq[index:index + self.chunk_len]) + self.current_chunk += 1 + return return_chunk + + def throw(self, typ, val=None, tb=None): + raise StopIteration + + def close(self) -> None: + raise GeneratorExit + + +class SensitiveKeysStrippingFilter(logging.Filter): + instance = None + sensitive_pairs = None # type: dict + sensitive_values_to_be_masked = None # type: re + + def __new__(cls) -> 'SensitiveKeysStrippingFilter': + if cls.instance is None: + cls.instance = super().__new__(cls) + cls.sensitive_pairs = cls.gather_sensitive_pairs() + cls.sensitive_values_to_be_masked = list(cls.sensitive_pairs.values()) + return cls.instance + + @classmethod + def build_sensitive_values_regexp(cls) -> re: + return re.compile( + "|".join([fr"{var}" + for var in cls.sensitive_pairs.values()])) + + @classmethod + def gather_sensitive_pairs(cls) -> dict: + return dict([(var, getattr(config, var, None)) + for var in dir(config) + if cls.is_matching_variable(var)]) + + @staticmethod + def is_matching_variable(var) -> bool: + if config.sensitive_keys_to_be_masked.match(var): + var_value = getattr(config, var, UNDEFINED) + if var_value is not UNDEFINED and \ + isinstance(var_value, str) and \ + len(var_value) > 0: + return True + return False + + def filter(self, record: logging.LogRecord) -> bool: + record.msg = self.strip_sensitive_data(record.msg) + record.args = self.filter_args(record.args) + return True + + def filter_args(self, args: Union[dict, tuple]) -> Union[dict, tuple]: + if not isinstance(args, (dict, tuple)): + return args + if isinstance(args, dict): + args = self.strip_sensitive_data(args) + else: + args = tuple(self.strip_sensitive_data(arg) for arg in args) + return args + + def strip_sensitive_data(self, data: Union[dict, str]) -> Union[dict, str]: + if config.strip_sensitive_data: + if isinstance(data, str) and len(data) > 0: + data = self.strip_sensitive_str_values(data) + elif isinstance(data, dict): + data = self.strip_sensitive_dict_values(data.copy()) + return data + + def strip_sensitive_dict_values(self, data: dict) -> dict: + for key, value in data.items(): + if value in self.sensitive_values_to_be_masked: + data[key] = "******" + return data + + def strip_sensitive_str_values(self, data: str) -> str: + stripped_data = data + for sensitive_value_to_be_masked in self.sensitive_values_to_be_masked: + stripped_data = stripped_data.replace(sensitive_value_to_be_masked, "******") + return stripped_data + + +class LoggerType(object): + """Logger types definitions""" + HTTP_REQUEST = "http_request" + HTTP_RESPONSE = "http_response" + REMOTE_LOGGER = "remote logger" + SHELL_COMMAND = "shell command" + STEP_LOGGER = "STEP" + FIXTURE_LOGGER = "FIXTURE" + FINALIZER_LOGGER = "FINALIZER" + + +class Logger(logging.Logger): + """src: https://stackoverflow.com/a/22586200""" + MIN_NUMBER_OF_LINES_TO_PRESENT_FINAL_MSG = 20 + VERBOSE = 5 + + def __init__(self, name, level=logging.NOTSET): + super().__init__(name, level) + # noinspection PyTypeChecker + self.last_record = None # type: weakref.ReferenceType + logging.addLevelName(self.VERBOSE, "VERBOSE") + + # pylint: disable=too-many-arguments + def makeRecord(self, name, level, fn, lno, msg, args, exc_info, + func=None, extra=None, sinfo: Union[None, bool] = None): + record = super().makeRecord(name, level, fn, lno, msg, args, exc_info, func, extra, sinfo) + self.last_record = weakref.ref(record) # type: weakref.ReferenceType + return record + + # pylint: disable=too-many-arguments + def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False, + list_of_strings: List[str] = None, + chunk_len: int = 1000, + chunk_msg: str = None, + final_msg: str = None): + try: + super()._log(level, msg, args, exc_info, extra, stack_info) + except Exception as exc: + print(str(exc)) + print(msg) + self.log_list_of_strings(level, chunk_msg, args, exc_info, extra, + stack_info, list_of_strings, chunk_len, final_msg) + + def findCaller(self, stack_info: bool = False, stacklevel: int = 1): + last_record = self.last_record() if self.last_record is not None else None # type: logging.LogRecord + if last_record is not None: + return last_record.pathname, last_record.lineno, last_record.funcName, last_record.stack_info + return super().findCaller(stack_info=stack_info) + + # pylint: disable=too-many-arguments + def log_list_of_strings(self, level, chunk_msg, args, exc_info=None, extra=None, stack_info=False, + list_of_strings: List[str] = None, + chunk_len: int = 1000, + final_msg: str = None): + fn, lno, func, sinfo = self.findCaller(stack_info=stack_info) + if list_of_strings is not None and len(list_of_strings): + chunks = Chunks(list_of_strings, max_number_of_elements=chunk_len) + if chunks.no_of_chunks > 1: + chunk_msg = chunk_msg.rstrip() if chunk_msg is not None else "Presenting chunk" + chunk_msg = " ".join([chunk_msg.rstrip(), "({index}/{no_of_chunks}):\n{chunk}\n"]) + else: + chunk_msg = "\n{chunk}\n" + list_chunk = [] + for chunk_number, no_of_chunks, list_chunk in chunks: + formatted_chunk_msg = chunk_msg.format(index=chunk_number, + no_of_chunks=no_of_chunks, + chunk="\n".join(list_chunk)) + chunk_record = self.makeRecord(self.name, level, fn, lno, formatted_chunk_msg, args, + exc_info, func, extra, sinfo) + self.handle(chunk_record) + else: + if len(list_chunk) > self.MIN_NUMBER_OF_LINES_TO_PRESENT_FINAL_MSG: + final_msg = final_msg.rstrip() if final_msg is not None else "End of presenting chunks" + if chunks.no_of_chunks > 1: + final_msg = " ".join([final_msg, f"Presented {chunks.no_of_chunks} chunks."]) + final_record = self.makeRecord(self.name, level, fn, lno, final_msg, args, + exc_info, func, extra, sinfo) + self.handle(final_record) + + def verbose(self, msg, *args, **kwargs): + if self.isEnabledFor(self.VERBOSE): + self._log(self.VERBOSE, msg, args, **kwargs) + + +logging.setLoggerClass(Logger) +logging.addLevelName(Logger.VERBOSE, "VERBOSE") + +__LOGGING_LEVEL = config.logging_level + + +def get_logger(name) -> Logger: + logger = logging.getLogger(name) + logger.addFilter(SensitiveKeysStrippingFilter()) + logger.setLevel(__LOGGING_LEVEL) + return cast(Logger, logger) + + +def step(message): + caller = inspect.stack()[1][3] + _log_separator(logger_type=LoggerType.STEP_LOGGER, separator=SEPARATOR, caller=caller, message=message) + + +def log_fixture(message, separator=FIXTURE_SEPARATOR): + caller = inspect.stack()[1][3] + _log_separator(logger_type=LoggerType.FIXTURE_LOGGER, separator=separator, caller=caller, message=message) + + +def _log_separator(logger_type, separator, caller, message): + get_logger(logger_type).info(f"{separator} {caller}: {message} {separator}") + + +def line_trimmer(line: str, max_number_of_elements: int = 1 * ONE_K // 4): + if len(line) > max_number_of_elements: + line = "t: " + \ + line[:max_number_of_elements // 2] + \ + "[...]" + \ + line[-max_number_of_elements // 2:] + return line + + +def list_trimmer(seq: list, max_number_of_elements: int = 4 * ONE_K): + if len(seq) > max_number_of_elements: + first_element = [f"Too long output was trimmed! Original len {len(seq)}, " + f"showing first and last {max_number_of_elements // 2} lines:"] + seq = first_element + seq[:max_number_of_elements // 2] + ["", "[...]", ""] + seq[-max_number_of_elements // 2:] + return seq + + +def log_trimmer(logs: str): + logs_list = logs.split(sep="\n") + logs_list = [line_trimmer(line) for line in logs_list] + logs_trimmed = list_trimmer(seq=logs_list, max_number_of_elements=ONE_K) + logs = "\n".join(logs_trimmed) + return logs + + +def sanitize_node(name_or_node_id): + name_or_node_id = "__".join(name_or_node_id.split("/")) + name_or_node_id = "..".join(name_or_node_id.split("::")) + name_or_node_id = "-".join(name_or_node_id.split(" ")) + return name_or_node_id + + +class OvmsFileHandler(logging.FileHandler): + def __init__(self, item: Union["Item", str] = None, + mode='a', encoding=None, delay=False): + self.filename = self.log_file_name(item) + file_path = os.path.join(config.test_log_directory, self.filename) + if item is not None: + item._log_file_name = self.filename + super().__init__(file_path, mode, encoding, delay) + fmt = logging.Formatter(logger_format) + self.setFormatter(fmt) + + @staticmethod + def safe_node_id(item: "Item"): + return sanitize_node(item.nodeid) + + @classmethod + def log_file_name(cls, item: "Item" = None): + if isinstance(item, str): + file_name = item + else: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + worker = f"_{worker_id}" if worker_count > 0 else "" + prefix = f"{cls.safe_node_id(item)}" if item is not None else "api_tests" + file_name = f"{prefix}{worker}_{timestamp}.log" + if len(file_name.encode('utf-8')) > 255: + file_name = cls.generate_short_log_file_name(file_name) + if get_host_os() == OsType.Windows: + # replace characters not supported by Windows: \/:*?"<>| + file_name = re.sub(r'[\\\/\:\*\?\\"\<\>\|]', "_", file_name) + return file_name + + @classmethod + def generate_short_log_file_name(cls, file_name): + encoded_str = file_name.encode() + hash_obj = hashlib.sha1(encoded_str) + hexa_value = hash_obj.hexdigest() + return f"{file_name.split('.log')[0][:246]}_{hexa_value[:4]}.log" diff --git a/tests/functional/utils/marks.py b/tests/functional/utils/marks.py new file mode 100644 index 0000000000..f6ffbf11c5 --- /dev/null +++ b/tests/functional/utils/marks.py @@ -0,0 +1,323 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from enum import Enum +from itertools import chain +from re import Pattern +from typing import Union + +from _pytest.nodes import Item + +from tests.functional.utils.logger import get_logger +from tests.functional.config import repository_name + +logger = get_logger(__name__) + + +class MarkMeta(str, Enum): + def __new__(cls, mark: str, description: str = None, *args): + obj = str.__new__(cls, mark) # noqa + obj._value_ = mark + obj.description = description + return obj + + def __init__(self, *args): + super(MarkMeta, self).__init__() + + def __hash__(self) -> int: + return hash(self.mark) + + def __format__(self, format_spec): + return self.mark + + def __repr__(self): + return self.mark + + def __str__(self): + return self.mark + + @classmethod + def get_by_name(cls, name): + return name + + @property + def mark(self): + return self._value_ + + @property + def marker_with_description(self): + return f"{self.mark}{f': {self.description}' if self.description is not None else ''}" + + def __eq__(self, o: object) -> bool: + if isinstance(o, str): + return self.mark.__eq__(o) + return super().__eq__(o) + + +class ConditionalMark(MarkMeta): + @classmethod + def get_conditional_marks_from_item(cls, name, item): + marks = list(filter(lambda x: x.name == name and x.args is not None, item.keywords.node.own_markers)) + return marks + + @classmethod + def _params_phrase_match_test_params(cls, params, item): + """ + Verify if current 'item' parameter match pytest Mark from test case + """ + if params is None: # no filtering -> any param will match + return True + if hasattr(item.keywords.node, "callspec"): + test_params = item.keywords.node.callspec.id + if isinstance(params, Pattern): + return bool(params.match(test_params)) + elif isinstance(params, str): + return params == test_params + else: + raise AttributeError(f"Unexpected conditional marker params {params}") + return True + + @classmethod + def _process_single_entry(cls, entry, item): + """ + Check if mark 'condition' is meet and item parameters match re/str phrase. + Then return mark value + """ + value, condition, params = None, True, None + if isinstance(entry, str): + # Simple string do not have condition nor parameters. + value = entry + elif isinstance(entry, dict): + value = entry.get('value') # required + condition = entry.get('condition', True) + params = entry.get('params', None) + elif isinstance(entry, tuple): + value, *_optional = entry + if isinstance(value, list): + return cls._process_single_entry(value, item) + + if len(_optional) > 0: + condition = _optional[0] + if len(_optional) > 1: + params = _optional[1] + elif isinstance(entry, list): + for _element in entry: + value = cls._process_single_entry(_element, item) + if value: # Return first match + return value + return None + else: + raise AttributeError(f"Unexpected conditional marker entry {entry}") + + if not condition: + return None + return value if cls._params_phrase_match_test_params(params, item) else None + + @classmethod + def get_all_marks_values_from_item(cls, item, marks): + mark_values = [] + for mark in marks: + values = cls.get_all_marker_values_from_item(item, mark) + if values: + mark_values.extend(values) + return mark_values + + @classmethod + def get_all_marker_values_from_item(cls, item, mark, _args=None): + """ + Marker can be set as 'str', 'list', 'tuple', 'dict'. + Process it accordingly and list of values. + """ + marker_values = [] + args = _args if _args else mark.args + if isinstance(args, list): + for entry in args: + value = cls._process_single_entry(entry, item) + if not value: + continue + marker_values.append(value) + elif isinstance(args, tuple): + value = cls._process_single_entry(args, item) + if value: + marker_values.append(value) + elif isinstance(args, str): + marker_values.append(args) + elif isinstance(args, dict): + for params, value in args.items(): + if not cls._params_phrase_match_test_params(params, item): + continue + if isinstance(value, list): + marker_values.extend(value) + else: + marker_values.append(value) + else: + raise AttributeError(f"Unrecognized conditional marker {mark}") + return marker_values + + @classmethod + def get_markers_values_from_item(cls, item, marks): + result = [] + for mark in marks: + result.extend(cls.get_all_marker_values_from_item(item, mark)) + return result + + @classmethod + def get_markers_values_via_conditional_marker(cls, item, name): + conditional_marks = cls.get_conditional_marks_from_item(name, item) + markers_values = cls.get_markers_values_from_item(item, conditional_marks) + return markers_values + + @classmethod + def get_mark_from_item(cls, item: Item, conditional_marker_name=None): + marks = cls.get_markers_values_via_conditional_marker(item, conditional_marker_name) + if not marks: + return cls.get_closest_mark(item) + marks = marks[0] + return marks + + @classmethod + def get_closest_mark(cls, item: Item): + for mark in cls: # type: 'MarkRunType' + if item.get_closest_marker(mark.mark): + return mark + return None + + @classmethod + def get_by_name(cls, name): + mark = list(filter(lambda x: x.value == name, list(cls))) + return mark[0] + + +class MarkBugs(ConditionalMark): + @classmethod + def get_all_bug_marks_values_from_item(cls, item: Item, conditional_marks=None): + if not conditional_marks: + conditional_marks = cls.get_conditional_marks_from_item("bugs", item) + bugs = cls.get_all_marks_values_from_item(item, conditional_marks) + return bugs + + +class MarkGeneral(MarkMeta): + COMPONENTS = "components" + REQIDS = "reqids", "Mark requirements tested" + + +class MarkPriority(MarkMeta): + HIGH = "priority_high", "Mark as a priority high" + MEDIUM = "priority_medium", "Mark as a priority medium" + LOW = "priority_low", "Mark as a priority low" + + +class MarkSupportedDevices(MarkMeta): + DEVICES_SUPPORTED = "devices_supported_for_test", "Mark devices that are supported for test" + DEVICES_NOT_SUPPORTED = "devices_not_supported_for_test", "Mark devices that are not supported for test" + + +class MarkSupportedOvmsTypes(MarkMeta): + OVMS_TYPES_SUPPORTED = "ovms_types_supported_for_test", "Mark ovms types that are supported for test" + OVMS_TYPES_NOT_SUPPORTED = "ovms_types_not_supported_for_test", "Mark ovms types that are not supported for test" + + +class MarkSupportedOsTypes(MarkMeta): + OS_TYPES_SUPPORTED = "os_types_supported_for_test", "Mark os types that are supported for test" + OS_TYPES_NOT_SUPPORTED = "os_types_not_supported_for_test", "Mark os types that are not supported for test" + + +class MarkTestParameters(MarkMeta): + MODEL_TYPE = "model_type" + MODEL_AUX_TYPE = "model_aux_type" + ALL_MODELS = "all_models" + MANY_MODELS = "many_models" + ITERATION_INFO = "iteration_info" + INPUT_SHAPE = "input_shape" + INPUT_SHAPE_NO_AUTO = "input_shape_no_auto" + PLUGIN_CONFIG = "plugin_config" + + +class MarkConditionalRunType(MarkMeta): + CONDITIONAL_RUN_TYPE = "conditional_run_type", \ + "Conditionally assign single/non-single run type mark based on device and OS" + CONDITIONAL_RUN_TYPE_BY_MODEL = "conditional_run_type_by_model", \ + "Conditionally assign run type mark based on model_type membership in model collections" + + +class MarkRunType(ConditionalMark): + TEST_MARK_COMPONENT = "component", "run component tests", "component" + TEST_MARK_SMOKE = "api_smoke", "run api-smoke tests", "api_smoke" + TEST_MARK_ON_COMMIT = "api_on_commit", "run api-on-commit tests", "api_on-commit" + TEST_MARK_REGRESSION = "api_regression", "run api-regression tests", "api_regression" + TEST_MARK_REGRESSION_SINGLE = "api_regression_single", "run api-regression-single tests", "api_regression-single" + TEST_MARK_REGRESSION_WEEKLY = "api_regression_weekly", "run api-regression-weekly tests", "api_regression-weekly" + TEST_MARK_REGRESSION_WEEKLY_SINGLE = "api_regression_weekly_single", "run api-regression-weekly-single tests", \ + "api_regression-weekly-single" + TEST_MARK_ENABLING = "api_enabling", "run api-enabling tests", "api_enabling" + TEST_MARK_MANUAL = "manual", "run api-manual tests", "api_manual" + TEST_MARK_STRESS_AND_LOAD = "api_stress_and_load", "run api-stress-and-load tests", "api_stress-and-load" + TEST_MARK_STRESS_AND_LOAD_SINGLE = "api_stress_and_load_single", "run api-stress-and-load-single tests",\ + "api_stress-and-load-single" + TEST_MARK_LONG = "api_long", "run api-long tests", "api_long" + TEST_MARK_PERFORMANCE = "api_performance", "run api-performance tests", "api_performance" + TEST_MARK_UNSTABLE = "api_unstable", "run api_api_unstable tests", "api_unstable" + + def __init__(self, mark: str, description: str = None, run_type: str = None) -> None: + super().__init__(self, mark, description) + self.run_type = f"{repository_name}_{run_type}" if repository_name is not None else run_type + + @classmethod + def test_mark_to_test_run_type(cls, test_type_mark: Union['MarkRunType', str]): + if isinstance(test_type_mark, str): + return MarkRunType(test_type_mark).run_type + return test_type_mark.run_type + + @classmethod + def get_test_type_mark(cls, item: Item): + return cls.get_mark_from_item(item, "test_group") + + @classmethod + def test_type_mark_to_int(cls, item): + mark = cls.get_test_type_mark(item) + assert mark, "Cannot find test_type mark from {item}" + return list(cls).index(mark) + + +API_COMPONENT = MarkRunType.TEST_MARK_COMPONENT +API_ON_COMMIT = MarkRunType.TEST_MARK_ON_COMMIT +API_REGRESSION = MarkRunType.TEST_MARK_REGRESSION +API_REGRESSION_SINGLE = MarkRunType.TEST_MARK_REGRESSION_SINGLE +API_REGRESSION_WEEKLY = MarkRunType.TEST_MARK_REGRESSION_WEEKLY +API_REGRESSION_WEEKLY_SINGLE = MarkRunType.TEST_MARK_REGRESSION_WEEKLY_SINGLE +API_ENABLING = MarkRunType.TEST_MARK_ENABLING +API_MANUAL = MarkRunType.TEST_MARK_MANUAL +API_STRESS_AND_LOAD = MarkRunType.TEST_MARK_STRESS_AND_LOAD +API_STRESS_AND_LOAD_SINGLE = MarkRunType.TEST_MARK_STRESS_AND_LOAD_SINGLE +API_LONG = MarkRunType.TEST_MARK_LONG +API_PERFORMANCE = MarkRunType.TEST_MARK_PERFORMANCE +API_UNSTABLE = MarkRunType.TEST_MARK_UNSTABLE + + +class MarksRegistry(tuple): + MARKERS = "markers" + MARK_ENUMS = [MarkGeneral, MarkRunType, MarkPriority, MarkBugs, MarkSupportedDevices, MarkTestParameters, + MarkSupportedOvmsTypes, MarkSupportedOsTypes, MarkConditionalRunType] + + def __new__(cls) -> 'MarksRegistry': + # noinspection PyTypeChecker + return tuple.__new__(cls, [mark for mark in chain(*cls.MARK_ENUMS)]) + + @staticmethod + def register(pytest_config): + for mark in MarksRegistry(): + pytest_config.addinivalue_line(MarksRegistry.MARKERS, mark.marker_with_description) diff --git a/tests/functional/utils/model_management.py b/tests/functional/utils/model_management.py deleted file mode 100644 index 9c5537f7bf..0000000000 --- a/tests/functional/utils/model_management.py +++ /dev/null @@ -1,118 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import re -import shutil -from datetime import datetime -from pathlib import Path - -import logging -from tests.functional.utils.parametrization import get_tests_suffix, generate_test_object_name -from tests.functional.utils.process import Process - -from tests.functional.config import converted_models_expire_time - -logger = logging.getLogger(__name__) - -def wget_file(url, dst): - Path(dst).parent.mkdir(parents=True, exist_ok=True) - logger.info(f"Downloading file via wget\n{url} => {dst}") - proc = Process() - proc.set_log_silence() - cmd = f"wget {url} -O {dst}" - proc.policy['log-check-output']['stderr'] = False - proc.run_and_check(cmd) - - -def _get_file_size_from_url(url): - cmd = f"wget -e robots=off --spider -r --server-response --no-parent {url}" - # .group(1) - url - # .group(2) - wget spider details - # .group(3) - Content-Lenght: - spider_entry_re = re.compile("-- (https?://[\S]+)\n(.+?)Content-Length: (\S+)", flags=re.MULTILINE | re.DOTALL) - proc = Process() - proc.set_log_silence() - proc.policy['log-check-output']['stderr'] = False - code, out, err = proc.run_and_check_return_all(cmd) - - all_entries_match = spider_entry_re.findall(err) - # filter only files (entries specified Length: ) - model_files_match = list(filter(lambda x: x[2] != "unspecified" and x[2] != '0', all_entries_match)) - assert(len(model_files_match) == 1) - return int(model_files_match[0][2]) - -def download_missing_file(url, output_path): - size = _get_file_size_from_url(url) - while not Path(output_path).exists() or Path(output_path).stat().st_size != size: - wget_file(url, output_path) - return output_path - -def copy_model(model, version, destination_path): - dir_to_cpy = destination_path + str(version) - if not os.path.exists(dir_to_cpy): - os.makedirs(dir_to_cpy) - shutil.copy(model[0], dir_to_cpy + '/model.bin') - shutil.copy(model[1], dir_to_cpy + '/model.xml') - return dir_to_cpy - - -def convert_model(client, - model, - output_dir, - model_name, - input_shape, - framework="tf"): - - - files = (os.path.join(output_dir, model_name) + '.bin', - os.path.join(output_dir, model_name) + '.xml') - - # Check if file exists and is not expired - if all(map(lambda x: os.path.exists(x) and \ - datetime.now().timestamp() - os.path.getmtime(x) < converted_models_expire_time, - files)): - return files - - Path(output_dir).mkdir(parents=True, exist_ok=True) - - input_shape_str = '[{}]'.format(','.join(str(i) for i in input_shape)) - logger.info("Converting {model} to IR with input shape {input_shape}...".format(model=model, - input_shape=input_shape_str)) - - input_dir = os.path.dirname(model) - - image = 'openvino/ubuntu20_dev:2022.1.0' - volumes = {input_dir: {'bind': '/mnt/input_dir', 'mode': 'ro'}, - output_dir: {'bind': '/mnt/output_dir', 'mode': 'rw'}} - user_id = os.getuid() - - command = ' '.join([ - 'mo', - '--input_model /mnt/input_dir/' + os.path.basename(model), - '--model_name ' + model_name, - '--output_dir /mnt/output_dir/', - '--input_shape ' + input_shape_str, - '--framework ' + framework - ]) - - client.containers.run(image=image, - name='convert-model-{}-{}'.format(get_tests_suffix(), generate_test_object_name(short=True)), - volumes=volumes, - user=user_id, - command=command, - remove=True) - return files diff --git a/tests/functional/utils/models_utils.py b/tests/functional/utils/models_utils.py deleted file mode 100644 index 41eeac888e..0000000000 --- a/tests/functional/utils/models_utils.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) 2019-2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -class ErrorCode: - OK = 0 - # CANCELLED = 1 - UNKNOWN = 2 - # INVALID_ARGUMENT = 3 - # DEADLINE_EXCEEDED = 4 - # NOT_FOUND = 5 - # ALREADY_EXISTS = 6 - # PERMISSION_DENIED = 7 - # UNAUTHENTICATED = 16 - # RESOURCE_EXHAUSTED = 8 - # FAILED_PRECONDITION = 9 - # ABORTED = 10 - # OUT_OF_RANGE = 11 - # UNIMPLEMENTED = 12 - # INTERNAL = 13 - # UNAVAILABLE = 14 - # DATA_LOSS = 15 - # DO_NOT_USE_RESERVED_FOR_FUTURE_EXPANSION_USE_DEFAULT_IN_SWITCH_INSTEAD_ \ - # = 20 - - -class ModelVersionState: - # UNKNOWN = 0 - START = 10 - LOADING = 20 - AVAILABLE = 30 - UNLOADING = 40 - END = 50 - - -ERROR_MESSAGE = { - ModelVersionState.START: { - ErrorCode.OK: "OK", # "Version detected" - }, - ModelVersionState.LOADING: { - ErrorCode.OK: "OK", # "Version is being loaded", - ErrorCode.UNKNOWN: "Error occurred while loading version" - }, - ModelVersionState.AVAILABLE: { - ErrorCode.OK: "OK", # "Version available" - }, - ModelVersionState.UNLOADING: { - ErrorCode.OK: "OK", # "Version is scheduled to be deleted" - }, - ModelVersionState.END: { - ErrorCode.OK: "OK", # "Version has been removed" - }, -} - -STATE_NAME = { - # ModelVersionState.UNKNOWN: "UNKNOWN", - ModelVersionState.START: "START", - ModelVersionState.LOADING: "LOADING", - ModelVersionState.AVAILABLE: "AVAILABLE", - ModelVersionState.UNLOADING: "UNLOADING", - ModelVersionState.END: "END", -} - -ERROR_CODE_NAME = { - ErrorCode.OK: "OK", - # ErrorCode.CANCELLED: "CANCELLED", - ErrorCode.UNKNOWN: "UNKNOWN", - # ErrorCode.INVALID_ARGUMENT: "INVALID_ARGUMENT", - # ErrorCode.DEADLINE_EXCEEDED: "DEADLINE_EXCEEDED", - # ErrorCode.NOT_FOUND: "NOT_FOUND", - # ErrorCode.ALREADY_EXISTS: "ALREADY_EXISTS", - # ErrorCode.PERMISSION_DENIED: "PERMISSION_DENIED", - # ErrorCode.UNAUTHENTICATED: "UNAUTHENTICATED", - # ErrorCode.RESOURCE_EXHAUSTED: "RESOURCE_EXHAUSTED", - # ErrorCode.FAILED_PRECONDITION: "FAILED_PRECONDITION", - # ErrorCode.ABORTED: "ABORTED", - # ErrorCode.OUT_OF_RANGE: "OUT_OF_RANGE", - # ErrorCode.UNIMPLEMENTED: "UNIMPLEMENTED", - # ErrorCode.INTERNAL: "INTERNAL", - # ErrorCode.UNAVAILABLE: "UNAVAILABLE", - # ErrorCode.DATA_LOSS: "DATA_LOSS", - # ErrorCode.DO_NOT_USE_RESERVED_FOR_FUTURE_EXPANSION_USE_DEFAULT_IN_SWITCH_INSTEAD_: - # "DO_NOT_USE_RESERVED_FOR_FUTURE_EXPANSION_USE_DEFAULT_IN_SWITCH_INSTEAD_" -} diff --git a/tests/functional/utils/numpy_loader.py b/tests/functional/utils/numpy_loader.py new file mode 100644 index 0000000000..8a982f91ff --- /dev/null +++ b/tests/functional/utils/numpy_loader.py @@ -0,0 +1,153 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import re + +import cv2 +import numpy as np + +from tests.functional.utils.logger import get_logger + +logger = get_logger(__name__) + + +def load_npy_labels(path): + if path is not None: + labels = np.load(path, mmap_mode='r', allow_pickle=False) + return labels + + +def adjust_to_batch_size(np_array, batch_size): + if batch_size > np_array.shape[0]: + array = np_array + for _ in range(int(batch_size / np_array.shape[0]) - 1): + np_array = np.append(np_array, array, axis=0) + np_array = np.append(np_array, np_array[:batch_size % array.shape[0]], axis=0) + else: + np_array = np_array[:batch_size, ...] + return np_array + + +def transpose_input(images, axes): + tuple_axes = [int(ax) for ax in axes] + return images.transpose(tuple_axes) + + +def crop_resize(img, cropx, cropy): + y, x, c = img.shape + if y < cropy: + img = cv2.resize(img, (x, cropy)) + y = cropy + if x < cropx: + img = cv2.resize(img, (cropx, y)) + x = cropx + startx = x//2-(cropx//2) + starty = y//2-(cropy//2) + return img[starty:starty+cropy, startx:startx+cropx, :] + + +def load_jpeg(path: str, height, width, ids, datatype=np.float32): + img = cv2.imread(path).astype(datatype) # BGR color format, shape HWC + if height > 0 and width > 0: + img = cv2.resize(img, (width, height)) + img = img.transpose(ids) + reshaped_img = img.reshape(1, *img.shape) + logger.debug(f"Image {path} shape: {reshaped_img.shape} ; " + f"data range: {np.amin(reshaped_img)} : {np.amax(reshaped_img)} ") + return reshaped_img + + +def load_labels(path): + labels_extension = path.split(sep=".")[-1] + if labels_extension == "npy": + return load_npy_labels(path=path) + elif labels_extension in ["txt", "json"]: + raise NotImplementedError() + else: + raise RuntimeError(f"Incorrect label data type: {labels_extension}") + + +def load_images(data_path, height, width, ids): + assert os.path.exists(data_path), "Error data path for images do not exists." + if os.path.isfile(data_path): + file_extension = os.path.basename(data_path).split(sep=".")[-1] + assert file_extension in ['jpg', 'jpeg'] + inputs = load_jpeg(data_path, height, width, ids) + return inputs + elif os.path.isdir(data_path): + inputs = [] + images = list(filter(lambda x: re.match(r".+\.jpe?g", x.lower()), os.listdir(data_path))) + for img in images: + path = os.path.join(data_path, img) + inputs.append(load_jpeg(path, height, width, ids)) + assert inputs, f"Lack of data to load with search path: {data_path}" + inputs = np.concatenate(inputs, axis=0) + return inputs + else: + raise AssertionError( + f"incorrect input_data_type value: {images} for provided input path (dir): {data_path}" + ) + else: + raise AssertionError(f"Invalida data_path={data_path}") + + +def load_numpy(data_path): + assert os.path.isfile(data_path) + file_extension = os.path.basename(data_path).split(sep=".")[-1] + # optional preprocessing depending on the model + data = np.load(data_path, mmap_mode='r+', allow_pickle=False) + data = data - np.min(data) # Normalization 0-255 + data = data / np.ptp(data) * 255 # Normalization 0-255 + # images = images[:,:,:,::-1] # RGB to BGR + logger.debug( + f'Input {os.path.basename(data_path)} shape: {data.shape}; data range: {np.amin(data)}: {np.amax(data)}' + ) + return data + + +def prepare_data(data_path, expected_shape, batch_size, transpose_axes=None, expected_layout=None, data_layout=None): + filename, file_extension = os.path.splitext(data_path) + if file_extension == '.npy': + data = load_numpy(data_path) + else: + if data_layout is None: + data_layout = "NHWC" + if expected_layout is None: + expected_layout = "NCHW" + + ids = [] + for dimension in expected_layout: + id = data_layout.lower().find(dimension.lower()) + ids.append(id - 1) + + height_index = expected_layout.lower().find("h") + width_index = expected_layout.lower().find("w") + expected_height = expected_shape[height_index] + expected_width = expected_shape[width_index] + + data = load_images(data_path, expected_height, expected_width, ids[1:]) + + if batch_size == -1: + batch_size = 1 + data = adjust_to_batch_size(np_array=data, batch_size=int(batch_size)) + if transpose_axes: + data = transpose_input(images=data, axes=transpose_axes) + + return data + +def is_dynamic_shape(shape): + return any(map(lambda x: x == -1, shape)) diff --git a/tests/functional/utils/other.py b/tests/functional/utils/other.py deleted file mode 100644 index 8c8ba14931..0000000000 --- a/tests/functional/utils/other.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -from collections import defaultdict - - -def get_server_fixtures_from_pytest_item(item): - server_fixtures = list(filter(lambda x: "start_server_" in x, item.fixturenames)) - return server_fixtures - -def reorder_items_by_fixtures_used(session): - """ - Reorder test items, group them by fixtures used - """ - - # Keep track how many tests use different container fixtures ('start_server_*') - server_fixtures_to_tests = defaultdict(lambda: []) - - # For each item (test case) collect used 'start_server_*' fixtures. - for test in session.items: - test._server_fixtures = get_server_fixtures_from_pytest_item(test) - if not test._server_fixtures: - server_fixtures_to_tests[''].append(test) - else: - for fixture in test._server_fixtures: - server_fixtures_to_tests[fixture].append(test) - session._server_fixtures_to_tests = server_fixtures_to_tests.copy() - - # Try to order test execution by minimal 'start_server_*' fixtures usage - ordered_tests = [] - - # Choose fixture with min tests assigned to be executed first. - number_of_tests_lambda = lambda x: len(x[1]) - fixture_with_min_number_of_cases = min(server_fixtures_to_tests.items(), key=number_of_tests_lambda)[0] - - # FIFO queue with processed fixtures - fixtures_working = [fixture_with_min_number_of_cases] - - while server_fixtures_to_tests: - current_fixture = fixtures_working[0] - for test in server_fixtures_to_tests[current_fixture]: - if test not in ordered_tests: - ordered_tests.append(test) - fixtures_used_by_test = get_server_fixtures_from_pytest_item(test) - - # Check all fixtures used by given test. - for fixture in fixtures_used_by_test: - if fixture not in fixtures_working: - # Test execute multiple fixtures, add it to queue, it to be processed next. - fixtures_working.append(fixture) - fixtures_working.remove(current_fixture) - del server_fixtures_to_tests[current_fixture] - - if server_fixtures_to_tests and not fixtures_working: - # If queue is empty add fixture with least tests (left). - fixtures_working.append(min(server_fixtures_to_tests.items(), key=number_of_tests_lambda)[0]) - - session.items = ordered_tests - return ordered_tests diff --git a/tests/functional/utils/ovms_testing_image/Dockerfile.redhat b/tests/functional/utils/ovms_testing_image/Dockerfile.redhat new file mode 100644 index 0000000000..6d89e5483c --- /dev/null +++ b/tests/functional/utils/ovms_testing_image/Dockerfile.redhat @@ -0,0 +1,188 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BASE_IMAGE=openvino-model-server:latest +FROM $BASE_IMAGE as base_image + +ARG ROOT_PATH_CPU_EXTENSIONS=/cpu_extensions +ARG OPENVINO_PATH=/opt/intel/openvino_2025 +ARG HEADER_FILE_PATH=customloaderinterface.hpp +ARG OPENCV_HEADER_FILE_PATH=/deps/opencv.hpp + +ARG OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Werror -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" +ARG NODES="east_ocr model_zoo_intel_object_detection image_transformation node_add_sub node_choose_maximum node_perform_different_operations node_dynamic_demultiplex demultiply demultiply_gather elastic_in_1t_out_1t" +ARG NODE_NAME=demultiply +ARG NODE_TYPE=cpp +ARG OPENCV_BRANCH=4.12.0 +ARG OV_PACKAGE=openvino_2025 +ARG DLDT_PACKAGE_URL=https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2026.2.0.0.dev20260331/openvino_genai_rhel8_2026.2.0.0.dev20260331_x86_64.tar.gz + +ENV OpenVINO_DIR=$OPENVINO_PATH/runtime/cmake +ENV TBB_DIR=/tmp/openvino_installer/oneapi-tbb-2021.13.0/lib/cmake/tbb + +USER root +# Prepare OS environment +RUN microdnf update -y && \ + microdnf install -y bzip2 cmake gcc-c++ git libcurl-devel make sudo tar unzip wget \ + && rm -rf /tmp/* + +# Install valgrind +WORKDIR / +RUN wget https://sourceware.org/pub/valgrind/valgrind-3.24.0.tar.bz2 +RUN tar xvf valgrind-3.24.0.tar.bz2 +WORKDIR /valgrind-3.24.0 +RUN ./configure +RUN make +RUN sudo make install + +# OV toolkit package +WORKDIR /opt/intel +RUN curl -L $DLDT_PACKAGE_URL --output $OV_PACKAGE.tgz +RUN tar -xf $OV_PACKAGE.tgz +RUN mv *openvino_genai* $OPENVINO_PATH +WORKDIR $OPENVINO_PATH + +# OpenCV +WORKDIR /opt +COPY opencv_cmake_flags.txt /opt +RUN rm -rf $OPENCV_BRANCH opencv_repo +RUN rm -rf $OPENCV_BRANCH opencv_contrib_repo +RUN git clone https://github.com/opencv/opencv.git --depth 1 -b $OPENCV_BRANCH opencv_repo +RUN git clone https://github.com/opencv/opencv_contrib.git --depth 1 -b $OPENCV_BRANCH opencv_contrib_repo +WORKDIR /opt/opencv_repo/build +RUN cmake $(cat /opt/opencv_cmake_flags.txt) /opt/opencv_repo && \ + make "-j$(nproc)" && \ + make install + +RUN /ovms/bin/ovms --version + +# Run benchmark app +WORKDIR $OPENVINO_PATH/samples/cpp +RUN ./build_samples.sh + +# Copy OV artifacts to build custom nodes and extensions +WORKDIR /ov +COPY ov /ov +WORKDIR /ov/ovms_basic/build +RUN cmake ../ovms +RUN cmake --build . + +WORKDIR /ov/ovms_reshape_model/build +RUN cmake ../ovms +RUN cmake --build . + +# Build cliloader +WORKDIR / +RUN git clone https://github.com/intel/opencl-intercept-layer/ +WORKDIR /opencl-intercept-layer/build +RUN cmake .. +RUN cmake --build . --config Debug --target cliloader +RUN cmake --build . --config RelWithDebInfo --target install + +### SimpleReluCpuExtension ### +WORKDIR $ROOT_PATH_CPU_EXTENSIONS/SampleCpuExtension +COPY SampleCpuExtension ./ +RUN make + +### CorruptedLibCpuExtension ### +WORKDIR $ROOT_PATH_CPU_EXTENSIONS/corrupted_lib +COPY SampleCpuExtension/Makefile corrupted_lib/CorruptedLib.cpp ./ +RUN sed -i "s|CustomReluOp.cpp|CorruptedLib.cpp|" Makefile +RUN sed -i "s|libcustom_relu_cpu_extension.so|libcorrupted_lib_cpu_extension.so|" Makefile +RUN sed -i "s|ov_extension.cpp||" Makefile +RUN make + +### ThrowExceptionCpuExtension ### +WORKDIR $ROOT_PATH_CPU_EXTENSIONS/throw_exceptions +COPY SampleCpuExtension/Makefile throw_exceptions/ThrowExceptions.cpp ./ +RUN sed -i "s|CustomReluOp.cpp|ThrowExceptions.cpp|" Makefile +RUN sed -i "s|libcustom_relu_cpu_extension.so|libthrow_exception_cpu_extension.so|" Makefile +RUN sed -i "s|ov_extension.cpp||" Makefile +RUN make + +### Custom nodes ### +WORKDIR /custom_nodes +COPY /custom_nodes /custom_nodes +COPY queue.hpp / +COPY custom_node_interface.h / +WORKDIR /deps +RUN cp /opt/opencv_repo/include/opencv2/opencv.hpp /deps/opencv.hpp +WORKDIR /custom_nodes/common +RUN g++ -c -std=c++17 *.cpp ${OPS} -I/opt/opencv/include/opencv4 + +# OvmsTestDevCustomNode - default cpp location: /custom_nodes// +# 1. east_ocr +WORKDIR /custom_nodes/east_ocr/ +RUN g++ -c -std=c++17 east_ocr.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/east_ocr/libcustom_node_east_ocr.so east_ocr.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 2. model_zoo_intel_object_detection +WORKDIR /custom_nodes/model_zoo_intel_object_detection/ +RUN g++ -c -std=c++17 model_zoo_intel_object_detection.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/model_zoo_intel_object_detection/libcustom_node_model_zoo_intel_object_detection.so model_zoo_intel_object_detection.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 3. image_transformation +WORKDIR /custom_nodes/image_transformation/ +RUN g++ -c -std=c++17 image_transformation.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/image_transformation/libcustom_node_image_transformation.so image_transformation.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + + +# OvmsTestDevCustomNode - default cpp location: /custom_nodes// +# 1. demultiply +WORKDIR /custom_nodes/demultiply/ +RUN g++ -c -std=c++17 demultiply.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/demultiply/libcustom_node_demultiply.so demultiply.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 2. demultiply_gather +WORKDIR /custom_nodes/demultiply_gather/ +RUN g++ -c -std=c++17 demultiply_gather.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/demultiply_gather/libcustom_node_demultiply_gather.so demultiply_gather.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 3. elastic_in_1t_out_1t +WORKDIR /custom_nodes/elastic_in_1t_out_1t/ +RUN g++ -c -std=c++17 elastic_in_1t_out_1t.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/elastic_in_1t_out_1t/libcustom_node_elastic_in_1t_out_1t.so elastic_in_1t_out_1t.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + + +# OvmsCUnitTestCustomNode - default cpp location: /custom_nodes/ +WORKDIR /custom_nodes +# 1. node_add_sub - NODE_TYPE=c +RUN g++ -c -std=c++17 node_add_sub.c ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/libcustom_node_node_add_sub.so node_add_sub.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 2. node_choose_maximum +RUN g++ -c -std=c++17 node_choose_maximum.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/libcustom_node_node_choose_maximum.so node_choose_maximum.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 3. node_perform_different_operations +RUN g++ -c -std=c++17 node_perform_different_operations.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/libcustom_node_node_perform_different_operations.so node_perform_different_operations.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 4. node_dynamic_demultiplex +RUN g++ -c -std=c++17 node_dynamic_demultiplex.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/libcustom_node_node_dynamic_demultiplex.so node_dynamic_demultiplex.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +ENTRYPOINT ["/ovms/bin/ovms"] diff --git a/tests/functional/utils/ovms_testing_image/Dockerfile.ubuntu b/tests/functional/utils/ovms_testing_image/Dockerfile.ubuntu new file mode 100644 index 0000000000..17d7c572ee --- /dev/null +++ b/tests/functional/utils/ovms_testing_image/Dockerfile.ubuntu @@ -0,0 +1,186 @@ +# +# INTEL CONFIDENTIAL +# Copyright (c) 2023-2025 Intel Corporation +# +# The source code contained or described herein and all documents related to +# the source code ("Material") are owned by Intel Corporation or its suppliers +# or licensors. Title to the Material remains with Intel Corporation or its +# suppliers and licensors. The Material contains trade secrets and proprietary +# and confidential information of Intel or its suppliers and licensors. The +# Material is protected by worldwide copyright and trade secret laws and treaty +# provisions. No part of the Material may be used, copied, reproduced, modified, +# published, uploaded, posted, transmitted, distributed, or disclosed in any way +# without Intel's prior express written permission. +# +# No license under any patent, copyright, trade secret or other intellectual +# property right is granted to or conferred upon you by disclosure or delivery +# of the Materials, either expressly, by implication, inducement, estoppel or +# otherwise. Any license under such intellectual property rights must be express +# and approved by Intel in writing. +# +ARG BASE_IMAGE=openvino/model_server:latest +FROM $BASE_IMAGE as base_image + +ARG ROOT_PATH_CPU_EXTENSIONS=/cpu_extensions +ARG OPENVINO_PATH=/opt/intel/openvino_2025 +ARG HEADER_FILE_PATH=customloaderinterface.hpp +ARG OPENCV_HEADER_FILE_PATH=/deps/opencv.hpp + +ARG OPS="-fpic -O2 -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -D_FORTIFY_SOURCE=1 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Werror -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security" +ARG NODES="east_ocr model_zoo_intel_object_detection image_transformation node_add_sub node_choose_maximum node_perform_different_operations node_dynamic_demultiplex demultiply demultiply_gather elastic_in_1t_out_1t" +ARG NODE_NAME=demultiply +ARG NODE_TYPE=cpp +ARG OPENCV_BRANCH=4.12.0 +ARG OV_PACKAGE=openvino_2025 +ARG DLDT_PACKAGE_URL=https://storage.openvinotoolkit.org/repositories/openvino_genai/packages/nightly/2026.2.0.0.dev20260323/openvino_genai_ubuntu24_2026.2.0.0.dev20260323_x86_64.tar.gz + +### Prevent any prompts during .deb installs ### +ENV DEBIAN_FRONTEND=noninteractive +ENV OpenVINO_DIR=$OPENVINO_PATH/runtime/cmake + +USER root +# Prepare OS environment +RUN apt-get update && apt-get install -y cmake curl g++ git make sudo tar unzip wget valgrind && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && rm -rf /tmp/* + +# OV toolkit package +WORKDIR /opt/intel +RUN curl -L $DLDT_PACKAGE_URL --output $OV_PACKAGE.tgz +RUN tar -xf $OV_PACKAGE.tgz +RUN mv *openvino_genai* $OPENVINO_PATH +WORKDIR $OPENVINO_PATH +RUN sudo -E ./install_dependencies/install_openvino_dependencies.sh -y +RUN ./setupvars.sh + +# OpenCV +WORKDIR /opt +COPY opencv_cmake_flags.txt /opt +RUN rm -rf $OPENCV_BRANCH opencv_repo +RUN rm -rf $OPENCV_BRANCH opencv_contrib_repo +RUN git clone http://github.com/opencv/opencv.git --depth 1 -b $OPENCV_BRANCH opencv_repo +RUN git clone http://github.com/opencv/opencv_contrib.git --depth 1 -b $OPENCV_BRANCH opencv_contrib_repo + +WORKDIR /opt/opencv_repo/build +RUN cmake $(cat /opt/opencv_cmake_flags.txt) /opt/opencv_repo && \ + make "-j$(nproc)" && \ + make install + +RUN /ovms/bin/ovms --version + +# Run benchmark app +WORKDIR $OPENVINO_PATH/samples/cpp +RUN ./build_samples.sh + +# Copy OV artifacts to build custom nodes and extensions +WORKDIR /ov +COPY ov /ov +WORKDIR /ov/ovms_basic/build +RUN cmake ../ovms +RUN cmake --build . + +WORKDIR /ov/ovms_reshape_model/build +RUN cmake ../ovms +RUN cmake --build . + +# Build cliloader +WORKDIR / +RUN git clone https://github.com/intel/opencl-intercept-layer/ +WORKDIR /opencl-intercept-layer/build +RUN cmake .. +RUN cmake --build . --config Debug --target cliloader +RUN cmake --build . --config RelWithDebInfo --target install + + +### SimpleReluCpuExtension ### +WORKDIR $ROOT_PATH_CPU_EXTENSIONS/SampleCpuExtension +COPY SampleCpuExtension ./ +RUN make + +### CorruptedLibCpuExtension ### +WORKDIR $ROOT_PATH_CPU_EXTENSIONS/corrupted_lib +COPY SampleCpuExtension/Makefile corrupted_lib/CorruptedLib.cpp ./ +RUN sed -i "s|CustomReluOp.cpp|CorruptedLib.cpp|" Makefile +RUN sed -i "s|libcustom_relu_cpu_extension.so|libcorrupted_lib_cpu_extension.so|" Makefile +RUN sed -i "s|ov_extension.cpp||" Makefile +RUN make + +### ThrowExceptionCpuExtension ### +WORKDIR $ROOT_PATH_CPU_EXTENSIONS/throw_exceptions +COPY SampleCpuExtension/Makefile throw_exceptions/ThrowExceptions.cpp ./ +RUN sed -i "s|CustomReluOp.cpp|ThrowExceptions.cpp|" Makefile +RUN sed -i "s|libcustom_relu_cpu_extension.so|libthrow_exception_cpu_extension.so|" Makefile +RUN sed -i "s|ov_extension.cpp||" Makefile +RUN make + +### Custom nodes ### +WORKDIR /custom_nodes +COPY /custom_nodes /custom_nodes +COPY queue.hpp / +COPY custom_node_interface.h / +WORKDIR /deps +RUN cp /opt/opencv_repo/include/opencv2/opencv.hpp /deps/opencv.hpp +WORKDIR /custom_nodes/common +RUN g++ -c -std=c++17 *.cpp ${OPS} -I/opt/opencv/include/opencv4 + +# OvmsTestDevCustomNode - default cpp location: /custom_nodes// +# 1. east_ocr +WORKDIR /custom_nodes/east_ocr/ +RUN g++ -c -std=c++17 east_ocr.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/east_ocr/libcustom_node_east_ocr.so east_ocr.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 2. model_zoo_intel_object_detection +WORKDIR /custom_nodes/model_zoo_intel_object_detection/ +RUN g++ -c -std=c++17 model_zoo_intel_object_detection.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/model_zoo_intel_object_detection/libcustom_node_model_zoo_intel_object_detection.so model_zoo_intel_object_detection.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 3. image_transformation +WORKDIR /custom_nodes/image_transformation/ +RUN g++ -c -std=c++17 image_transformation.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/image_transformation/libcustom_node_image_transformation.so image_transformation.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# OvmsTestDevCustomNode - default cpp location: /custom_nodes// +# 1. demultiply +WORKDIR /custom_nodes/demultiply/ +RUN g++ -c -std=c++17 demultiply.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/demultiply/libcustom_node_demultiply.so demultiply.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 2. demultiply_gather +WORKDIR /custom_nodes/demultiply_gather/ +RUN g++ -c -std=c++17 demultiply_gather.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/demultiply_gather/libcustom_node_demultiply_gather.so demultiply_gather.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 3. elastic_in_1t_out_1t +WORKDIR /custom_nodes/elastic_in_1t_out_1t/ +RUN g++ -c -std=c++17 elastic_in_1t_out_1t.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/elastic_in_1t_out_1t/libcustom_node_elastic_in_1t_out_1t.so elastic_in_1t_out_1t.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + + +# OvmsCUnitTestCustomNode - default cpp location: /custom_nodes/ +WORKDIR /custom_nodes +# 1. node_add_sub - NODE_TYPE=c +RUN g++ -c -std=c++17 node_add_sub.c ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/libcustom_node_node_add_sub.so node_add_sub.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 2. node_choose_maximum +RUN g++ -c -std=c++17 node_choose_maximum.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/libcustom_node_node_choose_maximum.so node_choose_maximum.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 3. node_perform_different_operations +RUN g++ -c -std=c++17 node_perform_different_operations.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/libcustom_node_node_perform_different_operations.so node_perform_different_operations.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +# 4. node_dynamic_demultiplex +RUN g++ -c -std=c++17 node_dynamic_demultiplex.${NODE_TYPE} ${OPS} -I/opt/opencv/include/opencv4 +RUN g++ -shared ${OPS} -o /custom_nodes/libcustom_node_node_dynamic_demultiplex.so node_dynamic_demultiplex.o /custom_nodes/common/*.o \ + -L/opt/opencv/lib/ -I/opt/opencv/include/opencv4 -lopencv_core -lopencv_imgproc -lopencv_imgcodecs + +ENTRYPOINT ["/ovms/bin/ovms"] diff --git a/tests/functional/utils/ovms_testing_image/cpu_extensions/corrupted_lib/CorruptedLib.cpp b/tests/functional/utils/ovms_testing_image/cpu_extensions/corrupted_lib/CorruptedLib.cpp new file mode 100644 index 0000000000..1ab27a2c8b --- /dev/null +++ b/tests/functional/utils/ovms_testing_image/cpu_extensions/corrupted_lib/CorruptedLib.cpp @@ -0,0 +1,41 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include + +#include +#include +#include + +//! [op:header] +namespace TemplateExtension { + +class CorruptedLib : public ov::op::Op { + +public: + OPENVINO_OP("Multiply", "opset1"); + CorruptedLib() = default; + CorruptedLib(const ov::Output& arg) : + Op({arg}) { + } + + bool evaluate(ov::TensorVector& outputs, const ov::TensorVector& inputs) const override { + std::cout << "Executing CorruptedLib evaluate()" << std::endl; + return true; + } +}; // class CorruptedLib +} // namespace TemplateExtension diff --git a/tests/functional/utils/ovms_testing_image/cpu_extensions/throw_exceptions/ThrowExceptions.cpp b/tests/functional/utils/ovms_testing_image/cpu_extensions/throw_exceptions/ThrowExceptions.cpp new file mode 100644 index 0000000000..d53418f7d7 --- /dev/null +++ b/tests/functional/utils/ovms_testing_image/cpu_extensions/throw_exceptions/ThrowExceptions.cpp @@ -0,0 +1,85 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include + +#include +#include +#include + +//! [op:header] +namespace TemplateExtension { + +class ThrowExceptions : public ov::op::Op { + +public: + OPENVINO_OP("Multiply", "opset1"); + /** + NOTE: + This extension was written against Resnet50-Binary model. + Intention is to "hijack" all layers with type="Multiply" (Our method 'evaluate(...)' will be called instead). + We use "Multiply" because it is first layer type that use input tensor values applied to model. + Whole purpose of this extension is to use insted "" from resnet50-binary-0001.xml: + + + ... // id="0" - input parameter layer. + ... + + ... + + **/ + ThrowExceptions() = default; + ThrowExceptions(const ov::Output& arg) : + Op({arg}) { + constructor_validate_and_infer_types(); + std::cout << "ThrowExceptions() constructor call"; + } + + void validate_and_infer_types() { + set_output_type(0, get_input_element_type(0), get_input_partial_shape(0)); + } + + std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override { + return std::make_shared(new_args.at(0)); + } + + bool visit_attributes(ov::AttributeVisitor& visitor) override { + return true; + } + + bool evaluate(ov::TensorVector& outputs, const ov::TensorVector& inputs) const override { + std::cout << "Executing ThrowExceptions evaluate()" << std::endl; + auto in = inputs[0]; + const float* in_data = in.data(); + switch(int(in_data[0])) { + case SIGFPE: // DIV by ZERO + auto foo = 0; + std::cout << "SIGFPE " << std::endl; + auto x = 0xDEADF00D / foo; // Goodbye cruel world. + } + return true; + } + bool has_evaluate() const override { + return true; + } +}; +//! [op:header] + +} // namespace TemplateExtension + +OPENVINO_CREATE_EXTENSIONS( + std::vector({std::make_shared>()})); diff --git a/tests/functional/utils/ovms_testing_image/custom_nodes/demultiply/demultiply.cpp b/tests/functional/utils/ovms_testing_image/custom_nodes/demultiply/demultiply.cpp new file mode 100644 index 0000000000..24b63cea46 --- /dev/null +++ b/tests/functional/utils/ovms_testing_image/custom_nodes/demultiply/demultiply.cpp @@ -0,0 +1,169 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include +#include + +#include "../../custom_node_interface.h" +#include "opencv2/opencv.hpp" + +#define NODE_ASSERT(cond, msg) \ + if (!(cond)) { \ + std::cout << "[" << __LINE__ << "] Assert: " << msg << std::endl; \ + return 1; \ + } + +#define LOG_ENTRY_M(method) std::cout << "[CustomNode] [" << __LINE__ << "] demultiply." << method << " entry" << std::endl; +#define LOG_EXIT_M(method) std::cout << "[CustomNode] [" << __LINE__ << "] demultiply." << method << " exit" << std::endl; + +static constexpr const char* IMAGE_TENSOR_NAME = "tensor"; +static constexpr const char* TENSOR_OUT = "tensor_out"; +const uint64_t NEW_LAYER_DIM = 3; +const uint64_t BATCH_SIZE = 1; +const uint64_t NR_OF_CHANNELS = 3; + +bool prepare_output_tensor(struct CustomNodeTensor** outputs, int* outputsCount, int img_height, int img_width) +{ + bool result = true; + *outputsCount = 1; + *outputs = (struct CustomNodeTensor*)malloc(*outputsCount * sizeof(CustomNodeTensor)); + NODE_ASSERT((*outputs) != nullptr, "malloc has failed"); + CustomNodeTensor& tensor = (*outputs)[0]; + uint64_t byte_size = sizeof(float) * NEW_LAYER_DIM * BATCH_SIZE * NR_OF_CHANNELS * img_height * img_width; + + float* buffer; + tensor.name = TENSOR_OUT; + buffer = (float*)malloc(byte_size); + NODE_ASSERT(buffer != nullptr, "malloc has failed"); + if(buffer == nullptr) { + result = false; + } else { + tensor.data = reinterpret_cast(buffer); + tensor.dataBytes = byte_size; + tensor.dimsCount = 5; + tensor.dims = (uint64_t*)malloc(tensor.dimsCount * sizeof(uint64_t)); + NODE_ASSERT(tensor.dims != nullptr, "malloc has failed"); + int dims_content[] = {NEW_LAYER_DIM, BATCH_SIZE, NR_OF_CHANNELS, img_height, img_width}; + for(uint64_t i = 0; i < tensor.dimsCount; i++) + tensor.dims[(int)i] = dims_content[(int)i]; + tensor.precision = FP32; + } + return result; +} + +void cleanup(CustomNodeTensor& tensor) { + free(tensor.data); + free(tensor.dims); +} + +int execute(const struct CustomNodeTensor* inputs, + int inputsCount, + struct CustomNodeTensor** outputs, + int* outputsCount, + const struct CustomNodeParam* params, + int paramsCount, + void* customNodeLibraryInternalManager){ + LOG_ENTRY_M("execute enter"); + int dimsCount = inputs[0].dimsCount; + int img_height = inputs[0].dims[dimsCount - 2]; + int img_width = inputs[0].dims[dimsCount - 1]; + std::cout << "demultiply load img_height: " << img_height << "; img_width: " << img_width << std::endl; + prepare_output_tensor(outputs, outputsCount, img_height, img_width); + LOG_EXIT_M("execute"); + return 0; +} + +int get_int_parameter(const std::string& name, const struct CustomNodeParam* params, int paramsCount, int defaultValue) { + int result = defaultValue; + for (int i = 0; i < paramsCount; i++) { + if (name == params[i].key) { + try { + result = std::stoi(params[i].value); + break; + } catch (std::invalid_argument& e) { + result = defaultValue; + } catch (std::out_of_range& e) { + result = defaultValue; + } + } + } + return result; +} + +int getInputsInfo(struct CustomNodeTensorInfo** info, + int* infoCount, + const struct CustomNodeParam* params, + int paramsCount, + void* customNodeLibraryInternalManager){ + LOG_ENTRY_M("getInputsInfo"); + *infoCount = 1; + *info = (struct CustomNodeTensorInfo*)malloc(*infoCount * sizeof(struct CustomNodeTensorInfo)); + NODE_ASSERT((*info) != nullptr, "malloc has failed"); + (*info)[0].name = IMAGE_TENSOR_NAME; + (*info)[0].dimsCount = 4; + (*info)[0].dims = (uint64_t*)malloc((*info)[0].dimsCount * sizeof(uint64_t)); + NODE_ASSERT(((*info)[0].dims) != nullptr, "malloc has failed"); + (*info)[0].dims[0] = 1; + (*info)[0].dims[1] = 3; + (*info)[0].dims[2] = 224; + (*info)[0].dims[3] = 224; + (*info)[0].precision = FP32; + LOG_EXIT_M("getInputsInfo"); + return 0; +} + +int getOutputsInfo(struct CustomNodeTensorInfo** info, + int* infoCount, + const struct CustomNodeParam* params, + int paramsCount, + void* customNodeLibraryInternalManager){ + LOG_ENTRY_M("getOutputsInfo"); + *infoCount = 1; + *info = (struct CustomNodeTensorInfo*)malloc(*infoCount * sizeof(struct CustomNodeTensorInfo)); + NODE_ASSERT((*info) != nullptr, "malloc has failed"); + (*info)[0].name = TENSOR_OUT; + (*info)[0].dimsCount = 5; + (*info)[0].dims = (uint64_t*)malloc((*info)[0].dimsCount * sizeof(uint64_t)); + NODE_ASSERT(((*info)[0].dims) != nullptr, "malloc has failed"); + (*info)[0].dims[0] = get_int_parameter("demultiply_size", params, paramsCount, 0); + (*info)[0].dims[1] = 1; + (*info)[0].dims[2] = 3; + (*info)[0].dims[3] = 224; + (*info)[0].dims[4] = 224; + (*info)[0].precision = FP32; + std::cout << "[CustomNode] [" << __LINE__ << "] demultiply.getOutputsInfo " << TENSOR_OUT << std::endl; + std::cout << " ["<<(*info)[0].dims[0]<<", "<<(*info)[0].dims[1]<<", "<<(*info)[0].dims[2]<<", "<<(*info)[0].dims[3]<<", "<<(*info)[0].dims[3]<<"]"<< std::endl; + LOG_EXIT_M("getOutputsInfo"); + return 0; +} + +int release(void* ptr, void* customNodeLibraryInternalManager) +{ + LOG_ENTRY_M("release"); + free(ptr); + LOG_EXIT_M("release"); + return 0; +} + +int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { + return 0; +} + +int deinitialize(void* customNodeLibraryInternalManager) { + return 0; +} diff --git a/tests/functional/utils/ovms_testing_image/custom_nodes/demultiply_gather/demultiply_gather.cpp b/tests/functional/utils/ovms_testing_image/custom_nodes/demultiply_gather/demultiply_gather.cpp new file mode 100644 index 0000000000..e36154df1d --- /dev/null +++ b/tests/functional/utils/ovms_testing_image/custom_nodes/demultiply_gather/demultiply_gather.cpp @@ -0,0 +1,155 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include + +#include "../../custom_node_interface.h" +#include "opencv2/opencv.hpp" + +#define NODE_ASSERT(cond, msg) \ + if (!(cond)) { \ + std::cout << "[" << __LINE__ << "] Assert: " << msg << std::endl; \ + return 1; \ + } + + +static constexpr const char* IMAGE_TENSOR_NAME = "tensor"; +static constexpr const char* TENSOR_OUT = "tensor_out"; +const uint64_t DIM_0 = 4; +const uint64_t DIM_1 = 4; +const uint64_t DIM_2 = 1; +const uint64_t DIM_3 = 10; + +bool prepare_output_tensor(struct CustomNodeTensor** outputs, int* outputsCount) +{ + bool result = true; + *outputsCount = 1; + *outputs = (struct CustomNodeTensor*)malloc(*outputsCount * sizeof(CustomNodeTensor)); + NODE_ASSERT((*outputs) != nullptr, "malloc has failed"); + CustomNodeTensor& tensor = (*outputs)[0]; + uint64_t byte_size = sizeof(float) * DIM_0 * DIM_1 * DIM_2 * DIM_3; + + float* buffer; + tensor.name = TENSOR_OUT; + buffer = (float*)malloc(byte_size); + NODE_ASSERT(buffer != nullptr, "malloc has failed"); + if(buffer == nullptr) { + result = false; + } else { + tensor.data = reinterpret_cast(buffer); + tensor.dataBytes = byte_size; + tensor.dimsCount = 4; + tensor.dims = (uint64_t*)malloc(tensor.dimsCount * sizeof(uint64_t)); + NODE_ASSERT(tensor.dims != nullptr, "malloc has failed"); + tensor.dims[0] = DIM_0; + tensor.dims[1] = DIM_1; + tensor.dims[2] = DIM_2; + tensor.dims[3] = DIM_3; + tensor.precision = FP32; + } + return result; +} + +void cleanup(CustomNodeTensor& tensor) { + free(tensor.data); + free(tensor.dims); +} + +int execute(const struct CustomNodeTensor* inputs, + int inputsCount, + struct CustomNodeTensor** outputs, + int* outputsCount, + const struct CustomNodeParam* params, + int paramsCount, + void* customNodeLibraryInternalManager) +{ + prepare_output_tensor(outputs, outputsCount); + return 0; +} + +int get_int_parameter(const std::string& name, const struct CustomNodeParam* params, int paramsCount, int defaultValue) { + int result = defaultValue; + for (int i = 0; i < paramsCount; i++) { + if (name == params[i].key) { + try { + result = std::stoi(params[i].value); + break; + } catch (std::invalid_argument& e) { + result = defaultValue; + } catch (std::out_of_range& e) { + result = defaultValue; + } + } + } + return result; +} + +int getInputsInfo(struct CustomNodeTensorInfo** info, + int* infoCount, + const struct CustomNodeParam* params, + int paramsCount, + void* customNodeLibraryInternalManager) +{ + *infoCount = 1; + *info = (struct CustomNodeTensorInfo*)malloc(*infoCount * sizeof(struct CustomNodeTensorInfo)); + NODE_ASSERT((*info) != nullptr, "malloc has failed"); + (*info)[0].name = IMAGE_TENSOR_NAME; + (*info)[0].dimsCount = 3; + (*info)[0].dims = (uint64_t*)malloc((*info)[0].dimsCount * sizeof(uint64_t)); + NODE_ASSERT(((*info)[0].dims) != nullptr, "malloc has failed"); + (*info)[0].dims[0] = DIM_1; + (*info)[0].dims[1] = DIM_2; + (*info)[0].dims[2] = DIM_3; + (*info)[0].precision = FP32; + return 0; +} + +int getOutputsInfo(struct CustomNodeTensorInfo** info, + int* infoCount, + const struct CustomNodeParam* params, + int paramsCount, + void* customNodeLibraryInternalManager) +{ + *infoCount = 1; + *info = (struct CustomNodeTensorInfo*)malloc(*infoCount * sizeof(struct CustomNodeTensorInfo)); + NODE_ASSERT((*info) != nullptr, "malloc has failed"); + (*info)[0].name = TENSOR_OUT; + (*info)[0].dimsCount = 4; + (*info)[0].dims = (uint64_t*)malloc((*info)[0].dimsCount * sizeof(uint64_t)); + NODE_ASSERT(((*info)[0].dims) != nullptr, "malloc has failed"); + (*info)[0].dims[0] = get_int_parameter("demultiply_size", params, paramsCount, 0); + (*info)[0].dims[1] = DIM_1; + (*info)[0].dims[2] = DIM_2; + (*info)[0].dims[3] = DIM_3; + (*info)[0].precision = FP32; + return 0; +} + +int release(void* ptr, void* customNodeLibraryInternalManager) +{ + free(ptr); + return 0; +} + +int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { + return 0; +} + +int deinitialize(void* customNodeLibraryInternalManager) { + return 0; +} diff --git a/tests/functional/utils/ovms_testing_image/custom_nodes/elastic_in_1t_out_1t/elastic_in_1t_out_1t.cpp b/tests/functional/utils/ovms_testing_image/custom_nodes/elastic_in_1t_out_1t/elastic_in_1t_out_1t.cpp new file mode 100644 index 0000000000..0f3f34ba0c --- /dev/null +++ b/tests/functional/utils/ovms_testing_image/custom_nodes/elastic_in_1t_out_1t/elastic_in_1t_out_1t.cpp @@ -0,0 +1,202 @@ +//***************************************************************************** +// Copyright 2021 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include + +#include "../../custom_node_interface.h" +#include "opencv2/opencv.hpp" + +#define NODE_ASSERT(cond, msg) \ + if (!(cond)) { \ + std::cout << "[" << __LINE__ << "] Assert: " << msg << std::endl; \ + return 1; \ + } + +#define LOG_PREFIX "[CustomNode] [" << __LINE__ << "] elastic: " + +#ifdef LOG +#define LOG_PREFIX "[CustomNode] [" << __LINE__ << "] elastic: " +#define LOG_ENTRY_M(method) std::cout << LOG_PREFIX << "Method: " << method << " entry" << std::endl; +#define LOG_EXIT_M(method) std::cout << LOG_PREFIX << "Method: " << method << " exit" << std::endl; +#else +#define LOG_ENTRY_M(method) +; +#define LOG_EXIT_M(method) ; +#endif + +static constexpr const char* TENSOR_IN = "tensor_in"; +static constexpr const char* TENSOR_OUT = "tensor_out"; + +const char* get_param_value(const std::string& param_name, const struct CustomNodeParam* params, int params_count, const char* default_value) { + LOG_ENTRY_M("get_param_value"); + const char* result = default_value; + for (int i = 0; i < params_count; i++) { + if(param_name == params[i].key) { + result = params[i].value; + break; + } + } + std::cout << LOG_PREFIX << "name: " << param_name << "; value: " << result << std::endl; + LOG_EXIT_M("get_param_value"); + return result; +} + +std::vector parse_tokens(const char* str, std::smatch &match, std::regex reg) +{ + std::vector result; + + std::string input_copy(str); + while(std::regex_search(input_copy, match, reg)) + { + result.push_back(match[1].str()); + input_copy = match.suffix().str(); + } + return result; +} + +std::vector parse_shape(const char* msg) +{ + std::vector result; + std::smatch match; + const std::regex dim_re_value("(\\d+)"); + + std::vector dim_string_value = parse_tokens(msg, match, dim_re_value); + for(size_t i = 0; i < dim_string_value.size(); i++){ + int dim_size = std::stoi(dim_string_value[(int)i]); + result.push_back(dim_size); + } + return result; +} + +int get_tensor_info(struct CustomNodeTensorInfo** info, + int* info_count, + const CustomNodeParam* params, + int params_count, + const char* param_name, + const char* default_value, + const char* tensor_name) +{ + LOG_ENTRY_M("get_tensor_info"); + const char* shape_str = get_param_value(param_name, params, params_count, default_value); + std::vector shape = parse_shape(shape_str); + + *info_count = 1; + *info = (struct CustomNodeTensorInfo*)malloc(*info_count * sizeof(struct CustomNodeTensorInfo)); + NODE_ASSERT((*info) != nullptr, "malloc has failed"); + std::cout << LOG_PREFIX << "allocate info: "<< info << std::endl; + + CustomNodeTensorInfo& tensor = (*info)[0]; + tensor.name = tensor_name; + tensor.dimsCount = static_cast(shape.size()); + tensor.dims = (uint64_t*)malloc(tensor.dimsCount * sizeof(uint64_t)); + NODE_ASSERT((tensor.dims) != nullptr, "malloc has failed"); + std::cout << LOG_PREFIX << "allocate dims: "<< tensor.dims << std::endl; + + for(int i = 0; i < (int)tensor.dimsCount; i++) + tensor.dims[i] = shape[i]; + tensor.precision = FP32; + + LOG_EXIT_M("get_tensor_info"); + return 0; +} + +void release_tensor_info(struct CustomNodeTensorInfo* info, int info_count) +{ + LOG_ENTRY_M("release_tensor_info"); + for(int i = 0; i < info_count; i ++) + free(info[i].dims); + free(info); + LOG_EXIT_M("release_tensor_info"); +} + +int execute(const struct CustomNodeTensor* inputs, + int inputsCount, + struct CustomNodeTensor** outputs, + int* outputsCount, + const struct CustomNodeParam* params, + int paramsCount, + void* customNodeLibraryInternalManager){ + LOG_ENTRY_M("execute"); + struct CustomNodeTensorInfo* info = NULL; + int result = get_tensor_info(&info, outputsCount, params, paramsCount, "output_shape", "[10, 10]", TENSOR_OUT); + + *outputs = (struct CustomNodeTensor*)malloc(*outputsCount * sizeof(CustomNodeTensor)); + NODE_ASSERT((*outputs) != nullptr, "malloc has failed"); + std::cout << LOG_PREFIX << "allocate outputs: "<< outputs << std::endl; + + CustomNodeTensor& tensor = (*outputs)[0]; + uint32_t byte_size = sizeof(uint32_t); + for(int i = 0; i < (int)info[0].dimsCount; i++) + byte_size *= (uint32_t)info[0].dims[i]; + tensor.name = TENSOR_OUT; + + float* buffer = (float*)malloc(byte_size); + NODE_ASSERT(buffer != nullptr, "malloc has failed"); + std::cout << LOG_PREFIX << "allocate data: " << buffer << std::endl; + + tensor.data = reinterpret_cast(buffer); + tensor.dataBytes = byte_size; + tensor.dimsCount = info[0].dimsCount; + tensor.dims = (uint64_t*)malloc(tensor.dimsCount * sizeof(uint64_t)); + NODE_ASSERT(tensor.dims != nullptr, "malloc has failed"); + std::cout << LOG_PREFIX << "allocate shape: " << tensor.dims << std::endl; + for(uint64_t i = 0; i < info[0].dimsCount; i++) + tensor.dims[i] = info[0].dims[(int)i]; + tensor.precision = info[0].precision; + + release_tensor_info(info, *outputsCount); + LOG_EXIT_M("execute"); + return result; +} + +int getInputsInfo(struct CustomNodeTensorInfo** info, + int* infoCount, + const struct CustomNodeParam* params, + int paramsCount, + void* customNodeLibraryInternalManager){ + LOG_ENTRY_M("getInputsInfo"); + int result = get_tensor_info(info, infoCount, params, paramsCount, "input_shape", "[1, 10]", TENSOR_IN); + LOG_EXIT_M("getInputsInfo"); + return result; +} + +int getOutputsInfo(struct CustomNodeTensorInfo** info, + int* infoCount, + const struct CustomNodeParam* params, + int paramsCount, + void* customNodeLibraryInternalManager){ + LOG_ENTRY_M("getOutputsInfo"); + int result = get_tensor_info(info, infoCount, params, paramsCount, "output_shape", "[1, 10]", TENSOR_OUT); + LOG_EXIT_M("getOutputsInfo"); + return result; +} + +int release(void* ptr, void* customNodeLibraryInternalManager) +{ + std::cout << LOG_PREFIX << "release(" << ptr << ")" << std::endl; + free(ptr); + return 0; +} + +int initialize(void** customNodeLibraryInternalManager, const struct CustomNodeParam* params, int paramsCount) { + return 0; +} + +int deinitialize(void* customNodeLibraryInternalManager) { + return 0; +} diff --git a/tests/functional/utils/files_operation.py b/tests/functional/utils/ovms_testing_image/ov/ovms_basic/ovms/CMakeLists.txt similarity index 58% rename from tests/functional/utils/files_operation.py rename to tests/functional/utils/ovms_testing_image/ov/ovms_basic/ovms/CMakeLists.txt index 817066fe6d..faf60e859c 100644 --- a/tests/functional/utils/files_operation.py +++ b/tests/functional/utils/ovms_testing_image/ov/ovms_basic/ovms/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 Intel Corporation +# Copyright (c) 2026 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,17 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os +cmake_minimum_required(VERSION 3.10) +set(CMAKE_CXX_STANDARD 11) -def get_path_friendly_test_name(location=None): - if location: - test_case = location[2].replace(".", "_") - else: - test_case = os.environ.get('PYTEST_CURRENT_TEST', "") - if test_case: - test_case = test_case.split(' ')[0].split("::") - test_case = "_".join(test_case[1:]) - return test_case +find_package(OpenVINO REQUIRED) +add_executable(test_ov_basic src/ov.cpp) +target_link_libraries(test_ov_basic PRIVATE openvino::runtime) diff --git a/tests/functional/utils/ovms_testing_image/ov/ovms_basic/ovms/src/ov.cpp b/tests/functional/utils/ovms_testing_image/ov/ovms_basic/ovms/src/ov.cpp new file mode 100644 index 0000000000..4974870fcc --- /dev/null +++ b/tests/functional/utils/ovms_testing_image/ov/ovms_basic/ovms/src/ov.cpp @@ -0,0 +1,51 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include +#include + +#include + +int main(int argc, char *argv[]){ + ov::Core ieCore; + auto model = ieCore.read_model(argv[1]); + + // Print info of first input layer + std::cout << model->input(0).get_partial_shape() << "\n"; + + // Compile model + ov::CompiledModel compiledModel = ieCore.compile_model(model, argv[2]); + + // Create an inference request + ov::InferRequest infer_request = compiledModel.create_infer_request(); + + // Get input port for model with one input + auto input_port = compiledModel.input(); + + // Create tensor from external memory + ov::Tensor input_tensor(input_port.get_element_type(), input_port.get_shape()); + + // Set input tensor for model with one input + infer_request.set_input_tensor(input_tensor); + infer_request.start_async(); + infer_request.wait(); + + std::this_thread::sleep_for(std::chrono::seconds(1)); + compiledModel = {}; + std::cout << "Hello\n" << std::flush; +} diff --git a/tests/functional/utils/ovms_testing_image/ov/ovms_reshape_model/ovms/CMakeLists.txt b/tests/functional/utils/ovms_testing_image/ov/ovms_reshape_model/ovms/CMakeLists.txt new file mode 100644 index 0000000000..e1e14c2019 --- /dev/null +++ b/tests/functional/utils/ovms_testing_image/ov/ovms_reshape_model/ovms/CMakeLists.txt @@ -0,0 +1,24 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +cmake_minimum_required(VERSION 3.10) +set(CMAKE_CXX_STANDARD 11) + +find_package(OpenVINO REQUIRED) + +add_executable(test_ov_reshape src/ov_reshape_model.cpp) + +target_link_libraries(test_ov_reshape PRIVATE openvino::runtime) diff --git a/tests/functional/utils/ovms_testing_image/ov/ovms_reshape_model/ovms/src/ov_reshape_model.cpp b/tests/functional/utils/ovms_testing_image/ov/ovms_reshape_model/ovms/src/ov_reshape_model.cpp new file mode 100644 index 0000000000..e5b86e7e77 --- /dev/null +++ b/tests/functional/utils/ovms_testing_image/ov/ovms_reshape_model/ovms/src/ov_reshape_model.cpp @@ -0,0 +1,50 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include +#include + +#include + +int main(int argc, char *argv[]){ + ov::Core ieCore; + auto model = ieCore.read_model(argv[1]); + + // Reshape model + model->reshape({0, 3, 300, 300}); + + ov::CompiledModel compiledModel = ieCore.compile_model(model, argv[2]); + + // Create an inference request + ov::InferRequest infer_request = compiledModel.create_infer_request(); + + // Get input port for model with one input + auto input_port = compiledModel.input(); + + // Create tensor from external memory + ov::Tensor input_tensor(input_port.get_element_type(), {0, 3, 300, 300}); + + // Set input tensor for model with one input + infer_request.set_input_tensor(input_tensor); + infer_request.start_async(); + infer_request.wait(); + + std::this_thread::sleep_for(std::chrono::seconds(1)); + compiledModel = {}; + std::cout << "Hello\n" << std::flush; +} diff --git a/tests/functional/utils/parametrization.py b/tests/functional/utils/parametrization.py deleted file mode 100644 index 878a3c8b11..0000000000 --- a/tests/functional/utils/parametrization.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2020 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import re -import socket -import logging -from datetime import datetime - -from tests.functional.utils.helpers import SingletonMeta - -logger = logging.getLogger(__name__) - - -class TestsSuffix(metaclass=SingletonMeta): - string = None - - -def get_tests_suffix(): - tests_suffix = TestsSuffix() - if not tests_suffix.string: - tests_suffix.string = os.environ.get("TESTS_SUFFIX", generate_test_object_name(prefix="suffix")) - return tests_suffix.string - - -class Suffix(metaclass=SingletonMeta): - index = 0 - - -class ObjectName: - _date_format, _time_format, _ms_format = "%d", "%H%M%S", "%f" - _NON_ALPHA_NUM = r'[^a-z0-9]+' - - def __init__(self, short=False, prefix=None, separator='_'): - worker_id = os.environ.get("PYTEST_XDIST_WORKER", "") - hostname = socket.gethostname().split(".", 1)[0].lower() - self._prefix = prefix if prefix else hostname - self._prefix = "{}{}".format(worker_id, self._prefix) - self._separator = separator - self._short = short - self._now = datetime.now() - - def __str__(self): - separator = '' if self._short else self._separator - parts = [self._prefix] + self.stem - name = separator.join(parts).lower() - return re.sub(self._NON_ALPHA_NUM, separator, name) - - @property - def stem(self) -> list: - seed = [ - self._now.strftime(self._date_format), - self._now.strftime(self._time_format), - self._now.strftime(self._ms_format) - ] - return seed[:2] if self._short else seed - - def build(self) -> str: - return str(self) - - -def generate_test_object_name(short=False, prefix=None, separator="_"): - name = ObjectName(short=short, prefix=prefix, separator=separator) - return name.build() diff --git a/tests/functional/utils/port_manager.py b/tests/functional/utils/port_manager.py index b9df4b5647..c3c20ddd02 100644 --- a/tests/functional/utils/port_manager.py +++ b/tests/functional/utils/port_manager.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Intel Corporation +# Copyright (c) 2026 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,21 +13,28 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import logging -import socket + import errno +import psutil +import socket +import threading +from tests.functional.utils.core import NamedSingletonMeta +from tests.functional.utils.logger import get_logger from tests.functional.utils.helpers import get_xdist_worker_count, get_xdist_worker_nr +from tests.functional.constants.os_type import OsType, get_host_os + +logger = get_logger(__name__) -logger = logging.getLogger(__name__) -class PortManager(): +class PortManager(metaclass=NamedSingletonMeta): + _thread_lock = threading.Lock() def __init__(self, name: str, starting_port: int = None, pool_size: int = None): self.name = name - assert starting_port is not None, "Lack of starting port while creating instance {} of PortManager".format(name) - assert pool_size is not None, "Lack of pool size while creating instance {} of PortManager".format(name) - assert pool_size > 0, "Not expected pool size given for manager {}, should be > 0.".format(name) + assert starting_port is not None, f"Lack of starting port while creating instance {name} of PortManager" + assert pool_size is not None, f"Lack of pool size while creating instance {name} of PortManager" + assert pool_size > 0, f"Not expected pool size given for manager {name}, should be > 0." self.xdist_worker_count = get_xdist_worker_count() self.xdist_current_worker = get_xdist_worker_nr() @@ -37,52 +44,73 @@ def __init__(self, name: str, starting_port: int = None, pool_size: int = None): self.reserved_ports = [] self.allowed_ports = list(range(self.starting_port, self.starting_port + self.pool_size)) + self.check_allowed_ports() - def get_port(self): - logger.debug("Getting port for Port Manager: {}\nallowed ports: {}\nreserved ports: {}" - .format(self.name, - ", ".join([str(port) for port in self.allowed_ports]), - ", ".join([str(port) for port in self.reserved_ports]))) - for port in self.allowed_ports[:]: - generated_port = self.reserve_port(port=port) - logger.debug("Generated port for Port Manager {}: {}".format(self.name, generated_port)) - if generated_port: - logger.debug("Reserved port for Port Manager {}: {}".format(self.name, generated_port)) - return generated_port + def check_allowed_ports(self): + unavailable_ports = [] + if get_host_os() == OsType.Windows: + for port in self.allowed_ports: + for conn in psutil.net_connections(): + if conn.laddr.port == port: + unavailable_ports.append(port) + break else: - raise Exception("Ports pool {} has been used up. " - "Consider release ports or increase pool size.".format(self.name)) + for port in self.allowed_ports: + try: + sock = socket.socket() + sock.bind(('', port)) + sock.close() + except socket.error as exc: + if exc.errno != errno.EADDRINUSE: + raise Exception(f"Not expected exception found in port manager {self.name}: {exc}") + unavailable_ports.append(port) + if unavailable_ports: + logger.warning(f"Unavailable ports found: {unavailable_ports}") - def reserve_port(self, port): - try: - sock = socket.socket() - sock.bind(('', port)) - sock.close() - self.reserved_ports.append(port) - self.allowed_ports.remove(port) - return port + def get_port(self): + with self._thread_lock: + logger.debug( + f"Getting port for Port Manager: {self.name}\n" + f"allowed ports: {', '.join([str(port) for port in self.allowed_ports])}\n" + f"reserved ports: {', '.join([str(port) for port in self.reserved_ports])}" + ) + for port in self.allowed_ports[:]: + generated_port = self.reserve_port(port=port) + logger.debug(f"Generated port for Port Manager {self.name}: {generated_port}") + if generated_port: + logger.debug(f"Reserved port for Port Manager {self.name}: {generated_port}") + return generated_port + else: + raise Exception(f"Ports pool {self.name} has been used up. " + "Consider releasing ports or increase the pool size.") - except socket.error as e: - if e.errno != errno.EADDRINUSE: - raise Exception("Not expected exception found in port manager {}: {}".format(self.name, e)) + def reserve_port(self, port): + if get_host_os() == OsType.Windows: + for conn in psutil.net_connections(): + if conn.laddr.port == port: + break + else: + self.reserved_ports.append(port) + self.allowed_ports.remove(port) + return port + else: + try: + sock = socket.socket() + sock.bind(('', port)) + sock.close() + self.reserved_ports.append(port) + self.allowed_ports.remove(port) + return port + except socket.error as exc: + if exc.errno != errno.EADDRINUSE: + raise Exception(f"Not expected exception found in port manager {self.name}: {exc}") self.allowed_ports.remove(port) return 0 def release_port(self, port: int): - logger.debug("Releasing port for Port Manager: {}\nport to release: {}\nallowed ports: {}\nreserved ports: {}" - .format(self.name, port, - ", ".join([str(port) for port in self.allowed_ports]), - ", ".join([str(port) for port in self.reserved_ports]))) - try: - sock = socket.socket() - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(('', port)) - sock.close() - except socket.error as e: - if e.errno == errno.EADDRINUSE: - raise Exception("Address has not been deleted for port manager {}".format(self.name)) - else: - raise Exception("Not expected exception found in port manager {}: {}".format(self.name, e)) - self.reserved_ports.remove(port) - self.allowed_ports.append(port) + logger.debug(f"Releasing port {port} from Port Manager:") + + with self._thread_lock: + self.reserved_ports.remove(port) + self.allowed_ports.append(port) diff --git a/tests/functional/utils/process.py b/tests/functional/utils/process.py index 6fb050e19e..66c9c95047 100644 --- a/tests/functional/utils/process.py +++ b/tests/functional/utils/process.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Intel Corporation +# Copyright (c) 2026 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,17 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import logging + from abc import ABC, abstractmethod from datetime import datetime, timedelta -from paramiko import SSHClient, WarningPolicy -import psutil +from pathlib import Path from queue import Queue -from subprocess import Popen, PIPE, TimeoutExpired, check_call, call, DEVNULL +from subprocess import DEVNULL, PIPE, Popen, TimeoutExpired, call, check_call from threading import Thread from time import sleep -logger = logging.getLogger(__name__) +import os +import psutil +from paramiko import SSHClient, WarningPolicy + +from tests.functional.utils.assertions import OvmsTestException +from tests.functional.utils.logger import LoggerType, get_logger +from tests.functional.constants.os_type import get_host_os, OsType + + +logger = get_logger(LoggerType.SHELL_COMMAND) TIMEOUT_CODE = 255 @@ -32,7 +40,7 @@ class AbstractProcess(ABC): def __init__(self): self.policy = { 'log-check-output': {'exit-code': True, 'stderr': True}, - 'log-run': {'verbose': True}, + 'log-run': {'verbose': True, 'trim-lines': None}, 'log-async-run': {'verbose': True}} self._std_stream = None self._err_stream = None @@ -48,8 +56,23 @@ def set_log_silence(self): self.policy['log-run']['verbose'] = False self.policy['log-async-run']['verbose'] = False + def set_log_trim_lines(self, value): + self.policy['log-run']['trim-lines'] = value + + def enable_check_stderr(self): + self.policy['log-check-output']['stderr'] = True + + def disable_check_stderr(self): + self.policy['log-check-output']['stderr'] = False + + def enable_check_exit_code(self): + self.policy['log-check-output']['exit-code'] = True + + def disable_check_exit_code(self): + self.policy['log-check-output']['exit-code'] = False + @abstractmethod - def async_run(self, cmd, cwd=None, daemon_mode=False, env=None): + def async_run(self, cmd, cwd=None, daemon_mode=False, env=None, sudo=False, use_stdin=False): if self.policy['log-async-run']['verbose']: logger.info(f'Executing cmd: {cmd} (cwd: {cwd}, daemon: {daemon_mode})') @@ -73,53 +96,119 @@ def _wait_for_output(self, timeout): def start_daemon(self, cmd, cwd=None): self.async_run(cmd, cwd, daemon_mode=True) - def run(self, cmd, cwd=None, timeout=600, env=None): - self.async_run(cmd, cwd=cwd, env=env) + def run(self, cmd, cwd=None, timeout=600, env=None, sudo=False, print_stdout=True): + self.async_run(cmd, cwd=cwd, env=env, sudo=sudo) exit_code, stdout, stderr = self._wait_for_output(timeout) - if self.policy['log-run']['verbose']: - logger.info(f"Finish process: \nExit code: {exit_code}\nStdout: {stdout}\nStderr: {stderr}") + self.log(exit_code, stdout, stderr, print_stdout=print_stdout) return exit_code, stdout, stderr - def run_and_check_return_all(self, cmd, cwd=None, env=None, timeout=600): - code, _stdout, stderr = self.run(cmd, cwd=cwd, env=env, timeout=timeout) - self._check_output(code, _stdout, stderr, cmd) + def log(self, exit_code, stdout, stderr, print_stdout=True): + if self.policy['log-run']['verbose']: + trim_lines = self.policy['log-run']['trim-lines'] + stdout_log = stdout + stderr_log = stderr + if isinstance(trim_lines, int): + stdout_lines = stdout.splitlines() + stderr_lines = stderr.splitlines() + if len(stdout_lines) > trim_lines: + stdout_log = "[stdout trimmed]\n" + "\n".join(stdout_lines[-trim_lines:]) + if len(stderr_lines) > trim_lines: + stderr_log = "[stderr trimmed]\n" + "\n".join(stderr_lines[-trim_lines:]) + if exit_code is not None and print_stdout: + logger.info(f"Finish process: \nExit code: {exit_code}\nStdout: {stdout_log}\nStderr: {stderr_log}") + elif exit_code is not None: + logger.info(f"Finish process: \nExit code: {exit_code}") + else: + logger.info(f"Finish process: \nStdout: {stdout_log}\nStderr: {stderr_log}") + + def run_and_check_return_all( + self, + cmd, + cwd=None, + env=None, + timeout=600, + sudo=False, + exception_type=OvmsTestException, + exit_code_check=0, + print_stdout=True, + ): + code, _stdout, stderr = self.run(cmd, cwd=cwd, env=env, timeout=timeout, sudo=sudo, print_stdout=print_stdout) + self._check_output(code, _stdout, stderr, cmd, exception_type, exit_code_check) return code, _stdout, stderr - def run_and_check(self, cmd, cwd=None, env=None, timeout=600, exception_type=AssertionError): - code, _stdout, stderr = self.run(cmd, cwd=cwd, env=env, timeout=timeout) - return self._check_output(code, _stdout, stderr, cmd, exception_type) - - def _check_output(self, code, _stdout, stderr, cmd, exception_type=AssertionError): - if self.policy['log-check-output']['exit-code'] and code != 0: - raise exception_type(f'Unexpected return code detected during executing cmd: {cmd} \n\tCode: {code}\n\tStderr: {stderr}') + def run_and_check( + self, + cmd, + cwd=None, + env=None, + timeout=600, + exception_type=OvmsTestException, + exit_code_check=0, + print_stdout=True, + ): + code, _stdout, stderr = self.run(cmd, cwd=cwd, env=env, timeout=timeout, print_stdout=print_stdout) + return self._check_output(code, _stdout, stderr, cmd, exception_type, exit_code_check) + + def _check_output(self, code, _stdout, stderr, cmd, exception_type=OvmsTestException, exit_code_check=0): + exception = None + if self.policy['log-check-output']['exit-code'] and code != exit_code_check: + exception = exception_type( + f'Unexpected return code detected during executing cmd: {cmd} \n\tCode: {code}\n\tStderr: {stderr}' + ) if self.policy['log-check-output']['stderr'] and stderr: - raise exception_type(f'Detect non empty stderr during executing cmd: {cmd}\n\tStderr: {stderr}') + exception = exception_type(f'Detect non empty stderr during executing cmd: {cmd}\n\tStderr: {stderr}') + if exception: + exception.set_process_details(cmd, code, _stdout, stderr) + raise exception return _stdout -class Process(AbstractProcess): +class CommonProcess(AbstractProcess): def __init__(self): super().__init__() self._proc = None - def async_run(self, cmd, cwd=None, daemon_mode=False, env=None): + @staticmethod + def _get_shell_settings(): + return False + + def async_run( + self, + cmd, + cwd=None, + daemon_mode=False, + env=None, + sudo=False, + use_stdin=False, + shell=None, + **kwargs, + ): super().async_run(cmd, cwd, daemon_mode) - cmd_mode = [] - cmd_mode += ['bash', '-c', cmd] - - self._proc = Popen(cmd_mode, - stdout=PIPE, - stderr=PIPE, - stdin=DEVNULL, - cwd=cwd, - env=env, - universal_newlines=(True if daemon_mode else None)) + stdin_redirect = PIPE if use_stdin else DEVNULL + + cmd_mode = self._get_cmd_mode(cmd, sudo) + + self._proc = Popen( + cmd_mode, + stdout=PIPE, + stderr=PIPE, + stdin=stdin_redirect, + cwd=cwd, + env=env, + universal_newlines=None, + shell=self._get_shell_settings() if shell is None else shell, + **kwargs, + ) self._err_stream = StreamReaderThread(self._proc.stderr) self._std_stream = StreamReaderThread(self._proc.stdout) self._err_stream.run() self._std_stream.run() + @staticmethod + def _get_cmd_mode(cmd, sudo): + raise NotImplementedError + def wait(self, timeout): try: self._proc.wait(timeout) @@ -138,31 +227,35 @@ def timeout_detected(self): def cleanup(self): self._proc.terminate() - def kill(self, force=False, timeout=300): + def _kill_by_shell(self, pid, end_time, force=False, sudo=False): + raise NotImplementedError + + def kill(self, force=False, timeout=30): logger.info(f'Killing process {self._proc.pid} (force: {force}, timeout: {timeout})!') end_time = datetime.now() + timedelta(seconds=timeout) if not self._proc: return None - parent_proc = psutil.Process(self._proc.pid) + try: + parent_proc = psutil.Process(self._proc.pid) + except psutil.NoSuchProcess as e: + return not self.is_alive() child_processes = parent_proc.children() for child_proc in child_processes: - child_proc.terminate() - _, alive = psutil.wait_procs(child_processes, + try: + child_proc.terminate() + except psutil.NoSuchProcess as e: + pass + except psutil.AccessDenied as e: + self._kill_by_shell(child_proc.pid, end_time=end_time, force=force, sudo=True) + + _, alive = psutil.wait_procs([child_proc], timeout=timeout, callback=lambda proc: f'Process {proc} exit code: {proc.returncode}') for proc in alive: proc.kill() - if self.is_alive(): - cmd = f'kill -9 {self._proc.pid}' if force else f'kill {self._proc.pid}' - check_call(cmd.split()) - cmd = 'ps --pid {}'.format(self._proc.pid) - while datetime.now() < end_time: - tmp_return = call(cmd.split()) - if tmp_return == 0: - break - sleep(1) + self._kill_by_shell(self._proc.pid, end_time, force=force) return not self.is_alive() @@ -179,7 +272,56 @@ def get_exitcode(self): return result -class RemoteProcess(SSHClient, Process): +class UnixProcess(CommonProcess): + + @staticmethod + def _get_cmd_mode(cmd, sudo): + cmd_mode = [] + if sudo: + cmd_mode += ['sudo', '-E'] + + cmd_mode += ['bash', '-c', cmd] + return cmd_mode + + def _kill_by_shell(self, pid, end_time, force=False, sudo=False): + if self.is_alive(): + cmd = f'kill -9 {pid}' if force else f'kill {pid}' + if sudo: + cmd = f"sudo {cmd}" + check_call(cmd.split()) + cmd = f'kill -s 0 {pid}' + while datetime.now() < end_time: + tmp_return = call(cmd.split()) + if tmp_return == 1: + break + sleep(1) + + +class WindowsProcess(CommonProcess): + + @staticmethod + def _get_cmd_mode(cmd, sudo): + if sudo: + raise NotImplementedError + return cmd + + @staticmethod + def _get_shell_settings(): + return True + + def _kill_by_shell(self, pid, end_time, force=False, sudo=False): + if self.is_alive(): + cmd = f'Taskkill /F /PID {pid}' if force else f'Taskkill /PID {pid}' + if sudo: + raise NotImplementedError + while datetime.now() < end_time: + tmp_return = call(cmd.split()) + if tmp_return == 0 or not self.is_alive(): + break + sleep(1) + + +class RemoteProcess(SSHClient, UnixProcess): def __init__(self, hostname, username=None, password=None, port=22): super(SSHClient, self).__init__() super(RemoteProcess, self).__init__() @@ -202,10 +344,10 @@ def reconnect(self, auth_timeout=None): def disconnect(self): self.close() - def async_run(self, cmd, cwd=None, daemon_mode=False, env=None): + def async_run(self, cmd, cwd=None, daemon_mode=False, env=None, sudo=False): super().async_run(cmd, cwd, daemon_mode) if cwd: - cmd = 'cd {} && {}'.format(cwd, cmd) + cmd = f"cd {cwd} && {cmd}" _, self._proc_stdout, stderr = self.exec_command(cmd, timeout=self._timeout) self._pid = self._proc_stdout.readline().strip() self._std_stream = StreamReaderThread(self._proc_stdout, False) @@ -225,7 +367,7 @@ def is_alive(self): def kill(self, force=False): if not self._pid: return None - logger.warning('Killing process (force: {})!'.format(force)) + logger.warning(f"Killing process (force: {force})!") self.exec_command("kill " + ("-9 " if force else "") + str(self._pid)) return not self.is_alive() @@ -241,11 +383,14 @@ def get_exitcode(self): if self._proc_stdout.channel.exit_status_ready(): result = self._proc_stdout.channel.recv_exit_status() else: - logger.warning('Timeout detected ({})...'.format(self.__class__.__name__)) + logger.warning(f'Timeout detected ({self.__class__.__name__})...') result = TIMEOUT_CODE return result +Process = WindowsProcess if get_host_os() == OsType.Windows else UnixProcess + + class StreamReaderThread: def __init__(self, stream, local_thread=True): self._queue = Queue() @@ -275,3 +420,81 @@ def pop(self): def wait_thread_end(self, timeout=None): self._thread.join(timeout) self._stream.close() + + +PID_STATE = "State" +PID_NAME = "Name" +PID_STATE_NONE = "" # Probably insufficient access +PID_STATE_SLEEPING = "S (sleeping)" +PID_STATE_ZOMBIE = "Z (zombie)" + + +def get_pid_details_as_dict(pid): + try: + proc_status = Path(f"/proc/{pid}/status").read_text() + proc_status_dict = {} + for line in proc_status.splitlines(): + key, *val = line.split(":") # if value contains multiple ':' len(val) > 1 + proc_status_dict[key.strip()] = ":".join(val).strip() # If len(val) > 1 again join val into string. + return proc_status_dict + except FileNotFoundError: + pass # Do not worry about it proc was killed definitly, most likely hazard during process killing + except Exception as exc: + logger.exception(str(exc)) + return None + + +def get_pid_name(pid): + results = get_pid_details_as_dict(pid) + name = results.get(PID_NAME, None) if results else None + return name + + +def get_pid_status(pid): + results = get_pid_details_as_dict(pid) + state = results.get(PID_STATE, None) if results else None + return state + + +class MountedShareDirectory: + def __init__(self, share, mount_point, read_only=True, share_type="nfs"): + self.share = share + self.mount_point = Path(mount_point) + self.read_only = read_only + self.share_type = share_type + self._proc = Process() + + def mount(self, allow_rw_permissions=False): + if not self.mount_point.exists(): + self._proc.run_and_check(f"sudo mkdir -p {self.mount_point}") + if list(self.mount_point.iterdir()): + logger.warning("Skip mounting.") + return + + if not allow_rw_permissions: + assert self.read_only, "Dynamic mounting share with RW can lead to unfortunate events.\ + Please reconsider statically mounted resource prior test session." + read_only = "-o ro" if self.read_only else "" + cmd = f"sudo mount -t {self.share_type} {read_only} {self.share} {self.mount_point}" + self._proc.run_and_check(cmd) + + def umount(self): + try: + mount_point_exists = self.mount_point.exists() + except OSError as e: + if "Stale file handle" in e.strerror: + self._proc.run_and_check(f"sudo umount -l {self.mount_point}") + else: + raise e + else: + if all([mount_point_exists, list(self.mount_point.iterdir())]): + self._proc.run_and_check(f"sudo umount -l {self.mount_point}") + else: + logger.warning("Skip unmounting.") + + def link(self, link_point): + link_point_dir = os.path.dirname(link_point) + if not os.path.exists(link_point_dir): + logger.warning(f"Creating directory: {link_point_dir}") + os.makedirs(link_point_dir) + self._proc.run_and_check(f"sudo ln -s {self.mount_point} {link_point}") diff --git a/tests/functional/utils/remote_test_environment.py b/tests/functional/utils/remote_test_environment.py new file mode 100644 index 0000000000..b4af08a222 --- /dev/null +++ b/tests/functional/utils/remote_test_environment.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from pathlib import Path + +from tests.functional.utils.process import Process + + +def copy_custom_lib_to_host(ovms_test_image, custom_library_path, new_library_path): + dirpath = Path(os.path.dirname(new_library_path)) + dirpath.mkdir(parents=True, exist_ok=True) + cmd = f"docker cp $(docker create --rm {ovms_test_image}):{custom_library_path} {new_library_path}" + proc = Process() + proc.run_and_check(cmd) diff --git a/tests/functional/utils/reservation_manager/__init__.py b/tests/functional/utils/reservation_manager/__init__.py new file mode 100644 index 0000000000..84cfbcf566 --- /dev/null +++ b/tests/functional/utils/reservation_manager/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/functional/utils/reservation_manager/__main__.py b/tests/functional/utils/reservation_manager/__main__.py new file mode 100644 index 0000000000..402ee8c5ca --- /dev/null +++ b/tests/functional/utils/reservation_manager/__main__.py @@ -0,0 +1,70 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.utils.logger import get_logger +from tests.functional.utils.reservation_manager.args import parse_args +from tests.functional.utils.reservation_manager.exceptions import ReservationNotAvailableError +from tests.functional.utils.reservation_manager.manager import Manager +from tests.functional.utils.reservation_manager.runner import reserve_and_run + + +def main(): + """Console script for manager.""" + + log = get_logger(__name__) + + try: + args = parse_args() + except Exception as exc: + log.error(f"While parsing arguments: {exc}") + return 1 + + if not args.reservation_action: + log.info("Nothing to do") + return + + log.info("Starting reservation manager") + reservation_mgr = Manager.manager_from_args(args) + + # Manage independent reservation + try: + if args.reservation_action == "create": + reservation_mgr.independent.create() + + elif args.reservation_action == "remove": + reservation_mgr.independent.remove() + + elif args.reservation_action == "cleanup": + reservation_mgr.independent.cleanup() + + elif args.reservation_action == "command": + return reserve_and_run(args.reservation_command, reservation_mgr) + + else: + raise ValueError(f"Provided reservation action " + f"is not allowed: {args.reservation_action}") + + except ReservationNotAvailableError as e: + log.error(f"{e}") + return 1 + + except Exception as e: + log.error(f"During handling reservation: {e}") + raise e + + +if __name__ == "__main__": + main() diff --git a/tests/functional/utils/reservation_manager/args.py b/tests/functional/utils/reservation_manager/args.py new file mode 100644 index 0000000000..70b5df482c --- /dev/null +++ b/tests/functional/utils/reservation_manager/args.py @@ -0,0 +1,166 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import argparse +import sys +from os import path + +from tests.functional.utils.logger import get_logger + + +def parse_args(args=sys.argv): + """Parse cli arguments""" + + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + parser.add_argument("-c", + "--config", + dest="config_path", + metavar="path", + default="./reservation_manager.json", + help="Path to YAML configuration file") + + parser.add_argument("-i", + "--init-config", + default=False, + action="store_true", + help="Init default config under path " + "pointed by --config") + + parser.add_argument("--pool-range-start", + type=int, + metavar="port", + help="Port number starting " + "allowed port reservation range") + + parser.add_argument("--pool-range-stop", + type=int, + metavar="port", + help="Port number ending " + "allowed port reservation range") + + parser.add_argument("--pool-part-size", + type=int, + metavar="number", + help="Port pool part size") + + parser.add_argument("--locks-dir", + type=str, + metavar="path", + help="Directory where lock files will be present") + + parser.add_argument("--reservation-file-json", + type=str, + metavar="filepath", + default="./reservation.json", + help="Path to file where the reservation information " + "will be saved in JSON file") + + parser.add_argument("--reservation-file-env", + type=str, + metavar="filepath", + default="./reservation.env", + help="Path to file where the reservation information " + "will be saved in ENV Variables format") + + parser.add_argument("--keep-env", + default=False, + action="store_true", + help="Keep environment variables when running " + "command, pass with configured reservation envs.") + + parser.add_argument("-l", + "--log-level", + choices=[ + "DEBUG", + "INFO", + "WARNING", + ], + default="INFO", + help="Enable debug mode") + + # Create subparser for functions + subparsers = parser.add_subparsers( + help="Action to be taken with %(prog)s", + dest="reservation_action", + ) + + # subparser for create + parser_for_create = subparsers.add_parser( + "create", + help="Create a long living reservation with reservation " + "files for JSON and Shell Environments. See --reservation-file-json " + "and --reservation-file-env.", + ) + parser_for_create.add_argument( + dest="reservation_create", + action="store_true", + ) + + # subparser for remove + parser_for_remove = subparsers.add_parser( + "remove", + help="Delete reservation pointed by --reservation-file-json on host.", + ) + parser_for_remove.add_argument( + dest="reservation_remove", + action="store_true", + ) + + # subparser for cleanup + parser_for_cleanup = subparsers.add_parser( + "cleanup", + help="Delete all reservations on host.", + ) + parser_for_cleanup.add_argument( + dest="reservation_cleanup", + action="store_true", + ) + + # subparser for command + parser_for_command = subparsers.add_parser( + "command", + help="Command to be run. Reservation will be " + "automatically cleaned up after command completion. ", + ) + parser_for_command.add_argument( + dest="reservation_command", + metavar="cmd_arg", + nargs="*", + help="example: " + "'%(prog)s -- pytest --mypytest opt'", + ) + + args = parser.parse_args(args=args) + + log = get_logger(__name__) + + log.info(f"Arguments: {args}") + + # Copy default configuration file if requested + if args.init_config: + if path.isfile(args.config_path): + raise Exception( + f"Can't init config in '{args.config_path}', file exists.") + # copyfile(paths.package_data_conf_filepath, args.config_path) + log.info("Config file initialized") + + # # If configuration not exists in config_path, use default from package + # if not path.exists(args.config_path): + # args.config_path = paths.package_data_conf_filepath + + return args diff --git a/tests/functional/utils/reservation_manager/env_manager.py b/tests/functional/utils/reservation_manager/env_manager.py new file mode 100644 index 0000000000..bc0a484f69 --- /dev/null +++ b/tests/functional/utils/reservation_manager/env_manager.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import inspect +import os +from math import floor + +from jinja2 import Template + +from tests.functional.utils.logger import get_logger + + +class EnvManager: + """ + Manage environment variables passed to Runner instance + params:: + - reservation=None + - pool_part_slices=None - if None, default: + [{ + "start": "TT_STARTING_PORT", + "end": "TT_STOPPING_PORT", + "size": "TT_PORTS_POOL_SIZE " + }] + - pool_part_ports_prefix=None + - keep_env=False + """ + def __init__( + self, + reservation=None, + pool_part_slices=None, + pool_part_ports_prefix=None, + keep_env=False, + ): + """Create environment mapping""" + self.keep_env = keep_env + + self.reservation = reservation + + if not pool_part_slices: + pool_part_slices = [{ + "start": "TT_STARTING_PORT", + "end": "TT_STOPPING_PORT", + "size": "TT_PORTS_POOL_SIZE " + }] + self.pool_part_slices = pool_part_slices + + self.pool_part_ports_prefix = pool_part_ports_prefix + + self.log = get_logger(__name__) + self.environment = {} + + def update_env_for_slice(self, slice_dict, key, value): + """Update env but don't fail if given key does not exists""" + try: + self.environment.update({slice_dict[key]: str(value)}) + except (KeyError, TypeError): + pass + + def manage_reservation_environments(self): + """Manages registered reservation environments""" + reservation_range_start = self.reservation.pool_part.start + reservation_range_size = self.reservation.pool_part.size() + + reservation_slice_count = len(self.pool_part_slices) + reservation_slice_size = floor(reservation_range_size / + reservation_slice_count) + + # Save ports prefixes in this string + ports_prefixes = "" + + for slice_index, pool_part_slice in enumerate(self.pool_part_slices): + slice_start = (reservation_range_start + + slice_index * reservation_slice_size) + slice_stop = (reservation_range_start + + (slice_index + 1) * reservation_slice_size) + slice_size = slice_stop - slice_start + + ports_prefixes += f"{str(slice_start)[:-2]} " + + # Update environemnt but don't fail if no slice keys were provided + self.update_env_for_slice(pool_part_slice, "start", slice_start) + self.update_env_for_slice(pool_part_slice, "end", slice_stop) + self.update_env_for_slice(pool_part_slice, "size", slice_size) + + # Don't force ports_prefixes to be present + if self.pool_part_ports_prefix: + # Trim string out of whitespace at end + ports_prefixes = ports_prefixes[:-1] + + self.environment.update( + {self.pool_part_ports_prefix: ports_prefixes}) + + def register_reservation(self, reservation): + """Set all attributes accordingly to given reservation""" + self.reservation = reservation + self.manage_reservation_environments() + if self.keep_env: + self.environment.update(os.environ) + + def get_json(self): + """ + Return env variables dict with primary key set to 'envs', for example: + ``` + "envs": { + "POOL_START": "31000", + "POOL_STOP": "32000", + "TT_REST_OVMS_STARTING_PORT": "31000", + "TT_REST_OVMS_END_PORT": "31500", + "TT_GRPC_OVMS_STARTING_PORT": "31500", + "TT_GRPC_OVMS_END_PORT": "32000", + "PORTS_PREFIX": "310 315" + } + + ``` + """ + return {'envs': self.environment} + + def get_shell_envs(self): + """ + Return env variables in shell sourceable format, + for example: + ``` + export KEY1=VALUE1 + export KEY2=VALUE2 + ``` + """ + template = Template(""" + {% for env, value in environment.items() %} + export {{ env }}='{{ value }}' + {%- endfor %} + + """) + + exported = template.render(environment=self.environment) + return inspect.cleandoc(exported) diff --git a/tests/functional/utils/reservation_manager/exceptions.py b/tests/functional/utils/reservation_manager/exceptions.py new file mode 100644 index 0000000000..3606f407a9 --- /dev/null +++ b/tests/functional/utils/reservation_manager/exceptions.py @@ -0,0 +1,50 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +class ReservationError(Exception): + pass + + +class ReservationNotExistsError(ReservationError): + """ + Exception raised when removing reservation fails because it does + not exists already. + """ + def __init__(self, msg=None): + if msg: + self.message = msg + else: + self.message = "Reservation not exists" + + def __str__(self): + """Return exception message""" + return self.message + + +class ReservationNotAvailableError(ReservationError): + """ + Exception raised when reservation fails because there is no more available. + """ + def __init__(self, msg=None): + if msg: + self.message = msg + else: + self.message = "No more reservation available" + + def __str__(self): + """Return exception message""" + return self.message diff --git a/tests/functional/utils/reservation_manager/locker.py b/tests/functional/utils/reservation_manager/locker.py new file mode 100644 index 0000000000..880b78293b --- /dev/null +++ b/tests/functional/utils/reservation_manager/locker.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import filelock + + +class Locker(object): + """Manage reservation lock across multiple reservation manager processes""" + def __init__( + self, + lock_path="/tmp/res_mgr.lock", + timeout=20, + ): + + self.lock_path = lock_path + self.timeout = timeout + self.lock = filelock.FileLock(lock_path) + + def acquire(self): + """ + Acquire lock or timeout for this reservation manager process + Unsuccessful acquiring after timeout raises exception. + + Exceptions: + - Timeout + """ + self.lock.acquire(timeout=self.timeout) + + def release(self): + self.lock.release() diff --git a/tests/functional/utils/reservation_manager/manager.py b/tests/functional/utils/reservation_manager/manager.py new file mode 100644 index 0000000000..5ebdf9af2a --- /dev/null +++ b/tests/functional/utils/reservation_manager/manager.py @@ -0,0 +1,639 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import os +import pathlib + +import yaml +from filelock import Timeout + +from tests.functional.utils.logger import get_logger +from tests.functional.utils.reservation_manager.env_manager import EnvManager +from tests.functional.utils.reservation_manager.exceptions import ReservationNotAvailableError, ReservationNotExistsError +from tests.functional.utils.reservation_manager.locker import Locker +from tests.functional.utils.reservation_manager.manager_config import ManagerConfig + +from tests.functional.config import global_tmp_dir + +logger = get_logger(__name__) + + +class Manager: + """ + Class Manager is responsible for gathering information about + possible port ranges, reserved ranges and supplying information + for next available range. + + params:: + - res_mgr_conf - ManagerConfig + - env_mgr - EnvManager + + """ + def __init__(self, res_mgr_conf, env_mgr): + """ + Get configuration from ManagerConfig and EnvManager, + init internal variables to manage ports reservation, + gather possible port ranges. + """ + + logger = get_logger(__name__) + + self.config = res_mgr_conf + self.env_mgr = env_mgr + + self.all_pool_parts = [] + self.calculate_all_pool_parts() + + logger.info(f"Calculated all possible port ranges, " + f"len: {len(self.all_pool_parts)}") + + logger.info("Create Locker instance") + self.reservation_lock = Locker(lock_path=os.path.join(global_tmp_dir, "res_mgr.lock")) + + self.reservations = [] + + self.owned_reservation = None + + def calculate_all_pool_parts(self): + self.all_pool_parts = [] + logger.info( + f"Calculating all reservable port ranges with following: " + f"pool range start: {self.config.pool_range_start}, " + f"pool range stop: {self.config.pool_range_stop}, " + f"pool part size: {self.config.pool_part_size}") + + for range_step in range(self.config.pool_range_start, + self.config.pool_range_stop, + self.config.pool_part_size): + range_start = range_step + range_stop = range_start + self.config.pool_part_size + + self.all_pool_parts.append(PoolPart(range_start, range_stop)) + + def register_existing_reservations(self): + reservation_files = [ + filename for filename in os.listdir(self.config.locks_dir) + if os.path.isfile(os.path.join(self.config.locks_dir, filename)) + and self.config.locks_prefix in filename + ] + + logger.info(f"Gathered reservation files: {reservation_files}; " + f"length: {len(reservation_files)}") + + for reservation_file in reservation_files: + logger.info(f"Creating Reservation instance " + f"from string: {reservation_file}") + try: + reservation = Reservation.from_str(reservation_file) + logger.info(f"Appending reservation instance " + f"'{reservation}' to reservations list") + self.reservations.append(reservation) + + except AssertionError as exc: + logger.error("While registering existing reservations") + raise AssertionError( + f"Can't create reservation from string: {reservation_file}; " + f"exception: {exc}") + + def _create_reservation_file(self, reservation): + """ + Create and save information about reservation file. + Concurently not safe. + Exceptions: + - pathlib.FileExistsError if file exists + """ + + reservation_lock_path = os.path.join(self.config.locks_dir, str(reservation)) + logger.info( + f"Creating reservation lock file: {reservation_lock_path}") + pathlib.Path(reservation_lock_path).touch(exist_ok=False) + return reservation_lock_path + + def register_as_owned(self, reservation): + """Registers as owned reservation in this instance.""" + self.owned_reservation = reservation + self.env_mgr.register_reservation(reservation) + + def reserve(self, reservation_candidate): + """Creates reservation file""" + + logger.info(f"Trying to create reservation for " + f"{reservation_candidate}") + try: + self._create_reservation_file(reservation_candidate) + + except FileExistsError as e: + logger.critical( + f"Creating reservation. This should have not happened, " + f"definitely a bug: {e}") + raise e + + def get_available_reservation(self): + """ + Get available reservation with respect to already existing + reservations. + Return None if not available. + """ + + logger.info("Get existing reservations") + self.register_existing_reservations() + + for pool_range_candidate in self.all_pool_parts: + logger.info(f"Considering pool range: {pool_range_candidate}") + pool_range_valid = True + for reservation in self.reservations: + if pool_range_candidate.is_intersect_with(reservation): + pool_range_valid = False + break + + if not pool_range_valid: + continue + + logger.info(f"Pool range is available for reservation: " + f"{pool_range_candidate}") + + available_reservation = Reservation( + self.config.locks_prefix, + pool_range_candidate, + self.config.reserver, + ) + return available_reservation + return None + + def reserve_and_return(self): + """ + Create reservation and return. + Concurently safe. + """ + + logger.info("Locking reservation procedure") + try: + self.reservation_lock.acquire() + logger.info("Reservation lock aquired") + + logger.info("Attempting reservation") + reservation = self.get_available_reservation() + if reservation: + self.reserve(reservation) + return reservation + raise ReservationNotAvailableError() + + except Timeout as exc: + raise Timeout(f"Can't lock reservation procedure: {exc}") from exc + except Exception as exc: + raise exc from exc + finally: + logger.info("Unlocking reservation procedure") + self.reservation_lock.release() + + def get_owned_reservation(self): + """Returns owned reservation or reserves if does not own any.""" + + logger.info("Return owned reservation or create") + + if not self.owned_reservation: + logger.info( + "Not registered any reservation, register and return") + reservation = self.reserve_and_return() + self.register_as_owned(reservation) + return reservation + + return self.owned_reservation + + def _remove_reservation_file(self, reservation): + """Remove reservation file""" + reservation_lock_path = os.path.join(self.config.locks_dir, str(reservation)) + logger.info(f"Removing reservation lock path: {reservation_lock_path}") + try: + os.remove(reservation_lock_path) + except FileNotFoundError: + logger.warning(f"Reservation file: {reservation_lock_path} lock already removed") + + def release_reservation(self, reservation): + """Delete reservation file""" + self._remove_reservation_file(reservation) + + def get_reservation_json(self): + """Return json data for owned reservation""" + + reservation_json = self.env_mgr.get_json() + reservation_file = f"{self.config.locks_dir}/{self.owned_reservation}" + reservation_json["reservation_file"] = reservation_file + reservation_json["shell_envs_file"] = self.config.reservation_file_env + return reservation_json + + def get_reservation_shell_envs(self): + """Return shell env variables for owned reservation""" + + reservation_shell_envs = self.env_mgr.get_shell_envs() + return reservation_shell_envs + + def reservation_from_json(self, json_path): + """Return reservation from json data""" + + try: + with open(json_path, "r") as json_file: + json_data = json.load(json_file) + logger.info(f"json_data: {json_data}") + + reservation_lock_path = json_data["reservation_file"] + self.config.reservation_file_env = json_data["shell_envs_file"] + logger.info( + f"reservation_lock_path: {reservation_lock_path}") + + self.env_mgr.locks_dir = os.path.dirname(reservation_lock_path) + reservation_str = os.path.basename(reservation_lock_path) + + reservation = Reservation.from_str(reservation_str) + return reservation + + except FileNotFoundError as exc: + raise FileNotFoundError(f"Reservation from json: {exc}") from exc + + except json.JSONDecodeError as exc: + raise json.JSONDecodeError(f"Loading reservation from json: {exc}") from exc + + @property + def independent(self): + class independent_class: + def __init__(self, res_mgr): + self.res_mgr = res_mgr + + def create(self, verbose=False): + """ + Create independent reservation, save info to json and shell env + file to filepaths pointed by + '--reservation-file-json' and '--reservation-file-env'. + """ + reservation = self.res_mgr.get_owned_reservation() + + reservation_json = self.res_mgr.get_reservation_json() + json_save_path = "./reservation.json" if not ( + self.res_mgr.config.reservation_file_json) else self.res_mgr.config.reservation_file_json + + res_shell_envs = self.res_mgr.get_reservation_shell_envs() + shell_env_save_path = "./reservation.env" if not ( + self.res_mgr.config.reservation_file_env) else self.res_mgr.config.reservation_file_env + if verbose: + print(reservation) + print(reservation_json) + print(res_shell_envs) + + # # Open exclusively, if file exists - throw exception + try: + with open(json_save_path, "x") as json_file: + json.dump(reservation_json, + json_file, + ensure_ascii=False, + indent=4) + except FileExistsError as exc: + self.res_mgr.release_reservation(reservation) + raise FileExistsError(f"Can't save reservation json: {exc}") from exc + + try: + with open(shell_env_save_path, "x") as shell_env_file: + shell_env_file.write(res_shell_envs) + + except FileExistsError as exc: + self.res_mgr.release_reservation(reservation) + raise FileExistsError(f"Can't save shell env variables: {exc}") from exc + + def remove(self): + """ + Remove independent reservation, use file pointed by + '--reservation-file-json' to get required information. + """ + + # Get reservation file json + json_path = self.res_mgr.config.reservation_file_json + try: + reservation = self.res_mgr.reservation_from_json(json_path) + except FileNotFoundError as exc: + raise FileNotFoundError(f"Cannot find reservation: {exc}") from exc + + reservation_file_env = self.res_mgr.config.reservation_file_env + + try: + self.res_mgr.release_reservation(reservation) + except ReservationNotExistsError: + pass + except Exception as exc: + logger.error(f"Error releasing reservation: {exc}") + + try: + os.remove(reservation_file_env) + except FileNotFoundError as exc: + logger.warning(f"Removing reservation shell env : {exc}") + + try: + os.remove(json_path) + except FileNotFoundError as exc: + logger.warning(f"Removing reservation json: {exc}") + + def cleanup(self): + """Find all reservations in locks_dir, delete. + Try to delete files pointed by + '--reservation-file-json' and '--reservation-file-env'. + """ + + # Try with remove first because there can be reservation-file{json,env} + try: + self.res_mgr.independent.remove() + except Exception: + pass + + self.res_mgr.register_existing_reservations() + existing_reservations = self.res_mgr.reservations + for reservation in existing_reservations: + logger.info(f"Removing reservation: {reservation}") + try: + self.res_mgr.release_reservation(reservation) + except Exception as exc: + raise Exception(f"Can't release reservation: {exc}") from exc + + return independent_class(self) + + + @staticmethod + def manager_from_args(args): + """ + Create and return new Manager instance + from cli arguments or config file. + + params:: + - args - parsed argparse namespace + """ + + log = get_logger(__name__) + + log.debug(f"args: {args}") + + config_path = args.config_path + reservation_action = None + + config = None + try: + with open(config_path, 'r') as file: + config = yaml.load(file, Loader=yaml.FullLoader)["config"] + + except FileNotFoundError as exc: + msg = f"Configuration file '{config_path}' not found" + log.warning(msg) + raise msg from exc + + log.debug(f"config: {config}") + + pool_range_start = 0 + pool_range_stop = 0 + pool_part_size = 0 + + pool_part_ports_prefix = None + pool_part_slices = None + locks_dir = None + + try: + # Port pool settings + pool_range_start = int(config["pool_range"]["start"]) + pool_range_stop = int(config["pool_range"]["stop"]) + pool_part_size = int(config["pool_part_size"]) + + # Pool parts envs settings + pool_part_slices = config["envs"]["slices"] + + # Allow passing number of slices as int, so it may be used + # easier when only prefix is needed. + # This behaviour will be possibly dropped in the future. + if isinstance(pool_part_slices, int): + pool_part_slices = [None] * pool_part_slices + + # Don't force ports_prefix to be present + pool_part_ports_prefix = config["envs"].get("ports_prefix", "") + locks_dir = config["locks_dir"] + + except KeyError as exc: + key = f"{exc}".replace("'", "") + raise KeyError(f"Following key not found in configuration file 'config/{key}'") from exc + + except Exception as exc: + raise Exception(f"Unknown exception while reading configuration file: {exc}") from exc + + try: # With CLI arguments, override config file + if args.pool_range_start: + pool_range_start = args.pool_range_start + + if args.pool_range_stop: + pool_range_stop = args.pool_range_stop + + if args.pool_part_size: + pool_part_size = args.pool_part_size + + if args.locks_dir: + locks_dir = args.locks_dir if args.locks_dir == global_tmp_dir else global_tmp_dir + + reservation_action = args.reservation_action + + reservation_file_json = args.reservation_file_json + reservation_file_env = args.reservation_file_env + + keep_env = args.keep_env + + command = None + reserver = "independent" + + if args.reservation_action == "command": + command = args.reservation_command + reserver = args.reservation_command[0].replace("-", "_") + + except Exception as exc: + log.error(f"Getting settings from cli arguments: {exc}") + raise Exception(f"While getting CLI arguments: {exc}") from exc + + log.info(f"All params:\n" + f"pool_range_start: {pool_range_start}\n" + f"pool_range_stop: {pool_range_stop}\n" + f"pool_part_size: {pool_part_size}\n" + f"pool_part_ports_prefix: {pool_part_ports_prefix}\n" + f"pool_part_slices: {pool_part_slices}\n" + f"locks_dir: {locks_dir}\n" + f"reservation_action: {reservation_action}\n" + f"reservation_file_json: {reservation_file_json}\n" + f"reservation_file_env: {reservation_file_env}\n" + f"command: {command}\n" + f"keep_env: {keep_env}\n" + f"reserver: {reserver}\n") + + env_mgr = EnvManager( + pool_part_slices=pool_part_slices, + pool_part_ports_prefix=pool_part_ports_prefix, + keep_env=keep_env, + ) + + res_mgr_conf = ManagerConfig( + pool_range_start=pool_range_start, + pool_range_stop=pool_range_stop, + pool_part_size=pool_part_size, + locks_dir=locks_dir, + reserver=reserver, + port_lock_cleanup=(False if reservation_action == "create" else True), + reservation_file_json=reservation_file_json, + reservation_file_env=reservation_file_env, + ) + + res_mgr = Manager(res_mgr_conf, env_mgr) + + log.info(f"env_mgr: {env_mgr}") + log.info(f"res_mgr_conf: {res_mgr_conf}") + log.info(f"res_mgr: {res_mgr}") + + return res_mgr + +class PoolPart: + """ + PoolPart represents reservable port pool part ranges. + Range start is inclusive, range stop is exclusive. + """ + def __init__(self, start, stop): + """ + Create with defined port stop and start range. + Stop port range value is exclusive. + + Throw an exception if failed to initialize: + 1. ValueError: range boundaries are not ints + 2. AssertionError: range boundaries are invalid + """ + + self.start = int(start) + self.stop = int(stop) + PoolPart.validate_range(self.start, self.stop) + self.range = range(self.start, self.stop) + + def size(self): + return self.stop - self.start + + @staticmethod + def validate_range(range_start, range_stop): + assert range_stop - range_start > 0, ( + "Invalid port range, must be greater than 0: " + f"start: {range_start}; stop: {range_stop}") + + def is_intersect_with(self, pool_part): + """ + Check if another PoolPart or Reservation instance + is in intersect with this instance. + """ + + if type(pool_part) is PoolPart: + self_is_subset = (self.range.start in pool_part.range + or self.range[-1] in pool_part.range) + pool_part_is_subset = (pool_part.range.start in self.range + or pool_part.range[-1] in self.range) + return self_is_subset or pool_part_is_subset + + elif type(pool_part) is Reservation: + res_pool_part = pool_part.pool_part.range + + self_is_subset = (self.range.start in res_pool_part + or self.range[-1] in res_pool_part) + + reservation_is_subset = (res_pool_part.start in self.range + or res_pool_part[-1] in self.range) + + return self_is_subset or reservation_is_subset + + def __str__(self): + return f"{self.start}-{self.stop}" + + +class Reservation: + """ + PoolPart reservation. + + params:: + - locks_prefix (string) - prefix for string representation and discovery + - pool_part (PoolPart) - create reservation for this pool part + - reserved_by (string) - suffix for string representation + + """ + def __init__(self, locks_prefix, pool_part, reserved_by): + self.locks_prefix = locks_prefix + self.pool_part = pool_part + self.reserved_by = reserved_by + + def __str__(self): + return_str = (f"{self.locks_prefix}-{self.pool_part.start}-" + f"{self.pool_part.stop}-{self.reserved_by}") + + try: + Reservation.validate_string(return_str) + return return_str + except AssertionError as exc: + raise AssertionError( + f"Reservation method '__str__' returned invalid " + f"string: {return_str}, exception: {exc}") from exc + + @staticmethod + def validate_string(reservation): + """Checks if string represents valid reservation""" + splitted = reservation.split("-") + + assert len(splitted) == 4, ( + "Reservation string should consist of 4 elements: " + "splitted with dash '-': " + "'locks prefix', 'pool part start', " + "'pool part stop', 'reserver identifier'") + + pool_part_start_str = splitted[1] + pool_part_stop_str = splitted[2] + + try: + pool_part_start = int(pool_part_start_str) + pool_part_stop = int(pool_part_stop_str) + PoolPart.validate_range(pool_part_start, pool_part_stop) + except AssertionError as e: + raise e + + except ValueError as exc: + raise AssertionError( + f"Range boundaries invalid, start: {pool_part_start_str}, " + f"stop: {pool_part_stop_str}; exception: {exc}") from exc + + @staticmethod + def from_str(string): + """Create Reservation instance from string""" + + try: + Reservation.validate_string(string) + except AssertionError as exc: + raise AssertionError( + f"Validating Reservation instance from string: {string}, exception: {exc}") from exc + + values = string.split("-") + locks_prefix = values[0] + pool_part_start = values[1] + pool_part_stop = values[2] + reserved_by = values[3] + + pool_part = PoolPart( + pool_part_start, + pool_part_stop, + ) + + return Reservation( + locks_prefix, + pool_part, + reserved_by, + ) diff --git a/tests/functional/utils/reservation_manager/manager_config.py b/tests/functional/utils/reservation_manager/manager_config.py new file mode 100644 index 0000000000..a03fff5e60 --- /dev/null +++ b/tests/functional/utils/reservation_manager/manager_config.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os + +from tests.functional.utils.logger import get_logger + + +# pylint: disable=too-many-instance-attributes +class ManagerConfig: + """ + ManagerConfig + + params:: + - pool_range_start=30000 + - pool_range_stop=60000 + - pool_part_size=1000 + - locks_dir="/tmp" + - locks_prefix="reservation_manager" + - reserver="default" + - port_lock_cleanup=True + - reservation_file_json="reservation.json" + - reservation_file_env="reservation.env" + """ + + # pylint: disable=too-many-arguments + def __init__( + self, + pool_range_start=30000, + pool_range_stop=60000, + pool_part_size=1000, + locks_dir="/tmp", + locks_prefix="reservation_manager", + reserver=None, + port_lock_cleanup=True, + reservation_file_json="reservation.json", + reservation_file_env="reservation.env", + ): + + self.pool_range_start = pool_range_start + self.pool_range_stop = pool_range_stop + self.pool_part_size = pool_part_size + + self.locks_dir = locks_dir + self.locks_prefix = locks_prefix + self.reserver = "default" if not reserver else reserver + + self.port_lock_cleanup = port_lock_cleanup + self.reservation_file_json = reservation_file_json + self.reservation_file_env = reservation_file_env + + self.log = get_logger(__name__) + self.validate() + + def validate(self): + """ + Validate ManagerConfig. + """ + + assert (self.pool_range_start is not None + and self.pool_range_stop is not None + and self.pool_part_size is not None + and self.locks_dir is not None), ( + "Following values must be provided: " + "pool_range_start, " + "pool_range_stop, " + "pool_part_size, " + "locks_dir") + + assert self.pool_range_start > 1024, ( + "Port range start must be greater than " + "1024") + assert self.pool_range_stop > 1024, ( + "Port range stop must be greater than " + "1024") + assert self.pool_range_stop - self.pool_range_start > 0, ( + "Port range must be greater than 0") + + assert self.pool_part_size <= ( + self.pool_range_stop - self.pool_range_start), ( + "Port pool size must be smaller or equal to port range") + + assert os.path.exists(self.locks_dir), ( + f"Path for locks dir must exist: {self.locks_dir}") + + assert os.access(self.locks_dir, os.W_OK), ( + f"Path for locks dir must be writeable: {self.locks_dir}") + + assert len(self.reserver.split("-")) == 1, ( + f"Reserver invalid: cannot have dashed in name: {self.reserver}") diff --git a/tests/functional/utils/reservation_manager/reservation_manager.yml b/tests/functional/utils/reservation_manager/reservation_manager.yml new file mode 100644 index 0000000000..daa032fcee --- /dev/null +++ b/tests/functional/utils/reservation_manager/reservation_manager.yml @@ -0,0 +1,32 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--- +config: + pool_range: + start: 30000 + stop: 60000 + pool_part_size: 1000 + locks_dir: /tmp + envs: + ports_prefix: PORTS_PREFIX + # As an alternative .slices can be a number and .ports_prefix will + # have a number of prefixes matching this number. + # slices: 2 + slices: + - start: TT_STARTING_PORT + size: TT_PORTS_POOL_SIZE + end: TT_ENDING_PORT diff --git a/tests/functional/utils/reservation_manager/runner.py b/tests/functional/utils/reservation_manager/runner.py new file mode 100644 index 0000000000..6132ee1fd1 --- /dev/null +++ b/tests/functional/utils/reservation_manager/runner.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import subprocess + +from tests.functional.utils.logger import get_logger +from tests.functional.utils.reservation_manager.exceptions import ReservationNotAvailableError + + +class Runner: + """ + Manage running command, setup evironment variables + accoarding to Manager.Reservation instance. + """ + def __init__(self, command, env_mgr): + """Prepare all necessary initialization to run command.""" + self.command = command + self.env_mgr = env_mgr + + self.reservation = env_mgr.reservation + self.log = get_logger(__name__) + + def run(self): + """Run and pipe output""" + + self.log.info(f"running with command: {self.command} and " + f"environment: {self.env_mgr.environment}") + + popen = subprocess.Popen( + self.command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + bufsize=1, + env=self.env_mgr.environment, + ) + + for stdout_line in iter(popen.stdout.readline, ""): + print(stdout_line, end="") + + popen.communicate() + return popen.returncode + + +def reserve_and_run(command, reservation_mgr): + """Use Manager to reserve port pool and run command""" + + log = get_logger(__name__) + + try: + # Reservation + log.info("Trying to acquire port reservation") + reservation = reservation_mgr.get_owned_reservation() + log.info(f"Acquired port reservation: {reservation}") + + # Runner + runner = Runner(command, reservation_mgr.env_mgr) + + return_code = runner.run() + log.info(f"Command returned code: {return_code}") + + if reservation_mgr.config.port_lock_cleanup: + log.info(f"Releasing reservation: {reservation}") + reservation_mgr.release_reservation(reservation) + + return return_code + + except ReservationNotAvailableError as e: + raise e + + except Exception as e: + log.error(f"Running command with reservation failed: {e}") + raise e diff --git a/tests/functional/utils/reservation_manager/unittests/__init__.py b/tests/functional/utils/reservation_manager/unittests/__init__.py new file mode 100644 index 0000000000..84cfbcf566 --- /dev/null +++ b/tests/functional/utils/reservation_manager/unittests/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/functional/utils/reservation_manager/unittests/test_manager.py b/tests/functional/utils/reservation_manager/unittests/test_manager.py new file mode 100644 index 0000000000..96975dad77 --- /dev/null +++ b/tests/functional/utils/reservation_manager/unittests/test_manager.py @@ -0,0 +1,163 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest + +from tests.functional.utils.reservation_manager.env_manager import EnvManager +from tests.functional.utils.reservation_manager.manager import Manager, PoolPart, Reservation +from tests.functional.utils.reservation_manager.manager_config import ManagerConfig + + +class TestManager: + pool_part_ranges = [ + (2000, 3000), + (3000, 4000), + (4000, 5000), + (5000, 6000), + (6000, 7000), + (7000, 8000), + ] + + def test_calculate_all_pool_parts(self, mocker): + conf_mgr = mocker.MagicMock() + conf_mgr.pool_range_start = 20000 + conf_mgr.pool_range_stop = 60000 + conf_mgr.pool_part_size = 1000 + expected_pool_size = 40 + + env_mgr = mocker.MagicMock() + + mgr = Manager(conf_mgr, env_mgr) + mgr.calculate_all_pool_parts() + + assert len(mgr.all_pool_parts) == expected_pool_size + + def test_get_reservation_json(self, mocker): + conf_mgr = ManagerConfig() + env_mgr = EnvManager() + mgr = Manager(conf_mgr, env_mgr) + mocker.spy(env_mgr, 'get_json') + + json = mgr.get_reservation_json() + + assert "envs" in json + assert "reservation_file" in json + assert "shell_envs_file" in json + assert env_mgr.get_json.call_count == 1 + + def test_get_reservation_shell_envs(self, mocker): + conf_mgr = ManagerConfig() + env_mgr = EnvManager() + mgr = Manager(conf_mgr, env_mgr) + + env_mgr.environment = { + "test_key_string": "test_value", + "test_key_with_int": 0, + } + + expected_envs = "" + for key, value in env_mgr.environment.items(): + expected_envs += f"export {key}='{value}'\n" + expected_envs = expected_envs.strip() + + mocker.spy(env_mgr, 'get_shell_envs') + + shell_envs = mgr.get_reservation_shell_envs() + + assert shell_envs == expected_envs + assert env_mgr.get_shell_envs.call_count == 1 + + +class TestPoolPart: + intersect_test_set = [ + ((1900, 1950), (2000, 2100), (True)), + ((1950, 2000), (2000, 2100), (True)), + ((1980, 2030), (2000, 2100), (False)), + ((2030, 2040), (2000, 2100), (False)), + ((2050, 2100), (2000, 2100), (False)), + ((2090, 2140), (2000, 2100), (False)), + ((2100, 2150), (2000, 2100), (True)), + ((2150, 2200), (2000, 2100), (True)), + ] + + def test_ranges_good(self): + for start, stop in TestManager.pool_part_ranges: + try: + PoolPart(start, stop) + except AssertionError as e: + pytest.fail(f"Creating PoolPart should succeed with range: " + f"start {start}, stp: {stop}") + + def test_ranges_bad(self): + for stop, start in TestManager.pool_part_ranges: + with pytest.raises(AssertionError): + PoolPart(start, stop) + + def test_is_intersect_with(self): + for range1, range2, should_be_valid in self.intersect_test_set: + pool_part_range1 = PoolPart(range1[0], range1[1]) + pool_part_range2 = PoolPart(range2[0], range2[1]) + + if should_be_valid: + assert not pool_part_range1.is_intersect_with(pool_part_range2) + else: + assert pool_part_range1.is_intersect_with(pool_part_range2) + + +class TestReservation: + prefix = "reservation_manager" + suffix = "unittests" + reservation = Reservation + + bad_reservation_strings = [ + "singlepart", + "double-part", + "tri-ple-part", + "with-four-sep-strings", + "1000-2000-wrong-order1", + "1000-wrong-order2-2000", + "wrong-order3-1000-2000", + "with-one-1000-number", + "wrong-2000-1000-range", + ] + + def test_validate_string_good(self): + for start, stop in TestManager.pool_part_ranges: + test_str = (f"{self.prefix}-" f"{start}-{stop}-" f"{self.suffix}") + + try: + self.reservation.validate_string(test_str) + except Exception as exc: + pytest.fail(f"Validating string should succeed: " + f"string {test_str}, exception: {exc}") + + def test_validate_string_bad(self): + for bad_string in self.bad_reservation_strings: + with pytest.raises(AssertionError): + self.reservation.validate_string(bad_string) + + def test_reservation_from_string_good(self): + for start, stop in TestManager.pool_part_ranges: + test_str = (f"{self.prefix}-" f"{start}-{stop}-" f"{self.suffix}") + + reservation = self.reservation.from_str(test_str) + assert test_str == f"{reservation}" + + def test_reservation_from_string_bad(self): + for stop, start in TestManager.pool_part_ranges: + test_str = (f"{self.prefix}-" f"{start}-{stop}-" f"{self.suffix}") + with pytest.raises(AssertionError): + self.reservation.from_str(test_str) diff --git a/tests/functional/utils/rest.py b/tests/functional/utils/rest.py deleted file mode 100644 index 8aa98648ce..0000000000 --- a/tests/functional/utils/rest.py +++ /dev/null @@ -1,147 +0,0 @@ -# -# Copyright (c) 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import numpy as np -import requests -from google.protobuf.json_format import Parse -from tensorflow_serving.apis import get_model_metadata_pb2, \ - get_model_status_pb2 -import logging - -from tests.functional.config import infer_timeout -from tests.functional.utils.port_manager import PortManager -from tests.functional.config import rest_ovms_starting_port, ports_pool_size - -logger = logging.getLogger(__name__) - -DEFAULT_ADDRESS = 'localhost' -DEFAULT_REST_PORT = "{}".format(rest_ovms_starting_port) -PREDICT = ':predict' -METADATA = '/metadata' - -port_manager_rest = PortManager("rest", starting_port=rest_ovms_starting_port, pool_size=ports_pool_size) - - -def get_url(model: str, address: str = DEFAULT_ADDRESS, port: str = DEFAULT_REST_PORT, - version: str = None, service: str = None): - version_string = "" - if version is not None: - version_string = "/versions/{}".format(version) - if version == "all": - version_string = "/all" - - service_string = "" - if service is not None: - service_string = service - rest_url = 'http://{}:{}/v1/models/{}{}{}'.format(address, port, model, version_string, service_string) - return rest_url - - -def get_predict_url(model: str, address: str = DEFAULT_ADDRESS, port: str = DEFAULT_REST_PORT, version: str = None): - return get_url(model=model, address=address, port=port, version=version, service=PREDICT) - - -def get_metadata_url(model: str, address: str = DEFAULT_ADDRESS, port: str = DEFAULT_REST_PORT, version: str = None): - return get_url(model=model, address=address, port=port, version=version, service=METADATA) - - -def get_status_url(model: str, address: str = DEFAULT_ADDRESS, port: str = DEFAULT_REST_PORT, version: str = None): - return get_url(model=model, address=address, port=port, version=version) - - -def prepare_body_format(img, request_format, input_name): - signature = "serving_default" - if request_format == "row_name": - instances = [] - for i in range(0, img.shape[0], 1): - instances.append({input_name: img[i].tolist()}) - data_obj = {"signature_name": signature, "instances": instances} - elif request_format == "row_noname": - data_obj = {"signature_name": signature, 'instances': img.tolist()} - elif request_format == "column_name": - data_obj = {"signature_name": signature, - 'inputs': {input_name: img.tolist()}} - elif request_format == "column_noname": - data_obj = {"signature_name": signature, 'inputs': img.tolist()} - data_json = json.dumps(data_obj) - return data_json - - -def process_json_output(result_dict, output_tensors): - output = {} - if "outputs" in result_dict: - keyname = "outputs" - if type(result_dict[keyname]) is dict: - for output_tensor in output_tensors: - output[output_tensor] = np.asarray( - result_dict[keyname][output_tensor]) - else: - output[output_tensors[0]] = np.asarray(result_dict[keyname]) - elif "predictions" in result_dict: - keyname = "predictions" - if type(result_dict[keyname][0]) is dict: - for row in result_dict[keyname]: - logger.info(row.keys()) - for output_tensor in output_tensors: - if output_tensor not in output: - output[output_tensor] = [] - output[output_tensor].append(row[output_tensor]) - for output_tensor in output_tensors: - output[output_tensor] = np.asarray(output[output_tensor]) - else: - output[output_tensors[0]] = np.asarray(result_dict[keyname]) - else: - logger.debug("Missing required response in {}".format(result_dict)) - - return output - - -def _get_output_json(rest_url, method_to_call, raise_error = True, - img = None, input_tensor = None, request_format = None, timeout=None): - if img is not None: - data_json = prepare_body_format(img, request_format, input_tensor) - else: - data_json = None - result = method_to_call(rest_url, data=data_json, timeout=timeout) - if raise_error and (not result.ok or result.status_code != 200): - msg = f"REST call: {method_to_call.__name__}() failed {result}" - txt = getattr(result, "text", "") - msg += txt - logger.error(msg) - raise Exception(msg) - output_json = json.loads(result.text) - return result.text, output_json - -def infer_rest(img, input_tensor, rest_url, - output_tensors, request_format, raise_error=True): - _, _json = _get_output_json(rest_url, requests.post, raise_error, img, input_tensor, request_format, infer_timeout) - data = process_json_output(_json, output_tensors) - return data - - -def get_model_metadata_response_rest(rest_url): - _txt, _ = _get_output_json(rest_url, requests.get) - metadata_pb = get_model_metadata_pb2.GetModelMetadataResponse() - response = Parse(_txt, metadata_pb, ignore_unknown_fields=True) - return response - - -def get_model_status_response_rest(rest_url): - _txt, _ = _get_output_json(rest_url, requests.get) - status_pb = get_model_status_pb2.GetModelStatusResponse() - response = Parse(_txt, status_pb, ignore_unknown_fields=False) - return response diff --git a/tests/functional/utils/ssl.py b/tests/functional/utils/ssl.py new file mode 100644 index 0000000000..1e3df0becf --- /dev/null +++ b/tests/functional/utils/ssl.py @@ -0,0 +1,98 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from collections import namedtuple + +from grpc import ssl_channel_credentials as grpc_ssl_channel_credentials + +HttpsCerts = namedtuple("HttpsCerts", ["client_cert", "client_key", "server_cert"]) + + +# pylint: disable=too-many-instance-attributes +class SslCertificates: + """ + Load ssl certificates. + """ + + def __init__(self): + self.server_cert_path = None + self.server_cert = None + + self.server_key_path = None + self.server_key = None + + self.client_cert_ca_path = None + self.client_cert_ca = None + + self.client_cert_ca_crl_path = None + self.client_cert_ca_crl = None + + self.dhparam_path = None + self.dhparam = None + + self.client_cert_path = None + self.client_cert = None + + self.client_key_path = None + self.client_key = None + + def _load_file_into_attribute(self, attribute, filepath): + setattr(self, f"{attribute}_path", filepath) + with open(filepath, "rb") as f: + setattr(self, attribute, f.read()) + + def load_server_cert(self, filepath): + self._load_file_into_attribute("server_cert", filepath) + + def load_server_key(self, filepath): + self._load_file_into_attribute("server_key", filepath) + + def load_client_cert_ca(self, filepath): + self._load_file_into_attribute("client_cert_ca", filepath) + + def load_client_cert_ca_crl(self, filepath): + self._load_file_into_attribute("client_cert_ca_crl", filepath) + + def load_dhparam(self, filepath): + self._load_file_into_attribute("dhparam", filepath) + + def load_client_cert(self, filepath): + self._load_file_into_attribute("client_cert", filepath) + + def load_client_key(self, filepath): + self._load_file_into_attribute("client_key", filepath) + + def get_server_cert_bytes(self): + return self.server_cert + + def get_client_key_bytes(self): + return self.client_key + + def get_client_cert_bytes(self): + return self.client_cert + + def get_client_ca_bytes(self): + return self.client_cert_ca + + def get_https_cert(self): + return HttpsCerts(self.client_cert_path, self.client_key_path, self.server_cert_path) + + def get_grpc_ssl_channel_credentials(self): + return grpc_ssl_channel_credentials( + root_certificates=self.get_server_cert_bytes(), + private_key=self.get_client_key_bytes(), + certificate_chain=self.get_client_cert_bytes(), + ) diff --git a/tests/functional/utils/test_framework.py b/tests/functional/utils/test_framework.py new file mode 100644 index 0000000000..d839405a9c --- /dev/null +++ b/tests/functional/utils/test_framework.py @@ -0,0 +1,267 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import pytest +import re +import shutil +import traceback + +from tests.functional.utils.assertions import CreateVenvError, PipInstallError +from tests.functional.utils.git_operations import clone_git_repository +from tests.functional.utils.logger import get_logger +from tests.functional.utils.process import Process, WindowsProcess +from tests.functional.utils.helpers import get_xdist_worker_count, generate_test_object_name + +from tests.functional.config import ( + build_test_image, + language_models_enabled, + machine_is_reserved_for_test_session, + mediapipe_disable, + python_disable, + run_ovms_with_opencl_trace, + run_ovms_with_valgrind, + windows_python_version, +) + +logger = get_logger(__name__) + +STACK_TRACE_SEARCH_PATTERN = "site-packages" + + +class FrameworkMessages: + NOT_IMPLEMENTED = "NOT IMPLEMENTED" + NEXT_VERSION = "NEXT VERSION" + NOT_TO_BE_REPORTED_IF_SKIPPED = "NOT TO BE REPORTED IF SKIPPED" + TWO_USERS_CANT_MOUNT_ON_WINDOWS = "Cannot create mount point by two nctl users on one Windows user" + TWO_USERS_CANT_MOUNT_ON_NON_LINUX = "Cannot create mount point by two nctl users on one non-Linux user" + TEST_ISSUE = "TEST ISSUE" + FEATURE_NOT_READY = "FEATURE NOT READY" + NOT_SUPPORTED_IN_SAFARI = "Not possible to automate on Safari browser" + NOT_EXECUTED = "NOT EXECUTED" + OVMS_C_REPO_ABSENT = "OVMS-C REPO ABSENT" + OVMS_O_REPO_ABSENT = "OVMS-O REPO ABSENT" + FUZZING_TESTS_NOT_ENABLED = "FUZZING TESTS NOT ENABLED" + UAT_TESTS_NOT_ENABLED = "UAT TESTS NOT ENABLED" + OV_TESTS_NOT_ENABLED = "OV TESTS NOT ENABLED" + OVMS_MEDIA_TESTS_NOT_ENABLED = "OVMS MEDIAPIPE TESTS NOT ENABLED" + OS_NOT_SUPPORTED = "{} OS NOT SUPPORTED" + MACHINE_NOT_RESERVED_FOR_TEST_SESSION = "MACHINE NOT RESERVED FOR TEST SESSION" + RUN_IN_PARALLEL= "RUN IN PARALLEL" + TEST_IMAGE_NOT_BUILD = "TEST IMAGE NOT BUILD" + LANGUAGE_MODELS_DISABLED = "LANGUAGE MODELS DISABLED" + MEDIAPIPE_DISABLED = "MEDIAPIPE DISABLED" + PYTHON_DISABLED = "PYTHON DISABLED" + OPENSHIFT_SERVICE_MESH_ENABLED = "OPENSHIFT SERVICE MESH ENABLED" + OPENSHIFT_SERVICE_MESH_DISABLED = "OPENSHIFT SERVICE MESH DISABLED" + ADD_NOTEBOOK_K8S_DISABLED = "ADD NOTEBOOK K8S DISABLED" + BUILD_AND_VERIFY_PACKAGE_DISABLED = "BUILD_AND_VERIFY_PACKAGE_DISABLED" + VLLM_TESTS_NOT_ENABLED = "vLLM TESTS NOT ENABLED" + NGINX_IMAGE_NOT_SUPPORTED = "NGINX IMAGE NOT SUPPORTED" + KFS_GET_MODEL_STATUS_NOT_SUPPORTED = "GetModelStatus is not supported in KFS" + LM_ACCURACY_TESTS_NOT_ENABLED = "Language models accuracy tests not enabled" + CLILOADER_DISABLED = "Cliloader disabled" + VALGRIND_DISABLED = "Valgrind disabled" + + +class TestStatus: + RESULT_PASS = "PASS" + RESULT_FAIL = "FAIL" + RESULT_SKIPPED = "SKIPPED" + RESULT_NOT_EXECUTED = "NOT_EXECUTED" + RESULT_NOT_IMPLEMENTED = FrameworkMessages.NOT_IMPLEMENTED.replace(" ", "_") + RESULT_NEXT_VERSION = FrameworkMessages.NEXT_VERSION.replace(" ", "_") + RESULT_UNKNOWN = "UNKNOWN" + RESULT_NOT_TO_BE_REPORTED = FrameworkMessages.NOT_TO_BE_REPORTED_IF_SKIPPED.replace(" ", "_") + RESULT_TEST_ISSUE = FrameworkMessages.TEST_ISSUE.replace(" ", "_") + RESULT_FEATURE_NOT_READY = FrameworkMessages.FEATURE_NOT_READY.replace(" ", "_") + + +def get_msg_with_stack_trace(msg): + # get stack trace for ovms framework functions only + current_stack_trace = [] + for trace_line in traceback.format_stack()[::-1]: + if STACK_TRACE_SEARCH_PATTERN in trace_line: + break + current_stack_trace.append(trace_line) + return f"{msg}\nStackTrace:\n{' '.join(current_stack_trace[::-1])}" + + +def skip_if_runtime(condition, msg=FrameworkMessages.NOT_TO_BE_REPORTED_IF_SKIPPED): + if condition: + pytest.skip(reason=get_msg_with_stack_trace(msg)) + + +def skip_if(condition, msg=FrameworkMessages.NOT_TO_BE_REPORTED_IF_SKIPPED): + return pytest.mark.skipif(condition, reason=get_msg_with_stack_trace(msg)) + + +def skip_not_implemented(): + return pytest.mark.skip(reason=get_msg_with_stack_trace(FrameworkMessages.NOT_IMPLEMENTED)) + + +def skip_if_language_models_not_enabled(): + return skip_if(not language_models_enabled, msg=FrameworkMessages.LANGUAGE_MODELS_DISABLED) + + +def skip_if_mediapipe_disabled(): + return skip_if(mediapipe_disable, msg=FrameworkMessages.MEDIAPIPE_DISABLED) + + +def skip_if_python_disabled(): + return skip_if(python_disable, msg=FrameworkMessages.PYTHON_DISABLED) + + +def skip_if_cliloader_disabled(): + return skip_if(not run_ovms_with_opencl_trace, msg=FrameworkMessages.CLILOADER_DISABLED) + + +def skip_if_valgrind_disabled(): + return skip_if(not run_ovms_with_valgrind, msg=FrameworkMessages.VALGRIND_DISABLED) + + +def skip_if_build_test_image_not_enabled(): + return skip_if(not build_test_image, msg=FrameworkMessages.TEST_IMAGE_NOT_BUILD) + + +def skip_if_machine_is_not_reserved_for_test_session(): + return skip_if(not machine_is_reserved_for_test_session, + msg=FrameworkMessages.MACHINE_NOT_RESERVED_FOR_TEST_SESSION) + + +def skip_if_run_in_parallel(): + return skip_if(not is_single_threaded(), msg=FrameworkMessages.RUN_IN_PARALLEL) + + +def get_xdist_worker_string(): + return os.environ.get("PYTEST_XDIST_WORKER", "master") + + +def is_xdist_master(): + return get_xdist_worker_string() == "master" + + +def current_pytest_test_case(separator="_"): + test_case = os.environ.get("PYTEST_CURRENT_TEST") + test_case = test_case.split("::")[-1] + test_case = re.sub('[^a-z0-9]+', separator, test_case) + return test_case + + +def get_test_object_prefix(): + # recover e.g.: tmp_k8sworker_342_ from test_object_name + object_name_match = re.match(r"([^\W_]+.[^\W_]+.[^\W_]+.)[^\W_]+.[^\W_]+", generate_test_object_name()) + assert object_name_match, "Can't' find prefix match for test_object_name" + test_object_prefix = object_name_match.group(1) + return test_object_prefix + + +def is_single_threaded(): + return get_xdist_worker_count() <= 1 + + +def get_parameter_from_item(item, param_name, default=None): + _callspec = getattr(item, 'callspec', None) + return _callspec.params.get(param_name, default) if _callspec else default + + +def create_venv_and_install_packages( + work_dir, + requirements_string=None, + requirements_file_path=None, + venv_dir_name=".venv", + pip_additional_options="", + **kwargs +): + print(f"Creating virtualenv in path: {work_dir}") + venv_dir = os.path.join(work_dir, venv_dir_name) + os.makedirs(venv_dir, exist_ok=True) + + process = Process() + process.disable_check_stderr() + + python_version = kwargs.get("python_version", "3") + if isinstance(process, WindowsProcess): + activate_path = os.path.join(venv_dir, "Scripts", "activate.bat") + create_venv_cmd = f"virtualenv {venv_dir} --python=python{windows_python_version}" + install_pip_cmd = f"{activate_path} && python -m pip install -U pip" + else: + activate_path = os.path.join(venv_dir, "bin", "activate") + create_venv_cmd = f"python{python_version} -m venv {venv_dir}" + install_pip_cmd = f". {activate_path} && pip3 install -U pip" + + process.run_and_check(create_venv_cmd, exception_type=CreateVenvError) + process.run_and_check(install_pip_cmd, exception_type=PipInstallError, timeout=900) + if requirements_string is not None or requirements_file_path is not None: + if requirements_file_path is not None: + if isinstance(process, WindowsProcess): + requirements_string = (f"pip install -r {requirements_file_path} --trusted-host download.pytorch.org " + f"--trusted-host storage.openvinotoolkit.org") + else: + requirements_string = f"pip install -r {requirements_file_path} {pip_additional_options}" + + install_requirements_cmd = f"{activate_path} && {requirements_string}"\ + if isinstance(process, WindowsProcess) else \ + f". {activate_path} && {requirements_string}" + print(f"Installing requirements with command: {install_requirements_cmd}\n") + _, stdout, stderr = process.run_and_check_return_all( + install_requirements_cmd, + cwd=work_dir, + exception_type=PipInstallError, + ) + print(f"Stdout: {stdout}\nStderr {stderr}") + + return activate_path + + +def create_venv_and_install_packages_from_git_repo( + repo_url, + repo_path, + repo_branch, + requirements_string=None, + requirements_file_path=None, + venv_dir_name=".venv", + pip_additional_options="", + commit_sha=None, + **kwargs +): + print(f"Clone repository: {repo_url}") + clone_git_repository(repo_url=repo_url, repo_path=repo_path, repo_branch=repo_branch, commit_sha=commit_sha) + + create_venv_and_install_packages( + repo_path, + requirements_string, + requirements_file_path, + venv_dir_name, + pip_additional_options, + **kwargs + ) + + +def change_dir_permissions(dir_path, permissions=0o777): + for root, dirs, files in os.walk(dir_path): + for d in dirs: + os.chmod(os.path.join(root, d), permissions) + for f in files: + os.chmod(os.path.join(root, f), permissions) + + +def remove_dir_contents(dir_path): + for root, dirs, files in os.walk(dir_path): + for f in files: + os.unlink(os.path.join(root, f)) + for d in dirs: + shutil.rmtree(os.path.join(root, d)) diff --git a/tests/reservation_manager.yml b/tests/reservation_manager.yml index 1b94adfc00..8fa28e890b 100644 --- a/tests/reservation_manager.yml +++ b/tests/reservation_manager.yml @@ -16,10 +16,15 @@ # config: pool_range: - start: 10000 - stop: 30000 - pool_part_size: 600 + start: 30000 + stop: 60000 + pool_part_size: 1000 locks_dir: /tmp envs: - ports_prefix: PORTS_PREFIX - slices: 2 + slices: + - start: TT_REST_OVMS_STARTING_PORT + size: TT_PORTS_POOL_SIZE + - start: TT_GRPC_OVMS_STARTING_PORT + size: TT_PORTS_POOL_SIZE + - start: TT_HAPROXY_STARTING_PORT + size: TT_HAPROXY_PORT_POOL_SIZE From 53dadc39436c4d44efea4453485aba35f3b65ce4 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Sun, 10 May 2026 10:18:26 +0200 Subject: [PATCH 05/12] model imports update --- tests/functional/constants/pipelines.py | 6 +++--- tests/functional/fixtures/ovms.py | 2 +- tests/functional/object_model/custom_node.py | 2 +- tests/functional/object_model/inference_helpers.py | 4 ++-- tests/functional/object_model/mediapipe_calculators.py | 2 +- tests/functional/object_model/ovms_binary.py | 2 +- tests/functional/object_model/ovms_capi.py | 2 +- tests/functional/object_model/ovms_config.py | 2 +- tests/functional/object_model/ovms_instance.py | 2 +- tests/functional/object_model/ovms_params.py | 4 ++-- .../object_model/python_custom_nodes/python_custom_nodes.py | 2 +- tests/functional/utils/generative_ai/validation_utils.py | 2 +- tests/functional/utils/hooks.py | 2 +- 13 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/functional/constants/pipelines.py b/tests/functional/constants/pipelines.py index d466d67f4a..fd2ba8588f 100644 --- a/tests/functional/constants/pipelines.py +++ b/tests/functional/constants/pipelines.py @@ -23,9 +23,9 @@ import numpy as np from tests.functional.config import datasets_path -from tests.functional.models.models_datasets import RandomDataset -from tests.functional.models import ModelInfo -from tests.functional.models.models_static import ( +from ovms.constants.model_dataset import RandomDataset +from ovms.constants.models import ModelInfo +from ovms.constants.models import ( AgeGender, ArgMax, CrnnTf, diff --git a/tests/functional/fixtures/ovms.py b/tests/functional/fixtures/ovms.py index 5c86077615..c524929f99 100644 --- a/tests/functional/fixtures/ovms.py +++ b/tests/functional/fixtures/ovms.py @@ -38,7 +38,7 @@ run_ovms_with_opencl_trace, run_ovms_with_valgrind, ) -from tests.functional.models import ModelInfo +from ovms.constants.models import ModelInfo from tests.functional.constants.ovms import CurrentOvmsType, CurrentTarget from tests.functional.constants.ovms_binaries import calculate_ovms_binary_name from tests.functional.constants.ovms_images import ( diff --git a/tests/functional/object_model/custom_node.py b/tests/functional/object_model/custom_node.py index 88dad25660..5c16d43cad 100644 --- a/tests/functional/object_model/custom_node.py +++ b/tests/functional/object_model/custom_node.py @@ -25,7 +25,7 @@ from tests.functional.utils.logger import get_logger from tests.functional.utils.process import Process from tests.functional.config import custom_nodes_path, ovms_c_repo_path -from tests.functional.models import ModelInfo +from ovms.constants.models import ModelInfo from tests.functional.constants.ovms import CurrentOvmsType from tests.functional.constants.ovms_type import OvmsType from tests.functional.constants.paths import Paths diff --git a/tests/functional/object_model/inference_helpers.py b/tests/functional/object_model/inference_helpers.py index ed69824b1f..cc536355f7 100644 --- a/tests/functional/object_model/inference_helpers.py +++ b/tests/functional/object_model/inference_helpers.py @@ -67,14 +67,14 @@ from tests.functional.utils.test_framework import FrameworkMessages, skip_if_runtime from tests.functional.utils.generative_ai.validation_utils import GenerativeAIValidationUtils from tests.functional.config import binary_io_images_path, wait_for_messages_timeout -from tests.functional.models.models_datasets import ( +from ovms.constants.model_dataset import ( BinaryDummyModelDataset, DefaultBinaryDataset, ExactShapeBinaryDataset, LanguageModelDataset, ModelDataset, ) -from tests.functional.models import ModelInfo +from ovms.constants.models import ModelInfo from tests.functional.constants.ovms import CurrentTarget as ct from tests.functional.constants.ovms import MediaPipeConstants, Ovms from tests.functional.constants.pipelines import SimpleMediaPipe diff --git a/tests/functional/object_model/mediapipe_calculators.py b/tests/functional/object_model/mediapipe_calculators.py index d318b31b1a..9614188609 100644 --- a/tests/functional/object_model/mediapipe_calculators.py +++ b/tests/functional/object_model/mediapipe_calculators.py @@ -35,7 +35,7 @@ mediapipe_repo_branch, ovms_c_repo_path, ) -from tests.functional.models import ModelInfo +from ovms.constants.models import ModelInfo from tests.functional.constants.target_device import TargetDevice from tests.functional.constants.ovms import Config, MediaPipeConstants from tests.functional.constants.paths import Paths diff --git a/tests/functional/object_model/ovms_binary.py b/tests/functional/object_model/ovms_binary.py index 7e474adae9..d45c72ab1b 100644 --- a/tests/functional/object_model/ovms_binary.py +++ b/tests/functional/object_model/ovms_binary.py @@ -26,7 +26,7 @@ from tests.functional.utils.process import Process from tests.functional.constants.core import CONTAINER_STATUS_EXITED, CONTAINER_STATUS_RUNNING -from tests.functional.models.models_static import Muse +from ovms.constants.models import Muse from tests.functional.constants.ovms_binaries import get_ovms_binary_cmd_setup from tests.functional.constants.ovms_type import OvmsType from tests.functional.constants.paths import Paths diff --git a/tests/functional/object_model/ovms_capi.py b/tests/functional/object_model/ovms_capi.py index 67e934d364..010d33ac54 100644 --- a/tests/functional/object_model/ovms_capi.py +++ b/tests/functional/object_model/ovms_capi.py @@ -33,7 +33,7 @@ from tests.functional.utils.test_framework import generate_test_object_name, skip_if_runtime from tests.functional.config import ovms_c_repo_path from tests.functional.constants.core import CONTAINER_STATUS_RUNNING -from tests.functional.models import ModelInfo +from ovms.constants.models import ModelInfo from tests.functional.constants.ovms import Config from tests.functional.constants.ovms_messages import OvmsMessages from tests.functional.constants.ovms_type import OvmsType diff --git a/tests/functional/object_model/ovms_config.py b/tests/functional/object_model/ovms_config.py index 0e77953e1d..88230c7576 100644 --- a/tests/functional/object_model/ovms_config.py +++ b/tests/functional/object_model/ovms_config.py @@ -23,7 +23,7 @@ from tests.functional.constants.os_type import OsType from tests.functional.config import enable_plugin_config_target_device from tests.functional.constants.custom_loader import CustomLoaderConsts -from tests.functional.models import ModelInfo +from ovms.constants.models import ModelInfo from tests.functional.constants.ovms import Config, CurrentOvmsType, Ovms, set_plugin_config_boolean_value from tests.functional.constants.ovms_type import OvmsType from tests.functional.constants.paths import Paths diff --git a/tests/functional/object_model/ovms_instance.py b/tests/functional/object_model/ovms_instance.py index eecf2ea112..1f8ad15cfe 100644 --- a/tests/functional/object_model/ovms_instance.py +++ b/tests/functional/object_model/ovms_instance.py @@ -53,7 +53,7 @@ wait_for_messages_timeout, ) from tests.functional.constants.core import CONTAINER_STATUS_EXITED, CONTAINER_STATUS_RUNNING -from tests.functional.models import ModelInfo +from ovms.constants.models import ModelInfo from tests.functional.constants.target_device import MAX_WORKERS_PER_TARGET_DEVICE from tests.functional.constants.ovms import CurrentTarget as ct from tests.functional.constants.ovms import Ovms diff --git a/tests/functional/object_model/ovms_params.py b/tests/functional/object_model/ovms_params.py index 137083390f..8102a54395 100644 --- a/tests/functional/object_model/ovms_params.py +++ b/tests/functional/object_model/ovms_params.py @@ -25,8 +25,8 @@ from tests.functional.object_model.ovms_command import OvmsCommand from tests.functional.config import logging_level_ovms from tests.functional.constants.metrics import MetricsPolicy -from tests.functional.models.models_static import ModelInfo, Muse -from tests.functional.models.models_library import ModelsLib +from ovms.constants.models import ModelInfo, Muse +from ovms.constants.models_library import ModelsLib from tests.functional.object_model.cpu_extension import MuseModelExtension from tests.functional.object_model.custom_loader import CustomLoader diff --git a/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py b/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py index 43bcda8315..67a99890de 100644 --- a/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py +++ b/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py @@ -19,7 +19,7 @@ from tests.functional.utils.inference.communication import GRPC from tests.functional.utils.logger import get_logger from tests.functional.constants.generative_ai import GenerativeAIPluginConfig -from tests.functional.models.models_datasets import LanguageModelDataset +from ovms.constants.model_dataset import LanguageModelDataset from tests.functional.constants.ovms import Ovms from tests.functional.constants.pipelines import MediaPipe, NodesConnection, NodeType, PythonGraphNode from tests.functional.object_model.mediapipe_calculators import HttpLLMCalculator, PythonCalculator, \ diff --git a/tests/functional/utils/generative_ai/validation_utils.py b/tests/functional/utils/generative_ai/validation_utils.py index 0b4ffe60b8..3a3fa9a6a6 100644 --- a/tests/functional/utils/generative_ai/validation_utils.py +++ b/tests/functional/utils/generative_ai/validation_utils.py @@ -33,7 +33,7 @@ from tests.functional.utils.inference.serving.openai import OpenAIWrapper, OpenAIFinishReason from tests.functional.config import save_image_to_artifacts from tests.functional.config import artifacts_dir, pipeline_type -from tests.functional.models.models_datasets import FeatureExtractionModelDataset +from ovms.constants.model_dataset import FeatureExtractionModelDataset logger = get_logger(__name__) diff --git a/tests/functional/utils/hooks.py b/tests/functional/utils/hooks.py index 8601bfb333..e244ae77ee 100644 --- a/tests/functional/utils/hooks.py +++ b/tests/functional/utils/hooks.py @@ -23,7 +23,7 @@ from pathlib import Path from tests.functional import config -from tests.functional.models.models_library import ModelsLib +from ovms.constants.models_library import ModelsLib from tests.functional.utils.reservation_manager.args import parse_args from tests.functional.utils.reservation_manager.manager import Manager as ReservationManager from tests.functional.config import ( From 19ec25f094d9a6de6282b6860a81807ea0fb5d7c Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Sun, 10 May 2026 14:39:37 +0200 Subject: [PATCH 06/12] utils\assertions + remove old conftest --- tests/functional/conftest.py | 195 -------------- tests/functional/utils/assertions.py | 386 +++++++++++++++++++++++++++ 2 files changed, 386 insertions(+), 195 deletions(-) delete mode 100644 tests/functional/conftest.py create mode 100644 tests/functional/utils/assertions.py diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py deleted file mode 100644 index cb9e8c0917..0000000000 --- a/tests/functional/conftest.py +++ /dev/null @@ -1,195 +0,0 @@ -# -# Copyright (c) 2018-2026 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os -from logging import FileHandler - -import pytest -from _pytest._code import ExceptionInfo, filter_traceback # noqa -from _pytest.outcomes import OutcomeException - -from tests.functional.constants.constants import NOT_TO_BE_REPORTED_IF_SKIPPED -from tests.functional.object_model.server import Server -from tests.functional.utils.other import reorder_items_by_fixtures_used -from tests.functional.utils.cleanup import clean_hanging_docker_resources, delete_test_directory, \ - get_containers_with_tests_suffix -from tests.functional.utils.logger import init_logger -from tests.functional.utils.files_operation import get_path_friendly_test_name -from tests.functional.utils.parametrization import get_tests_suffix -from tests.functional.config import test_dir, test_dir_cleanup, artifacts_dir, target_device, enable_pytest_plugins - -logger = logging.getLogger(__name__) - - -if enable_pytest_plugins: - pytest_plugins = [ - 'tests.functional.fixtures.model_download_fixtures', - 'tests.functional.fixtures.model_conversion_fixtures', - 'tests.functional.fixtures.server_detection_model_fixtures', - 'tests.functional.fixtures.server_for_update_fixtures', - 'tests.functional.fixtures.server_local_models_fixtures', - 'tests.functional.fixtures.server_multi_model_fixtures', - 'tests.functional.fixtures.server_remote_models_fixtures', - 'tests.functional.fixtures.server_with_batching_fixtures', - 'tests.functional.fixtures.server_with_version_policy_fixtures', - 'tests.functional.fixtures.test_files_fixtures', - 'tests.functional.fixtures.common_fixtures', - ] - - - def pytest_sessionstart(): - for item in os.environ.items(): - logger.debug(item) - - - def pytest_configure(): - # Perform initial configuration. - init_logger() - - init_conf_logger = logging.getLogger("init_conf") - - container_names = get_containers_with_tests_suffix() - if container_names: - init_conf_logger.info("Possible conflicting container names: {} " - "for given tests_suffix: {}".format(container_names, get_tests_suffix())) - - if artifacts_dir: - os.makedirs(artifacts_dir, exist_ok=True) - - - def pytest_keyboard_interrupt (excinfo): - clean_hanging_docker_resources() - Server.stop_all_instances() - - - def pytest_unconfigure(): - # Perform cleanup. - cleanup_logger = logging.getLogger("cleanup") - - cleanup_logger.info("Cleaning hanging docker resources with suffix: {}".format(get_tests_suffix())) - clean_hanging_docker_resources() - - if test_dir_cleanup: - cleanup_logger.info("Deleting test directory: {}".format(test_dir)) - delete_test_directory() - - if len(Server.running_instances) > 0: - logger.warning("Test got unstopped docker instances") - Server.stop_all_instances() - - - @pytest.hookimpl(hookwrapper=True) - def pytest_collection_modifyitems(session, config, items): - yield - items = reorder_items_by_fixtures_used(session) - - - @pytest.hookimpl(tryfirst=True, hookwrapper=True) - def pytest_runtest_makereport(item, call): - outcome = yield - if call.when == "setup": - report = outcome.get_result() - report.test_metadata = {"start": call.start} - - - @pytest.hookimpl(hookwrapper=True) - def pytest_runtest_call(): - __tracebackhide__ = True - try: - outcome = yield - finally: - pass - exception_catcher("call", outcome) - - - @pytest.hookimpl(hookwrapper=True) - def pytest_runtest_setup(): - __tracebackhide__ = True - try: - outcome = yield - finally: - pass - exception_catcher("setup", outcome) - - - @pytest.hookimpl(hookwrapper=True) - def pytest_runtest_teardown(): - __tracebackhide__ = True - try: - outcome = yield - finally: - pass - exception_catcher("teardown", outcome) - - - @pytest.hookimpl(hookwrapper=True) - def pytest_runtest_teardown(item): - yield - # Test finished: remove test item for all fixtures that was used - for fixture in item._server_fixtures: - if item in item.session._server_fixtures_to_tests[fixture]: - item.session._server_fixtures_to_tests[fixture].remove(item) - if len(item.session._server_fixtures_to_tests[fixture]) == 0: - # No other tests will use this docker instance so we can close it. - Server.stop_by_fixture_name(fixture) - - - def exception_catcher(when: str, outcome): - if isinstance(outcome.excinfo, tuple): - if len(outcome.excinfo) > 1 and isinstance(outcome.excinfo[1], OutcomeException): - return - exception_logger = logging.getLogger("exception_logger") - exception_info = ExceptionInfo.from_exc_info(outcome.excinfo) - exception_info.traceback = exception_info.traceback.filter(filter_traceback) - exc_repr = exception_info.getrepr(style="short", chain=False)\ - if exception_info.traceback\ - else exception_info.exconly() - exception_logger.error('Unhandled Exception during {}: \n{}' - .format(when.capitalize(), str(exc_repr))) - - - @pytest.hookimpl(hookwrapper=True, tryfirst=True) - def pytest_runtest_logstart(nodeid, location): - if artifacts_dir: - test_name = get_path_friendly_test_name(location) - log_path = os.path.join(artifacts_dir, f"{test_name}.log") - _root_logger = logging.getLogger(None) - _root_logger._test_log_handler = FileHandler(log_path) - formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s") - _root_logger._test_log_handler.setFormatter(formatter) - _root_logger.addHandler(_root_logger._test_log_handler) - yield - - - @pytest.hookimpl(hookwrapper=True, tryfirst=True) - def pytest_runtest_logfinish(nodeid, location): - if artifacts_dir: - _root_logger = logging.getLogger(None) - _root_logger.removeHandler(_root_logger._test_log_handler) - yield - - -def devices_not_supported_for_test(not_supported_devices_list): - """ - Comma separated list of devices not supported for test. - Use as a test decorator. - Example use: - @devices_not_supported_for_test(["CPU", "GPU"]) - def test_example(): - # test implementation - """ - return pytest.mark.skipif(target_device in not_supported_devices_list, reason=NOT_TO_BE_REPORTED_IF_SKIPPED) diff --git a/tests/functional/utils/assertions.py b/tests/functional/utils/assertions.py new file mode 100644 index 0000000000..590dd52af3 --- /dev/null +++ b/tests/functional/utils/assertions.py @@ -0,0 +1,386 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import grpc +import json +import os +import pytest +import re +import yaml +from pathlib import Path +from typing import Callable, Type + +from tests.functional.utils.logger import get_logger +from tests.functional.constants.ovms import CurrentOvmsType +from tests.functional.constants.paths import Paths + +logger = get_logger(__name__) +CPP_STD_EXCEPTION = "std::exception" + + +# pylint: disable=too-many-instance-attributes +class OvmsTestException(AssertionError): + def __init__(self, msg=None, ovms_log=None, dmesg_log=None, context=None, **kwargs): + super().__init__(msg) + self.ovms_log = ovms_log + self.dmesg_log = dmesg_log + self.ovms_type = CurrentOvmsType.ovms_type + self.context = context + + def set_process_details(self, cmd=None, retcode=None, stdout=None, stderr=None): + self.cmd = cmd + self.retcode = retcode + self.stdout = stdout + self.stderr = stderr + + def get_process_details(self): + cmd = getattr(self, "cmd", "") + retcode = getattr(self, "retcode", "") + stdout = getattr(self, "stdout", "") + stderr = getattr(self, "stderr", "") + return cmd, retcode, stdout, stderr + + def __str__(self): + msg = super().__str__() + header = f"{self.ovms_type}" + msg = f"\n{header}\n{msg}" + return msg + + +class AggregatedMultipleOvmsTestExceptions(OvmsTestException): + def __init__(self, multiple_exceptions): + self.multiple_ovms_exceptions = multiple_exceptions + + def __str__(self): + return "\n".join(str(e) for e in self.multiple_ovms_exceptions) + + +class UnexpectedResponseError(OvmsTestException): + def __init__(self, status=None, error_message=None, message=None): + message = message or f"Code:{status} Message:{error_message}" + super(UnexpectedResponseError, self).__init__(message) + self.status = status + self.error_message = error_message + + +class TemplateMessageException(Exception): + TEMPLATE = "" + + def __init__(self, message=None): + super().__init__(self.TEMPLATE.format(message)) + + +def assert_raises_exception(exception: Type[BaseException], output, callable_obj, *args, **kwargs): + with pytest.raises(exception) as e: + callable_obj(*args, **kwargs) + assert output in str(e.value), \ + f"Expected output:\n{output}\nnot found in exception {exception.__class__.__name__} value:\n{str(e.value)}" + + +def assert_raises_exception_with_pattern(exception: Type[BaseException], pattern, callable_obj, *args, **kwargs): + with pytest.raises(exception) as e: + callable_obj(*args, **kwargs) + assert pattern.search(str(e.value)), \ + f"Expected output:\n{pattern}\nnot found in exception {exception.__class__.__name__} value:\n{str(e.value)}" + + +def get_mediapipe_details_from_context(context): + client_input_data, client_output_data = None, None + ovms_session = context.ovms_sessions[0] + log_monitor = ovms_session.ovms.create_log(True) + ovms_log = log_monitor.get_logs_as_txt() + config_file = os.path.join(ovms_session.ovms.container_folder, Paths.MODELS_PATH_NAME, Paths.CONFIG_FILE_NAME) + config = json.loads(Path(config_file).read_text()) + mediapipe_model = [model for model in ovms_session.models if model.is_mediapipe][0] + src_code = [calc.src_file_path for calc in mediapipe_model.calculators] + graphs = mediapipe_model.graphs + request = getattr(context, "request", None) + if request: + client_input_data = request.inputs + client_output_data = request.outputs + return ovms_log, config, graphs, src_code, client_input_data, client_output_data + + +def _assert_status_code_and_message(status, error_message_phrase, status_code, error_msg, e, context=None): + try: + error_msg = yaml.load(error_msg, Loader=yaml.Loader) # convert dict saved as string + except (yaml.scanner.ScannerError, yaml.parser.ParserError) as exception: + e = exception + pass + error_msg = error_msg["error"] if getattr(error_msg, "error", None) is not None else str(error_msg) + assert error_message_phrase in error_msg, \ + f"Expected output:\n{error_message_phrase}\nnot found in exception {e.__class__.__name__} value:\n{error_msg}" + assert status == status_code, f"Not expected status code found: got: {status_code}, expected: {status}" + + +def assert_raises_http_exception( + status: int, error_message_phrase: str, callable_obj: Callable, context=None, *args, **kwargs +): + with pytest.raises(UnexpectedResponseError) as e: + callable_obj(*args, **kwargs) + _assert_status_code_and_message(status, error_message_phrase, + e.value.status, e.value.error_message, e, context, *args) + + +def assert_raises_grpc_exception( + status, error_message_phrase: str, callable_obj: Callable, context=None, *args, **kwargs +): + with pytest.raises(grpc.RpcError) as e: + callable_obj(*args, **kwargs) + _assert_status_code_and_message(status, error_message_phrase, + e.value._state.code.value[0], e.value._state.details, e, context, *args) + + +class InstallPkgVersionException(OvmsTestException): + pass + + +class AptInstallException(OvmsTestException): + pass + + +class UpgradePkgException(OvmsTestException): + pass + + +class InvalidMetadataException(OvmsTestException): + pass + + +class ModelNotReadyException(OvmsTestException): + pass + + +class ServerNotLiveException(OvmsTestException): + pass + + +class ServerNotReadyException(OvmsTestException): + pass + + +class SdlException(OvmsTestException): + pass + + +class StatefulModelGeneralException(OvmsTestException): + pass + + +class ModelCacheFailure(OvmsTestException): + pass + + +class ModelCacheIncorrectNumberOfCacheFiles(ModelCacheFailure): + def __int__(self, msg=None): + if msg is None: + msg = "Incorrect number of expected cache files" + super().__init__(msg) + + +class PodNotReadyException(OvmsTestException): + pass + + +class PodCreationException(OvmsTestException): + pass + + +class LogMessageNotFoundException(OvmsTestException): + pass + + +class KubeCtlApplyException(OvmsTestException): + pass + + +class AutomaticCodeReviewException(OvmsTestException): + pass + + +class StartOvmsException(OvmsTestException): + pass + + +class ExampleClientsError(OvmsTestException): + pass + + +class CurlArtifactsError(OvmsTestException): + pass + + +class UnzipError(OvmsTestException): + pass + + +class CreateVenvError(OvmsTestException): + pass + + +class PipInstallError(OvmsTestException): + pass + + +class BuildCustomNodeError(OvmsTestException): + pass + + +class DockerBuildError(OvmsTestException): + pass + + +class OvmsCrashed(OvmsTestException): + pass + + +class DmesgError(OvmsTestException): + msg = "dmesg error" + + +class DockerCannotCloseProperly(OvmsTestException): + pass + + +class DmesgBpFilterFail(DmesgError): + regex = re.compile("bpfilter: .+ fail .+") + + +class BadRIPValue(DmesgError): + msg = "Bad RIP value" + + +class SegfaultError(DmesgError): + msg = "segfault" + + +class GPUHangError(DmesgError): + msg = "GPU HANG" + + +class GeneralProtectionFault(DmesgError): + msg = "general protection fault" + + +class TrapDivideError(DmesgError): + msg = "trap divide error" + + +class OOMKillError(DmesgError): + msg = "oom-kill" + + +class DownloadError(OvmsTestException): + pass + + +class UnwantedMessageError(OvmsTestException): + pass + + +class DocumentationError(OvmsTestException): + pass + + +class MissingLinks(OvmsTestException): + pass + + +class SpellingError(OvmsTestException): + pass + + +class NginxException(OvmsTestException): + phrase_re = re.compile(r"nginx: \[emerg\]") + + +class DocumentationCommandsException(OvmsTestException): + pass + + +class InvalidReturnCodeException(OvmsTestException): + pass + + +class ProvisioningFailure(OvmsTestException): + pass + + +class CloudException(OvmsTestException): + pass + + +class HTTPError(OvmsTestException): + pass + + +class GitCloneException(OvmsTestException): + pass + + +class GitBranchException(OvmsTestException): + pass + + +class FuzzingError(OvmsTestException): + pass + + +class NotSupported(OvmsTestException): + pass + + +class AccuracyException(OvmsTestException): + pass + + +class StreamingApiException(OvmsTestException): + pass + + +class ValgrindException(OvmsTestException): + pass + + +class ModelAnalyzerException(OvmsTestException): + pass + + +class CapiException(OvmsTestException): + pass + + +class CommonDockerException(OvmsTestException): + pass + + +class ConvertModelException(OvmsTestException): + pass + + +class OVVPException(OvmsTestException): + pass + + +def get_exception_by_ovms_log(ovms_log_lines): + exceptions_to_recognize = [NginxException] + + for line in ovms_log_lines: + for e in exceptions_to_recognize: + m = e.phrase_re.search(line) + if m: + return e, line + return None From 9b5ece801a629511efef1a705424536ed8606583 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Sun, 10 May 2026 15:58:09 +0200 Subject: [PATCH 07/12] update config.py --- tests/functional/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/config.py b/tests/functional/config.py index 65dfe2cc9a..3cbc5d8928 100644 --- a/tests/functional/config.py +++ b/tests/functional/config.py @@ -195,11 +195,11 @@ def get_uses_mapping(): is_nginx_mtls = get_bool("TT_IS_NGINX_MTLS", False) """ TT_SKIP_TEST_IF_IS_NGINX_MTLS """ -skip_nginx_test = get_bool("TT_SKIP_TEST_IF_IS_NGINX_MTLS", "True") +skip_nginx_test = get_bool("TT_SKIP_TEST_IF_IS_NGINX_MTLS", True) skip_nginx_test = skip_nginx_test and is_nginx_mtls """ TT_ENABLE_OVMS_C_PYTEST_PLUGINS - enable pytest plugins """ -enable_pytest_plugins = get_bool("TT_ENABLE_OVMS_C_PYTEST_PLUGINS", "True") +enable_pytest_plugins = get_bool("TT_ENABLE_OVMS_C_PYTEST_PLUGINS", True) """ TT_OVMS_C_REPO_PATH - path to ovms-c repository. Can be relative or absolute. """ ovms_c_repo_path = get_path("TT_OVMS_C_REPO_PATH", get_path("PWD", "./")) @@ -266,7 +266,7 @@ def get_uses_mapping(): server_address = os.environ.get("TT_SERVER_ADDRESS", "localhost") """ TT_RESOURCE_MONITOR_ENABLED - Dump ovms container resource statistics once per second """ -resource_monitor_enabled = get_bool("TT_RESOURCE_MONITOR_ENABLED", False) +resource_monitor_enabled = get_bool("TT_RESOURCE_MONITOR_ENABLED", True) """ TT_TEST_TEMP_DIR - directory path where all temporary files are stored, default is not set """ test_temp_dir = os.environ.get("TT_TEST_TEMP_DIR", None) From e3991b2ed8c2c0491dbcf90d142c3b9038e85366 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Mon, 11 May 2026 09:51:46 +0200 Subject: [PATCH 08/12] newest updates added --- tests/functional/constants/ovms_openai.py | 13 ++-- .../object_model/inference_helpers.py | 70 +++++++++++++------ .../object_model/mediapipe_calculators.py | 18 ++++- .../utils/generative_ai/validation_utils.py | 58 +++++++++++---- .../utils/inference/serving/openai.py | 10 ++- 5 files changed, 126 insertions(+), 43 deletions(-) diff --git a/tests/functional/constants/ovms_openai.py b/tests/functional/constants/ovms_openai.py index cc32158571..f3fb0f6871 100644 --- a/tests/functional/constants/ovms_openai.py +++ b/tests/functional/constants/ovms_openai.py @@ -178,17 +178,20 @@ def prepare_dict(self, set_null_values=False, use_extra_body=True): @dataclass class OvmsResponsesRequestParams(OpenAIResponsesRequestParams, OvmsCommonRequestParams): - ignore_eos: bool = None stop: Union[str, list] = None - top_k: int = None + ignore_eos: bool = None include_stop_str_in_output: bool = None + logprobs: bool = None + response_format: dict = None + chat_template_kwargs: dict = None + n: int = None + best_of: int = None + length_penalty: float = None + top_k: int = None repetition_penalty: float = None frequency_penalty: float = None presence_penalty: float = None seed: int = None - best_of: int = None - length_penalty: float = None - n: int = None num_assistant_tokens: int = None assistant_confidence_threshold: float = None max_ngram_size: int = None diff --git a/tests/functional/object_model/inference_helpers.py b/tests/functional/object_model/inference_helpers.py index cc536355f7..ca79e7e650 100644 --- a/tests/functional/object_model/inference_helpers.py +++ b/tests/functional/object_model/inference_helpers.py @@ -486,6 +486,8 @@ def create_audio_transcription(self, audio_file_path, model_name=None, timeout=N **self.request_parameters_dict, timeout=timeout, ) + if self.stream: + return self._collect_audio_stream_text(transcript) return transcript.text def create_audio_translation(self, audio_file_path, model_name=None, timeout=None): @@ -497,8 +499,27 @@ def create_audio_translation(self, audio_file_path, model_name=None, timeout=Non **self.request_parameters_dict, timeout=timeout, ) + if self.stream: + return self._collect_audio_stream_text(translation) return translation.text + @staticmethod + def _collect_audio_stream_text(stream): + """Accumulate text from an OpenAI audio transcription/translation stream. + + Concatenates ``delta`` from ``transcript.text.delta`` events and prefers + the final ``text`` from ``transcript.text.done`` when present. + """ + collected_delta_chunks = [] + final_text = None + for event in stream: + event_type = getattr(event, "type", None) + if event_type == "transcript.text.delta": + collected_delta_chunks.append(getattr(event, "delta", "") or "") + elif event_type == "transcript.text.done": + final_text = getattr(event, "text", None) + return final_text if final_text is not None else "".join(collected_delta_chunks) + @dataclass(frozen=False) class RerankLLMInferenceRequest(LLMInferenceRequest): @@ -1166,6 +1187,8 @@ def run_llm_inference( api_type, port, endpoint, + tools_enabled=False, + validate_tools=False, dataset=None, input_data_type=None, validate_outputs=True, @@ -1196,11 +1219,12 @@ def run_llm_inference( infer_request = infer_request if infer_request is not None else \ RerankLLMInferenceRequest(api_type=api_client, request_parameters=request_parameters) outputs = None + input_content = None if endpoint == OpenAIWrapper.CHAT_COMPLETIONS: - messages = ChatCompletionsApi.prepare_chat_completions_input_content(input_objects) + input_content = ChatCompletionsApi.prepare_chat_completions_input_content(input_objects) if log_request: - log_request_info(request_parameters, model_name, messages) - raw_outputs = infer_request.create_chat_completions(messages, model_name=model_name, timeout=timeout) + log_request_info(request_parameters, model_name, input_content) + raw_outputs = infer_request.create_chat_completions(input_content, model_name=model_name, timeout=timeout) raw_outputs = list(raw_outputs) if infer_request.stream and raw_outputs else raw_outputs if validate_outputs: outputs = GenerativeAIValidationUtils.validate_chat_completions_outputs( @@ -1208,14 +1232,17 @@ def run_llm_inference( outputs=raw_outputs, stream=infer_request.stream, allow_empty_response=allow_empty_response, + tools_enabled=tools_enabled, + validate_tools=validate_tools, + model_instance=model, ) if validate_outputs_ttr: validate_ttr(outputs[0], reference=ttr_reference) elif endpoint == OpenAIWrapper.COMPLETIONS: - prompt = CompletionsApi.prepare_completions_input_content(input_objects) + input_content = CompletionsApi.prepare_completions_input_content(input_objects) if log_request: - log_request_info(request_parameters, model_name, prompt) - raw_outputs = infer_request.create_completions(prompt, model_name=model_name, timeout=timeout) + log_request_info(request_parameters, model_name, input_content) + raw_outputs = infer_request.create_completions(input_content, model_name=model_name, timeout=timeout) raw_outputs = list(raw_outputs) if infer_request.stream and raw_outputs else raw_outputs if validate_outputs: outputs = GenerativeAIValidationUtils.validate_completions_outputs( @@ -1239,14 +1266,17 @@ def run_llm_inference( outputs=raw_outputs, stream=infer_request.stream, allow_empty_response=allow_empty_response, + tools_enabled=True, + validate_tools=validate_tools, + model_instance=model, ) if validate_outputs_ttr: validate_ttr(outputs[0], reference=ttr_reference) elif endpoint == OpenAIWrapper.EMBEDDINGS: - embeddings_input = EmbeddingsApi.prepare_embeddings_input_content(input_objects) + input_content = EmbeddingsApi.prepare_embeddings_input_content(input_objects) if log_request: - log_request_info(request_parameters, model_name, embeddings_input) - raw_outputs = infer_request.create_embeddings(embeddings_input, model_name=model_name, timeout=timeout) + log_request_info(request_parameters, model_name, input_content) + raw_outputs = infer_request.create_embeddings(input_content, model_name=model_name, timeout=timeout) if validate_outputs: outputs = GenerativeAIValidationUtils.validate_embeddings_outputs( model_name=model_name, @@ -1254,10 +1284,10 @@ def run_llm_inference( allow_empty_response=allow_empty_response, ) elif endpoint == CohereWrapper.RERANK: - rerank_input = RerankApi.prepare_rerank_input_content(input_objects) + input_content = RerankApi.prepare_rerank_input_content(input_objects) if log_request: - log_request_info(request_parameters, model_name, rerank_input) - raw_outputs = infer_request.create_rerank(rerank_input, model_name=model_name) + log_request_info(request_parameters, model_name, input_content) + raw_outputs = infer_request.create_rerank(input_content, model_name=model_name) if validate_outputs: outputs = GenerativeAIValidationUtils.validate_rerank_outputs( model_name=model_name, @@ -1265,10 +1295,10 @@ def run_llm_inference( allow_empty_response=allow_empty_response, ) elif endpoint == OpenAIWrapper.IMAGES_GENERATIONS: - prompt = ImagesApi.prepare_image_generation_input_content(input_objects) + input_content = ImagesApi.prepare_image_generation_input_content(input_objects) if log_request: - log_request_info(request_parameters, model_name, prompt) - raw_outputs = infer_request.create_image_generation(prompt, model_name=model_name, timeout=timeout) + log_request_info(request_parameters, model_name, input_content) + raw_outputs = infer_request.create_image_generation(input_content, model_name=model_name, timeout=timeout) if validate_outputs: outputs = GenerativeAIValidationUtils.validate_image_outputs( model_name=model_name, @@ -1277,16 +1307,16 @@ def run_llm_inference( request_parameters=request_parameters, ) elif endpoint == OpenAIWrapper.IMAGES_EDITS: - prompt, image_path = ImagesApi.prepare_image_edit_input_content(input_objects) + input_content, image_path = ImagesApi.prepare_image_edit_input_content(input_objects) mask_path = dataset.mask_path if hasattr(dataset, "mask_path") else None if log_request: message = f"Run request with parameters: '{request_parameters}' for model: '{model_name}' " \ - f"with prompt: '{prompt}', image_path: '{image_path}'" + f"with prompt: '{input_content}', image_path: '{image_path}'" if mask_path is not None: message += f", mask_path: '{mask_path}'" logger.info(message) raw_outputs = infer_request.create_image_edit( - prompt, image_path, mask_path=mask_path, model_name=model_name, timeout=timeout) + input_content, image_path, mask_path=mask_path, model_name=model_name, timeout=timeout) if validate_outputs: outputs = GenerativeAIValidationUtils.validate_image_outputs( model_name=model_name, @@ -1306,7 +1336,7 @@ def run_llm_inference( raise NotImplementedError else: raise NotImplementedError - return outputs, raw_outputs + return outputs, raw_outputs, infer_request, input_content def run_audio_inference( @@ -1381,7 +1411,7 @@ def run_llm_inference_and_validate_against_reference( request_parameters, reference_outputs, ): - outputs = run_llm_inference( + outputs, _, _, _ = run_llm_inference( model, api_type, port, diff --git a/tests/functional/object_model/mediapipe_calculators.py b/tests/functional/object_model/mediapipe_calculators.py index 9614188609..847318cbc1 100644 --- a/tests/functional/object_model/mediapipe_calculators.py +++ b/tests/functional/object_model/mediapipe_calculators.py @@ -908,7 +908,7 @@ class S2tCalculator(LLMCalculator): input_streams: str = None output_streams: str = None node_name: str = None - loopback: bool = False + loopback: bool = True kv_cache_size: int = kv_cache_size_value plugin_config: dict = None best_of_limit: int = None @@ -930,7 +930,13 @@ def create_node_content(self, header, input_streams, output_streams): calculator: "{self.name}" input_side_packet: "STT_NODE_RESOURCES:s2t_servable" {input_streams} + input_stream: "LOOPBACK:loopback" + input_stream_info: {{ + tag_index: "LOOPBACK:0", + back_edge: true + }} {output_streams} + output_stream: "LOOPBACK:loopback" node_options: {{ [type.googleapis.com / mediapipe.S2tCalculatorOptions]: {{ models_path: "{self.models_path_internal}", @@ -938,6 +944,16 @@ def create_node_content(self, header, input_streams, output_streams): {plugin_config_str} }} }} + input_stream_handler {{ + input_stream_handler: "SyncSetInputStreamHandler", + options {{ + [mediapipe.SyncSetInputStreamHandlerOptions.ext] {{ + sync_set {{ + tag_index: "LOOPBACK:0" + }} + }} + }} + }} }}""" return content diff --git a/tests/functional/utils/generative_ai/validation_utils.py b/tests/functional/utils/generative_ai/validation_utils.py index 3a3fa9a6a6..40a454e6de 100644 --- a/tests/functional/utils/generative_ai/validation_utils.py +++ b/tests/functional/utils/generative_ai/validation_utils.py @@ -108,8 +108,8 @@ def validate_finish_reason(endpoint, raw_outputs, request_params, finish_reason) if endpoint == OpenAIWrapper.RESPONSES: if request_params.stream: if hasattr(raw_output, "response"): - if finish_reason == OpenAIFinishReason.STOP: - stream_finish_reason = OpenAIFinishReason.STOP \ + if finish_reason in (OpenAIFinishReason.STOP, OpenAIFinishReason.TOOL_CALLS): + stream_finish_reason = finish_reason \ if raw_output.response.completed_at is not None else \ raw_output.response.completed_at else: @@ -119,7 +119,7 @@ def validate_finish_reason(endpoint, raw_outputs, request_params, finish_reason) assert stream_finish_reason in [None, finish_reason], \ error_message.format(stream_finish_reason, finish_reason) else: - if finish_reason == OpenAIFinishReason.STOP: + if finish_reason in (OpenAIFinishReason.STOP, OpenAIFinishReason.TOOL_CALLS): assert raw_output.completed_at is not None, \ error_message.format(raw_output.completed_at, finish_reason) else: @@ -256,8 +256,28 @@ def validate_choice( return cls.validate_llm_outputs(model_name, outputs, stream, validate_choice, allow_empty_response, **kwargs) + @staticmethod + def _validate_responses_tool_output(output_item, outputs_content): + if output_item.type == "function_call": + # When tools are enabled content might not be empty + assert output_item.arguments is not None, f"Empty tool calls: {output_item.arguments}" + logger.info(output_item) + outputs_content.append(output_item) + elif output_item.type == "message": + assert output_item.role == "assistant", f"Unexpected role: {output_item.role}" + assert output_item.status == "completed", f"Unexpected status: {output_item.status}" + @classmethod - def validate_responses_outputs(cls, model_name, outputs, stream=False, allow_empty_response=False, **kwargs): + def validate_responses_outputs( + cls, + model_name, + outputs, + stream=False, + allow_empty_response=False, + tools_enabled=False, + validate_tools=False, + **kwargs + ): logger.info(outputs) outputs_content = [] assert outputs is not None and len(outputs) > 0, f"No output collected for node with model: {model_name}" @@ -272,20 +292,28 @@ def validate_responses_outputs(cls, model_name, outputs, stream=False, allow_emp elif output.type in ("response.completed", "response.incomplete"): if not allow_empty_response: assert len(stream_content) > 0, f"Empty stream_content: {stream_content}" - assert "".join(stream_content) == output.response.output_text, \ - f"stream_content: {stream_content} does not match output_text: {output.response.output_text}" - logger.info(output.response.output_text) - outputs_content.append(output.response.output_text) + if tools_enabled and validate_tools: + for output_item in output.response.output: + cls._validate_responses_tool_output(output_item, outputs_content) + else: + assert "".join(stream_content) == output.response.output_text, \ + f"stream_content: {stream_content} does not match output: {output.response.output_text}" + logger.info(output.response.output_text) + outputs_content.append(output.response.output_text) else: assert output.model == model_name, f"Invalid model name: {output.model}; Expected: {model_name}" for output_item in output.output: - if output_item.type == "message": - for content_item in output_item.content: - if content_item.type == "output_text": - if not allow_empty_response: - assert content_item.text, f"Empty response content: {content_item}" - logger.info(content_item.text) - outputs_content.append(content_item.text) + if tools_enabled and validate_tools: + cls._validate_responses_tool_output(output_item, outputs_content) + else: + if output_item.type == "message": + for content_item in output_item.content: + if content_item.type == "output_text": + if not allow_empty_response: + assert content_item.text, f"Empty response content: {content_item}" + logger.info(content_item.text) + outputs_content.append(content_item.text) + return outputs_content @classmethod diff --git a/tests/functional/utils/inference/serving/openai.py b/tests/functional/utils/inference/serving/openai.py index 39f1b82a20..6558e6f59d 100644 --- a/tests/functional/utils/inference/serving/openai.py +++ b/tests/functional/utils/inference/serving/openai.py @@ -43,6 +43,7 @@ class OpenAIWrapper(LLMCommonWrapper): AUDIO_TRANSLATIONS = "audio/translations" AVAILABLE_COMPLETIONS_ENDPOINTS = [CHAT_COMPLETIONS, COMPLETIONS] AVAILABLE_TEXT_GENERATION_ENDPOINTS = [CHAT_COMPLETIONS, COMPLETIONS, RESPONSES] + AVAILABLE_TEXT_GENERATION_TOOLS_ENDPOINTS = [CHAT_COMPLETIONS, RESPONSES] AVAILABLE_IMAGES_ENDPOINTS = [IMAGES_GENERATIONS, IMAGES_EDITS] AVAILABLE_MODELS_ENDPOINTS = [MODELS_LIST, MODELS_RETRIEVE] AVAILABLE_AUDIO_ENDPOINTS = [AUDIO_SPEECH, AUDIO_TRANSCRIPTIONS, AUDIO_TRANSLATIONS] @@ -354,10 +355,11 @@ class OpenAIFinishReason: class OpenAIResponsesRequestParams(OpenAIRequestParams): stream: bool = None max_output_tokens: int = None - temperature: float = None - top_p: float = None tools: list = None tool_choice: Union[dict, str] = None + reasoning: Union[dict, str] = None + temperature: float = None + top_p: float = None def set_default_values(self, **kwargs): self.stream = kwargs.get("stream", False) @@ -386,16 +388,20 @@ class OpenAIAudioTranscriptionsRequestParams(OpenAIRequestParams): language: str = None temperature: float = None timestamp_granularities: list = None + stream: bool = None def set_default_values(self, **kwargs): self.language = "en" self.temperature = 0.0 self.timestamp_granularities = ["segment"] + self.stream = kwargs.get("stream", None) @dataclass class OpenAIAudioTranslationsRequestParams(OpenAIRequestParams): temperature: float = None + stream: bool = None def set_default_values(self, **kwargs): self.temperature = 0.0 + self.stream = kwargs.get("stream", None) From 82097cb72c6d589cef39706412cc877322f90431 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Mon, 11 May 2026 10:09:06 +0200 Subject: [PATCH 09/12] add api_types to unittests --- .../reservation_manager/unittests/test_manager.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/functional/utils/reservation_manager/unittests/test_manager.py b/tests/functional/utils/reservation_manager/unittests/test_manager.py index 96975dad77..49d2af731c 100644 --- a/tests/functional/utils/reservation_manager/unittests/test_manager.py +++ b/tests/functional/utils/reservation_manager/unittests/test_manager.py @@ -31,6 +31,7 @@ class TestManager: (7000, 8000), ] + @pytest.mark.api_enabling def test_calculate_all_pool_parts(self, mocker): conf_mgr = mocker.MagicMock() conf_mgr.pool_range_start = 20000 @@ -45,6 +46,7 @@ def test_calculate_all_pool_parts(self, mocker): assert len(mgr.all_pool_parts) == expected_pool_size + @pytest.mark.api_enabling def test_get_reservation_json(self, mocker): conf_mgr = ManagerConfig() env_mgr = EnvManager() @@ -58,6 +60,7 @@ def test_get_reservation_json(self, mocker): assert "shell_envs_file" in json assert env_mgr.get_json.call_count == 1 + @pytest.mark.api_enabling def test_get_reservation_shell_envs(self, mocker): conf_mgr = ManagerConfig() env_mgr = EnvManager() @@ -93,6 +96,7 @@ class TestPoolPart: ((2150, 2200), (2000, 2100), (True)), ] + @pytest.mark.api_enabling def test_ranges_good(self): for start, stop in TestManager.pool_part_ranges: try: @@ -101,11 +105,13 @@ def test_ranges_good(self): pytest.fail(f"Creating PoolPart should succeed with range: " f"start {start}, stp: {stop}") + @pytest.mark.api_enabling def test_ranges_bad(self): for stop, start in TestManager.pool_part_ranges: with pytest.raises(AssertionError): PoolPart(start, stop) + @pytest.mark.api_enabling def test_is_intersect_with(self): for range1, range2, should_be_valid in self.intersect_test_set: pool_part_range1 = PoolPart(range1[0], range1[1]) @@ -134,6 +140,7 @@ class TestReservation: "wrong-2000-1000-range", ] + @pytest.mark.api_enabling def test_validate_string_good(self): for start, stop in TestManager.pool_part_ranges: test_str = (f"{self.prefix}-" f"{start}-{stop}-" f"{self.suffix}") @@ -144,11 +151,13 @@ def test_validate_string_good(self): pytest.fail(f"Validating string should succeed: " f"string {test_str}, exception: {exc}") + @pytest.mark.api_enabling def test_validate_string_bad(self): for bad_string in self.bad_reservation_strings: with pytest.raises(AssertionError): self.reservation.validate_string(bad_string) + @pytest.mark.api_enabling def test_reservation_from_string_good(self): for start, stop in TestManager.pool_part_ranges: test_str = (f"{self.prefix}-" f"{start}-{stop}-" f"{self.suffix}") @@ -156,6 +165,7 @@ def test_reservation_from_string_good(self): reservation = self.reservation.from_str(test_str) assert test_str == f"{reservation}" + @pytest.mark.api_enabling def test_reservation_from_string_bad(self): for stop, start in TestManager.pool_part_ranges: test_str = (f"{self.prefix}-" f"{start}-{stop}-" f"{self.suffix}") From e8a6f22a4ccc330298a6abeb0f0f67562ec2290d Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Mon, 11 May 2026 20:42:01 +0200 Subject: [PATCH 10/12] update Paths --- tests/functional/constants/paths.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/functional/constants/paths.py b/tests/functional/constants/paths.py index c38b76add2..bb97f5775d 100644 --- a/tests/functional/constants/paths.py +++ b/tests/functional/constants/paths.py @@ -54,6 +54,11 @@ class Paths: OVMS_TEST_CAPI_WRAPPER_MAKEFILE = os.path.join(OVMS_TEST_CAPI_WRAPPER_DIR, "Makefile") OVMS_TEST_CAPI_WRAPPER_SETUP = os.path.join(OVMS_TEST_CAPI_WRAPPER_DIR, "setup.py") + # OVMS-C FILES + LLM_EXPORT_MODELS_DIR = os.path.join(config.ovms_c_repo_path, "demos", "common", "export_models") + LLM_EXPORT_MODELS_REQUIREMENTS = os.path.join(LLM_EXPORT_MODELS_DIR, "requirements.txt") + LLM_EXPORT_MODELS_SCRIPT = os.path.join(LLM_EXPORT_MODELS_DIR, "export_model.py") + @staticmethod def CAPI_WRAPPER_PACKAGE_CONTENT_PATH(base_os): return os.path.join(config.c_api_wrapper_dir, base_os, "ovms") From 8d7b9967df1fbcadad08945ab7f2e878d280ed26 Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Tue, 12 May 2026 11:13:24 +0200 Subject: [PATCH 11/12] update spelling + fixes --- tests/functional/config.py | 2 +- tests/functional/constants/ovms_messages.py | 2 +- tests/functional/constants/paths.py | 2 +- .../data/ovms_capi_wrapper/ovms_autopxd.py | 4 +-- tests/functional/object_model/custom_node.py | 4 ++- tests/functional/object_model/ovms_docker.py | 2 +- .../python_custom_nodes.py | 4 +-- .../object_model/resource_monitor.py | 4 +++ .../object_model/test_environment.py | 8 +++--- tests/functional/utils/docker.py | 4 +++ tests/functional/utils/hooks.py | 2 +- tests/functional/utils/log_monitor.py | 4 +++ .../ovms_testing_image/Dockerfile.ubuntu | 28 ++++++++----------- .../throw_exceptions/ThrowExceptions.cpp | 2 +- tests/functional/utils/process.py | 2 +- .../utils/reservation_manager/env_manager.py | 2 +- .../utils/reservation_manager/manager.py | 16 +++++------ .../utils/reservation_manager/runner.py | 2 +- 18 files changed, 52 insertions(+), 42 deletions(-) diff --git a/tests/functional/config.py b/tests/functional/config.py index 3cbc5d8928..90d0c863a0 100644 --- a/tests/functional/config.py +++ b/tests/functional/config.py @@ -243,7 +243,7 @@ def get_uses_mapping(): save_image_to_artifacts = get_bool("TT_SAVE_IMAGE_TO_ARTIFACTS", False) """TT_SET_NO_PROXY""" -set_no_proxy = os.environ.get("TT_SET_NO_PROXY", True) +set_no_proxy = get_bool("TT_SET_NO_PROXY", True) no_proxy = os.environ.get("no_proxy", "") if set_no_proxy: os.environ["NO_PROXY"] = no_proxy diff --git a/tests/functional/constants/ovms_messages.py b/tests/functional/constants/ovms_messages.py index 456b94d753..b318a9412f 100644 --- a/tests/functional/constants/ovms_messages.py +++ b/tests/functional/constants/ovms_messages.py @@ -256,7 +256,7 @@ class OvmsMessages: ERROR_EXCEPTION_CATCH = "Exception catch:" CPU_EXTENSION_LOADING_CUSTOM_CPU_EXT = "Loading custom CPU extension from {}" - CPU_EXTENSION_LOADED = "Custom CPU extention loaded. Adding it." + CPU_EXTENSION_LOADED = "Custom CPU extension loaded. Adding it." CPU_EXTENSION_ADDED = "Extension added." ERROR_CPU_EXTENSION_WILL_NOW_TERMINATE = "- will now terminate." diff --git a/tests/functional/constants/paths.py b/tests/functional/constants/paths.py index bb97f5775d..6a0636d3a5 100644 --- a/tests/functional/constants/paths.py +++ b/tests/functional/constants/paths.py @@ -68,7 +68,7 @@ def get_target_device_lock_file(target_device, i): if isinstance(target_device, str): assert not all(x in target_device for x in [TargetDevice.GPU, TargetDevice.NPU]) - # generalize HETERO/AUTO/MUTLI:X => `X` + # generalize HETERO/AUTO/MULTI:X => `X` if TargetDevice.GPU in target_device: return os.path.join(config.ovms_file_locks_dir, f"target_device_{TargetDevice.GPU}_{i}.lock") if TargetDevice.NPU in target_device: diff --git a/tests/functional/data/ovms_capi_wrapper/ovms_autopxd.py b/tests/functional/data/ovms_capi_wrapper/ovms_autopxd.py index 2380f0211b..081d9c40bf 100644 --- a/tests/functional/data/ovms_capi_wrapper/ovms_autopxd.py +++ b/tests/functional/data/ovms_capi_wrapper/ovms_autopxd.py @@ -73,5 +73,5 @@ def translate(self, code): input_file_path = Path(args.input_file) output_file_path = Path(args.output_file) - with open(output_file_path, "w") as fo: - fo.write(OvmsAutoPxd(input_file_path.name).translate(input_file_path.read_text())) + with open(output_file_path, "w") as file_object: + file_object.write(OvmsAutoPxd(input_file_path.name).translate(input_file_path.read_text())) diff --git a/tests/functional/object_model/custom_node.py b/tests/functional/object_model/custom_node.py index 5c16d43cad..6ffa56403f 100644 --- a/tests/functional/object_model/custom_node.py +++ b/tests/functional/object_model/custom_node.py @@ -140,7 +140,9 @@ def __post_init__(self): @dataclass class OvmsTestDevCustomNode(DevCustomNode): def __post_init__(self): - self.src_dir = os.path.join(ovms_test_repo_path, "data", "ovms_testing_image", Paths.CUSTOM_NODE_PATH_NAME) + self.src_dir = os.path.join( + ovms_c_repo_path, "tests", "functional", "utils", "ovms_testing_image", Paths.CUSTOM_NODE_PATH_NAME + ) self.src_file_path = os.path.join(self.src_dir, self.name, f"{self.name}.{self.src_type}") super().__post_init__() diff --git a/tests/functional/object_model/ovms_docker.py b/tests/functional/object_model/ovms_docker.py index 2796e0b115..dca63d28d5 100644 --- a/tests/functional/object_model/ovms_docker.py +++ b/tests/functional/object_model/ovms_docker.py @@ -643,7 +643,7 @@ def __init__(self, docker_id, name, container_folder, rest_port, grpc_port, targ ) def fetch_and_store_ovms_pid(self, timeout=60): - self._dmesg_log.ovms_pid = None # Not implemetned yet + self._dmesg_log.ovms_pid = None # Not implemented yet def _create_logger(self): return OvmsCmdLineDockerLogMonitor(self.docker_id) diff --git a/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py b/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py index 67a99890de..bd5947f6ae 100644 --- a/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py +++ b/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py @@ -137,8 +137,8 @@ def get_expected_output(self, input_data: dict, client_type: str = None): # For REST API and BYTES type, every batch is always preceding by the 4 bytes, that contains its size # [42, 0, 0, 0] is constant value for "Lorem ipsum dolor sit amet" # https://github.com/openvinotoolkit/model_server/blob/main/docs/model_server_rest_api_kfs.md - splitted = np.array_split(np.array(char_array, dtype=np.object_), multiply_value) - extended_with_length = [np.insert(elem, 0, [42, 0, 0, 0]) for elem in splitted] + elements = np.array_split(np.array(char_array, dtype=np.object_), multiply_value) + extended_with_length = [np.insert(elem, 0, [42, 0, 0, 0]) for elem in elements] output_data[self.output_names[i]] = np.concatenate(extended_with_length, dtype=np.object_) return output_data diff --git a/tests/functional/object_model/resource_monitor.py b/tests/functional/object_model/resource_monitor.py index 45f48ddc44..5787b69b1f 100644 --- a/tests/functional/object_model/resource_monitor.py +++ b/tests/functional/object_model/resource_monitor.py @@ -66,6 +66,8 @@ class DockerResourceMonitor(ResourceMonitor): # "CPU_USAGE": lambda x: # [cpu / x['cpu_stats']['cpu_usage']['total_usage'] for cpu in x['cpu_stats']['cpu_usage']['percpu_usage']], } + # Optional callback invoked after save_data with (log_path). + on_data_saved = None def __init__(self, container): super().__init__() @@ -96,6 +98,8 @@ def save_data(self): writer = csv.DictWriter(csvfile, fieldnames=DockerResourceMonitor.FIELDS) writer.writeheader() writer.writerows(self.rows) + if DockerResourceMonitor.on_data_saved: + DockerResourceMonitor.on_data_saved(log_path) return log_path def plot_fo_file(self, x, y, field, filename): diff --git a/tests/functional/object_model/test_environment.py b/tests/functional/object_model/test_environment.py index 3c10356943..c6663a51e4 100644 --- a/tests/functional/object_model/test_environment.py +++ b/tests/functional/object_model/test_environment.py @@ -37,11 +37,11 @@ def update_model_files(model, models_dir): if hasattr(model, "max_position_embeddings") and model.max_position_embeddings is not None: config_file_path = os.path.join(models_dir[0], model.name, "config.json") if os.path.exists(config_file_path): - with open(config_file_path, "r") as fo: - config_data = json.load(fo) + with open(config_file_path, "r") as file_object: + config_data = json.load(file_object) config_data["max_position_embeddings"] = model.max_position_embeddings - with open(config_file_path, "w") as fo: - json.dump(config_data, fo) + with open(config_file_path, "w") as file_object: + json.dump(config_data, file_object) logger.info( f"max_position_embeddings value was updated to {model.max_position_embeddings} " f"in model's config file: {config_file_path}." diff --git a/tests/functional/utils/docker.py b/tests/functional/utils/docker.py index 1cdbfae08f..51f86e70dd 100644 --- a/tests/functional/utils/docker.py +++ b/tests/functional/utils/docker.py @@ -107,6 +107,8 @@ class DockerContainer(metaclass=ABCMeta): NOT_ON_LIST_RETRY = {"tries": 10, "delay": 2} GETTING_LOGS_RETRY = COMMON_RETRY GETTING_STATUS_RETRY = COMMON_RETRY + # Optional callback invoked before log check with (container_name). + on_log_check = None def __init__( self, @@ -289,6 +291,8 @@ def check_non_empty_logs(self, specific_str: str, acceptable_logs_length_trigger def ensure_logs_contain_specific_str( self, specific_str: str, acceptable_logs_length_trigger: int = 0, retry_kwargs: dict = None, **kwargs ): + if DockerContainer.on_log_check: + DockerContainer.on_log_check(self.name) args = [specific_str, acceptable_logs_length_trigger] getting_logs_retry = self.GETTING_LOGS_RETRY.copy() if retry_kwargs: diff --git a/tests/functional/utils/hooks.py b/tests/functional/utils/hooks.py index e244ae77ee..3778f7ac02 100644 --- a/tests/functional/utils/hooks.py +++ b/tests/functional/utils/hooks.py @@ -92,7 +92,7 @@ def cleanup_docker(cleanup_docker_func): try: cleanup_docker_func() except docker_errors.APIError as error: - logger.warning(f"Error occured during docker cleanup: {error}") + logger.warning(f"Error occurred during docker cleanup: {error}") def cleanup_docker_containers(): diff --git a/tests/functional/utils/log_monitor.py b/tests/functional/utils/log_monitor.py index 87ff8a5e47..f7138891a6 100644 --- a/tests/functional/utils/log_monitor.py +++ b/tests/functional/utils/log_monitor.py @@ -27,6 +27,8 @@ class LogMonitor(ABC): + # Optional callback invoked after save_to_file with (file_path, filename). + on_file_saved = None def __init__(self, **kwargs): self.context = None @@ -102,6 +104,8 @@ def save_to_file(filename, logs): with open(file_path, "w", encoding="utf-8") as fd: for line in logs: fd.write(f"{line}\n") + if LogMonitor.on_file_saved: + LogMonitor.on_file_saved(filename) return file_path def reset_to_logger_creation(self): diff --git a/tests/functional/utils/ovms_testing_image/Dockerfile.ubuntu b/tests/functional/utils/ovms_testing_image/Dockerfile.ubuntu index 17d7c572ee..2b288b0176 100644 --- a/tests/functional/utils/ovms_testing_image/Dockerfile.ubuntu +++ b/tests/functional/utils/ovms_testing_image/Dockerfile.ubuntu @@ -1,23 +1,19 @@ # -# INTEL CONFIDENTIAL -# Copyright (c) 2023-2025 Intel Corporation +# Copyright (c) 2026 Intel Corporation # -# The source code contained or described herein and all documents related to -# the source code ("Material") are owned by Intel Corporation or its suppliers -# or licensors. Title to the Material remains with Intel Corporation or its -# suppliers and licensors. The Material contains trade secrets and proprietary -# and confidential information of Intel or its suppliers and licensors. The -# Material is protected by worldwide copyright and trade secret laws and treaty -# provisions. No part of the Material may be used, copied, reproduced, modified, -# published, uploaded, posted, transmitted, distributed, or disclosed in any way -# without Intel's prior express written permission. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# No license under any patent, copyright, trade secret or other intellectual -# property right is granted to or conferred upon you by disclosure or delivery -# of the Materials, either expressly, by implication, inducement, estoppel or -# otherwise. Any license under such intellectual property rights must be express -# and approved by Intel in writing. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + ARG BASE_IMAGE=openvino/model_server:latest FROM $BASE_IMAGE as base_image diff --git a/tests/functional/utils/ovms_testing_image/cpu_extensions/throw_exceptions/ThrowExceptions.cpp b/tests/functional/utils/ovms_testing_image/cpu_extensions/throw_exceptions/ThrowExceptions.cpp index d53418f7d7..3e12b29dc4 100644 --- a/tests/functional/utils/ovms_testing_image/cpu_extensions/throw_exceptions/ThrowExceptions.cpp +++ b/tests/functional/utils/ovms_testing_image/cpu_extensions/throw_exceptions/ThrowExceptions.cpp @@ -33,7 +33,7 @@ class ThrowExceptions : public ov::op::Op { This extension was written against Resnet50-Binary model. Intention is to "hijack" all layers with type="Multiply" (Our method 'evaluate(...)' will be called instead). We use "Multiply" because it is first layer type that use input tensor values applied to model. - Whole purpose of this extension is to use insted "" from resnet50-binary-0001.xml: + Whole purpose of this extension is to use instead "" from resnet50-binary-0001.xml: ... // id="0" - input parameter layer. diff --git a/tests/functional/utils/process.py b/tests/functional/utils/process.py index 66c9c95047..93f0ca3a24 100644 --- a/tests/functional/utils/process.py +++ b/tests/functional/utils/process.py @@ -438,7 +438,7 @@ def get_pid_details_as_dict(pid): proc_status_dict[key.strip()] = ":".join(val).strip() # If len(val) > 1 again join val into string. return proc_status_dict except FileNotFoundError: - pass # Do not worry about it proc was killed definitly, most likely hazard during process killing + pass # Do not worry about it proc was killed definitely, most likely hazard during process killing except Exception as exc: logger.exception(str(exc)) return None diff --git a/tests/functional/utils/reservation_manager/env_manager.py b/tests/functional/utils/reservation_manager/env_manager.py index bc0a484f69..008b5a19bf 100644 --- a/tests/functional/utils/reservation_manager/env_manager.py +++ b/tests/functional/utils/reservation_manager/env_manager.py @@ -90,7 +90,7 @@ def manage_reservation_environments(self): ports_prefixes += f"{str(slice_start)[:-2]} " - # Update environemnt but don't fail if no slice keys were provided + # Update environment but don't fail if no slice keys were provided self.update_env_for_slice(pool_part_slice, "start", slice_start) self.update_env_for_slice(pool_part_slice, "end", slice_stop) self.update_env_for_slice(pool_part_slice, "size", slice_size) diff --git a/tests/functional/utils/reservation_manager/manager.py b/tests/functional/utils/reservation_manager/manager.py index 5ebdf9af2a..0feb6177e3 100644 --- a/tests/functional/utils/reservation_manager/manager.py +++ b/tests/functional/utils/reservation_manager/manager.py @@ -112,7 +112,7 @@ def register_existing_reservations(self): def _create_reservation_file(self, reservation): """ Create and save information about reservation file. - Concurently not safe. + Concurrently not safe. Exceptions: - pathlib.FileExistsError if file exists """ @@ -177,13 +177,13 @@ def get_available_reservation(self): def reserve_and_return(self): """ Create reservation and return. - Concurently safe. + Concurrently safe. """ logger.info("Locking reservation procedure") try: self.reservation_lock.acquire() - logger.info("Reservation lock aquired") + logger.info("Reservation lock acquired") logger.info("Attempting reservation") reservation = self.get_available_reservation() @@ -588,16 +588,16 @@ def __str__(self): @staticmethod def validate_string(reservation): """Checks if string represents valid reservation""" - splitted = reservation.split("-") + elements = reservation.split("-") - assert len(splitted) == 4, ( + assert len(elements) == 4, ( "Reservation string should consist of 4 elements: " - "splitted with dash '-': " + "split with dash '-': " "'locks prefix', 'pool part start', " "'pool part stop', 'reserver identifier'") - pool_part_start_str = splitted[1] - pool_part_stop_str = splitted[2] + pool_part_start_str = elements[1] + pool_part_stop_str = elements[2] try: pool_part_start = int(pool_part_start_str) diff --git a/tests/functional/utils/reservation_manager/runner.py b/tests/functional/utils/reservation_manager/runner.py index 6132ee1fd1..2db703c0b8 100644 --- a/tests/functional/utils/reservation_manager/runner.py +++ b/tests/functional/utils/reservation_manager/runner.py @@ -22,7 +22,7 @@ class Runner: """ - Manage running command, setup evironment variables + Manage running command, setup environment variables accoarding to Manager.Reservation instance. """ def __init__(self, command, env_mgr): From 755d9e4c03d40c9c44158be95cc4185ed4a3637a Mon Sep 17 00:00:00 2001 From: Natalia Groza Date: Tue, 12 May 2026 13:09:01 +0200 Subject: [PATCH 12/12] pylint updates --- tests/functional/config.py | 4 +- .../constants/target_device_configuration.py | 41 +++++++++---------- tests/functional/utils/environment_info.py | 4 +- tests/functional/utils/helpers.py | 6 +-- tests/functional/utils/test_framework.py | 3 +- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/functional/config.py b/tests/functional/config.py index 90d0c863a0..8c46d0d4cd 100644 --- a/tests/functional/config.py +++ b/tests/functional/config.py @@ -49,7 +49,7 @@ def get_uses_mapping(): return _uses_mapping -""" +""" TT_USES_MAPPING - use mapping JSON for model inputs/output name aliasing Possible TT_USES_MAPPING values (case insensitive): - (empty)/""/NONE - Default leave mapping.json provided alongside model untouched (if exists). @@ -402,6 +402,6 @@ def get_ovms_types(): return ovms_types_list -""" TT_OVMS_TYPE - ovms type runtime to be executed: +""" TT_OVMS_TYPE - ovms type runtime to be executed: DOCKER, BINARY, BINARY_DOCKER, CAPI, CAPI_DOCKER, DOCKER_CMD_LINE """ ovms_types = get_ovms_types() diff --git a/tests/functional/constants/target_device_configuration.py b/tests/functional/constants/target_device_configuration.py index c4eb95328a..d63eaaed86 100644 --- a/tests/functional/constants/target_device_configuration.py +++ b/tests/functional/constants/target_device_configuration.py @@ -20,7 +20,7 @@ getgrnam = None try: - from os import getuid + from os import getuid # pylint: disable=unused-import except ImportError: getuid = None @@ -37,28 +37,27 @@ DEVICES = "devices" HOST = "host" DOCKER_PARAMS = "docker_params" -""" TARGET_DEVICE_CONFIGURATION: VOLUMES - this map stores a list of devices that should be - mounted for given target device. - String representing device should be in form of: - - :: +""" +TARGET_DEVICE_CONFIGURATION: + VOLUMES - this map stores a list of devices that should be + mounted for given target device. + String representing device should be in form of: + - :: - Another representations are possible - - : - cgroup_permissions will be set to `mrw` by default + Another representations are possible + - : + cgroup_permissions will be set to `mrw` by default - - - path_in_container will be the same as path_on_host - cgroup_permissions will be set to `mrw` - """ -""" TARGET_DEVICE_CONFIGURATION: NETWORK - Name of the network this container will be connected to at creation time. - :type str - """ -""" TARGET_DEVICE_CONFIGURATION: USER - Username or UID to run commands as inside the container. - :type int or str - """ -""" TARGET_DEVICE_CONFIGURATION: PRIVILEGED - Give extended privileges to this container. - :type bool - """ + - + path_in_container will be the same as path_on_host + cgroup_permissions will be set to `mrw` + NETWORK - Name of the network this container will be connected to at creation time. + :type str + USER - Username or UID to run commands as inside the container. + :type int or str + PRIVILEGED - Give extended privileges to this container. + :type bool +""" # Currently docker imports are mandatory (even for non-docker types) and this enforce getuid() & getgrnam(...) syscalls # for non-docker testruns. TARGET_DEVICE_CONFIGURATION = { diff --git a/tests/functional/utils/environment_info.py b/tests/functional/utils/environment_info.py index 24f8bd6bcd..9a7fa1e8f0 100644 --- a/tests/functional/utils/environment_info.py +++ b/tests/functional/utils/environment_info.py @@ -69,7 +69,7 @@ def get(cls): return {"version": DEFAULT_FULL_VERSION_NUMBER} -class EnvironmentInfo(object): +class EnvironmentInfo: _instances = {} @classmethod @@ -140,7 +140,7 @@ def get_os_distname(): if CurrentOsInfo.LINUX == platform.system(): if CurrentOsInfo.UBUNTU.lower() in platform.version().lower(): return CurrentOsInfo.UBUNTU - elif CurrentOsInfo.REDHAT.lower() in platform.version().lower(): + if CurrentOsInfo.REDHAT.lower() in platform.version().lower(): return CurrentOsInfo.REDHAT return platform.system() diff --git a/tests/functional/utils/helpers.py b/tests/functional/utils/helpers.py index 1bcc14a7f5..5e9ffbae1d 100644 --- a/tests/functional/utils/helpers.py +++ b/tests/functional/utils/helpers.py @@ -25,8 +25,8 @@ ALL_AVAILABLE_OPTIONS = "*" -def get_int(key_name, fallback=None, environ=os.environ): - value = environ.get(key_name, fallback) +def get_int(key_name, fallback=None): + value = os.environ.get(key_name, fallback) if value != fallback: try: value = int(value) @@ -54,7 +54,7 @@ def get_bool(key_name, fallback=None): elif value == "false": value = False else: - raise ValueError("Value of {} env variable is '{}'. Should be 'True' or 'False'.".format(key_name, value)) + raise ValueError(f"Value of {key_name} env variable is '{value}'. Should be 'True' or 'False'.") return value diff --git a/tests/functional/utils/test_framework.py b/tests/functional/utils/test_framework.py index d839405a9c..896471b8c2 100644 --- a/tests/functional/utils/test_framework.py +++ b/tests/functional/utils/test_framework.py @@ -15,11 +15,12 @@ # import os -import pytest import re import shutil import traceback +import pytest + from tests.functional.utils.assertions import CreateVenvError, PipInstallError from tests.functional.utils.git_operations import clone_git_repository from tests.functional.utils.logger import get_logger