diff --git a/ci/build_test_OnCommit.groovy b/ci/build_test_OnCommit.groovy index 5726188504..863f074895 100644 --- a/ci/build_test_OnCommit.groovy +++ b/ci/build_test_OnCommit.groovy @@ -23,13 +23,15 @@ def validation_branch = "develop" // Override agent node: // [test_agent_linux=] - Run Linux doc tests on (default: ovms_ptl) // [test_agent_windows=] - Run Windows doc tests on (default: ovms_win_ptl) +// example: [test_agent_windows=ovms_win_ptl] // // Override file list (space-separated, converted to pytest -k filter joined with ' or '): // [test_doc_files_linux=] - Use instead of auto-detected list (Linux) // [test_doc_files_windows=] - Use instead of auto-detected list (Windows) +// example: [test_doc_files_linux=demos/continuous_batching/README.md demos/audio/README.md] // // Override validation branch: - // [validation_branch=] - Use instead of default 'develop' for test repo checkout +// [validation_branch=] - Use instead of default 'develop' for test repo checkout // pipeline { @@ -341,7 +343,7 @@ pipeline { def test_doc_files_str = test_doc_files_linux.split('\n').join(' or ') sh "make create-venv && rm -f tests/functional && ln -s ${pwd}/../tests/functional tests/functional" def cmd_venv_activate = ". .venv/bin/activate" - def cmd_export = "export TT_OVMS_C_REPO_PATH=../ && export TT_RUN_REGRESSION_TESTS=True && export TT_REGRESSION_WEEKLY_TESTS=True && export TT_TARGET_DEVICE=CPU,GPU,NPU && export TT_ENABLE_UAT_TESTS=True && export TT_ENABLE_SMOKE_TESTS=False && export TT_OVMS_C_REPO_PATH=${ovms_c_repo_path} && export TT_WAIT_FOR_MESSAGES_TIMEOUT=1500" + def cmd_export = "export TT_OVMS_C_REPO_PATH=../ && export TT_RUN_REGRESSION_TESTS=True && export TT_REGRESSION_WEEKLY_TESTS=True && export TT_TARGET_DEVICE=CPU,GPU,NPU && export TT_ENABLE_UAT_TESTS=True && export TT_ENABLE_SMOKE_TESTS=False && export TT_OVMS_C_REPO_PATH=${ovms_c_repo_path} && export TT_LOGGING_LEVEL_OVMS=DEBUG && export TT_WAIT_FOR_MESSAGES_TIMEOUT=1500" def cmd_pytest = "pytest tests/non_functional/documentation -k '${test_doc_files_str}' -n 0 --dist loadgroup" def cmd = "" if ( image_build_needed == "true" ) { @@ -406,7 +408,7 @@ pipeline { def ovms_c_repo_path = bat(returnStdout: true, script: 'cd .. && cd').trim().split('\n').last().trim() def cmd_link_ovms = "(if exist ${current_path}\\tests\\functional rmdir ${current_path}\\tests\\functional) && mklink /D ${current_path}\\tests\\functional ${ovms_c_repo_path}\\tests\\functional" def cmd_requirements = "(if not exist .venv virtualenv .venv --python=python3.12) && call .venv\\Scripts\\activate.bat && pip install -r requirements.txt" - def cmd_export = "set \"TT_OVMS_C_REPO_PATH=../\" && set \"TT_RUN_REGRESSION_TESTS=True\" && set \"TT_REGRESSION_WEEKLY_TESTS=True\" && set \"TT_TARGET_DEVICE=CPU,GPU,NPU\" && set \"TT_BASE_OS=windows\" && set \"TT_OVMS_TYPE=BINARY\" && set \"TT_ENABLE_UAT_TESTS=True\" && set \"TT_ENABLE_SMOKE_TESTS=False\" && set \"TT_DISABLE_DMESG_LOG_MONITOR=True\" && set \"TT_OVMS_C_REPO_PATH=${ovms_c_repo_path}\" && set \"TT_WAIT_FOR_MESSAGES_TIMEOUT=1500\" && set \"PYTHONUTF8=1\" && set \"PYTHONIOENCODING=utf-8\"" + def cmd_export = "set \"TT_OVMS_C_REPO_PATH=../\" && set \"TT_LOGGING_LEVEL_OVMS=DEBUG\" && set \"TT_RUN_REGRESSION_TESTS=True\" && set \"TT_REGRESSION_WEEKLY_TESTS=True\" && set \"TT_TARGET_DEVICE=CPU,GPU,NPU\" && set \"TT_BASE_OS=windows\" && set \"TT_OVMS_TYPE=BINARY\" && set \"TT_ENABLE_UAT_TESTS=True\" && set \"TT_ENABLE_SMOKE_TESTS=False\" && set \"TT_DISABLE_DMESG_LOG_MONITOR=True\" && set \"TT_OVMS_C_REPO_PATH=${ovms_c_repo_path}\" && set \"TT_WAIT_FOR_MESSAGES_TIMEOUT=1500\" && set \"PYTHONUTF8=1\" && set \"PYTHONIOENCODING=utf-8\"" def cmd_pytest = "pytest tests/non_functional/documentation -k \"${test_doc_files_str}\" -n 0 --dist loadgroup --basetemp=\"C:\\tmp\\pytest-${BRANCH_NAME}-${BUILD_NUMBER}\"" def cmd = "" if ( win_image_build_needed == "true" ) { diff --git a/tests/functional/config.py b/tests/functional/config.py index 9083668f7c..df8d71c843 100644 --- a/tests/functional/config.py +++ b/tests/functional/config.py @@ -20,8 +20,7 @@ from tests.functional.constants.os_type import OsType from tests.functional.constants.ovms_type import OvmsType -from tests.functional.constants.target_device import TargetDevice -from tests.functional.utils.core import TmpDir +from tests.functional.utils.core import TmpDir, get_token_value from tests.functional.utils.helpers import ( generate_test_object_name, get_bool, @@ -65,13 +64,14 @@ def get_uses_mapping(): """TEST_DIR_CACHE - location where models and test data should be downloaded to and serve as cache for TEST_DIR""" test_dir_cache = os.environ.get("TEST_DIR_CACHE", "/tmp/ovms_models_cache") -"""TEST_DIR_CLEANUP - if set to True, TEST_DIR directory will be removed after tests execution""" -test_dir_cleanup = os.environ.get("TEST_DIR_CLEANUP", "True") -test_dir_cleanup = test_dir_cleanup.lower() == "true" - """ TT_OVMS_C_REPO_PATH - path to ovms-c repository. Can be relative or absolute. """ ovms_c_repo_path = get_path("TT_OVMS_C_REPO_PATH", get_path("PWD", "./")) +""" TT_SETUPVARS_SCRIPT_PATH - path to setupvars.bat script """ +setupvars_script_path = os.environ.get( + "TT_SETUPVARS_SCRIPT_PATH", os.path.join(ovms_c_repo_path, "setupvars.bat") +) + """BUILD_LOGS - path to dir where artifacts should be stored""" artifacts_dir = get_path("BUILD_LOGS", os.path.join(ovms_c_repo_path, "tests", "functional", "test_log_build")) @@ -84,6 +84,11 @@ def get_uses_mapping(): """ TT_DATASETS_PATH - Datasets local repo path""" datasets_path = get_path("TT_DATASETS_PATH", os.path.join("~", "ovms_datasets")) +""" TT_GENERATIVE_MODELS_LOCAL_PATH - local path for converted generative models """ +generative_models_local_path = get_path( + "TT_GENERATIVE_MODELS_LOCAL_PATH", os.path.join(models_path, "generative_models") +) + """ TT_CLEAN_ARTIFACTS_DIR """ clean_artifacts_dir = get_bool("TT_CLEAN_ARTIFACTS_DIR", False) @@ -141,23 +146,14 @@ def get_uses_mapping(): path_to_mount_cache = os.path.join(test_dir_cache, "saved_models") -"""TT_MINIO_IMAGE_NAME - Docker image for Minio""" -minio_image = os.environ.get("TT_MINIO_IMAGE_NAME", "minio/minio:latest") +""" TT_MINIO_IMAGE_NAME - Docker image for Minio""" +minio_image = os.environ.get( + "TT_MINIO_IMAGE_NAME", + f"{docker_registry}/minio/minio:latest" if docker_registry is not None else "minio/minio:latest", +) """ TT_TARGET_DEVICE - list of devices separated by a comma "CPU,GPU,NPU" """ target_devices = get_target_devices() -target_device = target_devices[0] - -"""IMAGE - docker image name which should be used to run tests""" -if target_device == TargetDevice.GPU: - _default_image = "openvino/model_server-gpu" -else: - _default_image = "openvino/model_server" -image = os.environ.get("IMAGE", _default_image) - -start_minio_container_command = 'server --address ":{}" /data' - -container_minio_log_line = "Console endpoint is listening on a dynamic port" # Reservation manager values, for details study tests.functional.utils.reservation_manager """ TT_GRPC_OVMS_STARTING_PORT - Grpc port where ovms should be exposed""" @@ -170,32 +166,12 @@ def get_uses_mapping(): ports_pool_size = get_int("TT_PORTS_POOL_SIZE", None) # NOTE: Above values will be validated and could be changed if invalid -""" TT_CONVERTED_MODELS_EXPIRE_TIME - Time after converted models are not up-to-date and needs to be refreshed(s) """ -converted_models_expire_time = get_int("TT_CONVERTED_MODELS_EXPIRE_TIME", 7*24*3600) # Set default to one week - -""" TT_DEFAULT_INFER_TIMEOUT - Timeout for CPU target device""" -default_infer_timeout = get_int("TT_DEFAULT_INFER_TIMEOUT", 10) - -""" TT_DEFAULT_GPU_INFER_TIMEOUT - Timeout for GPU target device""" -default_gpu_infer_timeout = get_int("TT_DEFAULT_GPU_INFER_TIMEOUT", 10*default_infer_timeout) - -""" TT_DEFAULT_NPU_INFER_TIMEOUT - Timeout for NPU target device""" -default_npu_infer_timeout = get_int("TT_DEFAULT_NPU_INFER_TIMEOUT", 10*default_infer_timeout) - -""" INFER TIMEOUT """ -infer_timeouts = { - TargetDevice.CPU: default_infer_timeout, - TargetDevice.GPU: default_gpu_infer_timeout, - TargetDevice.NPU: default_npu_infer_timeout, - TargetDevice.AUTO: default_gpu_infer_timeout, - TargetDevice.HETERO: default_gpu_infer_timeout, - TargetDevice.AUTO_CPU_GPU: default_gpu_infer_timeout, -} -infer_timeout = infer_timeouts[target_device] - """ TT_IS_NGINX_MTLS - Specify if given image is OVSA nginx mtls image. """ is_nginx_mtls = get_bool("TT_IS_NGINX_MTLS", False) +""" TT_FORCE_GENERATE_NEW_SSL_CERTIFICATES """ +force_generate_new_ssl_certs = get_bool("TT_FORCE_GENERATE_NEW_SSL_CERTIFICATES", True) + """ TT_SKIP_TEST_IF_IS_NGINX_MTLS """ skip_nginx_test = get_bool("TT_SKIP_TEST_IF_IS_NGINX_MTLS", True) skip_nginx_test = skip_nginx_test and is_nginx_mtls @@ -317,6 +293,25 @@ def get_uses_mapping(): """ TT_OVMS_IMAGE_LOCAL - ovms image can only be found locally """ ovms_image_local = get_bool("TT_OVMS_IMAGE_LOCAL", False) +""" TT_REQUIREMENTS - Requirements """ +req_ids = get_list("TT_REQUIREMENTS") + +""" TT_EXCLUDE_REQUIREMENTS - Requirements to exclude """ +exclude_req_ids = get_list("TT_EXCLUDE_REQUIREMENTS") + +""" TT_COMPONENTS - Components """ +components_ids = get_list("TT_COMPONENTS") + +""" TT_EXCLUDE_COMPONENTS - Components to exclude """ +exclude_components_ids = get_list("TT_EXCLUDE_COMPONENTS") + +""" TT_TESTS_PRIORITY_LIST - tests priority to run - high, medium or low """ +tests_priority_list_raw = get_list("TT_TESTS_PRIORITY_LIST", fallback=["high", "medium", "low"]) +tests_priority_list = [f"priority_{p}" for p in tests_priority_list_raw if "priority" not in p] + +""" TT_PERFORMANCE_TEST_TIMEOUT_MINUTES - timeout (in minutes) for each performance test """ +performance_test_timeout_minutes = get_int("TT_PERFORMANCE_TEST_TIMEOUT_MINUTES", 10) + """ TT_BASE_OS - os type used for calculating ovms_image name (if not given explicitly). Possible options (case insensitive): ubuntu22 - use default Ubuntu 22.04 image @@ -328,6 +323,11 @@ def get_uses_mapping(): __base_os = os.environ.get("BASE_OS", OsType.Ubuntu24) base_os = get_list("TT_BASE_OS", fallback=[__base_os]) +""" BASE_IMAGE - Docker image used during OVMS image creation """ +base_image = os.environ.get("BASE_IMAGE", None) +if base_image is not None: + assert len(base_os) == 1, "If you wish to iterate by TT_BASE_OS: do not set BASE_IMAGE explicitly." + """ GLOBAL_TEMP_DIR - global temporary directory """ global_tmp_dir_default = os.path.join("~", "AppData", "Local", "Temp") if OsType.Windows in base_os else "/tmp" global_tmp_dir = get_path("GLOBAL_TEMP_DIR", global_tmp_dir_default) @@ -416,3 +416,17 @@ def get_ovms_types(): """ TT_OVMS_TYPE - ovms type runtime to be executed: DOCKER, BINARY, BINARY_DOCKER, CAPI, CAPI_DOCKER, DOCKER_CMD_LINE """ ovms_types = get_ovms_types() + +""" TT_DIVIDE_TARGET_DEVICE_PER_WORKER - spread tests across pytest workers based on target device """ +divide_target_device_per_worker = get_bool("TT_DIVIDE_TARGET_DEVICE_PER_WORKER", False) + +""" TT_PYTEST_KEYWORD_FILTER """ +pytest_keyword_filter = os.environ.get("TT_PYTEST_KEYWORD_FILTER", None) + +""" TT_HUGGINGFACE_TOKEN_FILE_PATH - path to file containing huggingface token """ +huggingface_token_file_path = get_path( + "TT_HUGGINGFACE_TOKEN_FILE_PATH", os.path.join("~", "ovms_tokens", "huggingface_token") +) + +""" TT_HUGGINGFACE_TOKEN - huggingface token value. Env var takes priority, then file. """ +huggingface_token = os.environ.get("TT_HUGGINGFACE_TOKEN") or get_token_value(huggingface_token_file_path, "") diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py new file mode 100644 index 0000000000..1bb8d68b10 --- /dev/null +++ b/tests/functional/conftest.py @@ -0,0 +1,180 @@ +# +# Copyright (c) 2018-2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import random +import sys + +from tests.functional.config import enable_pytest_plugins, pytest_keyword_filter, machine_is_reserved_for_test_session +from tests.functional.constants.ovms import ( + CURRENT_TARGET_DEVICE_DICT_ARGUMENT, + TMP_REPOS_DIR_ARGUMENT, +) +from tests.functional.utils import hooks +from tests.functional.utils.logger import OvmsFileHandler, get_logger +from tests.functional.utils.marks import MarksRegistry +from tests.functional.utils.test_framework import is_xdist_master + +logger = get_logger(__name__) + + +if enable_pytest_plugins: + + raise NotImplementedError("OVMS tests not enabled") + + pytest_plugins = [ # pylint: disable=unreachable + "tests.functional.fixtures.ovms", + "tests.functional.fixtures.server", + "tests.functional.fixtures.api_type", + "tests.functional.fixtures.params", + ] + + + def pytest_configure(config): + """ + Allow plugins and conftest files to perform initial configuration. + This hook is called for every plugin and initial conftest file after command line options have been parsed. + After that, the hook is called for other conftest files as they are imported. + + NOTE: + This hook is called multiple times: + 1) for master process prior spawning workers + (PYTEST_XDIST_WORKER_COUNT and PYTEST_XDIST_WORKER env variable unset) + 2) for each spawned worker process + + LIMITATIONS: + Internal pytest logging mechanisms are initialized in `pytest_sessionstart` hook. + Please avoid usage of logger in all hooks used in this function. + Please simple print(...) call for printing messages. + """ + hooks.mute_warnings() + MarksRegistry.register(config) + + if is_xdist_master(): + hooks.setup_tmp_repos_dir(config) + hooks.validate_port_pool(config) + # master thread pytest_configure call. No xdist worker process spawned yet. + hooks.init_environment(config) + hooks.clear_ovms_capi_artifacts() + hooks.setup_artifacts_dir() + hooks.prepare_ovms_package() + hooks.download_resources_master() + hooks.build_local_resources() + hooks.validate_lock_files() + hooks.list_host_zombie_processes() + else: # Xdist worker thread + hooks.download_docker_images() + hooks.init_ovms_config_retrieved_from_master(config) + + hooks.setup_nginx() + + # Let know that pytest was successfully configured + config.configured = True + + + def pytest_unconfigure(config): + if getattr(config, "configured", None) is not True: + # Check if pytest_configure() was done successfully, if not: logger would be in invalid state so disable. + for _logger in logger.manager.loggerDict.values(): + _logger.disabled = True + + try: + if is_xdist_master(): + hooks.remove_ports_reservation(config) + hooks.cleanup_tmp_repos_dir(config) + hooks.teardown_environment() + if machine_is_reserved_for_test_session: + hooks.clear_lockfiles() + except Exception as e: # pylint: disable=broad-exception-caught + error_msg = str(e) + print(error_msg) + sys.exit(error_msg) + + + def pytest_configure_node(node): + node.workerinput[TMP_REPOS_DIR_ARGUMENT] = node.config.tmp_repos_dir + node.workerinput[CURRENT_TARGET_DEVICE_DICT_ARGUMENT] = node.config.current_target_device_dict + + + MarksRegistry.MARK_ENUMS.extend([OvmsComponents]) + + + def pytest_sessionstart(session): + hooks.get_session_start_info(session) + + + @pytest.hookimpl(hookwrapper=True) + def pytest_collection_modifyitems(session, config, items): + """ + Support for running tests with component tags. + Report all test component markers to mongo_reporter. + """ + logger.info(f"Preparing tests for test session in the following folder: {session.startdir}") + + if pytest_keyword_filter: + # Filter case insensitive + deselected = [_item for _item in items if pytest_keyword_filter.lower() not in _item.name.lower()] + if deselected: + hooks.deselect_items(items, config, deselected) + + yield # deselect items in default hook way via keyword ('-k') + + if config.option.collectonly: + hooks.log_skip_statistic(items) + + deselected = hooks.preprocess_collected_items(items) + if deselected: + hooks.deselect_items(items, config, deselected) + + hooks.set_divide_target_device_per_worker(items) + + random.Random(7).shuffle(items) + + + @pytest.hookimpl(hookwrapper=True, tryfirst=True) + def pytest_runtest_protocol(item: "Item"): + """ + Perform the runtest protocol for a single test item. + The default runtest protocol is this (see individual hooks for full details): + pytest_runtest_logstart(nodeid, location) + Setup phase: + call = pytest_runtest_setup(item) (wrapped in CallInfo(when="setup")) + report = pytest_runtest_makereport(item, call) + pytest_runtest_logreport(report) + pytest_exception_interact(call, report) if an interactive exception occurred + Call phase, if the setup passed and the setuponly pytest option is not set: + call = pytest_runtest_call(item) (wrapped in CallInfo(when="call")) + report = pytest_runtest_makereport(item, call) + pytest_runtest_logreport(report) + pytest_exception_interact(call, report) if an interactive exception occurred + Teardown phase: + call = pytest_runtest_teardown(item, nextitem) (wrapped in CallInfo(when="teardown")) + report = pytest_runtest_makereport(item, call) + pytest_runtest_logreport(report) + pytest_exception_interact(call, report) if an interactive exception occurred + pytest_runtest_logfinish(nodeid, location) + """ + __root_logger = get_logger(None) + if not item.keywords.get("skip"): + fh = OvmsFileHandler(item) + __root_logger.addHandler(fh) + yield + if not item.keywords.get("skip"): + fh.close() + __root_logger.removeHandler(fh) + + + def pytest_generate_tests(metafunc): + hooks.parametrize_tests(metafunc) diff --git a/tests/functional/constants/os_version.py b/tests/functional/constants/os_version.py new file mode 100644 index 0000000000..bdbbf4e796 --- /dev/null +++ b/tests/functional/constants/os_version.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.constants.os_type import OsType + +REDHAT_MINIMAL_BASE_IMAGE = "registry.access.redhat.com/ubi9/ubi-minimal:9.7" +REDHAT_COMMON_BASE_IMAGE = "registry.access.redhat.com/ubi9/ubi:9.7" +UBUNTU_22_BASE_IMAGE = "ubuntu:22.04" +UBUNTU_24_BASE_IMAGE = "ubuntu:24.04" + +os_type_to_base_image = { + OsType.Redhat: REDHAT_COMMON_BASE_IMAGE, + OsType.Ubuntu22: UBUNTU_22_BASE_IMAGE, + OsType.Ubuntu24: UBUNTU_24_BASE_IMAGE, +} + +os_type_to_base_image_binary_docker = { + OsType.Redhat: REDHAT_COMMON_BASE_IMAGE, + OsType.Ubuntu22: UBUNTU_22_BASE_IMAGE, + OsType.Ubuntu24: UBUNTU_24_BASE_IMAGE +} + +OPENVINO_UBUNTU_20_DEV_IMAGE = "openvino/ubuntu20_dev:2024.6.0" +OPENVINO_MODEL_SERVER_LATEST = "openvino/model_server:latest" +OPENVINO_MODEL_SERVER_LATEST_GPU = "openvino/model_server:latest-gpu" +OPENVINO_MODEL_SERVER_LATEST_PY = "openvino/model_server:latest-py" +OPENVINO_MODEL_SERVER_WEEKLY = "openvino/model_server:weekly" +NO_DOC_UPDATE_IMAGES = [OPENVINO_MODEL_SERVER_LATEST, OPENVINO_MODEL_SERVER_LATEST_GPU, + OPENVINO_MODEL_SERVER_LATEST_PY, OPENVINO_MODEL_SERVER_WEEKLY] diff --git a/tests/functional/constants/ovms_cohere.py b/tests/functional/constants/ovms_cohere.py new file mode 100644 index 0000000000..ad6c9ab030 --- /dev/null +++ b/tests/functional/constants/ovms_cohere.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from tests.functional.utils.inference.serving.cohere import CohereWrapper, OvmsRerankRequestParams + + +class OvmsCohereRequestParamsBuilder: + def __init__(self, endpoint, **kwargs): + self.endpoint = endpoint + if self.endpoint == CohereWrapper.RERANK: + self.request_params = OvmsRerankRequestParams(**kwargs) + else: + raise NotImplementedError diff --git a/tests/functional/constants/ovms_images.py b/tests/functional/constants/ovms_images.py index 1e268965b4..dcfc06367e 100644 --- a/tests/functional/constants/ovms_images.py +++ b/tests/functional/constants/ovms_images.py @@ -19,13 +19,15 @@ from tests.functional.utils.environment_info import EnvironmentInfo from tests.functional.constants.os_type import OsType, UBUNTU from tests.functional.config import ( + base_os, docker_registry, + force_use_ovms_image, is_nginx_mtls, ovms_cpp_docker_image, ovms_image, ovms_image_tag, ovms_test_image_name, - force_use_ovms_image, + target_devices, ) from tests.functional.constants.target_device import TargetDevice from tests.functional.constants.ovms import CurrentOvmsType @@ -172,3 +174,11 @@ def calculate_ovms_capi_image_name(ovms_image_name): def calculate_ovms_test_image_name(ovms_image_name): test_image_name = ovms_test_image_name if ovms_test_image_name is not None else f"{ovms_image_name}-test" return test_image_name + + +def get_ovms_calculated_images(): + ovms_images = set() + for _os in base_os: + for _target_device in target_devices: + ovms_images.add((calculate_ovms_image_name(_target_device, _os), _os)) + return ovms_images diff --git a/tests/functional/constants/ovms_type.py b/tests/functional/constants/ovms_type.py index 02a1ea9873..458e7c2974 100644 --- a/tests/functional/constants/ovms_type.py +++ b/tests/functional/constants/ovms_type.py @@ -26,7 +26,6 @@ class OvmsType: BINARY_DOCKER = "BINARY_DOCKER" CAPI = "CAPI" CAPI_DOCKER = "CAPI_DOCKER" - KUBERNETES = "KUBERNETES" # legacy # https://github.com/openvinotoolkit/model_server/blob/main/docs/deploying_server.md#deploying-model-server-on-baremetal-without-container diff --git a/tests/functional/constants/paths.py b/tests/functional/constants/paths.py index 6a0636d3a5..9824023a9f 100644 --- a/tests/functional/constants/paths.py +++ b/tests/functional/constants/paths.py @@ -63,6 +63,17 @@ class Paths: def CAPI_WRAPPER_PACKAGE_CONTENT_PATH(base_os): return os.path.join(config.c_api_wrapper_dir, base_os, "ovms") + COMMON_GIT_CLONE_LOCK_FILE = os.path.join(config.ovms_file_locks_dir, "common_git_clone.lock") + + COMMON_BUILD_LOCK_FILE = os.path.join(config.ovms_file_locks_dir, "common_build.lock") + + # Use single shared lock file until ensure that those builds can be done concurrently. + DOCKER_BUILD_LOCK_FILE = COMMON_BUILD_LOCK_FILE + CUSTOM_NODE_BUILD_LOCK_FILE = COMMON_BUILD_LOCK_FILE + CPU_EXTENSION_BUILD_LOCK_FILE = COMMON_BUILD_LOCK_FILE + + COMMON_DOWNLOAD_LOCK_FILE = os.path.join(config.ovms_file_locks_dir, "common_download.lock") + @staticmethod def get_target_device_lock_file(target_device, i): if isinstance(target_device, str): diff --git a/tests/functional/constants/pipelines.py b/tests/functional/constants/pipelines.py index fd2ba8588f..0881e0636d 100644 --- a/tests/functional/constants/pipelines.py +++ b/tests/functional/constants/pipelines.py @@ -24,7 +24,7 @@ from tests.functional.config import datasets_path from ovms.constants.model_dataset import RandomDataset -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from ovms.constants.models import ( AgeGender, ArgMax, @@ -1461,7 +1461,7 @@ def add_mediapipe_graphs_to_config(self, config, use_subconfig=False, mediapipe_ ) for i, model in enumerate(mediapipe_models): model_name = model.name - graph_name = model_name if (not model.is_llm and model.pbtxt_name is None) \ + graph_name = model_name if (not model.is_generative and model.pbtxt_name is None) \ else getattr(model, "pbtxt_name", None) graph_filename = f"{graph_name}.pbtxt" mediapipe_base_path = str(Path(Paths.MODELS_PATH_INTERNAL, model_name)) diff --git a/tests/functional/constants/requirements.py b/tests/functional/constants/requirements.py index 70161c6916..479689c0a2 100644 --- a/tests/functional/constants/requirements.py +++ b/tests/functional/constants/requirements.py @@ -38,7 +38,13 @@ class Requirements: kfservin_api = "CVS-81053 KFServing api" metrics = "CVS-43549 metrics" custom_nodes = "CVS-44359 custom nodes" + llm = "CVS-129298 LLM execution in ovms based on c++ code only" + embeddings_endpoint = "CVS-147460 embeddings endpoint" + rerank_endpoint = "CVS-147460 rerank endpoint" + images_endpoint = "CVS-169110 images endpoint" audio_endpoint = "CVS-174282 audio endpoint" + hf_imports = "CVS-162541 Direct models import from HF Hub in OVMS" + tools = "CVS-166514 Structured response with tools support in chat/completions" # test types sdl = "CVS-59335 SDL" @@ -59,7 +65,4 @@ class Requirements: streaming_api = "CVS-118064 streaming API extension" mediapipe = "CVS-103194 mediapipe" python_custom_node = "CVS-117210 python support" - llm = "CVS-129298 LLM execution in ovms based on c++ code only" openai_api = "CVS-138033 OpenAI API in OVMS" - hf_imports = "CVS-162541 Direct models import from HF Hub in OVMS" - tools = "CVS-166514 Structured response with tools support in chat/completions" diff --git a/tests/functional/fixtures/ovms.py b/tests/functional/fixtures/ovms.py index c524929f99..1dfb4f91e5 100644 --- a/tests/functional/fixtures/ovms.py +++ b/tests/functional/fixtures/ovms.py @@ -38,7 +38,7 @@ run_ovms_with_opencl_trace, run_ovms_with_valgrind, ) -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from tests.functional.constants.ovms import CurrentOvmsType, CurrentTarget from tests.functional.constants.ovms_binaries import calculate_ovms_binary_name from tests.functional.constants.ovms_images import ( diff --git a/tests/functional/models/__init__.py b/tests/functional/models/__init__.py new file mode 100644 index 0000000000..84cfbcf566 --- /dev/null +++ b/tests/functional/models/__init__.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/functional/models/models.py b/tests/functional/models/models.py new file mode 100644 index 0000000000..cad6f47ae5 --- /dev/null +++ b/tests/functional/models/models.py @@ -0,0 +1,599 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=too-many-instance-attributes +# pylint: disable=too-many-public-methods +# pylint: disable=unused-argument + +import json +import math +import os +import shutil +import stat +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +import numpy as np + +from tests.functional.config import is_nginx_mtls, models_path, xdist_workers +from tests.functional.constants.os_type import OsType, get_host_os +from tests.functional.constants.ovms import Ovms +from tests.functional.constants.paths import Paths +from tests.functional.constants.target_device import TargetDevice +from tests.functional.object_model.cpu_extension import CpuExtension +from tests.functional.object_model.custom_loader import CustomLoader +from tests.functional.object_model.ovms_mapping_config import OvmsMappingConfig +from tests.functional.object_model.shape import Shape +from tests.functional.object_model.test_environment import TestEnvironment +from tests.functional.utils.logger import get_logger + +logger = get_logger(__name__) + + +class ModelType(str, Enum): + IR = "IR" + ONNX = "ONNX" + PDPD = "PDPD" + TFSM = "TFSM" # tensorflow savedmodel + + +CLOUD_HEADERS = {"azure-blob": "az://", "azure-fs": "azfs://", "google": "gs://", "s3_minio": "s3://"} + +DEVICE_LOADING_SPEED = { + TargetDevice.CPU: { + ModelType.IR: 97.4, + ModelType.ONNX: 58.1, + ModelType.PDPD: 97.4, + ModelType.TFSM: 97.4, + }, # For now TFSM works only with CPU. + TargetDevice.GPU: {ModelType.IR: 10, ModelType.ONNX: 25, ModelType.PDPD: 10}, + TargetDevice.NPU: {ModelType.IR: 10, ModelType.ONNX: 25, ModelType.PDPD: 10}, + TargetDevice.AUTO: {ModelType.IR: 2, ModelType.ONNX: 5, ModelType.PDPD: 2}, + TargetDevice.HETERO: {ModelType.IR: 2, ModelType.ONNX: 5, ModelType.PDPD: 2}, + TargetDevice.AUTO_CPU_GPU: {ModelType.IR: 2, ModelType.ONNX: 5, ModelType.PDPD: 2}, +} + + +@dataclass +class ModelInfo: + name: str = None + version: int = 1 + inputs: dict = None + outputs: dict = None + batch_size: int = None + model_version_policy: str = None + nireq: int = None + plugin_config: object = None + transpose_axes: str = None + custom_loader: CustomLoader = None + expected_batch_size: int = None + base_path: str = None + model_path_on_host = None + model_type: ModelType = ModelType.IR + input_shape_for_ovms: Any = None + _compiled_layout: str = None + _layout_for_ovms: str = None + allow_cache: str = None + use_mapping: bool = None + target_device: str = None # Currently used target device. Set up prior test in context fixture + base_os: str = None + is_mediapipe: bool = False + is_language: bool = False + use_relative_paths: bool = False + use_subconfig: bool = False + xml_name: str = None + onnx_name: str = None + is_generative: bool = False + is_llm: bool = False + is_vision_language: bool = False + is_hf_direct_load: bool = False # model can be loaded directly from HuggingFace without conversion with optimum-cli + gguf_filename: str = None + is_local: bool = False + model_subpath: str = None + single_mediapipe_model_mode: bool = False + tool_parser: str = None + jinja_template: str = None + tools_enabled: bool = False + apply_gorilla_patch: bool = False + gorilla_patch_name: str = None + is_agentic: bool = False + enable_tool_guided_generation: bool = False + reasoning_parser: str = None + pooling: str = None + extra_quantization_params: str = None + pipeline_type: str = None + bfcl_num_threads: int = None + max_num_batched_tokens: int = None + is_audio: bool = False + is_asr_model: bool = False + is_tts_model: bool = False + predict_timeout: Optional[int] = None + copy_all_model_versions: bool = False + cpu_extension: CpuExtension = None + + def __post_init__(self): + if self.use_relative_paths: + self.base_path = self.name + else: + if self.use_subconfig: + self.base_path = os.path.join(Paths.MODELS_PATH_INTERNAL, f"{self.name}_mediapipe", self.name) + else: + self.base_path = os.path.join(Paths.MODELS_PATH_INTERNAL, self.name) + self.model_path_on_host = os.path.join(models_path, self.name, str(self.version)) + self.set_additional_model_params() + if get_host_os() == OsType.Windows: + # enable model deletion https://jira.devtools.intel.com/browse/CVS-160412 + self.plugin_config = Ovms.PLUGIN_CONFIG_WINDOWS + + def set_additional_model_params(self): + if not self.base_os: + self.base_os = ModelInfo.base_os + if not self.target_device: + self.target_device = ModelInfo.target_device + + @staticmethod + def get_list_of_config_fields(): + return [ + "name", + "base_path", + "batch_size", + "model_version_policy", + "nireq", + "plugin_config", + "allow_cache", + ] + + def get_model_file_path(self): + filepath = None + if self.model_type == ModelType.IR: + ext = "xml" + xml_name = self.xml_name if self.xml_name is not None else self.name + filepath = f"{models_path}/{self.name}/{self.version}/{xml_name}.{ext}" + elif self.model_type == ModelType.ONNX: + ext = "onnx" + onnx_name = self.onnx_name if self.onnx_name is not None else self.name + filepath = f"{models_path}/{self.name}/{self.version}/{onnx_name}.{ext}" + else: + raise NotImplementedError(self.model_type) + return filepath + + def get_bin_path(self): + ext = "bin" + bin_path = None + if self.model_type == ModelType.IR: + bin_path = f"{models_path}/{self.name}/{self.version}/{self.name}.{ext}" + else: + raise NotImplementedError(self.model_type) + return bin_path + + def get_config(self): + config = {} + + for field_name in self.get_list_of_config_fields(): + if field_name == "name": + if self.is_mediapipe: + base_path = getattr(self, "base_path", None) + value = os.path.basename(base_path) + else: + value = getattr(self, field_name, None) + else: + value = getattr(self, field_name, None) + if value is not None: + config[field_name] = value + + if self.target_device is not None: + config["target_device"] = self.target_device + + if self.custom_loader is not None: + config.update(self.custom_loader.model_options) + + config_shape = self._calculate_shape_for_config() + if config_shape: + config["shape"] = config_shape + + config_layout = self._calculate_layout_for_config() + if config_layout: + config["layout"] = config_layout + + return {"config": config} + + def set_input_shape_for_ovms(self, input_shape: Union[str, List, Dict[str, List]] = None): + if input_shape is None: + input_shape = self.input_shapes + + if isinstance(input_shape, dict): + result = {} + for input_name, shape in input_shape.items(): + if isinstance(shape, str): + result[input_name] = shape + elif isinstance(shape, (list, tuple)): + shape_dims_str = f"{','.join([str(shape_dim) for shape_dim in shape])}" + result[input_name] = f"({shape_dims_str})" + self.input_shape_for_ovms = result + elif isinstance(input_shape, list): + self.input_shape_for_ovms = f"({','.join([str(shape_dim) for shape_dim in input_shape])})" + else: + self.input_shape_for_ovms = input_shape + + return self.input_shape_for_ovms + + def set_layout_for_ovms(self, layout: Union[str, dict]): + if isinstance(layout, str): + self._layout_for_ovms = layout + elif isinstance(layout, dict): + self._layout_for_ovms = json.dumps(layout) + else: + raise NotImplementedError() + + return self._layout_for_ovms + + def try_to_update_batch_size(self, shape): + if shape is not None and isinstance(shape, list) and isinstance(shape[0], int): + self.batch_size = shape[0] + + def _calculate_shape_for_config(self): + return self.input_shape_for_ovms + + def _calculate_layout_for_config(self): + layout_for_config = None + if self._layout_for_ovms is not None: + layout_for_config = self._layout_for_ovms + else: + layouts = [] + + inputs_outputs = {} + if self.inputs is not None: + inputs_outputs.update(self.inputs) + + if self.outputs is not None: + inputs_outputs.update(self.outputs) + + for input_name, input_info in inputs_outputs.items(): + if input_info is not None: + layout = input_info.get("layout", None) + if layout is not None: + layouts.append(f'"{input_name}": "{layout}"') + + if layouts: + layout_for_config = ", ".join(layouts) + else: + layout_for_config = None + + if layout_for_config is not None: + layout_for_config = f"{{{layout_for_config}}}" + + return layout_for_config + + def get_expected_output(self, input_data: dict, client_type: str = None): + return None + + @staticmethod + def is_pipeline(): + return False + + def get_model_path(self): + return self.base_path + + def prepare_input_data(self, batch_size=None, random_data=False, input_key=None): + result = {} + for input_name, input_data in self.inputs.items(): + if batch_size is not None: + input_data["shape"][0] = batch_size + if input_data["shape"] and input_data["shape"][0] == -1 and input_data.get("dataset", None): + result[input_name] = input_data["dataset"].get_data(input_data["shape"], input_data["shape"][0], False) + else: + if random_data: + result[input_name] = np.random.uniform(-100.0, 100.0, input_data["shape"]).astype( + input_data["dtype"] + ) + else: + result[input_name] = np.ones(input_data["shape"], dtype=input_data["dtype"]) + return result + + def prepare_input_data_from_model_datasets(self, batch_size=None): + result = {} + for param_name, param_data in self.inputs.items(): + if batch_size is None: + batch_size = self.get_expected_batch_size() if self.batch_size is None else self.batch_size + result[param_name] = param_data["dataset"].get_data( + shape=param_data["shape"], + batch_size=batch_size, + transpose_axes=self.transpose_axes, + datatype=param_data["dtype"], + ) + return result + + def get_model_path_with_version(self, version=None): + if version is None: + version = self.version + return os.path.join(self.get_model_path(), str(version)) + + def get_model_files(self): + model_dir = self.model_path_on_host + all_files = os.listdir(model_dir) + return all_files + + def get_expected_batch_size(self) -> int: + expected_batch_size = None + + def get_batch_size_form_input_shape(): + model = self + if self.inputs is None: + model = self.clone() + if not any(v["shape"] for v in model.inputs.values()): + return Ovms.SCALAR_BATCH_SIZE + + return [v["shape"] for v in model.inputs.values()][0][0] + + if self.input_shape_for_ovms is not None or self.batch_size is None or self.batch_size == "auto": + expected_batch_size = get_batch_size_form_input_shape() + else: + expected_batch_size = self.batch_size + + try: + if expected_batch_size == Ovms.SCALAR_BATCH_SIZE: + return expected_batch_size + + if isinstance(expected_batch_size, str) and ":" in expected_batch_size: + expected_batch_size = expected_batch_size.split(":")[0] + + expected_batch_size = int(expected_batch_size) + except (TypeError, ValueError) as e: + raise e.__class__(f"Calculated expected_batch_size: `{expected_batch_size}` is not a number: {e}") + + return expected_batch_size + + def clone(self, clone_model_name=None, model_path_on_host=None): + clone = type(self)() + if clone_model_name is not None: + clone.name = clone_model_name + clone.base_path = os.path.join(Paths.MODELS_PATH_INTERNAL, clone_model_name) + if model_path_on_host is not None: + clone.model_path_on_host = model_path_on_host + + shutil.copytree(self.model_path_on_host, clone.model_path_on_host, dirs_exist_ok=True) + return clone + + def create_new_version(self, container_folder, new_version, copy_from_host_path=False, model_name=None): + model_name = model_name if model_name is not None else self.name + result = type(self)() + + if copy_from_host_path: + source = self.model_path_on_host + else: + source = Path(container_folder, Paths.MODELS_PATH_NAME, model_name, str(self.version)) + + destination = Path(container_folder, Paths.MODELS_PATH_NAME, model_name, str(new_version)) + if source != destination: + # This check is for negative tests from TestOnlineModification + shutil.copytree(source, destination, dirs_exist_ok=True) + for file in destination.glob("*"): + # resource files from shared folder should be read only. + # Add proper access for test container folder manipulations. + file.chmod(file.stat().st_mode | stat.S_IWRITE) + + result.version = new_version + + # Copy inputs/outputs (be aware that inputs/outputs can be mapped) + result.inputs = self.inputs + result.outputs = self.outputs + + if result.is_mediapipe: + for model in result.regular_models: + if model_name == model.name: + model.version = new_version + + return result + + def delete(self, container_folder, model_name=None): + model_name = model_name if model_name is not None else self.name + shutil.rmtree(os.path.join(container_folder, Paths.MODELS_PATH_NAME, model_name)) + + def delete_version(self, container_folder): + shutil.rmtree(os.path.join(container_folder, Paths.MODELS_PATH_NAME, self.name, str(self.version))) + + def restore_input_names(self): + model = self.clone() + self.inputs = {} + for input_name in model.inputs: + self.inputs[input_name] = None + + def change_input_name(self, old_name, new_name): + tmp = self.inputs.pop(old_name) + self.inputs[new_name] = tmp + + def change_output_name(self, old_name, new_name): + tmp = self.outputs.pop(old_name) + self.outputs[new_name] = tmp + + def validate_outputs(self, outputs, expected_output_shapes=None, provided_input=None): + assert outputs, "Prediction returned no output" + if expected_output_shapes is None: + expected_output_shapes = list(self.output_shapes.values()) + for i, shape in enumerate(expected_output_shapes): # Check for dynamic shape + for j, val in enumerate(shape): + if val == -1: + expected_output_shapes[i][j] = 1 + + for output_name in self.output_names: + assert ( + output_name in outputs + ), f"Incorrect output name, expected: {output_name}, found: {', '.join(outputs.keys())}" + output_shapes = [list(o.shape) for o in outputs.values()] + assert any( + shape in expected_output_shapes for shape in output_shapes + ), f"Incorrect output shape, expected: {expected_output_shapes}, found: {output_shapes}." + + def get_ovms_loading_time(self): + loading_file_speed = 0.60 if self.is_on_cloud else 300.0 + model_loading_speed = DEVICE_LOADING_SPEED[self.target_device][self.model_type] + if self.custom_loader: + model_loading_speed = model_loading_speed * 0.10 + size = self.size + if size > 0.0: + size_in_mb = math.ceil(size / (1024.0 * 1024.0)) + else: + size_in_mb = 100 # Model files not provided yet assume 100MB model + + timeout = size_in_mb * (1 / loading_file_speed + 1 / model_loading_speed) + timeout += 60 if self.is_on_cloud else 10 # required for very small models on cloud + timeout += 10 if self.custom_loader else 0 # required for additional overhead of custom loader invocation + timeout += 10 if is_nginx_mtls else 0 # required for additional overhead of nginx invocation + + timeout += timeout * 0.2 * xdist_workers + + return timeout + + def prepare_resources(self, base_location): + result = [] + resource_destination = ( + os.path.join(base_location, Paths.MODELS_PATH_NAME) + if not self.use_subconfig + else os.path.join(base_location, Paths.MODELS_PATH_NAME, self.name) + ) + if not self.is_on_cloud and self.model_path_on_host is not None: + if self.copy_all_model_versions: + src_model_path = Path(self.model_path_on_host).parent + target_model_dir = Path(resource_destination, src_model_path.name) + else: + src_model_path = Path(self.model_path_on_host) + # model_name/version_num + model_subpath = src_model_path.parts[-2:] if self.model_subpath is None else \ + Path(self.model_subpath).parts + target_model_dir = Path(resource_destination, *model_subpath) + if not os.path.exists(target_model_dir): + logger.debug(f"Copying {self.name} to container: {target_model_dir}") + shutil.copytree(src_model_path, target_model_dir, dirs_exist_ok=True) + for file in target_model_dir.glob("*"): + # resource files from shared folder should be read only. + # Add proper access for test container folder manipulations. + file.chmod(file.stat().st_mode | stat.S_IWRITE) + result = [resource_destination] + else: + if CLOUD_HEADERS["google"] in self.base_path: + # WA for GoogleCloud credential folder + result = [resource_destination, os.path.join(TestEnvironment.current.base_dir, "credentials")] + + if self.custom_loader: + assert not self.is_on_cloud, "Test framework not ready for models on cloud with custom_loader!" + if self.custom_loader.prepare_custom_loader_resources: + result.append(self.custom_loader.prepare_resources(base_location)) + return result + + def get_input_shape(self, input_name): + return Shape(self.inputs[input_name]["shape"], self._compiled_layout) + + def set_shape_for_input(self, input_name, shape): + _layout = self._compiled_layout.split(":")[0] if self._compiled_layout else None + self.inputs[input_name]["shape"] = shape.get_shape_by_layout(_layout) + + def change_input_layout(self, new_layout): + new_layout = new_layout.split(":")[0] if ":" in new_layout else new_layout + for _, val in self.inputs.items(): + s = val["shape"] + new_shape = [s[0], s[2], s[3], s[1]] + val["shape"] = new_shape + + def change_input_type(self, input_name, dtype): + self.inputs[input_name]["dtype"] = dtype + + def get_regular_models(self): + return [self] + + def get_demultiply_count(self): + return None + + def get_mapping_config_path(self, container_folder): + return OvmsMappingConfig.mapping_config_path(container_folder, self) + + def get_mapping_dict(self, container_folder): + mapping_config_path = self.get_mapping_config_path(container_folder) + return OvmsMappingConfig.load_config(mapping_config_path) + + @property + def is_on_cloud(self): + result = False + for header in CLOUD_HEADERS.values(): + if header in self.base_path: + result = True + break + return result + + @property + def size(self): + if self.model_path_on_host is None or not os.path.exists(self.model_path_on_host): + return 0.0 + file_list = os.listdir(self.model_path_on_host) + file_ext = ".bin" if self.model_type == ModelType.IR else ".onnx" + detected = [x for x in file_list if file_ext in x] + result = 1 + if len(detected) > 0: + result = Path(self.model_path_on_host, detected[0]).stat().st_size + + return result + + @property + def input_names(self): + return list(self.inputs.keys()) + + @property + def output_names(self): + return list(self.outputs.keys()) + + @property + def input_shapes(self): + return {k: v["shape"] for k, v in self.inputs.items()} + + @input_shapes.setter + def input_shapes(self, shape): + for _, v in self.inputs.items(): + v["shape"] = shape + + @property + def input_layouts(self): + return {k: v.get("layout", None) for k, v in self.inputs.items()} if self.inputs else {} + + @input_layouts.setter + def input_layouts(self, layout): + for _, v in self.inputs.items(): + v["layout"] = layout + + @property + def output_shapes(self): + return {k: v["shape"] for k, v in self.outputs.items()} + + @property + def input_types(self): + return {k: v["dtype"] for k, v in self.inputs.items()} + + @property + def output_types(self): + return {k: v["dtype"] for k, v in self.outputs.items()} + + @property + def input_datasets(self): + return {k: v["dataset"] if "dataset" in v else None for k, v in self.inputs.items()} + + def is_dynamic(self): + return False + + @staticmethod + def rename_input_output_data(data, src_name, dst_name): + data[dst_name] = data[src_name] + del data[src_name] + return data diff --git a/tests/functional/models/models_datasets.py b/tests/functional/models/models_datasets.py new file mode 100644 index 0000000000..26e1bdc428 --- /dev/null +++ b/tests/functional/models/models_datasets.py @@ -0,0 +1,265 @@ + +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=arguments-renamed +# pylint: disable=super-init-not-called +# pylint: disable=too-many-positional-arguments +# pylint: disable=unused-argument + +import json +import os +import re +from dataclasses import dataclass, field +from io import BytesIO +from random import choice +from string import ascii_lowercase + +import cv2 +import numpy as np +from PIL import Image + +from tests.functional.config import binary_io_images_path, datasets_path +from tests.functional.constants.ovms import Ovms +from tests.functional.utils.inference.serving.openai import ChatCompletionsApi +from tests.functional.utils.numpy_loader import prepare_data + + +class ModelDataset: + + @staticmethod + def create(data_str): + result = None + ext = os.path.splitext(data_str)[1] + if ext == ".npy": + result = NumPyDataset(data_str) + return result + + def __init__(self): + self.data_path = None + self.name = None + self.shape = None + + def get_data(self, shape, batch_size, transpose_axes, layout=None, datatype=np.float32): + data = prepare_data( + data_path=self.data_path, + expected_shape=shape, + batch_size=batch_size, + transpose_axes=transpose_axes, + expected_layout=layout, + data_layout=None, + ) + self.shape = data.shape + return data + + def to_str(self): + return json.dumps(self.__dict__) + + +class NumPyDataset(ModelDataset): + + def __init__(self, *data_path): + self.name = data_path[0] + self.data_path = os.path.join(datasets_path, *data_path) + + +class LanguageModelDataset(ModelDataset): + str_input_data = ["Lorem ipsum dolor sit amet", "consectetur adipiscing elit", "sed do eiusmod tempor"] + + def __init__(self, data_sample=0): + try: + self.default_str_input_data = self.str_input_data[data_sample] + except IndexError: + self.default_str_input_data = self.str_input_data[-1] + + def get_data(self, shape, batch_size, transpose_axes, layout=None, datatype=np.float32): + # https://github.com/triton-inference-server/client/blob/main/src/python/examples/simple_grpc_string_infer_client.py + str_input_data = [""] if batch_size == 0 else [s.split() for s in self.get_str_input_data()] + return np.array([str(x).encode("utf-8") for x in str_input_data], dtype=np.object_) + + def get_str_input_data(self): + return [self.default_str_input_data] + + def get_source_data(self, shape, dtype=np.float32): + return {} + + def create_data(self, tmp_file_location, shape, img_format): + return {} + + @staticmethod + def generate_random_text_list(inputs_number, word_length=5): + return ["".join(choice(ascii_lowercase) for _ in range(word_length)) for _ in range(inputs_number)] + + +class LargeLanguageModelDataset(ModelDataset): + user_content = "What is OpenVINO?" + system_content = "You are a helpful assistant." + user_data = [ChatCompletionsApi.ROLE_USER, user_content] + system_data = [ChatCompletionsApi.ROLE_SYSTEM, system_content] + input_data = [system_data, user_data] + + def __init__(self, data_sample=0): + self.default_input_data = self.input_data + + def get_data(self, shape, batch_size, transpose_axes, layout=None, datatype=np.float32): + return self.default_input_data + + def get_source_data(self, shape, dtype=np.float32): + return {} + + def create_data(self, tmp_file_location, shape, img_format): + return {} + + +class FeatureExtractionModelDataset(LargeLanguageModelDataset): + input_data_1 = "That is a happy person." + input_data_2 = "That is a very happy person." + input_data = [input_data_1, input_data_2] + + def __init__(self, data_sample=0): + self.default_input_data = self.input_data + + def get_string_data(self): + return self.input_data_1 + + +class RerankModelDataset(LargeLanguageModelDataset): + query = "hello" + document_1 = "welcome" + document_2 = "farewell" + input_data = { + "query": query, + "documents": [document_1, document_2] + } + + def __init__(self, data_sample=0): + self.default_input_data = self.input_data + + def get_string_data(self): + return str(self.input_data) + + +class SingleMessageLanguageModelDataset(LargeLanguageModelDataset): + user_data = [ChatCompletionsApi.ROLE_USER, LargeLanguageModelDataset.user_content] + input_data = [user_data] + + +def load_image_data_from_path(full_path, img_format, img_mode=None, size=None): + img_byte_arr = BytesIO() + image_obj = Image.open(full_path, mode="r", formats=None) + if img_mode: + image_obj = image_obj.convert(img_mode) + if size: + image_obj = image_obj.resize(size) + image_obj.save(img_byte_arr, format=img_format) + return img_byte_arr.getvalue() + + +class BinaryDummyModelDataset(ModelDataset): + def get_data(self, shape, batch_size, transpose_axes, layout=None, datatype=np.float32): + return np.ones(shape, datatype) + + def get_source_data(self, shape, dtype=np.float32): + return np.ones(shape, dtype) + + def create_data(self, tmp_file_location, shape, img_format): + data = self.get_source_data(shape) + fname = f'generated_ones_{"x".join([str(x) for x in shape])}.{img_format.lower()}' + os.makedirs(tmp_file_location, exist_ok=True) + cv2.imwrite(os.path.join(tmp_file_location, fname), data) # pylint: disable=no-member + return load_image_data_from_path(os.path.join(tmp_file_location, fname), img_format) + + +@dataclass +class DefaultBinaryDataset(ModelDataset): + _saved_labels_to_path_mapping: dict = None + image_format: str = None + image_mode: str = None + max_num_of_images: int = None + offset: int = 0 + + def get_path(self): + return os.path.join(binary_io_images_path, "input_images.txt") + + def _get_image_label_mapping(self): + image_list_path = self.get_path() + image_labels = {} + with open(image_list_path, "r", encoding="utf-8") as f: + for line in f.readlines(): + path, label = line.strip().split(" ") + image_labels[path] = label + return image_labels + + def get_data(self, shape, batch_size, transpose_axes, layout, reshape=False, datatype=np.float32): + labels_to_path_mapping = self._get_image_label_mapping() + + i = 0 + images = [] + size = shape[-2:] if reshape else None + for path, _ in labels_to_path_mapping.items(): + i += 1 + if i <= self.offset: + continue + + full_path = os.path.join(binary_io_images_path, path) + img_data = load_image_data_from_path(full_path, self.image_format, size=size) + images.append(img_data) + + if len(images) == batch_size: + break + + self._saved_labels_to_path_mapping = labels_to_path_mapping + return images + + def verify_match(self, response): + result = True + + if self._saved_labels_to_path_mapping is None: + return result + + labels = list(self._saved_labels_to_path_mapping.values()) + + nu = list(response.items()) + assert len(nu) == 1 # We expect single dimension result + nu = nu[0][1] + + if nu.shape[-1] == 1001: + model_offset = 1 + else: + model_offset = 0 + + for i in range(nu.shape[0]): + label = int(labels[i + self.offset]) + single_result = nu[[i], ...] + ma = np.argmax(single_result) - model_offset + if label != ma: + result = False + return result + + +@dataclass +class ExactShapeBinaryDataset(DefaultBinaryDataset): + shape: dict = field(default_factory=lambda: []) + image_format: str = Ovms.JPG_IMAGE_FORMAT + + def get_path(self): + file_path = os.path.join(binary_io_images_path, "images", "_".join([str(x) for x in self.shape])) + file_names = list(filter(lambda x: re.match(r".+\.jpe?g$", x.lower()), os.listdir(file_path))) + assert len(file_names) == 1, f"Unable to find images file with shape: {self.shape}" + return os.path.join(file_path, file_names[0]) + + def get_data(self, shape, batch_size, transpose_axes, layout=None, reshape=False, datatype=np.float32): + path = self.get_path() + return [load_image_data_from_path(path, self.image_format)] diff --git a/tests/functional/models/models_generative.py b/tests/functional/models/models_generative.py new file mode 100644 index 0000000000..4ea438283c --- /dev/null +++ b/tests/functional/models/models_generative.py @@ -0,0 +1,188 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=arguments-renamed +# pylint: disable=too-many-instance-attributes + +import os +import shutil + +from dataclasses import dataclass +from pathlib import Path + +from tests.functional.config import generative_models_local_path +from tests.functional.constants.paths import Paths +from tests.functional.models.models import ModelInfo, ModelType +from tests.functional.models.models_datasets import ( + FeatureExtractionModelDataset, + LargeLanguageModelDataset, + RerankModelDataset, + SingleMessageLanguageModelDataset, +) + + +@dataclass +class GenerativeModel(ModelInfo): + model_type: ModelType = ModelType.IR + is_generative: bool = True + is_local: bool = True + precision: str = "INT8" + precision_dir: str = "INT8" + parent_name: str = None + parent_base_dir: str = os.path.join("pytorch", "ov") + parent_precision_dir: str = "OV_FP16-INT8_ASYM" + max_position_embeddings: int = None + model_path_on_parent_host: str = None + model_subpath: str = None + single_message_dataset: bool = False + allows_reasoning: bool = False + is_llm: bool = False + is_feature_extraction: bool = False + is_rerank: bool = False + is_image_generation: bool = False + is_audio: bool = False + is_hf_direct_load: bool = False + is_agentic: bool = False + gguf_filename: str = None + pooling: str = None + transformers_v4_required: bool = False + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls._own_field_defaults = {} + for name in getattr(cls, '__annotations__', {}): + if name in cls.__dict__: + cls._own_field_defaults[name] = cls.__dict__[name] + + def __post_init__(self): + self.model_base_path_on_host = generative_models_local_path + self.model_subpath = os.path.join(self.precision_dir, Path(self.name)) + self.model_path_on_host = os.path.join(self.model_base_path_on_host, self.model_subpath) + self.base_path = os.path.join(Paths.MODELS_PATH_INTERNAL, self.model_subpath) + + def get_default_dataset(self): + return + + def prepare_input_data(self, batch_size=None, input_key=None, dataset=None, input_data_type=None): + if dataset is not None: + input_data = {input_name: dataset().get_data(None, None, None) for input_name in self.input_names} + elif input_data_type == "string": + input_data = { + input_name: self.inputs[input_name]["dataset"].get_string_data() + for input_name in self.input_names + } + else: + input_data = { + input_name: self.inputs[input_name]["dataset"].get_data(None, None, None) + for input_name in self.input_names + } + return input_data + + +@dataclass +class LargeLanguageModel(GenerativeModel): + is_llm: bool = True + + def get_default_dataset(self): + if self.single_message_dataset: + return SingleMessageLanguageModelDataset + return LargeLanguageModelDataset + + +@dataclass +class FeatureExtractionModel(GenerativeModel): + use_subconfig: bool = True + is_feature_extraction: bool = True + pooling: str = "CLS" + + def get_default_dataset(self): + return FeatureExtractionModelDataset + + +@dataclass +class RerankModel(GenerativeModel): + is_rerank: bool = True + use_subconfig: bool = True + + def get_default_dataset(self): + return RerankModelDataset + + +@dataclass +class GenerativeModelHuggingFace(GenerativeModel): + is_local: bool = False + is_hf_direct_load: bool = True + input_name: str = "input" + precision: str = "INT4" + model_timeout: int = 900 + + def _apply_diamond_defaults(self): + """Fix field defaults for diamond inheritance. + + When a class inherits from both LargeLanguageModelHuggingFace and a specialized + type (e.g. ImageGenerationModel), LargeLanguageModelHuggingFace's inherited field + defaults override the specialized type's directly-defined defaults. This method + restores the correct defaults from specialized parent classes. + """ + cls = type(self) + seen_fields = set() + seen_fields.update(getattr(cls, '_own_field_defaults', {}).keys()) + seen_fields.update(getattr(GenerativeModelHuggingFace, '_own_field_defaults', {}).keys()) + for base in cls.__mro__: + if base in (cls, object, GenerativeModelHuggingFace, GenerativeModel, ModelInfo): + continue + own_defaults = getattr(base, '_own_field_defaults', {}) + for field_name, default_value in own_defaults.items(): + if field_name not in seen_fields: + setattr(self, field_name, default_value) + seen_fields.add(field_name) + + def __post_init__(self): + self._apply_diamond_defaults() + if self.is_local: + self.model_base_path_on_host = generative_models_local_path + self.model_path_on_host = os.path.join(self.model_base_path_on_host, Path(self.name)) + self.model_subpath = os.path.join("models_ov_hf", Path(self.name)) + self.base_path = os.path.join(Paths.MODELS_PATH_INTERNAL, self.name) + self.set_additional_model_params() + + def prepare_resources(self, base_location): + models_dir = Path(base_location, Paths.MODELS_PATH_NAME) + models_dir.mkdir(exist_ok=True, parents=True) + if self.is_local: + models_sub_dir = Path(models_dir, self.name) + if not models_sub_dir.exists(): + shutil.copytree(self.model_path_on_host, models_sub_dir) + return [str(models_dir)] + + def prepare_input_data(self, batch_size=None, input_key=None, dataset=None, input_data_type=None): + if dataset is not None: + dataset_obj = dataset if not isinstance(dataset, type) else dataset() + else: + dataset_obj = self.get_default_dataset()() # pylint: disable=not-callable + if input_data_type == "string": + input_data = {self.input_name: dataset_obj.get_string_data()} + else: + input_data = {self.input_name: dataset_obj.get_data(None, None, None)} + return input_data + + +@dataclass +class Qwen3Embedding06BFp16OvHf(GenerativeModelHuggingFace, FeatureExtractionModel): + name: str = "OpenVINO/Qwen3-Embedding-0.6B-fp16-ov" + precision: str = "FP16" + pooling: str = "LAST" + is_local: bool = True diff --git a/tests/functional/models/models_library.py b/tests/functional/models/models_library.py new file mode 100644 index 0000000000..a095ad7caf --- /dev/null +++ b/tests/functional/models/models_library.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from tests.functional.models.models_generative import Qwen3Embedding06BFp16OvHf + + +class ModelsLibrary: + + @property + def various_feature_extraction_models(self): + return [Qwen3Embedding06BFp16OvHf] + + +ModelsLib = ModelsLibrary() # pylint: disable=invalid-name diff --git a/tests/functional/object_model/custom_node.py b/tests/functional/object_model/custom_node.py index a2be9f13d4..295d8eb95a 100644 --- a/tests/functional/object_model/custom_node.py +++ b/tests/functional/object_model/custom_node.py @@ -25,7 +25,7 @@ from tests.functional.utils.logger import get_logger from tests.functional.utils.process import Process from tests.functional.config import custom_nodes_path, ovms_c_repo_path -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from tests.functional.constants.ovms import CurrentOvmsType from tests.functional.constants.paths import Paths diff --git a/tests/functional/object_model/inference_helpers.py b/tests/functional/object_model/inference_helpers.py index ab14c05968..da04acc62a 100644 --- a/tests/functional/object_model/inference_helpers.py +++ b/tests/functional/object_model/inference_helpers.py @@ -67,14 +67,14 @@ from tests.functional.utils.test_framework import FrameworkMessages, skip_if_runtime from tests.functional.utils.generative_ai.validation_utils import GenerativeAIValidationUtils from tests.functional.config import binary_io_images_path, wait_for_messages_timeout -from ovms.constants.model_dataset import ( +from tests.functional.models.models import ModelInfo +from tests.functional.models.models_datasets import ( BinaryDummyModelDataset, DefaultBinaryDataset, ExactShapeBinaryDataset, LanguageModelDataset, ModelDataset, ) -from ovms.constants.models import ModelInfo from tests.functional.constants.ovms import CurrentTarget as ct from tests.functional.constants.ovms import MediaPipeConstants, Ovms from tests.functional.constants.pipelines import SimpleMediaPipe @@ -744,7 +744,7 @@ def predict_and_assert(inference_infos: List[InferenceInfo], validate_results=Tr MediaPipeInferenceResponse.create(inference_info, outputs).validate( inference_info.input_data, output_key=output_key ) - elif inference_info.model.is_llm: + elif inference_info.model.is_generative: LLMInferenceResponse.create(inference_info, outputs).validate() else: InferenceResponse.create(inference_info, outputs).validate(inference_info.input_data) diff --git a/tests/functional/object_model/mediapipe_calculators.py b/tests/functional/object_model/mediapipe_calculators.py index 81f1eea0bf..50cd69fd8a 100644 --- a/tests/functional/object_model/mediapipe_calculators.py +++ b/tests/functional/object_model/mediapipe_calculators.py @@ -35,7 +35,7 @@ mediapipe_repo_branch, ovms_c_repo_path, ) -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from tests.functional.constants.target_device import TargetDevice from tests.functional.constants.ovms import Config, MediaPipeConstants from tests.functional.constants.paths import Paths diff --git a/tests/functional/object_model/ovms_binary.py b/tests/functional/object_model/ovms_binary.py index 7f61beca9c..1d4ef610ac 100644 --- a/tests/functional/object_model/ovms_binary.py +++ b/tests/functional/object_model/ovms_binary.py @@ -28,12 +28,10 @@ from tests.functional.config import artifacts_dir from tests.functional.constants.core import CONTAINER_STATUS_EXITED, CONTAINER_STATUS_RUNNING -from ovms.constants.models import Muse from tests.functional.constants.ovms_binaries import get_ovms_binary_cmd_setup from tests.functional.constants.ovms_type import OvmsType from tests.functional.constants.paths import Paths from tests.functional.utils.log_monitor import LogMonitor -from tests.functional.object_model.cpu_extension import MuseModelExtension from tests.functional.object_model.mediapipe_calculators import MediaPipeCalculator from tests.functional.object_model.ovms_command import create_ovms_command from tests.functional.object_model.ovms_config import OvmsConfig @@ -100,10 +98,12 @@ def start_binary_ovms( MediaPipeCalculator.prepare_proto_calculator(parameters, config_dir_path_on_host, config_path_on_host) cpu_extension_path = None - if parameters.models is not None and any(isinstance(model, Muse) for model in parameters.models): - cpu_extension = MuseModelExtension() - cpu_extension_path = cpu_extension.lib_path[1:] - elif parameters.cpu_extension: + if parameters.models is not None: + for model in parameters.models: + if getattr(model, "cpu_extension", None) is not None: + cpu_extension = model.cpu_extension() + cpu_extension_path = cpu_extension.lib_path[1:] + if parameters.cpu_extension: if kwargs.get("replace_cpu_extension_params_for_binary", True): host_dir = os.path.join(resources_dir, Paths.CPU_EXTENSIONS) host_lib_path = os.path.join(host_dir, parameters.cpu_extension.lib_name) diff --git a/tests/functional/object_model/ovms_capi.py b/tests/functional/object_model/ovms_capi.py index 010d33ac54..6726295155 100644 --- a/tests/functional/object_model/ovms_capi.py +++ b/tests/functional/object_model/ovms_capi.py @@ -33,7 +33,7 @@ from tests.functional.utils.test_framework import generate_test_object_name, skip_if_runtime from tests.functional.config import ovms_c_repo_path from tests.functional.constants.core import CONTAINER_STATUS_RUNNING -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from tests.functional.constants.ovms import Config from tests.functional.constants.ovms_messages import OvmsMessages from tests.functional.constants.ovms_type import OvmsType diff --git a/tests/functional/object_model/ovms_config.py b/tests/functional/object_model/ovms_config.py index 88230c7576..8595d0771b 100644 --- a/tests/functional/object_model/ovms_config.py +++ b/tests/functional/object_model/ovms_config.py @@ -23,7 +23,7 @@ from tests.functional.constants.os_type import OsType from tests.functional.config import enable_plugin_config_target_device from tests.functional.constants.custom_loader import CustomLoaderConsts -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from tests.functional.constants.ovms import Config, CurrentOvmsType, Ovms, set_plugin_config_boolean_value from tests.functional.constants.ovms_type import OvmsType from tests.functional.constants.paths import Paths diff --git a/tests/functional/object_model/ovms_info.py b/tests/functional/object_model/ovms_info.py index 036c9d21f6..8ad621b103 100644 --- a/tests/functional/object_model/ovms_info.py +++ b/tests/functional/object_model/ovms_info.py @@ -28,6 +28,7 @@ from tests.functional.config import tmp_dir from tests.functional.constants.ovms_binaries import get_binaries, get_ovms_binary_cmd_setup from tests.functional.constants.ovms_type import OvmsType, OVMS_BINARY_PACKAGE_EXTENSIONS +from tests.functional.utils.docker import DockerClient logger = get_logger(__name__) @@ -219,8 +220,6 @@ def docker_pull_image_cli(image_to_pool): def pull_latest_image(cls, image_to_pull, force_pull=False): cls.docker_pull_image_cli(image_to_pull) # ensure image is available on host. - from tests.functional.utils.docker import DockerClient # pylint: disable=import-outside-toplevel - if image_to_pull not in cls.IMAGES or force_pull: repository, tag = image_to_pull.split(":") logger.info("Pulling image: {} tag: {}".format(repository, tag)) diff --git a/tests/functional/object_model/ovms_instance.py b/tests/functional/object_model/ovms_instance.py index 1f8ad15cfe..6ab0630a23 100644 --- a/tests/functional/object_model/ovms_instance.py +++ b/tests/functional/object_model/ovms_instance.py @@ -53,7 +53,7 @@ wait_for_messages_timeout, ) from tests.functional.constants.core import CONTAINER_STATUS_EXITED, CONTAINER_STATUS_RUNNING -from ovms.constants.models import ModelInfo +from tests.functional.models.models import ModelInfo from tests.functional.constants.target_device import MAX_WORKERS_PER_TARGET_DEVICE from tests.functional.constants.ovms import CurrentTarget as ct from tests.functional.constants.ovms import Ovms diff --git a/tests/functional/object_model/ovms_params.py b/tests/functional/object_model/ovms_params.py index 253737cb80..e732516f4b 100644 --- a/tests/functional/object_model/ovms_params.py +++ b/tests/functional/object_model/ovms_params.py @@ -25,9 +25,8 @@ from tests.functional.object_model.ovms_command import OvmsCommand from tests.functional.config import logging_level_ovms from tests.functional.constants.metrics import MetricsPolicy -from ovms.constants.models import ModelInfo, Muse -from ovms.constants.models_library import ModelsLib -from tests.functional.object_model.cpu_extension import MuseModelExtension +from tests.functional.models.models import ModelInfo +from tests.functional.models.models_library import ModelsLib from tests.functional.object_model.custom_loader import CustomLoader logger = get_logger(__name__) @@ -82,8 +81,11 @@ class OvmsParams(object): cache_size: int = None def __post_init__(self): - if self.models is not None and any(isinstance(model, Muse) for model in self.models): - self.cpu_extension = MuseModelExtension() + if self.models is not None: + for model in self.models: + if getattr(model, "cpu_extension", None) is not None: + self.cpu_extension = model.cpu_extension() + break def get_shape_param(self): result = self.shape diff --git a/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py b/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py index bd5947f6ae..4646889f53 100644 --- a/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py +++ b/tests/functional/object_model/python_custom_nodes/python_custom_nodes.py @@ -19,9 +19,9 @@ from tests.functional.utils.inference.communication import GRPC from tests.functional.utils.logger import get_logger from tests.functional.constants.generative_ai import GenerativeAIPluginConfig -from ovms.constants.model_dataset import LanguageModelDataset from tests.functional.constants.ovms import Ovms from tests.functional.constants.pipelines import MediaPipe, NodesConnection, NodeType, PythonGraphNode +from tests.functional.models.models_datasets import LanguageModelDataset from tests.functional.object_model.mediapipe_calculators import HttpLLMCalculator, PythonCalculator, \ ImageGenCalculator, EmbeddingsCalculatorOV, RerankCalculatorOV, S2tCalculator, T2sCalculator from tests.functional import config @@ -47,7 +47,7 @@ class SimplePythonCustomNodeMediaPipe(MediaPipe): def __init__(self, handler_path, node_name="upper_text", loopback=False, initialize_graphs=True, **kwargs): self.node_name = node_name self.config = {} - self.prepare_llm_model_inputs_outputs(model=self, dataset=LanguageModelDataset, kwargs=kwargs) + self.prepare_model_inputs_outputs(model=self, dataset=LanguageModelDataset, kwargs=kwargs) super().__init__(**kwargs) self.calculators = [ @@ -65,7 +65,7 @@ def __init__(self, handler_path, node_name="upper_text", loopback=False, initial self.handler_path = handler_path @staticmethod - def prepare_llm_model_inputs_outputs(model, dataset, **kwargs): + def prepare_model_inputs_outputs(model, dataset, **kwargs): inputs_number = kwargs.get("inputs_number", None) model.inputs_number = inputs_number if inputs_number is not None else model.inputs_number model.inputs = { @@ -253,7 +253,7 @@ def is_pipeline(): return True -class SimpleLLM(SimplePythonCustomNodeMediaPipe): +class SimpleGenerativeNode(SimplePythonCustomNodeMediaPipe): inputs_number = 1 input_name: str = "input" outputs_number = 1 @@ -265,17 +265,18 @@ class SimpleLLM(SimplePythonCustomNodeMediaPipe): name: str = "" pbtxt_name: str = "simple_llm" is_python_custom_node: bool = True - is_llm: bool = True + is_generative: bool = True + is_llm: bool = False calculator_class = HttpLLMCalculator + use_subconfig: bool = False precision: str = None - allows_reasoning: bool = False def __init__(self, models_path, node_name="LLMExecutor", loopback=True, initialize_graphs=True, **kwargs): model = kwargs["model"] self.node_name = node_name self.config = {} dataset = model.get_default_dataset() - self.prepare_llm_model_inputs_outputs(model=self, dataset=dataset, kwargs=kwargs) + self.prepare_model_inputs_outputs(model=self, dataset=dataset, kwargs=kwargs) self.regular_models = [] self.is_mediapipe = True @@ -315,28 +316,23 @@ def __init__(self, models_path, node_name="LLMExecutor", loopback=True, initiali self.model_timeout = getattr(model, "model_timeout", None) -class SimpleImageGenerationLLM(SimpleLLM): - inputs_number = 1 - input_name: str = "input" - outputs_number = 1 - output_name: str = "output" - child_nodes: list = None - inputs: dict = None - outputs: list = None - base_path: str = "" - name: str = "" - pbtxt_name: str = "simple_llm" - is_python_custom_node: bool = True +class SimpleLLM(SimpleGenerativeNode): is_llm: bool = True + allows_reasoning: bool = False + + def __init__(self, models_path, node_name="LLMExecutor", loopback=False, initialize_graphs=True, **kwargs): + super().__init__(models_path, node_name, loopback, initialize_graphs, **kwargs) + + +class SimpleImageGeneration(SimpleGenerativeNode): calculator_class = ImageGenCalculator - precision: str = None def __init__(self, models_path, node_name="ImageGenExecutor", initialize_graphs=True, **kwargs): model = kwargs["model"] self.node_name = node_name self.config = {} dataset = model.get_default_dataset() - self.prepare_llm_model_inputs_outputs(model=self, dataset=dataset, kwargs=kwargs) + self.prepare_model_inputs_outputs(model=self, dataset=dataset, kwargs=kwargs) self.regular_models = [] self.is_mediapipe = True @@ -368,61 +364,25 @@ def __init__(self, models_path, node_name="ImageGenExecutor", initialize_graphs= self.model_timeout = getattr(model, "model_timeout", None) -class SimpleFeatureExtractionLLM(SimpleLLM): - inputs_number: int = 1 - input_name: str = "input" - outputs_number: int = 1 - output_name: str = "output" - child_nodes: list = None - inputs: dict = None - outputs: list = None - base_path: str = "" - name: str = "" - pbtxt_name: str = "simple_llm" - is_python_custom_node: bool = True - is_llm: bool = True - use_subconfig: bool = True +class SimpleFeatureExtraction(SimpleGenerativeNode): is_feature_extraction: bool = True calculator_class = EmbeddingsCalculatorOV + use_subconfig: bool = True def __init__(self, models_path, node_name="LLMExecutor", loopback=False, initialize_graphs=True, **kwargs): super().__init__(models_path, node_name, loopback, initialize_graphs, **kwargs) -class SimpleRerankLLM(SimpleLLM): - inputs_number: int = 1 - input_name: str = "input" - outputs_number: int = 1 - output_name: str = "output" - child_nodes: list = None - inputs: dict = None - outputs: list = None - base_path: str = "" - name: str = "" - pbtxt_name: str = "simple_llm" - is_python_custom_node: bool = True - is_llm: bool = True - use_subconfig: bool = True +class SimpleRerank(SimpleGenerativeNode): is_rerank: bool = True calculator_class = RerankCalculatorOV + use_subconfig: bool = True def __init__(self, models_path, node_name="LLMExecutor", loopback=False, initialize_graphs=True, **kwargs): super().__init__(models_path, node_name, loopback, initialize_graphs, **kwargs) -class SimpleAsrModel(SimpleLLM): - inputs_number: int = 1 - input_name: str = "input" - outputs_number: int = 1 - output_name: str = "output" - child_nodes: list = None - inputs: dict = None - outputs: list = None - base_path: str = "" - name: str = "" - pbtxt_name: str = "simple_llm" - is_python_custom_node: bool = True - is_llm: bool = False +class SimpleAsrModel(SimpleGenerativeNode): is_asr_model: bool = True calculator_class = S2tCalculator @@ -430,19 +390,7 @@ def __init__(self, models_path, node_name="S2tExecutor", loopback=False, initial super().__init__(models_path, node_name, loopback, initialize_graphs, **kwargs) -class SimpleTtsModel(SimpleLLM): - inputs_number: int = 1 - input_name: str = "input" - outputs_number: int = 1 - output_name: str = "output" - child_nodes: list = None - inputs: dict = None - outputs: list = None - base_path: str = "" - name: str = "" - pbtxt_name: str = "simple_llm" - is_python_custom_node: bool = True - is_llm: bool = False +class SimpleTtsModel(SimpleGenerativeNode): is_tts_model: bool = True calculator_class = T2sCalculator diff --git a/tests/functional/pylintrc b/tests/functional/pylintrc new file mode 100644 index 0000000000..90c83a19e1 --- /dev/null +++ b/tests/functional/pylintrc @@ -0,0 +1,768 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +# ignore= + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + config.py, + constants/metrics.py, + constants/ovms.py, + constants/ovms_images.py, + constants/ovms_messages.py, # W0511: + constants/ovms_openai.py, + constants/paths.py, + constants/pipelines.py, # (R0801: duplicate code with models.py) + constants/target_device_configuration.py, # W0105: String statement has no effect (pointless-string-statement) + data/ovms_capi_wrapper/ovms_autopxd.py, + data/ovms_capi_wrapper/setup.py, + data/python_custom_nodes/incrementer/incrementer.py, + data/python_custom_nodes/ovms_basic/python_model.py, + data/python_custom_nodes/ovms_basic/python_model_loopback.py, + data/python_custom_nodes/ovms_corrupted/python_model_corrupted_import.py, + data/python_custom_nodes/ovms_corrupted/python_model_exceptions.py, + data/python_custom_nodes/ovms_corrupted/python_model_loopback_multiple_use_of_valid_outputs.py, + data/python_custom_nodes/ovms_corrupted/python_model_loopback_return_instead_of_yield.py, + data/python_custom_nodes/ovms_corrupted/python_model_writing_to_loopback_output_in_execute.py, + fixtures/*, + object_model/cpu_extension.py, # C0103: Attribute name doesn't conform to naming style (invalid-name) + object_model/custom_loader.py, + object_model/custom_node.py, + object_model/dmesg_log_monitor.py, + object_model/inference_helpers.py, + object_model/mediapipe_calculators.py, + object_model/ovms_binary.py, + object_model/ovms_capi.py, + object_model/ovms_command.py, + object_model/ovms_config.py, # (R0801: duplicate code with ovms_mapping_config.py) + object_model/ovms_docker.py, + object_model/ovms_info.py, + object_model/ovms_instance.py, + object_model/ovms_log_monitor.py, + object_model/ovms_mapping_config.py, # (R0801: duplicate code with ovms_config.py) + object_model/ovms_params.py, + object_model/ovsa.py, + object_model/package_manager.py, + object_model/python_custom_nodes/python_custom_nodes.py, + object_model/resource_monitor.py, + object_model/shape.py, + object_model/test_environment.py, # R0205: (useless-object-inheritance) + object_model/test_helpers.py, + utils/assertions.py, + utils/context.py, + utils/core.py, + utils/docker.py, + utils/git_operations.py, + utils/hooks.py, + utils/http/base.py, + utils/http/client_auth/auth.py, + utils/http/client_auth/base.py, + utils/http/http_client.py, + utils/http/http_client_configuration.py, + utils/http/http_client_factory.py, + utils/http/http_session.py, + utils/http/http_socket_wrapper.py, + utils/inference/capi.py, + utils/inference/communication/base.py, + utils/inference/communication/grpc.py, + utils/inference/communication/rest.py, + utils/inference/inference_client_factory.py, + utils/inference/serving/base.py, + utils/inference/serving/kf.py, + utils/inference/serving/openai.py, + utils/inference/serving/tf.py, + utils/inference/serving/triton.py, + utils/log_monitor.py, + utils/logger.py, + utils/marks.py, + utils/numpy_loader.py, + utils/port_manager.py, + utils/process.py, + utils/reservation_manager/__init__.py, + utils/reservation_manager/__main__.py, + utils/reservation_manager/args.py, + utils/reservation_manager/locker.py, + utils/reservation_manager/manager.py, + utils/reservation_manager/manager_config.py, + utils/reservation_manager/runner.py, + utils/reservation_manager/unittests/test_manager.py, + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules=openai + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +init-hook='import sys; sys.path.insert(0, ".")' + + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=0 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.12 + +# Discover python modules and packages in the file system subtree. +recursive=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names= + foo, + bar, + baz, + toto, + tutu, + tata, + dupa, + kaczka + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names= + i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods= + __init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected= + _asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=8 + +# Maximum number of positional arguments for function / method. +max-positional-arguments=20 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions= + builtins.BaseException, + builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence= + HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable= + raw-checker-failed, # (I0001) + bad-inline-option, # (I0010) + locally-disabled, # (I0011) + file-ignored, # (I0013) + suppressed-message, # (I0020) + useless-suppression, # (I0021) + deprecated-pragma, # (I0022) + use-symbolic-message-instead, # (I0023) + logging-fstring-interpolation, # (W1203, to be FIXED) + logging-format-interpolation, # (W1202, to be FIXED) + missing-function-docstring, # (C0116, to be FIXED) + missing-class-docstring, # (C0115, to be FIXED) + missing-module-docstring, # (C0114, to be FIXED) + too-few-public-methods, # (R0903, to be FIXED) + too-many-lines, # (allowed intentionally) + too-many-statements, # (allowed intentionally) + too-many-locals, # (allowed intentionally) + too-many-branches, # (allowed intentionally) + + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods= + requests.api.delete, + requests.api.get, + requests.api.head, + requests.api.options, + requests.api.patch, + requests.api.post, + requests.api.put, + requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions= + sys.exit, + argparse.parse_error + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation= + max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format=colorized + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. Available dictionaries: none. To make it work, +# install the 'python-enchant' package. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives= + fmt: on, + fmt: off, + noqa:, + noqa, + nosec, + isort:skip, + mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members=random.getrandbits + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins= + no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes= + optparse.Values, + thread._local, + _thread._local, + argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks= + cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules= + six.moves, + past.builtins, + future.builtins, + builtins,io diff --git a/tests/functional/utils/assertions.py b/tests/functional/utils/assertions.py index 8e5299311d..15c3a0477c 100644 --- a/tests/functional/utils/assertions.py +++ b/tests/functional/utils/assertions.py @@ -369,6 +369,10 @@ class OVVPException(OvmsTestException): pass +class OVHfDownloadException(OvmsTestException): + pass + + def get_exception_by_ovms_log(ovms_log_lines): exceptions_to_recognize = [NginxException] diff --git a/tests/functional/utils/docker.py b/tests/functional/utils/docker.py index 51f86e70dd..35d5a472b0 100644 --- a/tests/functional/utils/docker.py +++ b/tests/functional/utils/docker.py @@ -33,6 +33,8 @@ from tests.functional.config import docker_client_timeout from tests.functional.constants.core import CONTAINER_STATUS_RUNNING +DOCKER_CONTAINER_TMP_PATH = "/tmp" + logger = get_logger(__name__) diff --git a/tests/functional/utils/download.py b/tests/functional/utils/download.py new file mode 100644 index 0000000000..8f589a9acc --- /dev/null +++ b/tests/functional/utils/download.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from pathlib import Path + +from tests.functional.utils.core import SelfDeletingFileLock +from tests.functional.utils.logger import get_logger +from tests.functional.utils.process import Process + +logger = get_logger(__name__) + + +def wget_item(dst, cmd): + Path(dst).parent.mkdir(parents=True, exist_ok=True) + proc = Process() + proc.set_log_silence() + proc.policy["log-check-output"]["stderr"] = False + + with SelfDeletingFileLock(f"{dst}.lock", self_delete=True) as _: + proc.run_and_check(cmd) + + +def wget_file(url, dst): + logger.info(f"Downloading file via wget\n{url} => {dst}") + cmd = f"wget {url} -O {dst}" + wget_item(dst, cmd) + + +def curl_file(url, dst, user, token): + logger.info(f"Downloading file via curl\n{url} => {dst}") + cmd = f'curl --insecure -L --user {user}:{token} "{url}" -o {dst}' + proc = Process() + proc.set_log_silence() + proc.policy["log-check-output"]["stderr"] = False + with SelfDeletingFileLock(f"{dst}.lock", self_delete=True) as _: + proc.run_and_check(cmd) + + +def wget_folder(url, dst, depth=2, reject=".html,.tmp", extra_options=None): + logger.info(f"Downloading folder via wget\n{url} => {dst}") + options = f"-r --directory-prefix={dst} --no-parent --no-host-directories --cut-dirs={depth} --reject={reject}" + if extra_options is not None: + options += extra_options + cmd = f"wget {options} {url}" + wget_item(dst, cmd) diff --git a/tests/functional/utils/generative_ai/utils.py b/tests/functional/utils/generative_ai/utils.py new file mode 100644 index 0000000000..83245004a9 --- /dev/null +++ b/tests/functional/utils/generative_ai/utils.py @@ -0,0 +1,194 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=too-many-arguments +# pylint: disable=too-many-positional-arguments + +from openai import NotFoundError + +from tests.functional.config import ( + logging_level_ovms, + kv_cache_size_value, + kv_cache_precision_value, +) +from tests.functional.constants.generative_ai import GenerativeAIPluginConfig +from tests.functional.constants.ovms import CurrentTarget as ct +from tests.functional.constants.ovms_cohere import OvmsCohereRequestParamsBuilder +from tests.functional.constants.ovms_openai import OvmsOpenAIRequestParamsBuilder +from tests.functional.constants.ovms_messages import OvmsMessages +from tests.functional.fixtures.server import start_ovms +from tests.functional.object_model.inference_helpers import run_llm_inference +from tests.functional.object_model.ovms_params import OvmsParams +from tests.functional.utils.assertions import assert_raises_exception +from tests.functional.utils.context import Context +from tests.functional.utils.hooks import timeout_dict +from tests.functional.utils.inference.serving.cohere import CohereWrapper +from tests.functional.utils.logger import step +from tests.functional.utils.marks import MarkRunType +from tests.functional.object_model.python_custom_nodes.python_custom_nodes import ( + SimpleLLM, + SimpleFeatureExtraction, + SimpleRerank, + SimpleImageGeneration, + SimpleAsrModel, + SimpleTtsModel, +) + +INITIALIZE_LLM_TIMEOUT = 900 + + +def calculate_generative_test_timeout(lm_time_sec): + test_timeout_sec = lm_time_sec + INITIALIZE_LLM_TIMEOUT + return ( + test_timeout_sec + if test_timeout_sec > timeout_dict[MarkRunType.TEST_MARK_REGRESSION_WEEKLY_SINGLE] or ct.is_gpu_target() + else timeout_dict[MarkRunType.TEST_MARK_REGRESSION_WEEKLY_SINGLE] + ) + + +class GenerativeAIUtils: + + @staticmethod + def prepare_request_params(endpoint, **request_params_kwargs): + request_params_builder_class = OvmsCohereRequestParamsBuilder if endpoint == CohereWrapper.RERANK \ + else OvmsOpenAIRequestParamsBuilder + request_params_builder = request_params_builder_class( + endpoint=endpoint, + **request_params_kwargs + ) + request_params = request_params_builder.request_params + return request_params + + @staticmethod + def prepare_model( + model_type, + kv_cache_size=kv_cache_size_value, + plugin_config=None, + max_position_embeddings=None, + tools_enabled=None, + apply_gorilla_patch=False, + enable_tool_guided_generation=False, + target_device=None, + resolution=None, + apply_short_name=False, + **kwargs + ): + if plugin_config is None: + plugin_config = {GenerativeAIPluginConfig.KV_CACHE_PRECISION: kv_cache_precision_value} + + stream = kwargs.get("stream", False) + + step("Prepare language model instance") + llm = model_type() + llm.apply_gorilla_patch = apply_gorilla_patch + if apply_gorilla_patch: + llm.name = f"{llm.gorilla_patch_name}-stream" if stream else llm.gorilla_patch_name + if apply_short_name: + llm.name = llm.name.split("/")[-1] if "/" in llm.name else llm.name + if tools_enabled: + llm.tools_enabled = True + + node_name = "LLMExecutor" + if hasattr(llm, "is_feature_extraction") and llm.is_feature_extraction: + node_class = SimpleFeatureExtraction + elif hasattr(llm, "is_rerank") and llm.is_rerank: + node_class = SimpleRerank + elif hasattr(llm, "is_image_generation") and llm.is_image_generation: + node_class = SimpleImageGeneration + node_name = "ImageGenExecutor" + elif hasattr(llm, "is_asr_model") and llm.is_asr_model: + node_class = SimpleAsrModel + node_name = "S2tExecutor" + elif hasattr(llm, "is_tts_model") and llm.is_tts_model: + node_class = SimpleTtsModel + node_name = "T2sExecutor" + else: + node_class = SimpleLLM + + model = node_class( + model=llm, + node_name=node_name, + models_path=llm.model_path_on_host, + kv_cache_size=kv_cache_size, + plugin_config=plugin_config, + enable_tool_guided_generation=enable_tool_guided_generation, + target_device=target_device, + resolution=resolution, + ) + model.precision = llm.precision + model.max_position_embeddings = max_position_embeddings + model.jinja_template = llm.jinja_template + model.allows_reasoning = llm.allows_reasoning + model.apply_gorilla_patch = apply_gorilla_patch + model.gorilla_patch_name = llm.gorilla_patch_name + model.enable_tool_guided_generation = enable_tool_guided_generation + model.bfcl_num_threads = llm.bfcl_num_threads + return model + + @classmethod + def prepare_resources( + cls, context: Context, model_type, openai_rest_api_type, endpoint, log_level=logging_level_ovms, + kv_cache_size=kv_cache_size_value, plugin_config=None, max_position_embeddings=None, env=None, + allowed_local_media_path=None, allowed_media_domains=None, target_device=None, resolution=None, + apply_short_name=False, **request_params_kwargs, + ): + if plugin_config is None: + plugin_config = {GenerativeAIPluginConfig.KV_CACHE_PRECISION: kv_cache_precision_value} + + model = cls.prepare_model( + model_type, + kv_cache_size, + plugin_config, + max_position_embeddings, + target_device=target_device, + resolution=resolution, + apply_short_name=apply_short_name, + ) + + step("Start OVMS") + result = start_ovms( + context, + OvmsParams(models=[model], use_config=True, use_subconfig=model.use_subconfig, log_level=log_level, + allowed_local_media_path=allowed_local_media_path, allowed_media_domains=allowed_media_domains), + timeout=model.model_timeout, environment=env, + ) + port = result.ovms.get_port(openai_rest_api_type) + + request_params = cls.prepare_request_params(endpoint, **request_params_kwargs) + + return model, result, port, request_params + + @staticmethod + def unload_model_and_verify(model, result, port, openai_rest_api_type, endpoint, request_parameters, + dataset=None, error_type=NotFoundError): + step("Unload model") + ovms_log_monitor = result.ovms.create_log(False) + result.ovms.unload_all_models() + ovms_log_monitor.models_unloaded([model]) + result.models = [] + + step("Verify model is unreachable") + assert_raises_exception( + error_type, + OvmsMessages.MEDIAPIPE_IS_RETIRED, + run_llm_inference, + model=model, + api_type=openai_rest_api_type, + port=port, + endpoint=endpoint, + dataset=dataset, + request_parameters=request_parameters, + ) diff --git a/tests/functional/utils/generative_ai/validation_utils.py b/tests/functional/utils/generative_ai/validation_utils.py index 3cfbd55a04..7bb4b6b92b 100644 --- a/tests/functional/utils/generative_ai/validation_utils.py +++ b/tests/functional/utils/generative_ai/validation_utils.py @@ -15,6 +15,7 @@ # # pylint: disable=too-many-nested-blocks +# pylint: disable=too-many-positional-arguments # pylint: disable=unused-argument import base64 @@ -33,7 +34,7 @@ from tests.functional.utils.inference.serving.openai import OpenAIWrapper, OpenAIFinishReason from tests.functional.config import save_image_to_artifacts from tests.functional.config import artifacts_dir, pipeline_type -from ovms.constants.model_dataset import FeatureExtractionModelDataset +from tests.functional.models.models_datasets import FeatureExtractionModelDataset logger = get_logger(__name__) @@ -545,7 +546,7 @@ def create_embeddings_getter( # pylint: disable=import-outside-toplevel api_type: OpenAI REST API type. port: OVMS port where the embeddings model is served. request_parameters: Optional pre-built request parameters for embeddings endpoint. - If None, will be built automatically via LLMUtils.prepare_request_params. + If None, will be built automatically via GenerativeAIUtils.prepare_request_params. inference_fn: Callable to run LLM inference (e.g. run_llm_inference). Injected to avoid circular import between this module and inference_helpers. @@ -557,8 +558,8 @@ def create_embeddings_getter( # pylint: disable=import-outside-toplevel ) if request_parameters is None: - from llm.utils import LLMUtils - request_parameters = LLMUtils.prepare_request_params(OpenAIWrapper.EMBEDDINGS) + from tests.functional.utils.generative_ai.utils import GenerativeAIUtils + request_parameters = GenerativeAIUtils.prepare_request_params(OpenAIWrapper.EMBEDDINGS) def getter(text): class TextDataset(FeatureExtractionModelDataset): diff --git a/tests/functional/utils/hooks.py b/tests/functional/utils/hooks.py index d9aa74c25f..557889f063 100644 --- a/tests/functional/utils/hooks.py +++ b/tests/functional/utils/hooks.py @@ -14,52 +14,132 @@ # limitations under the License. # +import itertools import os +import re import shutil +import sys +import time import warnings +import pytest -from collections import defaultdict +from collections import Counter, defaultdict, namedtuple from docker import errors as docker_errors +from itertools import groupby from pathlib import Path +from _pytest.mark import Mark, MarkDecorator +from _pytest.python import Function from tests.functional import config -from ovms.constants.models_library import ModelsLib +from tests.functional.models.models_library import ModelsLib, ModelsLibrary +from tests.functional.utils.download import wget_file from tests.functional.utils.reservation_manager.args import parse_args from tests.functional.utils.reservation_manager.manager import Manager as ReservationManager from tests.functional.config import ( + build_test_image, c_api_wrapper_dir, cleanup_env_on_startup, + components_ids, + exclude_components_ids, + exclude_req_ids, + force_generate_new_ssl_certs, global_tmp_dir_default, + http_proxy, + https_proxy, + is_nginx_mtls, + no_proxy, ovms_c_repo_path, + ovms_file_locks_dir, + performance_test_timeout_minutes, machine_is_reserved_for_test_session, + req_ids, + run_ovms_with_opencl_trace, + run_ovms_with_valgrind, + target_devices, + tests_priority_list, tmp_dir, ) -from tests.functional.constants.os_type import get_host_os, OsType +from tests.functional.constants.os_type import get_host_os, OsType, UBUNTU +from tests.functional.constants.os_version import os_type_to_base_image_binary_docker from tests.functional.constants.ovms import ( BASE_OS_PARAM_NAME, + CURRENT_TARGET_DEVICE_DICT_ARGUMENT, OVMS_TYPE_PARAM_NAME, TARGET_DEVICE_PARAM_NAME, + TMP_REPOS_DIR_ARGUMENT, USES_MAPPING_PARAM_NAME, ) -from tests.functional.constants.ovms_images import calculate_ovms_image_name +from tests.functional.constants.ovms_images import ( + calculate_ovms_binary_image_name, + calculate_ovms_capi_image_name, + calculate_ovms_test_image_name, + calculate_ovms_image_name, + get_ovms_calculated_images, + GPU_INSTALL_DRIVER_VERSION, + GPU_INSTALL_SCRIPTS, +) +from tests.functional.constants.ovms_type import ( + OvmsType, + OVMS_BINARY_DEPENDENCIES, + OVMS_BINARY_PACKAGE_EXTENSIONS, + OVMS_BINARY_PACKAGE_NAME, + OVMS_CAPI_DEPENDENCIES, + OVMS_CAPI_TOOLS_DEPENDENCIES, +) from tests.functional.constants.paths import Paths -from tests.functional.constants.target_device import TargetDevice +from tests.functional.constants.target_device import MAX_WORKERS_PER_TARGET_DEVICE, TargetDevice +from tests.functional.constants.ovms_binaries import calculate_ovms_binary_name from tests.functional.object_model.ovms_info import OvmsInfo from tests.functional.utils.core import TmpDir -from tests.functional.utils.docker import DockerClient, DockerContainer +from tests.functional.utils.docker import DockerClient, DockerContainer, DOCKER_CONTAINER_TMP_PATH from tests.functional.utils.environment_info import EnvironmentInfo from tests.functional.utils.logger import get_logger -from tests.functional.utils.marks import MarkTestParameters -from tests.functional.utils.process import Process -from tests.functional.utils.test_framework import get_test_object_prefix +from tests.functional.utils.marks import ( + MarkConditionalRunType, + MarkGeneral, + MarkPriority, + MarkRunType, + MarkTestParameters, +) +from tests.functional.utils.ov_hf_downloader import OVHfDownloader +from tests.functional.utils.process import PID_STATE_ZOMBIE, Process, get_pid_name, get_pid_status +from tests.functional.utils.test_framework import change_dir_permissions, get_test_object_prefix, is_xdist_master +from tests.functional.object_model.ovsa import OvsaCerts logger = get_logger(__name__) +timeout_dict = defaultdict( + lambda: 5 * 60, + { + MarkRunType.TEST_MARK_ON_COMMIT: 3 * 60, + MarkRunType.TEST_MARK_REGRESSION: 5 * 60, + MarkRunType.TEST_MARK_REGRESSION_SINGLE: 5 * 60, + MarkRunType.TEST_MARK_REGRESSION_WEEKLY: 5 * 60, + MarkRunType.TEST_MARK_REGRESSION_WEEKLY_SINGLE: 5 * 60, + MarkRunType.TEST_MARK_ENABLING: 10 * 60, + MarkRunType.TEST_MARK_STRESS_AND_LOAD: 40 * 60, + MarkRunType.TEST_MARK_STRESS_AND_LOAD_SINGLE: 40 * 60, + MarkRunType.TEST_MARK_LONG: 48 * 60 * 60, + MarkRunType.TEST_MARK_SMOKE: 5 * 60, + MarkRunType.TEST_MARK_MANUAL: 5 * 60, + MarkRunType.TEST_MARK_PERFORMANCE: performance_test_timeout_minutes * 60, + }, +) + +TIMEOUT_MULTIPLIER: dict = { + TargetDevice.GPU: 1.5, + TargetDevice.NPU: 1.5, + "TRACE_TOOLS": 2, + "AUTO_HETERO_MULTI": 3, +} + CURRENT_TARGET_DEVICE_DICT = {} DEVICE_ID_TO_DETAILED_TARGET_DEVICE_NAME_MAP = defaultdict(lambda: ("", []), {}) +SkippedItem = namedtuple("SkippedItem", "test_name reason") + def init_environment(_config): global CURRENT_TARGET_DEVICE_DICT @@ -123,6 +203,15 @@ def cleanup_docker_images(): logger.info(f"Removed docker image: {image.id}") +def cleanup_tmp_repos_dir(config): + try: + shutil.rmtree(config.tmp_repos_dir) + except PermissionError as e: + if get_host_os() == OsType.Windows and type(e) == PermissionError: + change_dir_permissions(config.tmp_repos_dir) + shutil.rmtree(config.tmp_repos_dir) + + def teardown_environment(): if get_host_os() == OsType.Windows: if config.teardown_ovms_processes: @@ -169,6 +258,242 @@ def setup_artifacts_dir(): file.unlink() +def setup_capi_wrapper(package_content): + if OvmsType.CAPI not in config.ovms_types: + return + for _src, _dst in [ + (Paths.OVMS_TEST_CAPI_WRAPPER_PYX, Path(package_content, "include")), + (Paths.OVMS_TEST_CAPI_WRAPPER_MAKEFILE, package_content), + (Paths.OVMS_TEST_CAPI_WRAPPER_SETUP, package_content), + (Paths.OVMS_TEST_CAPI_AUTOPXD_PY, Path(package_content, "include")), + ]: + shutil.copy(_src, _dst) + + proc = Process() + proc.disable_check_stderr() + + # Example: + # >>> sys.executable + # '/usr/local/ovms-test/.venv/bin/python3' + # >>> venv_path + # '/usr/local/ovms-test/.venv' + venv_path = str(Path(*Path(sys.executable).parts[:-2])) + _stdout = proc.run_and_check(f"PYVENV={venv_path} make", cwd=package_content) + + +def download_binary_package(binary_package_src_file_path, binary_package_dst_file_path): + wget_file(binary_package_src_file_path, binary_package_dst_file_path) + + +def get_binary_artifacts( + binary_package_src_file_path, binary_package_dst_path, ovms_binary_name=OVMS_BINARY_PACKAGE_NAME +): + proc = Process() + proc.disable_check_stderr() + print(f"Preparing OVMS package: {binary_package_src_file_path}") + used_extensions = [extension for extension in OVMS_BINARY_PACKAGE_EXTENSIONS + if binary_package_src_file_path.endswith(extension)] + if not used_extensions: + raise NotImplementedError( + f"OVMS binary supported only with .tar.gz or .zip formats. " + f"Current package name: {binary_package_src_file_path}" + ) + ovms_binary_full_name = f"{ovms_binary_name}{used_extensions[0]}" + + if binary_package_src_file_path.startswith("http"): + binary_package_dst_file_path = os.path.join(binary_package_dst_path, ovms_binary_full_name) + download_binary_package(binary_package_src_file_path, binary_package_dst_file_path) + else: + ovms_binary_src_path = os.path.realpath(os.path.expanduser(binary_package_src_file_path)) + if not os.path.exists(binary_package_dst_path): + os.makedirs(binary_package_dst_path) + shutil.copy(ovms_binary_src_path, os.path.join(binary_package_dst_path, ovms_binary_full_name)) + proc.run_and_check(f"tar -xf {ovms_binary_full_name}", cwd=binary_package_dst_path) + setupvars_script_dst = os.path.join(binary_package_dst_path, "ovms", "setupvars.bat") + if not os.path.exists(setupvars_script_dst): + shutil.copy2(config.setupvars_script_path, setupvars_script_dst) + + +def run_docker_build_ovms_image(cmd, ovms_image_name, cwd, timeout=None): + print(f"Building {ovms_image_name} image using cmd: {cmd}") + proc = Process() + proc.disable_check_stderr() + code, stdout, stderr = proc.run_and_check_return_all(cmd, cwd=cwd, timeout=timeout) + assert (f"naming to {ovms_image_name}" in stderr) or ( + f"Successfully tagged {ovms_image_name}" in stdout + ), f"Image was not built successfully; stderr: {stderr}" + print(f"Ovms-test image {ovms_image_name} successfully created") + + +def get_ovms_capi_docker_build_cmd(ovms_image, base_os, dockerfile, ovms_binary_image_name): + base_image = config.base_image if config.base_image else os_type_to_base_image_binary_docker[base_os] + ovms_test_image = calculate_ovms_test_image_name(ovms_image) if build_test_image else base_image + target_device = TargetDevice.GPU if TargetDevice.GPU.lower() in ovms_image else TargetDevice.CPU + cmd = ( + f"docker build -f {dockerfile} -t {ovms_binary_image_name} . " + f"--build-arg BASE_IMAGE={base_image} " + f"--build-arg OVMS_IMAGE={ovms_image} " + f"--build-arg OVMS_TEST_IMAGE={ovms_test_image} " + f"--build-arg OVMS_DEPENDENCIES='{OVMS_CAPI_DEPENDENCIES[base_os]}' " + f"--build-arg TOOLS_DEPENDENCIES='{OVMS_CAPI_TOOLS_DEPENDENCIES[target_device][base_os]}' " + f"--build-arg INSTALL_DRIVER_VERSION='{GPU_INSTALL_DRIVER_VERSION[base_os]}' " + f"--build-arg http_proxy={http_proxy} " + f"--build-arg https_proxy={https_proxy} " + f"--build-arg no_proxy={no_proxy} " + ) + return cmd + + +def get_ovms_binary_docker_build_cmd(ovms_image, base_os, dockerfile, ovms_binary_image_name): + base_image = config.base_image if config.base_image else os_type_to_base_image_binary_docker[base_os] + ovms_test_image = calculate_ovms_test_image_name(ovms_image) if build_test_image else base_image + cpu_extensions_path = Paths.ROOT_PATH_CPU_EXTENSIONS if build_test_image else DOCKER_CONTAINER_TMP_PATH + custom_loader_path = Paths.CUSTOM_LOADER_LIBRARIES_PATH_INTERNAL if build_test_image else DOCKER_CONTAINER_TMP_PATH + custom_nodes_path = Paths.CUSTOM_NODE_LIBRARIES_PATH_INTERNAL if build_test_image else DOCKER_CONTAINER_TMP_PATH + cmd = ( + f"docker build -f {dockerfile} -t {ovms_binary_image_name} . " + f"--build-arg BASE_IMAGE={base_image} " + f"--build-arg OVMS_IMAGE={ovms_image} " + f"--build-arg OVMS_TEST_IMAGE={ovms_test_image} " + f"--build-arg OVMS_DEPENDENCIES='{OVMS_BINARY_DEPENDENCIES[base_os]}' " + f"--build-arg CPU_EXTENSIONS_PATH={cpu_extensions_path} " + f"--build-arg CUSTOM_LOADER_PATH={custom_loader_path} " + f"--build-arg CUSTOM_NODES_PATH={custom_nodes_path} " + f"--build-arg http_proxy={http_proxy} " + f"--build-arg https_proxy={https_proxy} " + f"--build-arg no_proxy={no_proxy} " + ) + return cmd + + +def build_ovms_binary_image(): + ovms_c_artifacts = {_base_os: config.ovms_c_release_artifacts_path[index] for index, _base_os in enumerate(config.base_os)} + + for ovms_image, base_os in get_ovms_calculated_images(): + ovms_binary_dst_path = os.path.join(tmp_dir, "ovms_binary", base_os) + get_binary_artifacts(ovms_c_artifacts[base_os], ovms_binary_dst_path) + + dockerfile = f"Dockerfile.{UBUNTU if UBUNTU in base_os else base_os}" + shutil.copy( + os.path.join(os.path.dirname(__file__), "ovms_binary_image", dockerfile), + ovms_binary_dst_path, + ) + + ovms_binary_image_name = calculate_ovms_binary_image_name(ovms_image) + cmd = get_ovms_binary_docker_build_cmd(ovms_image, base_os, dockerfile, ovms_binary_image_name) + run_docker_build_ovms_image(cmd, ovms_binary_image_name, cwd=ovms_binary_dst_path, timeout=None) + + +def build_ovms_capi_image(): + ovms_c_artifacts = {_base_os: config.ovms_c_release_artifacts_path[index] for index, _base_os in enumerate(config.base_os)} + + for ovms_image, base_os in get_ovms_calculated_images(): + ovms_capi_dst_path = os.path.join(tmp_dir, "ovms_capi", base_os) + get_binary_artifacts(ovms_c_artifacts[base_os], ovms_capi_dst_path) + if TargetDevice.GPU.lower() in ovms_image: + for gpu_install_script in GPU_INSTALL_SCRIPTS[base_os]: + shutil.copy(os.path.join(ovms_c_repo_path, gpu_install_script), ovms_capi_dst_path) + else: + for gpu_install_script in GPU_INSTALL_SCRIPTS[base_os]: + with open(os.path.join(ovms_capi_dst_path, gpu_install_script), "a"): + pass + + dockerfile = f"Dockerfile.{UBUNTU if UBUNTU in base_os else base_os}" + shutil.copy( + os.path.join(os.path.dirname(__file__), "ovms_capi_image", dockerfile), + ovms_capi_dst_path, + ) + + ovms_capi_image_name = calculate_ovms_capi_image_name(ovms_image) + cmd = get_ovms_capi_docker_build_cmd(ovms_image, base_os, dockerfile, ovms_capi_image_name) + run_docker_build_ovms_image(cmd, ovms_capi_image_name, cwd=ovms_capi_dst_path, timeout=None) + + +def prepare_ovms_package(): + if all([ + all([OvmsType.CAPI not in ovms_type for ovms_type in config.ovms_types]), + all([OvmsType.BINARY not in ovms_type for ovms_type in config.ovms_types]), + ]): + return + + if any([OvmsType.CAPI in config.ovms_types, OvmsType.BINARY in config.ovms_types]): + assert ( + len(config.base_os) == 1 and get_host_os() == config.base_os[0] + ), f"Mismatch between config base os: {config.base_os}; host os: {get_host_os()}" + + for base_os in config.base_os: + ovms_binary_dst_path = os.path.join(c_api_wrapper_dir, base_os) + get_binary_artifacts(config.ovms_c_release_artifacts_path[0], ovms_binary_dst_path) + + package_content = Path(Paths.CAPI_WRAPPER_PACKAGE_CONTENT_PATH(base_os)) + setup_capi_wrapper(package_content) + + +def get_models_to_download(): + models_to_download = [] + for various_models_name in [name for name, obj in vars(ModelsLibrary).items() if isinstance(obj, property)]: + various_models_value = getattr(ModelsLib, various_models_name) + if isinstance(various_models_value, dict): + for target_device in target_devices: + models_to_download.extend(various_models_value[target_device]) + else: + models_to_download.extend(various_models_value) + return list(set(models_to_download)) + + +def download_models(): + models_to_download = get_models_to_download() + for model_type in models_to_download: + if model_type.is_local: + ov_hf_downloader = OVHfDownloader(model_type) + ov_hf_downloader.check_and_update_hf_model() + + +def get_docker_images(images_to_download): + images_to_download.add(config.minio_image) + for target_device, base_os in itertools.product(config.target_devices, config.base_os): + ovms_image = calculate_ovms_image_name(target_device, base_os) + if config.ovms_image_local: + OvmsInfo.get_local_image(ovms_image) + else: + images_to_download.add(ovms_image) + return images_to_download + + +def download_docker_images(): + docker_ovms_types = [ + OvmsType.DOCKER, OvmsType.DOCKER_CMD_LINE, OvmsType.BINARY_DOCKER, OvmsType.CAPI_DOCKER + ] + if not any(_ovms_type in docker_ovms_types for _ovms_type in config.ovms_types): + return + + images_to_download = set() + images_to_download = get_docker_images(images_to_download) + + for image in images_to_download: + if image: + OvmsInfo.pull_latest_image(image) + + +def download_resources_master(): + print("Download required resources") + download_models() + download_docker_images() + + +def init_ovms_config_retrieved_from_master(pytest_config): + config.tmp_repos_dir = pytest_config.workerinput[TMP_REPOS_DIR_ARGUMENT] + global CURRENT_TARGET_DEVICE_DICT + CURRENT_TARGET_DEVICE_DICT = pytest_config.workerinput[CURRENT_TARGET_DEVICE_DICT_ARGUMENT] + + +def build_local_resources(): + if OvmsType.BINARY_DOCKER in config.ovms_types: + build_ovms_binary_image() + if OvmsType.CAPI_DOCKER in config.ovms_types: + build_ovms_capi_image() + + def setup_tmp_repos_dir(config): config.tmp_repos_dir = TmpDir() @@ -314,6 +639,35 @@ def parametrize_target_device(metafunc): metafunc.parametrize(TARGET_DEVICE_PARAM_NAME, config.target_devices, ids=ids) +def validate_lock_files(): + """Ensure that target_device locks files exists""" + if not machine_is_reserved_for_test_session: + return # Cannot validate locks validity since other testing session could acquire device lock + + locks = [value for key, value in vars(Paths).items() if "LOCK_FILE" in key] + for target_device in config.target_devices: + n = MAX_WORKERS_PER_TARGET_DEVICE[target_device] + locks += [Paths.get_target_device_lock_file(target_device, i) for i in range(n)] + for lock_path in [Path(x) for x in locks]: + if lock_path.exists(): + logger.warning(f"Hanging lock file discovered:\n{lock_path.name}") + logger.warning(f"Deleting lock file:\n{lock_path.name}") + lock_path.unlink() + + +def list_host_zombie_processes(): + zombie_pids = [] + if OsType.Windows in config.base_os: + return zombie_pids + all_pids = [x.name for x in Path("/proc").iterdir() if str(x.name).isnumeric()] + zombie_pids = [x for x in all_pids if get_pid_status(x) == PID_STATE_ZOMBIE] + if len(zombie_pids) > 0: + logger.warning(f"Found {len(zombie_pids)} zombie processes.") + for zombie in zombie_pids: + logger.warning(f"Zombie:\t{get_pid_name(zombie)}") + return zombie_pids + + def parametrize_ovms_type(metafunc): metafunc.parametrize(OVMS_TYPE_PARAM_NAME, config.ovms_types) @@ -414,3 +768,383 @@ def mute_warnings(): warnings.filterwarnings( action="ignore", message="`np.bool8` is a deprecated alias for `np.bool_`", category=DeprecationWarning ) + + +def setup_nginx(): + if not is_nginx_mtls: + return + print("Setup nginx certificates") + if is_xdist_master(): + OvsaCerts.generate_ovsa_certs(skip_if_valid=not force_generate_new_ssl_certs) + OvsaCerts.init_ovsa_certs() + + +def remove_ports_reservation(_config): + reservation_manager = getattr(_config, "reservation_manager", None) + if reservation_manager is None: + return + logger.info("Removing self reserved ports") + reservation_manager.independent.remove() + + if machine_is_reserved_for_test_session: + # If machine is reserved, no other test session should active + # So clean dangling reservations (if any occurs during previous fatal errors). + # If machine never be reserved exclusively (ie. builder0x) + # you can clean dangling reservation manually by deleting files: + # `/tmp/reservation_manager-*-*-independent` + # (after ensuring no test session is active) + reservation_manager.independent.cleanup() + + +def clear_lockfiles(): + if not Path(ovms_file_locks_dir).exists(): + return + for file in Path(ovms_file_locks_dir).iterdir(): + print(f"Delete hanging lock: {str(file)}") # logger could be unavailable by now + file.unlink() + + +def deselect_items(items, config, deselected): + config.hook.pytest_deselected(items=deselected) + for item in deselected: + test_name = item.parent.nodeid + # nodeid comes in a way: + # 1) test.py::TestClass::() + # 2) test.py:: + if test_name[-2:] == "()": + test_name = test_name[:-2] + elif "::" not in test_name: + test_name += "::" + + test_name += item.name + logger.debug("Deselecting test: " + test_name) + items.remove(item) + + +def set_divide_target_device_per_worker(items): + # Assign xdist_group per target_device so --dist loadgroup routes + # all tests for a given device to the same worker. + # Must be done before yield so xdist sees the markers during scheduling. + if config.divide_target_device_per_worker: + num_devices = len(config.target_devices) + if num_devices and config.xdist_workers > 0 and config.xdist_workers % num_devices != 0: + raise ValueError( + f"xdist_workers ({config.xdist_workers}) must be a multiple of " + f"the number of target devices ({num_devices}): {config.target_devices}" + ) + for item in items: + if hasattr(item, "callspec") and TARGET_DEVICE_PARAM_NAME in item.callspec.params: + td = item.callspec.params[TARGET_DEVICE_PARAM_NAME] + item.add_marker(pytest.mark.xdist_group(name=f"device_{td}")) + logger.debug(f"Assigned {item.nodeid} to xdist_group device_{td}") + + +def preprocess_collected_items(items): + deselected = [] + all_components = {} + all_requirements = {} + try: + required_marker_ids, excluded_marker_ids = get_marker_ids_for_test_run() + for item in items: + set_item_image_parameter(item) + preprocess_collected_item( + item, + deselected, + all_components, + all_requirements, + required_marker_ids, + excluded_marker_ids, + ) + + except RuntimeError as e: + error_msg = str(e) + logger.exception(error_msg) + sys.exit(error_msg) + + return deselected + + +def get_marker_ids_for_test_run(): + # requirements + if req_ids and exclude_req_ids: + raise RuntimeError("Can't both include and exclude requirements!") + # components + if components_ids and exclude_components_ids: + raise RuntimeError("Can't both include and exclude components!") + + required_marker_ids = generate_marker_ids(req_ids, components_ids, tests_priority_list) + excluded_marker_ids = generate_marker_ids(exclude_req_ids, exclude_components_ids) + return required_marker_ids, excluded_marker_ids + + +def generate_marker_ids(*args): + ids_lists = [ids_list for ids_list in args if ids_list] + marker_ids = [] + if len(ids_lists) > 1: + marker_ids = list(itertools.product(*ids_lists)) + elif len(ids_lists) == 1: + marker_ids = [(id_value,) for id_value in ids_lists[0]] + return marker_ids + + +def preprocess_collected_item( + item, deselected, all_components, all_requirements, required_marker_ids, excluded_marker_ids +): + apply_conditional_run_type_marks(item) + test_type = MarkRunType.get_test_type_mark(item) + set_timeout_per_test_type(item, test_type) + update_parent_markers( + item, ( + MarkGeneral.COMPONENTS.mark, + MarkGeneral.REQIDS.mark, + MarkPriority.HIGH.mark, + MarkPriority.MEDIUM.mark, + MarkPriority.LOW.mark, + ) + ) + if deselect(item, test_type, required_marker_ids, excluded_marker_ids): + deselected.append(item) + else: + update_markers(item, test_type, all_components, MarkGeneral.COMPONENTS.mark) + update_markers(item, test_type, all_requirements, MarkGeneral.REQIDS.mark) + return deselected + + +def set_item_image_parameter(item): + if getattr(item, "callspec", None): + # Store calculated image for later use. + ovms_type = item.callspec.params.get(OVMS_TYPE_PARAM_NAME, OvmsType.DOCKER) + base_os = item.callspec.params.get(BASE_OS_PARAM_NAME, OsType.Ubuntu22) + if ovms_type == OvmsType.BINARY or ovms_type == OvmsType.CAPI: + item._image = calculate_ovms_binary_name(base_os=base_os) + else: + target_device = item.callspec.params.get(TARGET_DEVICE_PARAM_NAME, TargetDevice.CPU) + item._image = calculate_ovms_image_name(target_device=target_device, base_os=base_os) + + +def apply_conditional_run_type_marks(item): + """Resolve conditional_run_type and conditional_run_type_by_model meta-markers. + + conditional_run_type: assigns single_mark when device+OS match, default_mark otherwise. + conditional_run_type_by_model: assigns mark based on model_type membership in model collections. + """ + params = getattr(getattr(item, 'callspec', None), 'params', {}) + + for marker in item.iter_markers(MarkConditionalRunType.CONDITIONAL_RUN_TYPE): + single_mark = marker.kwargs["single_mark"] + default_mark = marker.kwargs["default_mark"] + single_if_device = marker.kwargs.get("single_if_device") + single_if_os = marker.kwargs.get("single_if_os") + + device = params.get(TARGET_DEVICE_PARAM_NAME, "") + base_os = str(params.get(BASE_OS_PARAM_NAME, "")).lower() + + is_single = True + if single_if_device and device not in single_if_device: + is_single = False + if single_if_os and base_os not in single_if_os: + is_single = False + + mark_name = single_mark if is_single else default_mark + item.add_marker(getattr(pytest.mark, mark_name)) + return # only first conditional_run_type marker is applied + + for marker in item.iter_markers(MarkConditionalRunType.CONDITIONAL_RUN_TYPE_BY_MODEL): + model_type = params.get(MarkTestParameters.MODEL_TYPE) + if model_type is None: + continue + device = params.get(TARGET_DEVICE_PARAM_NAME, "") + for mark_name, model_collection in marker.kwargs.get("model_mark_map", {}).items(): + device_models = set(model_collection.get(device, [])) + if model_type in device_models: + item.add_marker(getattr(pytest.mark, mark_name)) + return + default_mark = marker.kwargs.get("default_mark") + if default_mark: + item.add_marker(getattr(pytest.mark, default_mark)) + return + + +def set_timeout_per_test_type(item, test_type): + if item.get_closest_marker("timeout") is None: + value = timeout_dict[test_type] + if any([test_type == MarkRunType.TEST_MARK_REGRESSION, + test_type == MarkRunType.TEST_MARK_ON_COMMIT, + ]): + if any(["AUTO" in item.name, "HETERO" in item.name, "MULTI" in item.name]): + value *= TIMEOUT_MULTIPLIER["AUTO_HETERO_MULTI"] + elif TargetDevice.GPU in item.name: + value *= TIMEOUT_MULTIPLIER[TargetDevice.GPU] + elif TargetDevice.NPU in item.name: + value *= TIMEOUT_MULTIPLIER[TargetDevice.NPU] + if run_ovms_with_valgrind or run_ovms_with_opencl_trace: + value *= TIMEOUT_MULTIPLIER["TRACE_TOOLS"] + item.add_marker(pytest.mark.timeout(value)) + + +def update_parent_markers(item, marker_types): + for marker_type in marker_types: + components = item.get_closest_marker(marker_type) + if components is not None: + current_components = next( + (component for component in item.own_markers if component.name == marker_type), + None, + ) + if current_components is None: + item.own_markers.append(components) + + +def deselect(item, test_type, required_marker_ids, excluded_marker_ids): + # Validate different scenarios where test should be deselected from execution during `collect` stage. + if isinstance(item, Function): + if test_type is None: + raise RuntimeError("Test do not have test_type: " + item.name) + + if required_marker_ids: + for required_marker_id_list in required_marker_ids: + if _is_test_marker_id_is_matched_with_id(item, required_marker_id_list): + # make sure that item is not deselected by other marker + return deselect_by_excluded_marker_ids(item, excluded_marker_ids) + return True + elif excluded_marker_ids: + return deselect_by_excluded_marker_ids(item, excluded_marker_ids) + + return False + + +def deselect_by_excluded_marker_ids(item, excluded_marker_ids): + for excluded_marker_ids_list in excluded_marker_ids: + if _is_test_marker_id_is_matched_with_id(item, excluded_marker_ids_list): + return True + return False + + +def _is_test_marker_id_is_matched_with_id(test, ids_to_check: list): + markers_to_check = [] + + for marker in test.own_markers: + if any([ + marker.name is MarkGeneral.REQIDS.value, + marker.name is MarkGeneral.COMPONENTS.value, + marker.name is MarkPriority.HIGH.mark, + marker.name is MarkPriority.MEDIUM.mark, + marker.name is MarkPriority.LOW.mark, + ]): + if marker.args: + for marker_arg in marker.args: + if isinstance(marker_arg, dict): + for param in marker_arg: + if param is None: + markers_to_check.append(str(marker_arg.values)) + elif param in test.name: + markers_to_check.append(str(marker_arg.values())) + elif isinstance(marker_arg, str): + markers_to_check.append(marker_arg) + else: + raise RuntimeError( + f"Test {test.name} do not have mark in correct form. Form: {type(marker_arg)}" + ) + else: + markers_to_check.append(marker.name) + + check_list = [] + for id_to_check in ids_to_check: + check_list.append(any(id_to_check.lower() in marker_to_check.lower() for marker_to_check in markers_to_check)) + + return all(check_list) + + +def update_markers(item, test_type, markers, marker_type): + marker = item.get_closest_marker(marker_type) + if marker is not None: + if test_type not in markers: + markers[test_type] = set() + markers[test_type].update(set(marker.args)) + + +def get_skipped_items(items): + skipped_items = [item for item in items if item.keywords.get("skip") is not None] + items = [] + for item in skipped_items: + skip_info = item.keywords.get("skip") + if isinstance(skip_info, (Mark, MarkDecorator)): + if "reason" in skip_info.kwargs: + reason = skip_info.kwargs["reason"] + elif skip_info.args: + reason = skip_info.args[0] + else: + reason = "" + items.append(SkippedItem(item.nodeid, reason)) + return items + + +def calc_statistics(items): + skipped_items = get_skipped_items(items) + issue_numbers = [] + other_tests = [] + for item in skipped_items: + match = re.search(r"DPNG-\d+", item.reason) + if match: + issue_numbers.append(match.group(0)) + else: + issue_numbers.append("others") + other_tests.append(item) + return Counter(issue_numbers), other_tests + + +def log_labeled_stats(issues): + msg = ["Skipped tests statistic:"] + issues_sorted_by_quantity = sorted(issues.items(), key=lambda i: i[1], reverse=True) + for issue, quantity in issues_sorted_by_quantity: + msg.append("{:>11}: {:>6}".format(issue, quantity)) + logger.info("\n".join(msg)) + + +def log_others(other_items): + msg = ["Skipped tests not labeled with issue:"] + items_grouped_by_reason = groupby(other_items, key=lambda i: i.reason) + for reason, items in list(items_grouped_by_reason): + msg.append("{}:".format(reason)) + msg.extend("|---{}".format(item.test_name) for item in list(items)) + logger.info("\n".join(msg)) + + +def log_skip_statistic(items): + issue_stats, other_tests = calc_statistics(items) + log_labeled_stats(issue_stats) + log_others(other_tests) + + +def get_session_start_info(session): + logger.info(f"Starting test session in the following folder: {session.startdir}") + log_configuration_variables() + session.start_time = time.time() + + +def parametrize_tests(metafunc): + if OVMS_TYPE_PARAM_NAME in metafunc.fixturenames: + parametrize_ovms_type(metafunc) + + if USES_MAPPING_PARAM_NAME in metafunc.fixturenames: + parametrize_uses_mapping(metafunc) + + if BASE_OS_PARAM_NAME in metafunc.fixturenames: + parametrize_base_os(metafunc) + + if MarkTestParameters.MODEL_TYPE in metafunc.fixturenames: + parametrize_model_type(metafunc) + elif MarkTestParameters.ALL_MODELS in metafunc.fixturenames: + parametrize_all_models(metafunc) + elif MarkTestParameters.MANY_MODELS in metafunc.fixturenames: + parametrize_many_models(metafunc) + elif MarkTestParameters.ITERATION_INFO in metafunc.fixturenames: + parametrize_iteration_info(metafunc) + elif MarkTestParameters.INPUT_SHAPE in metafunc.fixturenames: + parametrize_input_shape(metafunc) + elif MarkTestParameters.PLUGIN_CONFIG in metafunc.fixturenames: + parametrize_plugin_config(metafunc) + elif TARGET_DEVICE_PARAM_NAME in metafunc.fixturenames: + parametrize_target_device(metafunc) + + if MarkTestParameters.MODEL_AUX_TYPE in metafunc.fixturenames: + parametrize_model_aux_type(metafunc) diff --git a/tests/functional/utils/ov_hf_downloader.py b/tests/functional/utils/ov_hf_downloader.py new file mode 100644 index 0000000000..ab61475b38 --- /dev/null +++ b/tests/functional/utils/ov_hf_downloader.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from datetime import datetime, timezone +from huggingface_hub import HfApi, snapshot_download + +from tests.functional.config import huggingface_token +from tests.functional.utils.assertions import OVHfDownloadException +from tests.functional.utils.logger import get_logger +from tests.functional.utils.test_framework import get_dir_latest_mtime, remove_dir_tree, swap_directory + +logger = get_logger(__name__) + + +class OVHfDownloader: + + def __init__(self, model_type, model_base_path=None): + if not huggingface_token: + raise OVHfDownloadException( + "Provide huggingface_token with TT_HUGGINGFACE_TOKEN or TT_HUGGINGFACE_TOKEN_FILE_PATH envs" + ) + self.api = HfApi(token=huggingface_token) + self.model = model_type() + self.model_name = self.model.name + if model_base_path is None: + self.model_local_path = self.model.model_path_on_host + else: + self.model_local_path = os.path.join(model_base_path, self.model.name) + + def check_and_update_hf_model(self): + repo_info = self.api.repo_info(self.model_name) + local_latest_mtime = get_dir_latest_mtime(self.model_local_path) + local_latest_dt = datetime.fromtimestamp(local_latest_mtime, tz=timezone.utc) if local_latest_mtime else None + + if local_latest_dt and repo_info.last_modified <= local_latest_dt: + print(f"No files to update for model: {self.model_name}") + return False + + print(f"Download OVHf model: {self.model_name}") + staging_path = self.model_local_path + "_staging" + if os.path.exists(staging_path): + remove_dir_tree(staging_path) + self.download_model(model_dir=staging_path) + swap_directory(self.model_local_path, staging_path) + return True + + def download_model(self, model_name=None, model_dir=None, force_download=False): + snapshot_download( + repo_id=self.model_name if model_name is None else model_name, + local_dir=self.model_local_path if model_dir is None else model_dir, + token=huggingface_token, + force_download=force_download, + ) diff --git a/tests/functional/utils/ovms_binary_image/Dockerfile.redhat b/tests/functional/utils/ovms_binary_image/Dockerfile.redhat new file mode 100644 index 0000000000..31e2da11d3 --- /dev/null +++ b/tests/functional/utils/ovms_binary_image/Dockerfile.redhat @@ -0,0 +1,47 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:9.7 +ARG OVMS_TEST_IMAGE=openvino-model-server:latest-test +ARG OVMS_IMAGE=registry.connect.redhat.com/intel/openvino-model-server:latest + +FROM $OVMS_IMAGE as ovms_image +FROM $OVMS_TEST_IMAGE as ovms_test_image +FROM $BASE_IMAGE as base_image + +# libtbb is now available in /ovms/lib +#ARG OVMS_DEPENDENCIES="https://vault.centos.org/centos/8/AppStream/x86_64/os/Packages/tbb-2018.2-9.el8.x86_64.rpm" +#RUN dnf install -y pkg-config && rpm -ivh ${OVMS_DEPENDENCIES} + +WORKDIR / +COPY ovms/ ovms/ + +ENV LD_LIBRARY_PATH=/ovms/lib + +RUN /ovms/bin/ovms --version + +# Copy all dependent libraries for enabling GPU based images (be cautious) +COPY --from=ovms_image /etc /etc +COPY --from=ovms_image /usr/lib64/ /usr/lib64/ +COPY --from=ovms_image /usr/local/ /usr/local/ + +ARG CPU_EXTENSIONS_PATH=/cpu_extensions +ARG CUSTOM_NODES_PATH=/custom_nodes + +COPY --from=ovms_test_image ${CPU_EXTENSIONS_PATH} ${CPU_EXTENSIONS_PATH} +COPY --from=ovms_test_image ${CUSTOM_NODES_PATH} ${CUSTOM_NODES_PATH} + +ENTRYPOINT ["/ovms/bin/ovms"] diff --git a/tests/functional/utils/ovms_binary_image/Dockerfile.ubuntu b/tests/functional/utils/ovms_binary_image/Dockerfile.ubuntu new file mode 100644 index 0000000000..bdc6bab436 --- /dev/null +++ b/tests/functional/utils/ovms_binary_image/Dockerfile.ubuntu @@ -0,0 +1,46 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BASE_IMAGE=ubuntu:24.04 +ARG OVMS_TEST_IMAGE=openvino/model_server:latest-test +ARG OVMS_IMAGE=openvino/model_server:latest + +FROM $OVMS_IMAGE as ovms_image +FROM $OVMS_TEST_IMAGE as ovms_test_image +FROM $BASE_IMAGE as base_image + +ARG OVMS_DEPENDENCIES="libcurl4-openssl-dev libpugixml1v5 libtbb12 libxml2" +RUN apt-get update && apt-get install -y --no-install-recommends ${OVMS_DEPENDENCIES} && rm -rf /var/lib/apt/lists/* + +WORKDIR / +COPY ovms/ ovms/ + +ENV LD_LIBRARY_PATH=/ovms/lib + +RUN /ovms/bin/ovms --version + +# Copy all dependent libraries for enabling GPU based images (be cautious) +COPY --from=ovms_image /etc /etc +COPY --from=ovms_image /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu +COPY --from=ovms_image /usr/local/ /usr/local/ + +ARG CPU_EXTENSIONS_PATH=/cpu_extensions +ARG CUSTOM_NODES_PATH=/custom_nodes + +COPY --from=ovms_test_image ${CPU_EXTENSIONS_PATH} ${CPU_EXTENSIONS_PATH} +COPY --from=ovms_test_image ${CUSTOM_NODES_PATH} ${CUSTOM_NODES_PATH} + +ENTRYPOINT ["/ovms/bin/ovms"] diff --git a/tests/functional/utils/ovms_capi_image/Dockerfile.redhat b/tests/functional/utils/ovms_capi_image/Dockerfile.redhat new file mode 100644 index 0000000000..393121890f --- /dev/null +++ b/tests/functional/utils/ovms_capi_image/Dockerfile.redhat @@ -0,0 +1,47 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BASE_IMAGE=registry.access.redhat.com/ubi9/ubi:9.7 +ARG OVMS_TEST_IMAGE=openvino-model-server:latest-test +ARG OVMS_IMAGE=registry.connect.redhat.com/intel/openvino-model-server:latest + +FROM $OVMS_IMAGE as ovms_image +FROM $OVMS_TEST_IMAGE as ovms_test_image +FROM $BASE_IMAGE as base_image + +# libtbb is now available in /ovms/lib +#ARG OVMS_DEPENDENCIES="https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/tbb-2020.3-8.el9.x86_64.rpm" +#RUN dnf install -y pkg-config && rpm -ivh ${OVMS_DEPENDENCIES} + +WORKDIR / +COPY ovms/ ovms/ + +ENV LD_LIBRARY_PATH=/ovms/lib + +RUN /ovms/bin/ovms --version +RUN dnf install -y https://rpmfind.net/linux/centos-stream/9-stream/BaseOS/x86_64/os/Packages/libnl3-3.11.0-1.el9.x86_64.rpm && dnf clean all +RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && dnf clean all && \ + yum update -d6 -y && yum install -d6 -y gcc-c++ +RUN dnf install -y pkg-config && rpm -ivh https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/opencl-headers-3.0-6.20201007gitd65bcc5.el9.noarch.rpm && dnf clean all + +# Enable GPU +ARG INSTALL_DRIVER_VERSION="24.52.32224" +ARG DNF_TOOL="dnf" +COPY install_redhat_gpu_drivers.sh /tmp/install_redhat_gpu_drivers.sh +RUN chmod 775 /tmp/install_redhat_gpu_drivers.sh && /tmp/install_redhat_gpu_drivers.sh + +# use for debug with GPU target device +#RUN yum update -d6 -y && yum install -d6 -y clinfo diff --git a/tests/functional/utils/ovms_capi_image/Dockerfile.ubuntu b/tests/functional/utils/ovms_capi_image/Dockerfile.ubuntu new file mode 100644 index 0000000000..35a2007078 --- /dev/null +++ b/tests/functional/utils/ovms_capi_image/Dockerfile.ubuntu @@ -0,0 +1,44 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +ARG BASE_IMAGE=ubuntu:24.04 +ARG OVMS_TEST_IMAGE=openvino/model_server:latest-test +ARG OVMS_IMAGE=openvino/model_server:latest + +FROM $OVMS_IMAGE as ovms_image +FROM $OVMS_TEST_IMAGE as ovms_test_image +FROM $BASE_IMAGE as base_image + +ARG OVMS_DEPENDENCIES="libcurl4-openssl-dev libpugixml1v5 libtbb12 libxml2" +RUN apt-get update && apt-get install -y --no-install-recommends ${OVMS_DEPENDENCIES} && rm -rf /var/lib/apt/lists/* + +WORKDIR / +COPY ovms/ ovms/ + +ENV LD_LIBRARY_PATH=/ovms/lib + +RUN /ovms/bin/ovms --version + +ARG TOOLS_DEPENDENCIES="build-essential" +RUN apt-get update && apt-get install -y --no-install-recommends ${TOOLS_DEPENDENCIES} && rm -rf /var/lib/apt/lists/* + +# Enable GPU +ARG INSTALL_DRIVER_VERSION="25.35.35096" +COPY install_ubuntu_gpu_drivers.sh /tmp/install_ubuntu_gpu_drivers.sh +RUN chmod 775 /tmp/install_ubuntu_gpu_drivers.sh && /tmp/install_ubuntu_gpu_drivers.sh + +#COPY install_va.sh /tmp/install_va.sh +#RUN chmod 775 /tmp/install_va.sh && /tmp/install_va.sh diff --git a/tests/functional/utils/ovms_testing_image/Dockerfile.redhat b/tests/functional/utils/ovms_testing_image/Dockerfile.redhat index 6d89e5483c..c38e61f277 100644 --- a/tests/functional/utils/ovms_testing_image/Dockerfile.redhat +++ b/tests/functional/utils/ovms_testing_image/Dockerfile.redhat @@ -14,7 +14,7 @@ # limitations under the License. # -ARG BASE_IMAGE=openvino-model-server:latest +ARG BASE_IMAGE=registry.connect.redhat.com/intel/openvino-model-server:latest FROM $BASE_IMAGE as base_image ARG ROOT_PATH_CPU_EXTENSIONS=/cpu_extensions diff --git a/tests/functional/utils/test_framework.py b/tests/functional/utils/test_framework.py index 26efff66f7..66622bc1cb 100644 --- a/tests/functional/utils/test_framework.py +++ b/tests/functional/utils/test_framework.py @@ -14,6 +14,9 @@ # limitations under the License. # +# pylint: disable=deprecated-argument +# pylint: disable=too-many-positional-arguments + import os import re import shutil diff --git a/tests/requirements.txt b/tests/requirements.txt index c4424528ce..84f9cc2e63 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,15 +1,31 @@ -boto3==1.33.13 -checksec-py==0.7.4 +# supported python version: 3.12 +cohere==7.0.4 +dataclasses-json==0.6.7 +distro==1.9.0 docker==7.1.0 -grpcio==1.60.0 +filelock==3.29.4 +GitPython==3.1.50 +grpcio==1.67.1 +Jinja2==3.1.6 +jiwer>=4.0.0 +openai==2.43.0 +opencv-python==4.13.0.92 paramiko==5.0.0 -psutil==5.9.6 -pytest==9.0.3 -pytest-json==0.4.0 -tensorflow-serving-api>=2.16.1 -tensorflow>=2.16.1 -requests==2.33.0 +Pillow==12.2.0 +protobuf==6.33.6 +psutil==7.2.2 +pylint==4.0.6 +pytest==9.1.0 +pytest-timeout==2.4.0 +pytest-xdist==3.8.0 +python-dateutil==2.8.2 +pyyaml==6.0.3 +requests==2.34.2 +requests-toolbelt==1.0.0 retry==0.9.2 -protobuf<=5.29.6 -jsonschema<=4.23.0 -openai<=1.84.0 +setuptools==81.0.0 +soundfile==0.14.0 +tensorboard==2.20.0 +tensorflow==2.21.0 +tensorflow-serving-api==2.20.0 +tritonclient[all]==2.69.0