CVS-169692_enable_on_commit_tests_part_II

ci/build_test_OnCommit.groovy

@@ -23,13 +23,15 @@ def validation_branch = "develop"
 // Override agent node:
 //   [test_agent_linux=<node>]          - Run Linux doc tests on <node> (default: ovms_ptl)
 //   [test_agent_windows=<node>]        - Run Windows doc tests on <node> (default: ovms_win_ptl)
+//   example: [test_agent_windows=ovms_win_ptl]
 //
 // Override file list (space-separated, converted to pytest -k filter joined with ' or '):
 //   [test_doc_files_linux=<files>]      - Use <files> instead of auto-detected list (Linux)
 //   [test_doc_files_windows=<files>]    - Use <files> instead of auto-detected list (Windows)
+//   example: [test_doc_files_linux=demos/continuous_batching/README.md demos/audio/README.md]
 //
 // Override validation branch:
- //   [validation_branch=<branch>]       - Use <branch> instead of default 'develop' for test repo checkout
+//   [validation_branch=<branch>]       - Use <branch> instead of default 'develop' for test repo checkout
 //
 
 pipeline {
@@ -341,7 +343,7 @@ pipeline {
                       def test_doc_files_str = test_doc_files_linux.split('\n').join(' or ')
                       sh "make create-venv && rm -f tests/functional && ln -s ${pwd}/../tests/functional tests/functional"
                       def cmd_venv_activate = ". .venv/bin/activate"
-                      def cmd_export = "export TT_OVMS_C_REPO_PATH=../ && export TT_RUN_REGRESSION_TESTS=True && export TT_REGRESSION_WEEKLY_TESTS=True && export TT_TARGET_DEVICE=CPU,GPU,NPU && export TT_ENABLE_UAT_TESTS=True && export TT_ENABLE_SMOKE_TESTS=False && export TT_OVMS_C_REPO_PATH=${ovms_c_repo_path} && export TT_WAIT_FOR_MESSAGES_TIMEOUT=1500"
+                      def cmd_export = "export TT_OVMS_C_REPO_PATH=../ && export TT_RUN_REGRESSION_TESTS=True && export TT_REGRESSION_WEEKLY_TESTS=True && export TT_TARGET_DEVICE=CPU,GPU,NPU && export TT_ENABLE_UAT_TESTS=True && export TT_ENABLE_SMOKE_TESTS=False && export TT_OVMS_C_REPO_PATH=${ovms_c_repo_path} && export TT_LOGGING_LEVEL_OVMS=DEBUG && export TT_WAIT_FOR_MESSAGES_TIMEOUT=1500"
                       def cmd_pytest = "pytest tests/non_functional/documentation -k '${test_doc_files_str}' -n 0 --dist loadgroup"
                       def cmd = ""
                       if ( image_build_needed == "true" ) {
@@ -406,7 +408,7 @@ pipeline {
                       def ovms_c_repo_path = bat(returnStdout: true, script: 'cd .. && cd').trim().split('\n').last().trim()
                       def cmd_link_ovms = "(if exist ${current_path}\\tests\\functional rmdir ${current_path}\\tests\\functional) && mklink /D ${current_path}\\tests\\functional ${ovms_c_repo_path}\\tests\\functional"
                       def cmd_requirements = "(if not exist .venv virtualenv .venv --python=python3.12) && call .venv\\Scripts\\activate.bat && pip install -r requirements.txt"
-                      def cmd_export = "set \"TT_OVMS_C_REPO_PATH=../\" && set \"TT_RUN_REGRESSION_TESTS=True\" && set \"TT_REGRESSION_WEEKLY_TESTS=True\" && set \"TT_TARGET_DEVICE=CPU,GPU,NPU\" && set \"TT_BASE_OS=windows\" && set \"TT_OVMS_TYPE=BINARY\" && set \"TT_ENABLE_UAT_TESTS=True\" && set \"TT_ENABLE_SMOKE_TESTS=False\" && set \"TT_DISABLE_DMESG_LOG_MONITOR=True\" && set \"TT_OVMS_C_REPO_PATH=${ovms_c_repo_path}\" && set \"TT_WAIT_FOR_MESSAGES_TIMEOUT=1500\" && set \"PYTHONUTF8=1\" && set \"PYTHONIOENCODING=utf-8\""
+                      def cmd_export = "set \"TT_OVMS_C_REPO_PATH=../\" && set \"TT_LOGGING_LEVEL_OVMS=DEBUG\" && set \"TT_RUN_REGRESSION_TESTS=True\" && set \"TT_REGRESSION_WEEKLY_TESTS=True\" && set \"TT_TARGET_DEVICE=CPU,GPU,NPU\" && set \"TT_BASE_OS=windows\" && set \"TT_OVMS_TYPE=BINARY\" && set \"TT_ENABLE_UAT_TESTS=True\" && set \"TT_ENABLE_SMOKE_TESTS=False\" && set \"TT_DISABLE_DMESG_LOG_MONITOR=True\" && set \"TT_OVMS_C_REPO_PATH=${ovms_c_repo_path}\" && set \"TT_WAIT_FOR_MESSAGES_TIMEOUT=1500\" && set \"PYTHONUTF8=1\" && set \"PYTHONIOENCODING=utf-8\""
                       def cmd_pytest = "pytest tests/non_functional/documentation -k \"${test_doc_files_str}\" -n 0 --dist loadgroup --basetemp=\"C:\\tmp\\pytest-${BRANCH_NAME}-${BUILD_NUMBER}\""
                       def cmd = ""
                       if ( win_image_build_needed == "true" ) {

tests/functional/config.py

@@ -20,8 +20,7 @@
 
 from tests.functional.constants.os_type import OsType
 from tests.functional.constants.ovms_type import OvmsType
-from tests.functional.constants.target_device import TargetDevice
-from tests.functional.utils.core import TmpDir
+from tests.functional.utils.core import TmpDir, get_token_value
 from tests.functional.utils.helpers import (
     generate_test_object_name,
     get_bool,
@@ -65,13 +64,14 @@ def get_uses_mapping():
 """TEST_DIR_CACHE -  location where models and test data should be downloaded to and serve as cache for TEST_DIR"""
 test_dir_cache = os.environ.get("TEST_DIR_CACHE", "/tmp/ovms_models_cache")
 
-"""TEST_DIR_CLEANUP - if set to True, TEST_DIR directory will be removed after tests execution"""
-test_dir_cleanup = os.environ.get("TEST_DIR_CLEANUP", "True")
-test_dir_cleanup = test_dir_cleanup.lower() == "true"
-
 """ TT_OVMS_C_REPO_PATH - path to ovms-c repository. Can be relative or absolute. """
 ovms_c_repo_path = get_path("TT_OVMS_C_REPO_PATH", get_path("PWD", "./"))
 
+""" TT_SETUPVARS_SCRIPT_PATH - path to setupvars.bat script """
+setupvars_script_path = os.environ.get(
+    "TT_SETUPVARS_SCRIPT_PATH", os.path.join(ovms_c_repo_path, "setupvars.bat")
+)
+
 """BUILD_LOGS -  path to dir where artifacts should be stored"""
 artifacts_dir = get_path("BUILD_LOGS", os.path.join(ovms_c_repo_path, "tests", "functional", "test_log_build"))
 
@@ -84,6 +84,11 @@ def get_uses_mapping():
 """ TT_DATASETS_PATH - Datasets local repo path"""
 datasets_path = get_path("TT_DATASETS_PATH", os.path.join("~", "ovms_datasets"))
 
+""" TT_GENERATIVE_MODELS_LOCAL_PATH - local path for converted generative models """
+generative_models_local_path = get_path(
+    "TT_GENERATIVE_MODELS_LOCAL_PATH", os.path.join(models_path, "generative_models")
+)
+
 """ TT_CLEAN_ARTIFACTS_DIR """
 clean_artifacts_dir = get_bool("TT_CLEAN_ARTIFACTS_DIR", False)
 
@@ -141,23 +146,14 @@ def get_uses_mapping():
 
 path_to_mount_cache = os.path.join(test_dir_cache, "saved_models")
 
-"""TT_MINIO_IMAGE_NAME - Docker image for Minio"""
-minio_image = os.environ.get("TT_MINIO_IMAGE_NAME", "minio/minio:latest")
+""" TT_MINIO_IMAGE_NAME - Docker image for Minio"""
+minio_image = os.environ.get(
+    "TT_MINIO_IMAGE_NAME",
+    f"{docker_registry}/minio/minio:latest" if docker_registry is not None else "minio/minio:latest",
+)
 
 """ TT_TARGET_DEVICE - list of devices separated by a comma "CPU,GPU,NPU" """
 target_devices = get_target_devices()
-target_device = target_devices[0]
-
-"""IMAGE - docker image name which should be used to run tests"""
-if target_device == TargetDevice.GPU:
-    _default_image = "openvino/model_server-gpu"
-else:
-    _default_image = "openvino/model_server"
-image = os.environ.get("IMAGE", _default_image)
-
-start_minio_container_command = 'server --address ":{}" /data'
-
-container_minio_log_line = "Console endpoint is listening on a dynamic port"
 
 # Reservation manager values, for details study tests.functional.utils.reservation_manager
 """ TT_GRPC_OVMS_STARTING_PORT - Grpc port where ovms should be exposed"""
@@ -170,32 +166,12 @@ def get_uses_mapping():
 ports_pool_size = get_int("TT_PORTS_POOL_SIZE", None)
 # NOTE: Above values will be validated and could be changed if invalid
 
-""" TT_CONVERTED_MODELS_EXPIRE_TIME - Time after converted models are not up-to-date and needs to be refreshed(s) """
-converted_models_expire_time = get_int("TT_CONVERTED_MODELS_EXPIRE_TIME", 7*24*3600)  # Set default to one week
-
-""" TT_DEFAULT_INFER_TIMEOUT - Timeout for CPU target device"""
-default_infer_timeout = get_int("TT_DEFAULT_INFER_TIMEOUT", 10)
-
-""" TT_DEFAULT_GPU_INFER_TIMEOUT - Timeout for GPU target device"""
-default_gpu_infer_timeout = get_int("TT_DEFAULT_GPU_INFER_TIMEOUT", 10*default_infer_timeout)
-
-""" TT_DEFAULT_NPU_INFER_TIMEOUT - Timeout for NPU target device"""
-default_npu_infer_timeout = get_int("TT_DEFAULT_NPU_INFER_TIMEOUT", 10*default_infer_timeout)
-
-""" INFER TIMEOUT """
-infer_timeouts = {
-    TargetDevice.CPU: default_infer_timeout,
-    TargetDevice.GPU: default_gpu_infer_timeout,
-    TargetDevice.NPU: default_npu_infer_timeout,
-    TargetDevice.AUTO: default_gpu_infer_timeout,
-    TargetDevice.HETERO: default_gpu_infer_timeout,
-    TargetDevice.AUTO_CPU_GPU: default_gpu_infer_timeout,
-}
-infer_timeout = infer_timeouts[target_device]
-
 """ TT_IS_NGINX_MTLS - Specify if given image is OVSA nginx mtls image. """
 is_nginx_mtls = get_bool("TT_IS_NGINX_MTLS", False)
 
+""" TT_FORCE_GENERATE_NEW_SSL_CERTIFICATES """
+force_generate_new_ssl_certs = get_bool("TT_FORCE_GENERATE_NEW_SSL_CERTIFICATES", True)
+
 """ TT_SKIP_TEST_IF_IS_NGINX_MTLS """
 skip_nginx_test = get_bool("TT_SKIP_TEST_IF_IS_NGINX_MTLS", True)
 skip_nginx_test = skip_nginx_test and is_nginx_mtls
@@ -317,6 +293,25 @@ def get_uses_mapping():
 """ TT_OVMS_IMAGE_LOCAL - ovms image can only be found locally """
 ovms_image_local = get_bool("TT_OVMS_IMAGE_LOCAL", False)
 
+""" TT_REQUIREMENTS - Requirements """
+req_ids = get_list("TT_REQUIREMENTS")
+
+""" TT_EXCLUDE_REQUIREMENTS - Requirements to exclude """
+exclude_req_ids = get_list("TT_EXCLUDE_REQUIREMENTS")
+
+""" TT_COMPONENTS - Components """
+components_ids = get_list("TT_COMPONENTS")
+
+""" TT_EXCLUDE_COMPONENTS - Components to exclude """
+exclude_components_ids = get_list("TT_EXCLUDE_COMPONENTS")
+
+""" TT_TESTS_PRIORITY_LIST - tests priority to run - high, medium or low """
+tests_priority_list_raw = get_list("TT_TESTS_PRIORITY_LIST", fallback=["high", "medium", "low"])
+tests_priority_list = [f"priority_{p}" for p in tests_priority_list_raw if "priority" not in p]
+
+""" TT_PERFORMANCE_TEST_TIMEOUT_MINUTES - timeout (in minutes) for each performance test """
+performance_test_timeout_minutes = get_int("TT_PERFORMANCE_TEST_TIMEOUT_MINUTES", 10)
+
 """ TT_BASE_OS - os type used for calculating ovms_image name (if not given explicitly). 
         Possible options (case insensitive): 
         ubuntu22 - use default Ubuntu 22.04 image
@@ -328,6 +323,11 @@ def get_uses_mapping():
 __base_os = os.environ.get("BASE_OS", OsType.Ubuntu24)
 base_os = get_list("TT_BASE_OS", fallback=[__base_os])
 
+""" BASE_IMAGE - Docker image used during OVMS image creation """
+base_image = os.environ.get("BASE_IMAGE", None)
+if base_image is not None:
+    assert len(base_os) == 1, "If you wish to iterate by TT_BASE_OS: do not set BASE_IMAGE explicitly."
+
 """ GLOBAL_TEMP_DIR - global temporary directory """
 global_tmp_dir_default = os.path.join("~", "AppData", "Local", "Temp") if OsType.Windows in base_os else "/tmp"
 global_tmp_dir = get_path("GLOBAL_TEMP_DIR", global_tmp_dir_default)
@@ -416,3 +416,17 @@ def get_ovms_types():
 """ TT_OVMS_TYPE - ovms type runtime to be executed:
 DOCKER, BINARY, BINARY_DOCKER, CAPI, CAPI_DOCKER, DOCKER_CMD_LINE """
 ovms_types = get_ovms_types()
+
+""" TT_DIVIDE_TARGET_DEVICE_PER_WORKER - spread tests across pytest workers based on target device """
+divide_target_device_per_worker = get_bool("TT_DIVIDE_TARGET_DEVICE_PER_WORKER", False)
+
+""" TT_PYTEST_KEYWORD_FILTER """
+pytest_keyword_filter = os.environ.get("TT_PYTEST_KEYWORD_FILTER", None)
+
+""" TT_HUGGINGFACE_TOKEN_FILE_PATH - path to file containing huggingface token """
+huggingface_token_file_path = get_path(
+    "TT_HUGGINGFACE_TOKEN_FILE_PATH", os.path.join("~", "ovms_tokens", "huggingface_token")
+)
+
+""" TT_HUGGINGFACE_TOKEN - huggingface token value. Env var takes priority, then file. """
+huggingface_token = os.environ.get("TT_HUGGINGFACE_TOKEN") or get_token_value(huggingface_token_file_path, "")

tests/functional/conftest.py

@@ -0,0 +1,180 @@
+#
+# Copyright (c) 2018-2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import random
+import sys
+
+from tests.functional.config import enable_pytest_plugins, pytest_keyword_filter, machine_is_reserved_for_test_session
+from tests.functional.constants.ovms import (
+    CURRENT_TARGET_DEVICE_DICT_ARGUMENT,
+    TMP_REPOS_DIR_ARGUMENT,
+)
+from tests.functional.utils import hooks
+from tests.functional.utils.logger import OvmsFileHandler, get_logger
+from tests.functional.utils.marks import MarksRegistry
+from tests.functional.utils.test_framework import is_xdist_master
+
+logger = get_logger(__name__)
+
+
+if enable_pytest_plugins:
+
+    raise NotImplementedError("OVMS tests not enabled")
+
+    pytest_plugins = [      # pylint: disable=unreachable
+        "tests.functional.fixtures.ovms",
+        "tests.functional.fixtures.server",
+        "tests.functional.fixtures.api_type",
+        "tests.functional.fixtures.params",
+    ]
+
+
+    def pytest_configure(config):
+        """
+        Allow plugins and conftest files to perform initial configuration.
+        This hook is called for every plugin and initial conftest file after command line options have been parsed.
+        After that, the hook is called for other conftest files as they are imported.
+
+        NOTE:
+            This hook is called multiple times:
+            1) for master process prior spawning workers
+                (PYTEST_XDIST_WORKER_COUNT and PYTEST_XDIST_WORKER env variable unset)
+            2) for each spawned worker process
+
+        LIMITATIONS:
+            Internal pytest logging mechanisms are initialized in `pytest_sessionstart` hook.
+            Please avoid usage of logger in all hooks used in this function.
+            Please simple print(...) call for printing messages.
+        """
+        hooks.mute_warnings()
+        MarksRegistry.register(config)
+
+        if is_xdist_master():
+            hooks.setup_tmp_repos_dir(config)
+            hooks.validate_port_pool(config)
+            # master thread pytest_configure call. No xdist worker process spawned yet.
+            hooks.init_environment(config)
+            hooks.clear_ovms_capi_artifacts()
+            hooks.setup_artifacts_dir()
+            hooks.prepare_ovms_package()
+            hooks.download_resources_master()
+            hooks.build_local_resources()
+            hooks.validate_lock_files()
+            hooks.list_host_zombie_processes()
+        else:  # Xdist worker thread
+            hooks.download_docker_images()
+            hooks.init_ovms_config_retrieved_from_master(config)
+
+        hooks.setup_nginx()
+
+        # Let know that pytest was successfully configured
+        config.configured = True
+
+
+    def pytest_unconfigure(config):
+        if getattr(config, "configured", None) is not True:
+            # Check if pytest_configure() was done successfully, if not: logger would be in invalid state so disable.
+            for _logger in logger.manager.loggerDict.values():
+                _logger.disabled = True
+
+        try:
+            if is_xdist_master():
+                hooks.remove_ports_reservation(config)
+                hooks.cleanup_tmp_repos_dir(config)
+                hooks.teardown_environment()
+                if machine_is_reserved_for_test_session:
+                    hooks.clear_lockfiles()
+        except Exception as e:      # pylint: disable=broad-exception-caught
+            error_msg = str(e)
+            print(error_msg)
+            sys.exit(error_msg)
+
+
+    def pytest_configure_node(node):
+        node.workerinput[TMP_REPOS_DIR_ARGUMENT] = node.config.tmp_repos_dir
+        node.workerinput[CURRENT_TARGET_DEVICE_DICT_ARGUMENT] = node.config.current_target_device_dict
+
+
+    MarksRegistry.MARK_ENUMS.extend([OvmsComponents])
+
+
+    def pytest_sessionstart(session):
+        hooks.get_session_start_info(session)
+
+
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_collection_modifyitems(session, config, items):
+        """
+        Support for running tests with component tags.
+        Report all test component markers to mongo_reporter.
+        """
+        logger.info(f"Preparing tests for test session in the following folder: {session.startdir}")
+
+        if pytest_keyword_filter:
+            # Filter case insensitive
+            deselected = [_item for _item in items if pytest_keyword_filter.lower() not in _item.name.lower()]
+            if deselected:
+                hooks.deselect_items(items, config, deselected)
+
+        yield  # deselect items in default hook way via keyword ('-k')
+
+        if config.option.collectonly:
+            hooks.log_skip_statistic(items)
+
+        deselected = hooks.preprocess_collected_items(items)
+        if deselected:
+            hooks.deselect_items(items, config, deselected)
+
+        hooks.set_divide_target_device_per_worker(items)
+
+        random.Random(7).shuffle(items)
+
+
+    @pytest.hookimpl(hookwrapper=True, tryfirst=True)
+    def pytest_runtest_protocol(item: "Item"):
+        """
+         Perform the runtest protocol for a single test item.
+         The default runtest protocol is this (see individual hooks for full details):
+             pytest_runtest_logstart(nodeid, location)
+             Setup phase:
+                     call = pytest_runtest_setup(item) (wrapped in CallInfo(when="setup"))
+                     report = pytest_runtest_makereport(item, call)
+                     pytest_runtest_logreport(report)
+                     pytest_exception_interact(call, report) if an interactive exception occurred
+             Call phase, if the setup passed and the setuponly pytest option is not set:
+                     call = pytest_runtest_call(item) (wrapped in CallInfo(when="call"))
+                     report = pytest_runtest_makereport(item, call)
+                     pytest_runtest_logreport(report)
+                     pytest_exception_interact(call, report) if an interactive exception occurred
+             Teardown phase:
+                     call = pytest_runtest_teardown(item, nextitem) (wrapped in CallInfo(when="teardown"))
+                     report = pytest_runtest_makereport(item, call)
+                     pytest_runtest_logreport(report)
+                     pytest_exception_interact(call, report) if an interactive exception occurred
+             pytest_runtest_logfinish(nodeid, location)
+         """
+        __root_logger = get_logger(None)
+        if not item.keywords.get("skip"):
+            fh = OvmsFileHandler(item)
+            __root_logger.addHandler(fh)
+        yield
+        if not item.keywords.get("skip"):
+            fh.close()
+            __root_logger.removeHandler(fh)
+
+
+    def pytest_generate_tests(metafunc):
+        hooks.parametrize_tests(metafunc)