diff --git a/modules/ducktests/README.md b/modules/ducktests/README.md index 3d56ca33f292d..25c7d938dcca5 100644 --- a/modules/ducktests/README.md +++ b/modules/ducktests/README.md @@ -205,6 +205,7 @@ You can modify test environments at execution time using global flags injected t |---------------------|------------|----------------------| | **persistent_root** | Root directory for persistent storage in tests. Used for storing test data and logs. Default is typically the test working directory. | ```{"persistent_root": "/opt/ignite/test-data"}``` | | **install_root** | Root directory for Ignite installation. Points to the base directory where Ignite is installed for testing. Default is typically "/opt/ignite". | ```{"install_root": "/opt/ignite-testing"}``` | +| **dev_binary_distribution** | Boolean flag declaring that dev folders (e.g. "ignite-dev") contain a complete binary distribution with a release layout, rather than a compiled source tree. Affects classpath building on service startup: all modules, including "ducktests" and "-ext" extensions, are resolved from `libs/optional/ignite-` inside the distribution instead of `modules//target` and the neighbouring "ignite-extensions" directory. Default is false. | ```{"dev_binary_distribution": true}``` | ### Custom Ignites & Forks Testing You can test arbitrary binary releases or custom compiled forks by overriding the execution path: diff --git a/modules/ducktests/tests/checks/utils/check_ignite_spec.py b/modules/ducktests/tests/checks/utils/check_ignite_spec.py index a3c8e52104939..4a6336d6053f2 100644 --- a/modules/ducktests/tests/checks/utils/check_ignite_spec.py +++ b/modules/ducktests/tests/checks/utils/check_ignite_spec.py @@ -16,12 +16,14 @@ """ Checks Spec class that describes config and command line to start Ignite-aware service. """ +import os from unittest.mock import Mock import pytest -from ignitetest.services.utils.ignite_spec import IgniteApplicationSpec +from ignitetest.services.utils.ignite_spec import IgniteApplicationSpec, IgniteNodeSpec, DEV_BINARY_DISTRIBUTION from ignitetest.utils.ignite_test import JFR_ENABLED +from ignitetest.utils.version import IgniteVersion @pytest.fixture @@ -38,6 +40,19 @@ def service(): return service +def with_version(service, version, install_root='/opt', modules=None): + """ + Prepare the service mock for classpath (libs) resolution against the given version. + """ + service.context.globals = {} + service.install_root = install_root + service.modules = modules + service.config.version = IgniteVersion(version) + service.product = str(service.config.version) + + return service + + """ Checks that the JVM options passed via constructor are not overriden by the default ones. """ @@ -90,10 +105,74 @@ def check_boolean_options__go_after_default_ones_and_overwrite_them__if_passed_v def check_colon_options__go_after_default_ones_and_overwrite_them__if_passed_via_jvm_opt(service): service.log_dir = "/default-path" + default_opt = "-Xlog:gc*=debug,gc+stats*=debug,gc+ergo*=debug:%s:uptime,time,level,tags" \ + % os.path.join("/default-path", "gc.log") spec = IgniteApplicationSpec(service, jvm_opts=["-Xlog:gc:/some-non-default-path/gc.log"]) assert "-Xlog:gc:/some-non-default-path/gc.log" in spec.jvm_opts - assert "-Xlog:gc*=debug,gc+stats*=debug,gc+ergo*=debug:/default-path/gc.log:uptime,time,level,tags" \ - in spec.jvm_opts - assert spec.jvm_opts.index("-Xlog:gc:/some-non-default-path/gc.log") > \ - spec.jvm_opts.index( - "-Xlog:gc*=debug,gc+stats*=debug,gc+ergo*=debug:/default-path/gc.log:uptime,time,level,tags") + assert default_opt in spec.jvm_opts + assert spec.jvm_opts.index("-Xlog:gc:/some-non-default-path/gc.log") > spec.jvm_opts.index(default_opt) + + +""" +Checks that libs() resolves module classpath according to the version layout and +the 'dev_binary_distribution' global flag. +""" + + +def module_target(home, module_name): + """Source-tree layout classpath entry of a module.""" + return os.path.join('/opt', home, 'modules', module_name, 'target', '*') + + +def module_optional_libs(home, module_name): + """Binary-release layout classpath entry of a module.""" + return os.path.join('/opt', home, 'libs', 'optional', 'ignite-%s' % module_name, '*') + + +def check_libs__resolved_from_source_tree__if_dev_version(service): + libs = IgniteNodeSpec(with_version(service, 'dev')).libs() + + assert module_target('ignite-dev', 'log4j2') in libs + assert module_target('ignite-dev', 'ducktests') in libs + assert not [lib for lib in libs if os.path.join('libs', 'optional') in lib] + + +def check_libs__resolved_from_distribution__if_dev_version_and_binary_distribution(service): + with_version(service, 'dev').context.globals[DEV_BINARY_DISTRIBUTION] = True + + libs = IgniteNodeSpec(service).libs() + + assert module_optional_libs('ignite-dev', 'log4j2') in libs + assert module_optional_libs('ignite-dev', 'ducktests') in libs + assert not [lib for lib in libs if os.path.join('modules', '') in lib] + + +def check_ducktests_libs__resolved_from_dev_source_tree__if_release_version(service): + libs = IgniteNodeSpec(with_version(service, '2.17.0')).libs() + + assert module_optional_libs('ignite-2.17.0', 'log4j2') in libs + assert module_target('ignite-dev', 'ducktests') in libs + + +def check_ducktests_libs__resolved_from_dev_distribution__if_release_version_and_binary_distribution(service): + with_version(service, '2.17.0').context.globals[DEV_BINARY_DISTRIBUTION] = True + + libs = IgniteNodeSpec(service).libs() + + assert module_optional_libs('ignite-2.17.0', 'log4j2') in libs + assert module_optional_libs('ignite-dev', 'ducktests') in libs + + +def check_ext_libs__resolved_from_neighbouring_extensions__if_dev_version(service): + libs = IgniteNodeSpec(with_version(service, 'dev', modules=['cdc-ext'])).libs() + + assert module_target('ignite-extensions', 'cdc-ext') in libs + + +def check_ext_libs__resolved_from_distribution__if_dev_version_and_binary_distribution(service): + with_version(service, 'dev', modules=['cdc-ext']).context.globals[DEV_BINARY_DISTRIBUTION] = True + + libs = IgniteNodeSpec(service).libs() + + assert module_optional_libs('ignite-dev', 'cdc-ext') in libs + assert not [lib for lib in libs if 'ignite-extensions' in lib] diff --git a/modules/ducktests/tests/ignitetest/services/utils/cdc/kafka_to_ignite.py b/modules/ducktests/tests/ignitetest/services/utils/cdc/kafka_to_ignite.py index 1ffb501ecd1f5..b4d98da299656 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/cdc/kafka_to_ignite.py +++ b/modules/ducktests/tests/ignitetest/services/utils/cdc/kafka_to_ignite.py @@ -214,11 +214,11 @@ def config_file_path(self): else self.service.thin_client_config_file def script(self, cmd): - if self.service.config.version.is_dev: - return os.path.join(self.service.spec.extensions_home(), "modules", "cdc-ext", "bin", cmd) - else: + if self._is_binary_layout(self.service.config.version): return self.service.script(cmd) + return os.path.join(self.service.spec.extensions_home(), "modules", "cdc-ext", "bin", cmd) + return KafkaToIgniteSpec(service, service.spec.jvm_opts) diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py index a99aa163026f4..495e63fc10422 100644 --- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py +++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py @@ -41,6 +41,8 @@ SHARED_PREPARED_FILE = ".ignite_prepared" +DEV_BINARY_DISTRIBUTION = "dev_binary_distribution" + def resolve_spec(service, **kwargs): """ @@ -208,19 +210,30 @@ def __get_module_libs(project_dir, module_name, is_dev): def extensions_home(self): return os.path.join(self.service.install_root, "ignite-extensions") + def _is_binary_layout(self, version): + """ + True if the home directory of the product for the given version has a binary release layout. + Dev versions are compiled source trees unless the 'dev_binary_distribution' global declares + the dev folder to be a complete binary distribution. + """ + return not version.is_dev or self.service.context.globals.get(DEV_BINARY_DISTRIBUTION, False) + def _module_libs(self, module_name): """ Get list of paths to be added to classpath for the passed module for current spec. """ if module_name == "ducktests": - return self.__get_module_libs(self.__home(str(DEV_BRANCH)), module_name, is_dev=True) + return self.__get_module_libs(self.__home(str(DEV_BRANCH)), module_name, + is_dev=not self._is_binary_layout(DEV_BRANCH)) + + is_dev = not self._is_binary_layout(self.service.config.version) - if module_name.endswith("-ext") and self.service.config.version.is_dev: + if module_name.endswith("-ext") and is_dev: home = self.extensions_home() else: home = self.__home() - return self.__get_module_libs(home, module_name, self.service.config.version.is_dev) + return self.__get_module_libs(home, module_name, is_dev) @abstractmethod def command(self, node): diff --git a/modules/ducktests/tests/tox.ini b/modules/ducktests/tests/tox.ini index e0fa4a74df8aa..f3b2f81dfa676 100644 --- a/modules/ducktests/tests/tox.ini +++ b/modules/ducktests/tests/tox.ini @@ -14,7 +14,6 @@ # limitations under the License. [tox] envlist = codestyle, py{38,39,310,311,312,313} -skipsdist = True [testenv] usedevelop = True @@ -25,6 +24,7 @@ commands = pytest {env:PYTESTARGS:} {posargs} [testenv:codestyle] basepython = python3 +skip_install = True deps = flake8 commands = flake8