IGNITE-28905 [ducktests] Add dev_binary_distribution flag to treat ignite-dev as a binary release layout

modules/ducktests/README.md

@@ -205,6 +205,7 @@ You can modify test environments at execution time using global flags injected t
 |---------------------|------------|----------------------|
 | **persistent_root** | Root directory for persistent storage in tests. Used for storing test data and logs. Default is typically the test working directory. | ```{"persistent_root": "/opt/ignite/test-data"}``` |
 | **install_root** | Root directory for Ignite installation. Points to the base directory where Ignite is installed for testing. Default is typically "/opt/ignite". | ```{"install_root": "/opt/ignite-testing"}``` |
+| **dev_binary_distribution** | Boolean flag declaring that dev folders (e.g. "ignite-dev") contain a complete binary distribution with a release layout, rather than a compiled source tree. Affects classpath building on service startup: all modules, including "ducktests" and "-ext" extensions, are resolved from `libs/optional/ignite-<module>` inside the distribution instead of `modules/<module>/target` and the neighbouring "ignite-extensions" directory. Default is false. | ```{"dev_binary_distribution": true}``` |
 
 ### Custom Ignites & Forks Testing
 You can test arbitrary binary releases or custom compiled forks by overriding the execution path:

modules/ducktests/tests/checks/utils/check_ignite_spec.py

@@ -16,12 +16,14 @@
 """
 Checks Spec class that describes config and command line to start Ignite-aware service.
 """
+import os
 from unittest.mock import Mock
 
 import pytest
 
-from ignitetest.services.utils.ignite_spec import IgniteApplicationSpec
+from ignitetest.services.utils.ignite_spec import IgniteApplicationSpec, IgniteNodeSpec, DEV_BINARY_DISTRIBUTION
 from ignitetest.utils.ignite_test import JFR_ENABLED
+from ignitetest.utils.version import IgniteVersion
 
 
 @pytest.fixture
@@ -38,6 +40,19 @@ def service():
     return service
 
 
+def with_version(service, version, install_root='/opt', modules=None):
+    """
+    Prepare the service mock for classpath (libs) resolution against the given version.
+    """
+    service.context.globals = {}
+    service.install_root = install_root
+    service.modules = modules
+    service.config.version = IgniteVersion(version)
+    service.product = str(service.config.version)
+
+    return service
+
+
 """
 Checks that the JVM options passed via constructor are not overriden by the default ones.
 """
@@ -90,10 +105,74 @@ def check_boolean_options__go_after_default_ones_and_overwrite_them__if_passed_v
 
 def check_colon_options__go_after_default_ones_and_overwrite_them__if_passed_via_jvm_opt(service):
     service.log_dir = "/default-path"
+    default_opt = "-Xlog:gc*=debug,gc+stats*=debug,gc+ergo*=debug:%s:uptime,time,level,tags" \
+                  % os.path.join("/default-path", "gc.log")
     spec = IgniteApplicationSpec(service, jvm_opts=["-Xlog:gc:/some-non-default-path/gc.log"])
     assert "-Xlog:gc:/some-non-default-path/gc.log" in spec.jvm_opts
-    assert "-Xlog:gc*=debug,gc+stats*=debug,gc+ergo*=debug:/default-path/gc.log:uptime,time,level,tags" \
-           in spec.jvm_opts
-    assert spec.jvm_opts.index("-Xlog:gc:/some-non-default-path/gc.log") > \
-           spec.jvm_opts.index(
-               "-Xlog:gc*=debug,gc+stats*=debug,gc+ergo*=debug:/default-path/gc.log:uptime,time,level,tags")
+    assert default_opt in spec.jvm_opts
+    assert spec.jvm_opts.index("-Xlog:gc:/some-non-default-path/gc.log") > spec.jvm_opts.index(default_opt)
+
+
+"""
+Checks that libs() resolves module classpath according to the version layout and
+the 'dev_binary_distribution' global flag.
+"""
+
+
+def module_target(home, module_name):
+    """Source-tree layout classpath entry of a module."""
+    return os.path.join('/opt', home, 'modules', module_name, 'target', '*')
+
+
+def module_optional_libs(home, module_name):
+    """Binary-release layout classpath entry of a module."""
+    return os.path.join('/opt', home, 'libs', 'optional', 'ignite-%s' % module_name, '*')
+
+
+def check_libs__resolved_from_source_tree__if_dev_version(service):
+    libs = IgniteNodeSpec(with_version(service, 'dev')).libs()
+
+    assert module_target('ignite-dev', 'log4j2') in libs
+    assert module_target('ignite-dev', 'ducktests') in libs
+    assert not [lib for lib in libs if os.path.join('libs', 'optional') in lib]
+
+
+def check_libs__resolved_from_distribution__if_dev_version_and_binary_distribution(service):
+    with_version(service, 'dev').context.globals[DEV_BINARY_DISTRIBUTION] = True
+
+    libs = IgniteNodeSpec(service).libs()
+
+    assert module_optional_libs('ignite-dev', 'log4j2') in libs
+    assert module_optional_libs('ignite-dev', 'ducktests') in libs
+    assert not [lib for lib in libs if os.path.join('modules', '') in lib]
+
+
+def check_ducktests_libs__resolved_from_dev_source_tree__if_release_version(service):
+    libs = IgniteNodeSpec(with_version(service, '2.17.0')).libs()
+
+    assert module_optional_libs('ignite-2.17.0', 'log4j2') in libs
+    assert module_target('ignite-dev', 'ducktests') in libs
+
+
+def check_ducktests_libs__resolved_from_dev_distribution__if_release_version_and_binary_distribution(service):
+    with_version(service, '2.17.0').context.globals[DEV_BINARY_DISTRIBUTION] = True
+
+    libs = IgniteNodeSpec(service).libs()
+
+    assert module_optional_libs('ignite-2.17.0', 'log4j2') in libs
+    assert module_optional_libs('ignite-dev', 'ducktests') in libs
+
+
+def check_ext_libs__resolved_from_neighbouring_extensions__if_dev_version(service):
+    libs = IgniteNodeSpec(with_version(service, 'dev', modules=['cdc-ext'])).libs()
+
+    assert module_target('ignite-extensions', 'cdc-ext') in libs
+
+
+def check_ext_libs__resolved_from_distribution__if_dev_version_and_binary_distribution(service):
+    with_version(service, 'dev', modules=['cdc-ext']).context.globals[DEV_BINARY_DISTRIBUTION] = True
+
+    libs = IgniteNodeSpec(service).libs()
+
+    assert module_optional_libs('ignite-dev', 'cdc-ext') in libs
+    assert not [lib for lib in libs if 'ignite-extensions' in lib]

modules/ducktests/tests/ignitetest/services/utils/cdc/kafka_to_ignite.py

@@ -214,11 +214,11 @@ def config_file_path(self):
                 else self.service.thin_client_config_file
 
         def script(self, cmd):
-            if self.service.config.version.is_dev:
-                return os.path.join(self.service.spec.extensions_home(), "modules", "cdc-ext", "bin", cmd)
-            else:
+            if self._is_binary_layout(self.service.config.version):
                 return self.service.script(cmd)
 
+            return os.path.join(self.service.spec.extensions_home(), "modules", "cdc-ext", "bin", cmd)
+
     return KafkaToIgniteSpec(service, service.spec.jvm_opts)
 
 

modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py

@@ -41,6 +41,8 @@
 
 SHARED_PREPARED_FILE = ".ignite_prepared"
 
+DEV_BINARY_DISTRIBUTION = "dev_binary_distribution"
+
 
 def resolve_spec(service, **kwargs):
     """
@@ -208,19 +210,30 @@ def __get_module_libs(project_dir, module_name, is_dev):
     def extensions_home(self):
         return os.path.join(self.service.install_root, "ignite-extensions")
 
+    def _is_binary_layout(self, version):
+        """
+        True if the home directory of the product for the given version has a binary release layout.
+        Dev versions are compiled source trees unless the 'dev_binary_distribution' global declares
+        the dev folder to be a complete binary distribution.
+        """
+        return not version.is_dev or self.service.context.globals.get(DEV_BINARY_DISTRIBUTION, False)
+
     def _module_libs(self, module_name):
         """
         Get list of paths to be added to classpath for the passed module for current spec.
         """
         if module_name == "ducktests":
-            return self.__get_module_libs(self.__home(str(DEV_BRANCH)), module_name, is_dev=True)
+            return self.__get_module_libs(self.__home(str(DEV_BRANCH)), module_name,
+                                          is_dev=not self._is_binary_layout(DEV_BRANCH))
+
+        is_dev = not self._is_binary_layout(self.service.config.version)
 
-        if module_name.endswith("-ext") and self.service.config.version.is_dev:
+        if module_name.endswith("-ext") and is_dev:
             home = self.extensions_home()
         else:
             home = self.__home()
 
-        return self.__get_module_libs(home, module_name, self.service.config.version.is_dev)
+        return self.__get_module_libs(home, module_name, is_dev)
 
     @abstractmethod
     def command(self, node):

modules/ducktests/tests/tox.ini

@@ -14,7 +14,6 @@
 # limitations under the License.
 [tox]
 envlist = codestyle, py{38,39,310,311,312,313}
-skipsdist = True
 
 [testenv]
 usedevelop = True
@@ -25,6 +24,7 @@ commands = pytest {env:PYTESTARGS:} {posargs}
 
 [testenv:codestyle]
 basepython = python3
+skip_install = True
 deps = flake8
 commands = flake8