Resolve ros_distro selectors in package tests

README.md

@@ -50,6 +50,57 @@ A ROS package whose name a `conda_index` maps to a conda package (for example
 `tl_expected` mapped to `cpp-expected`) is not built: dependencies on it resolve to the
 conda package, and a ROS build would shadow that package under the same name.
 
+## Sharing configuration between distributions
+
+A `vinca.yaml` can build on another one with `extends: <path>` (relative to the file),
+for example one configuration per ROS distribution on top of a shared one:
+
+```yaml
+# distros/jazzy/vinca.yaml
+extends: ../../shared/vinca.yaml
+ros_distro: jazzy
+mutex_package:
+  version: 0.19.0
+  run_constraints:
+    - libprotobuf 7.35.*   # replaces the shared libprotobuf constraint
+packages_exclude:
+  - if: win
+    then: [some_package]
+```
+
+- `packages_select_by_deps`, `packages_exclude` and `packages_skip` are combined (the
+  base's entries first).
+- `mutex_package` is merged: the extending file's keys win, and its `run_constraints`
+  replace the base's constraint on the same package and add the others.
+- Any other key of the extending file replaces the base's.
+- Files next to each configuration are combined too, relative to that
+  configuration's directory: `pkg_additional_info.yaml` and `<patch_dir>/dependencies.yaml`
+  per package (the extending file's keys win), patches and `tests/` per package (the
+  extending file's file wins), and `conda_index` files (the extending file's mappings
+  are looked up first).
+
+A base configuration can itself extend another one. Paths (`patch_dir`,
+`rosdistro_snapshot`, `rosdistro_additional_recipes`, `conda_index` files) are relative
+to the file that sets them.
+
+## Package tests
+
+`tests/<package>.yaml` (next to `vinca.yaml`, or next to a configuration it extends)
+becomes the `tests` section of the package's recipe. Selectors that test `ros_distro`
+are resolved by vinca, so one test file can serve several distributions; all other
+selectors are left to rattler-build:
+
+```yaml
+tests:
+  - script:
+      - if: ros_distro in ["humble", "jazzy"]
+        then:
+          - if: osx
+            then: export ROS_LOCALHOST_ONLY=1 && launch_test smoke_test.py
+        else:
+          - launch_test smoke_test.py
+```
+
 ## Package naming
 
 The optional `package_name_mode` setting controls the transition from legacy distro-qualified names such as `ros-humble-rclcpp` to ROS-major-version names such as `ros2-rclcpp`:

vinca/configuration.py

@@ -12,6 +12,7 @@
 
 from __future__ import annotations
 
+import re
 import warnings
 from pathlib import Path
 from typing import Any
@@ -150,41 +151,144 @@ def _apply_package_exclusions(vinca_conf: dict[str, Any]) -> None:
     ]
 
 
+# paths in a configuration, relative to the file that sets them
+_PATH_KEYS = ("patch_dir", "rosdistro_snapshot", "rosdistro_additional_recipes")
+
+# package lists that a configuration adds to the ones of the configuration it extends
+_LAYERED_LISTS = ("packages_select_by_deps", "packages_exclude", "packages_skip")
+
+
+def _constraint_name(spec: Any) -> str:
+    return re.split(r"[\s=<>!~]", str(spec).strip(), maxsplit=1)[0]
+
+
+def _merge_mutex(base: dict[str, Any], own: dict[str, Any]) -> dict[str, Any]:
+    """``mutex_package``: own keys win; own ``run_constraints`` replace the base's
+    constraint on the same package and add the others."""
+    merged = {**base, **own}
+    mine = list(own.get("run_constraints") or [])
+    names = {_constraint_name(c) for c in mine}
+    merged["run_constraints"] = [
+        c for c in base.get("run_constraints") or [] if _constraint_name(c) not in names
+    ] + mine
+    return merged
+
+
+def _merge_configs(base: dict[str, Any], own: dict[str, Any]) -> dict[str, Any]:
+    """A configuration on top of the one it extends: package lists are combined,
+    ``mutex_package`` is merged, any other key of ``own`` replaces the base's."""
+    merged = dict(base)
+    for key, value in own.items():
+        if key in _LAYERED_LISTS:
+            merged[key] = list(dict.fromkeys(_names(base.get(key)) + _names(value)))
+        elif key == "mutex_package" and isinstance(base.get(key), dict):
+            merged[key] = _merge_mutex(base[key], value or {})
+        else:
+            merged[key] = value
+    return merged
+
+
+def _merge_per_package(base: dict[str, Any], own: dict[str, Any]) -> dict[str, Any]:
+    """Per-package settings: own keys of a package's entry win over the base's."""
+    merged = {k: (dict(v) if isinstance(v, dict) else v) for k, v in base.items()}
+    for package, entry in own.items():
+        if isinstance(entry, dict) and isinstance(merged.get(package), dict):
+            merged[package] = {**merged[package], **entry}
+        else:
+            merged[package] = entry
+    return merged
+
+
+def _load_layers(
+    filepath: Path, target_platform: str, seen: tuple[Path, ...] = ()
+) -> list[tuple[Path, dict[str, Any]]]:
+    """The configuration and the ones it ``extends`` (resolved relative to the
+    file), base first, each as (directory, selector-resolved content)."""
+    filepath = filepath.resolve()
+    if filepath in seen:
+        chain = " -> ".join(str(p) for p in (*seen, filepath))
+        raise ValueError(f"vinca.yaml extends itself: {chain}")
+    conf = _load_selected_yaml(filepath, target_platform) or {}
+    for key in _PATH_KEYS:
+        if conf.get(key):
+            conf[key] = str((filepath.parent / conf[key]).resolve())
+    base = conf.pop("extends", None)
+    layers = (
+        _load_layers(filepath.parent / base, target_platform, (*seen, filepath))
+        if base
+        else []
+    )
+    return [*layers, (filepath.parent, conf)]
+
+
 def read_vinca_yaml(filepath: str | Path, target_platform: str) -> dict[str, Any]:
     """Read a vinca configuration and populate its derived internal fields."""
     filepath = Path(filepath)
     config_dir = filepath.parent
-    vinca_conf = _load_selected_yaml(filepath, target_platform)
+    layers = _load_layers(filepath, target_platform)
+    vinca_conf: dict[str, Any] = {}
+    for _, layer in layers:
+        vinca_conf = _merge_configs(vinca_conf, layer)
     _apply_package_exclusions(vinca_conf)
     vinca_conf["package_name_mode"] = get_package_name_mode(vinca_conf).value
     vinca_conf["variants_mode"] = get_variants_mode(vinca_conf).value
-    vinca_conf["conda_index"] = _normalize_conda_indexes(vinca_conf["conda_index"])
 
-    patch_dir = Path(vinca_conf["patch_dir"]).absolute()
-    vinca_conf["_patch_dir"] = patch_dir
-    vinca_conf["_patches"] = _discover_patches(patch_dir, vinca_conf["ros_distro"])
-    vinca_conf["_tests"], vinca_conf["_test_folders"] = _discover_tests(
-        config_dir, vinca_conf["ros_distro"]
+    # Files of each layer: patches and dependencies.yaml (patch_dir), tests,
+    # pkg_additional_info.yaml and conda_index files, relative to that layer's file.
+    patches: dict[str, Any] = {}
+    tests: dict[str, Path] = {}
+    test_folders: dict[str, Path] = {}
+    depmods: dict[str, Any] = {}
+    additional_info: dict[str, Any] = {}
+    conda_indexes: list[Any] = []
+    patch_dir: Path | None = None
+    for layer_dir, layer in layers:
+        if layer.get("patch_dir"):
+            layer_patch_dir = Path(layer["patch_dir"])
+            patches.update(_discover_patches(layer_patch_dir, vinca_conf["ros_distro"]))
+            dependencies_path = layer_patch_dir / "dependencies.yaml"
+            if dependencies_path.exists():
+                depmods = _merge_per_package(
+                    depmods,
+                    _load_selected_yaml(dependencies_path, target_platform) or {},
+                )
+            patch_dir = layer_patch_dir
+        layer_tests, layer_test_folders = _discover_tests(
+            layer_dir, vinca_conf["ros_distro"]
+        )
+        tests.update(layer_tests)
+        test_folders.update(layer_test_folders)
+        additional_info_path = layer_dir / "pkg_additional_info.yaml"
+        if additional_info_path.exists():
+            additional_info = _merge_per_package(
+                additional_info,
+                _load_selected_yaml(additional_info_path, target_platform) or {},
+            )
+        if layer.get("conda_index"):
+            layer_indexes = [
+                str(layer_dir / i) if (layer_dir / i).is_file() else i
+                for i in layer["conda_index"]
+            ]
+            # the extending configuration's mappings are looked up first
+            conda_indexes = (
+                get_conda_index({"conda_index": layer_indexes}, str(layer_dir))
+                + conda_indexes
+            )
+    vinca_conf["conda_index"] = _normalize_conda_indexes(
+        vinca_conf.get("conda_index") or []
     )
-
-    dependencies_path = patch_dir / "dependencies.yaml"
-    if dependencies_path.exists():
-        vinca_conf["depmods"] = _load_selected_yaml(dependencies_path, target_platform)
-    vinca_conf["depmods"] = vinca_conf.get("depmods") or {}
+    vinca_conf["_patch_dir"] = patch_dir or Path(vinca_conf["patch_dir"]).absolute()
+    vinca_conf["_patches"] = patches
+    vinca_conf["_tests"], vinca_conf["_test_folders"] = tests, test_folders
+    vinca_conf["depmods"] = depmods or vinca_conf.get("depmods") or {}
 
     config.ros_distro = vinca_conf["ros_distro"]
     config.skip_testing = vinca_conf.get("skip_testing", True)
     config.setup_pixi_version = vinca_conf.get("setup_pixi_version")
     config.pixi_version = vinca_conf.get("pixi_version")
-    vinca_conf["_conda_indexes"] = get_conda_index(vinca_conf, str(config_dir))
+    vinca_conf["_conda_indexes"] = conda_indexes
     vinca_conf["trigger_new_versions"] = vinca_conf.get("trigger_new_versions", False)
-
-    additional_info_path = config_dir / "pkg_additional_info.yaml"
-    vinca_conf["_pkg_additional_info"] = (
-        _load_selected_yaml(additional_info_path, target_platform)
-        if additional_info_path.exists()
-        else {}
-    )
+    vinca_conf["_pkg_additional_info"] = additional_info
 
     vinca_conf["_variant_config"] = {}
     if get_variants_mode(vinca_conf) is VariantsMode.LOCAL:

vinca/generate_gha.py

@@ -100,9 +100,7 @@ def add_additional_recipes(args):
     if not os.path.exists(additional_recipes_path):
         return []
 
-    with open("vinca.yaml", encoding="utf-8") as vinca_yaml:
-        vinca_conf = yaml.safe_load(vinca_yaml)
-
+    vinca_conf = read_vinca_yaml("vinca.yaml")
     repodatas = get_skip_existing(vinca_conf, args.platform)
 
     additional_recipes = []

vinca/main.py

@@ -51,6 +51,7 @@
     source_reference,
 )
 from .template import write_recipe, write_recipe_package
+from .v1_selectors import evaluate_distro_selectors
 
 unsatisfied_deps = set()
 distro = None
@@ -489,8 +490,10 @@ def parse_package(pkg, distro, vinca_conf, path):
 
     if test := vinca_conf.get("_tests", {}).get(final_name):
         # parse as yaml
-        text = test.read_text()
-        test_content = ruamel.yaml.safe_load(text)
+        text = test.read_text(encoding="utf-8")
+        test_content = evaluate_distro_selectors(
+            ruamel.yaml.safe_load(text), ros_distro=config.ros_distro
+        )
         recipe["test"] = test_content
 
     for p in pkg["authors"]:

vinca/sources.py

@@ -13,7 +13,6 @@
 
 from __future__ import annotations
 
-import os
 from pathlib import Path
 from typing import Any
 
@@ -41,7 +40,7 @@ def _is_built(package_name: str, version: str, vinca_conf: dict[str, Any]) -> bo
     return key in skipped
 
 
-def _package_patches(
+def package_patches(
     package_name: str, vinca_conf: dict[str, Any], platform: str
 ) -> list[str]:
     """Return the generic patches for a package followed by its platform-specific ones."""
@@ -51,27 +50,23 @@ def _package_patches(
     return [*configured.get("any", []), *configured.get(platform.split("-")[0], [])]
 
 
-def _relative_patch_paths(patches: list[str]) -> list[str]:
-    """Rewrite patch paths relative to their closest shared root with the recipe.
+def _recipe_patch_paths(patches: list[str]) -> list[str]:
+    """Where the recipe refers to its patches: ``patch/<file>`` next to the recipe.
 
-    Falls back to absolute paths when no shared root exists, which on Windows happens
-    whenever the patches live on a different drive than the working directory.
+    The template copies each patch from where it lives (the configuration's patch
+    directory, or one of a configuration it extends), so patches outside the working
+    directory, or on another drive on Windows, work too.
     """
-    try:
-        common_root = os.path.commonpath([os.getcwd(), *patches])
-    except ValueError:
-        return [Path(patch).as_posix() for patch in patches]
-    return [Path(os.path.relpath(patch, common_root)).as_posix() for patch in patches]
+    return [f"patch/{Path(patch).name}" for patch in patches]
 
 
 def generate_source(
     distro: Distro, vinca_conf: dict[str, Any], platform: str
 ) -> dict[str, dict[str, Any]]:
     """Generate per-package release sources for a multi-recipe build.
 
-    Patch paths are rewritten relative to the closest shared ancestor of the working
-    directory and the patches themselves, because rattler-build resolves them
-    relative to the recipe.
+    Patches are referred to as ``patch/<file>`` next to the recipe, because
+    rattler-build resolves them relative to the recipe; the template copies them.
     """
     sources: dict[str, dict[str, Any]] = {}
     for shortname in vinca_conf["_selected_pkgs"]:
@@ -92,10 +87,10 @@ def generate_source(
         entry: dict[str, Any] = source_reference(url=url, ref=ref, ref_type=ref_type)
         entry["target_directory"] = f"{package_name}/src/work"
 
-        patches = _package_patches(package_name, vinca_conf, platform)
+        patches = package_patches(package_name, vinca_conf, platform)
         if patches:
             print(patches)
-            entry["patches"] = _relative_patch_paths(patches)
+            entry["patches"] = _recipe_patch_paths(patches)
         sources[package_name] = entry
 
     mutex_recipe = generate_mutex_package_recipe(vinca_conf, distro)
@@ -129,7 +124,7 @@ def generate_source_version(
         package_name = package_names[0]
         entry: dict[str, Any] = source_reference(url=url, ref=ref, ref_type=ref_type)
         entry["target_directory"] = f"{package_name}/src/work"
-        if patches := _package_patches(package_name, vinca_conf, platform):
+        if patches := package_patches(package_name, vinca_conf, platform):
             entry["patches"] = patches
         sources[package_name] = entry
     return sources

vinca/template.py

@@ -8,12 +8,15 @@
 
 from ruamel import yaml
 
+from vinca import config
 from vinca.naming import get_package_prefix, is_legacy_compatibility_output
+from vinca.sources import package_patches
 from vinca.utils import (
     ensure_name_is_without_distro_prefix_and_with_underscores,
     get_pkg_additional_info,
     get_pkg_build_number,
 )
+from vinca.v1_selectors import evaluate_distro_selectors
 from vinca.variants import VariantsMode, get_recipe_variants, get_variants_mode
 
 TEMPLATE = """\
@@ -140,8 +143,10 @@ def write_recipe(source, outputs, vinca_conf, distro, single_file=True):
                 test := vinca_conf["_tests"].get(package_name)
             ):
                 print("Using test: ", test)
-                text = test.read_text()
-                test_content = yaml.safe_load(text)
+                text = test.read_text(encoding="utf-8")
+                test_content = evaluate_distro_selectors(
+                    yaml.safe_load(text), ros_distro=config.ros_distro
+                )
                 meta["tests"] = test_content["tests"]
 
             recipe_dir = (Path("recipes") / package_name).absolute()
@@ -174,10 +179,16 @@ def write_recipe(source, outputs, vinca_conf, distro, single_file=True):
                     file.dump(variants, stream)
 
             if meta.get("source") and meta["source"].get("patches"):
+                # the recipe refers to patch/<file>; copy each from where it lives
+                files = {
+                    Path(p).name: p
+                    for p in package_patches(
+                        package_name, vinca_conf, config.selected_platform or ""
+                    )
+                }
                 for p in meta["source"]["patches"]:
-                    patch_dir, _ = os.path.split(p)
-                    os.makedirs(recipe_dir / patch_dir, exist_ok=True)
-                    shutil.copyfile(p, recipe_dir / p)
+                    os.makedirs((recipe_dir / p).parent, exist_ok=True)
+                    shutil.copyfile(files.get(Path(p).name, p), recipe_dir / p)
 
             build_scripts = re.findall(r"'(.*?)'", meta["build"]["script"])
             for script in build_scripts: