From ee00b2060f40f66d5daab3cbb87e35a31811561e Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Thu, 8 Oct 2026 10:16:52 +1000 Subject: [PATCH 1/5] extends: build a vinca.yaml on top of a shared one A configuration can extend another one (relative path). Package lists are combined, mutex_package is merged (run_constraints replaced per package), other keys of the extending file win. pkg_additional_info.yaml, dependencies.yaml, patches, tests and conda_index files of every layer are combined, each relative to its configuration. Co-Authored-By: Claude Opus 5.5 --- README.md | 31 +++++++ vinca/configuration.py | 149 +++++++++++++++++++++++++++++----- vinca/test_config_layering.py | 143 ++++++++++++++++++++++++++++++++ 3 files changed, 303 insertions(+), 20 deletions(-) create mode 100644 vinca/test_config_layering.py diff --git a/README.md b/README.md index c8d5f6a..8898b37 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,37 @@ A ROS package whose name a `conda_index` maps to a conda package (for example `tl_expected` mapped to `cpp-expected`) is not built: dependencies on it resolve to the conda package, and a ROS build would shadow that package under the same name. +## Sharing configuration between distributions + +A `vinca.yaml` can build on another one with `extends: ` (relative to the file), +for example one configuration per ROS distribution on top of a shared one: + +```yaml +# distros/jazzy/vinca.yaml +extends: ../../shared/vinca.yaml +ros_distro: jazzy +mutex_package: + version: 0.19.0 + run_constraints: + - libprotobuf 7.35.* # replaces the shared libprotobuf constraint +packages_exclude: + - if: win + then: [some_package] +``` + +- `packages_select_by_deps`, `packages_exclude` and `packages_skip` are combined (the + base's entries first). +- `mutex_package` is merged: the extending file's keys win, and its `run_constraints` + replace the base's constraint on the same package and add the others. +- Any other key of the extending file replaces the base's. +- Files next to each configuration are combined too, relative to that + configuration's directory: `pkg_additional_info.yaml` and `/dependencies.yaml` + per package (the extending file's keys win), patches and `tests/` per package (the + extending file's file wins), and `conda_index` files (the extending file's mappings + are looked up first). + +A base configuration can itself extend another one. + ## Package naming The optional `package_name_mode` setting controls the transition from legacy distro-qualified names such as `ros-humble-rclcpp` to ROS-major-version names such as `ros2-rclcpp`: diff --git a/vinca/configuration.py b/vinca/configuration.py index 6554e8c..494cffb 100644 --- a/vinca/configuration.py +++ b/vinca/configuration.py @@ -12,6 +12,7 @@ from __future__ import annotations +import re import warnings from pathlib import Path from typing import Any @@ -150,41 +151,149 @@ def _apply_package_exclusions(vinca_conf: dict[str, Any]) -> None: ] +# package lists that a configuration adds to the ones of the configuration it extends +_LAYERED_LISTS = ("packages_select_by_deps", "packages_exclude", "packages_skip") + + +def _constraint_name(spec: Any) -> str: + return re.split(r"[\s=<>!~]", str(spec).strip(), maxsplit=1)[0] + + +def _merge_mutex(base: dict[str, Any], own: dict[str, Any]) -> dict[str, Any]: + """``mutex_package``: own keys win; own ``run_constraints`` replace the base's + constraint on the same package and add the others.""" + merged = {**base, **own} + mine = list(own.get("run_constraints") or []) + names = {_constraint_name(c) for c in mine} + merged["run_constraints"] = [ + c for c in base.get("run_constraints") or [] if _constraint_name(c) not in names + ] + mine + return merged + + +def _merge_configs(base: dict[str, Any], own: dict[str, Any]) -> dict[str, Any]: + """A configuration on top of the one it extends: package lists are combined, + ``mutex_package`` is merged, any other key of ``own`` replaces the base's.""" + merged = dict(base) + for key, value in own.items(): + if key in _LAYERED_LISTS: + merged[key] = list(dict.fromkeys(_names(base.get(key)) + _names(value))) + elif key == "mutex_package" and isinstance(base.get(key), dict): + merged[key] = _merge_mutex(base[key], value or {}) + else: + merged[key] = value + return merged + + +def _merge_per_package(base: dict[str, Any], own: dict[str, Any]) -> dict[str, Any]: + """Per-package settings: own keys of a package's entry win over the base's.""" + merged = {k: (dict(v) if isinstance(v, dict) else v) for k, v in base.items()} + for package, entry in own.items(): + if isinstance(entry, dict) and isinstance(merged.get(package), dict): + merged[package] = {**merged[package], **entry} + else: + merged[package] = entry + return merged + + +def _load_layers( + filepath: Path, target_platform: str, seen: tuple[Path, ...] = () +) -> list[tuple[Path, dict[str, Any]]]: + """The configuration and the ones it ``extends`` (resolved relative to the + file), base first, each as (directory, selector-resolved content).""" + filepath = filepath.resolve() + if filepath in seen: + chain = " -> ".join(str(p) for p in (*seen, filepath)) + raise ValueError(f"vinca.yaml extends itself: {chain}") + conf = _load_selected_yaml(filepath, target_platform) or {} + base = conf.pop("extends", None) + layers = ( + _load_layers(filepath.parent / base, target_platform, (*seen, filepath)) + if base + else [] + ) + return [*layers, (filepath.parent, conf)] + + def read_vinca_yaml(filepath: str | Path, target_platform: str) -> dict[str, Any]: """Read a vinca configuration and populate its derived internal fields.""" filepath = Path(filepath) config_dir = filepath.parent - vinca_conf = _load_selected_yaml(filepath, target_platform) + layers = _load_layers(filepath, target_platform) + vinca_conf: dict[str, Any] = {} + for _, layer in layers: + vinca_conf = _merge_configs(vinca_conf, layer) _apply_package_exclusions(vinca_conf) vinca_conf["package_name_mode"] = get_package_name_mode(vinca_conf).value vinca_conf["variants_mode"] = get_variants_mode(vinca_conf).value - vinca_conf["conda_index"] = _normalize_conda_indexes(vinca_conf["conda_index"]) - patch_dir = Path(vinca_conf["patch_dir"]).absolute() - vinca_conf["_patch_dir"] = patch_dir - vinca_conf["_patches"] = _discover_patches(patch_dir, vinca_conf["ros_distro"]) - vinca_conf["_tests"], vinca_conf["_test_folders"] = _discover_tests( - config_dir, vinca_conf["ros_distro"] + # Files next to each layer: patches and dependencies.yaml (patch_dir), tests, + # pkg_additional_info.yaml and conda_index files. Paths of a base configuration + # are relative to its own file; the extending configuration (the root) keeps + # resolving patch_dir against the working directory, as without extends. + patches: dict[str, Any] = {} + tests: dict[str, Path] = {} + test_folders: dict[str, Path] = {} + depmods: dict[str, Any] = {} + additional_info: dict[str, Any] = {} + conda_indexes: list[Any] = [] + patch_dir: Path | None = None + for layer_dir, layer in layers: + is_root = layer_dir == config_dir.resolve() + if layer.get("patch_dir"): + layer_patch_dir = ( + Path(layer["patch_dir"]).absolute() + if is_root + else (layer_dir / layer["patch_dir"]).resolve() + ) + patches.update(_discover_patches(layer_patch_dir, vinca_conf["ros_distro"])) + dependencies_path = layer_patch_dir / "dependencies.yaml" + if dependencies_path.exists(): + depmods = _merge_per_package( + depmods, + _load_selected_yaml(dependencies_path, target_platform) or {}, + ) + patch_dir = layer_patch_dir + layer_tests, layer_test_folders = _discover_tests( + layer_dir, vinca_conf["ros_distro"] + ) + tests.update(layer_tests) + test_folders.update(layer_test_folders) + additional_info_path = layer_dir / "pkg_additional_info.yaml" + if additional_info_path.exists(): + additional_info = _merge_per_package( + additional_info, + _load_selected_yaml(additional_info_path, target_platform) or {}, + ) + if layer.get("conda_index"): + layer_indexes = ( + _normalize_conda_indexes(layer["conda_index"]) + if is_root + else [ + str(layer_dir / i) if (layer_dir / i).is_file() else i + for i in layer["conda_index"] + ] + ) + # the extending configuration's mappings are looked up first + conda_indexes = ( + get_conda_index({"conda_index": layer_indexes}, str(layer_dir)) + + conda_indexes + ) + vinca_conf["conda_index"] = _normalize_conda_indexes( + vinca_conf.get("conda_index") or [] ) - - dependencies_path = patch_dir / "dependencies.yaml" - if dependencies_path.exists(): - vinca_conf["depmods"] = _load_selected_yaml(dependencies_path, target_platform) - vinca_conf["depmods"] = vinca_conf.get("depmods") or {} + vinca_conf["_patch_dir"] = patch_dir or Path(vinca_conf["patch_dir"]).absolute() + vinca_conf["_patches"] = patches + vinca_conf["_tests"], vinca_conf["_test_folders"] = tests, test_folders + vinca_conf["depmods"] = depmods or vinca_conf.get("depmods") or {} config.ros_distro = vinca_conf["ros_distro"] config.skip_testing = vinca_conf.get("skip_testing", True) config.setup_pixi_version = vinca_conf.get("setup_pixi_version") config.pixi_version = vinca_conf.get("pixi_version") - vinca_conf["_conda_indexes"] = get_conda_index(vinca_conf, str(config_dir)) + vinca_conf["_conda_indexes"] = conda_indexes vinca_conf["trigger_new_versions"] = vinca_conf.get("trigger_new_versions", False) - - additional_info_path = config_dir / "pkg_additional_info.yaml" - vinca_conf["_pkg_additional_info"] = ( - _load_selected_yaml(additional_info_path, target_platform) - if additional_info_path.exists() - else {} - ) + vinca_conf["_pkg_additional_info"] = additional_info vinca_conf["_variant_config"] = {} if get_variants_mode(vinca_conf) is VariantsMode.LOCAL: diff --git a/vinca/test_config_layering.py b/vinca/test_config_layering.py new file mode 100644 index 0000000..71725ba --- /dev/null +++ b/vinca/test_config_layering.py @@ -0,0 +1,143 @@ +"""Tests for ``extends:``: a configuration on top of a shared base configuration.""" + +import pytest + +from vinca.configuration import read_vinca_yaml + + +def _write(path, text): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + + +@pytest.fixture +def layered(tmp_path, monkeypatch): + shared, distro = tmp_path / "shared", tmp_path / "distros" / "humble" + _write( + shared / "vinca.yaml", + """\ +package_name_mode: new +conda_index: [robostack.yaml] +patch_dir: patch +skip_testing: true +mutex_package: + name: ros2-distro-mutex + upper_bound: x.x + run_constraints: + - libboost 1.90.* + - pcl 1.15.* +packages_select_by_deps: + - desktop + - if: win + then: [win_only] +packages_exclude: + - broken +""", + ) + _write( + shared / "robostack.yaml", + "eigen:\n robostack: [eigen]\nfoo:\n robostack: [foo]\n", + ) + _write( + shared / "patch" / "dependencies.yaml", + "demo:\n add_host: [shared-dep]\n add_run: [x]\n", + ) + _write(shared / "patch" / "ros2-demo.patch", "shared patch") + _write(shared / "patch" / "ros2-other.patch", "shared other patch") + _write( + shared / "pkg_additional_info.yaml", + "demo:\n additional_cmake_args: -DFOO=ON\n", + ) + _write(shared / "tests" / "ros2-demo.yaml", "tests: [shared]\n") + _write(shared / "tests" / "ros2-other.yaml", "tests: [shared]\n") + + _write( + distro / "vinca.yaml", + """\ +extends: ../../shared/vinca.yaml +ros_distro: humble +conda_index: [robostack.yaml] +patch_dir: patch +mutex_package: + version: 0.10.0 + run_constraints: + - pcl 1.16.* + - krb5 1.22.* +packages_select_by_deps: + - extra +packages_skip: + - desktop +""", + ) + _write(distro / "robostack.yaml", "foo:\n robostack: [foo-own]\n") + _write(distro / "patch" / "dependencies.yaml", "demo:\n add_run: [own-dep]\n") + _write(distro / "patch" / "ros2-demo.patch", "own patch") + _write(distro / "pkg_additional_info.yaml", "demo:\n build_number: 3\n") + _write(distro / "tests" / "ros2-demo.yaml", "tests: [own]\n") + + monkeypatch.chdir(distro) + return shared, distro + + +def test_lists_and_scalars_are_layered(layered): + conf = read_vinca_yaml(layered[1] / "vinca.yaml", "linux-64") + + assert conf["packages_select_by_deps"] == ["extra"] # desktop is skipped + assert conf["packages_skip"] == ["desktop"] + assert conf["packages_exclude"] == ["broken"] + assert conf["package_name_mode"] == "new" + assert conf["skip_testing"] is True + assert "extends" not in conf + + +def test_selectors_of_the_base_are_resolved(layered): + conf = read_vinca_yaml(layered[1] / "vinca.yaml", "win-64") + + assert conf["packages_select_by_deps"] == ["win_only", "extra"] + + +def test_mutex_constraints_are_overridden_by_package(layered): + mutex = read_vinca_yaml(layered[1] / "vinca.yaml", "linux-64")["mutex_package"] + + assert mutex["name"] == "ros2-distro-mutex" + assert mutex["version"] == "0.10.0" + assert mutex["upper_bound"] == "x.x" + assert mutex["run_constraints"] == ["libboost 1.90.*", "pcl 1.16.*", "krb5 1.22.*"] + + +def test_files_of_each_layer(layered): + shared, distro = layered + conf = read_vinca_yaml(distro / "vinca.yaml", "linux-64") + + # per-package settings: the distribution's keys win + assert conf["depmods"]["demo"] == { + "add_host": ["shared-dep"], + "add_run": ["own-dep"], + } + assert conf["_pkg_additional_info"]["demo"] == { + "additional_cmake_args": "-DFOO=ON", + "build_number": 3, + } + # patches and tests: the distribution's file replaces the shared one + assert conf["_patches"]["ros2-demo"]["any"] == [ + str(distro / "patch" / "ros2-demo.patch") + ] + assert conf["_patches"]["ros2-other"]["any"] == [ + str(shared / "patch" / "ros2-other.patch") + ] + assert conf["_tests"]["ros2-demo"] == distro / "tests" / "ros2-demo.yaml" + assert conf["_tests"]["ros2-other"] == shared / "tests" / "ros2-other.yaml" + # conda_index: the distribution's mappings are looked up first + assert [list(index) for index in conf["_conda_indexes"]] == [ + ["foo"], + ["eigen", "foo"], + ] + + +def test_extends_cycle_is_rejected(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _write(tmp_path / "a.yaml", "extends: b.yaml\n") + _write(tmp_path / "b.yaml", "extends: a.yaml\n") + + with pytest.raises(ValueError, match="extends itself"): + read_vinca_yaml(tmp_path / "a.yaml", "linux-64") From c6cb34bf3f3219bde1a346c850fc912aaaaf348c Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Thu, 8 Oct 2026 10:19:54 +1000 Subject: [PATCH 2/5] Resolve configuration paths relative to the file that sets them; vinca-gha reads the layered configuration Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +++- vinca/configuration.py | 31 +++++++++++++------------------ vinca/generate_gha.py | 4 +--- vinca/test_config_layering.py | 18 ++++++++++++++++++ 4 files changed, 35 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 8898b37..6303bc4 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,9 @@ packages_exclude: extending file's file wins), and `conda_index` files (the extending file's mappings are looked up first). -A base configuration can itself extend another one. +A base configuration can itself extend another one. Paths (`patch_dir`, +`rosdistro_snapshot`, `rosdistro_additional_recipes`, `conda_index` files) are relative +to the file that sets them. ## Package naming diff --git a/vinca/configuration.py b/vinca/configuration.py index 494cffb..c9806ca 100644 --- a/vinca/configuration.py +++ b/vinca/configuration.py @@ -151,6 +151,9 @@ def _apply_package_exclusions(vinca_conf: dict[str, Any]) -> None: ] +# paths in a configuration, relative to the file that sets them +_PATH_KEYS = ("patch_dir", "rosdistro_snapshot", "rosdistro_additional_recipes") + # package lists that a configuration adds to the ones of the configuration it extends _LAYERED_LISTS = ("packages_select_by_deps", "packages_exclude", "packages_skip") @@ -206,6 +209,9 @@ def _load_layers( chain = " -> ".join(str(p) for p in (*seen, filepath)) raise ValueError(f"vinca.yaml extends itself: {chain}") conf = _load_selected_yaml(filepath, target_platform) or {} + for key in _PATH_KEYS: + if conf.get(key): + conf[key] = str((filepath.parent / conf[key]).resolve()) base = conf.pop("extends", None) layers = ( _load_layers(filepath.parent / base, target_platform, (*seen, filepath)) @@ -227,10 +233,8 @@ def read_vinca_yaml(filepath: str | Path, target_platform: str) -> dict[str, Any vinca_conf["package_name_mode"] = get_package_name_mode(vinca_conf).value vinca_conf["variants_mode"] = get_variants_mode(vinca_conf).value - # Files next to each layer: patches and dependencies.yaml (patch_dir), tests, - # pkg_additional_info.yaml and conda_index files. Paths of a base configuration - # are relative to its own file; the extending configuration (the root) keeps - # resolving patch_dir against the working directory, as without extends. + # Files of each layer: patches and dependencies.yaml (patch_dir), tests, + # pkg_additional_info.yaml and conda_index files, relative to that layer's file. patches: dict[str, Any] = {} tests: dict[str, Path] = {} test_folders: dict[str, Path] = {} @@ -239,13 +243,8 @@ def read_vinca_yaml(filepath: str | Path, target_platform: str) -> dict[str, Any conda_indexes: list[Any] = [] patch_dir: Path | None = None for layer_dir, layer in layers: - is_root = layer_dir == config_dir.resolve() if layer.get("patch_dir"): - layer_patch_dir = ( - Path(layer["patch_dir"]).absolute() - if is_root - else (layer_dir / layer["patch_dir"]).resolve() - ) + layer_patch_dir = Path(layer["patch_dir"]) patches.update(_discover_patches(layer_patch_dir, vinca_conf["ros_distro"])) dependencies_path = layer_patch_dir / "dependencies.yaml" if dependencies_path.exists(): @@ -266,14 +265,10 @@ def read_vinca_yaml(filepath: str | Path, target_platform: str) -> dict[str, Any _load_selected_yaml(additional_info_path, target_platform) or {}, ) if layer.get("conda_index"): - layer_indexes = ( - _normalize_conda_indexes(layer["conda_index"]) - if is_root - else [ - str(layer_dir / i) if (layer_dir / i).is_file() else i - for i in layer["conda_index"] - ] - ) + layer_indexes = [ + str(layer_dir / i) if (layer_dir / i).is_file() else i + for i in layer["conda_index"] + ] # the extending configuration's mappings are looked up first conda_indexes = ( get_conda_index({"conda_index": layer_indexes}, str(layer_dir)) diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index 83f8cc3..c1b5f4d 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -100,9 +100,7 @@ def add_additional_recipes(args): if not os.path.exists(additional_recipes_path): return [] - with open("vinca.yaml", encoding="utf-8") as vinca_yaml: - vinca_conf = yaml.safe_load(vinca_yaml) - + vinca_conf = read_vinca_yaml("vinca.yaml") repodatas = get_skip_existing(vinca_conf, args.platform) additional_recipes = [] diff --git a/vinca/test_config_layering.py b/vinca/test_config_layering.py index 71725ba..320a9df 100644 --- a/vinca/test_config_layering.py +++ b/vinca/test_config_layering.py @@ -141,3 +141,21 @@ def test_extends_cycle_is_rejected(tmp_path, monkeypatch): with pytest.raises(ValueError, match="extends itself"): read_vinca_yaml(tmp_path / "a.yaml", "linux-64") + + +def test_paths_are_relative_to_the_file_that_sets_them(layered, tmp_path, monkeypatch): + shared, distro = layered + _write(distro / "rosdistro_snapshot.yaml", "demo:\n version: 1.0.0\n") + text = (distro / "vinca.yaml").read_text() + (distro / "vinca.yaml").write_text( + text + "rosdistro_snapshot: rosdistro_snapshot.yaml\n" + ) + monkeypatch.chdir(tmp_path) # not the configuration's directory + + conf = read_vinca_yaml(distro / "vinca.yaml", "linux-64") + + assert conf["_snapshot"]["demo"]["version"] == "1.0.0" + assert conf["_patch_dir"] == distro / "patch" + assert conf["_patches"]["ros2-demo"]["any"] == [ + str(distro / "patch" / "ros2-demo.patch") + ] From b70f0f02d15b3efd604470e939c3f24af7fd6e5a Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Thu, 8 Oct 2026 12:15:29 +1000 Subject: [PATCH 3/5] Copy each patch into the recipe from where it lives The recipe refers to patch/; the template copies the patch from its configuration's patch directory. Before, a path relative to a common root was copied relative to the working directory, which broke for patches outside it (e.g. with extends). Co-Authored-By: Claude Opus 5.5 --- vinca/sources.py | 29 ++++++++++++----------------- vinca/template.py | 14 +++++++++++--- vinca/test_sources.py | 29 +++++++++++------------------ 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/vinca/sources.py b/vinca/sources.py index 04014e8..17c3c15 100644 --- a/vinca/sources.py +++ b/vinca/sources.py @@ -13,7 +13,6 @@ from __future__ import annotations -import os from pathlib import Path from typing import Any @@ -41,7 +40,7 @@ def _is_built(package_name: str, version: str, vinca_conf: dict[str, Any]) -> bo return key in skipped -def _package_patches( +def package_patches( package_name: str, vinca_conf: dict[str, Any], platform: str ) -> list[str]: """Return the generic patches for a package followed by its platform-specific ones.""" @@ -51,17 +50,14 @@ def _package_patches( return [*configured.get("any", []), *configured.get(platform.split("-")[0], [])] -def _relative_patch_paths(patches: list[str]) -> list[str]: - """Rewrite patch paths relative to their closest shared root with the recipe. +def _recipe_patch_paths(patches: list[str]) -> list[str]: + """Where the recipe refers to its patches: ``patch/`` next to the recipe. - Falls back to absolute paths when no shared root exists, which on Windows happens - whenever the patches live on a different drive than the working directory. + The template copies each patch from where it lives (the configuration's patch + directory, or one of a configuration it extends), so patches outside the working + directory, or on another drive on Windows, work too. """ - try: - common_root = os.path.commonpath([os.getcwd(), *patches]) - except ValueError: - return [Path(patch).as_posix() for patch in patches] - return [Path(os.path.relpath(patch, common_root)).as_posix() for patch in patches] + return [f"patch/{Path(patch).name}" for patch in patches] def generate_source( @@ -69,9 +65,8 @@ def generate_source( ) -> dict[str, dict[str, Any]]: """Generate per-package release sources for a multi-recipe build. - Patch paths are rewritten relative to the closest shared ancestor of the working - directory and the patches themselves, because rattler-build resolves them - relative to the recipe. + Patches are referred to as ``patch/`` next to the recipe, because + rattler-build resolves them relative to the recipe; the template copies them. """ sources: dict[str, dict[str, Any]] = {} for shortname in vinca_conf["_selected_pkgs"]: @@ -92,10 +87,10 @@ def generate_source( entry: dict[str, Any] = source_reference(url=url, ref=ref, ref_type=ref_type) entry["target_directory"] = f"{package_name}/src/work" - patches = _package_patches(package_name, vinca_conf, platform) + patches = package_patches(package_name, vinca_conf, platform) if patches: print(patches) - entry["patches"] = _relative_patch_paths(patches) + entry["patches"] = _recipe_patch_paths(patches) sources[package_name] = entry mutex_recipe = generate_mutex_package_recipe(vinca_conf, distro) @@ -129,7 +124,7 @@ def generate_source_version( package_name = package_names[0] entry: dict[str, Any] = source_reference(url=url, ref=ref, ref_type=ref_type) entry["target_directory"] = f"{package_name}/src/work" - if patches := _package_patches(package_name, vinca_conf, platform): + if patches := package_patches(package_name, vinca_conf, platform): entry["patches"] = patches sources[package_name] = entry return sources diff --git a/vinca/template.py b/vinca/template.py index 6873139..4aea56a 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -8,7 +8,9 @@ from ruamel import yaml +from vinca import config from vinca.naming import get_package_prefix, is_legacy_compatibility_output +from vinca.sources import package_patches from vinca.utils import ( ensure_name_is_without_distro_prefix_and_with_underscores, get_pkg_additional_info, @@ -174,10 +176,16 @@ def write_recipe(source, outputs, vinca_conf, distro, single_file=True): file.dump(variants, stream) if meta.get("source") and meta["source"].get("patches"): + # the recipe refers to patch/; copy each from where it lives + files = { + Path(p).name: p + for p in package_patches( + package_name, vinca_conf, config.selected_platform or "" + ) + } for p in meta["source"]["patches"]: - patch_dir, _ = os.path.split(p) - os.makedirs(recipe_dir / patch_dir, exist_ok=True) - shutil.copyfile(p, recipe_dir / p) + os.makedirs((recipe_dir / p).parent, exist_ok=True) + shutil.copyfile(files.get(Path(p).name, p), recipe_dir / p) build_scripts = re.findall(r"'(.*?)'", meta["build"]["script"]) for script in build_scripts: diff --git a/vinca/test_sources.py b/vinca/test_sources.py index a092b5a..e61d73a 100644 --- a/vinca/test_sources.py +++ b/vinca/test_sources.py @@ -1,6 +1,4 @@ -import os - -from vinca.sources import _relative_patch_paths, generate_source, source_reference +from vinca.sources import _recipe_patch_paths, generate_source, source_reference class FakeDistro: @@ -58,24 +56,19 @@ def test_generate_source_combines_generic_and_platform_patches(tmp_path, monkeyp "url": "https://example.com/demo.tar.gz", "sha256": "abc123", "target_directory": "ros-humble-demo/src/work", - "patches": ["patches/demo.patch", "patches/demo.linux.patch"], + "patches": ["patch/demo.patch", "patch/demo.linux.patch"], } -def test_relative_patch_paths_use_posix_separators(tmp_path, monkeypatch): - monkeypatch.chdir(tmp_path) - patch = tmp_path / "patches" / "demo.patch" - - assert _relative_patch_paths([str(patch)]) == ["patches/demo.patch"] - - -def test_relative_patch_paths_fall_back_to_absolute_without_a_shared_root(monkeypatch): - def explode(_paths): - raise ValueError("paths don't have the same drive") - - monkeypatch.setattr(os.path, "commonpath", explode) - - assert _relative_patch_paths(["/elsewhere/demo.patch"]) == ["/elsewhere/demo.patch"] +def test_recipe_refers_to_patches_next_to_it(): + # wherever the patches live (another configuration's directory, another drive), + # the recipe refers to patch/; the template copies them there + assert _recipe_patch_paths( + ["/elsewhere/patch/demo.patch", "C:/x/demo.win.patch"] + ) == [ + "patch/demo.patch", + "patch/demo.win.patch", + ] def test_generate_source_skips_already_built_package(): From 9e89d1e3a222149596887f62d07f3fa184c9c376 Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Thu, 8 Oct 2026 10:17:45 +1000 Subject: [PATCH 4/5] Resolve ros_distro selectors in package tests Selectors that test ros_distro (e.g. if: ros_distro in ["humble", "jazzy"]) are resolved when a test file is put into a recipe, so one test file serves several distributions; platform selectors stay for rattler-build. Co-Authored-By: Claude Opus 5.5 --- README.md | 18 +++++++++++++++ vinca/main.py | 5 ++++- vinca/template.py | 5 ++++- vinca/test_distro_selectors.py | 32 ++++++++++++++++++++++++++ vinca/v1_selectors.py | 41 ++++++++++++++++++++++++++++++++++ 5 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 vinca/test_distro_selectors.py diff --git a/README.md b/README.md index 6303bc4..43e0ece 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,24 @@ A base configuration can itself extend another one. Paths (`patch_dir`, `rosdistro_snapshot`, `rosdistro_additional_recipes`, `conda_index` files) are relative to the file that sets them. +## Package tests + +`tests/.yaml` (next to `vinca.yaml`, or next to a configuration it extends) +becomes the `tests` section of the package's recipe. Selectors that test `ros_distro` +are resolved by vinca, so one test file can serve several distributions; all other +selectors are left to rattler-build: + +```yaml +tests: + - script: + - if: ros_distro in ["humble", "jazzy"] + then: + - if: osx + then: export ROS_LOCALHOST_ONLY=1 && launch_test smoke_test.py + else: + - launch_test smoke_test.py +``` + ## Package naming The optional `package_name_mode` setting controls the transition from legacy distro-qualified names such as `ros-humble-rclcpp` to ROS-major-version names such as `ros2-rclcpp`: diff --git a/vinca/main.py b/vinca/main.py index 8aa17e2..8d08eca 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -51,6 +51,7 @@ source_reference, ) from .template import write_recipe, write_recipe_package +from .v1_selectors import evaluate_distro_selectors unsatisfied_deps = set() distro = None @@ -490,7 +491,9 @@ def parse_package(pkg, distro, vinca_conf, path): if test := vinca_conf.get("_tests", {}).get(final_name): # parse as yaml text = test.read_text() - test_content = ruamel.yaml.safe_load(text) + test_content = evaluate_distro_selectors( + ruamel.yaml.safe_load(text), ros_distro=config.ros_distro + ) recipe["test"] = test_content for p in pkg["authors"]: diff --git a/vinca/template.py b/vinca/template.py index 4aea56a..9aae91a 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -16,6 +16,7 @@ get_pkg_additional_info, get_pkg_build_number, ) +from vinca.v1_selectors import evaluate_distro_selectors from vinca.variants import VariantsMode, get_recipe_variants, get_variants_mode TEMPLATE = """\ @@ -143,7 +144,9 @@ def write_recipe(source, outputs, vinca_conf, distro, single_file=True): ): print("Using test: ", test) text = test.read_text() - test_content = yaml.safe_load(text) + test_content = evaluate_distro_selectors( + yaml.safe_load(text), ros_distro=config.ros_distro + ) meta["tests"] = test_content["tests"] recipe_dir = (Path("recipes") / package_name).absolute() diff --git a/vinca/test_distro_selectors.py b/vinca/test_distro_selectors.py new file mode 100644 index 0000000..7ebc0c9 --- /dev/null +++ b/vinca/test_distro_selectors.py @@ -0,0 +1,32 @@ +from vinca.v1_selectors import evaluate_distro_selectors + +TEST = { + "tests": [ + { + "script": [ + { + "if": 'ros_distro in ["humble", "jazzy"]', + "then": [{"if": "osx", "then": "old-osx"}], + "else": [{"if": "osx", "then": "new-osx"}], + }, + {"if": 'ros_distro == "humble"', "then": ["humble-only"]}, + {"if": "linux", "then": "linux-cmd"}, + ] + } + ] +} + + +def test_distro_selectors_are_resolved_platform_ones_kept(): + humble = evaluate_distro_selectors(TEST, ros_distro="humble") + assert humble["tests"][0]["script"] == [ + {"if": "osx", "then": "old-osx"}, + "humble-only", + {"if": "linux", "then": "linux-cmd"}, + ] + + rolling = evaluate_distro_selectors(TEST, ros_distro="rolling") + assert rolling["tests"][0]["script"] == [ + {"if": "osx", "then": "new-osx"}, + {"if": "linux", "then": "linux-cmd"}, + ] diff --git a/vinca/v1_selectors.py b/vinca/v1_selectors.py index 49720c7..b9da3c8 100644 --- a/vinca/v1_selectors.py +++ b/vinca/v1_selectors.py @@ -168,3 +168,44 @@ def evaluate_selectors( ctx = _platform_flags(target_platform) return _process_node(copy.deepcopy(data), ctx) + + +def _process_distro_node(node: Any, ctx: Mapping[str, Any]) -> Any | None: + """Like _process_node, but only for selectors that mention ``ros_distro``; + other selectors (e.g. platform ones) are left for rattler-build.""" + if ( + isinstance(node, Mapping) + and "if" in node + and "then" in node + and "ros_distro" in str(node["if"]) + ): + branch = "then" if _eval_condition(str(node["if"]), ctx) else "else" + if branch in node: + return _process_distro_node(node[branch], ctx) + return None + if isinstance(node, MutableMapping): + mapping_out: dict[str, Any] = {} + for k, v in node.items(): + new = _process_distro_node(v, ctx) + if new is not None: + mapping_out[k] = new + return mapping_out + if isinstance(node, Sequence) and not isinstance(node, (str, bytes)): + sequence_out: list[Any] = [] + for item in node: + new = _process_distro_node(item, ctx) + if new is None: + continue + if isinstance(new, list): + sequence_out.extend(new) + else: + sequence_out.append(new) + return sequence_out + return node + + +def evaluate_distro_selectors(data: Any, *, ros_distro: str | None) -> Any: + """Return *data* with the selectors that test ``ros_distro`` evaluated, e.g. + ``- if: ros_distro in ["humble", "jazzy"]``, so one test file can serve several + distributions. Selectors without ``ros_distro`` are kept unchanged.""" + return _process_distro_node(copy.deepcopy(data), {"ros_distro": ros_distro}) From e5b291cfb6a37563779675b352f9ee0f85a50a5e Mon Sep 17 00:00:00 2001 From: Tobias Fischer Date: Thu, 8 Oct 2026 13:45:28 +1000 Subject: [PATCH 5/5] Read package tests as UTF-8 Co-Authored-By: Claude Opus 5.5 --- vinca/main.py | 2 +- vinca/template.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index 8d08eca..2df4356 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -490,7 +490,7 @@ def parse_package(pkg, distro, vinca_conf, path): if test := vinca_conf.get("_tests", {}).get(final_name): # parse as yaml - text = test.read_text() + text = test.read_text(encoding="utf-8") test_content = evaluate_distro_selectors( ruamel.yaml.safe_load(text), ros_distro=config.ros_distro ) diff --git a/vinca/template.py b/vinca/template.py index 9aae91a..d03d4ee 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -143,7 +143,7 @@ def write_recipe(source, outputs, vinca_conf, distro, single_file=True): test := vinca_conf["_tests"].get(package_name) ): print("Using test: ", test) - text = test.read_text() + text = test.read_text(encoding="utf-8") test_content = evaluate_distro_selectors( yaml.safe_load(text), ros_distro=config.ros_distro )