From db78614bda6739e4acff565bb260bf069906bd2a Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 20 Aug 2026 19:09:26 +0200 Subject: [PATCH 1/5] refactor: split recipe generation into focused modules --- vinca/configuration.py | 120 ++++++ vinca/main.py | 803 +++--------------------------------- vinca/mutex.py | 107 +++++ vinca/platforms.py | 33 ++ vinca/recipes.py | 405 ++++++++++++++++++ vinca/sources.py | 125 ++++++ vinca/test_configuration.py | 55 +++ vinca/test_mutex.py | 72 ++++ vinca/test_platforms.py | 34 ++ vinca/test_recipes.py | 30 ++ vinca/test_sources.py | 74 ++++ 11 files changed, 1110 insertions(+), 748 deletions(-) create mode 100644 vinca/configuration.py create mode 100644 vinca/mutex.py create mode 100644 vinca/platforms.py create mode 100644 vinca/recipes.py create mode 100644 vinca/sources.py create mode 100644 vinca/test_configuration.py create mode 100644 vinca/test_mutex.py create mode 100644 vinca/test_platforms.py create mode 100644 vinca/test_recipes.py create mode 100644 vinca/test_sources.py diff --git a/vinca/configuration.py b/vinca/configuration.py new file mode 100644 index 0000000..d813c19 --- /dev/null +++ b/vinca/configuration.py @@ -0,0 +1,120 @@ +"""Loading and normalization of ``vinca.yaml`` configuration.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ruamel.yaml import YAML + +from vinca import config +from vinca.naming import get_package_name_mode +from vinca.resolve import get_conda_index +from vinca.utils import add_package_name_variants +from vinca.v1_selectors import evaluate_selectors + +_PATCH_PLATFORMS = ("osx", "linux", "win", "emscripten") + + +def _load_yaml(path: Path) -> Any: + yaml = YAML() + with path.open(encoding="utf-8") as stream: + return yaml.load(stream) + + +def _load_selected_yaml(path: Path, target_platform: str) -> Any: + return evaluate_selectors(_load_yaml(path), target_platform=target_platform) + + +def _normalize_conda_indexes(indexes: list[str]) -> list[str]: + return [ + str(Path(index).absolute()) if Path(index).is_file() else index + for index in indexes + ] + + +def _discover_patches( + patch_dir: Path, ros_distro: str +) -> dict[str, dict[str, list[str]]]: + patches: dict[str, dict[str, list[str]]] = {} + for path in sorted(patch_dir.glob("*.patch")): + parts = path.name.split(".") + package_patches = patches.setdefault( + parts[0], {"any": [], **{platform: [] for platform in _PATCH_PLATFORMS}} + ) + destination_platforms = ["any"] + if len(parts) == 3: + if parts[1] in _PATCH_PLATFORMS: + destination_platforms = [parts[1]] + elif parts[1] == "unix": + destination_platforms = ["linux", "osx"] + for platform in destination_platforms: + package_patches[platform].append(str(path)) + + add_package_name_variants(patches, ros_distro) + return patches + + +def _discover_tests( + config_dir: Path, ros_distro: str +) -> tuple[dict[str, Path], dict[str, Path]]: + test_dir = config_dir / "tests" + tests = {path.stem: path for path in test_dir.glob("*.yaml")} + test_folders = {path.name: path for path in test_dir.glob("*") if path.is_dir()} + add_package_name_variants(tests, ros_distro) + add_package_name_variants(test_folders, ros_distro) + return tests, test_folders + + +def read_snapshot( + vinca_conf: dict[str, Any], +) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: + """Load the primary and optional additional package snapshots.""" + snapshot_path = vinca_conf.get("rosdistro_snapshot") + if not snapshot_path: + return None, None + + snapshot = _load_yaml(Path(snapshot_path)) or {} + additional_path = vinca_conf.get("rosdistro_additional_recipes") + additional = _load_yaml(Path(additional_path)) or {} if additional_path else None + if additional: + snapshot.update(additional) + return snapshot, additional + + +def read_vinca_yaml(filepath: str | Path, target_platform: str) -> dict[str, Any]: + """Read a vinca configuration and populate its derived internal fields.""" + filepath = Path(filepath) + config_dir = filepath.parent + vinca_conf = _load_selected_yaml(filepath, target_platform) + vinca_conf["package_name_mode"] = get_package_name_mode(vinca_conf).value + vinca_conf["conda_index"] = _normalize_conda_indexes(vinca_conf["conda_index"]) + + patch_dir = Path(vinca_conf["patch_dir"]).absolute() + vinca_conf["_patch_dir"] = patch_dir + vinca_conf["_patches"] = _discover_patches(patch_dir, vinca_conf["ros_distro"]) + vinca_conf["_tests"], vinca_conf["_test_folders"] = _discover_tests( + config_dir, vinca_conf["ros_distro"] + ) + + dependencies_path = patch_dir / "dependencies.yaml" + if dependencies_path.exists(): + vinca_conf["depmods"] = _load_selected_yaml(dependencies_path, target_platform) + vinca_conf["depmods"] = vinca_conf.get("depmods") or {} + + config.ros_distro = vinca_conf["ros_distro"] + config.skip_testing = vinca_conf.get("skip_testing", True) + vinca_conf["_conda_indexes"] = get_conda_index(vinca_conf, str(config_dir)) + vinca_conf["trigger_new_versions"] = vinca_conf.get("trigger_new_versions", False) + + additional_info_path = config_dir / "pkg_additional_info.yaml" + vinca_conf["_pkg_additional_info"] = ( + _load_selected_yaml(additional_info_path, target_platform) + if additional_info_path.exists() + else {} + ) + + snapshot, additional = read_snapshot(vinca_conf) + vinca_conf["_snapshot"] = snapshot or {} + vinca_conf["_additional_packages_snapshot"] = additional or {} + return vinca_conf diff --git a/vinca/main.py b/vinca/main.py index 4325d28..3a9869d 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -1,38 +1,53 @@ #!/usr/bin/env python import argparse -import catkin_pkg -import sys -import os import glob -import platform +import os +import sys + +import catkin_pkg import ruamel.yaml -from pathlib import Path -from vinca import __version__ -from .resolve import get_conda_index -from .resolve import resolve_pkgname -from .template import write_recipe, write_recipe_package +from vinca import __version__, config +from vinca.utils import get_pkg_build_number, get_repodata + +from .configuration import read_snapshot as read_snapshot +from .configuration import read_vinca_yaml as load_vinca_yaml from .distro import Distro +from .mutex import ( + generate_mutex_package_recipe as generate_mutex_package_recipe, +) +from .mutex import ( + get_mutex_package_dependency as get_mutex_package_dependency, +) +from .mutex import ( + parse_mutex_package_config as parse_mutex_package_config, +) +from .mutex import ( + should_skip_mutex_package as should_skip_mutex_package, +) from .naming import ( generate_legacy_compatibility_output, get_package_name, - get_package_name_mode, - get_package_prefix, is_legacy_compatibility_output, - normalize_package_dependency, ) -from .v1_selectors import evaluate_selectors - -from vinca import config -from vinca.utils import ( - add_package_name_variants, - get_repodata, - get_pkg_build_number, - get_pkg_additional_info, - is_dummy_metapackage, +from .platforms import get_conda_subdir as detect_conda_subdir +from .recipes import generate_output as build_output +from .recipes import get_depmods as get_depmods +from .resolve import resolve_pkgname +from .sources import ( + generate_fat_source as build_fat_source, +) +from .sources import ( + generate_source as build_source, +) +from .sources import ( + generate_source_version as build_source_version, +) +from .sources import ( + source_reference, ) -from vinca.license_utils import convert_to_spdx_license +from .template import write_recipe, write_recipe_package unsatisfied_deps = set() distro = None @@ -46,25 +61,9 @@ def ensure_list(obj): def get_conda_subdir(): - if config.parsed_args.platform: - return config.parsed_args.platform - - sys_platform = sys.platform - machine = platform.machine() - if sys_platform.startswith("linux"): - if machine == "aarch64": - return "linux-aarch64" - elif machine == "x86_64": - return "linux-64" - else: - raise RuntimeError("Unknown machine!") - elif sys_platform == "darwin": - if machine == "arm64": - return "osx-arm64" - else: - return "osx-64" - elif sys_platform == "win32": - return "win-64" + """Return the configured target platform, or detect the host platform.""" + selected = getattr(config.parsed_args, "platform", None) + return detect_conda_subdir(selected) def parse_command_line(argv): @@ -149,134 +148,9 @@ def parse_command_line(argv): return arguments -def get_depmods(vinca_conf, pkg_name, distro): - depmods = vinca_conf["depmods"].get(pkg_name, {}) - rm_deps, add_deps = ( - {"build": [], "host": [], "run": []}, - {"build": [], "host": [], "run": []}, - ) - - for dep_type in ["build", "host", "run"]: - for dependency in depmods.get("remove_" + dep_type, []): - rm_deps[dep_type].append( - normalize_package_dependency(dependency, distro, vinca_conf) - ) - - for dependency in depmods.get("add_" + dep_type, []): - add_deps[dep_type].append( - normalize_package_dependency(dependency, distro, vinca_conf) - ) - - return rm_deps, add_deps - - def read_vinca_yaml(filepath): - yaml = ruamel.yaml.YAML() - vinca_conf = evaluate_selectors( - yaml.load(open(filepath, "r")), target_platform=get_conda_subdir() - ) - vinca_conf["package_name_mode"] = get_package_name_mode(vinca_conf).value - - # normalize paths to absolute paths - conda_index = [] - for i in vinca_conf["conda_index"]: - if os.path.isfile(i): - conda_index.append(os.path.abspath(i)) - else: - conda_index.append(i) - - vinca_conf["conda_index"] = conda_index - patch_dir = Path(vinca_conf["patch_dir"]).absolute() - vinca_conf["_patch_dir"] = patch_dir - patches = {} - - for x in sorted(glob.glob(os.path.join(vinca_conf["_patch_dir"], "*.patch"))): - splitted = os.path.basename(x).split(".") - if splitted[0] not in patches: - patches[splitted[0]] = { - "any": [], - "osx": [], - "linux": [], - "win": [], - "emscripten": [], - } - if len(splitted) == 3: - if splitted[1] in ("osx", "linux", "win", "emscripten"): - patches[splitted[0]][splitted[1]].append(x) - continue - if splitted[1] == "unix": - patches[splitted[0]]["linux"].append(x) - patches[splitted[0]]["osx"].append(x) - continue - - patches[splitted[0]]["any"].append(x) - - add_package_name_variants(patches, vinca_conf["ros_distro"]) - vinca_conf["_patches"] = patches - - tests = {} - test_dir = Path(filepath).parent / "tests" - for x in test_dir.glob("*.yaml"): - tests[os.path.basename(x).split(".")[0]] = x - add_package_name_variants(tests, vinca_conf["ros_distro"]) - vinca_conf["_tests"] = tests - - test_folders = {path.name: path for path in test_dir.glob("*") if path.is_dir()} - add_package_name_variants(test_folders, vinca_conf["ros_distro"]) - vinca_conf["_test_folders"] = test_folders - - if (patch_dir / "dependencies.yaml").exists(): - vinca_conf["depmods"] = evaluate_selectors( - yaml.load(open(patch_dir / "dependencies.yaml")), - target_platform=get_conda_subdir(), - ) - if not vinca_conf.get("depmods"): - vinca_conf["depmods"] = {} - - config.ros_distro = vinca_conf["ros_distro"] - config.skip_testing = vinca_conf.get("skip_testing", True) - - vinca_conf["_conda_indexes"] = get_conda_index( - vinca_conf, os.path.dirname(filepath) - ) - - vinca_conf["trigger_new_versions"] = vinca_conf.get("trigger_new_versions", False) - - if (Path(filepath).parent / "pkg_additional_info.yaml").exists(): - vinca_conf["_pkg_additional_info"] = evaluate_selectors( - yaml.load(open(Path(filepath).parent / "pkg_additional_info.yaml")), - target_platform=get_conda_subdir(), - ) - else: - vinca_conf["_pkg_additional_info"] = {} - - # snapshot contains both rosdistro_snapshot.yaml and - # rosdistro_additional_recipes.yaml - snapshot, additional_packages_snapshot = read_snapshot(vinca_conf) - - # Store additional_packages_snapshot in vinca_conf for template access - vinca_conf["_snapshot"] = snapshot or {} - vinca_conf["_additional_packages_snapshot"] = additional_packages_snapshot or {} - - return vinca_conf - - -def read_snapshot(vinca_conf): - if "rosdistro_snapshot" not in vinca_conf: - return None, None - - yaml = ruamel.yaml.YAML() - # load primary snapshot - snapshot = yaml.load(open(vinca_conf["rosdistro_snapshot"], "r")) or {} - # if additional snapshot file specified, load and merge - additional_key = "rosdistro_additional_recipes" - additional = None - if additional_key in vinca_conf and vinca_conf[additional_key]: - additional = yaml.load(open(vinca_conf[additional_key], "r")) or {} - # merge additional entries, overriding or adding - snapshot.update(additional) - - return snapshot, additional + """Load configuration for the currently selected conda platform.""" + return load_vinca_yaml(filepath, target_platform=get_conda_subdir()) def should_skip_output(output, vinca_conf): @@ -308,352 +182,14 @@ def append_output_with_compatibility( def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): - if not all_pkgs: - all_pkgs = [] - - if pkg_shortname not in vinca_conf["_selected_pkgs"]: - return None - - pkg_names = resolve_pkgname(pkg_shortname, vinca_conf, distro) - if not pkg_names: - return None - - # handle dummy recipe generation for vendored packages - output = {} - if is_dummy_metapackage(pkg_shortname, vinca_conf): - pkg_additional_info = get_pkg_additional_info(pkg_shortname, vinca_conf) - gen = pkg_additional_info["generate_dummy_package_with_run_deps"] - dep_name = gen.get("dep_name") - # dep_name is required to specify which dependency to pin in the dummy recipe - if not dep_name: - runerr = f"Missing 'dep_name' for dummy recipe of {pkg_shortname}" - raise RuntimeError(runerr) - # upper_bound is required for dummy recipe pinning - upper_bound = gen.get("upper_bound") - if not upper_bound: - upper_bound = gen.get("max_pin") - if not upper_bound: - runerr = f"Missing 'upper_bound' or 'max_pin' for dummy recipe of {pkg_shortname}" - raise RuntimeError(runerr) - # Compute rattler-build-compatible version constraint based on upper_bound: - # - lower bound: allow the exact package version (or override_version if specified) - # - upper bound: increment the segment of version defined by upper_bound length, - # then append 'a0' to ensure the constraint captures pre-releases correctly - - # Use override_version if specified, otherwise use the ROS package version - dummy_package_version = gen.get("override_version", version) - lower = dummy_package_version - parts = [int(p) for p in lower.split(".")] - seg = len(upper_bound.split(".")) - upper_parts = parts[:seg] - upper_parts[-1] += 1 - upper_parts += [0] * (len(parts) - seg) - upper = ".".join(str(p) for p in upper_parts) + "a0" - constraint = f"{dep_name} >={lower}, <{upper}" - output = { - "package": {"name": pkg_names[0], "version": dummy_package_version}, - "build": { - "number": get_pkg_build_number( - vinca_conf.get("build_number", 0), pkg_names[0], vinca_conf - ), - "script": "", - }, - "requirements": {"build": [], "host": [], "run": [constraint]}, - } - else: - # If the package is not a dummy recipe, we generate a full recipe - output = { - "package": {"name": pkg_names[0], "version": version}, - "requirements": { - "build": [ - "${{ compiler('cxx') }}", - "${{ compiler('c') }}", - { - "if": "target_platform!='emscripten-wasm32'", - "then": ["${{ stdlib('c') }}"], - }, - "ninja", - "python", - "setuptools", - "git", - "git-lfs", - {"if": "unix", "then": ["patch", "make", "coreutils"]}, - {"if": "win", "then": ["m2-patch"]}, - {"if": "osx", "then": ["tapi"]}, - {"if": "build_platform != target_platform", "then": ["pkg-config"]}, - "cmake", - "cython", - { - "if": "build_platform != target_platform", - "then": [ - "python", - "cross-python_${{ target_platform }}", - "numpy", - ], - }, - ], - "host": [ - {"if": "build_platform == target_platform", "then": ["pkg-config"]}, - "python", - "numpy", - "pip", - ], - "run": [], - }, - "build": {"script": ""}, - } - - xml = distro.get_release_package_xml(pkg_shortname) - - # If the snapshot is not aligned with the latest rosdistro (for example if a package is removed, - # see https://github.com/RoboStack/ros-jazzy/pull/107#issuecomment-3338962041), get_release_package_xml can return none, - # in that case we can just skip the package, remove if https://github.com/RoboStack/vinca/issues/93 is fixed - if not xml: - print( - "Skip " - + pkg_shortname - + " as it is present in our snapshot, but not in the latest rosdistro cache." - ) - return None - - pkg = catkin_pkg.package.parse_package_string(xml) - - pkg.evaluate_conditions(os.environ) - - resolved_python = resolve_pkgname("python", vinca_conf, distro) - output["requirements"]["run"].extend(resolved_python) - output["requirements"]["host"].extend(resolved_python) - - is_dummy_package = is_dummy_metapackage(pkg_shortname, vinca_conf) - build_type = pkg.get_build_type() - - if is_dummy_package: - # Dummy recipes do not actually build anything, so we set the script to empty - output["build"]["script"] = "" - elif build_type in ["cmake", "catkin"]: - output["build"]["script"] = ( - "${{ '$RECIPE_DIR/build_catkin.sh' if unix or wasm32 else '%RECIPE_DIR%\\\\bld_catkin.bat' }}" - ) - elif build_type in ["ament_cmake"]: - output["build"]["script"] = ( - "${{ '$RECIPE_DIR/build_ament_cmake.sh' if unix or wasm32 else '%RECIPE_DIR%\\\\bld_ament_cmake.bat' }}" - ) - elif build_type in ["ament_python"]: - output["build"]["script"] = ( - "${{ '$RECIPE_DIR/build_ament_python.sh' if unix or wasm32 else '%RECIPE_DIR%\\\\bld_ament_python.bat' }}" - ) - resolved_setuptools = resolve_pkgname("python-setuptools", vinca_conf, distro) - output["requirements"]["host"].extend(resolved_setuptools) - else: - print(f"Unknown build type for {pkg_shortname}: {build_type}") - return None - - if not is_dummy_package and build_type in ["cmake", "catkin", "ament_cmake"]: - output["requirements"]["build"].append( - { - "if": "osx", - "then": [ - "clang-tools ${{ (cxx_compiler_version ~ '.*') if " - "cxx_compiler_version is defined else '*' }}" - ], - } - ) - - if vinca_conf.get("mutex_package"): - mutex_dep = get_mutex_package_dependency(vinca_conf, distro) - if mutex_dep: - output["requirements"]["host"].append(mutex_dep) - output["requirements"]["run"].append(mutex_dep) - - if not distro.check_ros1() and pkg_shortname not in [ - "ament_cmake_core", - "ament_package", - "ros_workspace", - "ros_environment", - ]: - package_prefix = get_package_prefix(distro, vinca_conf) - output["requirements"]["host"].append(f"{package_prefix}-ros-environment") - output["requirements"]["host"].append(f"{package_prefix}-ros-workspace") - output["requirements"]["run"].append(f"{package_prefix}-ros-workspace") - - rm_deps, add_deps = get_depmods(vinca_conf, pkg.name, distro) - gdeps = [] - if pkg.group_depends: - for gdep in pkg.group_depends: - gdep.extract_group_members(all_pkgs) - gdeps += gdep.members - - build_tool_deps = pkg.buildtool_depends - build_tool_deps += pkg.buildtool_export_depends - build_tool_deps = [d.name for d in build_tool_deps if d.evaluated_condition] - - build_deps = pkg.build_depends - build_deps += pkg.build_export_depends - build_deps += pkg.test_depends - build_deps = [d.name for d in build_deps if d.evaluated_condition] - build_deps += gdeps - - # we stick some build tools into the `build` section to make cross compilation work - # right now it's only `git`. - for dep in build_tool_deps: - resolved_dep = resolve_pkgname(dep, vinca_conf, distro) - if not resolved_dep: - unsatisfied_deps.add(dep) - continue - - if "git" in resolved_dep: - output["requirements"]["build"].extend(resolved_dep) - else: - # remove duplicate cmake - if dep not in ["cmake"]: - build_deps.append(dep) - - # Hack to add cyclonedds into build for cross compilation - if pkg_shortname == "cyclonedds" or "cyclonedds" in ( - build_deps + build_tool_deps - ): - output["requirements"]["build"].append( - { - "if": "build_platform != target_platform", - "then": [f"{get_package_prefix(distro, vinca_conf)}-cyclonedds"], - } - ) - - for dep in build_deps: - resolved_dep = resolve_pkgname(dep, vinca_conf, distro) - if not resolved_dep: - unsatisfied_deps.add(dep) - continue - output["requirements"]["host"].extend(resolved_dep) - - run_deps = pkg.run_depends - run_deps += pkg.exec_depends - run_deps += pkg.build_export_depends - run_deps += pkg.buildtool_export_depends - run_deps = [d.name for d in run_deps if d.evaluated_condition] - run_deps += gdeps - - for dep in run_deps: - resolved_dep = resolve_pkgname(dep, vinca_conf, distro, is_rundep=True) - if not resolved_dep: - unsatisfied_deps.add(dep) - continue - output["requirements"]["run"].extend(resolved_dep) - - for dep_type in ["build", "host", "run"]: - for dep in add_deps[dep_type]: - output["requirements"][dep_type].append(dep) - for dep in rm_deps[dep_type]: - while dep in output["requirements"][dep_type]: - output["requirements"][dep_type].remove(dep) - - def sortkey(k): - if isinstance(k, dict): - return list(k.values())[0] - return k - - # For Emscripten, only install cmake as a build dependency. - # This should be ok as cmake is only really needed during builds, not when running packages. - if "cmake" in output["requirements"]["run"]: - output["requirements"]["run"].remove("cmake") - output["requirements"]["run"].append( - {"if": "target_platform != 'emscripten-wasm32'", "then": ["cmake"]} - ) - - if "cmake" in output["requirements"]["host"]: - output["requirements"]["host"].remove("cmake") - if "cmake" not in output["requirements"]["build"]: - output["requirements"]["build"].append("cmake") - - package_prefix = get_package_prefix(distro, vinca_conf) - mimick_vendor_name = f"{package_prefix}-mimick-vendor" - if mimick_vendor_name in output["requirements"]["build"]: - output["requirements"]["build"].remove(mimick_vendor_name) - output["requirements"]["build"].append( - { - "if": "target_platform != 'emscripten-wasm32'", - "then": [mimick_vendor_name], - } - ) - - if mimick_vendor_name in output["requirements"]["host"]: - output["requirements"]["host"].remove(mimick_vendor_name) - output["requirements"]["build"].append( - { - "if": "target_platform != 'emscripten-wasm32'", - "then": [mimick_vendor_name], - } - ) - - rosidl_generators_name = f"{package_prefix}-rosidl-default-generators" - if rosidl_generators_name in output["requirements"]["host"]: - output["requirements"]["build"].append( - { - "if": "target_platform == 'emscripten-wasm32'", - "then": [rosidl_generators_name], - } - ) - - output["requirements"]["run"] = sorted(output["requirements"]["run"], key=sortkey) - output["requirements"]["host"] = sorted(output["requirements"]["host"], key=sortkey) - - pybind11_vendor_name = f"{package_prefix}-pybind11-vendor" - if pybind11_vendor_name in output["requirements"]["host"]: - output["requirements"]["host"] += ["pybind11"] - if "pybind11" in output["requirements"]["host"]: - output["requirements"]["build"] += [ - {"if": "build_platform != target_platform", "then": ["pybind11"]} - ] - if "qt-main" in output["requirements"]["host"]: - output["requirements"]["build"] += [ - {"if": "build_platform != target_platform", "then": ["qt-main"]} - ] - # pyqt-builder + git + doxygen must be in build, not host for cross-compile - pkgs_move_to_build = ["pyqt-builder", "git", "doxygen", "git-lfs"] - for pkg_move_to_build in pkgs_move_to_build: - if pkg_move_to_build in output["requirements"]["host"]: - output["requirements"]["build"] += [ - {"if": "build_platform != target_platform", "then": [pkg_move_to_build]} - ] - while pkg_move_to_build in output["requirements"]["host"]: - output["requirements"]["host"].remove(pkg_move_to_build) - output["requirements"]["host"] += [ - {"if": "build_platform == target_platform", "then": [pkg_move_to_build]} - ] - - # remove duplicates - for dep_type in ["build", "host", "run"]: - tmp_nonduplicate = [] - [ - tmp_nonduplicate.append(x) - for x in output["requirements"][dep_type] - if x not in tmp_nonduplicate - ] - output["requirements"][dep_type] = tmp_nonduplicate - - # Add "about" section with license and package metadata - output["about"] = {} - - # Add URLs from package.xml based on their type - for u in pkg.urls: - if u.type == "website": - output["about"]["homepage"] = u.url - elif u.type == "repository": - output["about"]["repository"] = u.url - - # Add license if available (convert to SPDX format) - if pkg.licenses: - spdx_license = convert_to_spdx_license( - [str(lic) for lic in pkg.licenses], package_name=pkg_shortname - ) - if spdx_license: - output["about"]["license"] = spdx_license - - # Add summary/description if available - if pkg.description: - output["about"]["summary"] = pkg.description - - return output + return build_output( + pkg_shortname, + vinca_conf, + distro, + version, + all_packages=all_pkgs, + unsatisfied=unsatisfied_deps, + ) def generate_outputs(distro, vinca_conf): @@ -722,131 +258,20 @@ def generate_outputs_version(distro, vinca_conf): def _source_reference(*, url, ref, ref_type): - """Return the source keys locating url at ref, for a git repository or an archive. - - An archive is identified by a sha256 ref_type and is fetched by URL, so it carries - a checksum instead of a git revision. - """ - if ref_type == "sha256": - return {"url": url, "sha256": ref} - return {"git": url, ref_type: ref} + """Backward-compatible entry point for source reference generation.""" + return source_reference(url=url, ref=ref, ref_type=ref_type) def generate_source(distro, vinca_conf): - source = {} - for pkg_shortname in vinca_conf["_selected_pkgs"]: - if not distro.check_package(pkg_shortname): - print(f"Could not generate source for {pkg_shortname}") - continue - # skip cloning source for dummy recipes - if is_dummy_metapackage(pkg_shortname, vinca_conf): - continue - url, ref, ref_type = distro.get_released_repo(pkg_shortname) - entry = _source_reference(url=url, ref=ref, ref_type=ref_type) - pkg_names = resolve_pkgname(pkg_shortname, vinca_conf, distro) - pkg_version = distro.get_version(pkg_shortname) - print("Checking ", pkg_shortname, pkg_version) - if not pkg_names: - continue - if vinca_conf.get("trigger_new_versions"): - if (pkg_names[0], pkg_version) in vinca_conf["skip_built_packages"]: - continue - else: - if pkg_names[0] in vinca_conf["skip_built_packages"]: - continue - pkg_name = pkg_names[0] - entry["target_directory"] = "%s/src/work" % pkg_name - - patches = [] - pd = vinca_conf["_patches"].get(pkg_name) - if pd: - patches.extend(pd["any"]) - - # find specific patches - plat = get_conda_subdir().split("-")[0] - patches.extend(pd[plat]) - if len(patches): - print(patches) - common_prefix = os.path.commonprefix((os.getcwd(), patches[0])) - print(common_prefix) - entry["patches"] = [os.path.relpath(p, common_prefix) for p in patches] - - source[pkg_name] = entry - - # Generate empty source for mutex package (if generated) since it's a meta-package - mutex_recipe = generate_mutex_package_recipe(vinca_conf, distro) - if mutex_recipe: - # Check if mutex package should be skipped - mutex_name = mutex_recipe["package"]["name"] - mutex_version = mutex_recipe["package"]["version"] - - if not should_skip_mutex_package(vinca_conf, mutex_name, mutex_version): - source[mutex_name] = {} - - return source + return build_source(distro, vinca_conf, get_conda_subdir()) def generate_source_version(distro, vinca_conf): - source = {} - for pkg_shortname in vinca_conf["_selected_pkgs"]: - if not distro.check_package(pkg_shortname): - print(f"Could not generate source for {pkg_shortname}") - continue - - url, ref, ref_type = distro.get_released_repo(pkg_shortname) - - entry = _source_reference(url=url, ref=ref, ref_type=ref_type) - pkg_names = resolve_pkgname(pkg_shortname, vinca_conf, distro) - version = distro.get_version(pkg_shortname) - if vinca_conf.get("trigger_new_versions"): - if ( - not pkg_names - or (pkg_names[0], version) in vinca_conf["skip_built_packages"] - ): - continue - else: - if not pkg_names or pkg_names[0] in vinca_conf["skip_built_packages"]: - continue - pkg_name = pkg_names[0] - entry["target_directory"] = "%s/src/work" % pkg_name - - patches = [] - pd = vinca_conf["_patches"].get(pkg_name) - if pd: - patches.extend(pd["any"]) - - # find specific patches - plat = get_conda_subdir().split("-")[0] - patches.extend(pd[plat]) - if len(patches): - entry["patches"] = patches - - source[pkg_name] = entry - - return source + return build_source_version(distro, vinca_conf, get_conda_subdir()) def generate_fat_source(distro, vinca_conf): - source = [] - for pkg_shortname in vinca_conf["_selected_pkgs"]: - if not distro.check_package(pkg_shortname): - print(f"Could not generate source for {pkg_shortname}") - continue - - url, ref, ref_type = distro.get_released_repo(pkg_shortname) - entry = _source_reference(url=url, ref=ref, ref_type=ref_type) - pkg_names = resolve_pkgname(pkg_shortname, vinca_conf, distro) - if not pkg_names: - continue - pkg_name = pkg_names[0] - entry["target_directory"] = "src/%s" % pkg_name - patch_path = os.path.join(vinca_conf["_patch_dir"], "%s.patch" % pkg_name) - if os.path.exists(patch_path): - entry["patches"] = [ - "%s/%s" % (vinca_conf["patch_dir"], "%s.patch" % pkg_name) - ] - source.append(entry) - return source + return build_fat_source(distro, vinca_conf) def get_selected_packages(distro, vinca_conf): @@ -941,124 +366,6 @@ def get_selected_packages(distro, vinca_conf): return result -def parse_mutex_package_config(vinca_conf): - """Parse and validate mutex package configuration. - - Returns: - dict: Parsed mutex configuration with all required fields, or None if mutex_package is a string - - Raises: - ValueError: If mutex_package is a dict but missing required fields - """ - mutex_pkg = vinca_conf.get("mutex_package") - if not mutex_pkg: - return None - - if isinstance(mutex_pkg, str): - # Backward compatibility: return None to indicate string format - return None - - if isinstance(mutex_pkg, dict): - # Validate required fields - required_fields = ["name", "version", "upper_bound", "run_constraints"] - missing_fields = [field for field in required_fields if field not in mutex_pkg] - - if missing_fields: - raise ValueError( - f"mutex_package configuration is missing required fields: {missing_fields}" - ) - - # Return validated config with build_number from vinca_conf if not specified - config = dict(mutex_pkg) - if "build_number" not in config: - config["build_number"] = vinca_conf.get("build_number", 1) - - return config - - raise ValueError( - f"mutex_package must be either a string or a dictionary, got {type(mutex_pkg)}" - ) - - -def get_mutex_package_dependency(vinca_conf, distro): - """Get the mutex package dependency string, handling both string and dict formats.""" - mutex_pkg = vinca_conf.get("mutex_package") - if not mutex_pkg: - return None - - if isinstance(mutex_pkg, str): - # Backward compatibility: return the string as-is - return mutex_pkg - - # Try to parse as dict configuration - try: - config = parse_mutex_package_config(vinca_conf) - if config is None: - # This shouldn't happen since we already checked isinstance(mutex_pkg, str) above - return None - - # New format: construct the dependency string - # Compute the pin from version and upper_bound - version_parts = config["version"].split(".") - upper_bound_parts = config["upper_bound"].split(".") - - # Take as many version parts as specified by upper_bound - pin_parts = version_parts[: len(upper_bound_parts)] - pin = ".".join(pin_parts) + ".*" - - return f"{config['name']} {pin} {distro.name}_*" - except ValueError as e: - raise ValueError(f"Error parsing mutex_package configuration: {e}") - - return None - - -def should_skip_mutex_package(vinca_conf, mutex_name, mutex_version): - """Check if the mutex package should be skipped based on skip_built_packages logic.""" - if vinca_conf.get("trigger_new_versions"): - return (mutex_name, mutex_version) in vinca_conf["skip_built_packages"] - else: - return mutex_name in vinca_conf["skip_built_packages"] - - -def generate_mutex_package_recipe(vinca_conf, distro): - """Generate a mutex package recipe if mutex_package is defined as a dict.""" - try: - config = parse_mutex_package_config(vinca_conf) - if config is None: - # mutex_package is a string or not configured, don't generate recipe - return None - except ValueError as e: - raise ValueError(f"Cannot generate mutex package recipe: {e}") - - # Create build string using distro name - build_string = f"{distro.name}_{config['build_number']}" - - recipe = { - "package": {"name": config["name"], "version": config["version"]}, - "build": { - "number": config["build_number"], - "string": build_string, - "script": "", - }, - "requirements": { - "run_constraints": config["run_constraints"], - "run_exports": { - "weak": [ - f"${{{{ pin_subpackage('{config['name']}', upper_bound='{config['upper_bound']}') }}}}" - ] - }, - }, - "about": { - "homepage": f"https://github.com/robostack/ros-{distro.name}", - "license": "BSD-3-Clause", - "summary": f"The ROS2 distro mutex. To switch between ROS2 versions, you need to change the mutex.\nE.g. mamba install {config['name']}=*={distro.name} to switch to {distro.name}.", - }, - } - - return recipe - - def parse_package(pkg, distro, vinca_conf, path): name = pkg["name"].replace("_", "-") final_name = get_package_name(name, distro, vinca_conf) diff --git a/vinca/mutex.py b/vinca/mutex.py new file mode 100644 index 0000000..4b6968e --- /dev/null +++ b/vinca/mutex.py @@ -0,0 +1,107 @@ +"""ROS distribution mutex package configuration and recipe generation.""" + +from __future__ import annotations + +from typing import Any + +_REQUIRED_FIELDS = ("name", "version", "upper_bound", "run_constraints") + + +def parse_mutex_package_config(vinca_conf: dict[str, Any]) -> dict[str, Any] | None: + """Validate and normalize dictionary-style mutex configuration. + + String values are legacy dependency declarations and therefore do not produce a + standalone mutex recipe. + """ + mutex_package = vinca_conf.get("mutex_package") + if not mutex_package or isinstance(mutex_package, str): + return None + if not isinstance(mutex_package, dict): + raise ValueError( + "mutex_package must be either a string or a dictionary, " + f"got {type(mutex_package).__name__}" + ) + + missing = [field for field in _REQUIRED_FIELDS if field not in mutex_package] + if missing: + raise ValueError( + f"mutex_package configuration is missing required fields: {missing}" + ) + + result = dict(mutex_package) + result.setdefault("build_number", vinca_conf.get("build_number", 1)) + return result + + +def get_mutex_package_dependency(vinca_conf: dict[str, Any], distro: Any) -> str | None: + """Return the dependency spec for legacy or dictionary-style configuration.""" + mutex_package = vinca_conf.get("mutex_package") + if not mutex_package: + return None + if isinstance(mutex_package, str): + return mutex_package + + try: + mutex_config = parse_mutex_package_config(vinca_conf) + except ValueError as error: + raise ValueError( + f"Error parsing mutex_package configuration: {error}" + ) from error + + version_parts = mutex_config["version"].split(".") + pin_depth = len(mutex_config["upper_bound"].split(".")) + pin = ".".join(version_parts[:pin_depth]) + ".*" + return f"{mutex_config['name']} {pin} {distro.name}_*" + + +def should_skip_mutex_package( + vinca_conf: dict[str, Any], mutex_name: str, mutex_version: str +) -> bool: + """Return whether the configured skip set already contains this mutex.""" + skipped = vinca_conf.get("skip_built_packages", []) + key = ( + (mutex_name, mutex_version) + if vinca_conf.get("trigger_new_versions") + else mutex_name + ) + return key in skipped + + +def generate_mutex_package_recipe( + vinca_conf: dict[str, Any], distro: Any +) -> dict[str, Any] | None: + """Generate a recipe for dictionary-style mutex configuration.""" + try: + mutex_config = parse_mutex_package_config(vinca_conf) + except ValueError as error: + raise ValueError(f"Cannot generate mutex package recipe: {error}") from error + if mutex_config is None: + return None + + name = mutex_config["name"] + distro_name = distro.name + return { + "package": {"name": name, "version": mutex_config["version"]}, + "build": { + "number": mutex_config["build_number"], + "string": f"{distro_name}_{mutex_config['build_number']}", + "script": "", + }, + "requirements": { + "run_constraints": mutex_config["run_constraints"], + "run_exports": { + "weak": [ + f"${{{{ pin_subpackage('{name}', upper_bound='{mutex_config['upper_bound']}') }}}}" + ] + }, + }, + "about": { + "homepage": f"https://github.com/robostack/ros-{distro_name}", + "license": "BSD-3-Clause", + "summary": ( + "The ROS2 distro mutex. To switch between ROS2 versions, you need " + f"to change the mutex.\nE.g. mamba install {name}=*={distro_name} " + f"to switch to {distro_name}." + ), + }, + } diff --git a/vinca/platforms.py b/vinca/platforms.py new file mode 100644 index 0000000..b1aa1db --- /dev/null +++ b/vinca/platforms.py @@ -0,0 +1,33 @@ +"""Conda platform detection utilities.""" + +from __future__ import annotations + +import platform +import sys + +_PLATFORM_BY_SYSTEM_AND_MACHINE = { + ("linux", "aarch64"): "linux-aarch64", + ("linux", "x86_64"): "linux-64", + ("darwin", "arm64"): "osx-arm64", + ("darwin", "x86_64"): "osx-64", + ("win32", "AMD64"): "win-64", + ("win32", "x86_64"): "win-64", +} + + +def get_conda_subdir(selected_platform: str | None = None) -> str: + """Return an explicit target platform or detect the host conda subdir.""" + if selected_platform: + return selected_platform + + system = sys.platform + if system.startswith("linux"): + system = "linux" + + machine = platform.machine() + try: + return _PLATFORM_BY_SYSTEM_AND_MACHINE[system, machine] + except KeyError as error: + raise RuntimeError( + f"Unsupported platform: system={system!r}, machine={machine!r}" + ) from error diff --git a/vinca/recipes.py b/vinca/recipes.py new file mode 100644 index 0000000..9348680 --- /dev/null +++ b/vinca/recipes.py @@ -0,0 +1,405 @@ +"""Generation of package outputs for rattler-build recipes.""" + +from __future__ import annotations + +import copy +import os +from typing import Any + +import catkin_pkg + +from vinca.license_utils import convert_to_spdx_license +from vinca.mutex import get_mutex_package_dependency +from vinca.naming import get_package_prefix, normalize_package_dependency +from vinca.resolve import resolve_pkgname +from vinca.utils import ( + get_pkg_additional_info, + get_pkg_build_number, + is_dummy_metapackage, +) + +_BUILD_SCRIPTS = { + "cmake": "${{ '$RECIPE_DIR/build_catkin.sh' if unix or wasm32 else '%RECIPE_DIR%\\\\bld_catkin.bat' }}", + "catkin": "${{ '$RECIPE_DIR/build_catkin.sh' if unix or wasm32 else '%RECIPE_DIR%\\\\bld_catkin.bat' }}", + "ament_cmake": "${{ '$RECIPE_DIR/build_ament_cmake.sh' if unix or wasm32 else '%RECIPE_DIR%\\\\bld_ament_cmake.bat' }}", + "ament_python": "${{ '$RECIPE_DIR/build_ament_python.sh' if unix or wasm32 else '%RECIPE_DIR%\\\\bld_ament_python.bat' }}", +} + +_BASE_REQUIREMENTS = { + "build": [ + "${{ compiler('cxx') }}", + "${{ compiler('c') }}", + { + "if": "target_platform!='emscripten-wasm32'", + "then": ["${{ stdlib('c') }}"], + }, + "ninja", + "python", + "setuptools", + "git", + "git-lfs", + {"if": "unix", "then": ["patch", "make", "coreutils"]}, + {"if": "win", "then": ["m2-patch"]}, + {"if": "osx", "then": ["tapi"]}, + {"if": "build_platform != target_platform", "then": ["pkg-config"]}, + "cmake", + "cython", + { + "if": "build_platform != target_platform", + "then": ["python", "cross-python_${{ target_platform }}", "numpy"], + }, + ], + "host": [ + {"if": "build_platform == target_platform", "then": ["pkg-config"]}, + "python", + "numpy", + "pip", + ], + "run": [], +} + + +def get_depmods(vinca_conf: dict[str, Any], package_name: str, distro: Any): + """Return normalized dependency removals and additions by requirement section.""" + configured = vinca_conf.get("depmods", {}).get(package_name, {}) + removed = {section: [] for section in ("build", "host", "run")} + added = {section: [] for section in ("build", "host", "run")} + for section in removed: + removed[section] = [ + normalize_package_dependency(dependency, distro, vinca_conf) + for dependency in configured.get(f"remove_{section}", []) + ] + added[section] = [ + normalize_package_dependency(dependency, distro, vinca_conf) + for dependency in configured.get(f"add_{section}", []) + ] + return removed, added + + +def _dummy_constraint( + config: dict[str, Any], version: str, shortname: str +) -> tuple[str, str]: + dependency = config.get("dep_name") + if not dependency: + raise RuntimeError(f"Missing 'dep_name' for dummy recipe of {shortname}") + upper_bound = config.get("upper_bound") or config.get("max_pin") + if not upper_bound: + raise RuntimeError( + f"Missing 'upper_bound' or 'max_pin' for dummy recipe of {shortname}" + ) + + package_version = config.get("override_version", version) + parts = [int(part) for part in package_version.split(".")] + pin_depth = len(upper_bound.split(".")) + upper_parts = parts[:pin_depth] + upper_parts[-1] += 1 + upper_parts += [0] * (len(parts) - pin_depth) + upper = ".".join(map(str, upper_parts)) + "a0" + return package_version, f"{dependency} >={package_version}, <{upper}" + + +def _initial_output( + shortname: str, + package_name: str, + version: str, + vinca_conf: dict[str, Any], +) -> dict[str, Any]: + if not is_dummy_metapackage(shortname, vinca_conf): + return { + "package": {"name": package_name, "version": version}, + "requirements": copy.deepcopy(_BASE_REQUIREMENTS), + "build": {"script": ""}, + } + + dummy_config = get_pkg_additional_info(shortname, vinca_conf)[ + "generate_dummy_package_with_run_deps" + ] + package_version, constraint = _dummy_constraint(dummy_config, version, shortname) + return { + "package": {"name": package_name, "version": package_version}, + "build": { + "number": get_pkg_build_number( + vinca_conf.get("build_number", 0), package_name, vinca_conf + ), + "script": "", + }, + "requirements": {"build": [], "host": [], "run": [constraint]}, + } + + +def _evaluated_dependency_names(dependencies) -> list[str]: + return [ + dependency.name for dependency in dependencies if dependency.evaluated_condition + ] + + +def _resolve_into( + dependencies: list[str], + destination: list, + vinca_conf: dict[str, Any], + distro: Any, + unsatisfied: set[str], + *, + runtime: bool = False, +) -> None: + for dependency in dependencies: + resolved = resolve_pkgname(dependency, vinca_conf, distro, is_rundep=runtime) + if resolved: + destination.extend(resolved) + else: + unsatisfied.add(dependency) + + +def _apply_depmods(requirements: dict[str, list], removed: dict, added: dict) -> None: + for section in requirements: + requirements[section].extend(added[section]) + requirements[section] = [ + dependency + for dependency in requirements[section] + if dependency not in removed[section] + ] + + +def _replace_with_selector( + requirements: list, + dependency: str, + condition: str, + *, + destination: list | None = None, +) -> None: + if dependency not in requirements: + return + while dependency in requirements: + requirements.remove(dependency) + (destination if destination is not None else requirements).append( + {"if": condition, "then": [dependency]} + ) + + +def _adjust_requirements(requirements: dict[str, list], package_prefix: str) -> None: + _replace_with_selector( + requirements["run"], "cmake", "target_platform != 'emscripten-wasm32'" + ) + if "cmake" in requirements["host"]: + requirements["host"].remove("cmake") + if "cmake" not in requirements["build"]: + requirements["build"].append("cmake") + + mimick_vendor = f"{package_prefix}-mimick-vendor" + _replace_with_selector( + requirements["build"], mimick_vendor, "target_platform != 'emscripten-wasm32'" + ) + _replace_with_selector( + requirements["host"], + mimick_vendor, + "target_platform != 'emscripten-wasm32'", + destination=requirements["build"], + ) + + rosidl_generators = f"{package_prefix}-rosidl-default-generators" + if rosidl_generators in requirements["host"]: + requirements["build"].append( + { + "if": "target_platform == 'emscripten-wasm32'", + "then": [rosidl_generators], + } + ) + + requirements["run"].sort(key=_requirement_sort_key) + requirements["host"].sort(key=_requirement_sort_key) + + if f"{package_prefix}-pybind11-vendor" in requirements["host"]: + requirements["host"].append("pybind11") + for dependency in ("pybind11", "qt-main"): + if dependency in requirements["host"]: + requirements["build"].append( + {"if": "build_platform != target_platform", "then": [dependency]} + ) + + for dependency in ("pyqt-builder", "git", "doxygen", "git-lfs"): + _replace_with_selector( + requirements["host"], + dependency, + "build_platform == target_platform", + ) + if any( + item == {"if": "build_platform == target_platform", "then": [dependency]} + for item in requirements["host"] + ): + requirements["build"].append( + {"if": "build_platform != target_platform", "then": [dependency]} + ) + + for section, dependencies in requirements.items(): + unique = [] + for dependency in dependencies: + if dependency not in unique: + unique.append(dependency) + requirements[section] = unique + + +def _requirement_sort_key(requirement): + return ( + next(iter(requirement.values())) + if isinstance(requirement, dict) + else requirement + ) + + +def _add_metadata(output: dict[str, Any], package: Any, shortname: str) -> None: + about = output["about"] = {} + for url in package.urls: + if url.type == "website": + about["homepage"] = url.url + elif url.type == "repository": + about["repository"] = url.url + if package.licenses: + license_expression = convert_to_spdx_license( + [str(license) for license in package.licenses], package_name=shortname + ) + if license_expression: + about["license"] = license_expression + if package.description: + about["summary"] = package.description + + +def generate_output( + shortname: str, + vinca_conf: dict[str, Any], + distro: Any, + version: str, + all_packages: list | None = None, + unsatisfied: set[str] | None = None, +) -> dict[str, Any] | None: + """Generate one package output from ROS metadata and vinca configuration.""" + all_packages = all_packages or [] + unsatisfied = unsatisfied if unsatisfied is not None else set() + if shortname not in vinca_conf["_selected_pkgs"]: + return None + + package_names = resolve_pkgname(shortname, vinca_conf, distro) + if not package_names: + return None + output = _initial_output(shortname, package_names[0], version, vinca_conf) + + xml = distro.get_release_package_xml(shortname) + if not xml: + print( + f"Skip {shortname} as it is present in our snapshot, but not in the " + "latest rosdistro cache." + ) + return None + package = catkin_pkg.package.parse_package_string(xml) + package.evaluate_conditions(os.environ) + + python_dependencies = resolve_pkgname("python", vinca_conf, distro) + output["requirements"]["run"].extend(python_dependencies) + output["requirements"]["host"].extend(python_dependencies) + + is_dummy = is_dummy_metapackage(shortname, vinca_conf) + build_type = package.get_build_type() + if not is_dummy: + try: + output["build"]["script"] = _BUILD_SCRIPTS[build_type] + except KeyError: + print(f"Unknown build type for {shortname}: {build_type}") + return None + if build_type == "ament_python": + output["requirements"]["host"].extend( + resolve_pkgname("python-setuptools", vinca_conf, distro) + ) + if build_type in ("cmake", "catkin", "ament_cmake"): + output["requirements"]["build"].append( + { + "if": "osx", + "then": [ + "clang-tools ${{ (cxx_compiler_version ~ '.*') if " + "cxx_compiler_version is defined else '*' }}" + ], + } + ) + + mutex_dependency = get_mutex_package_dependency(vinca_conf, distro) + if mutex_dependency: + output["requirements"]["host"].append(mutex_dependency) + output["requirements"]["run"].append(mutex_dependency) + + package_prefix = get_package_prefix(distro, vinca_conf) + if not distro.check_ros1() and shortname not in { + "ament_cmake_core", + "ament_package", + "ros_workspace", + "ros_environment", + }: + output["requirements"]["host"].extend( + [f"{package_prefix}-ros-environment", f"{package_prefix}-ros-workspace"] + ) + output["requirements"]["run"].append(f"{package_prefix}-ros-workspace") + + group_dependencies = [] + for dependency in package.group_depends: + dependency.extract_group_members(all_packages) + group_dependencies.extend(dependency.members) + + build_tools = _evaluated_dependency_names( + [*package.buildtool_depends, *package.buildtool_export_depends] + ) + build_dependencies = ( + _evaluated_dependency_names( + [ + *package.build_depends, + *package.build_export_depends, + *package.test_depends, + ] + ) + + group_dependencies + ) + for dependency in build_tools: + resolved = resolve_pkgname(dependency, vinca_conf, distro) + if not resolved: + unsatisfied.add(dependency) + elif "git" in resolved: + output["requirements"]["build"].extend(resolved) + elif dependency != "cmake": + build_dependencies.append(dependency) + + if shortname == "cyclonedds" or "cyclonedds" in build_dependencies + build_tools: + output["requirements"]["build"].append( + { + "if": "build_platform != target_platform", + "then": [f"{package_prefix}-cyclonedds"], + } + ) + + _resolve_into( + build_dependencies, + output["requirements"]["host"], + vinca_conf, + distro, + unsatisfied, + ) + run_dependencies = ( + _evaluated_dependency_names( + [ + *package.run_depends, + *package.exec_depends, + *package.build_export_depends, + *package.buildtool_export_depends, + ] + ) + + group_dependencies + ) + _resolve_into( + run_dependencies, + output["requirements"]["run"], + vinca_conf, + distro, + unsatisfied, + runtime=True, + ) + + removed, added = get_depmods(vinca_conf, package.name, distro) + _apply_depmods(output["requirements"], removed, added) + _adjust_requirements(output["requirements"], package_prefix) + _add_metadata(output, package, shortname) + return output diff --git a/vinca/sources.py b/vinca/sources.py new file mode 100644 index 0000000..7be467d --- /dev/null +++ b/vinca/sources.py @@ -0,0 +1,125 @@ +"""Source sections for generated rattler-build recipes.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +from vinca.mutex import generate_mutex_package_recipe, should_skip_mutex_package +from vinca.resolve import resolve_pkgname +from vinca.utils import is_dummy_metapackage + + +def source_reference(*, url: str, ref: str, ref_type: str) -> dict[str, str]: + """Create source keys for either a git reference or a checksummed archive.""" + if ref_type == "sha256": + return {"url": url, "sha256": ref} + return {"git": url, ref_type: ref} + + +def _is_built(package_name: str, version: str, vinca_conf: dict[str, Any]) -> bool: + skipped = vinca_conf.get("skip_built_packages", []) + key = ( + (package_name, version) + if vinca_conf.get("trigger_new_versions") + else package_name + ) + return key in skipped + + +def _package_patches( + package_name: str, vinca_conf: dict[str, Any], platform: str +) -> list[str]: + configured = vinca_conf.get("_patches", {}).get(package_name) + if not configured: + return [] + return [*configured.get("any", []), *configured.get(platform.split("-")[0], [])] + + +def generate_source( + distro: Any, vinca_conf: dict[str, Any], platform: str +) -> dict[str, dict]: + """Generate per-package release sources for a multi-recipe build.""" + sources: dict[str, dict] = {} + for shortname in vinca_conf["_selected_pkgs"]: + if not distro.check_package(shortname): + print(f"Could not generate source for {shortname}") + continue + if is_dummy_metapackage(shortname, vinca_conf): + continue + + package_names = resolve_pkgname(shortname, vinca_conf, distro) + version = distro.get_version(shortname) + print("Checking ", shortname, version) + if not package_names or _is_built(package_names[0], version, vinca_conf): + continue + + url, ref, ref_type = distro.get_released_repo(shortname) + package_name = package_names[0] + entry = source_reference(url=url, ref=ref, ref_type=ref_type) + entry["target_directory"] = f"{package_name}/src/work" + + patches = _package_patches(package_name, vinca_conf, platform) + if patches: + print(patches) + common_root = os.path.commonpath([os.getcwd(), *patches]) + entry["patches"] = [ + os.path.relpath(patch, common_root) for patch in patches + ] + sources[package_name] = entry + + mutex_recipe = generate_mutex_package_recipe(vinca_conf, distro) + if mutex_recipe: + mutex = mutex_recipe["package"] + if not should_skip_mutex_package(vinca_conf, mutex["name"], mutex["version"]): + sources[mutex["name"]] = {} + return sources + + +def generate_source_version( + distro: Any, vinca_conf: dict[str, Any], platform: str +) -> dict[str, dict]: + """Generate release sources while retaining absolute patch paths.""" + sources: dict[str, dict] = {} + for shortname in vinca_conf["_selected_pkgs"]: + if not distro.check_package(shortname): + print(f"Could not generate source for {shortname}") + continue + + package_names = resolve_pkgname(shortname, vinca_conf, distro) + version = distro.get_version(shortname) + if not package_names or _is_built(package_names[0], version, vinca_conf): + continue + + url, ref, ref_type = distro.get_released_repo(shortname) + package_name = package_names[0] + entry = source_reference(url=url, ref=ref, ref_type=ref_type) + entry["target_directory"] = f"{package_name}/src/work" + if patches := _package_patches(package_name, vinca_conf, platform): + entry["patches"] = patches + sources[package_name] = entry + return sources + + +def generate_fat_source(distro: Any, vinca_conf: dict[str, Any]) -> list[dict]: + """Generate sources for a single fat recipe containing all packages.""" + sources = [] + for shortname in vinca_conf["_selected_pkgs"]: + if not distro.check_package(shortname): + print(f"Could not generate source for {shortname}") + continue + + package_names = resolve_pkgname(shortname, vinca_conf, distro) + if not package_names: + continue + url, ref, ref_type = distro.get_released_repo(shortname) + package_name = package_names[0] + entry = source_reference(url=url, ref=ref, ref_type=ref_type) + entry["target_directory"] = f"src/{package_name}" + + patch_path = Path(vinca_conf["_patch_dir"]) / f"{package_name}.patch" + if patch_path.exists(): + entry["patches"] = [f"{vinca_conf['patch_dir']}/{package_name}.patch"] + sources.append(entry) + return sources diff --git a/vinca/test_configuration.py b/vinca/test_configuration.py new file mode 100644 index 0000000..5b0aefc --- /dev/null +++ b/vinca/test_configuration.py @@ -0,0 +1,55 @@ +from vinca.configuration import read_snapshot, read_vinca_yaml + + +def test_read_vinca_yaml_discovers_companion_files(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + patches = tmp_path / "patches" + patches.mkdir() + (patches / "demo.patch").write_text("generic") + (patches / "demo.unix.patch").write_text("unix") + (patches / "demo.win.patch").write_text("windows") + (patches / "dependencies.yaml").write_text("demo: {}\n") + + tests = tmp_path / "tests" + tests.mkdir() + (tests / "demo.yaml").write_text("tests: []\n") + (tests / "demo").mkdir() + (tmp_path / "pkg_additional_info.yaml").write_text("demo:\n build_number: 2\n") + (tmp_path / "vinca.yaml").write_text( + "ros_distro: humble\nconda_index: []\npatch_dir: patches\nskip_testing: false\n" + ) + + config = read_vinca_yaml(tmp_path / "vinca.yaml", "linux-64") + + assert config["_patches"]["demo"]["any"] == [str(patches / "demo.patch")] + assert config["_patches"]["demo"]["linux"] == [str(patches / "demo.unix.patch")] + assert config["_patches"]["demo"]["osx"] == [str(patches / "demo.unix.patch")] + assert config["_patches"]["demo"]["win"] == [str(patches / "demo.win.patch")] + assert config["_tests"]["demo"] == tests / "demo.yaml" + assert config["_test_folders"]["demo"] == tests / "demo" + assert config["_pkg_additional_info"]["demo"]["build_number"] == 2 + assert config["depmods"] == {"demo": {}} + + +def test_read_snapshot_merges_additional_packages(tmp_path): + snapshot = tmp_path / "snapshot.yaml" + additional = tmp_path / "additional.yaml" + snapshot.write_text("existing:\n version: 1\noverridden:\n version: 1\n") + additional.write_text("overridden:\n version: 2\nnew:\n version: 1\n") + + merged, loaded_additional = read_snapshot( + { + "rosdistro_snapshot": str(snapshot), + "rosdistro_additional_recipes": str(additional), + } + ) + + assert merged == { + "existing": {"version": 1}, + "overridden": {"version": 2}, + "new": {"version": 1}, + } + assert loaded_additional == { + "overridden": {"version": 2}, + "new": {"version": 1}, + } diff --git a/vinca/test_mutex.py b/vinca/test_mutex.py new file mode 100644 index 0000000..043c8c6 --- /dev/null +++ b/vinca/test_mutex.py @@ -0,0 +1,72 @@ +import pytest + +from vinca.mutex import ( + generate_mutex_package_recipe, + get_mutex_package_dependency, + parse_mutex_package_config, + should_skip_mutex_package, +) + + +class Distro: + name = "jazzy" + + +def mutex_config(**overrides): + config = { + "mutex_package": { + "name": "ros2-distro-mutex", + "version": "1.2.0", + "upper_bound": "x.x", + "run_constraints": ["ros2-distro-mutex 1.2.*"], + }, + "build_number": 3, + "skip_built_packages": [], + } + config.update(overrides) + return config + + +def test_parse_mutex_config_adds_default_build_number(): + parsed = parse_mutex_package_config(mutex_config()) + + assert parsed["build_number"] == 3 + + +def test_parse_mutex_config_reports_missing_fields(): + with pytest.raises(ValueError, match="upper_bound"): + parse_mutex_package_config({"mutex_package": {"name": "mutex"}}) + + +def test_legacy_mutex_is_used_as_dependency_without_generating_recipe(): + config = {"mutex_package": "ros2-distro-mutex 1.*"} + + assert get_mutex_package_dependency(config, Distro()) == "ros2-distro-mutex 1.*" + assert generate_mutex_package_recipe(config, Distro()) is None + + +def test_dictionary_mutex_dependency_and_recipe(): + config = mutex_config() + + assert ( + get_mutex_package_dependency(config, Distro()) + == "ros2-distro-mutex 1.2.* jazzy_*" + ) + recipe = generate_mutex_package_recipe(config, Distro()) + assert recipe["build"] == {"number": 3, "string": "jazzy_3", "script": ""} + assert ( + "pin_subpackage('ros2-distro-mutex', upper_bound='x.x')" + in recipe["requirements"]["run_exports"]["weak"][0] + ) + + +def test_skip_mutex_uses_name_or_name_and_version(): + assert should_skip_mutex_package({"skip_built_packages": ["mutex"]}, "mutex", "1.0") + assert should_skip_mutex_package( + { + "trigger_new_versions": True, + "skip_built_packages": [("mutex", "1.0")], + }, + "mutex", + "1.0", + ) diff --git a/vinca/test_platforms.py b/vinca/test_platforms.py new file mode 100644 index 0000000..5e5f602 --- /dev/null +++ b/vinca/test_platforms.py @@ -0,0 +1,34 @@ +import pytest + +from vinca import platforms + + +def test_explicit_conda_subdir_wins(monkeypatch): + monkeypatch.setattr(platforms.sys, "platform", "unsupported") + + assert platforms.get_conda_subdir("emscripten-wasm32") == "emscripten-wasm32" + + +@pytest.mark.parametrize( + ("system", "machine", "expected"), + [ + ("linux", "x86_64", "linux-64"), + ("linux", "aarch64", "linux-aarch64"), + ("darwin", "arm64", "osx-arm64"), + ("darwin", "x86_64", "osx-64"), + ("win32", "AMD64", "win-64"), + ], +) +def test_detect_conda_subdir(monkeypatch, system, machine, expected): + monkeypatch.setattr(platforms.sys, "platform", system) + monkeypatch.setattr(platforms.platform, "machine", lambda: machine) + + assert platforms.get_conda_subdir() == expected + + +def test_unknown_conda_subdir_has_actionable_error(monkeypatch): + monkeypatch.setattr(platforms.sys, "platform", "plan9") + monkeypatch.setattr(platforms.platform, "machine", lambda: "mips") + + with pytest.raises(RuntimeError, match="system='plan9'.*machine='mips'"): + platforms.get_conda_subdir() diff --git a/vinca/test_recipes.py b/vinca/test_recipes.py new file mode 100644 index 0000000..9a415f6 --- /dev/null +++ b/vinca/test_recipes.py @@ -0,0 +1,30 @@ +import pytest + +from vinca.recipes import _dummy_constraint + + +def test_dummy_constraint_uses_pin_depth_and_override_version(): + version, constraint = _dummy_constraint( + { + "dep_name": "vendor-library", + "upper_bound": "x.x", + "override_version": "2.4.1", + }, + "1.0.0", + "demo_vendor", + ) + + assert version == "2.4.1" + assert constraint == "vendor-library >=2.4.1, <2.5.0a0" + + +@pytest.mark.parametrize( + ("config", "message"), + [ + ({"upper_bound": "x"}, "dep_name"), + ({"dep_name": "vendor-library"}, "upper_bound.*max_pin"), + ], +) +def test_dummy_constraint_validates_required_settings(config, message): + with pytest.raises(RuntimeError, match=message): + _dummy_constraint(config, "1.0.0", "demo_vendor") diff --git a/vinca/test_sources.py b/vinca/test_sources.py new file mode 100644 index 0000000..e58be4a --- /dev/null +++ b/vinca/test_sources.py @@ -0,0 +1,74 @@ +from vinca.sources import generate_source, source_reference + + +class FakeDistro: + name = "humble" + + def check_package(self, _name): + return True + + def get_version(self, _name): + return "1.2.3" + + def get_released_repo(self, _name): + return "https://example.com/demo.tar.gz", "abc123", "sha256" + + def get_legacy_package_prefix(self): + return "ros-humble" + + +def test_source_reference_supports_archives_and_git(): + assert source_reference(url="archive", ref="sum", ref_type="sha256") == { + "url": "archive", + "sha256": "sum", + } + assert source_reference(url="repo", ref="v1", ref_type="git_tag") == { + "git": "repo", + "git_tag": "v1", + } + + +def test_generate_source_combines_generic_and_platform_patches(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + generic = tmp_path / "patches" / "demo.patch" + platform_patch = tmp_path / "patches" / "demo.linux.patch" + generic.parent.mkdir() + generic.touch() + platform_patch.touch() + config = { + "_selected_pkgs": ["demo"], + "_conda_indexes": [], + "_pkg_additional_info": {}, + "_patches": { + "ros-humble-demo": { + "any": [str(generic)], + "linux": [str(platform_patch)], + } + }, + "ros_distro": "humble", + "package_name_mode": "legacy", + "skip_built_packages": [], + } + + sources = generate_source(FakeDistro(), config, "linux-64") + + assert sources["ros-humble-demo"] == { + "url": "https://example.com/demo.tar.gz", + "sha256": "abc123", + "target_directory": "ros-humble-demo/src/work", + "patches": ["patches/demo.patch", "patches/demo.linux.patch"], + } + + +def test_generate_source_skips_already_built_package(): + config = { + "_selected_pkgs": ["demo"], + "_conda_indexes": [], + "_pkg_additional_info": {}, + "_patches": {}, + "ros_distro": "humble", + "package_name_mode": "legacy", + "skip_built_packages": ["ros-humble-demo"], + } + + assert generate_source(FakeDistro(), config, "linux-64") == {} From 27a45928f7e0082ad9f6de45c1330e80341b88c8 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 20 Aug 2026 19:09:31 +0200 Subject: [PATCH 2/5] refactor: share CI pipeline helpers --- vinca/generate_azure.py | 129 ++++++---------------------------------- vinca/generate_gha.py | 110 ++++------------------------------ vinca/pipeline.py | 94 +++++++++++++++++++++++++++++ vinca/test_pipeline.py | 43 ++++++++++++++ 4 files changed, 166 insertions(+), 210 deletions(-) create mode 100644 vinca/pipeline.py create mode 100644 vinca/test_pipeline.py diff --git a/vinca/generate_azure.py b/vinca/generate_azure.py index 31f32ab..8bb5c1a 100644 --- a/vinca/generate_azure.py +++ b/vinca/generate_azure.py @@ -1,34 +1,34 @@ -import networkx as nx -import yaml -import re +import argparse import glob -import sys import os -import argparse -from importlib import resources +import sys from distutils.dir_util import copy_tree +from importlib import resources +import networkx as nx +import yaml from rich import print -from vinca.utils import extract_dependency_names, get_repodata -from vinca.utils import literal_unicode as lu +from vinca import config from vinca.distro import Distro from vinca.main import ( - get_selected_packages, generate_outputs, - read_vinca_yaml, get_conda_subdir, + get_selected_packages, + read_vinca_yaml, ) -from vinca import config +from vinca.pipeline import batch_stages, get_all_ancestors, get_skip_existing +from vinca.utils import extract_dependency_names +from vinca.utils import literal_unicode as lu def read_azure_script(fn): - return (resources.files("vinca") / "azure_templates" / fn).read_text() + return (resources.files("vinca") / "azure_templates" / fn).read_text( + encoding="utf-8" + ) -azure_linux_script = lu(read_azure_script("linux.sh")) -azure_osx_script = lu(read_azure_script("osx_64.sh")) -azure_osx_arm64_script = lu(read_azure_script("osx_arm64.sh")) +azure_unix_script = lu(read_azure_script("unix.sh")) azure_win_preconfig_script = lu(read_azure_script("win_preconfig.bat")) azure_win_script = lu(read_azure_script("win_build.bat")) @@ -71,94 +71,6 @@ def parse_command_line(argv): return arguments -def normalize_name(s): - s = s.replace("-", "_") - return re.sub("[^a-zA-Z0-9_]+", "", s) - - -def batch_stages(stages, max_batch_size=5): - with open("vinca.yaml", "r") as vinca_yaml: - vinca_conf = yaml.safe_load(vinca_yaml) - - # this reduces the number of individual builds to try to save some time - stage_lengths = [len(s) for s in stages] - merged_stages = [] - curr_stage = [] - build_individually = vinca_conf.get("build_in_own_azure_stage", []) - - def chunks(lst, n): - """Yield successive n-sized chunks from lst.""" - for i in range(0, len(lst), n): - yield lst[i : i + n] - - i = 0 - while i < len(stages): - for build_individually_pkg in build_individually: - if build_individually_pkg in stages[i]: - merged_stages.append([[build_individually_pkg]]) - stages[i].remove(build_individually_pkg) - - if ( - stage_lengths[i] < max_batch_size - and len(curr_stage) + stage_lengths[i] < max_batch_size - ): - # merge with previous stage - curr_stage += stages[i] - else: - if len(curr_stage): - merged_stages.append([curr_stage]) - curr_stage = [] - if stage_lengths[i] < max_batch_size: - curr_stage += stages[i] - else: - # split this stage into multiple - merged_stages.append(list(chunks(stages[i], max_batch_size))) - i += 1 - if len(curr_stage): - merged_stages.append([curr_stage]) - return merged_stages - - -def get_skip_existing(vinca_conf, platform): - fn = vinca_conf.get("skip_existing") - repodatas = [] - if fn is not None: - fns = list(fn) - else: - fns = [] - - for fn in fns: - print(f"Fetching repodata: {fn}") - repodata = get_repodata(fn, platform) - repodatas.append(repodata) - - return repodatas - - -def get_all_ancestors(graph, node): - ancestors = set() - visited = set() - current_node = node - - while True: - a = { - a - for a in graph.get(node, []) - if a.startswith("ros-") or a.startswith("ros2") - } - if not graph.get(node): - print(f"[yellow]{node} not found") - - ancestors |= a - visited.add(current_node) - - if len(ancestors - visited) == 0: - print(f"Returning all ancestors for {node} : {ancestors}") - return ancestors - else: - current_node = list(ancestors - visited)[0] - - def add_additional_recipes(args): additional_recipes_path = os.path.abspath( os.path.join(args.dir, "..", "additional_recipes") @@ -210,7 +122,7 @@ def add_additional_recipes(args): def build_linux_pipeline( stages, trigger_branch, - script=azure_linux_script, + script=azure_unix_script, azure_template=None, docker_image=None, outfile="linux.yml", @@ -268,7 +180,7 @@ def build_osx_pipeline( trigger_branch, vm_imagename="macOS-10.15", outfile="osx.yml", - script=azure_osx_script, + script=azure_unix_script, ): # Build OSX pipeline azure_template = {"pool": {"vmImage": vm_imagename}} @@ -525,11 +437,7 @@ def main(): build_linux_pipeline(stages, args.trigger_branch, outfile="linux.yml") if args.platform == "osx-64": - build_osx_pipeline( - stages, - args.trigger_branch, - script=azure_osx_script, - ) + build_osx_pipeline(stages, args.trigger_branch) if args.platform == "osx-arm64": build_osx_pipeline( @@ -537,7 +445,6 @@ def main(): args.trigger_branch, vm_imagename="macOS-11", outfile="osx_arm64.yml", - script=azure_osx_arm64_script, ) if args.platform == "linux-aarch64": diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index 3978f4b..a36cc04 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -1,25 +1,25 @@ -import networkx as nx -import yaml -import re +import argparse import glob -import sys import os -import argparse -from importlib import resources +import sys from distutils.dir_util import copy_tree +from importlib import resources +import networkx as nx +import yaml from rich import print -from vinca.utils import extract_dependency_names, get_repodata, NoAliasDumper -from vinca.utils import literal_unicode as lu +from vinca import config from vinca.distro import Distro from vinca.main import ( - get_selected_packages, generate_outputs, - read_vinca_yaml, get_conda_subdir, + get_selected_packages, + read_vinca_yaml, ) -from vinca import config +from vinca.pipeline import batch_stages, get_all_ancestors, get_skip_existing +from vinca.utils import NoAliasDumper, extract_dependency_names +from vinca.utils import literal_unicode as lu def read_azure_script(fn): @@ -80,94 +80,6 @@ def parse_command_line(argv): return arguments -def normalize_name(s): - s = s.replace("-", "_") - return re.sub("[^a-zA-Z0-9_]+", "", s) - - -def batch_stages(stages, max_batch_size=5): - with open("vinca.yaml", "r") as vinca_yaml: - vinca_conf = yaml.safe_load(vinca_yaml) - - # this reduces the number of individual builds to try to save some time - stage_lengths = [len(s) for s in stages] - merged_stages = [] - curr_stage = [] - build_individually = vinca_conf.get("build_in_own_azure_stage", []) - - def chunks(lst, n): - """Yield successive n-sized chunks from lst.""" - for i in range(0, len(lst), n): - yield lst[i : i + n] - - i = 0 - while i < len(stages): - for build_individually_pkg in build_individually: - if build_individually_pkg in stages[i]: - merged_stages.append([[build_individually_pkg]]) - stages[i].remove(build_individually_pkg) - - if ( - stage_lengths[i] < max_batch_size - and len(curr_stage) + stage_lengths[i] < max_batch_size - ): - # merge with previous stage - curr_stage += stages[i] - else: - if len(curr_stage): - merged_stages.append([curr_stage]) - curr_stage = [] - if stage_lengths[i] < max_batch_size: - curr_stage += stages[i] - else: - # split this stage into multiple - merged_stages.append(list(chunks(stages[i], max_batch_size))) - i += 1 - if len(curr_stage): - merged_stages.append([curr_stage]) - return merged_stages - - -def get_skip_existing(vinca_conf, platform): - fn = vinca_conf.get("skip_existing") - repodatas = [] - if fn is not None: - fns = list(fn) - else: - fns = [] - - for fn in fns: - print(f"Fetching repodata: {fn}") - repodata = get_repodata(fn, platform) - repodatas.append(repodata) - - return repodatas - - -def get_all_ancestors(graph, node): - ancestors = set() - visited = set() - current_node = node - - while True: - a = { - a - for a in graph.get(node, []) - if a.startswith("ros-") or a.startswith("ros2") - } - if not graph.get(node): - print(f"[yellow]{node} not found") - - ancestors |= a - visited.add(current_node) - - if len(ancestors - visited) == 0: - print(f"Returning all ancestors for {node} : {ancestors}") - return ancestors - else: - current_node = list(ancestors - visited)[0] - - def add_additional_recipes(args): additional_recipes_path = os.path.abspath( os.path.join(args.dir, "..", "additional_recipes") diff --git a/vinca/pipeline.py b/vinca/pipeline.py new file mode 100644 index 0000000..54e5822 --- /dev/null +++ b/vinca/pipeline.py @@ -0,0 +1,94 @@ +"""Shared helpers for CI pipeline generation.""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Any + +import yaml + +from vinca.utils import get_repodata + + +def _chunks(items: list[str], size: int) -> list[list[str]]: + return [items[index : index + size] for index in range(0, len(items), size)] + + +def batch_stages( + stages: Iterable[Iterable[str]], + max_batch_size: int = 5, + *, + config_path: str | Path = "vinca.yaml", +) -> list[list[list[str]]]: + """Group dependency stages into CI batches without mutating the input. + + The outer list preserves sequential dependency stages. Each inner list contains + batches which can execute in parallel. + """ + if max_batch_size < 1: + raise ValueError("max_batch_size must be at least 1") + + with Path(config_path).open(encoding="utf-8") as stream: + vinca_conf = yaml.safe_load(stream) or {} + build_individually = set(vinca_conf.get("build_in_own_azure_stage", [])) + + result: list[list[list[str]]] = [] + pending: list[str] = [] + for original_stage in stages: + stage = list(original_stage) + own_stage = [package for package in stage if package in build_individually] + stage = [package for package in stage if package not in build_individually] + + if own_stage: + if pending: + result.append([pending]) + pending = [] + for package in own_stage: + result.append([[package]]) + + if len(stage) >= max_batch_size: + if pending: + result.append([pending]) + pending = [] + result.append(_chunks(stage, max_batch_size)) + elif len(pending) + len(stage) < max_batch_size: + pending.extend(stage) + else: + if pending: + result.append([pending]) + pending = stage + + if pending: + result.append([pending]) + return result + + +def get_skip_existing(vinca_conf: Mapping[str, Any], platform: str) -> list[dict]: + """Fetch repodata files configured for skip-existing checks.""" + repositories = vinca_conf.get("skip_existing") or [] + repodatas = [] + for repository in repositories: + print(f"Fetching repodata: {repository}") + repodatas.append(get_repodata(repository, platform)) + return repodatas + + +def get_all_ancestors(graph: Mapping[str, Iterable[str]], node: str) -> set[str]: + """Return all transitive ROS dependencies of *node*. + + Missing nodes and dependency cycles are handled safely. + """ + ancestors: set[str] = set() + visited = {node} + pending = list(graph.get(node, ())) + while pending: + dependency = pending.pop() + if dependency in visited: + continue + visited.add(dependency) + if not (dependency.startswith("ros-") or dependency.startswith("ros2-")): + continue + ancestors.add(dependency) + pending.extend(graph.get(dependency, ())) + return ancestors diff --git a/vinca/test_pipeline.py b/vinca/test_pipeline.py new file mode 100644 index 0000000..b946cac --- /dev/null +++ b/vinca/test_pipeline.py @@ -0,0 +1,43 @@ +from vinca import generate_azure, generate_gha +from vinca.pipeline import batch_stages, get_all_ancestors + + +def test_ci_generators_load_packaged_build_scripts(): + assert "build_unix.sh" in generate_azure.azure_unix_script + assert "build_unix.sh" in generate_gha.azure_unix_script + + +def test_get_all_ancestors_follows_transitive_dependencies_and_cycles(): + graph = { + "ros-app": ["ros-library", "python"], + "ros-library": ["ros-core"], + "ros-core": ["ros-app"], + } + + assert get_all_ancestors(graph, "ros-app") == {"ros-library", "ros-core"} + + +def test_batch_stages_does_not_mutate_input(tmp_path): + config = tmp_path / "vinca.yaml" + config.write_text("build_in_own_azure_stage: [ros-special]\n") + stages = [["ros-a", "ros-special"], ["ros-b", "ros-c"], ["ros-d", "ros-e"]] + original = [stage.copy() for stage in stages] + + batches = batch_stages(stages, 3, config_path=config) + + assert stages == original + assert batches == [ + [["ros-special"]], + [["ros-a"]], + [["ros-b", "ros-c"]], + [["ros-d", "ros-e"]], + ] + + +def test_batch_stages_splits_large_stages(tmp_path): + config = tmp_path / "vinca.yaml" + config.write_text("{}\n") + + assert batch_stages([["a", "b", "c", "d", "e"]], 2, config_path=config) == [ + [["a", "b"], ["c", "d"], ["e"]] + ] From b1aa9938d5f4ce45457c1f83671a331680da5ec4 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 20 Aug 2026 19:09:38 +0200 Subject: [PATCH 3/5] chore: enforce import and bugbear lint rules --- ruff.toml | 1 + vinca/distro.py | 28 ++++++++++++++++++---------- vinca/generate_gitlab.py | 12 +++++++----- vinca/license_utils.py | 5 +++-- vinca/migrate.py | 19 +++++++++++-------- vinca/resolve.py | 1 + vinca/snapshot.py | 4 +++- vinca/template.py | 9 +++++---- vinca/test_license_utils.py | 1 - vinca/utils.py | 5 +++-- 10 files changed, 52 insertions(+), 33 deletions(-) diff --git a/ruff.toml b/ruff.toml index 1219b52..3d03d6c 100644 --- a/ruff.toml +++ b/ruff.toml @@ -2,6 +2,7 @@ line-length = 88 target-version = "py39" [lint] +extend-select = ["B", "I"] # Ignore rules that were excluded in .flake8: # D104: Missing docstring in public package # E501: Line too long (handled by line-length setting) diff --git a/vinca/distro.py b/vinca/distro.py index 9ce23fb..5d92455 100644 --- a/vinca/distro.py +++ b/vinca/distro.py @@ -84,8 +84,8 @@ def _read_archive_member(*, payload, url, member): ) try: archive = tarfile.open(fileobj=io.BytesIO(payload), mode="r:*") - except tarfile.TarError: - raise RuntimeError(f"Unsupported archive format: {url}") + except tarfile.TarError as error: + raise RuntimeError(f"Unsupported archive format: {url}") from error with archive: entries = [(m.name, m.isdir()) for m in archive.getmembers()] extracted = archive.extractfile(_resolve_member(entries=entries, member=member)) @@ -342,8 +342,10 @@ def _download_raw_pkg_xml_or_cached(self, url): return self._additional_xml_cache[url] try: xml_content = self._get(url).text - except Exception as e: - raise RuntimeError(f"Failed to fetch package.xml from {url}: {e}") + except Exception as error: + raise RuntimeError( + f"Failed to fetch package.xml from {url}: {error}" + ) from error self._additional_xml_cache[url] = xml_content return xml_content @@ -364,10 +366,14 @@ def _package_xml_from_archive_or_cached(self, pkg_info): payload = self._download_archive_or_cached(url) try: xml_content = _read_archive_member(payload=payload, url=url, member=member) - except KeyError: - raise RuntimeError(f"Could not find '{member}' inside the archive {url}") - except Exception as e: - raise RuntimeError(f"Failed to read '{member}' from the archive {url}: {e}") + except KeyError as error: + raise RuntimeError( + f"Could not find '{member}' inside the archive {url}" + ) from error + except Exception as error: + raise RuntimeError( + f"Failed to read '{member}' from the archive {url}: {error}" + ) from error self._additional_xml_cache[cache_key] = xml_content return xml_content @@ -381,8 +387,10 @@ def _download_archive_or_cached(self, url): return self._last_archive[1] try: payload = self._get(url).content - except Exception as e: - raise RuntimeError(f"Failed to download the archive {url}: {e}") + except Exception as error: + raise RuntimeError( + f"Failed to download the archive {url}: {error}" + ) from error self._last_archive = (url, payload) return payload diff --git a/vinca/generate_gitlab.py b/vinca/generate_gitlab.py index 11415e4..0410c26 100644 --- a/vinca/generate_gitlab.py +++ b/vinca/generate_gitlab.py @@ -1,15 +1,17 @@ -import networkx as nx -import yaml import glob -import sys import os +import sys + +import networkx as nx +import yaml from vinca.utils import extract_dependency_names try: - from yaml import CLoader as Loader, CDumper as Dumper + from yaml import CDumper as Dumper + from yaml import CLoader as Loader except ImportError: - from yaml import Loader, Dumper + from yaml import Dumper, Loader # def setup_yaml(): # """ https://stackoverflow.com/a/8661021 """ diff --git a/vinca/license_utils.py b/vinca/license_utils.py index 44335ae..9f750bf 100644 --- a/vinca/license_utils.py +++ b/vinca/license_utils.py @@ -1,8 +1,9 @@ """Utilities for converting ROS package licenses to SPDX format.""" import re -from typing import List, Optional, Dict -from license_expression import get_spdx_licensing, ExpressionError +from typing import Dict, List, Optional + +from license_expression import ExpressionError, get_spdx_licensing # Lookup table for common non-SPDX license strings to SPDX identifiers # Note: Keys are lowercase for case-insensitive matching diff --git a/vinca/migrate.py b/vinca/migrate.py index 4f7635e..2a87025 100644 --- a/vinca/migrate.py +++ b/vinca/migrate.py @@ -1,17 +1,20 @@ -import yaml -import sys -import os import argparse +import os import re -import networkx as nx -import subprocess import shutil +import subprocess +import sys +from distutils.dir_util import copy_tree + +import networkx as nx import ruamel.yaml -from .naming import PackageNameMode, get_package_name_mode, get_package_prefix -from .utils import get_repodata +import yaml + from vinca import config from vinca.distro import Distro -from distutils.dir_util import copy_tree + +from .naming import PackageNameMode, get_package_name_mode, get_package_prefix +from .utils import get_repodata distro_version = None ros_prefix = None diff --git a/vinca/resolve.py b/vinca/resolve.py index b0438b9..e0a36a0 100644 --- a/vinca/resolve.py +++ b/vinca/resolve.py @@ -1,5 +1,6 @@ import os from urllib.request import urlopen + from vinca import config from vinca.naming import get_package_name diff --git a/vinca/snapshot.py b/vinca/snapshot.py index 28793fd..cdf10b3 100644 --- a/vinca/snapshot.py +++ b/vinca/snapshot.py @@ -1,6 +1,8 @@ import argparse -import yaml import datetime + +import yaml + from .distro import Distro diff --git a/vinca/template.py b/vinca/template.py index 6d8c4c0..3b89ea2 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -1,12 +1,12 @@ import datetime -from importlib import resources -import shutil import os import re +import shutil import stat +from importlib import resources +from pathlib import Path from ruamel import yaml -from pathlib import Path from vinca.naming import get_package_prefix, is_legacy_compatibility_output from vinca.utils import ( @@ -217,7 +217,8 @@ def write_recipe(source, outputs, vinca_conf, distro, single_file=True): def generate_template(template_in, template_out, extra_globals=None): import em - from vinca.config import skip_testing, ros_distro + + from vinca.config import ros_distro, skip_testing g = {"ros_distro": ros_distro, "skip_testing": "ON" if skip_testing else "OFF"} diff --git a/vinca/test_license_utils.py b/vinca/test_license_utils.py index 0550c3d..c3da996 100644 --- a/vinca/test_license_utils.py +++ b/vinca/test_license_utils.py @@ -1,6 +1,5 @@ from vinca.license_utils import convert_to_spdx_license, is_valid_spdx_license - # Tests for is_valid_spdx_license function diff --git a/vinca/utils.py b/vinca/utils.py index 88cac32..d835aaf 100644 --- a/vinca/utils.py +++ b/vinca/utils.py @@ -1,9 +1,10 @@ -import yaml import hashlib +import json import os import time -import json + import requests +import yaml class folded_unicode(str): From 9e703ab136662cf2e6398df2b641cd6a2e69974e Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 20 Aug 2026 19:11:16 +0200 Subject: [PATCH 4/5] fix: normalize recipe patch paths across platforms --- vinca/sources.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vinca/sources.py b/vinca/sources.py index 7be467d..1ab54ab 100644 --- a/vinca/sources.py +++ b/vinca/sources.py @@ -65,7 +65,8 @@ def generate_source( print(patches) common_root = os.path.commonpath([os.getcwd(), *patches]) entry["patches"] = [ - os.path.relpath(patch, common_root) for patch in patches + Path(os.path.relpath(patch, common_root)).as_posix() + for patch in patches ] sources[package_name] = entry From 3f4a99d2ff8bbc844b94a1af4f853a16f84b3bfa Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Thu, 20 Aug 2026 19:16:35 +0200 Subject: [PATCH 5/5] refactor: remove obsolete Azure pipeline generator --- MANIFEST.in | 2 +- pyproject.toml | 3 +- vinca/azure_templates/unix.sh | 4 - vinca/ci_templates/unix.sh | 4 + .../windows_build.bat} | 48 +- .../windows_preconfig.bat} | 50 +- vinca/generate_azure.py | 472 ------------------ vinca/generate_gha.py | 71 +-- vinca/migrate.py | 9 +- vinca/pipeline.py | 2 +- vinca/test_pipeline.py | 9 +- 11 files changed, 102 insertions(+), 572 deletions(-) delete mode 100644 vinca/azure_templates/unix.sh create mode 100644 vinca/ci_templates/unix.sh rename vinca/{azure_templates/win_build.bat => ci_templates/windows_build.bat} (96%) rename vinca/{azure_templates/win_preconfig.bat => ci_templates/windows_preconfig.bat} (92%) delete mode 100644 vinca/generate_azure.py diff --git a/MANIFEST.in b/MANIFEST.in index 130b2dd..ed97739 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ recursive-include vinca/templates * -recursive-include vinca/azure_templates * +recursive-include vinca/ci_templates * diff --git a/pyproject.toml b/pyproject.toml index 3fc0560..cd2eec4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,6 @@ GitHub = "https://github.com/RoboStack/vinca" vinca = "vinca.main:main" vinca-glab = "vinca.generate_gitlab:main" vinca-gha = "vinca.generate_gha:main" -vinca-azure = "vinca.generate_azure:main" vinca-migrate = "vinca.migrate:main" vinca-snapshot = "vinca.snapshot:main" vinca-sort-vinca-lists = "vinca.sort_vinca_lists:main" @@ -60,7 +59,7 @@ path = "vinca/__init__.py" [tool.hatch.build] include = [ "vinca/templates/**", - "vinca/azure_templates/**", + "vinca/ci_templates/**", ] [tool.hatch.build.targets.wheel] diff --git a/vinca/azure_templates/unix.sh b/vinca/azure_templates/unix.sh deleted file mode 100644 index 499ceac..0000000 --- a/vinca/azure_templates/unix.sh +++ /dev/null @@ -1,4 +0,0 @@ -export CI=azure -export GIT_BRANCH=$BUILD_SOURCEBRANCHNAME -export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME}) -.scripts/build_unix.sh --target $BUILD_TARGET diff --git a/vinca/ci_templates/unix.sh b/vinca/ci_templates/unix.sh new file mode 100644 index 0000000..8783f1f --- /dev/null +++ b/vinca/ci_templates/unix.sh @@ -0,0 +1,4 @@ +export CI=github +export GIT_BRANCH=${GITHUB_REF_NAME} +export FEEDSTOCK_NAME=${GITHUB_REPOSITORY##*/} +.scripts/build_unix.sh --target $BUILD_TARGET diff --git a/vinca/azure_templates/win_build.bat b/vinca/ci_templates/windows_build.bat similarity index 96% rename from vinca/azure_templates/win_build.bat rename to vinca/ci_templates/windows_build.bat index aeb50f9..1d0ec71 100644 --- a/vinca/azure_templates/win_build.bat +++ b/vinca/ci_templates/windows_build.bat @@ -1,24 +1,24 @@ -setlocal EnableExtensions EnableDelayedExpansion -call activate base - -set "FEEDSTOCK_ROOT=%cd%" - -call conda config --add channels conda-forge -call conda config --add channels robostack-staging -call conda config --set channel_priority strict - -:: Enable long path names on Windows -reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f - -:: conda remove --force m2-git - -for %%X in (%CURRENT_RECIPES%) do ( - echo "BUILDING RECIPE %%X" - cd %FEEDSTOCK_ROOT%\\recipes\\%%X\\ - copy %FEEDSTOCK_ROOT%\\conda_build_config.yaml .\\conda_build_config.yaml - boa build . - if errorlevel 1 exit 1 -) - -anaconda -t %ANACONDA_API_TOKEN% upload "C:\\bld\\win-64\\*.tar.bz2" --force -if errorlevel 1 exit 1 +setlocal EnableExtensions EnableDelayedExpansion +call activate base + +set "FEEDSTOCK_ROOT=%cd%" + +call conda config --add channels conda-forge +call conda config --add channels robostack-staging +call conda config --set channel_priority strict + +:: Enable long path names on Windows +reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f + +:: conda remove --force m2-git + +for %%X in (%CURRENT_RECIPES%) do ( + echo "BUILDING RECIPE %%X" + cd %FEEDSTOCK_ROOT%\\recipes\\%%X\\ + copy %FEEDSTOCK_ROOT%\\conda_build_config.yaml .\\conda_build_config.yaml + boa build . + if errorlevel 1 exit 1 +) + +anaconda -t %ANACONDA_API_TOKEN% upload "C:\\bld\\win-64\\*.tar.bz2" --force +if errorlevel 1 exit 1 diff --git a/vinca/azure_templates/win_preconfig.bat b/vinca/ci_templates/windows_preconfig.bat similarity index 92% rename from vinca/azure_templates/win_preconfig.bat rename to vinca/ci_templates/windows_preconfig.bat index 3bae13e..660739d 100644 --- a/vinca/azure_templates/win_preconfig.bat +++ b/vinca/ci_templates/windows_preconfig.bat @@ -1,25 +1,25 @@ -set "CI=true" - -:: 4 cores available on GHA: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners -:: CPU_COUNT is passed through conda build: https://github.com/conda/conda-build/pull/1149 -set CPU_COUNT=4 - -set PYTHONUNBUFFERED=1 - -call setup_x64 - -:: Set the conda-build working directory to a smaller path -if "%CONDA_BLD_PATH%" == "" ( - set "CONDA_BLD_PATH=C:\\bld\\" -) - -:: On azure, there are libcrypto*.dll & libssl*.dll under -:: C:\\Windows\\System32, which should not be there (no vendor dlls in windows folder). -:: They would be found before the openssl libs of the conda environment, so we delete them. -if defined CI ( - DEL C:\\Windows\\System32\\libcrypto-1_1-x64.dll || (Echo Ignoring failure to delete C:\\Windows\\System32\\libcrypto-1_1-x64.dll) - DEL C:\\Windows\\System32\\libssl-1_1-x64.dll || (Echo Ignoring failure to delete C:\\Windows\\System32\\libssl-1_1-x64.dll) -) - -:: Make paths like C:\\hostedtoolcache\\windows\\Ruby\\2.5.7\\x64\\bin garbage -set "PATH=%PATH:ostedtoolcache=%" +set "CI=true" + +:: 4 cores available on GHA: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners +:: CPU_COUNT is passed through conda build: https://github.com/conda/conda-build/pull/1149 +set CPU_COUNT=4 + +set PYTHONUNBUFFERED=1 + +call setup_x64 + +:: Set the conda-build working directory to a smaller path +if "%CONDA_BLD_PATH%" == "" ( + set "CONDA_BLD_PATH=C:\\bld\\" +) + +:: Hosted runners may have libcrypto*.dll & libssl*.dll under +:: C:\\Windows\\System32, which should not be there (no vendor dlls in windows folder). +:: They would be found before the openssl libs of the conda environment, so we delete them. +if defined CI ( + DEL C:\\Windows\\System32\\libcrypto-1_1-x64.dll || (Echo Ignoring failure to delete C:\\Windows\\System32\\libcrypto-1_1-x64.dll) + DEL C:\\Windows\\System32\\libssl-1_1-x64.dll || (Echo Ignoring failure to delete C:\\Windows\\System32\\libssl-1_1-x64.dll) +) + +:: Make paths like C:\\hostedtoolcache\\windows\\Ruby\\2.5.7\\x64\\bin garbage +set "PATH=%PATH:ostedtoolcache=%" diff --git a/vinca/generate_azure.py b/vinca/generate_azure.py deleted file mode 100644 index 8bb5c1a..0000000 --- a/vinca/generate_azure.py +++ /dev/null @@ -1,472 +0,0 @@ -import argparse -import glob -import os -import sys -from distutils.dir_util import copy_tree -from importlib import resources - -import networkx as nx -import yaml -from rich import print - -from vinca import config -from vinca.distro import Distro -from vinca.main import ( - generate_outputs, - get_conda_subdir, - get_selected_packages, - read_vinca_yaml, -) -from vinca.pipeline import batch_stages, get_all_ancestors, get_skip_existing -from vinca.utils import extract_dependency_names -from vinca.utils import literal_unicode as lu - - -def read_azure_script(fn): - return (resources.files("vinca") / "azure_templates" / fn).read_text( - encoding="utf-8" - ) - - -azure_unix_script = lu(read_azure_script("unix.sh")) -azure_win_preconfig_script = lu(read_azure_script("win_preconfig.bat")) -azure_win_script = lu(read_azure_script("win_build.bat")) - - -def parse_command_line(argv): - parser = argparse.ArgumentParser( - description="Conda recipe Azure pipeline generator for ROS packages" - ) - - default_dir = "./recipes" - parser.add_argument( - "-d", - "--dir", - dest="dir", - default=default_dir, - help="The recipes directory to process (default: {}).".format(default_dir), - ) - - parser.add_argument( - "-t", "--trigger-branch", dest="trigger_branch", help="Trigger branch for Azure" - ) - - parser.add_argument( - "-p", - "--platform", - dest="platform", - default="linux-64", - help="Platform to emit build pipeline for", - ) - - parser.add_argument( - "-a", - "--additional-recipes", - action="store_true", - help="search for additional_recipes folder?", - ) - - arguments = parser.parse_args(argv[1:]) - config.parsed_args = arguments - return arguments - - -def add_additional_recipes(args): - additional_recipes_path = os.path.abspath( - os.path.join(args.dir, "..", "additional_recipes") - ) - - print("Searching additional recipes in ", additional_recipes_path) - - if not os.path.exists(additional_recipes_path): - return - - with open("vinca.yaml", "r") as vinca_yaml: - vinca_conf = yaml.safe_load(vinca_yaml) - - repodatas = get_skip_existing(vinca_conf, args.platform) - - additional_recipes = [] - for recipe_path in glob.glob(additional_recipes_path + "/**/recipe.yaml"): - with open(recipe_path) as recipe: - additional_recipe = yaml.safe_load(recipe) - - name, version, bnumber = ( - additional_recipe["package"]["name"], - additional_recipe["package"]["version"], - additional_recipe["build"]["number"], - ) - print("Checking if ", name, version, bnumber, " exists") - skip = False - for repo in repodatas: - for _, pkg in repo.get("packages", {}).items(): - if ( - pkg["name"] == name - and pkg["version"] == version - and pkg["build_number"] == bnumber - ): - skip = True - print(f"{name}=={version}=={bnumber} already exists. Skipping.") - break - - if not skip: - print("Adding ", os.path.dirname(recipe_path)) - goal_folder = os.path.join(args.dir, name) - os.makedirs(goal_folder, exist_ok=True) - copy_tree(os.path.dirname(recipe_path), goal_folder) - additional_recipes.append(additional_recipe) - - return additional_recipes - - -def build_linux_pipeline( - stages, - trigger_branch, - script=azure_unix_script, - azure_template=None, - docker_image=None, - outfile="linux.yml", -): - # Build Linux pipeline - if azure_template is None: - azure_template = {"pool": {"vmImage": "ubuntu-latest"}} - - if docker_image is None: - docker_image = "condaforge/linux-anvil-cos7-x86_64" - azure_stages = [] - - stage_names = [] - for i, s in enumerate(stages): - stage_name = f"stage_{i}" - stage = {"stage": stage_name, "jobs": []} - stage_names.append(stage_name) - - for batch in s: - stage["jobs"].append( - { - "job": f"stage_{i}_job_{len(stage['jobs'])}", - "steps": [ - { - "script": script, - "env": { - "ANACONDA_API_TOKEN": "$(ANACONDA_API_TOKEN)", - "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", - "DOCKER_IMAGE": docker_image, - }, - "displayName": f"Build {' '.join([pkg for pkg in batch])}", - } - ], - } - ) - - if len(stage["jobs"]) != 0: - # all packages skipped ... - azure_stages.append(stage) - - azure_template["trigger"] = [trigger_branch] - azure_template["pr"] = "none" - if azure_stages: - azure_template["stages"] = azure_stages - - if not len(azure_stages): - return - - with open(outfile, "w") as fo: - fo.write(yaml.dump(azure_template, sort_keys=False)) - - -def build_osx_pipeline( - stages, - trigger_branch, - vm_imagename="macOS-10.15", - outfile="osx.yml", - script=azure_unix_script, -): - # Build OSX pipeline - azure_template = {"pool": {"vmImage": vm_imagename}} - - azure_stages = [] - - stage_names = [] - for i, s in enumerate(stages): - stage_name = f"stage_{i}" - stage = {"stage": stage_name, "jobs": []} - stage_names.append(stage_name) - - for batch in s: - stage["jobs"].append( - { - "job": f"stage_{i}_job_{len(stage['jobs'])}", - "steps": [ - { - "script": script, - "env": { - "ANACONDA_API_TOKEN": "$(ANACONDA_API_TOKEN)", - "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", - }, - "displayName": f"Build {' '.join([pkg for pkg in batch])}", - } - ], - } - ) - - if len(stage["jobs"]) != 0: - # all packages skipped ... - azure_stages.append(stage) - - azure_template["trigger"] = [trigger_branch] - azure_template["pr"] = "none" - if azure_stages: - azure_template["stages"] = azure_stages - - if not len(azure_stages): - return - - with open(outfile, "w") as fo: - fo.write(yaml.dump(azure_template, sort_keys=False)) - - -def build_win_pipeline(stages, trigger_branch, outfile="win.yml"): - azure_template = {"pool": {"vmImage": "windows-2019"}} - - azure_stages = [] - script = azure_win_script - - # overwrite with what we're finding in the repo! - if os.path.exists(".scripts/build_win.bat"): - with open(".scripts/build_win.bat", "r") as fi: - script = lu(fi.read()) - - stage_names = [] - for i, s in enumerate(stages): - stage_name = f"stage_{i}" - stage = {"stage": stage_name, "jobs": []} - stage_names.append(stage_name) - - for batch in s: - stage["jobs"].append( - { - "job": f"stage_{i}_job_{len(stage['jobs'])}", - "variables": {"CONDA_BLD_PATH": "C:\\\\bld\\\\"}, - "steps": [ - { - "task": "PythonScript@0", - "displayName": "Download Miniforge", - "inputs": { - "scriptSource": "inline", - "script": lu( - """import urllib.request -url = 'https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Windows-x86_64.exe' -path = r"$(Build.ArtifactStagingDirectory)/Miniforge.exe" -urllib.request.urlretrieve(url, path)""" - ), - }, - }, - { - "script": lu( - """start /wait "" %BUILD_ARTIFACTSTAGINGDIRECTORY%\\Miniforge.exe /InstallationType=JustMe /RegisterPython=0 /S /D=C:\\Miniforge""" - ), - "displayName": "Install Miniforge", - }, - { - "powershell": 'Write-Host "##vso[task.prependpath]C:\\Miniforge\\Scripts"', - "displayName": "Add conda to PATH", - }, - { - "script": lu( - """call activate base -mamba.exe install -c conda-forge --yes --quiet conda-build pip ruamel.yaml anaconda-client""" - ), - "displayName": "Install conda-build, boa and activate environment", - }, - { - "script": azure_win_preconfig_script, - "displayName": "conda-forge build setup", - }, - { - "script": script, - "env": { - "ANACONDA_API_TOKEN": "$(ANACONDA_API_TOKEN)", - "CURRENT_RECIPES": f"{' '.join([pkg for pkg in batch])}", - "PYTHONUNBUFFERED": 1, - }, - "displayName": f"Build {' '.join([pkg for pkg in batch])}", - }, - ], - } - ) - - if len(stage["jobs"]) != 0: - # all packages skipped ... - azure_stages.append(stage) - - azure_template["trigger"] = [trigger_branch] - azure_template["pr"] = "none" - if azure_stages: - azure_template["stages"] = azure_stages - - if not len(azure_stages): - return - - with open(outfile, "w") as fo: - fo.write(yaml.dump(azure_template, sort_keys=False)) - - -def get_full_tree(): - recipes_dir = config.parsed_args.dir - - vinca_yaml = os.path.join(os.path.dirname(recipes_dir), "vinca.yaml") - - temp_vinca_conf = read_vinca_yaml(vinca_yaml) - temp_vinca_conf["build_all"] = True - temp_vinca_conf["skip_built_packages"] = [] - config.selected_platform = get_conda_subdir() - - python_version = temp_vinca_conf.get("python_version", None) - distro = Distro( - temp_vinca_conf["ros_distro"], - python_version, - temp_vinca_conf["_snapshot"], - temp_vinca_conf["_additional_packages_snapshot"], - ) - - all_packages = get_selected_packages(distro, temp_vinca_conf) - temp_vinca_conf["_selected_pkgs"] = all_packages - - all_outputs = generate_outputs(distro, temp_vinca_conf) - return all_outputs - - -def main(): - args = parse_command_line(sys.argv) - - full_tree = get_full_tree() - - metas = [] - - additional_recipes = [] - if args.additional_recipes: - additional_recipes = add_additional_recipes(args) - - if not os.path.exists(args.dir): - print(f"{args.dir} not found. Not generating a pipeline.") - - all_recipes = glob.glob(os.path.join(args.dir, "**", "*.yaml")) - for f in all_recipes: - with open(f) as fi: - metas.append(yaml.safe_load(fi.read())) - - if len(metas) >= 1: - requirements = {} - - for pkg in full_tree + additional_recipes: - requirements[pkg["package"]["name"]] = pkg["requirements"].get( - "host", [] - ) + pkg["requirements"].get("run", []) - - # Normalize direct and conditional requirements to package names. - for pkg_name, reqs in requirements.items(): - requirements[pkg_name] = extract_dependency_names(reqs) - - G = nx.DiGraph() - for pkg, reqs in requirements.items(): - G.add_node(pkg) - for r in reqs: - if r.startswith("ros-") or r.startswith("ros2-"): - G.add_edge(pkg, r) - - # print(requirements) - # import matplotlib.pyplot as plt - # nx.draw(G, with_labels=True, font_weight='bold') - # plt.show() - - tg = list(reversed(list(nx.topological_sort(G)))) - - names_to_build = {pkg["package"]["name"] for pkg in metas} - print("Names to build: ", names_to_build) - tg_slimmed = [el for el in tg if el in names_to_build] - - stages = [] - current_stage = [] - for pkg in tg_slimmed: - reqs = get_all_ancestors(requirements, pkg) - - sort_in_stage = 0 - for r in reqs: - # sort up the stages, until first stage found where all requirements are fulfilled. - for sidx, _ in enumerate(stages): - if r in stages[sidx]: - sort_in_stage = max(sidx + 1, sort_in_stage) - - if sort_in_stage >= len(stages): - stages.append([pkg]) - else: - stages[sort_in_stage].append(pkg) - - if len(current_stage): - stages.append(current_stage) - - elif len(metas) == 1: - fn_wo_yaml = os.path.splitext(os.path.basename(all_recipes[0]))[0] - stages = [[fn_wo_yaml]] - requirements = [fn_wo_yaml] - else: - stages = [] - requirements = [] - - # filter out packages that we are not actually building - filtered_stages = [] - for stage in stages: - filtered = [pkg for pkg in stage if pkg in requirements] - if len(filtered): - filtered_stages.append(filtered) - - stages = batch_stages(filtered_stages) - print(stages) - - with open("buildorder.txt", "w") as fo: - order = [] - for stage in filtered_stages: - for el in stage: - print(el) - order.append(el) - - fo.write("\n".join(order)) - - if args.platform == "linux-64": - build_linux_pipeline(stages, args.trigger_branch, outfile="linux.yml") - - if args.platform == "osx-64": - build_osx_pipeline(stages, args.trigger_branch) - - if args.platform == "osx-arm64": - build_osx_pipeline( - stages, - args.trigger_branch, - vm_imagename="macOS-11", - outfile="osx_arm64.yml", - ) - - if args.platform == "linux-aarch64": - # Build aarch64 pipeline - aarch64_azure_template = { - "pool": { - "name": "Default", - "demands": [ - "Agent.OS -equals linux", - "Agent.OSArchitecture -equals ARM64", - ], - } - } - - build_linux_pipeline( - stages, - args.trigger_branch, - azure_template=aarch64_azure_template, - docker_image="condaforge/linux-anvil-aarch64", - outfile="linux_aarch64.yml", - ) - - # windows - if args.platform == "win-64": - build_win_pipeline(stages, args.trigger_branch, outfile="win.yml") diff --git a/vinca/generate_gha.py b/vinca/generate_gha.py index a36cc04..023eac5 100644 --- a/vinca/generate_gha.py +++ b/vinca/generate_gha.py @@ -22,20 +22,18 @@ from vinca.utils import literal_unicode as lu -def read_azure_script(fn): - return (resources.files("vinca") / "azure_templates" / fn).read_text( - encoding="utf-8" - ) +def read_ci_script(fn): + return (resources.files("vinca") / "ci_templates" / fn).read_text(encoding="utf-8") -azure_unix_script = lu(read_azure_script("unix.sh")) -azure_win_preconfig_script = lu(read_azure_script("win_preconfig.bat")) -azure_win_script = lu(read_azure_script("win_build.bat")) +unix_build_script = lu(read_ci_script("unix.sh")) +windows_preconfig_script = lu(read_ci_script("windows_preconfig.bat")) +windows_build_script = lu(read_ci_script("windows_build.bat")) def parse_command_line(argv): parser = argparse.ArgumentParser( - description="Conda recipe Azure pipeline generator for ROS packages" + description="GitHub Actions workflow generator for ROS package recipes" ) default_dir = "./recipes" @@ -48,7 +46,10 @@ def parse_command_line(argv): ) parser.add_argument( - "-t", "--trigger-branch", dest="trigger_branch", help="Trigger branch for Azure" + "-t", + "--trigger-branch", + dest="trigger_branch", + help="Branch that triggers the generated workflow", ) parser.add_argument( @@ -156,8 +157,8 @@ def get_stage_name(batch): def build_unix_pipeline( stages, trigger_branch, - script=azure_unix_script, - azure_template=None, + script=unix_build_script, + workflow=None, runs_on="ubuntu-latest", outfile="linux.yml", pipeline_name="build_unix", @@ -165,8 +166,8 @@ def build_unix_pipeline( ): blurb = {"jobs": {}, "name": pipeline_name} - if azure_template is None: - azure_template = blurb + if workflow is None: + workflow = blurb prev_batch_keys = [] @@ -174,7 +175,7 @@ def build_unix_pipeline( stage_name = f"stage_{i}" batch_keys = [] for batch in s: - batch_key = f"{stage_name}_job_{len(azure_template['jobs'])}" + batch_key = f"{stage_name}_job_{len(workflow['jobs'])}" batch_keys.append(batch_key) pretty_stage_name = get_stage_name(batch) @@ -210,23 +211,23 @@ def build_unix_pipeline( "attestations": "write", } - azure_template["jobs"][batch_key] = job + workflow["jobs"][batch_key] = job prev_batch_keys = batch_keys - if len(azure_template.get("jobs", [])) == 0: + if len(workflow.get("jobs", [])) == 0: return - azure_template["on"] = {"push": {"branches": [trigger_branch]}} + workflow["on"] = {"push": {"branches": [trigger_branch]}} - dump_for_gha(azure_template, outfile) + dump_for_gha(workflow, outfile) def build_linux_pipeline( stages, trigger_branch, - script=azure_unix_script, - azure_template=None, + script=unix_build_script, + workflow=None, runs_on="ubuntu-latest", outfile="linux.yml", pipeline_name="build_linux", @@ -235,7 +236,7 @@ def build_linux_pipeline( stages, trigger_branch, script=script, - azure_template=azure_template, + workflow=workflow, runs_on=runs_on, outfile=outfile, pipeline_name=pipeline_name, @@ -248,8 +249,8 @@ def build_osx_pipeline( trigger_branch, vm_imagename="macos-15-intel", outfile="osx.yml", - azure_template=None, - script=azure_unix_script, + workflow=None, + script=unix_build_script, target="osx-64", pipeline_name="build_osx64", ): @@ -257,7 +258,7 @@ def build_osx_pipeline( stages, trigger_branch, script=script, - azure_template=azure_template, + workflow=workflow, runs_on=vm_imagename, outfile=outfile, target=target, @@ -265,15 +266,15 @@ def build_osx_pipeline( ) -def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template=None): +def build_win_pipeline(stages, trigger_branch, outfile="win.yml", workflow=None): vm_imagename = "windows-2022" # Build Win pipeline blurb = {"jobs": {}, "name": "build_win"} - if azure_template is None: - azure_template = blurb + if workflow is None: + workflow = blurb - script = azure_win_script + script = windows_build_script # overwrite with what we're finding in the repo! if os.path.exists(".scripts/build_win.bat"): @@ -285,7 +286,7 @@ def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template stage_name = f"stage_{i}" batch_keys = [] for batch in s: - batch_key = f"{stage_name}_job_{len(azure_template['jobs'])}" + batch_key = f"{stage_name}_job_{len(workflow['jobs'])}" batch_keys.append(batch_key) pretty_stage_name = get_stage_name(batch) @@ -314,7 +315,7 @@ def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template }, { "shell": "cmd", - "run": azure_win_preconfig_script, + "run": windows_preconfig_script, "name": "conda-forge build setup", }, { @@ -342,16 +343,16 @@ def build_win_pipeline(stages, trigger_branch, outfile="win.yml", azure_template "attestations": "write", } - azure_template["jobs"][batch_key] = job + workflow["jobs"][batch_key] = job prev_batch_keys = batch_keys - if len(azure_template.get("jobs", [])) == 0: + if len(workflow.get("jobs", [])) == 0: return - azure_template["on"] = {"push": {"branches": [trigger_branch]}} + workflow["on"] = {"push": {"branches": [trigger_branch]}} - dump_for_gha(azure_template, outfile) + dump_for_gha(workflow, outfile) def get_full_tree(): @@ -501,7 +502,7 @@ def main(): args.trigger_branch, vm_imagename="macos-15", outfile="osx_arm64.yml", - script=azure_unix_script, + script=unix_build_script, target=platform, pipeline_name="build_osx_arm64", ) diff --git a/vinca/migrate.py b/vinca/migrate.py index 2a87025..00c2b62 100644 --- a/vinca/migrate.py +++ b/vinca/migrate.py @@ -150,7 +150,7 @@ def create_migration_instructions(arch, packages_to_migrate, trigger_branch): recipe_dir = os.path.join(config.parsed_args.dir, "recipes") subprocess.check_call( [ - "vinca-azure", + "vinca-gha", "--platform", arch, "--trigger-branch", @@ -164,7 +164,7 @@ def create_migration_instructions(arch, packages_to_migrate, trigger_branch): def parse_command_line(argv): parser = argparse.ArgumentParser( - description="Conda recipe Azure pipeline generator for ROS packages" + description="Generate migration recipes and GitHub Actions workflows" ) default_dir = "./recipes" @@ -177,7 +177,10 @@ def parse_command_line(argv): ) parser.add_argument( - "-t", "--trigger-branch", dest="trigger_branch", help="Trigger branch for Azure" + "-t", + "--trigger-branch", + dest="trigger_branch", + help="Branch that triggers the generated workflow", ) parser.add_argument( diff --git a/vinca/pipeline.py b/vinca/pipeline.py index 54e5822..ea0dcb0 100644 --- a/vinca/pipeline.py +++ b/vinca/pipeline.py @@ -31,7 +31,7 @@ def batch_stages( with Path(config_path).open(encoding="utf-8") as stream: vinca_conf = yaml.safe_load(stream) or {} - build_individually = set(vinca_conf.get("build_in_own_azure_stage", [])) + build_individually = set(vinca_conf.get("build_in_own_stage", [])) result: list[list[list[str]]] = [] pending: list[str] = [] diff --git a/vinca/test_pipeline.py b/vinca/test_pipeline.py index b946cac..b5d7d4d 100644 --- a/vinca/test_pipeline.py +++ b/vinca/test_pipeline.py @@ -1,10 +1,9 @@ -from vinca import generate_azure, generate_gha +from vinca import generate_gha from vinca.pipeline import batch_stages, get_all_ancestors -def test_ci_generators_load_packaged_build_scripts(): - assert "build_unix.sh" in generate_azure.azure_unix_script - assert "build_unix.sh" in generate_gha.azure_unix_script +def test_github_generator_loads_packaged_build_scripts(): + assert "build_unix.sh" in generate_gha.unix_build_script def test_get_all_ancestors_follows_transitive_dependencies_and_cycles(): @@ -19,7 +18,7 @@ def test_get_all_ancestors_follows_transitive_dependencies_and_cycles(): def test_batch_stages_does_not_mutate_input(tmp_path): config = tmp_path / "vinca.yaml" - config.write_text("build_in_own_azure_stage: [ros-special]\n") + config.write_text("build_in_own_stage: [ros-special]\n") stages = [["ros-a", "ros-special"], ["ros-b", "ros-c"], ["ros-d", "ros-e"]] original = [stage.copy() for stage in stages]