vinca-gha: stage the generated recipes without resolving the distribution again

vinca/generate_gha.py

@@ -12,13 +12,7 @@
 from rich import print
 
 from vinca import config
-from vinca.distro import Distro
-from vinca.main import (
-    generate_outputs,
-    get_conda_subdir,
-    get_selected_packages,
-    read_vinca_yaml,
-)
+from vinca.main import get_conda_subdir, read_vinca_yaml
 from vinca.pipeline import batch_stages, get_all_ancestors, get_skip_existing
 from vinca.utils import (
     NoAliasDumper,
@@ -437,72 +431,64 @@ def build_win_pipeline(
     dump_for_gha(workflow, outfile)
 
 
-def get_full_tree():
+def load_configuration():
+    """Read vinca.yaml next to the recipes directory for the few settings the
+    workflow needs (distribution, pixi versions); the recipes themselves are
+    already generated, so the packages are not resolved again."""
     parsed_args = config.parsed_args
     if parsed_args is None:
-        raise RuntimeError("Pipeline arguments must be parsed before generating a tree")
-    recipes_dir = parsed_args.dir
-
-    vinca_yaml = os.path.join(os.path.dirname(recipes_dir), "vinca.yaml")
-
-    temp_vinca_conf = read_vinca_yaml(vinca_yaml)
-    temp_vinca_conf["build_all"] = True
-    temp_vinca_conf["skip_built_packages"] = []
+        raise RuntimeError(
+            "Pipeline arguments must be parsed before reading vinca.yaml"
+        )
+    vinca_yaml = os.path.join(os.path.dirname(parsed_args.dir), "vinca.yaml")
+    vinca_conf = read_vinca_yaml(vinca_yaml)
     config.selected_platform = get_conda_subdir()
+    return vinca_conf
 
-    python_version = temp_vinca_conf.get("python_version", None)
-    distro = Distro(
-        temp_vinca_conf["ros_distro"],
-        python_version,
-        temp_vinca_conf["_snapshot"],
-        temp_vinca_conf["_additional_packages_snapshot"],
-    )
-
-    all_packages = get_selected_packages(distro, temp_vinca_conf)
-    temp_vinca_conf["_selected_pkgs"] = all_packages
 
-    all_outputs = generate_outputs(distro, temp_vinca_conf)
-    return all_outputs
+def get_recipe_requirements(recipes):
+    """Host and run dependency names of each recipe, keyed by package name."""
+    requirements = {}
+    for pkg in recipes:
+        if "outputs" in pkg:
+            req_section = pkg["outputs"][0]["requirements"]
+        else:
+            req_section = pkg.get("requirements") or {}
+        requirements[pkg["package"]["name"]] = extract_dependency_names(
+            (req_section.get("host") or []) + (req_section.get("run") or [])
+        )
+    return requirements
 
 
 def main():
     args = parse_command_line(sys.argv)
 
-    full_tree = get_full_tree()
+    load_configuration()
     setup_pixi_version = config.setup_pixi_version or DEFAULT_SETUP_PIXI_VERSION
     pixi_version = config.pixi_version or DEFAULT_PIXI_VERSION
 
     metas = []
 
-    additional_recipes = []
     if args.additional_recipes:
-        additional_recipes = add_additional_recipes(args)
+        # copied into args.dir, so they are picked up with the generated recipes
+        add_additional_recipes(args)
 
     if not os.path.exists(args.dir):
         print(f"{args.dir} not found. Not generating a pipeline.")
 
-    all_recipes = glob.glob(os.path.join(args.dir, "**", "*.yaml"))
+    # sorted: the stage and batch order follows the order the recipes are read in
+    all_recipes = sorted(glob.glob(os.path.join(args.dir, "**", "*.yaml")))
     for f in all_recipes:
         with open(f, encoding="utf-8") as fi:
             metas.append(yaml.safe_load(fi.read()))
 
     platform = args.platform
 
     if len(metas) >= 1:
-        requirements = {}
-
-        for pkg in full_tree + additional_recipes:
-            if "outputs" in pkg:
-                req_section = pkg["outputs"][0]["requirements"]
-            else:
-                req_section = pkg["requirements"]
-            requirements[pkg["package"]["name"]] = req_section.get(
-                "host", []
-            ) + req_section.get("run", [])
-
-        # Normalize direct and conditional requirements to package names.
-        for pkg_name, reqs in requirements.items():
-            requirements[pkg_name] = extract_dependency_names(reqs)
+        # The stages only order the recipes being built, so their own requirements
+        # are enough: resolving every package of the distribution again (as
+        # generate-recipes already did) would only add packages that are not built.
+        requirements = get_recipe_requirements(metas)
 
         test_requirements = add_test_requirements(requirements, metas)
 

vinca/test_generate_gha.py

@@ -1,11 +1,14 @@
 """Tests for the GitHub Actions pipeline generation."""
 
+import sys
+
 import pytest
 
-from vinca import config
+from vinca import config, generate_gha
 from vinca.generate_gha import (
     build_unix_pipeline,
     build_win_pipeline,
+    get_recipe_requirements,
     get_setup_pixi_step,
     get_stage_name,
 )
@@ -122,6 +125,57 @@ def test_win_pipeline_requires_the_repository_build_script(tmp_path, monkeypatch
         )
 
 
+def test_recipe_requirements_are_host_and_run_names():
+    recipe = {
+        "package": {"name": "ros2-b", "version": "1.0.0"},
+        "requirements": {
+            "build": ["cmake"],
+            "host": ["ros2-a ==1.0.0", {"if": "linux", "then": ["ros2-linux-only"]}],
+            "run": ["ros2-a", "python"],
+        },
+    }
+
+    assert get_recipe_requirements([recipe]) == {
+        "ros2-b": ["ros2-a", "ros2-linux-only", "python"]
+    }
+
+
+def test_stages_are_derived_from_the_generated_recipes(tmp_path, monkeypatch):
+    yaml = pytest.importorskip("yaml")
+    monkeypatch.chdir(tmp_path)
+    (tmp_path / "vinca.yaml").write_text("{}\n")
+    recipes = {
+        "ros2-a": [],
+        # ros2-published is not in ./recipes (already built): no stage of its own
+        "ros2-b": ["ros2-a", "ros2-published"],
+        "ros2-c": ["ros2-b"],
+    }
+    for name, deps in recipes.items():
+        (tmp_path / "recipes" / name).mkdir(parents=True)
+        recipe = {
+            "package": {"name": name, "version": "1.0.0"},
+            "requirements": {"host": deps, "run": deps},
+        }
+        (tmp_path / "recipes" / name / "recipe.yaml").write_text(yaml.safe_dump(recipe))
+    # no resolution of the ROS distribution: only the recipes are read
+    monkeypatch.setattr(generate_gha, "load_configuration", lambda: None)
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        # batch size 1: consecutive small stages are not merged into one job
+        ["vinca-gha", "-d", "./recipes", "-t", "branch", "-p", "linux-64", "-b", "1"],
+    )
+
+    generate_gha.main()
+
+    jobs = yaml.safe_load((tmp_path / "linux.yml").read_text())["jobs"]
+    assert [job["steps"][-1]["env"]["CURRENT_RECIPES"] for job in jobs.values()] == [
+        "ros2-a",
+        "ros2-b",
+        "ros2-c",
+    ]
+
+
 def test_only_later_stages_declare_needs(tmp_path, monkeypatch):
     monkeypatch.chdir(tmp_path)
     (tmp_path / ".scripts").mkdir()