refactor: modularize vinca recipe and pipeline generation

MANIFEST.in

@@ -1,2 +1,2 @@
 recursive-include vinca/templates *
-recursive-include vinca/azure_templates *
+recursive-include vinca/ci_templates *

pyproject.toml

@@ -48,7 +48,6 @@ GitHub = "https://github.com/RoboStack/vinca"
 vinca = "vinca.main:main"
 vinca-glab = "vinca.generate_gitlab:main"
 vinca-gha = "vinca.generate_gha:main"
-vinca-azure = "vinca.generate_azure:main"
 vinca-migrate = "vinca.migrate:main"
 vinca-snapshot = "vinca.snapshot:main"
 vinca-sort-vinca-lists = "vinca.sort_vinca_lists:main"
@@ -60,7 +59,7 @@ path = "vinca/__init__.py"
 [tool.hatch.build]
 include = [
   "vinca/templates/**",
-  "vinca/azure_templates/**",
+  "vinca/ci_templates/**",
 ]
 
 [tool.hatch.build.targets.wheel]

ruff.toml

@@ -2,6 +2,7 @@ line-length = 88
 target-version = "py39"
 
 [lint]
+extend-select = ["B", "I"]
 # Ignore rules that were excluded in .flake8:
 # D104: Missing docstring in public package
 # E501: Line too long (handled by line-length setting)

vinca/ci_templates/unix.sh

@@ -0,0 +1,4 @@
+export CI=github
+export GIT_BRANCH=${GITHUB_REF_NAME}
+export FEEDSTOCK_NAME=${GITHUB_REPOSITORY##*/}
+.scripts/build_unix.sh --target $BUILD_TARGET

vinca/azure_templates/win_build.bat → vinca/ci_templates/windows_build.bat

@@ -1,24 +1,24 @@
-setlocal EnableExtensions EnableDelayedExpansion
-call activate base
-
-set "FEEDSTOCK_ROOT=%cd%"
-
-call conda config --add channels conda-forge
-call conda config --add channels robostack-staging
-call conda config --set channel_priority strict
-
-:: Enable long path names on Windows
-reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f
-
-:: conda remove --force m2-git
-
-for %%X in (%CURRENT_RECIPES%) do (
-    echo "BUILDING RECIPE %%X"
-    cd %FEEDSTOCK_ROOT%\\recipes\\%%X\\
-    copy %FEEDSTOCK_ROOT%\\conda_build_config.yaml .\\conda_build_config.yaml
-    boa build .
-    if errorlevel 1 exit 1
-)
-
-anaconda -t %ANACONDA_API_TOKEN% upload "C:\\bld\\win-64\\*.tar.bz2" --force
-if errorlevel 1 exit 1
+setlocal EnableExtensions EnableDelayedExpansion
+call activate base
+
+set "FEEDSTOCK_ROOT=%cd%"
+
+call conda config --add channels conda-forge
+call conda config --add channels robostack-staging
+call conda config --set channel_priority strict
+
+:: Enable long path names on Windows
+reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f
+
+:: conda remove --force m2-git
+
+for %%X in (%CURRENT_RECIPES%) do (
+    echo "BUILDING RECIPE %%X"
+    cd %FEEDSTOCK_ROOT%\\recipes\\%%X\\
+    copy %FEEDSTOCK_ROOT%\\conda_build_config.yaml .\\conda_build_config.yaml
+    boa build .
+    if errorlevel 1 exit 1
+)
+
+anaconda -t %ANACONDA_API_TOKEN% upload "C:\\bld\\win-64\\*.tar.bz2" --force
+if errorlevel 1 exit 1

vinca/azure_templates/win_preconfig.bat → vinca/ci_templates/windows_preconfig.bat

@@ -1,25 +1,25 @@
-set "CI=true"
-
-:: 4 cores available on GHA: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners
-:: CPU_COUNT is passed through conda build: https://github.com/conda/conda-build/pull/1149
-set CPU_COUNT=4
-
-set PYTHONUNBUFFERED=1
-
-call setup_x64
-
-:: Set the conda-build working directory to a smaller path
-if "%CONDA_BLD_PATH%" == "" (
-    set "CONDA_BLD_PATH=C:\\bld\\"
-)
-
-:: On azure, there are libcrypto*.dll & libssl*.dll under
-:: C:\\Windows\\System32, which should not be there (no vendor dlls in windows folder).
-:: They would be found before the openssl libs of the conda environment, so we delete them.
-if defined CI (
-    DEL C:\\Windows\\System32\\libcrypto-1_1-x64.dll || (Echo Ignoring failure to delete C:\\Windows\\System32\\libcrypto-1_1-x64.dll)
-    DEL C:\\Windows\\System32\\libssl-1_1-x64.dll || (Echo Ignoring failure to delete C:\\Windows\\System32\\libssl-1_1-x64.dll)
-)
-
-:: Make paths like C:\\hostedtoolcache\\windows\\Ruby\\2.5.7\\x64\\bin garbage
-set "PATH=%PATH:ostedtoolcache=%"
+set "CI=true"
+
+:: 4 cores available on GHA: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners
+:: CPU_COUNT is passed through conda build: https://github.com/conda/conda-build/pull/1149
+set CPU_COUNT=4
+
+set PYTHONUNBUFFERED=1
+
+call setup_x64
+
+:: Set the conda-build working directory to a smaller path
+if "%CONDA_BLD_PATH%" == "" (
+    set "CONDA_BLD_PATH=C:\\bld\\"
+)
+
+:: Hosted runners may have libcrypto*.dll & libssl*.dll under
+:: C:\\Windows\\System32, which should not be there (no vendor dlls in windows folder).
+:: They would be found before the openssl libs of the conda environment, so we delete them.
+if defined CI (
+    DEL C:\\Windows\\System32\\libcrypto-1_1-x64.dll || (Echo Ignoring failure to delete C:\\Windows\\System32\\libcrypto-1_1-x64.dll)
+    DEL C:\\Windows\\System32\\libssl-1_1-x64.dll || (Echo Ignoring failure to delete C:\\Windows\\System32\\libssl-1_1-x64.dll)
+)
+
+:: Make paths like C:\\hostedtoolcache\\windows\\Ruby\\2.5.7\\x64\\bin garbage
+set "PATH=%PATH:ostedtoolcache=%"

vinca/configuration.py

@@ -0,0 +1,120 @@
+"""Loading and normalization of ``vinca.yaml`` configuration."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from ruamel.yaml import YAML
+
+from vinca import config
+from vinca.naming import get_package_name_mode
+from vinca.resolve import get_conda_index
+from vinca.utils import add_package_name_variants
+from vinca.v1_selectors import evaluate_selectors
+
+_PATCH_PLATFORMS = ("osx", "linux", "win", "emscripten")
+
+
+def _load_yaml(path: Path) -> Any:
+    yaml = YAML()
+    with path.open(encoding="utf-8") as stream:
+        return yaml.load(stream)
+
+
+def _load_selected_yaml(path: Path, target_platform: str) -> Any:
+    return evaluate_selectors(_load_yaml(path), target_platform=target_platform)
+
+
+def _normalize_conda_indexes(indexes: list[str]) -> list[str]:
+    return [
+        str(Path(index).absolute()) if Path(index).is_file() else index
+        for index in indexes
+    ]
+
+
+def _discover_patches(
+    patch_dir: Path, ros_distro: str
+) -> dict[str, dict[str, list[str]]]:
+    patches: dict[str, dict[str, list[str]]] = {}
+    for path in sorted(patch_dir.glob("*.patch")):
+        parts = path.name.split(".")
+        package_patches = patches.setdefault(
+            parts[0], {"any": [], **{platform: [] for platform in _PATCH_PLATFORMS}}
+        )
+        destination_platforms = ["any"]
+        if len(parts) == 3:
+            if parts[1] in _PATCH_PLATFORMS:
+                destination_platforms = [parts[1]]
+            elif parts[1] == "unix":
+                destination_platforms = ["linux", "osx"]
+        for platform in destination_platforms:
+            package_patches[platform].append(str(path))
+
+    add_package_name_variants(patches, ros_distro)
+    return patches
+
+
+def _discover_tests(
+    config_dir: Path, ros_distro: str
+) -> tuple[dict[str, Path], dict[str, Path]]:
+    test_dir = config_dir / "tests"
+    tests = {path.stem: path for path in test_dir.glob("*.yaml")}
+    test_folders = {path.name: path for path in test_dir.glob("*") if path.is_dir()}
+    add_package_name_variants(tests, ros_distro)
+    add_package_name_variants(test_folders, ros_distro)
+    return tests, test_folders
+
+
+def read_snapshot(
+    vinca_conf: dict[str, Any],
+) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
+    """Load the primary and optional additional package snapshots."""
+    snapshot_path = vinca_conf.get("rosdistro_snapshot")
+    if not snapshot_path:
+        return None, None
+
+    snapshot = _load_yaml(Path(snapshot_path)) or {}
+    additional_path = vinca_conf.get("rosdistro_additional_recipes")
+    additional = _load_yaml(Path(additional_path)) or {} if additional_path else None
+    if additional:
+        snapshot.update(additional)
+    return snapshot, additional
+
+
+def read_vinca_yaml(filepath: str | Path, target_platform: str) -> dict[str, Any]:
+    """Read a vinca configuration and populate its derived internal fields."""
+    filepath = Path(filepath)
+    config_dir = filepath.parent
+    vinca_conf = _load_selected_yaml(filepath, target_platform)
+    vinca_conf["package_name_mode"] = get_package_name_mode(vinca_conf).value
+    vinca_conf["conda_index"] = _normalize_conda_indexes(vinca_conf["conda_index"])
+
+    patch_dir = Path(vinca_conf["patch_dir"]).absolute()
+    vinca_conf["_patch_dir"] = patch_dir
+    vinca_conf["_patches"] = _discover_patches(patch_dir, vinca_conf["ros_distro"])
+    vinca_conf["_tests"], vinca_conf["_test_folders"] = _discover_tests(
+        config_dir, vinca_conf["ros_distro"]
+    )
+
+    dependencies_path = patch_dir / "dependencies.yaml"
+    if dependencies_path.exists():
+        vinca_conf["depmods"] = _load_selected_yaml(dependencies_path, target_platform)
+    vinca_conf["depmods"] = vinca_conf.get("depmods") or {}
+
+    config.ros_distro = vinca_conf["ros_distro"]
+    config.skip_testing = vinca_conf.get("skip_testing", True)
+    vinca_conf["_conda_indexes"] = get_conda_index(vinca_conf, str(config_dir))
+    vinca_conf["trigger_new_versions"] = vinca_conf.get("trigger_new_versions", False)
+
+    additional_info_path = config_dir / "pkg_additional_info.yaml"
+    vinca_conf["_pkg_additional_info"] = (
+        _load_selected_yaml(additional_info_path, target_platform)
+        if additional_info_path.exists()
+        else {}
+    )
+
+    snapshot, additional = read_snapshot(vinca_conf)
+    vinca_conf["_snapshot"] = snapshot or {}
+    vinca_conf["_additional_packages_snapshot"] = additional or {}
+    return vinca_conf

vinca/distro.py

@@ -84,8 +84,8 @@ def _read_archive_member(*, payload, url, member):
             )
     try:
         archive = tarfile.open(fileobj=io.BytesIO(payload), mode="r:*")
-    except tarfile.TarError:
-        raise RuntimeError(f"Unsupported archive format: {url}")
+    except tarfile.TarError as error:
+        raise RuntimeError(f"Unsupported archive format: {url}") from error
     with archive:
         entries = [(m.name, m.isdir()) for m in archive.getmembers()]
         extracted = archive.extractfile(_resolve_member(entries=entries, member=member))
@@ -342,8 +342,10 @@ def _download_raw_pkg_xml_or_cached(self, url):
             return self._additional_xml_cache[url]
         try:
             xml_content = self._get(url).text
-        except Exception as e:
-            raise RuntimeError(f"Failed to fetch package.xml from {url}: {e}")
+        except Exception as error:
+            raise RuntimeError(
+                f"Failed to fetch package.xml from {url}: {error}"
+            ) from error
         self._additional_xml_cache[url] = xml_content
         return xml_content
 
@@ -364,10 +366,14 @@ def _package_xml_from_archive_or_cached(self, pkg_info):
         payload = self._download_archive_or_cached(url)
         try:
             xml_content = _read_archive_member(payload=payload, url=url, member=member)
-        except KeyError:
-            raise RuntimeError(f"Could not find '{member}' inside the archive {url}")
-        except Exception as e:
-            raise RuntimeError(f"Failed to read '{member}' from the archive {url}: {e}")
+        except KeyError as error:
+            raise RuntimeError(
+                f"Could not find '{member}' inside the archive {url}"
+            ) from error
+        except Exception as error:
+            raise RuntimeError(
+                f"Failed to read '{member}' from the archive {url}: {error}"
+            ) from error
         self._additional_xml_cache[cache_key] = xml_content
         return xml_content
 
@@ -381,8 +387,10 @@ def _download_archive_or_cached(self, url):
             return self._last_archive[1]
         try:
             payload = self._get(url).content
-        except Exception as e:
-            raise RuntimeError(f"Failed to download the archive {url}: {e}")
+        except Exception as error:
+            raise RuntimeError(
+                f"Failed to download the archive {url}: {error}"
+            ) from error
         self._last_archive = (url, payload)
         return payload