diff --git a/action.yaml b/action.yaml index 0d9105b..6043bcb 100644 --- a/action.yaml +++ b/action.yaml @@ -58,26 +58,32 @@ runs: fileName: "schedule.json" - name: Run update script shell: bash + env: + PROJECT_FILE_NAME: ${{ inputs.project_file_name }} + SCHEDULE_INPUT: ${{ inputs.schedule_path }} + UPDATE_ALL: ${{ inputs.update_all }} run: | set -e - if [ -n "${{ inputs.schedule_path }}" ]; then - SCHEDULE_PATH="${{ github.workspace }}/${{ inputs.schedule_path }}" + if [ -n "$SCHEDULE_INPUT" ]; then + SCHEDULE_PATH="${GITHUB_WORKSPACE}/${SCHEDULE_INPUT}" else - SCHEDULE_PATH="${{ github.workspace }}/schedule.json" + SCHEDULE_PATH="${GITHUB_WORKSPACE}/schedule.json" fi - echo "Updating ${{ inputs.project_file_name }} using schedule $SCHEDULE_PATH" - UPDATE_ALL_FLAG="" - if [ -n "${{ inputs.update_all }}" ]; then - UPDATE_ALL_FLAG="--update-all ${{ inputs.update_all }}" + echo "Updating ${PROJECT_FILE_NAME} using schedule ${SCHEDULE_PATH}" + UPDATE_ALL_ARGS=() + if [ -n "$UPDATE_ALL" ]; then + UPDATE_ALL_ARGS=(--update-all "$UPDATE_ALL") fi - pixi run --manifest-path ${{ github.action_path }}/pyproject.toml update-dependencies "${{ github.workspace }}/${{ inputs.project_file_name }}" "$SCHEDULE_PATH" $UPDATE_ALL_FLAG + pixi run --manifest-path "${GITHUB_ACTION_PATH}/pyproject.toml" update-dependencies "${GITHUB_WORKSPACE}/${PROJECT_FILE_NAME}" "$SCHEDULE_PATH" "${UPDATE_ALL_ARGS[@]}" - name: Changes id: changes shell: bash + env: + PROJECT_FILE_NAME: ${{ inputs.project_file_name }} run: | echo "Showing changes that would be committed" - git --no-pager diff ${{ inputs.project_file_name }} - if git diff --quiet ${{ inputs.project_file_name }}; then + git --no-pager diff -- "$PROJECT_FILE_NAME" + if git diff --quiet -- "$PROJECT_FILE_NAME"; then echo "changes_detected=false" >> "$GITHUB_OUTPUT" else echo "changes_detected=true" >> "$GITHUB_OUTPUT" diff --git a/pyproject.toml b/pyproject.toml index 2f58399..e822daa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ name = "spec0-action" description = "Python code to update the lower bounds of Scientific Python libraries according to SPEC 0" requires-python = ">= 3.11" version = "1.0.0" -dependencies = ["packaging>=25.0", "pandas>=2.3.3", "requests>=2.32.5,<3"] +dependencies = ["packaging>=25.0", "pandas>=2.3.3", "requests>=2.32.5,<3", "tomlkit>=0.13.3,<0.14"] [build-system] build-backend = "hatchling.build" @@ -16,7 +16,6 @@ platforms = ["linux-64"] [tool.pixi.pypi-dependencies] spec0-action= { path = ".", editable = true } -tomlkit = ">=0.13.3,<0.14" [tool.pixi.tasks] update-dependencies = { cmd = ["python", "run_spec0_update.py"] } diff --git a/spec0_action/__init__.py b/spec0_action/__init__.py index 6a89f70..5e3c192 100644 --- a/spec0_action/__init__.py +++ b/spec0_action/__init__.py @@ -1,4 +1,5 @@ import contextlib +from collections import defaultdict from packaging.specifiers import SpecifierSet from typing import Sequence, Dict import datetime @@ -15,7 +16,14 @@ read_toml, write_toml, ) -from packaging.version import Version, InvalidVersion +from packaging.version import Version +from packaging.utils import ( + InvalidSdistFilename, + InvalidWheelFilename, + canonicalize_name, + parse_sdist_filename, + parse_wheel_filename, +) __all__ = ["read_schedule", "read_toml", "write_toml", "update_pyproject_toml"] @@ -37,17 +45,12 @@ def _get_oldest_version_in_window(package: str, years: float) -> Version | None: data = resp.json() except Exception: return None - candidates: list[Version] = [] + release_dates: dict[Version, list[datetime.datetime]] = defaultdict(list) for f in data.get("files", []): - parts = f.get("filename", "").split("-") - if len(parts) < 2: - continue - try: - ver = Version(parts[1]) - except InvalidVersion: - continue - if ver.is_prerelease: + ver = _version_from_filename(f.get("filename", "")) + if ver is None or ver.is_prerelease: continue + upload_str = f.get("upload-time", "") upload_time = None for fmt in ["%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ"]: @@ -56,21 +59,43 @@ def _get_oldest_version_in_window(package: str, years: float) -> Version | None: tzinfo=datetime.timezone.utc ) break - if upload_time is None or upload_time < cutoff: + if upload_time is None: continue - candidates.append(ver) + release_dates[ver].append(upload_time) + candidates = [ + ver + for ver, upload_times in release_dates.items() + if min(upload_times) >= cutoff + ] return min(candidates, default=None) -def update_pyproject_dependencies(dependencies: dict, schedule: Dict[str, str]): +def _version_from_filename(filename: str) -> Version | None: + try: + _, version, _, _ = parse_wheel_filename(filename) + return version + except InvalidWheelFilename: + pass + + try: + _, version = parse_sdist_filename(filename) + return version + except InvalidSdistFilename: + return None + + +def update_pyproject_dependencies(dependencies: list, schedule: Dict[str, str]): # Iterate by idx because we want to update it inplace for i in range(len(dependencies)): dep_str = dependencies[i] + if not isinstance(dep_str, str): + continue pkg, extras, spec, env = parse_pep_dependency(dep_str) - if isinstance(spec, Url) or pkg not in schedule: + schedule_key = canonicalize_name(pkg) + if isinstance(spec, Url) or schedule_key not in schedule: continue - new_lower_bound = Version(schedule[pkg]) + new_lower_bound = Version(schedule[schedule_key]) try: spec = tighten_lower_bound(spec or SpecifierSet(), new_lower_bound) # Will raise a value error if bound is already tighter, in this case we just do nothing and continue @@ -83,29 +108,54 @@ def update_pyproject_dependencies(dependencies: dict, schedule: Dict[str, str]): dependencies[i] = new_dep_str +def iter_pep_dependency_lists(pyproject_data: dict, project_data: dict): + dependencies = project_data.get("dependencies") + if isinstance(dependencies, list): + yield dependencies + + optional_dependencies = project_data.get("optional-dependencies", {}) + if isinstance(optional_dependencies, dict): + for dependencies in optional_dependencies.values(): + if isinstance(dependencies, list): + yield dependencies + + dependency_groups = pyproject_data.get("dependency-groups", {}) + if isinstance(dependency_groups, dict): + for dependencies in dependency_groups.values(): + if isinstance(dependencies, list): + yield dependencies + + def update_dependency_table(dep_table: dict, new_versions: dict): for pkg, pkg_data in dep_table.items(): + schedule_key = canonicalize_name(pkg) # Don't do anything for pkgs that aren't in our schedule - if pkg not in new_versions: + if schedule_key not in new_versions: continue # Like pkg = ">x.y.z, SpecifierSet: + if new_lower_bound not in spec_set: + raise ValueError(f"{new_lower_bound} does not satisfy {spec_set}") + out = [] contains_lower_bound = False diff --git a/tests/test_data/pyproject_pixi_updated.toml b/tests/test_data/pyproject_pixi_updated.toml index 69cfc85..7d62ffa 100644 --- a/tests/test_data/pyproject_pixi_updated.toml +++ b/tests/test_data/pyproject_pixi_updated.toml @@ -31,4 +31,4 @@ xarray = ">=2024.1.0" bar = ["foo"] [tool.pixi.dependencies] -numpy = ">=2.0.0,<2" +numpy = ">=1.10.0,<2" diff --git a/tests/test_update_pyproject_toml.py b/tests/test_update_pyproject_toml.py index dd69c42..4c00cb9 100644 --- a/tests/test_update_pyproject_toml.py +++ b/tests/test_update_pyproject_toml.py @@ -2,6 +2,7 @@ from unittest.mock import patch import pytest +from packaging.specifiers import SpecifierSet from packaging.version import Version from spec0_action.parsing import read_schedule, read_toml @@ -89,3 +90,157 @@ def test_update_all_noop_when_not_set(patch_datetime_now): with patch.object(spec0_action, "_get_oldest_version_in_window") as mock_pypi: update_pyproject_toml(pyproject, schedule) mock_pypi.assert_not_called() + + +def test_update_all_updates_optional_dependency_groups_and_unbounded( + patch_datetime_now, +): + pyproject = _minimal_pyproject("requests") + pyproject["project"]["optional-dependencies"] = { + "test": ["idna>=3.0.0"], + } + pyproject["dependency-groups"] = { + "dev": ["charset-normalizer>=3.0.0", {"include-group": "test"}], + } + schedule = read_schedule("tests/test_data/test_schedule.json") + with patch.object( + spec0_action, "_get_oldest_version_in_window", return_value=Version("9.0.0") + ): + update_pyproject_toml(pyproject, schedule, update_all=2.0) + + assert pyproject["project"]["dependencies"] == ["requests>=9.0.0"] + assert pyproject["project"]["optional-dependencies"]["test"] == ["idna>=9.0.0"] + assert pyproject["dependency-groups"]["dev"] == [ + "charset-normalizer>=9.0.0", + {"include-group": "test"}, + ] + + +def test_requires_python_preserves_existing_restrictions(patch_datetime_now): + pyproject = _minimal_pyproject() + pyproject["project"]["requires-python"] = ">=3.9,<3.14,!=3.13.*" + schedule = read_schedule("tests/test_data/test_schedule.json") + + update_pyproject_toml(pyproject, schedule) + + assert SpecifierSet(pyproject["project"]["requires-python"]) == SpecifierSet( + ">=3.12,<3.14,!=3.13.*" + ) + + +def test_requires_python_keeps_incompatible_existing_restrictions(patch_datetime_now): + pyproject = _minimal_pyproject() + pyproject["project"]["requires-python"] = ">=3.9,<3.12" + schedule = read_schedule("tests/test_data/test_schedule.json") + + update_pyproject_toml(pyproject, schedule) + + assert SpecifierSet(pyproject["project"]["requires-python"]) == SpecifierSet( + ">=3.9,<3.12" + ) + + +def test_canonical_package_names_match_schedule(patch_datetime_now): + pyproject = _minimal_pyproject("Numpy>=1.20", "scikit_learn>=1.0") + schedule = read_schedule("tests/test_data/test_schedule.json") + + update_pyproject_toml(pyproject, schedule) + + assert pyproject["project"]["dependencies"] == [ + "Numpy>=2.0.0", + "scikit_learn>=1.4.0", + ] + + +def test_optional_dependencies_and_dependency_groups_are_updated(patch_datetime_now): + pyproject = _minimal_pyproject() + pyproject["project"]["optional-dependencies"] = { + "test": ["Numpy>=1.20"], + } + pyproject["dependency-groups"] = { + "dev": ["numpy>=1.20", {"include-group": "test"}], + } + schedule = read_schedule("tests/test_data/test_schedule.json") + + update_pyproject_toml(pyproject, schedule) + + assert pyproject["project"]["optional-dependencies"]["test"] == ["Numpy>=2.0.0"] + assert pyproject["dependency-groups"]["dev"] == [ + "numpy>=2.0.0", + {"include-group": "test"}, + ] + + +def test_missing_project_dependencies_is_noop(patch_datetime_now): + pyproject = {"project": {"requires-python": ">=3.9"}} + schedule = read_schedule("tests/test_data/test_schedule.json") + + update_pyproject_toml(pyproject, schedule) + + assert SpecifierSet(pyproject["project"]["requires-python"]) == SpecifierSet( + ">=3.12" + ) + + +def test_pixi_feature_pypi_dependencies_and_non_version_tables(patch_datetime_now): + pyproject = _minimal_pyproject() + pyproject["tool"] = { + "pixi": { + "dependencies": { + "scikit-learn": {"git": "https://example.invalid/scikit-learn.git"} + }, + "feature": { + "test": { + "pypi-dependencies": {"Numpy": ">=1.20"}, + "dependencies": { + "xarray": {"url": "https://example.invalid/pkg.whl"} + }, + } + }, + } + } + schedule = read_schedule("tests/test_data/test_schedule.json") + + update_pyproject_toml(pyproject, schedule) + + assert ( + pyproject["tool"]["pixi"]["feature"]["test"]["pypi-dependencies"]["Numpy"] + == ">=2.0.0" + ) + assert pyproject["tool"]["pixi"]["dependencies"]["scikit-learn"] == { + "git": "https://example.invalid/scikit-learn.git" + } + assert pyproject["tool"]["pixi"]["feature"]["test"]["dependencies"]["xarray"] == { + "url": "https://example.invalid/pkg.whl" + } + + +def test_update_all_uses_version_release_date_not_new_file_upload( + patch_datetime_now, +): + class Response: + def raise_for_status(self): + pass + + def json(self): + return { + "files": [ + { + "filename": "example-1.0.0.tar.gz", + "upload-time": "2020-01-01T00:00:00Z", + }, + { + "filename": "example-1.0.0-py3-none-any.whl", + "upload-time": "2025-01-01T00:00:00Z", + }, + { + "filename": "example-2.0.0-py3-none-any.whl", + "upload-time": "2024-01-01T00:00:00Z", + }, + ] + } + + with patch.object(spec0_action.requests, "get", return_value=Response()): + assert spec0_action._get_oldest_version_in_window("example", 2) == Version( + "2.0.0" + ) diff --git a/tests/test_versions.py b/tests/test_versions.py index e364f7b..80ac2f8 100644 --- a/tests/test_versions.py +++ b/tests/test_versions.py @@ -1,6 +1,7 @@ from packaging.version import Version from spec0_action.versions import repr_spec_set, tighten_lower_bound from packaging.specifiers import SpecifierSet +import pytest def test_repr_specset(): @@ -16,14 +17,22 @@ def test_tighter_lower_bound_any(): def test_tighter_lower_bound_leaves_other_restrictions(): - spec = SpecifierSet("~= 0.9,>=1.0,!= 1.3.4.*,< 2.0") - lower_bound = Version("3.8.0") + spec = SpecifierSet(">=1.0,!= 1.3.4.*,< 2.0") + lower_bound = Version("1.4.0") tightened = tighten_lower_bound(spec, lower_bound) - assert tightened == SpecifierSet("~= 0.9,>=3.8.0,!=1.3.4.*,<2.0") + assert tightened == SpecifierSet(">=1.4.0,!=1.3.4.*,<2.0") def test_tighter_lower_bound_adds_lower_bound_if_not_present(): - spec = SpecifierSet("~=0.9,!=1.3.4.*,<2.0") - lower_bound = Version("3.8.0") + spec = SpecifierSet("!=1.3.4.*,<2.0") + lower_bound = Version("1.4.0") tightened = tighten_lower_bound(spec, lower_bound) - assert tightened == SpecifierSet("~= 0.9, != 1.3.4.*, < 2.0, >=3.8.0") + assert tightened == SpecifierSet("!=1.3.4.*,<2.0,>=1.4.0") + + +def test_tighter_lower_bound_rejects_incompatible_restrictions(): + spec = SpecifierSet(">=1.0,<2.0") + lower_bound = Version("2.0.0") + + with pytest.raises(ValueError): + tighten_lower_bound(spec, lower_bound)