diff --git a/.github/workflows/test_action.yaml b/.github/workflows/test_action.yaml index e539b83..3ec9fb6 100644 --- a/.github/workflows/test_action.yaml +++ b/.github/workflows/test_action.yaml @@ -19,8 +19,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - excluded_packages: ["", "numpy,\nscikit-learn"] - name: "Run action with exclusions: ${{ matrix.excluded_packages }}" + include: + - excluded_packages: "" + spec0_support_years: "" + - excluded_packages: "numpy,\nscikit-learn" + spec0_support_years: "" + - excluded_packages: "ipython matplotlib networkx numpy pandas scikit-image scikit-learn scipy xarray zarr python" + spec0_support_years: "3" + name: "Run action with support years: '${{ matrix.spec0_support_years }}', exclusions: '${{ matrix.excluded_packages }}'" steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -31,10 +37,16 @@ jobs: create_pr: false schedule_path: tests/test_data/test_schedule.json excluded_packages: ${{ matrix.excluded_packages }} + spec0_support_years: ${{ matrix.spec0_support_years }} - name: Check dependency updates env: EXCLUDED_PACKAGES: ${{ matrix.excluded_packages }} + SPEC0_SUPPORT_YEARS: ${{ matrix.spec0_support_years }} run: | + if [ -n "$SPEC0_SUPPORT_YEARS" ]; then + git diff --exit-code -- tests/test_data/pyproject.toml + exit 0 + fi git diff -U0 -- tests/test_data/pyproject.toml > changes.diff grep -q '^+.*pandas' changes.diff if [ -n "$EXCLUDED_PACKAGES" ]; then diff --git a/action.yaml b/action.yaml index 4c8d316..259709e 100644 --- a/action.yaml +++ b/action.yaml @@ -35,13 +35,17 @@ inputs: description: "GitHub token with pull-requests write permission to create pull requests. Defaults to the built-in GITHUB_TOKEN." required: false update_all: - description: "If set, also update all non-SPEC0 dependencies to versions released within the last N years (e.g., 2)." + description: "If set, update PEP dependencies without an applicable schedule floor or explicit core-package support period to versions first released within the last N years (e.g., 2)." required: false default: "" excluded_packages: description: "Comma- or whitespace-separated package names to leave unchanged, including with update_all. Use python to exclude Python requirements." required: false default: "" + spec0_support_years: + description: "If set, compute SPEC 0 core-package floors from PyPI using this many years of support (e.g., 3). Python still follows the supplied schedule; exclusions take precedence." + required: false + default: "" runs: using: "composite" steps: @@ -70,6 +74,7 @@ runs: PROJECT_FILE_NAME: ${{ inputs.project_file_name }} SCHEDULE_INPUT: ${{ inputs.schedule_path }} UPDATE_ALL: ${{ inputs.update_all }} + SPEC0_SUPPORT_YEARS: ${{ inputs.spec0_support_years }} EXCLUDED_PACKAGES: ${{ inputs.excluded_packages }} run: | set -e @@ -79,11 +84,14 @@ runs: SCHEDULE_PATH="${GITHUB_WORKSPACE}/schedule.json" fi echo "Updating ${PROJECT_FILE_NAME} using schedule ${SCHEDULE_PATH}" - UPDATE_ALL_ARGS=() + UPDATE_ARGS=() if [ -n "$UPDATE_ALL" ]; then - UPDATE_ALL_ARGS=(--update-all "$UPDATE_ALL") + UPDATE_ARGS+=(--update-all "$UPDATE_ALL") + fi + if [ -n "$SPEC0_SUPPORT_YEARS" ]; then + UPDATE_ARGS+=(--spec0-support-years "$SPEC0_SUPPORT_YEARS") fi - pixi run --manifest-path "${GITHUB_ACTION_PATH}/pyproject.toml" update-dependencies "${GITHUB_WORKSPACE}/${PROJECT_FILE_NAME}" "$SCHEDULE_PATH" --excluded-packages "$EXCLUDED_PACKAGES" "${UPDATE_ALL_ARGS[@]}" + pixi run --manifest-path "${GITHUB_ACTION_PATH}/pyproject.toml" update-dependencies "${GITHUB_WORKSPACE}/${PROJECT_FILE_NAME}" "$SCHEDULE_PATH" --excluded-packages "$EXCLUDED_PACKAGES" "${UPDATE_ARGS[@]}" - name: Changes id: changes shell: bash diff --git a/readme.md b/readme.md index 2349689..d95c5b8 100644 --- a/readme.md +++ b/readme.md @@ -14,7 +14,8 @@ name: Update SPEC 0 dependencies on: schedule: - # Day 3 of each quarter. Allows one day buffer after the quarterly schedule release on day 1 + # Day 3 of each quarter. +Allows one day buffer after the quarterly schedule release on day 1 - cron: "0 0 3 1,4,7,10 *" workflow_dispatch: @@ -40,17 +41,18 @@ The built-in `GITHUB_TOKEN` is used by default as long as the workflow has `pull ### Parameters -| Input | Required | Default | Description | -| ------------------- | -------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | -| `token` | no | `GITHUB_TOKEN` | Token with `pull-requests: write` permission to open PRs | -| `project_file_name` | no | `pyproject.toml` | Path to the file to update, relative to repository root | -| `schedule_path` | no | — | Path to a custom `schedule.json`, relative to repository root. Uses the latest release if unset | -| `target_branch` | no | `main` | Branch to open the PR against | -| `create_pr` | no | `true` | Set to `false` for a dry run | -| `pr_title` | no | `chore: Drop support for unsupported packages conform SPEC 0` | Title of the opened PR | -| `commit_msg` | no | `chore: Drop support for unsupported packages conform SPEC 0` | Commit message for the version update commit | -| `update_all` | no | — | If set to a number N, also update non-SPEC0 dependencies to versions released within the last N years (e.g. `2`) | -| `excluded_packages` | no | — | Comma- or whitespace-separated package names to leave unchanged, including with `update_all` | +| Input | Required | Default | Description | +| --------------------- | -------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `token` | no | `GITHUB_TOKEN` | Token with `pull-requests: write` permission to open PRs | +| `project_file_name` | no | `pyproject.toml` | Path to the file to update, relative to repository root | +| `schedule_path` | no | — | Path to a custom `schedule.json`, relative to repository root. Uses the latest release if unset | +| `target_branch` | no | `main` | Branch to open the PR against | +| `create_pr` | no | `true` | Set to `false` for a dry run | +| `pr_title` | no | `chore: Drop support for unsupported packages conform SPEC 0` | Title of the opened PR | +| `commit_msg` | no | `chore: Drop support for unsupported packages conform SPEC 0` | Commit message for the version update commit | +| `update_all` | no | — | If set to N years, update PEP dependencies without a schedule floor or custom core-package policy to the oldest stable version first released within that window | +| `spec0_support_years` | no | — | Override the support period for SPEC 0 core packages using current PyPI release history (e.g. `3`); Python still follows the supplied schedule | +| `excluded_packages` | no | — | Comma- or whitespace-separated package names to leave unchanged, including with `update_all` | For examples of before/after see [tests/test_data/pyproject.toml](./tests/test_data/pyproject.toml) and [tests/test_data/pyproject_updated.toml](./tests/test_data/pyproject_updated.toml). @@ -58,7 +60,8 @@ SPEC 0 packages include `ipython`, `matplotlib`, `networkx`, `numpy`, `pandas`, ### Excluding packages -To keep a package's lower bound as-is, for example to stay compatible with NumPy 1.26 while everything else updates, list it in `excluded_packages` before the action raises its bound (bounds are never lowered). Separate names with commas or whitespace; `python` excludes the Python requirement: +To keep a package's lower bound as-is, for example to stay compatible with NumPy 1.26 while everything else updates, list it in `excluded_packages` before the action raises its bound (bounds are never lowered). +Separate names with commas or whitespace; `python` excludes the Python requirement: ```yaml with: @@ -68,18 +71,46 @@ with: python ``` -Exclusions win over the schedule and `update_all`. The CLI takes the same value via `--excluded-packages`. +Exclusions win over the schedule, `spec0_support_years`, and `update_all`, and excluded packages cause no PyPI lookup. +The CLI takes the same value via `--excluded-packages`. + +### Changing the core-package support period + +The supplied schedule gives SPEC 0 packages two years of support. +To use three years for those packages while updating other dependencies independently: + +```yaml +with: + spec0_support_years: 3 + update_all: 2 +``` + +For each release (`X.Y.0`; pre-, post-, and patch releases are ignored), support ends at the start of the quarter containing its release date plus the period, and the floor moves to the next release. +The schedule generator and custom support periods use the earliest PyPI upload of a release, whether a source distribution or wheel; later uploads do not reset its age. +Floors come from the PyPI release history, so any explicit value, can differ from a published schedule snapshot. +Python always follows the schedule. + +Precedence is `excluded_packages`, then `spec0_support_years` for SPEC 0 packages, then the schedule, then `update_all` for the remaining PEP dependencies. +If PyPI cannot be reached, the affected dependency stays unchanged with a warning rather than falling back to the schedule or `update_all`. + +The CLI takes `--spec0-support-years 3` and the Python API `spec0_support_years=3`. +Extras such as `xarray[io]>=2026.5.1` and environment markers are preserved when bounds change. ## Limitations -1. The action only tightens lower bounds and leaves upper bounds untouched. An update can produce an unsolvable environment — for example `numpy = ">=1.25.0,<2"` becomes `numpy = ">=2.0.0,<2"`. Keeping the environment solvable is out of scope; adjust upper bounds manually if needed. +1. The action only tightens lower bounds and leaves upper bounds untouched. + A proposed floor that conflicts with an existing constraint is skipped; for example, `numpy = ">=1.25.0,<2"` stays unchanged when the proposed floor is `2.0.0`. + It does not solve the full dependency graph or guarantee a compatible environment. 2. Only `pyproject.toml` is currently supported. + This includes Pixi tables within the file; standalone Conda environment files are unsupported. + Conda-only version expressions such as `>=1.26|>=2.0` are left unchanged. ## Maintainer notes ### Releasing a new action version -Action versions are **git tags only**, do not create a GitHub Release for them. GitHub Releases in this repository are reserved for the quarterly schedule data. +Action versions are **git tags only**, do not create a GitHub Release for them. +GitHub Releases in this repository are reserved for the quarterly schedule data. ```bash git tag v1.x @@ -88,13 +119,15 @@ git push origin v1.x ### Schedule releases -The SPEC 0 schedule (`schedule.json` and `schedule.md`) is published as a GitHub Release quarterly by the [Update SPEC 0 schedule](./.github/workflows/update_schedule.yml) workflow. Releases are tagged `schedule-YYYY-QN` (e.g. `schedule-2026-Q2`). +The SPEC 0 schedule (`schedule.json` and `schedule.md`) is published as a GitHub Release quarterly by the [Update SPEC 0 schedule](./.github/workflows/update_schedule.yml) workflow. +Releases are tagged `schedule-YYYY-QN` (e.g. `schedule-2026-Q2`). The action always fetches `schedule.json` from the **latest** GitHub Release in this repository, which will always be a schedule release as long as action versions are never published as releases. #### Bootstrap -Before the first quarterly schedule release exists, the action will fail. To create the initial release, trigger the workflow manually: +Before the first quarterly schedule release exists, the action will fail. +To create the initial release, trigger the workflow manually: 1. Go to **Actions → Update SPEC 0 schedule** 2. Click **Run workflow** diff --git a/run_spec0_update.py b/run_spec0_update.py index ca9312a..a228f78 100644 --- a/run_spec0_update.py +++ b/run_spec0_update.py @@ -22,7 +22,14 @@ type=float, default=None, metavar="YEARS", - help="Also update all non-SPEC0 dependencies to versions released within the last YEARS years (e.g., 2).", + help="Update PEP dependencies without an applicable schedule floor or explicit core-package support period to versions first released within the last YEARS years (e.g., 2).", + ) + parser.add_argument( + "--spec0-support-years", + type=float, + default=None, + metavar="YEARS", + help="Compute SPEC 0 core-package floors from PyPI using YEARS of support; Python still follows the supplied schedule.", ) parser.add_argument( "--excluded-packages", @@ -47,6 +54,7 @@ project_data, schedule_data, update_all=args.update_all, + spec0_support_years=args.spec0_support_years, excluded_packages=args.excluded_packages.replace(",", " ").split(), ) write_toml(toml_path, project_data) diff --git a/spec0_action/__init__.py b/spec0_action/__init__.py index 9c2ff4b..e79e840 100644 --- a/spec0_action/__init__.py +++ b/spec0_action/__init__.py @@ -1,9 +1,11 @@ import datetime +import logging from collections.abc import Callable, Sequence from functools import cache +from itertools import pairwise import requests -from packaging.specifiers import SpecifierSet +from packaging.requirements import Requirement from packaging.utils import ( InvalidSdistFilename, InvalidWheelFilename, @@ -15,9 +17,7 @@ from spec0_action.parsing import ( SupportSchedule, - Url, is_url_spec, - parse_pep_dependency, parse_version_spec, read_schedule, read_toml, @@ -25,17 +25,33 @@ ) from spec0_action.versions import repr_spec_set, tighten_lower_bound -__all__ = ["read_schedule", "read_toml", "update_pyproject_toml", "write_toml"] +__all__ = [ + "CORE_PACKAGES", + "read_schedule", + "read_toml", + "update_pyproject_toml", + "write_toml", +] + +CORE_PACKAGES = [ + "ipython", + "matplotlib", + "networkx", + "numpy", + "pandas", + "scikit-image", + "scikit-learn", + "scipy", + "xarray", + "zarr", +] + +logger = logging.getLogger(__name__) @cache -def _get_oldest_version_in_window(package: str, years: float) -> Version | None: - """ - Query PyPI, return oldest non-pre release version uploaded within the last ``years`` years. - """ - cutoff = datetime.datetime.now(tz=datetime.UTC) - datetime.timedelta( - days=int(365 * years) - ) +def _get_release_dates(package: str) -> dict[Version, datetime.datetime]: + """Fetch each stable version's earliest distribution upload from PyPI.""" try: resp = requests.get( f"https://pypi.org/simple/{package}", @@ -44,40 +60,66 @@ def _get_oldest_version_in_window(package: str, years: float) -> Version | None: ) resp.raise_for_status() data = resp.json() - except requests.RequestException: - return None + except requests.RequestException as exc: + logger.warning("Could not fetch %s releases from PyPI: %s", package, exc) + return {} first_uploads: dict[Version, datetime.datetime] = {} - for f in data.get("files", []): - ver = _version_from_filename(f.get("filename", "")) + for f in data["files"]: + ver = _version_from_filename(f["filename"]) if ver is None or ver.is_prerelease: continue - try: - upload_time = datetime.datetime.fromisoformat(f.get("upload-time", "")) - except ValueError: - continue + upload_time = datetime.datetime.fromisoformat(f["upload-time"]) + first_uploads[ver] = min(first_uploads.get(ver, upload_time), upload_time) + return first_uploads - previous = first_uploads.get(ver) - if previous is None or upload_time < previous: - first_uploads[ver] = upload_time + +def _get_feature_release_dates(package: str) -> list[tuple[Version, datetime.datetime]]: + """Return stable feature releases in version order, with their first uploads.""" + return sorted( + (version, release_date) + for version, release_date in _get_release_dates(package).items() + if not version.is_postrelease and not any(version.release[2:]) + ) + + +def _get_oldest_version_in_window(package: str, years: float) -> Version | None: + """Return the oldest stable version first uploaded within ``years`` years.""" + cutoff = datetime.datetime.now(tz=datetime.UTC) - datetime.timedelta( + days=int(365 * years) + ) candidates = [ - ver for ver, first_upload in first_uploads.items() if first_upload >= cutoff + ver + for ver, first_upload in _get_release_dates(package).items() + if first_upload >= cutoff ] return min(candidates, default=None) -def _version_from_filename(filename: str) -> Version | None: - try: - _, version, _, _ = parse_wheel_filename(filename) - return version - except InvalidWheelFilename: - pass +def _get_spec0_floor( + package: str, support_time: datetime.timedelta, now: datetime.datetime +) -> Version | None: + """Advance to each successor at the quarter start of its predecessor's drop.""" + floor = None + for (_, release_date), (successor, _) in pairwise( + _get_feature_release_dates(package) + ): + if _quarter(release_date + support_time) <= _quarter(now): + floor = successor + return floor + + +def _quarter(date: datetime.datetime) -> tuple[int, int]: + return date.year, (date.month - 1) // 3 + +def _version_from_filename(filename: str) -> Version | None: try: - _, version = parse_sdist_filename(filename) - return version - except InvalidSdistFilename: + if filename.endswith(".whl"): + return parse_wheel_filename(filename)[1] + return parse_sdist_filename(filename)[1] + except (InvalidWheelFilename, InvalidSdistFilename): return None @@ -90,18 +132,24 @@ def update_pyproject_dependencies( for i, dep_str in enumerate(dependencies): if not isinstance(dep_str, str): continue - pkg, extras, spec, env = parse_pep_dependency(dep_str) - package_key = canonicalize_name(pkg) - if isinstance(spec, Url) or package_key in skip: + requirement = Requirement(dep_str) + package_key = canonicalize_name(requirement.name) + if requirement.url or package_key in skip: continue new_lower_bound = resolve_lower_bound(package_key) if new_lower_bound is None: continue - new_spec = tighten_lower_bound(spec or SpecifierSet(), new_lower_bound) - if new_spec is None or new_spec == spec: + new_spec = tighten_lower_bound(requirement.specifier, new_lower_bound) + if new_spec is None or new_spec == requirement.specifier: # Skip no-op updates so unchanged specs keep their original formatting continue - dependencies[i] = f"{pkg}{extras or ''}{repr_spec_set(new_spec)}{env or ''}" + # Keep the original extras and marker spelling when rewriting the bound. + suffix = dep_str.strip()[len(requirement.name) :].lstrip() + extras = suffix[: suffix.index("]") + 1] if requirement.extras else "" + _, separator, marker = dep_str.partition(";") + dependencies[i] = ( + f"{requirement.name}{extras}{repr_spec_set(new_spec)}{separator}{marker}" + ) def iter_pep_dependency_lists(pyproject_data: dict): @@ -118,11 +166,13 @@ def iter_pep_dependency_lists(pyproject_data: dict): def update_dependency_table( - dep_table: dict, new_versions: dict[str, Version], skip: set[str] + dep_table: dict, + resolve_lower_bound: Callable[[str], Version | None], + skip: set[str], ): for pkg, pkg_data in dep_table.items(): package_key = canonicalize_name(pkg) - if package_key in skip or package_key not in new_versions: + if package_key in skip: continue # Like pkg = ">x.y.z, Version | None: + def resolve_lower_bound( + package_key: str, *, use_update_all: bool = True + ) -> Version | None: + if support_time is not None and package_key in CORE_PACKAGES: + return _get_spec0_floor(package_key, support_time, now) if package_key in new_version: return new_version[package_key] - if update_all is not None: + if use_update_all and update_all is not None: return _get_oldest_version_in_window(package_key, update_all) return None @@ -226,4 +300,8 @@ def resolve_lower_bound(package_key: str) -> Version | None: update_pyproject_dependencies(dependencies, resolve_lower_bound, skip) if "tool" in pyproject_data and "pixi" in pyproject_data["tool"]: - update_pixi_dependencies(pyproject_data["tool"]["pixi"], new_version, skip) + update_pixi_dependencies( + pyproject_data["tool"]["pixi"], + lambda package: resolve_lower_bound(package, use_update_all=False), + skip, + ) diff --git a/spec0_action/parsing.py b/spec0_action/parsing.py index 6ab0976..8ffc1d3 100644 --- a/spec0_action/parsing.py +++ b/spec0_action/parsing.py @@ -1,22 +1,12 @@ import json from collections.abc import Sequence from pathlib import Path -from re import compile -from typing import TypeAlias, TypedDict -from urllib.parse import ParseResult, urlparse +from typing import TypedDict from packaging.specifiers import InvalidSpecifier, SpecifierSet -from packaging.version import InvalidVersion, Version +from packaging.version import Version from tomlkit import dumps, loads -# We won't actually do anything with URLs we just need to detect them -Url: TypeAlias = ParseResult - -# Slightly modified version of https://packaging.python.org/en/latest/specifications/dependency-specifiers/#names -PEP_PACKAGE_IDENT_RE = compile( - r"(?im)^([A-Z0-9][A-Z0-9._-]*)(\[[A-Z0-9._,-]+\])?([^;]*)(;.*)?$" -) - class SupportSchedule(TypedDict): start_date: str @@ -31,20 +21,7 @@ def parse_version_spec(s: str) -> SpecifierSet: try: return SpecifierSet(s) except InvalidSpecifier: - try: - ver = Version(s) - except InvalidVersion: - if "*" in s: - # pixi sometimes uses things like python = "3.11.*" - try: - return SpecifierSet(f"=={s}") - except InvalidVersion: - # fall through to the raise below - pass - - raise ValueError(f"{s} is not a version or specifyer") - - return SpecifierSet(f">={ver}") + return SpecifierSet(f"=={s}" if "*" in s else f">={Version(s)}") def write_toml(path: Path | str, data: dict): @@ -64,27 +41,6 @@ def read_schedule(path: Path | str) -> Sequence[SupportSchedule]: return json.load(file) -def parse_pep_dependency( - dep_str: str, -) -> tuple[str, str | None, SpecifierSet | Url | None, str | None]: - match = PEP_PACKAGE_IDENT_RE.match(dep_str) - if match is None: - raise ValueError("Could not find any valid python package identifier") - - pkg, extras, spec_str, env = match.groups() - - extras = extras or None - - if is_url_spec(spec_str): - spec = urlparse(spec_str.split("@")[1]) - elif not spec_str: - spec = None - else: - spec = SpecifierSet(spec_str) - - return (pkg, extras, spec, env) - - def is_url_spec(str_spec: str | None) -> bool: if str_spec is None: return False diff --git a/spec0_action/versions.py b/spec0_action/versions.py index 3798b8c..f575459 100644 --- a/spec0_action/versions.py +++ b/spec0_action/versions.py @@ -14,22 +14,10 @@ def tighten_lower_bound( if new_lower_bound not in spec_set: return None - out = [] - contains_lower_bound = False - - for spec in spec_set: - if spec.operator in (">", ">="): - # new_lower_bound satisfies every specifier in the set, so it can - # simply replace any existing lower bound - out.append(Specifier(f">={new_lower_bound}")) - contains_lower_bound = True - else: - out.append(spec) - - if not contains_lower_bound: - out.append(Specifier(f">={new_lower_bound}")) - - return SpecifierSet(out) + return SpecifierSet( + [spec for spec in spec_set if spec.operator not in {">", ">="}] + + [Specifier(f">={new_lower_bound}")] + ) def repr_spec_set(spec: SpecifierSet) -> str: diff --git a/spec0_versions.py b/spec0_versions.py index 457cfab..8858d40 100644 --- a/spec0_versions.py +++ b/spec0_versions.py @@ -1,10 +1,9 @@ -import collections import json from datetime import datetime, timedelta import pandas as pd -import requests -from packaging.version import InvalidVersion, Version + +from spec0_action import CORE_PACKAGES, _get_feature_release_dates PY_RELEASES = { "3.8": "Oct 14, 2019", @@ -15,18 +14,6 @@ "3.13": "Oct 7, 2024", "3.14": "Oct 7, 2025", } -CORE_PACKAGES = [ - "ipython", - "matplotlib", - "networkx", - "numpy", - "pandas", - "scikit-image", - "scikit-learn", - "scipy", - "xarray", - "zarr", -] PLUS_36_MONTHS = timedelta(days=365 * 3) PLUS_24_MONTHS = timedelta(days=365 * 2) @@ -43,41 +30,15 @@ def get_release_dates(package, support_time=PLUS_24_MONTHS): releases = {} - print(f"Querying pypi.org for {package} versions...", end="", flush=True) - response = requests.get( - f"https://pypi.org/simple/{package}", - headers={"Accept": "application/vnd.pypi.simple.v1+json"}, - ).json() - print("OK") - file_date = collections.defaultdict(list) - for f in response["files"]: - if f["filename"].endswith(".tar.gz") or f["filename"].endswith(".zip"): - continue - ver = f["filename"].split("-")[1] - try: - version = Version(ver) - except InvalidVersion as e: - print(f"Error: '{ver}' is an invalid version for '{package}'. Reason: {e}") - continue - if version.is_prerelease or version.micro != 0: - continue - release_date = None - for format in ["%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ"]: - try: - release_date = datetime.strptime(f["upload-time"], format) - except ValueError as e: - print(f"Error parsing invalid date: {e}") - if not release_date: - continue - file_date[version].append(release_date) - release_dates = {v: min(file_date[v]) for v in file_date} - for ver, release_date in sorted(release_dates.items()): - drop_date = release_date + support_time - if drop_date >= CUTOFF: - releases[ver] = { - "release_date": release_date, - "drop_date": drop_date, - } + for version, release_date in _get_feature_release_dates(package): + # Pandas quarter calculations below use naive UTC dates. + release_date = release_date.replace(tzinfo=None) + releases[version] = { + "release_date": release_date, + "drop_date": release_date + support_time, + } + if not releases: + raise RuntimeError(f"Could not find feature releases for {package}") return releases diff --git a/tests/test_cli.py b/tests/test_cli.py index 023f30f..5f8961f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,9 +1,13 @@ +import datetime import runpy import sys from pathlib import Path +from unittest.mock import patch import pytest +from packaging.version import Version +import spec0_action from spec0_action import read_toml SCRIPT = Path(__file__).resolve().parents[1] / "run_spec0_update.py" @@ -55,3 +59,69 @@ def test_cli_invalid_exclusion_preserves_file(tmp_path, monkeypatch): runpy.run_path(str(SCRIPT), run_name="__main__") assert project.read_bytes() == PROJECT.encode() + + +def test_cli_custom_support_period(tmp_path, monkeypatch): + project = tmp_path / "pyproject.toml" + project.write_text(PROJECT) + schedule = tmp_path / "schedule.json" + schedule.write_text(SCHEDULE) + monkeypatch.setattr( + sys, + "argv", + [ + str(SCRIPT), + str(project), + str(schedule), + "--spec0-support-years", + "3", + "--update-all", + "2", + "--excluded-packages", + "pandas,\nscikit-learn", + ], + ) + + with ( + patch.object( + spec0_action, "_get_spec0_floor", return_value=Version("1.27") + ) as floor, + patch.object(spec0_action, "_get_oldest_version_in_window") as fallback, + ): + runpy.run_path(str(SCRIPT), run_name="__main__") + + assert read_toml(project)["project"] == { + "requires-python": ">=3.12", + "dependencies": ["numpy>=1.27", "scikit-learn >= 1.0", "pandas>=1.0"], + } + floor.assert_called_once() + assert floor.call_args.args[:2] == ("numpy", datetime.timedelta(days=1095)) + fallback.assert_not_called() + + +@pytest.mark.parametrize("value", ["invalid", "0"]) +def test_cli_invalid_support_period_preserves_file(tmp_path, monkeypatch, value): + project = tmp_path / "pyproject.toml" + project.write_text(PROJECT) + schedule = tmp_path / "schedule.json" + schedule.write_text(SCHEDULE) + monkeypatch.setattr( + sys, + "argv", + [ + str(SCRIPT), + str(project), + str(schedule), + "--spec0-support-years", + value, + ], + ) + + with ( + pytest.raises((ValueError, SystemExit)), + patch.object(spec0_action.requests, "get") as get, + ): + runpy.run_path(str(SCRIPT), run_name="__main__") + + assert project.read_bytes() == PROJECT.encode() + get.assert_not_called() diff --git a/tests/test_parsing.py b/tests/test_parsing.py index 8eb53bf..132764b 100644 --- a/tests/test_parsing.py +++ b/tests/test_parsing.py @@ -1,11 +1,7 @@ -from urllib.parse import urlparse - import pytest from packaging.specifiers import SpecifierSet -from spec0_action.parsing import parse_pep_dependency, parse_version_spec - -URL = "https://github.com/pypa/pip/archive/1.3.1.zip#sha1=da9234ee9982d4bbb3c72346a6de940a148ea686" +from spec0_action.parsing import parse_version_spec @pytest.mark.parametrize( @@ -26,29 +22,3 @@ def test_parse_version_spec(spec_str, expected): def test_parse_version_spec_invalid(spec_str): with pytest.raises(ValueError): parse_version_spec(spec_str) - - -@pytest.mark.parametrize( - ("dep_str", "expected"), - [ - ("matplotlib", ("matplotlib", None, None, None)), - ("ruamel.yaml", ("ruamel.yaml", None, None, None)), - ("matplotlib >= 3.7.0", ("matplotlib", None, SpecifierSet(">=3.7.0"), None)), - ( - "matplotlib[foo,bar]>=3.7.0,<4", - ("matplotlib", "[foo,bar]", SpecifierSet(">=3.7.0,<4"), None), - ), - ( - "matplotlib>=3.7.0,<4;sys_platform != 'win32'", - ( - "matplotlib", - None, - SpecifierSet(">=3.7.0,<4"), - ";sys_platform != 'win32'", - ), - ), - (f"matplotlib @ {URL}", ("matplotlib", None, urlparse(f" {URL}"), None)), - ], -) -def test_parse_pep_dependency(dep_str, expected): - assert parse_pep_dependency(dep_str) == expected diff --git a/tests/test_spec0_versions.py b/tests/test_spec0_versions.py index 38c7220..a9001d3 100644 --- a/tests/test_spec0_versions.py +++ b/tests/test_spec0_versions.py @@ -4,10 +4,15 @@ from unittest.mock import Mock import pandas as pd +import pytest import requests +from packaging.version import Version + +import spec0_action def test_generator_schedule(tmp_path, monkeypatch): + spec0_action._get_release_dates.cache_clear() script = Path(__file__).resolve().parents[1] / "spec0_versions.py" monkeypatch.chdir(tmp_path) @@ -18,11 +23,14 @@ def now(tz=None): monkeypatch.setattr(pd.Timestamp, "now", now) files = [ - {"filename": f"example-{version}-py3-none-any.whl", "upload-time": date} - for version, date in [ - ("1.0.0", "2023-07-15T00:00:00.000Z"), - ("1.1.0", "2024-01-10T00:00:00Z"), - ("1.2.0", "2024-07-01T00:00:00Z"), + {"filename": filename, "upload-time": date} + for filename, date in [ + ("example-1.2.0-py3-none-any.whl", "2024-07-01T00:00:00Z"), + ("example-1.1.0.tar.gz", "2024-01-10T00:00:00Z"), + ("example-1.0.0-py3-none-any.whl", "2023-10-01T00:00:00Z"), + ("example-1.0.0.tar.gz", "2023-07-15T00:00:00.000Z"), + ("example-1.1.0.post1-py3-none-any.whl", "2025-01-01T00:00:00Z"), + ("example-1.1.1-py3-none-any.whl", "2025-01-01T00:00:00Z"), ] ] response = Mock(json=Mock(return_value={"files": files})) @@ -39,5 +47,23 @@ def now(tz=None): assert schedule["2025-07-01T00:00:00Z"] == dict.fromkeys(core, "1.1.0") assert schedule["2026-01-01T00:00:00Z"] == dict.fromkeys(core, "1.2.0") assert schedule["2025-10-01T00:00:00Z"] == {"python": "3.12"} + assert spec0_action._get_spec0_floor( + "numpy", result["PLUS_24_MONTHS"], now("UTC").to_pydatetime() + ) == Version("1.1.0") assert "gantt" in (tmp_path / "chart.md").read_text() assert "2026 - Quarter 1" in (tmp_path / "schedule.md").read_text() + + +def test_generator_pypi_failure_preserves_outputs(tmp_path, monkeypatch): + spec0_action._get_release_dates.cache_clear() + script = Path(__file__).resolve().parents[1] / "spec0_versions.py" + monkeypatch.chdir(tmp_path) + outputs = [tmp_path / name for name in ("schedule.json", "schedule.md", "chart.md")] + for path in outputs: + path.write_text("previous output") + monkeypatch.setattr( + requests, "get", Mock(side_effect=requests.ConnectionError("offline")) + ) + with pytest.raises(RuntimeError, match="Could not find feature releases"): + runpy.run_path(str(script)) + assert all(path.read_text() == "previous output" for path in outputs) diff --git a/tests/test_update_pyproject_toml.py b/tests/test_update_pyproject_toml.py index 55316a0..74e5901 100644 --- a/tests/test_update_pyproject_toml.py +++ b/tests/test_update_pyproject_toml.py @@ -30,7 +30,7 @@ def schedule(): @pytest.fixture(autouse=True) def clear_pypi_cache(): - spec0_action._get_oldest_version_in_window.cache_clear() + spec0_action._get_release_dates.cache_clear() def _minimal_pyproject(*deps): @@ -54,10 +54,34 @@ def _pypi_response(files): @pytest.mark.parametrize("name", ["pyproject", "pyproject_pixi"]) def test_update_pyproject_toml(patch_datetime_now, schedule, name): pyproject = read_toml(f"tests/test_data/{name}.toml") - update_pyproject_toml(pyproject, schedule) + with patch.object(spec0_action.requests, "get") as get: + update_pyproject_toml(pyproject, schedule) + get.assert_not_called() assert pyproject == read_toml(f"tests/test_data/{name}_updated.toml") +@pytest.mark.parametrize( + ("dependency", "expected"), + [ + ("xarray[io]>=2026.5.1", "xarray[io]>=2026.7.0"), + ( + " xarray [io, parallel] (>=2026.5.1) ; python_version < '4'", + "xarray[io, parallel]>=2026.7.0; python_version < '4'", + ), + ("xarray[io] >= 2026.8.0", "xarray[io] >= 2026.8.0"), + ], +) +def test_pep_requirement_extras_and_whitespace( + patch_datetime_now, dependency, expected +): + pyproject = _minimal_pyproject(dependency) + schedule = [ + {"start_date": "2000-01-01T00:00:00Z", "packages": {"xarray": "2026.7.0"}} + ] + update_pyproject_toml(pyproject, schedule) + assert pyproject["project"]["dependencies"] == [expected] + + @pytest.mark.parametrize( ("update_all", "expected"), [(None, "requests>=2.0.0"), (2.0, "requests>=2.28.0")] ) @@ -164,6 +188,7 @@ def test_pixi_feature_pypi_dependencies_and_non_version_tables( "dependencies": { "scikit-learn": {"git": "https://example.invalid/scikit-learn.git"}, "pandas": {"version": ">=1.0", "channel": "conda-forge"}, + "numpy": ">=1.26|>=2.0", }, "feature": { "test": { @@ -180,6 +205,7 @@ def test_pixi_feature_pypi_dependencies_and_non_version_tables( pixi = pyproject["tool"]["pixi"] assert pixi["feature"]["test"]["pypi-dependencies"]["Numpy"] == ">=2.0.0" + assert pixi["dependencies"]["numpy"] == ">=1.26|>=2.0" # version tables are updated in place, other keys kept assert pixi["dependencies"]["pandas"] == { "version": ">=2.2.0", @@ -229,16 +255,25 @@ def test_update_all_uses_version_release_date_not_new_file_upload(patch_datetime ), ], ) +@pytest.mark.parametrize("spec0_support_years", [None, 3]) def test_update_all_preserves_dependency_on_pypi_failure( - patch_datetime_now, schedule, stage, error + patch_datetime_now, schedule, stage, error, spec0_support_years, caplog ): - pyproject = _minimal_pyproject("requests >= 2.0") + package = "numpy" if spec0_support_years else "requests" + dependency = f"{package} >= 1.0" + pyproject = _minimal_pyproject(dependency) + pyproject["tool"] = {"pixi": {"dependencies": {package: ">= 1.0"}}} with patch.object(spec0_action.requests, "get") as get: operation = get if stage == "get" else getattr(get.return_value, stage) operation.side_effect = error - update_pyproject_toml(pyproject, schedule, update_all=2.0) + update_pyproject_toml( + pyproject, schedule, update_all=2.0, spec0_support_years=spec0_support_years + ) - assert pyproject["project"]["dependencies"] == ["requests >= 2.0"] + assert pyproject["project"]["dependencies"] == [dependency] + assert pyproject["tool"]["pixi"]["dependencies"] == {package: ">= 1.0"} + get.assert_called_once() + assert f"Could not fetch {package} releases" in caplog.text def test_update_all_queries_pypi_once_per_package(patch_datetime_now, schedule): @@ -332,11 +367,30 @@ def test_excluded_python(patch_datetime_now, schedule, current): assert pyproject["tool"] == pixi -def test_excluding_every_package_is_a_noop(patch_datetime_now, schedule): +@pytest.mark.parametrize("spec0_support_years", [None, 3]) +def test_excluding_every_package_is_a_noop( + patch_datetime_now, schedule, spec0_support_years +): pyproject = _minimal_pyproject("numpy>=1.10.0") excluded = [pkg for entry in schedule for pkg in entry["packages"]] - update_pyproject_toml(pyproject, schedule, excluded_packages=excluded) + with patch.object(spec0_action.requests, "get") as get: + update_pyproject_toml( + pyproject, + schedule, + excluded_packages=excluded, + spec0_support_years=spec0_support_years, + ) assert pyproject == _minimal_pyproject("numpy>=1.10.0") + get.assert_not_called() + + +def test_update_all_includes_unscheduled_core_packages(patch_datetime_now): + pyproject = _minimal_pyproject("numpy>=1.0") + schedule = [{"start_date": "2025-10-01T00:00:00Z", "packages": {"python": "3.12"}}] + with _mock_pypi("1.26") as fallback: + update_pyproject_toml(pyproject, schedule, update_all=3) + assert pyproject["project"]["dependencies"] == ["numpy>=1.26"] + fallback.assert_called_once_with("numpy", 3) @pytest.mark.parametrize("invalid", ["numpy>=1", "numpy[extra]"]) @@ -357,3 +411,208 @@ def test_exclusions_do_not_hide_invalid_schedule(patch_datetime_now): pyproject = _minimal_pyproject("numpy>=1.10.0") with pytest.raises(RuntimeError, match="Could not find schedule"): update_pyproject_toml(pyproject, [], excluded_packages=["numpy", "python"]) + + +@pytest.fixture +def feature_files(): + # Full history, a source-only release, duplicate distributions, and an old + # feature's late wheel. Non-feature versions must never advance the floor. + return [ + {"filename": filename, "upload-time": date} + for filename, date in [ + ("example-1.4.0.tar.gz", "2025-09-01T00:00:00Z"), + ("example-1.3.0.tar.gz", "2024-11-25T00:00:00Z"), + ("example-1.2.0.tar.gz", "2023-11-25T00:00:00Z"), + ("example-1.1.0-py3-none-any.whl", "2025-01-01T00:00:00Z"), + ("example-1.1.0.tar.gz", "2022-11-25T00:00:00.000Z"), + ("example-1.1.0.zip", "2022-11-26T00:00:00Z"), + ("example-1.0.0.tar.gz", "2019-12-31T00:00:00Z"), + ("example-1.3.0rc1.tar.gz", "2020-01-01T00:00:00Z"), + ("example-1.3.0.dev1.tar.gz", "2020-01-01T00:00:00Z"), + ("example-1.3.0.post1.tar.gz", "2020-01-01T00:00:00Z"), + ("example-1.3.1.tar.gz", "2020-01-01T00:00:00Z"), + ("example-1.3.0.1.tar.gz", "2020-01-01T00:00:00Z"), + ("example-1.3.0-py2.7.egg", "2020-01-01T00:00:00Z"), + ] + ] + + +@pytest.mark.parametrize( + ("years", "expected"), + [(1, "1.4.0"), (2, "1.3.0"), (3, "1.2.0"), (5, "1.1.0"), (8, "1.0")], +) +def test_custom_support_period( + patch_datetime_now, schedule, feature_files, years, expected +): + pyproject = _minimal_pyproject("numpy>=1.0") + with patch.object( + spec0_action.requests, "get", return_value=_pypi_response(feature_files) + ) as get: + update_pyproject_toml(pyproject, schedule, spec0_support_years=years) + assert pyproject["project"]["dependencies"] == [f"numpy>={expected}"] + assert pyproject["project"]["requires-python"] == ">=3.12" + get.assert_called_once_with( + "https://pypi.org/simple/numpy", + headers={"Accept": "application/vnd.pypi.simple.v1+json"}, + timeout=15, + ) + + +@pytest.mark.parametrize( + ("release_date", "years", "now", "expected"), + [ + ("2024-12-31T12:00:00Z", 1, "2025-09-30T23:59:59Z", "numpy >= 1.0"), + ("2024-12-31T12:00:00Z", 1, "2025-10-01T00:00:00Z", "numpy>=1.1"), + # Crossing leap day makes the anniversary March 31, so the drop is Q1. + ("2023-04-01T00:00:00Z", 1, "2023-12-31T23:59:59Z", "numpy >= 1.0"), + ("2023-04-01T00:00:00Z", 1, "2024-01-01T00:00:00Z", "numpy>=1.1"), + ("2025-09-30T00:00:00Z", 1 / 365, "2025-10-01T00:00:00Z", "numpy>=1.1"), + ], +) +def test_custom_support_quarter_boundaries( + patch_datetime_now, monkeypatch, release_date, years, now, expected +): + monkeypatch.setattr(f"{__name__}.FAKE_TIME", datetime.datetime.fromisoformat(now)) + pyproject = _minimal_pyproject("numpy >= 1.0") + schedule = [{"start_date": "2000-01-01T00:00:00Z", "packages": {"python": "3.12"}}] + files = [ + {"filename": f"numpy-{version}.tar.gz", "upload-time": release_date} + for version in ("1.0", "1.1") + ] + with patch.object(spec0_action.requests, "get", return_value=_pypi_response(files)): + update_pyproject_toml(pyproject, schedule, spec0_support_years=years) + assert pyproject["project"]["dependencies"] == [expected] + + +def test_custom_support_covers_pep_and_pixi_without_core_schedule_entries( + patch_datetime_now, feature_files +): + # A custom schedule still owns Python and additional packages; membership + # in the core policy cannot depend on entries in this schedule. + schedule = [ + { + "start_date": "2025-10-01T00:00:00Z", + "packages": {"python": "3.12", "custom-pkg": "4.0"}, + } + ] + pyproject = _minimal_pyproject( + "SCIKIT_LEARN[extra]>=1.0;python_version<'4'", "requests>=1", "custom-pkg>=1" + ) + pixi = { + "dependencies": { + "SCIKIT_LEARN": ">=1.0", + "requests": ">=1", + "custom-pkg": ">=1", + "python": ">=3.9", + }, + } + target = { + "pypi-dependencies": {"scikit-learn": {"version": ">=1.0", "extras": ["test"]}} + } + pixi["feature"] = {"test": {"target": {"linux-64": target}}} + pyproject["tool"] = {"pixi": pixi} + + with patch.object( + spec0_action.requests, "get", return_value=_pypi_response(feature_files) + ) as get: + update_pyproject_toml(pyproject, schedule, update_all=2, spec0_support_years=3) + + assert pyproject["project"]["dependencies"] == [ + "SCIKIT_LEARN[extra]>=1.2.0;python_version<'4'", + "requests>=1.2.0", # Existing update_all policy selects oldest in window. + "custom-pkg>=4.0", + ] + assert pyproject["project"]["requires-python"] == ">=3.12" + assert pixi["dependencies"] == { + "SCIKIT_LEARN": ">=1.2.0", + "requests": ">=1", + "custom-pkg": ">=4.0", + "python": ">=3.12", + } + assert target["pypi-dependencies"] == { + "scikit-learn": {"version": ">=1.2.0", "extras": ["test"]} + } + assert [c.args[0] for c in get.call_args_list] == [ + "https://pypi.org/simple/scikit-learn", + "https://pypi.org/simple/requests", + ] + + +def test_custom_support_skips_excluded_and_non_version_dependencies( + patch_datetime_now, schedule +): + pyproject = _minimal_pyproject( + "scipy @ https://example.invalid/scipy.whl", + "scikit_LEARN[tests]", + "pandas >= 1", + "requests >= 1", + ) + pyproject["project"]["name"] = "scikit-learn" + pixi = { + "dependencies": {"pandas": ">= 1", "python": ">= 3.9"}, + "pypi-dependencies": { + "scipy": "@ https://example.invalid/scipy.whl", + "scikit-learn": {"version": "*", "extras": ["tests"]}, + "xarray": {"git": "https://example.invalid/xarray.git"}, + }, + } + pyproject["tool"] = {"pixi": pixi} + expected = deepcopy(pyproject) + with patch.object(spec0_action.requests, "get") as get: + update_pyproject_toml( + pyproject, + schedule, + update_all=2, + spec0_support_years=3, + excluded_packages=["Pandas", "requests", "PYTHON"], + ) + assert pyproject == expected + get.assert_not_called() + + +@pytest.mark.parametrize( + "files", + [ + [], + [ + { + "filename": "numpy-1.0.post1.tar.gz", + "upload-time": "2020-01-01T00:00:00Z", + } + ], + [{"filename": "numpy-1.0.tar.gz", "upload-time": "2000-01-01T00:00:00Z"}], + ], +) +def test_custom_support_without_a_floor_never_falls_back( + patch_datetime_now, schedule, files +): + pyproject = _minimal_pyproject("numpy >= 0.5") + pyproject["tool"] = {"pixi": {"dependencies": {"numpy": ">= 0.5"}}} + with ( + patch.object( + spec0_action.requests, + "get", + return_value=_pypi_response(files), + ) as get, + _mock_pypi() as fallback, + ): + update_pyproject_toml(pyproject, schedule, update_all=2, spec0_support_years=3) + assert pyproject["project"]["dependencies"] == ["numpy >= 0.5"] + assert pyproject["tool"]["pixi"]["dependencies"] == {"numpy": ">= 0.5"} + get.assert_called_once() + fallback.assert_not_called() + + +@pytest.mark.parametrize("years", [0, -1, float("nan"), float("inf"), 3_000_000]) +def test_invalid_support_period_fails_before_mutation( + patch_datetime_now, schedule, years +): + pyproject = _minimal_pyproject("numpy>=1") + expected = deepcopy(pyproject) + with ( + patch.object(spec0_action.requests, "get") as get, + pytest.raises(ValueError, match="spec0_support_years"), + ): + update_pyproject_toml(pyproject, schedule, spec0_support_years=years) + assert pyproject == expected + get.assert_not_called()