From 4e492653e84e7e697695b7698074b49630e7d049 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Thu, 27 Aug 2026 12:46:55 +0200 Subject: [PATCH] feat: warn when running on an unsupported python version Print a warning in the pytest header when the interpreter is outside the version range pytest-codspeed claims to support, so customers upgrading Python are told their measurements run on an untested interpreter instead of getting silent, unvalidated results. On GitHub Actions the warning is emitted as a `::warning` workflow command so it becomes an annotation shown outside the job log; on GitLab CI, which has no annotation mechanism, the line is coloured instead. Mirrors the `warnCi` helper of the codspeed-node counterpart, including the percent-encoding of workflow-command syntax. The bounds mirror `requires-python` and the `Programming Language :: Python` classifiers in pyproject.toml, and the CI matrix. Refs COD-3400 --- src/pytest_codspeed/plugin.py | 36 +++++++++++++++++++++++++++++++++ tests/test_pytest_plugin.py | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/src/pytest_codspeed/plugin.py b/src/pytest_codspeed/plugin.py index ca0db18..6995306 100644 --- a/src/pytest_codspeed/plugin.py +++ b/src/pytest_codspeed/plugin.py @@ -5,6 +5,7 @@ import json import os import random +import sys from dataclasses import dataclass, field from pathlib import Path from time import time @@ -37,6 +38,39 @@ T = TypeVar("T") P = ParamSpec("P") +# Mirrors `requires-python` and the `Programming Language :: Python` classifiers +# in pyproject.toml, and the CI test matrix. Bumping a version means updating all +# three. Outside this range nothing is validated, so measurements may be wrong. +MIN_SUPPORTED_PYTHON_VERSION = (3, 9) +MAX_SUPPORTED_PYTHON_VERSION = (3, 15) + + +def get_python_version_warning(version: tuple[int, int]) -> str | None: + """Warn when the interpreter is outside the supported version range. + + GitHub Actions renders a workflow command as an annotation shown outside the + job log, which requires it to start a line. Elsewhere the line is only + coloured, since no annotation mechanism exists. + """ + if MIN_SUPPORTED_PYTHON_VERSION <= version <= MAX_SUPPORTED_PYTHON_VERSION: + return None + + supported = " to ".join( + f"{major}.{minor}" + for major, minor in ( + MIN_SUPPORTED_PYTHON_VERSION, + MAX_SUPPORTED_PYTHON_VERSION, + ) + ) + message = ( + f"Python {version[0]}.{version[1]} is not officially supported by " + f"pytest-codspeed (supported: {supported}). Support is experimental and " + "untested, measurements may be unreliable." + ) + if os.environ.get("GITHUB_ACTIONS") == "true": + return f"::warning title=Unsupported Python version::{message}" + return f"\033[93m{message}\033[0m" + @pytest.hookimpl(trylast=True) def pytest_addoption(parser: pytest.Parser): @@ -185,12 +219,14 @@ def pytest_plugin_registered(plugin, manager: pytest.PytestPluginManager): def pytest_report_header(config: pytest.Config): plugin = get_plugin(config) config_str, warns = plugin.instrument.get_instrument_config_str_and_warns() + python_version_warning = get_python_version_warning(sys.version_info[:2]) out = [ ( f"codspeed: {__version__} (" f"{'enabled' if plugin.is_codspeed_enabled else 'disabled'}, {config_str}" ")" ), + *([python_version_warning] if python_version_warning else []), *warns, ] if len(plugin.disabled_plugins) > 0: diff --git a/tests/test_pytest_plugin.py b/tests/test_pytest_plugin.py index 452e9da..abfc9ad 100644 --- a/tests/test_pytest_plugin.py +++ b/tests/test_pytest_plugin.py @@ -8,6 +8,8 @@ skip_without_valgrind, ) +from pytest_codspeed import plugin + @pytest.mark.parametrize("mode", [*MeasurementMode]) def test_plugin_enabled_with_kwargs( @@ -38,6 +40,42 @@ def test_bench_enabled_header_without_perf( ) +@pytest.mark.parametrize("version", [(3, 9), (3, 14), (3, 15)]) +def test_supported_python_version_does_not_warn(version) -> None: + assert plugin.get_python_version_warning(version) is None + + +@pytest.mark.parametrize("version", [(3, 8), (3, 16), (4, 0)]) +def test_unsupported_python_version_warns(version, monkeypatch) -> None: + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + assert plugin.get_python_version_warning(version) == ( + f"\033[93mPython {version[0]}.{version[1]} is not officially supported by " + "pytest-codspeed (supported: 3.9 to 3.15). Support is experimental and " + "untested, measurements may be unreliable.\033[0m" + ) + + +def test_unsupported_python_version_warns_as_annotation(monkeypatch) -> None: + monkeypatch.setenv("GITHUB_ACTIONS", "true") + warning = plugin.get_python_version_warning((3, 16)) + assert warning is not None + assert warning.startswith("::warning title=Unsupported Python version::Python 3.16") + + +def test_python_version_warning_is_reported_in_header( + pytester: pytest.Pytester, monkeypatch +) -> None: + # Faking `sys.version_info` would break pytest's own version-gated + # introspection, so the lookup is stubbed instead. The line has to start the + # output line for GitHub Actions to pick the annotation up. + monkeypatch.setattr( + plugin, "get_python_version_warning", lambda version: "::warning ::too new" + ) + pytester.copy_example("tests/examples/test_addition_fixture.py") + result = pytester.runpytest("--codspeed") + result.stdout.fnmatch_lines(["::warning ::too new"]) + + @skip_without_valgrind def test_plugin_enabled_by_env(pytester: pytest.Pytester, codspeed_env) -> None: pytester.copy_example("tests/examples/test_addition_fixture.py")