feat: warn when running on an unsupported Python version

src/pytest_codspeed/plugin.py

@@ -5,6 +5,7 @@
 import json
 import os
 import random
+import sys
 from dataclasses import dataclass, field
 from pathlib import Path
 from time import time
@@ -37,6 +38,39 @@
     T = TypeVar("T")
     P = ParamSpec("P")
 
+# Mirrors `requires-python` and the `Programming Language :: Python` classifiers
+# in pyproject.toml, and the CI test matrix. Bumping a version means updating all
+# three. Outside this range nothing is validated, so measurements may be wrong.
+MIN_SUPPORTED_PYTHON_VERSION = (3, 9)
+MAX_SUPPORTED_PYTHON_VERSION = (3, 15)
+
+
+def get_python_version_warning(version: tuple[int, int]) -> str | None:
+    """Warn when the interpreter is outside the supported version range.
+
+    GitHub Actions renders a workflow command as an annotation shown outside the
+    job log, which requires it to start a line. Elsewhere the line is only
+    coloured, since no annotation mechanism exists.
+    """
+    if MIN_SUPPORTED_PYTHON_VERSION <= version <= MAX_SUPPORTED_PYTHON_VERSION:
+        return None
+
+    supported = " to ".join(
+        f"{major}.{minor}"
+        for major, minor in (
+            MIN_SUPPORTED_PYTHON_VERSION,
+            MAX_SUPPORTED_PYTHON_VERSION,
+        )
+    )
+    message = (
+        f"Python {version[0]}.{version[1]} is not officially supported by "
+        f"pytest-codspeed (supported: {supported}). Support is experimental and "
+        "untested, measurements may be unreliable."
+    )
+    if os.environ.get("GITHUB_ACTIONS") == "true":
+        return f"::warning title=Unsupported Python version::{message}"
+    return f"\033[93m{message}\033[0m"
+
 
 @pytest.hookimpl(trylast=True)
 def pytest_addoption(parser: pytest.Parser):
@@ -185,12 +219,14 @@ def pytest_plugin_registered(plugin, manager: pytest.PytestPluginManager):
 def pytest_report_header(config: pytest.Config):
     plugin = get_plugin(config)
     config_str, warns = plugin.instrument.get_instrument_config_str_and_warns()
+    python_version_warning = get_python_version_warning(sys.version_info[:2])
     out = [
         (
             f"codspeed: {__version__} ("
             f"{'enabled' if plugin.is_codspeed_enabled else 'disabled'}, {config_str}"
             ")"
         ),
+        *([python_version_warning] if python_version_warning else []),
         *warns,
     ]
     if len(plugin.disabled_plugins) > 0:

tests/test_pytest_plugin.py

@@ -8,6 +8,8 @@
     skip_without_valgrind,
 )
 
+from pytest_codspeed import plugin
+
 
 @pytest.mark.parametrize("mode", [*MeasurementMode])
 def test_plugin_enabled_with_kwargs(
@@ -38,6 +40,42 @@ def test_bench_enabled_header_without_perf(
     )
 
 
+@pytest.mark.parametrize("version", [(3, 9), (3, 14), (3, 15)])
+def test_supported_python_version_does_not_warn(version) -> None:
+    assert plugin.get_python_version_warning(version) is None
+
+
+@pytest.mark.parametrize("version", [(3, 8), (3, 16), (4, 0)])
+def test_unsupported_python_version_warns(version, monkeypatch) -> None:
+    monkeypatch.delenv("GITHUB_ACTIONS", raising=False)
+    assert plugin.get_python_version_warning(version) == (
+        f"\033[93mPython {version[0]}.{version[1]} is not officially supported by "
+        "pytest-codspeed (supported: 3.9 to 3.15). Support is experimental and "
+        "untested, measurements may be unreliable.\033[0m"
+    )
+
+
+def test_unsupported_python_version_warns_as_annotation(monkeypatch) -> None:
+    monkeypatch.setenv("GITHUB_ACTIONS", "true")
+    warning = plugin.get_python_version_warning((3, 16))
+    assert warning is not None
+    assert warning.startswith("::warning title=Unsupported Python version::Python 3.16")
+
+
+def test_python_version_warning_is_reported_in_header(
+    pytester: pytest.Pytester, monkeypatch
+) -> None:
+    # Faking `sys.version_info` would break pytest's own version-gated
+    # introspection, so the lookup is stubbed instead. The line has to start the
+    # output line for GitHub Actions to pick the annotation up.
+    monkeypatch.setattr(
+        plugin, "get_python_version_warning", lambda version: "::warning ::too new"
+    )
+    pytester.copy_example("tests/examples/test_addition_fixture.py")
+    result = pytester.runpytest("--codspeed")
+    result.stdout.fnmatch_lines(["::warning ::too new"])
+
+
 @skip_without_valgrind
 def test_plugin_enabled_by_env(pytester: pytest.Pytester, codspeed_env) -> None:
     pytester.copy_example("tests/examples/test_addition_fixture.py")