From dc19cfab75dc47d751486837a599aaacf906170e Mon Sep 17 00:00:00 2001 From: Adam Johnson Date: Tue, 1 Sep 2026 23:26:40 +0100 Subject: [PATCH] Shuffle before other plugins' collection hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #701. Make pytest-randomly run first, even before other plugin’s collection hooks marked "tryfirst", by turning the hook into a hook wrapper. This ensures that pytest-django, and other plugins that group tests with a stable sort, apply their grouping on top of the shuffled order, making the final order reproducible from the seed. Now we require pytest 8 and pluggy 1.0, the first version to support hook wrappers. --- CHANGELOG.rst | 17 +++++++++++++ pyproject.toml | 2 +- src/pytest_randomly/__init__.py | 20 +++++++++++---- tests/test_pytest_randomly.py | 44 +++++++++++++++++++++++++++++++++ uv.lock | 16 ++++++------ 5 files changed, 85 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8b61da3..9feebb7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,23 @@ Unreleased * Support Python 3.15. +* Shuffle tests before other plugins’ ``pytest_collection_modifyitems`` hooks run. + + Previously, pytest-randomly used a plain ``tryfirst`` hook, so another plugin using ``tryfirst``, such as pytest-django, could reorder tests before or after pytest-randomly’s shuffle, depending on the plugin registration order. + Since that order can vary between environments, this would make the final test order non-repeatable. + + Now, pytest-randomly marks its hook as a hook wrapper, making it run before any other plugin’s ``pytest_collection_modifyitems`` hooks, regardless of registration order. + Plugins that group tests with a stable sort can apply their grouping on top of the shuffled order, making the final test order reproducible from the seed. + + This is a low-impact breaking change because it you may not be able to reprduce a test order from the last pytest-randomly version by reusing a ``--randomly-seed`` value. + + `PR #746 `__. + Thanks to milssky for the report in `Issue #701 `__. + +* Require pytest 8+, the first version to require a version of pluggy that supports hook wrappers, as used by the above fix. + + `PR #746 `__. + * Reset `Polyfactory `__\’s default random state at the start of every test, if it is installed. Thanks to Rahul Kumar in `PR #735 `__. diff --git a/pyproject.toml b/pyproject.toml index 2f8518c..3c20734 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "pytest", + "pytest>=8", ] urls.Changelog = "https://github.com/pytest-dev/pytest-randomly/blob/main/CHANGELOG.rst" urls.Funding = "https://adamj.eu/books/" diff --git a/src/pytest_randomly/__init__.py b/src/pytest_randomly/__init__.py index 2557786..f9f724a 100644 --- a/src/pytest_randomly/__init__.py +++ b/src/pytest_randomly/__init__.py @@ -2,7 +2,7 @@ import argparse import random -from collections.abc import Callable +from collections.abc import Callable, Generator from functools import lru_cache from importlib.metadata import entry_points from itertools import groupby @@ -205,11 +205,21 @@ def pytest_runtest_teardown(item: Item) -> None: _reseed(item.config, (_crc32(item.nodeid) + 1) % 2**32) -@hookimpl(tryfirst=True) -def pytest_collection_modifyitems(config: Config, items: list[Item]) -> None: - if not config.getoption("randomly_reorganize"): - return +@hookimpl(wrapper=True, tryfirst=True) +def pytest_collection_modifyitems( + config: Config, items: list[Item] +) -> Generator[None, None, None]: + # Reorganize within this wrapper, before the yield, so it runs before all + # non-wrapper implementations of this hook, regardless of plugin + # registration order. Plugins that group tests with a stable sort, like + # pytest-django, then apply their grouping on top of the shuffled order, + # making the final order reproducible from the seed. + if config.getoption("randomly_reorganize"): + _reorganize_items(config, items) + return (yield) + +def _reorganize_items(config: Config, items: list[Item]) -> None: seed = _reseed(config) modules_items: list[tuple[ModuleType | None, list[Item]]] = [] diff --git a/tests/test_pytest_randomly.py b/tests/test_pytest_randomly.py index 45bf7d2..b9b142e 100644 --- a/tests/test_pytest_randomly.py +++ b/tests/test_pytest_randomly.py @@ -409,6 +409,50 @@ def test_d(): ] +def test_reordered_before_other_tryfirst_hook(ourtester): + """ + The reorganization should run before other plugins’ + pytest_collection_modifyitems hooks, even tryfirst ones registered after + pytest-randomly, so plugins that group tests with a stable sort, like + pytest-django, preserve the shuffled order within their groups. + """ + ourtester.makeconftest( + """ + import pytest + + @pytest.hookimpl(tryfirst=True) + def pytest_collection_modifyitems(items): + items.sort(key=lambda item: 0 if "db" in item.name else 1) + """ + ) + ourtester.makepyfile( + test_one=""" + def test_db_a(): + pass + + def test_db_b(): + pass + + def test_plain_c(): + pass + + def test_plain_d(): + pass + """ + ) + args = ["-v", "--randomly-seed=15"] + + out = ourtester.runpytest(*args) + + out.assert_outcomes(passed=4, failed=0) + assert out.outlines[9:13] == [ + "test_one.py::test_db_a PASSED", + "test_one.py::test_db_b PASSED", + "test_one.py::test_plain_d PASSED", + "test_one.py::test_plain_c PASSED", + ] + + def test_doctests_reordered(ourtester): ourtester.makepyfile( test_one=""" diff --git a/uv.lock b/uv.lock index daf1810..b2b0a94 100644 --- a/uv.lock +++ b/uv.lock @@ -170,9 +170,9 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "asgiref" }, - { name = "sqlparse" }, - { name = "tzdata", marker = "sys_platform == 'win32'" }, + { name = "asgiref", marker = "python_full_version < '3.12'" }, + { name = "sqlparse", marker = "python_full_version < '3.12'" }, + { name = "tzdata", marker = "python_full_version < '3.12' and sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d5/d8/43e9d000519adceb189620b6869ff88031e046df91c2e9da72f8f6918399/django-5.2.17.tar.gz", hash = "sha256:9d4d93be539a18ab80d058eb515900e10951e04c537c5a6b394fc49528d3251f", size = 10889740, upload-time = "2026-08-04T15:04:03.173Z" } wheels = [ @@ -187,9 +187,9 @@ resolution-markers = [ "python_full_version >= '3.12'", ] dependencies = [ - { name = "asgiref" }, - { name = "sqlparse" }, - { name = "tzdata", marker = "sys_platform == 'win32'" }, + { name = "asgiref", marker = "python_full_version >= '3.12'" }, + { name = "sqlparse", marker = "python_full_version >= '3.12'" }, + { name = "tzdata", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e2/42/6cb20996733984c1f6661daeda3877990836c76c633c6c8879d39f7120eb/django-6.1.tar.gz", hash = "sha256:86a2aacd59b817e4d6ac2ebfe22356c58f66f7b24e503f71b7c2fead677ee48b", size = 11223034, upload-time = "2026-08-05T19:21:53.789Z" } wheels = [ @@ -201,7 +201,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -567,7 +567,7 @@ test = [ ] [package.metadata] -requires-dist = [{ name = "pytest" }] +requires-dist = [{ name = "pytest", specifier = ">=8" }] [package.metadata.requires-dev] test = [