From 9944870deae69bee522c0d18b102d2cfe2887956 Mon Sep 17 00:00:00 2001 From: Fabian Henze <1144183+flyser@users.noreply.github.com> Date: Mon, 1 Jun 2026 23:29:50 +0200 Subject: [PATCH 1/4] Support rerunning on subtest errors in pytest 9.0 and newer --- CHANGES.rst | 4 +- src/pytest_rerunfailures.py | 70 +++++++++++++++++++++++++++++- tests/test_pytest_rerunfailures.py | 31 +++++++++++++ 3 files changed, 103 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7960238..10556a7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,9 @@ Changelog 16.4 (unreleased) ----------------- -- Nothing changed yet. +- Rerun tests with failed subtests. This feature is only available on pytest 9.0 + and later. The pytest-subtests plugin is *not* supported. + Fixes `#315 `_. 16.3 (2026-05-22) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 127cdc9..c597a98 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -16,6 +16,12 @@ from _pytest.runner import runtestprotocol from packaging.version import parse as parse_version +try: + from _pytest.subtests import SubtestReport, failed_subtests_key +except ImportError: + failed_subtests_key = None + SubtestReport = None + try: from xdist.newhooks import pytest_handlecrashitem @@ -293,6 +299,62 @@ def _remove_failed_setup_state_from_session(item): del setup_state.stack[item] +def _remove_failed_subtests_from_report(item, report): + """ + Clean up failed subtests stash entry. + + Note: This function does nothing on pytest versions without subtests support. + """ + if failed_subtests_key is None: + return + + failed_subtests = item.config.stash.get(failed_subtests_key, None) + if failed_subtests is not None and report.nodeid in failed_subtests: + del failed_subtests[report.nodeid] + + +def _remove_failed_subtest_reports_from_stats(item): + """ + Remove already-logged SubtestReports for this item from the terminal reporter's + 'failed' stats bucket. + + SubtestReports are logged immediately during runtestprotocol (independent of + log=False), so when a rerun is triggered they must be retroactively removed + from the 'failed' category to avoid counting them as permanent failures. + + Note: This function does nothing on pytest versions without subtests support. + """ + if SubtestReport is None: + return + + tr = item.config.pluginmanager.get_plugin("terminalreporter") + if tr is None: + return + + if "failed" in tr.stats: + tr.stats["failed"] = [ + report + for report in tr.stats["failed"] + if not isinstance(report, SubtestReport) or report.nodeid != item.nodeid + ] + + +def _get_num_failed_subtests(item, report): + """ + Return the number of failed subtests. + + Note: Returns 0 on pytest versions without subtests support. + """ + if failed_subtests_key is None: + return 0 + + failed_subtests = item.config.stash.get(failed_subtests_key, None) + if failed_subtests is not None: + return failed_subtests.get(report.nodeid, 0) + + return 0 + + def _get_rerun_filter_regex(item, regex_name): rerun_marker = _get_marker(item) @@ -358,9 +420,13 @@ def _should_not_rerun(item, report, reruns): xfail = hasattr(report, "wasxfail") is_terminal_error = item._terminal_errors[report.when] condition = get_reruns_condition(item) + has_failed_subtests = ( + report.when == "call" and _get_num_failed_subtests(item, report) > 0 + ) + return ( item.execution_count > reruns - or not report.failed + or (not report.failed and not has_failed_subtests) or xfail or is_terminal_error or not condition @@ -648,6 +714,8 @@ def pytest_runtest_protocol(item, nextitem): # cleanin item's cashed results from any level of setups _remove_cached_results_from_failed_fixtures(item) _remove_failed_setup_state_from_session(item) + _remove_failed_subtests_from_report(item, report) + _remove_failed_subtest_reports_from_stats(item) break # trigger rerun else: diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index b60716c..8dfb6e0 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1,5 +1,6 @@ import random import time +from textwrap import indent from unittest import mock import pytest @@ -1526,3 +1527,33 @@ def test_pass(): result = testdir.runpytest("--reruns-mode", "bogus") assert result.ret != 0 + + +def test_failing_subtests_are_rerun(testdir): + testdir.makepyfile( + f""" + import pytest + + def test_subtests(subtests): + with subtests.test("Fails on first attempt"): + {indent(temporary_failure(), " ")} + """ + ) + + result = testdir.runpytest("--reruns", "1") + assert_outcomes(result, passed=1, rerun=1) + + +def test_too_many_failing_subtests_are_failures(testdir): + testdir.makepyfile( + """ + import pytest + + def test_subtests(subtests): + with subtests.test("Always fails"): + assert False + """ + ) + + result = testdir.runpytest("--reruns", "1") + assert_outcomes(result, passed=0, failed=2, rerun=1) From 166a9fa65abe025cbdca664210b52345fb793c51 Mon Sep 17 00:00:00 2001 From: Fabian Henze <1144183+flyser@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:52:02 +0200 Subject: [PATCH 2/4] subtests: Fix running unit tests on pytest 8.x --- src/pytest_rerunfailures.py | 2 ++ tests/test_pytest_rerunfailures.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index c597a98..f2c3268 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -19,6 +19,8 @@ try: from _pytest.subtests import SubtestReport, failed_subtests_key except ImportError: + if pytest.version_tuple >= (9, 0, 0): + raise failed_subtests_key = None SubtestReport = None diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index 8dfb6e0..e4354bb 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -5,11 +5,12 @@ import pytest -from pytest_rerunfailures import HAS_PYTEST_HANDLECRASHITEM +from pytest_rerunfailures import HAS_PYTEST_HANDLECRASHITEM, SubtestReport pytest_plugins = "pytester" has_xdist = HAS_PYTEST_HANDLECRASHITEM +has_subtests = SubtestReport is not None def temporary_failure(count=1): @@ -1529,6 +1530,7 @@ def test_pass(): assert result.ret != 0 +@pytest.mark.skipif(not has_subtests, reason="Only supported on pytest 9.0 and newer") def test_failing_subtests_are_rerun(testdir): testdir.makepyfile( f""" @@ -1544,6 +1546,7 @@ def test_subtests(subtests): assert_outcomes(result, passed=1, rerun=1) +@pytest.mark.skipif(not has_subtests, reason="Only supported on pytest 9.0 and newer") def test_too_many_failing_subtests_are_failures(testdir): testdir.makepyfile( """ From ec50adb375ecaad3da4d32c8c0b311911cb5c2de Mon Sep 17 00:00:00 2001 From: Fabian Henze <1144183+flyser@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:01:09 +0200 Subject: [PATCH 3/4] subtests support: Fix exit code when subtests are rerun --- src/pytest_rerunfailures.py | 4 ++++ tests/test_pytest_rerunfailures.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index f2c3268..d77dbe8 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -334,11 +334,15 @@ def _remove_failed_subtest_reports_from_stats(item): return if "failed" in tr.stats: + num_failed_before = len(tr.stats["failed"]) tr.stats["failed"] = [ report for report in tr.stats["failed"] if not isinstance(report, SubtestReport) or report.nodeid != item.nodeid ] + removed_count = num_failed_before - len(tr.stats["failed"]) + if removed_count > 0: + item.session.testsfailed = max(0, item.session.testsfailed - removed_count) def _get_num_failed_subtests(item, report): diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index e4354bb..21a3bf5 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1543,6 +1543,7 @@ def test_subtests(subtests): ) result = testdir.runpytest("--reruns", "1") + assert result.ret == 0 assert_outcomes(result, passed=1, rerun=1) @@ -1559,4 +1560,5 @@ def test_subtests(subtests): ) result = testdir.runpytest("--reruns", "1") + assert result.ret != 0 assert_outcomes(result, passed=0, failed=2, rerun=1) From a11c816e6c3674dac3fe0f69a8bb928fedddb582 Mon Sep 17 00:00:00 2001 From: Fabian Henze <1144183+flyser@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:54:07 +0200 Subject: [PATCH 4/4] subtests support: Fix two additional edge cases Fixes: * Wrong color in the test summary, because pytest would only check the existance, but not the value of the 'failed' key in the tr.stats * Wrong count of the passed subtests --- src/pytest_rerunfailures.py | 51 +++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index d77dbe8..0117647 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -318,11 +318,16 @@ def _remove_failed_subtests_from_report(item, report): def _remove_failed_subtest_reports_from_stats(item): """ Remove already-logged SubtestReports for this item from the terminal reporter's - 'failed' stats bucket. + stats buckets. SubtestReports are logged immediately during runtestprotocol (independent of log=False), so when a rerun is triggered they must be retroactively removed - from the 'failed' category to avoid counting them as permanent failures. + from all stat categories to avoid double-counting on the subsequent run. + + Concretely: + - Failed SubtestReports land in tr.stats["failed"]. + - Passed SubtestReports land in tr.stats["subtests passed"]. + Both must be removed so the final tally only reflects the last (successful) run. Note: This function does nothing on pytest versions without subtests support. """ @@ -333,16 +338,40 @@ def _remove_failed_subtest_reports_from_stats(item): if tr is None: return - if "failed" in tr.stats: - num_failed_before = len(tr.stats["failed"]) - tr.stats["failed"] = [ - report - for report in tr.stats["failed"] - if not isinstance(report, SubtestReport) or report.nodeid != item.nodeid + def _remove_subtest_reports(key): + """ + Remove SubtestReports for item.nodeid from tr.stats[key]. + + Returns the number of removed reports, and deletes the key entirely when + the list becomes empty, because some code just checks the presence of + the 'failed' key, but doesn't check the content. + """ + if key not in tr.stats: + return 0 + + num_items_before = len(tr.stats[key]) + tr.stats[key] = [ + r + for r in tr.stats[key] + if not isinstance(r, SubtestReport) or r.nodeid != item.nodeid ] - removed_count = num_failed_before - len(tr.stats["failed"]) - if removed_count > 0: - item.session.testsfailed = max(0, item.session.testsfailed - removed_count) + num_items_removed = num_items_before - len(tr.stats[key]) + + if not tr.stats[key]: + del tr.stats[key] + + return num_items_removed + + failed_removed = _remove_subtest_reports("failed") + if failed_removed > 0: + # Decrement session.testsfailed which was incremented when the + # SubtestReport was originally logged via pytest_runtest_logreport. + item.session.testsfailed = max(0, item.session.testsfailed - failed_removed) + + # When a test is rerun, subtests that already passed on the first attempt + # will run again and produce a second SUBPASSED report. Remove the first + # run's SUBPASSED entries so the count reflects each subtest exactly once. + _remove_subtest_reports("subtests passed") def _get_num_failed_subtests(item, report):