diff --git a/changes/356.bugfix.rst b/changes/356.bugfix.rst new file mode 100644 index 0000000..ec69a55 --- /dev/null +++ b/changes/356.bugfix.rst @@ -0,0 +1 @@ +Restore module, class, and session scoped fixture teardown when an error raised during teardown rules out a re-run. diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index b294296..22aef22 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -13,10 +13,13 @@ from typing import Any import pytest -from _pytest.outcomes import fail +from _pytest.outcomes import Exit, fail from _pytest.runner import runtestprotocol from packaging.version import parse as parse_version +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + failed_subtests_key: Any SubtestReport: Any _failed_subtests_key: Any = None @@ -898,7 +901,55 @@ def _is_rerun_path_excluded(item): ) +def _teardown_suspended_finalizers(item, call, report): + """Tear down the scopes held back for a re-run that will not happen. + + pytest_runtest_teardown takes the module, class and session finalizers off + the setup stack when it expects the test to be re-run. Whether the error a + teardown raised rules that re-run out is only known here, where the report + of the phase says how the error was classified, so put the finalizers back + and tear down whatever this item was the last user of. + + Returns the report of the teardown phase. + """ + if not getattr(item, "_finalizers_suspended", False): + return report + + item._finalizers_suspended = False + _restore_suspended_finalizers(item) + + def teardown_higher_scopes(): + try: + item.session._setupstate.teardown_exact(item._teardown_nextitem) + except (Exit, KeyboardInterrupt): + # A fixture that ends the session has to keep ending it. + raise + except BaseException as exc: + if call.excinfo is None: + raise + # Both errors come from the teardown of this item, so report them + # together, the way pytest reports several failing finalizers. + raise BaseExceptionGroup( + "errors during test teardown", [exc, call.excinfo.value] + ) from None + + teardown_call = pytest.CallInfo.from_call( + teardown_higher_scopes, when="teardown", reraise=(Exit, KeyboardInterrupt) + ) + if teardown_call.excinfo is None: + return report + + # The report of the phase is already built, so the only way for the error + # above to reach the terminal is a report that replaces it. + return pytest.TestReport.from_item_and_call(item, teardown_call) + + def pytest_runtest_teardown(item, nextitem): + # pytest_runtest_makereport needs both of these to finish a teardown this + # hook left half done because it expected a re-run. + item._finalizers_suspended = False + item._teardown_nextitem = nextitem + reruns = get_reruns_count(item) if reruns is None: # global setting is not specified, and this test is not marked with @@ -940,15 +991,17 @@ def pytest_runtest_teardown(item, nextitem): if key not in suspended_finalizers: suspended_finalizers[key] = item.session._setupstate.stack[key] del item.session._setupstate.stack[key] + item._finalizers_suspended = True else: # restore suspended finalizers _restore_suspended_finalizers(item) -@pytest.hookimpl(hookwrapper=True) +# A wrapper rather than an old-style hookwrapper because it has to be able to +# let an Exit or a KeyboardInterrupt out of _teardown_suspended_finalizers. +@pytest.hookimpl(wrapper=True) def pytest_runtest_makereport(item, call): - outcome = yield - result = outcome.get_result() + result = yield if result.when == "setup": # clean failed statuses at the beginning of each test/rerun setattr(item, "_test_failed_statuses", {}) @@ -963,6 +1016,11 @@ def pytest_runtest_makereport(item, call): item, result, call.excinfo ) + if result.when == "teardown" and item._terminal_errors["teardown"]: + result = _teardown_suspended_finalizers(item, call, result) + + return result + def pytest_runtest_protocol(item, nextitem): """ diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index c09e3f1..c257bbf 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1431,6 +1431,204 @@ def test_fail(): assert result.stdout.str().count("module teardown") == 1 +@pytest.mark.parametrize("scope", ["class", "module", "session"]) +def test_terminal_teardown_error_preserves_higher_scope_teardown(testdir, scope): + testdir.makepyfile( + f""" + import pytest + + @pytest.fixture(scope="{scope}", autouse=True) + def higher_scope_fixture(): + yield + print("{scope} teardown") + + @pytest.fixture + def broken_fixture(): + yield + raise ValueError("teardown error") + + class TestFlaky: + @pytest.mark.flaky(reruns=2, rerun_except=["ValueError"]) + def test_fail(self, broken_fixture): + assert False""" + ) + + result = testdir.runpytest("-s") + assert_outcomes(result, passed=0, failed=1, error=1, rerun=0) + result.stdout.fnmatch_lines(f"*{scope} teardown*") + + +def test_terminal_teardown_error_preserves_teardown_of_earlier_module(testdir): + testdir.makepyfile( + test_flaky_module=""" + import pytest + + @pytest.fixture(scope="module", autouse=True) + def flaky_module_fixture(): + yield + print("flaky module teardown") + + @pytest.fixture + def broken_fixture(): + yield + raise ValueError("teardown error") + + @pytest.mark.flaky(reruns=2, rerun_except=["ValueError"]) + def test_fail(broken_fixture): + assert False""", + test_later_module=""" + def test_pass(): + print("later module test")""", + ) + + result = testdir.runpytest("-s") + assert_outcomes(result, passed=1, failed=1, error=1, rerun=0) + result.stdout.fnmatch_lines( + ["*flaky module teardown*", "*later module test*"], + ) + + +def test_terminal_teardown_error_reports_failing_higher_scope_teardown(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture(scope="module", autouse=True) + def module_fixture(): + yield + raise RuntimeError("module teardown error") + + @pytest.fixture + def broken_fixture(): + yield + raise ValueError("teardown error") + + @pytest.mark.flaky(reruns=2, rerun_except=["ValueError"]) + def test_fail(broken_fixture): + assert False""" + ) + + result = testdir.runpytest() + assert_outcomes(result, passed=0, failed=1, error=1, rerun=0) + result.stdout.fnmatch_lines( + ["*RuntimeError: module teardown error*", "*ValueError: teardown error*"], + ) + + +def test_rerunnable_teardown_error_tears_down_module_fixture_once(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture(scope="module", autouse=True) + def module_fixture(): + yield + print("module teardown") + + @pytest.fixture + def broken_fixture(): + yield + raise ValueError("teardown error") + + @pytest.mark.flaky(reruns=2, only_rerun=["AssertionError", "ValueError"]) + def test_fail(broken_fixture): + assert False""" + ) + + result = testdir.runpytest("-s") + assert_outcomes(result, passed=0, failed=1, error=3, rerun=2) + assert result.stdout.str().count("module teardown") == 1 + + +@pytest.mark.parametrize( + "outcome,skipped,xfailed", + [("skip", 3, 0), ("xfail", 0, 3)], +) +def test_teardown_error_that_is_not_a_failure_does_not_stop_reruns( + testdir, outcome, skipped, xfailed +): + testdir.makepyfile( + f""" + import pytest + + @pytest.fixture(scope="module", autouse=True) + def module_fixture(): + yield + print("module teardown") + + @pytest.fixture + def not_failing_fixture(): + yield + pytest.{outcome}("{outcome} in teardown") + + @pytest.mark.flaky(reruns=2, only_rerun=["AssertionError"]) + def test_fail(not_failing_fixture): + assert False""" + ) + + result = testdir.runpytest("-s") + assert_outcomes( + result, passed=0, skipped=skipped, xfailed=xfailed, failed=1, rerun=2 + ) + assert result.stdout.str().count("module teardown") == 1 + + +def test_terminal_teardown_error_lets_a_higher_scope_teardown_exit(testdir): + testdir.makepyfile( + test_flaky_module=""" + import pytest + + @pytest.fixture(scope="module", autouse=True) + def module_fixture(): + yield + pytest.exit("exit from teardown") + + @pytest.fixture + def broken_fixture(): + yield + raise ValueError("teardown error") + + @pytest.mark.flaky(reruns=2, rerun_except=["ValueError"]) + def test_fail(broken_fixture): + assert False""", + test_later_module=""" + def test_pass(): + print("later module test")""", + ) + + result = testdir.runpytest("-s") + assert result.ret == pytest.ExitCode.INTERRUPTED + result.stdout.fnmatch_lines("*Exit: exit from teardown*") + assert "later module test" not in result.stdout.str() + + +def test_terminal_teardown_error_reports_higher_scope_teardown_without_context(testdir): + testdir.makepyfile( + """ + import pytest + + @pytest.fixture(scope="module", autouse=True) + def module_fixture(): + yield + raise RuntimeError("module teardown error") + + @pytest.fixture + def broken_fixture(): + yield + raise ValueError("teardown error") from None + + @pytest.mark.flaky(reruns=2, rerun_except=["ValueError"]) + def test_fail(broken_fixture): + assert False""" + ) + + result = testdir.runpytest() + assert_outcomes(result, passed=0, failed=1, error=1, rerun=0) + result.stdout.fnmatch_lines( + ["*RuntimeError: module teardown error*", "*ValueError: teardown error*"], + ) + + @pytest.mark.parametrize( "marker_only_rerun,cli_only_rerun,should_rerun", [