diff --git a/changelog/13617.improvement.rst b/changelog/13617.improvement.rst new file mode 100644 index 00000000000..7a1951e1d7b --- /dev/null +++ b/changelog/13617.improvement.rst @@ -0,0 +1,12 @@ +Running with ``-v`` now lists the deselected items and, for deselections made by +pytest itself, the reason for them -- the mark or keyword expression that did not +match, ``--deselect``, ``--lf`` or ``--stepwise``: + +.. code-block:: text + + ================================ deselected ================================ + -m 'not slow' did not match: + test_it.py::test_slow + +Items deselected by a plugin are listed without a reason, because +:hook:`pytest_deselected` has no way to carry one. diff --git a/changelog/15035.feature.rst b/changelog/15035.feature.rst new file mode 100644 index 00000000000..37f80cc46c4 --- /dev/null +++ b/changelog/15035.feature.rst @@ -0,0 +1,3 @@ +New :meth:`Stash.replaced() ` context manager, which sets a +key to a value for the duration of a block and restores the previous value -- or +removes the key again if it had none -- on exit. diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py index 5fbc6771b29..a49d310825b 100644 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -26,6 +26,7 @@ from _pytest.config import hookimpl from _pytest.config.argparsing import Parser from _pytest.deprecated import check_ispytest +from _pytest.deselect import deselect_items from _pytest.fixtures import fixture from _pytest.fixtures import FixtureRequest from _pytest.main import Session @@ -406,7 +407,9 @@ def pytest_collection_modifyitems( else: if self.config.getoption("lf"): items[:] = previously_failed - config.hook.pytest_deselected(items=previously_passed) + deselect_items( + config, previously_passed, "passed in the last run (--lf)" + ) else: # --failedfirst items[:] = previously_failed + previously_passed @@ -423,7 +426,12 @@ def pytest_collection_modifyitems( self._report_status = "no previously failed tests, " if self.config.getoption("last_failed_no_failures") == "none": self._report_status += "deselecting all items." - config.hook.pytest_deselected(items=items[:]) + deselect_items( + config, + items[:], + "no test failed in the last run" + " (--lf --last-failed-no-failures=none)", + ) items[:] = [] else: self._report_status += "not deselecting items." diff --git a/src/_pytest/deselect.py b/src/_pytest/deselect.py new file mode 100644 index 00000000000..3d914dca56c --- /dev/null +++ b/src/_pytest/deselect.py @@ -0,0 +1,58 @@ +"""Recording of the reason why items were deselected. + +The reason is *side-channeled* through the config stash for the duration of a +:hook:`pytest_deselected` call instead of being passed to the hook as an +argument. + +That is a workaround, not a design: the natural spelling is +``pytest_deselected(items, reason)``. It is not available because +``pytest_deselected`` is a hook third party plugins *call* -- calling it from a +``pytest_collection_modifyitems`` implementation is part of the documented +contract -- and pluggy cannot yet evolve the arguments of a hook *call*. +Adding ``reason`` to the hookspec would leave every existing caller passing no +reason, and a caller that cannot pass one is indistinguishable from a caller +that has nothing to say, so the argument could never become required either. + +Consequently nothing here is public: pytest records reasons for its own +deselections and reads them back in its own reporting. A plugin can neither +supply a reason nor read one, and items a plugin deselects are reported without +one. Making the channel public is pointless while the underlying hook cannot +carry the value; see https://github.com/pytest-dev/pytest/issues/15036 and +https://github.com/pytest-dev/pluggy/issues/170 before building on it. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING + +from _pytest.stash import StashKey + + +if TYPE_CHECKING: + from _pytest.config import Config + from _pytest.nodes import Item + + +#: Set only while a ``pytest_deselected`` call started by :func:`deselect_items` +#: is in progress. +deselection_reason_key = StashKey[str]() + + +def deselect_items(config: Config, items: Sequence[Item], reason: str) -> None: + """Call :hook:`pytest_deselected` for *items*, recording *reason*. + + *reason* is phrased as the answer to "why is this item not selected?", e.g. + ``"-m 'slow' did not match"``. + """ + with config.stash.replaced(deselection_reason_key, reason): + config.hook.pytest_deselected(items=items) + + +def get_deselection_reason(config: Config) -> str | None: + """The reason for the ``pytest_deselected`` call currently in progress. + + ``None`` when the caller did not record one, which is the case for every + caller outside of pytest itself. + """ + return config.stash.get(deselection_reason_key, None) diff --git a/src/_pytest/hookspec.py b/src/_pytest/hookspec.py index 350b333144a..3a22668da42 100644 --- a/src/_pytest/hookspec.py +++ b/src/_pytest/hookspec.py @@ -444,6 +444,13 @@ def pytest_deselected(items: Sequence[Item]) -> None: May be called multiple times. + The hook carries no reason for the deselection, and cannot grow one: since + plugins call it, a new argument would be one that no existing caller passes, + and pluggy has no way to evolve the arguments of a hook *call*. pytest + reports a reason for its own deselections by passing it next to the call + (see ``_pytest.deselect``); items deselected by a plugin are reported + without one. + :param items: The items. diff --git a/src/_pytest/main.py b/src/_pytest/main.py index d43a68b4679..a1f1a44e71f 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -34,6 +34,7 @@ from _pytest.config import UsageError from _pytest.config.argparsing import OverrideIniAction from _pytest.config.argparsing import Parser +from _pytest.deselect import deselect_items from _pytest.nodeid import NodeId from _pytest.outcomes import exit from _pytest.pathlib import absolutepath @@ -495,7 +496,7 @@ def pytest_collection_modifyitems(items: list[nodes.Item], config: Config) -> No remaining.append(colitem) if deselected: - config.hook.pytest_deselected(items=deselected) + deselect_items(config, deselected, "node id matched --deselect") items[:] = remaining diff --git a/src/_pytest/mark/__init__.py b/src/_pytest/mark/__init__.py index 73354506df3..d3f19039152 100644 --- a/src/_pytest/mark/__init__.py +++ b/src/_pytest/mark/__init__.py @@ -26,6 +26,7 @@ from _pytest.config import hookimpl from _pytest.config import UsageError from _pytest.config.argparsing import Parser +from _pytest.deselect import deselect_items from _pytest.stash import StashKey @@ -223,7 +224,7 @@ def deselect_by_keyword(items: list[Item], config: Config) -> None: remaining.append(colitem) if deselected: - config.hook.pytest_deselected(items=deselected) + deselect_items(config, deselected, f"-k {keywordexpr!r} did not match") items[:] = remaining @@ -271,7 +272,7 @@ def deselect_by_mark(items: list[Item], config: Config) -> None: else: deselected.append(item) if deselected: - config.hook.pytest_deselected(items=deselected) + deselect_items(config, deselected, f"-m {matchexpr!r} did not match") items[:] = remaining diff --git a/src/_pytest/stash.py b/src/_pytest/stash.py index 6a9ff884e04..cb1b2bdc161 100644 --- a/src/_pytest/stash.py +++ b/src/_pytest/stash.py @@ -1,5 +1,7 @@ from __future__ import annotations +from collections.abc import Generator +import contextlib from typing import Any from typing import cast from typing import Generic @@ -91,6 +93,33 @@ def get(self, key: StashKey[T], default: D) -> T | D: except KeyError: return default + @contextlib.contextmanager + def replaced(self, key: StashKey[T], value: T) -> Generator[None]: + """Context manager which sets key to value for the duration of the block. + + On exit the previous value is restored, or the key is deleted again if + it had no value before. Nested replacements of the same key restore the + value of the enclosing block. + + .. code-block:: python + + with config.stash.replaced(some_str_key, "value"): + ... + + .. versionadded:: 9.2 + """ + absent = key not in self + previous = self._storage.get(key) + self[key] = value + try: + yield + finally: + if absent: + # The block may have deleted the key itself. + self._storage.pop(key, None) + else: + self._storage[key] = previous + def setdefault(self, key: StashKey[T], default: T) -> T: """Return the value of key if already set, otherwise set the value of key to default and return default.""" diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index c14b6fc250b..0d6291f7010 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -10,6 +10,7 @@ from _pytest.cacheprovider import Cache from _pytest.config import Config from _pytest.config.argparsing import Parser +from _pytest.deselect import deselect_items from _pytest.main import Session from _pytest.nodeid import NodeId from _pytest.reports import TestReport @@ -171,7 +172,11 @@ def pytest_collection_modifyitems( ) deselected = items[:failed_index] del items[:failed_index] - config.hook.pytest_deselected(items=deselected) + deselect_items( + config, + deselected, + "already passed before the last failure (--stepwise)", + ) def pytest_runtest_logreport(self, report: TestReport) -> None: if report.failed: diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index 023fdcaabb8..e199cd66894 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -45,6 +45,7 @@ from _pytest.config import ExitCode from _pytest.config import hookimpl from _pytest.config.argparsing import Parser +from _pytest.deselect import get_deselection_reason from _pytest.nodeid import NodeId from _pytest.nodes import Item from _pytest.nodes import Node @@ -77,6 +78,10 @@ _REPORTCHARS_DEFAULT = "fE" +#: Shown for items deselected by a plugin, which currently has no way to record +#: a reason -- see :mod:`_pytest.deselect`. +_NO_DESELECTION_REASON = "deselected by a plugin, no reason recorded" + _ConsoleOutputStyle = Literal[ "classic", "progress", "count", "times", "progress-even-when-capture-no" ] @@ -413,6 +418,7 @@ def __init__(self, config: Config, file: TextIO | None = None) -> None: self._show_progress_info = self._determine_show_progress_info() self._collect_report_last_write = timing.Instant() self._already_displayed_warnings: int | None = None + self._deselections: list[tuple[str | None, list[Item]]] = [] self._keyboardinterrupt_memo: ExceptionRepr | None = None def _determine_show_progress_info( @@ -618,6 +624,10 @@ def pytest_plugin_registered(self, plugin: _PluggyPlugin) -> None: def pytest_deselected(self, items: Sequence[Item]) -> None: self._add_stats("deselected", items) + if items: + # The reason is side-channeled rather than passed to the hook; see + # _pytest.deselect for why it is None for everyone but pytest. + self._deselections.append((get_deselection_reason(self.config), [*items])) def pytest_runtest_logstart( self, nodeid: str, location: tuple[str, int | None, str] @@ -1023,6 +1033,7 @@ def pytest_terminal_summary(self) -> Generator[None]: return (yield) finally: if show_summary: + self.summary_deselected() self.short_test_summary() # Display any extra warnings from teardown here (if any). self.summary_warnings() @@ -1297,6 +1308,16 @@ def summary_stats(self) -> None: else: self.write_line(msg, **main_markup) + def summary_deselected(self) -> None: + """List the deselected items and, where known, why they were deselected.""" + if self.verbosity < 1 or not self._deselections: + return + self.write_sep("=", "deselected") + for reason, items in self._deselections: + self.write_line(f"{reason or _NO_DESELECTION_REASON}:", yellow=True) + for item in items: + self.write_line(f" {item.nodeid}") + def short_test_summary(self) -> None: if not self.reportchars: return diff --git a/testing/test_stash.py b/testing/test_stash.py index c7f6f4f95fe..24eabf7c28e 100644 --- a/testing/test_stash.py +++ b/testing/test_stash.py @@ -67,3 +67,57 @@ def test_stash() -> None: assert stash2[key2] + stash2[key3] == 300 assert stash[key2] == 1 assert key3 not in stash + + +def test_stash_replaced_restores_previous_value() -> None: + stash = Stash() + key = StashKey[str]() + stash[key] = "before" + + with stash.replaced(key, "during"): + assert stash[key] == "during" + + assert stash[key] == "before" + + +def test_stash_replaced_removes_a_key_that_was_absent() -> None: + stash = Stash() + key = StashKey[str]() + + with stash.replaced(key, "during"): + assert stash[key] == "during" + + assert key not in stash + + +def test_stash_replaced_restores_on_exception() -> None: + stash = Stash() + key = StashKey[str]() + stash[key] = "before" + + with pytest.raises(ValueError), stash.replaced(key, "during"): + raise ValueError + + assert stash[key] == "before" + + +def test_stash_replaced_nests() -> None: + stash = Stash() + key = StashKey[str]() + + with stash.replaced(key, "outer"): + with stash.replaced(key, "inner"): + assert stash[key] == "inner" + assert stash[key] == "outer" + + assert key not in stash + + +def test_stash_replaced_tolerates_deletion_inside_the_block() -> None: + stash = Stash() + key = StashKey[str]() + + with stash.replaced(key, "during"): + del stash[key] + + assert key not in stash diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index d2ad3bae500..d37d1b47272 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -141,8 +141,10 @@ def test_fail_and_continue_with_stepwise(stepwise_pytester: Pytester) -> None: assert _strip_resource_warnings(result.stderr.lines) == [] stdout = result.stdout.str() - # Make sure the latest failing test runs and then continues. - assert "test_success_before_fail" not in stdout + # Make sure the latest failing test runs and then continues. The already + # passed test is still named by the deselected summary, so check that it did + # not run rather than that it is absent. + assert "test_success_before_fail PASSED" not in stdout assert "test_fail_on_flag PASSED" in stdout assert "test_success_after_fail PASSED" in stdout diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 30208084ab2..40d2e453a9c 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -3576,3 +3576,175 @@ def test_session_lifecycle( # Session finish - should remove progress. plugin.pytest_sessionfinish() assert "\x1b]9;4;0;\x1b\\" in mock_file.getvalue() + + +class TestDeselectedSummary: + """The ``deselected`` section listing items and why they were deselected.""" + + @pytest.fixture + def pytester(self, pytester: Pytester) -> Pytester: + pytester.makeini("[pytest]\nmarkers = slow\n") + pytester.makepyfile( + test_it=""" + import pytest + + @pytest.mark.slow + def test_slow(): pass + + def test_fast(): pass + """ + ) + return pytester + + def test_not_shown_without_verbosity(self, pytester: Pytester) -> None: + result = pytester.runpytest("-m", "not slow") + result.stdout.no_fnmatch_line("*deselected*did not match*") + result.stdout.fnmatch_lines(["*1 passed, 1 deselected*"]) + + def test_mark_expression(self, pytester: Pytester) -> None: + result = pytester.runpytest("-v", "-m", "not slow") + result.stdout.fnmatch_lines( + [ + "*= deselected =*", + "-m 'not slow' did not match:", + " test_it.py::test_slow", + ] + ) + + def test_keyword_expression(self, pytester: Pytester) -> None: + result = pytester.runpytest("-v", "-k", "fast") + result.stdout.fnmatch_lines( + ["*= deselected =*", "-k 'fast' did not match:", " test_it.py::test_slow"] + ) + + def test_deselect_option(self, pytester: Pytester) -> None: + result = pytester.runpytest("-v", "--deselect", "test_it.py::test_slow") + result.stdout.fnmatch_lines( + [ + "*= deselected =*", + "node id matched --deselect:", + " test_it.py::test_slow", + ] + ) + + def test_shown_with_collect_only(self, pytester: Pytester) -> None: + result = pytester.runpytest("-v", "--collect-only", "-m", "not slow") + result.stdout.fnmatch_lines( + ["*= deselected =*", "-m 'not slow' did not match:"] + ) + + def test_last_failed(self, pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + def test_pass(): pass + + def test_fail(): assert False + """ + ) + pytester.runpytest() + # Pass the file explicitly so that the passing item survives collection + # and is deselected rather than never collected. + result = pytester.runpytest("-v", "--lf", "test_it.py") + result.stdout.fnmatch_lines( + [ + "*= deselected =*", + "passed in the last run (--lf):", + " test_it.py::test_pass", + ] + ) + + def test_last_failed_no_failures_none(self, pytester: Pytester) -> None: + result = pytester.runpytest( + "-v", "--lf", "--last-failed-no-failures=none", "test_it.py" + ) + result.stdout.fnmatch_lines( + [ + "*= deselected =*", + "no test failed in the last run (--lf --last-failed-no-failures=none):", + " test_it.py::test_slow", + " test_it.py::test_fast", + ] + ) + + def test_stepwise(self, pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + def test_pass(): pass + + def test_fail(): assert False + """ + ) + pytester.runpytest("--stepwise") + result = pytester.runpytest("-v", "--stepwise") + result.stdout.fnmatch_lines( + [ + "*= deselected =*", + "already passed before the last failure (--stepwise):", + " test_it.py::test_pass", + ] + ) + + def test_plugin_deselection_has_no_reason(self, pytester: Pytester) -> None: + """A plugin cannot record a reason -- see _pytest.deselect.""" + pytester.makeconftest( + """ + def pytest_collection_modifyitems(config, items): + deselected = [item for item in items if item.name == "test_slow"] + for item in deselected: + items.remove(item) + config.hook.pytest_deselected(items=deselected) + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "*= deselected =*", + "deselected by a plugin, no reason recorded:", + " test_it.py::test_slow", + ] + ) + + def test_reason_is_only_set_during_the_hook_call(self, pytester: Pytester) -> None: + """The side channel does not leak past the call that sets it.""" + pytester.makeconftest( + """ + from _pytest.deselect import get_deselection_reason + + seen = [] + + def pytest_deselected(items): + seen.append(get_deselection_reason(items[0].config)) + + def pytest_collection_finish(session): + assert seen == ["-m 'not slow' did not match"], seen + assert get_deselection_reason(session.config) is None + """ + ) + result = pytester.runpytest("-m", "not slow") + assert result.ret == 0 + + def test_nested_deselection_restores_the_outer_reason( + self, pytester: Pytester + ) -> None: + """An implementation that deselects further items does not clobber it.""" + pytester.makeconftest( + """ + from _pytest.deselect import deselect_items + from _pytest.deselect import get_deselection_reason + + seen = [] + + def pytest_deselected(items): + reason = get_deselection_reason(items[0].config) + seen.append(reason) + if reason != "inner": + deselect_items(items[0].config, items, "inner") + seen.append(get_deselection_reason(items[0].config)) + + def pytest_collection_finish(session): + assert seen == ["-m 'not slow' did not match", "inner", + "-m 'not slow' did not match"], seen + """ + ) + result = pytester.runpytest("-m", "not slow") + assert result.ret == 0