From 38d8eeeee44fa158968a9fd19d1d451da5d20f31 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Sat, 1 Aug 2026 14:09:53 +0200 Subject: [PATCH] Classify plugin startup import failures (#993) A plugin failing to load during startup escaped `_main` as an unhandled exception: Python printed a raw traceback and exited 1, which is indistinguishable from EXIT_TESTSFAILED. Meanwhile a conftest.py failing to import already returned EXIT_USAGEERROR, which is the inconsistency #993 was filed about. Split the failure into the two things it can actually mean: - the plugin cannot be found at all -- pytest was pointed at something which is not there, so this is a usage error (exit 4), matching what conftest.py import failures already do. - the plugin was found but raised while importing -- including a missing transitive dependency and a broken pytest11 entry point -- which is a defect in the plugin rather than a misuse of pytest, so it is reported as an internal error (exit 3). The plugin traceback is preserved in both the report and the PluginImportFailure cause; losing it was the main objection to the earlier attempt in #7290. Side effects: - pytest.main() now returns these exit codes instead of propagating the exception to its caller, matching its documented contract. - a bare `raise ImportError` in a plugin no longer crashes pytest's own internals with `IndexError: tuple index out of range` from `e.args[0]`. conftest.py import failures are deliberately left alone and keep returning exit 4. Co-Authored-By: Claude Opus 5 (1M context) --- changelog/993.bugfix.rst | 15 ++++ doc/en/reference/exit-codes.rst | 4 +- src/_pytest/config/__init__.py | 85 ++++++++++++++++++--- testing/acceptance_test.py | 130 +++++++++++++++++++++++++++++++- testing/python/collect.py | 3 +- testing/test_config.py | 4 +- testing/test_pluginmanager.py | 29 ++++--- testing/test_session.py | 3 +- 8 files changed, 241 insertions(+), 32 deletions(-) create mode 100644 changelog/993.bugfix.rst diff --git a/changelog/993.bugfix.rst b/changelog/993.bugfix.rst new file mode 100644 index 00000000000..aed557c6d1c --- /dev/null +++ b/changelog/993.bugfix.rst @@ -0,0 +1,15 @@ +Fixed a plugin which fails to load during startup escaping as an unhandled exception, printing a raw traceback and exiting with the accidental exit code ``1`` +-- indistinguishable from :attr:`ExitCode.TESTS_FAILED `. The failure is now reported and classified: + +* A plugin which cannot be found -- requested via ``-p``, ``pytest_plugins`` or the ``PYTEST_PLUGINS`` environment variable -- + now exits with :attr:`ExitCode.USAGE_ERROR ` (``4``), the same code already used for a ``conftest.py`` which fails to import. + +* A plugin which is found but raises while being imported -- including an installed plugin loaded through a ``pytest11`` entry point, + and a plugin whose own dependencies are missing -- now exits with :attr:`ExitCode.INTERNAL_ERROR ` (``3``), + since this is a defect in the plugin rather than a misuse of pytest. The plugin's traceback is still shown. + +As a consequence, :func:`pytest.main` now returns these exit codes instead of propagating the exception to its caller. + +A ``conftest.py`` which fails to import is unaffected and keeps returning :attr:`ExitCode.USAGE_ERROR ` (``4``). + +This also fixes an ``IndexError`` raised from pytest's internals when a plugin raised an exception with no arguments, such as a bare ``raise ImportError``. diff --git a/doc/en/reference/exit-codes.rst b/doc/en/reference/exit-codes.rst index 485bd4fe20a..b8461a40005 100644 --- a/doc/en/reference/exit-codes.rst +++ b/doc/en/reference/exit-codes.rst @@ -8,8 +8,8 @@ Running ``pytest`` can result in seven different exit codes: :Exit code 0: All tests were collected and passed successfully :Exit code 1: Tests were collected and run but some of the tests failed :Exit code 2: Test execution was interrupted by the user -:Exit code 3: Internal error happened while executing tests -:Exit code 4: pytest command line usage error +:Exit code 3: Internal error happened while executing tests, or a plugin failed to load +:Exit code 4: pytest command line usage error, including a ``conftest.py`` which fails to import or a plugin which cannot be found :Exit code 5: No tests were collected :Exit code 6: Maximum number of warnings exceeded (see :option:`--max-warnings`) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 688cc8a9054..4516bc005a8 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -141,7 +141,27 @@ def __str__(self) -> str: return f"{type(self.cause).__name__}: {self.cause} (from {self.path})" -def filter_traceback_for_conftest_import_failure( +class PluginImportFailure(Exception): + """A plugin was found, but raised while being imported. + + This is deliberately distinct from a plugin which could not be found at + all: not finding it means pytest was pointed at something that isn't there, + which is a :class:`UsageError`, while a plugin blowing up on import is a + defect in the plugin and reported as an internal error. + """ + + def __init__(self, modname: str, *, cause: BaseException) -> None: + self.modname = modname + self.cause = cause + + def __str__(self) -> str: + return ( + f"{type(self.cause).__name__}: {self.cause} " + f"(importing plugin {self.modname!r})" + ) + + +def filter_traceback_for_import_failure( entry: _pytest._code.TracebackEntry, ) -> bool: """Filter tracebacks entries which point to pytest internals or importlib. @@ -152,13 +172,11 @@ def filter_traceback_for_conftest_import_failure( return filter_traceback(entry) and "importlib" not in str(entry.path).split(os.sep) -def print_conftest_import_error(e: ConftestImportFailure, file: TextIO) -> None: - exc_info = ExceptionInfo.from_exception(e.cause) +def _print_import_error(header: str, cause: BaseException, file: TextIO) -> None: + exc_info = ExceptionInfo.from_exception(cause) tw = TerminalWriter(file) - tw.line(f"ImportError while loading conftest '{e.path}'.", red=True) - exc_info.traceback = exc_info.traceback.filter( - filter_traceback_for_conftest_import_failure - ) + tw.line(header, red=True) + exc_info.traceback = exc_info.traceback.filter(filter_traceback_for_import_failure) exc_repr = ( exc_info.getrepr(style="short", chain=False) if exc_info.traceback @@ -169,6 +187,16 @@ def print_conftest_import_error(e: ConftestImportFailure, file: TextIO) -> None: tw.line(line.rstrip(), red=True) +def print_conftest_import_error(e: ConftestImportFailure, file: TextIO) -> None: + _print_import_error( + f"ImportError while loading conftest '{e.path}'.", e.cause, file + ) + + +def print_plugin_import_error(e: PluginImportFailure, file: TextIO) -> None: + _print_import_error(f'Error while loading plugin "{e.modname}".', e.cause, file) + + def print_usage_error(e: UsageError, file: TextIO) -> None: tw = TerminalWriter(file) for msg in e.args: @@ -227,6 +255,9 @@ def _main( except ConftestImportFailure as e: print_conftest_import_error(e, file=sys.stderr) return ExitCode.USAGE_ERROR + except PluginImportFailure as e: + print_plugin_import_error(e, file=sys.stderr) + return ExitCode.INTERNAL_ERROR try: ret: ExitCode | int = config.hook.pytest_cmdline_main(config=config) @@ -919,16 +950,46 @@ def import_plugin(self, modname: str, consider_entry_points: bool = False) -> No # testing/test_config.py::test_disable_plugin_autoload. __import__(importspec) mod = sys.modules[importspec] - except ImportError as e: - raise ImportError( - f'Error importing plugin "{modname}": {e.args[0]}' - ).with_traceback(e.__traceback__) from e - except Skipped as e: self.skipped_plugins.append((modname, e.msg or "")) + except ModuleNotFoundError as e: + if _is_missing_module(e, importspec): + # The plugin itself is nowhere to be found - pytest was pointed + # at something which does not exist, so this is a usage error. + raise UsageError(f'Error importing plugin "{modname}": {e}') from e + # Some *other* module the plugin imports is missing: the plugin was + # found, so this is a defect in the plugin, not a usage error. + raise PluginImportFailure(modname, cause=e) from e + except UsageError: + raise + except Exception as e: + raise PluginImportFailure(modname, cause=e) from e else: self.register(mod, modname) + def load_setuptools_entrypoints(self, group: str, name: str | None = None) -> int: + """:meta private:""" + try: + return super().load_setuptools_entrypoints(group, name=name) + except UsageError: + raise + except Exception as e: + # An installed plugin which cannot be loaded is a defect in that + # plugin - the user did nothing wrong by having it installed. + raise PluginImportFailure(name or group, cause=e) from e + + +def _is_missing_module(e: ModuleNotFoundError, importspec: str) -> bool: + """Whether ``e`` means that ``importspec`` itself could not be found. + + A ``ModuleNotFoundError`` naming some other module means the plugin was + located but one of its own imports is unsatisfied. + """ + if e.name is None: + return False + # A missing parent package also means importspec cannot be found. + return e.name == importspec or importspec.startswith(f"{e.name}.") + def _get_plugin_specs_as_list( specs: None | types.ModuleType | str | Sequence[str], diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index f941cbe1921..af883aa61e6 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -13,6 +13,7 @@ import setuptools from _pytest.config import ExitCode +from _pytest.monkeypatch import MonkeyPatch from _pytest.pathlib import symlink_or_skip from _pytest.pytester import Pytester import pytest @@ -510,9 +511,10 @@ def test_plugins_given_as_strings( ) -> None: """Test that str values passed to main() as `plugins` arg are interpreted as module names to be imported and registered (#855).""" - with pytest.raises(ImportError) as excinfo: - pytest.main([str(pytester.path)], plugins=["invalid.module"]) - assert "invalid" in str(excinfo.value) + # A plugin which cannot be found is a usage error, reported through the + # return value rather than raised out of pytest.main() (#993). + ret = pytest.main([str(pytester.path)], plugins=["invalid.module"]) + assert ret == ExitCode.USAGE_ERROR p = pytester.path.joinpath("test_test_plugins_given_as_strings.py") p.write_text("def test_foo(): pass", encoding="utf-8") @@ -1088,6 +1090,128 @@ def main(): result.stdout.no_fnmatch_line("*INTERNALERROR>*") +class TestStartupPluginImportErrors: + """Exit codes for plugins which fail to load at startup (#993). + + A plugin which cannot be found means pytest was pointed at something which + is not there, which is a usage error; a plugin which is found but blows up + while importing is a defect in the plugin, reported as an internal error. + """ + + @pytest.fixture + def broken_plugin(self, pytester: Pytester) -> Pytester: + pytester.syspathinsert() + pytester.makepyfile(myplugin="raise ValueError('plugin is broken')") + pytester.makepyfile("def test_foo(): pass") + return pytester + + @pytest.fixture + def missing_plugin(self, pytester: Pytester) -> Pytester: + pytester.syspathinsert() + pytester.makepyfile("def test_foo(): pass") + return pytester + + def test_missing_via_cmdline(self, missing_plugin: Pytester) -> None: + result = missing_plugin.runpytest("-p", "nosuchplugin") + assert result.ret == ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(['*Error importing plugin "nosuchplugin"*']) + + def test_missing_via_conftest(self, missing_plugin: Pytester) -> None: + missing_plugin.makeconftest("pytest_plugins = ['nosuchplugin']") + result = missing_plugin.runpytest() + assert result.ret == ExitCode.USAGE_ERROR + + def test_missing_via_env( + self, missing_plugin: Pytester, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.setenv("PYTEST_PLUGINS", "nosuchplugin") + result = missing_plugin.runpytest() + assert result.ret == ExitCode.USAGE_ERROR + + def test_broken_via_cmdline(self, broken_plugin: Pytester) -> None: + result = broken_plugin.runpytest("-p", "myplugin") + assert result.ret == ExitCode.INTERNAL_ERROR + result.stderr.fnmatch_lines( + [ + 'Error while loading plugin "myplugin".', + "*myplugin.py:1: in *", + "E*ValueError: plugin is broken", + ] + ) + + def test_broken_via_conftest(self, broken_plugin: Pytester) -> None: + broken_plugin.makeconftest("pytest_plugins = ['myplugin']") + result = broken_plugin.runpytest() + assert result.ret == ExitCode.INTERNAL_ERROR + + def test_broken_via_env( + self, broken_plugin: Pytester, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.setenv("PYTEST_PLUGINS", "myplugin") + result = broken_plugin.runpytest() + assert result.ret == ExitCode.INTERNAL_ERROR + + def test_broken_via_entry_point( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + + class DummyEntryPoint: + name = "myplugin" + group = "pytest11" + + def load(self): + raise ValueError("plugin is broken") + + class Distribution: + version = "1.0" + files = ("foo.txt",) + metadata = {"name": "foo"} + entry_points = (DummyEntryPoint(),) + + monkeypatch.setattr( + importlib.metadata, "distributions", lambda: (Distribution(),) + ) + pytester.makepyfile("def test_foo(): pass") + result = pytester.runpytest() + assert result.ret == ExitCode.INTERNAL_ERROR + + def test_import_error_without_args(self, pytester: Pytester) -> None: + """A bare ``raise ImportError`` used to crash with an IndexError (#993).""" + pytester.syspathinsert() + pytester.makepyfile(myplugin="raise ImportError") + pytester.makepyfile("def test_foo(): pass") + result = pytester.runpytest("-p", "myplugin") + assert result.ret == ExitCode.INTERNAL_ERROR + result.stderr.no_fnmatch_line("*IndexError*") + result.stderr.fnmatch_lines(['Error while loading plugin "myplugin".']) + + def test_missing_dependency_is_not_a_usage_error(self, pytester: Pytester) -> None: + """The plugin was found; one of *its* imports is unsatisfied (#993).""" + pytester.syspathinsert() + pytester.makepyfile(myplugin="import nosuchdependency") + pytester.makepyfile("def test_foo(): pass") + result = pytester.runpytest("-p", "myplugin") + assert result.ret == ExitCode.INTERNAL_ERROR + + def test_missing_submodule_of_existing_package(self, pytester: Pytester) -> None: + """The package exists but the requested plugin module within it does not.""" + pytester.syspathinsert() + pytester.mkpydir("mypkg") + pytester.makepyfile("def test_foo(): pass") + result = pytester.runpytest("-p", "mypkg.nosuchmodule") + assert result.ret == ExitCode.USAGE_ERROR + + def test_conftest_import_failure_stays_a_usage_error( + self, pytester: Pytester + ) -> None: + """conftest.py is not a plugin; it keeps reporting a usage error (#993).""" + pytester.makeconftest("raise ValueError('conftest is broken')") + pytester.makepyfile("def test_foo(): pass") + result = pytester.runpytest() + assert result.ret == ExitCode.USAGE_ERROR + + def test_import_plugin_unicode_name(pytester: Pytester) -> None: pytester.makepyfile(myplugin="") pytester.makepyfile("def test(): pass") diff --git a/testing/python/collect.py b/testing/python/collect.py index c9023f98595..dddd15b8f67 100644 --- a/testing/python/collect.py +++ b/testing/python/collect.py @@ -8,6 +8,7 @@ import _pytest._code from _pytest.config import ExitCode +from _pytest.config.exceptions import UsageError from _pytest.main import Session from _pytest.monkeypatch import MonkeyPatch from _pytest.nodes import Collector @@ -80,7 +81,7 @@ def test_syntax_error_in_module(self, pytester: Pytester) -> None: def test_module_considers_pluginmanager_at_import(self, pytester: Pytester) -> None: modcol = pytester.getmodulecol("pytest_plugins='xasdlkj',") - with pytest.raises(ImportError): + with pytest.raises(UsageError): modcol.obj() def test_invalid_test_module_name(self, pytester: Pytester) -> None: diff --git a/testing/test_config.py b/testing/test_config.py index 5d19627bca7..15bb0e1002c 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -22,6 +22,7 @@ from _pytest.config import console_main from _pytest.config import ExitCode from _pytest.config import parse_warning_filter +from _pytest.config import PluginImportFailure from _pytest.config.argparsing import get_ini_default_for_type from _pytest.config.argparsing import Parser from _pytest.config.exceptions import UsageError @@ -1771,8 +1772,9 @@ def distributions(): return (Distribution(),) monkeypatch.setattr(importlib.metadata, "distributions", distributions) - with pytest.raises(ImportError): + with pytest.raises(PluginImportFailure) as excinfo: pytester.parseconfig() + assert "Don't hide me!" in str(excinfo.value) def test_importlib_metadata_broken_distribution( diff --git a/testing/test_pluginmanager.py b/testing/test_pluginmanager.py index 70c1cde2821..e2c40a38df6 100644 --- a/testing/test_pluginmanager.py +++ b/testing/test_pluginmanager.py @@ -7,8 +7,10 @@ import sys import types +from _pytest._code import ExceptionInfo from _pytest.config import Config from _pytest.config import ExitCode +from _pytest.config import PluginImportFailure from _pytest.config import PytestPluginManager from _pytest.config.exceptions import UsageError from _pytest.main import Session @@ -252,13 +254,16 @@ def test_traceback(): test_traceback() """ ) - with pytest.raises(ImportError) as excinfo: + with pytest.raises(PluginImportFailure) as excinfo: pytestpm.import_plugin("qwe") - assert str(excinfo.value).endswith( - 'Error importing plugin "qwe": Not possible to import: ☺' + assert str(excinfo.value) == ( + "ImportError: Not possible to import: ☺ (importing plugin 'qwe')" ) - assert "in test_traceback" in str(excinfo.traceback[-1]) + # The original traceback must stay reachable through the cause, otherwise + # there is no way to tell where in the plugin things went wrong. + cause = ExceptionInfo.from_exception(excinfo.value.cause) + assert "in test_traceback" in str(cause.traceback[-1]) class TestPytestPluginManager: @@ -322,7 +327,7 @@ def test_consider_env_fails_to_import( self, monkeypatch: MonkeyPatch, pytestpm: PytestPluginManager ) -> None: monkeypatch.setenv("PYTEST_PLUGINS", "nonexisting", prepend=",") - with pytest.raises(ImportError): + with pytest.raises(UsageError): pytestpm.consider_env() def test_consider_env_entry_point_name( @@ -458,9 +463,9 @@ def test_hello(pytestconfig): def test_import_plugin_importname( self, pytester: Pytester, pytestpm: PytestPluginManager ) -> None: - with pytest.raises(ImportError): + with pytest.raises(UsageError): pytestpm.import_plugin("qweqwex.y") - with pytest.raises(ImportError): + with pytest.raises(UsageError): pytestpm.import_plugin("pytest_qweqwx.y") pytester.syspathinsert() @@ -480,9 +485,9 @@ def test_import_plugin_importname( def test_import_plugin_dotted_name( self, pytester: Pytester, pytestpm: PytestPluginManager ) -> None: - with pytest.raises(ImportError): + with pytest.raises(UsageError): pytestpm.import_plugin("qweqwex.y") - with pytest.raises(ImportError): + with pytest.raises(UsageError): pytestpm.import_plugin("pytest_qweqwex.y") pytester.syspathinsert() @@ -503,17 +508,17 @@ def test_consider_conftest_deps( root=pytester.path, consider_namespace_packages=False, ) - with pytest.raises(ImportError): + with pytest.raises(UsageError): pytestpm.consider_conftest(mod, registration_name="unused") class TestPytestPluginManagerBootstrapping: def test_preparse_args(self, pytestpm: PytestPluginManager) -> None: - with pytest.raises(ImportError): + with pytest.raises(UsageError): pytestpm.consider_preparse(["xyz", "-p", "hello123"]) # Handles -p without space (#3532). - with pytest.raises(ImportError) as excinfo: + with pytest.raises(UsageError) as excinfo: pytestpm.consider_preparse(["-phello123"]) assert '"hello123"' in excinfo.value.args[0] pytestpm.consider_preparse(["-pno:hello123"]) diff --git a/testing/test_session.py b/testing/test_session.py index be1e66112d7..94738837831 100644 --- a/testing/test_session.py +++ b/testing/test_session.py @@ -2,6 +2,7 @@ from __future__ import annotations from _pytest.config import ExitCode +from _pytest.config.exceptions import UsageError from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester import pytest @@ -254,7 +255,7 @@ def test_minus_x_overridden_by_maxfail(self, pytester: Pytester) -> None: def test_plugin_specify(pytester: Pytester) -> None: - with pytest.raises(ImportError): + with pytest.raises(UsageError): pytester.parseconfig("-p", "nqweotexistent") # pytest.raises(ImportError, # "config.do_configure(config)"