diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index ac0969d3..5b829ca3 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -44,6 +44,39 @@ SimplePath, ) from .compat import py311 +from .compat.py314 import lazy_importer + +__lazy_modules__ = { + 'csv', + 'inspect', + 'json', + # 'zipfile' in the stdlib, which needs no overlay + 'zipp.compat.overlay', + f'{__name__}._adapters', +} +""" +Imports to defer until first use (PEP 810). + +Each of them costs more than the path that reads it is traveled, so +deferring keeps that cost off every other caller. Stating it here rather +than importing inside those functions leaves the dependencies visible. + +Python 3.15 honors this declaration itself; earlier versions reach the +same end through the context below. + +The adapters are imported by member rather than as ``from . import +_adapters``, which names this package as the module to defer and which +no emulation can honor from inside the package itself. +""" + +with lazy_importer(): + import csv + import inspect + import json + + from zipp.compat.overlay import zipfile + + from ._adapters import Message __all__ = [ 'Distribution', @@ -598,10 +631,7 @@ def _has_metadata(self) -> bool: @staticmethod def _assemble_message(text: str) -> PackageMetadata: - # deferred for performance (python/cpython#109829) - from . import _adapters - - return _adapters.Message(email.message_from_string(text)) + return Message(email.message_from_string(text)) def _ensure_metadata_present(self, text: str | None) -> str: if text is not None: @@ -662,10 +692,6 @@ def make_file(name, hash=None, size_str=None): @pass_none def make_files(lines): - # Delay csv import, since Distribution.files is not as widely used - # as other parts of importlib.metadata - import csv - return starmap(make_file, csv.reader(lines)) @pass_none @@ -787,9 +813,6 @@ def origin(self): return self._load_json('direct_url.json') def _load_json(self, filename): - # Deferred for performance (python/importlib_metadata#503) - import json - return pass_none(json.loads)( self.read_text(filename), object_hook=lambda data: types.SimpleNamespace(**data), @@ -907,9 +930,6 @@ def children(self): return [] def zip_children(self): - # deferred for performance (python/importlib_metadata#502) - from zipp.compat.overlay import zipfile - zip_path = zipfile.Path(self.root) names = zip_path.root.namelist() self.joinpath = zip_path.joinpath @@ -1296,9 +1316,6 @@ def _get_toplevel_name(name: IPackagePath) -> str: >>> _get_toplevel_name(PackagePath('foo.dist-info')) 'foo.dist-info' """ - # Defer import of inspect for performance (python/cpython#118761) - import inspect - return _topmost(name) or inspect.getmodulename(name) or str(name) diff --git a/importlib_metadata/_compat/localize.py b/importlib_metadata/_compat/localize.py index b08b4fbd..c45b84db 100644 --- a/importlib_metadata/_compat/localize.py +++ b/importlib_metadata/_compat/localize.py @@ -6,12 +6,24 @@ import importlib_metadata from ..compat import py312 +from ..compat.py314 import lazy_importer -if TYPE_CHECKING: - import email.message +__lazy_modules__ = {'importlib.metadata', 'importlib_metadata'} +""" +Imports to defer until first use (PEP 810). + +Only a foreign distribution reaches the code that reads these, so the +cost belongs to that caller and not to every import of the package. +""" + +with lazy_importer(): import importlib.metadata from .. import _adapters + +if TYPE_CHECKING: + import email.message + from .._meta import SimplePath @@ -55,10 +67,6 @@ def dist( if isinstance(dist, importlib_metadata.Distribution): return dist - # only a foreign dist gets this far, so defer the cost of the import - # (python/cpython#109829) - import importlib.metadata - if isinstance(dist, importlib.metadata.PathDistribution): return importlib_metadata.PathDistribution(cast('SimplePath', dist._path)) # Warn on what the interface requires and the class omits, rather than @@ -93,9 +101,6 @@ def message( >>> type(local) """ - # deferred for performance (python/cpython#109829) - from .. import _adapters - if isinstance(input, _adapters.Message): return input return _adapters.Message(input) diff --git a/importlib_metadata/compat/py314.py b/importlib_metadata/compat/py314.py new file mode 100644 index 00000000..f87554ea --- /dev/null +++ b/importlib_metadata/compat/py314.py @@ -0,0 +1,44 @@ +import sys + + +class nothing: + """ + A context that does nothing, the interpreter deferring the imports itself. + """ + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + return False + + +def backport(): # pragma: no cover + """ + Borrow lazy_importer's emulation of PEP 810. + """ + from lazy_importer import lazy_importer + + return lazy_importer() + + +lazy_importer = nothing if sys.version_info >= (3, 15) else backport +""" +Defer the imports in the block that follows, before 3.15 as after. + +Two properties of the emulation shape how a module may declare its lazy +imports. Neither holds from 3.15, which defers them natively. + +A lazy name is bound to a proxy. Attribute access and calls reach the +object it stands for, so ``csv.reader`` and ``_adapters.Message`` behave, +but ``isinstance`` and anything else handed the object itself see the +proxy and get the wrong answer in silence. Reach such an object through +a module (``_adapters.Message``), or resolve the name with +``lazy_importer.resolve`` before testing it. + +``from . import `` names the package as the module to defer. +That is fine from another module, but in the package's own ``__init__`` +the proxy stands for the module being executed and resolves itself +through the globals entry that is the proxy, until the stack is +exhausted. Import the member there instead. +""" diff --git a/mypy.ini b/mypy.ini index 533fe73d..b7bdaa71 100644 --- a/mypy.ini +++ b/mypy.ini @@ -22,5 +22,9 @@ ignore_missing_imports = True [mypy-zipp.*] ignore_missing_imports = True +# Not a dependency on Python 3.15 and later, which needs no backport +[mypy-lazy_importer.*] +ignore_missing_imports = True + [mypy-test.support.*] ignore_missing_imports = True diff --git a/newsfragments/554.feature.rst b/newsfragments/554.feature.rst new file mode 100644 index 00000000..ff70d877 --- /dev/null +++ b/newsfragments/554.feature.rst @@ -0,0 +1,8 @@ +Restored ``csv``, ``inspect``, ``json``, ``zipfile``, ``_adapters``, and +``importlib.metadata`` to module scope, where a reader can see what the +module depends on, instead of hiding them in the functions that use them +for performance. They are declared in ``__lazy_modules__`` and so stay +deferred, and deferred alike on every supported Python: 3.15 honors the +declaration natively (PEP 810) and earlier Pythons reach the same +behavior through the ``lazy-importer`` backport, which is now a +dependency there. diff --git a/pyproject.toml b/pyproject.toml index ad5eeb5b..cb19b2f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,8 @@ requires-python = ">=3.11" license = "Apache-2.0" dependencies = [ "zipp>=3.20", + # PEP 810 lazy imports, native from 3.15 (python/importlib_metadata#554) + "lazy-importer>=0.1; python_version < '3.15'", ] dynamic = ["version"] diff --git a/tests/test_main.py b/tests/test_main.py index b03911a3..8ec2c0aa 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,6 +3,8 @@ import io import pickle import re +import subprocess +import sys import unittest import pyfakefs.fake_filesystem_unittest as ffs @@ -108,6 +110,76 @@ def test_resolve_without_attr(self): assert ep.load() is importlib_metadata +class LazyImportTests(unittest.TestCase): + """ + The imports at module scope cost no more than the deferred ones did. + """ + + def test_declared_modules_stay_unloaded(self): + """ + Importing the package loads none of the modules it declares lazy. + + ``importlib.abc`` goes first because the package imports it + eagerly and, on some Pythons, it reaches ``inspect`` through + ``importlib.resources``. Taking the snapshot after it leaves only + what the declaration itself governs. + + Ref python/importlib_metadata#554. + """ + script = ( + 'import sys, importlib.abc; before = frozenset(sys.modules); ' + 'import importlib_metadata as md; ' + 'print(*sorted(md.__lazy_modules__ & (sys.modules.keys() - before)))' + ) + proc = subprocess.run( + [sys.executable, '-c', script], + capture_output=True, + encoding='utf-8', + check=True, + ) + assert not proc.stdout.split(), f'imported eagerly: {proc.stdout.strip()}' + + def test_reading_through_a_lazy_name(self): + """ + Attribute access on a lazy name reaches the real module. + + Note that the name itself need not be the module: before 3.15 the + backport binds a proxy that passes attribute access through, so + this holds while ``is`` comparison against the module does not. + """ + assert importlib_metadata.csv.reader is importlib.import_module('csv').reader + assert isinstance(importlib_metadata.json.loads('{}'), dict) + assert importlib_metadata.inspect.getmodulename('f.py') == 'f' + + def test_adapted_message_passes_through(self): + """ + An already adapted message survives localization unchanged. + + ``localize.message`` tests it with ``isinstance``, reaching the + class through a lazy module so that the name resolves first. A + proxy named directly answers as no match however the object + relates to it, and the message would be adapted a second time. + Run as a subprocess because reading a lazy name resolves it for + good, which every other test here would do first. + + Ref python/importlib_metadata#554. + """ + script = ( + 'import email.message, importlib; ' + 'from importlib_metadata._compat import localize; ' + 'adapters = importlib.import_module("importlib_metadata._adapters"); ' + 'msg = adapters.Message(email.message.Message()); ' + 'print(localize.message(msg) is msg)' + ) + proc = subprocess.run( + [sys.executable, '-c', script], + capture_output=True, + encoding='utf-8', + check=True, + ) + assert proc.stdout.strip() == 'True', 'the message was adapted twice' + + class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): @staticmethod def make_pkg(name):