From e770e7dee6274b9fea8b60e62f3cfa2f66a0fadf Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 31 Oct 2026 15:45:32 -0400 Subject: [PATCH 1/3] Restore the deferred imports to module scope. Six imports sat inside the functions that used them, each with a comment explaining why, and none where a reader looks for a module's dependencies. PEP 810 lets the language express that instead: declared in __lazy_modules__, they stay deferred while reading as ordinary imports. Python 3.15 honors the declaration natively. Earlier versions reach the same behavior through the lazy_importer backport, kept behind compat/py314 so the shared module carries only one import and one with statement, both removed when syncing to the stdlib. Costs about 1-2 ms of import time before 3.15, the price of importing lazy_importer, and nothing measurable on 3.15. Closes #554 Co-Authored-By: Claude Opus 5 --- importlib_metadata/__init__.py | 51 +++++++++++++++++--------- importlib_metadata/_compat/localize.py | 34 ++++++++++------- importlib_metadata/compat/py314.py | 27 ++++++++++++++ mypy.ini | 4 ++ newsfragments/554.feature.rst | 7 ++++ pyproject.toml | 2 + tests/test_main.py | 44 ++++++++++++++++++++++ 7 files changed, 139 insertions(+), 30 deletions(-) create mode 100644 importlib_metadata/compat/py314.py create mode 100644 newsfragments/554.feature.rst diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index ac0969d3..564a6867 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -44,6 +44,39 @@ SimplePath, ) from .compat import py311 +from .compat.py314 import lazy_importer + +__lazy_modules__ = { + 'csv', + 'inspect', + 'json', + # 'zipfile' in the stdlib, which needs no overlay + 'zipp.compat.overlay', + f'{__name__}._adapters', +} +""" +Imports to defer until first use (PEP 810). + +Each of them costs more than the path that reads it is traveled, so +deferring keeps that cost off every other caller. Stating it here rather +than importing inside those functions leaves the dependencies visible. + +Python 3.15 honors this declaration itself; earlier versions reach the +same end through the context below. + +Note that the adapters are imported by member and not as ``from . +import _adapters``, which names this package as the module to defer +and so would defer nothing, the package being mid-import. +""" + +with lazy_importer(): + import csv + import inspect + import json + + from zipp.compat.overlay import zipfile + + from ._adapters import Message __all__ = [ 'Distribution', @@ -598,10 +631,7 @@ def _has_metadata(self) -> bool: @staticmethod def _assemble_message(text: str) -> PackageMetadata: - # deferred for performance (python/cpython#109829) - from . import _adapters - - return _adapters.Message(email.message_from_string(text)) + return Message(email.message_from_string(text)) def _ensure_metadata_present(self, text: str | None) -> str: if text is not None: @@ -662,10 +692,6 @@ def make_file(name, hash=None, size_str=None): @pass_none def make_files(lines): - # Delay csv import, since Distribution.files is not as widely used - # as other parts of importlib.metadata - import csv - return starmap(make_file, csv.reader(lines)) @pass_none @@ -787,9 +813,6 @@ def origin(self): return self._load_json('direct_url.json') def _load_json(self, filename): - # Deferred for performance (python/importlib_metadata#503) - import json - return pass_none(json.loads)( self.read_text(filename), object_hook=lambda data: types.SimpleNamespace(**data), @@ -907,9 +930,6 @@ def children(self): return [] def zip_children(self): - # deferred for performance (python/importlib_metadata#502) - from zipp.compat.overlay import zipfile - zip_path = zipfile.Path(self.root) names = zip_path.root.namelist() self.joinpath = zip_path.joinpath @@ -1296,9 +1316,6 @@ def _get_toplevel_name(name: IPackagePath) -> str: >>> _get_toplevel_name(PackagePath('foo.dist-info')) 'foo.dist-info' """ - # Defer import of inspect for performance (python/cpython#118761) - import inspect - return _topmost(name) or inspect.getmodulename(name) or str(name) diff --git a/importlib_metadata/_compat/localize.py b/importlib_metadata/_compat/localize.py index b08b4fbd..03027d8b 100644 --- a/importlib_metadata/_compat/localize.py +++ b/importlib_metadata/_compat/localize.py @@ -6,12 +6,27 @@ import importlib_metadata from ..compat import py312 +from ..compat.py314 import lazy_importer + +__lazy_modules__ = {'importlib.metadata', 'importlib_metadata._adapters'} +""" +Imports to defer until first use (PEP 810). + +Only a foreign distribution reaches the code that reads these, so the +cost belongs to that caller and not to every import of the package. + +As in the parent module, the adapters are imported by member so that +the module to defer is ``_adapters`` and not this package. +""" + +with lazy_importer(): + import importlib.metadata + + from .._adapters import Message if TYPE_CHECKING: import email.message - import importlib.metadata - from .. import _adapters from .._meta import SimplePath @@ -55,10 +70,6 @@ def dist( if isinstance(dist, importlib_metadata.Distribution): return dist - # only a foreign dist gets this far, so defer the cost of the import - # (python/cpython#109829) - import importlib.metadata - if isinstance(dist, importlib.metadata.PathDistribution): return importlib_metadata.PathDistribution(cast('SimplePath', dist._path)) # Warn on what the interface requires and the class omits, rather than @@ -82,8 +93,8 @@ def unimplemented(dist) -> list[str]: def message( - input: _adapters.Message | email.message.Message, -) -> _adapters.Message: + input: Message | email.message.Message, +) -> Message: """ Ensure a message is adapted to an importlib_metadata.Message. @@ -93,12 +104,9 @@ def message( >>> type(local) """ - # deferred for performance (python/cpython#109829) - from .. import _adapters - - if isinstance(input, _adapters.Message): + if isinstance(input, Message): return input - return _adapters.Message(input) + return Message(input) def package_path( diff --git a/importlib_metadata/compat/py314.py b/importlib_metadata/compat/py314.py new file mode 100644 index 00000000..99af5fad --- /dev/null +++ b/importlib_metadata/compat/py314.py @@ -0,0 +1,27 @@ +import sys + + +class native: + """ + A context that does nothing, as the interpreter defers the imports itself. + """ + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + return False + + +def backport(): # pragma: no cover + """ + Borrow lazy_importer's emulation of PEP 810. + """ + from lazy_importer import lazy_importer + + return lazy_importer() + + +# Python 3.15 honors __lazy_modules__ natively (PEP 810); only earlier +# versions need the imports routed through the backport. +lazy_importer = native if sys.version_info >= (3, 15) else backport diff --git a/mypy.ini b/mypy.ini index 533fe73d..b7bdaa71 100644 --- a/mypy.ini +++ b/mypy.ini @@ -22,5 +22,9 @@ ignore_missing_imports = True [mypy-zipp.*] ignore_missing_imports = True +# Not a dependency on Python 3.15 and later, which needs no backport +[mypy-lazy_importer.*] +ignore_missing_imports = True + [mypy-test.support.*] ignore_missing_imports = True diff --git a/newsfragments/554.feature.rst b/newsfragments/554.feature.rst new file mode 100644 index 00000000..dbf7ff89 --- /dev/null +++ b/newsfragments/554.feature.rst @@ -0,0 +1,7 @@ +Restored ``csv``, ``inspect``, ``json``, ``zipfile``, ``_adapters``, and +``importlib.metadata`` to module scope, where a reader can see what the +module depends on, instead of hiding them in the functions that use them +for performance. They are declared in ``__lazy_modules__`` and so stay +deferred: Python 3.15 honors the declaration natively (PEP 810) and +earlier Pythons reach the same end through the ``lazy-importer`` +backport, which is now a dependency there. diff --git a/pyproject.toml b/pyproject.toml index ad5eeb5b..cb19b2f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,8 @@ requires-python = ">=3.11" license = "Apache-2.0" dependencies = [ "zipp>=3.20", + # PEP 810 lazy imports, native from 3.15 (python/importlib_metadata#554) + "lazy-importer>=0.1; python_version < '3.15'", ] dynamic = ["version"] diff --git a/tests/test_main.py b/tests/test_main.py index b03911a3..b41da89b 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -3,6 +3,8 @@ import io import pickle import re +import subprocess +import sys import unittest import pyfakefs.fake_filesystem_unittest as ffs @@ -108,6 +110,48 @@ def test_resolve_without_attr(self): assert ep.load() is importlib_metadata +class LazyImportTests(unittest.TestCase): + """ + The imports at module scope cost no more than the deferred ones did. + """ + + def test_declared_modules_stay_unloaded(self): + """ + Importing the package loads none of the modules it declares lazy. + + ``importlib.abc`` goes first because the package imports it + eagerly and, on some Pythons, it reaches ``inspect`` through + ``importlib.resources``. Taking the snapshot after it leaves only + what the declaration itself governs. + + Ref python/importlib_metadata#554. + """ + script = ( + 'import sys, importlib.abc; before = frozenset(sys.modules); ' + 'import importlib_metadata as md; ' + 'print(*sorted(md.__lazy_modules__ & (sys.modules.keys() - before)))' + ) + proc = subprocess.run( + [sys.executable, '-c', script], + capture_output=True, + encoding='utf-8', + check=True, + ) + assert not proc.stdout.split(), f'imported eagerly: {proc.stdout.strip()}' + + def test_declared_modules_resolve(self): + """ + Reading a lazy name yields the module itself, not a stand-in. + + The backport binds a proxy that resolves on first attribute + access, so a name read for anything else -- ``isinstance``, a + ``type()`` comparison -- could see the proxy instead. + """ + assert importlib_metadata.Message is importlib_metadata._adapters.Message + assert importlib_metadata.csv.reader is importlib.import_module('csv').reader + assert isinstance(importlib_metadata.json.loads('{}'), dict) + + class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): @staticmethod def make_pkg(name): From 8495a7f8cca07ab721f35b3e581e7e59c509a7b2 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 7 Nov 2026 14:28:37 -0500 Subject: [PATCH 2/3] Resolve a lazy name before isinstance. Before 3.15 the backport binds a proxy, which passes attribute access and calls through but answers isinstance() as no match however the object relates to the class it stands for. localize.message asks exactly that, and so adapted an already adapted message a second time on every Python before 3.15. Route such uses through compat.py314.resolve, the backport's resolve() before 3.15 and the identity after, so the behavior matches across versions. Co-Authored-By: Claude Opus 5 --- importlib_metadata/_compat/localize.py | 5 ++-- importlib_metadata/compat/py314.py | 39 ++++++++++++++++++++++---- newsfragments/554.feature.rst | 7 +++-- tests/test_main.py | 39 ++++++++++++++++++++++---- 4 files changed, 74 insertions(+), 16 deletions(-) diff --git a/importlib_metadata/_compat/localize.py b/importlib_metadata/_compat/localize.py index 03027d8b..a2e1c74b 100644 --- a/importlib_metadata/_compat/localize.py +++ b/importlib_metadata/_compat/localize.py @@ -6,7 +6,7 @@ import importlib_metadata from ..compat import py312 -from ..compat.py314 import lazy_importer +from ..compat.py314 import lazy_importer, resolve __lazy_modules__ = {'importlib.metadata', 'importlib_metadata._adapters'} """ @@ -104,7 +104,8 @@ def message( >>> type(local) """ - if isinstance(input, Message): + # resolve, as a lazy Message would answer isinstance() as no match + if isinstance(input, resolve(Message)): return input return Message(input) diff --git a/importlib_metadata/compat/py314.py b/importlib_metadata/compat/py314.py index 99af5fad..f25c5a64 100644 --- a/importlib_metadata/compat/py314.py +++ b/importlib_metadata/compat/py314.py @@ -1,9 +1,14 @@ import sys +native_lazy_imports = sys.version_info >= (3, 15) +""" +Whether the interpreter honors ``__lazy_modules__`` itself (PEP 810). +""" -class native: + +class nothing: """ - A context that does nothing, as the interpreter defers the imports itself. + A context that does nothing, the interpreter deferring the imports itself. """ def __enter__(self): @@ -22,6 +27,30 @@ def backport(): # pragma: no cover return lazy_importer() -# Python 3.15 honors __lazy_modules__ natively (PEP 810); only earlier -# versions need the imports routed through the backport. -lazy_importer = native if sys.version_info >= (3, 15) else backport +def itself(value): + """ + The value as bound, the interpreter having resolved it already. + """ + return value + + +def resolved(value): # pragma: no cover + """ + The object the backport's proxy stands for. + """ + from lazy_importer import resolve + + return resolve(value) + + +lazy_importer = nothing if native_lazy_imports else backport + +resolve = itself if native_lazy_imports else resolved +""" +Supply a lazy name where the object itself is what matters. + +A proxy passes attribute access and calls through to the object it +stands for, so most uses of a lazy name need nothing. ``isinstance`` +and other operations on the object itself see the proxy, and silently +get the wrong answer, so those must resolve the name first. +""" diff --git a/newsfragments/554.feature.rst b/newsfragments/554.feature.rst index dbf7ff89..ff70d877 100644 --- a/newsfragments/554.feature.rst +++ b/newsfragments/554.feature.rst @@ -2,6 +2,7 @@ Restored ``csv``, ``inspect``, ``json``, ``zipfile``, ``_adapters``, and ``importlib.metadata`` to module scope, where a reader can see what the module depends on, instead of hiding them in the functions that use them for performance. They are declared in ``__lazy_modules__`` and so stay -deferred: Python 3.15 honors the declaration natively (PEP 810) and -earlier Pythons reach the same end through the ``lazy-importer`` -backport, which is now a dependency there. +deferred, and deferred alike on every supported Python: 3.15 honors the +declaration natively (PEP 810) and earlier Pythons reach the same +behavior through the ``lazy-importer`` backport, which is now a +dependency there. diff --git a/tests/test_main.py b/tests/test_main.py index b41da89b..37c5922e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -139,17 +139,44 @@ def test_declared_modules_stay_unloaded(self): ) assert not proc.stdout.split(), f'imported eagerly: {proc.stdout.strip()}' - def test_declared_modules_resolve(self): + def test_reading_through_a_lazy_name(self): """ - Reading a lazy name yields the module itself, not a stand-in. + Attribute access on a lazy name reaches the real module. - The backport binds a proxy that resolves on first attribute - access, so a name read for anything else -- ``isinstance``, a - ``type()`` comparison -- could see the proxy instead. + Note that the name itself need not be the module: before 3.15 the + backport binds a proxy that passes attribute access through, so + this holds while ``is`` comparison against the module does not. """ - assert importlib_metadata.Message is importlib_metadata._adapters.Message assert importlib_metadata.csv.reader is importlib.import_module('csv').reader assert isinstance(importlib_metadata.json.loads('{}'), dict) + assert importlib_metadata.inspect.getmodulename('f.py') == 'f' + + def test_adapted_message_passes_through(self): + """ + An already adapted message survives localization unchanged. + + ``localize.message`` asks ``isinstance`` about a lazy name, which + a proxy answers as no match however the object relates to it, and + so wrongly adapts the message a second time. Run as a subprocess + because reading the name anywhere resolves it for good, which + every other test here would do first. + + Ref python/importlib_metadata#554. + """ + script = ( + 'import email.message, importlib; ' + 'from importlib_metadata._compat import localize; ' + 'adapters = importlib.import_module("importlib_metadata._adapters"); ' + 'msg = adapters.Message(email.message.Message()); ' + 'print(localize.message(msg) is msg)' + ) + proc = subprocess.run( + [sys.executable, '-c', script], + capture_output=True, + encoding='utf-8', + check=True, + ) + assert proc.stdout.strip() == 'True', 'the message was adapted twice' class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): From 61af6b32e434fd33bbea1e7a535c14fb619817c1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 8 Nov 2026 12:06:07 -0500 Subject: [PATCH 3/3] Import the adapters as a module where that is possible. Only the package's own __init__ has to import the member: naming the package as the module to defer is what no emulation can honor from inside the package itself. Elsewhere `from .. import _adapters` works, and reaching the class through the module resolves the lazy name, so isinstance sees the class and not the proxy without being told to. That leaves compat.py314.resolve with no callers, so it goes, and what it documented moves to lazy_importer, out of the shared module and into the shim the stdlib never sees. Co-Authored-By: Claude Opus 5 --- importlib_metadata/__init__.py | 6 ++-- importlib_metadata/_compat/localize.py | 18 ++++------ importlib_metadata/compat/py314.py | 48 ++++++++++---------------- tests/test_main.py | 11 +++--- 4 files changed, 34 insertions(+), 49 deletions(-) diff --git a/importlib_metadata/__init__.py b/importlib_metadata/__init__.py index 564a6867..5b829ca3 100644 --- a/importlib_metadata/__init__.py +++ b/importlib_metadata/__init__.py @@ -64,9 +64,9 @@ Python 3.15 honors this declaration itself; earlier versions reach the same end through the context below. -Note that the adapters are imported by member and not as ``from . -import _adapters``, which names this package as the module to defer -and so would defer nothing, the package being mid-import. +The adapters are imported by member rather than as ``from . import +_adapters``, which names this package as the module to defer and which +no emulation can honor from inside the package itself. """ with lazy_importer(): diff --git a/importlib_metadata/_compat/localize.py b/importlib_metadata/_compat/localize.py index a2e1c74b..c45b84db 100644 --- a/importlib_metadata/_compat/localize.py +++ b/importlib_metadata/_compat/localize.py @@ -6,23 +6,20 @@ import importlib_metadata from ..compat import py312 -from ..compat.py314 import lazy_importer, resolve +from ..compat.py314 import lazy_importer -__lazy_modules__ = {'importlib.metadata', 'importlib_metadata._adapters'} +__lazy_modules__ = {'importlib.metadata', 'importlib_metadata'} """ Imports to defer until first use (PEP 810). Only a foreign distribution reaches the code that reads these, so the cost belongs to that caller and not to every import of the package. - -As in the parent module, the adapters are imported by member so that -the module to defer is ``_adapters`` and not this package. """ with lazy_importer(): import importlib.metadata - from .._adapters import Message + from .. import _adapters if TYPE_CHECKING: import email.message @@ -93,8 +90,8 @@ def unimplemented(dist) -> list[str]: def message( - input: Message | email.message.Message, -) -> Message: + input: _adapters.Message | email.message.Message, +) -> _adapters.Message: """ Ensure a message is adapted to an importlib_metadata.Message. @@ -104,10 +101,9 @@ def message( >>> type(local) """ - # resolve, as a lazy Message would answer isinstance() as no match - if isinstance(input, resolve(Message)): + if isinstance(input, _adapters.Message): return input - return Message(input) + return _adapters.Message(input) def package_path( diff --git a/importlib_metadata/compat/py314.py b/importlib_metadata/compat/py314.py index f25c5a64..f87554ea 100644 --- a/importlib_metadata/compat/py314.py +++ b/importlib_metadata/compat/py314.py @@ -1,10 +1,5 @@ import sys -native_lazy_imports = sys.version_info >= (3, 15) -""" -Whether the interpreter honors ``__lazy_modules__`` itself (PEP 810). -""" - class nothing: """ @@ -27,30 +22,23 @@ def backport(): # pragma: no cover return lazy_importer() -def itself(value): - """ - The value as bound, the interpreter having resolved it already. - """ - return value - - -def resolved(value): # pragma: no cover - """ - The object the backport's proxy stands for. - """ - from lazy_importer import resolve - - return resolve(value) - - -lazy_importer = nothing if native_lazy_imports else backport - -resolve = itself if native_lazy_imports else resolved +lazy_importer = nothing if sys.version_info >= (3, 15) else backport """ -Supply a lazy name where the object itself is what matters. - -A proxy passes attribute access and calls through to the object it -stands for, so most uses of a lazy name need nothing. ``isinstance`` -and other operations on the object itself see the proxy, and silently -get the wrong answer, so those must resolve the name first. +Defer the imports in the block that follows, before 3.15 as after. + +Two properties of the emulation shape how a module may declare its lazy +imports. Neither holds from 3.15, which defers them natively. + +A lazy name is bound to a proxy. Attribute access and calls reach the +object it stands for, so ``csv.reader`` and ``_adapters.Message`` behave, +but ``isinstance`` and anything else handed the object itself see the +proxy and get the wrong answer in silence. Reach such an object through +a module (``_adapters.Message``), or resolve the name with +``lazy_importer.resolve`` before testing it. + +``from . import `` names the package as the module to defer. +That is fine from another module, but in the package's own ``__init__`` +the proxy stands for the module being executed and resolves itself +through the globals entry that is the proxy, until the stack is +exhausted. Import the member there instead. """ diff --git a/tests/test_main.py b/tests/test_main.py index 37c5922e..8ec2c0aa 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -155,11 +155,12 @@ def test_adapted_message_passes_through(self): """ An already adapted message survives localization unchanged. - ``localize.message`` asks ``isinstance`` about a lazy name, which - a proxy answers as no match however the object relates to it, and - so wrongly adapts the message a second time. Run as a subprocess - because reading the name anywhere resolves it for good, which - every other test here would do first. + ``localize.message`` tests it with ``isinstance``, reaching the + class through a lazy module so that the name resolves first. A + proxy named directly answers as no match however the object + relates to it, and the message would be adapted a second time. + Run as a subprocess because reading a lazy name resolves it for + good, which every other test here would do first. Ref python/importlib_metadata#554. """