diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 365b6e781bbc..7c86ce5e0638 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -13,6 +13,8 @@ from __future__ import annotations +import importlib.machinery +import os from collections import defaultdict from collections.abc import Iterable from typing import NamedTuple @@ -73,6 +75,7 @@ tuple_rprimitive, ) from mypyc.irbuild.mapper import Mapper +from mypyc.irbuild.safe_bases import SAFE_STUB_BASE_CLASSES from mypyc.irbuild.util import ( dataclass_type, default_attr_name, @@ -127,7 +130,9 @@ def build_type_map( for module, cdef in classes: with catch_errors(module.path, cdef.line): if mapper.type_to_ir[cdef.info].is_ext_class: - prepare_class_def(module.path, module.fullname, cdef, errors, mapper, options) + prepare_class_def( + module.path, module.fullname, cdef, errors, mapper, options, graph + ) else: prepare_non_ext_class_def( module.path, module.fullname, cdef, errors, mapper, options @@ -584,6 +589,56 @@ def check_unknown_base_classes(path: str, cdef: ClassDef, errors: Errors, mapper report_needs_non_native_class(errors, msg, path, cdef.line) +def find_unsupported_base( + info: TypeInfo, mapper: Mapper, graph: Graph +) -> tuple[TypeInfo, str] | None: + """Find a non-mypyc base class whose instance layout may conflict with a native class. + + Native classes assume that non-mypyc base classes have the layout of object (or of + dict/BaseException, which are checked elsewhere). Regular Python classes defined in + source files are fine unless they use __slots__, but stubs don't tell whether a class + is implemented in C, so classes defined in stubs must be explicitly allowed. + + Return the base class and a description of the problem, or None if all bases are ok. + """ + seen = set() + todo = [base.type for base in info.bases] + while todo: + base = todo.pop() + if base in seen or base.fullname.startswith("builtins."): + # Builtin base classes are checked in prepare_class_def + continue + seen.add(base) + base_ir = mapper.type_to_ir.get(base) + if base_ir is not None: + if not base_ir.is_ext_class: + # A compiled non-native class is a regular Python class + todo.extend(b.type for b in base.bases) + # Native base classes are checked separately + continue + state = graph.get(base.module_name) + path = state.xpath if state is not None else "" + if path.endswith(".pyi") or (state is not None and state.tree and state.tree.is_stub): + if base.fullname not in SAFE_STUB_BASE_CLASSES: + return base, "defined in a stub, so its instance layout is unknown" + # The listed classes have a known layout, including their base classes + continue + if base.slots: + return base, "defines __slots__" + if is_compiled_module_path(path): + return base, "its instance layout is unknown, since it's imported from a C extension" + todo.extend(b.type for b in base.bases) + return None + + +def is_compiled_module_path(path: str) -> bool: + """Is there a C extension next to a Python source file that would be imported instead?""" + if not path.endswith(".py"): + return False + stem = path[:-3] + return any(os.path.isfile(stem + suffix) for suffix in importlib.machinery.EXTENSION_SUFFIXES) + + def prepare_class_def( path: str, module_name: str, @@ -591,6 +646,7 @@ def prepare_class_def( errors: Errors, mapper: Mapper, options: CompilerOptions, + graph: Graph, ) -> None: """Populate the interface-level information in a class IR. @@ -614,6 +670,7 @@ def prepare_class_def( ir.is_acyclic = True # Check for subclassing from builtin types + reported_base = False for cls in info.mro: # Special case exceptions and dicts # XXX: How do we handle *other* things?? @@ -630,6 +687,18 @@ def prepare_class_def( report_needs_non_native_class( errors, "Inheriting from most builtin types is unimplemented", path, cdef.line ) + reported_base = True + + if not reported_base and not info.fallback_to_any: + unsupported = find_unsupported_base(info, mapper, graph) + if unsupported is not None: + base, reason = unsupported + report_needs_non_native_class( + errors, + f'Native classes cannot inherit from "{base.fullname}" ({reason})', + path, + cdef.line, + ) free_list_len = attrs.get("free_list_len") if free_list_len is not None: diff --git a/mypyc/irbuild/safe_bases.py b/mypyc/irbuild/safe_bases.py new file mode 100644 index 000000000000..260471e217b0 --- /dev/null +++ b/mypyc/irbuild/safe_bases.py @@ -0,0 +1,218 @@ +"""Standard library classes that native classes can inherit from. + +Stubs don't describe the instance layout of a class, and a class implemented +in C may have instance fields that native attributes would overlap. So a native +class can only inherit from a non-mypyc class defined in a stub if the class is +listed here (see find_unsupported_base in prepare.py). + +The list only covers classes that are commonly used as base classes in +practice, to keep it short and maintainable: + +- used as a base class in at least two of the 1000 most downloaded PyPI + projects, in a large private codebase (surveyed in November 2026), or in + mypy itself, +- documented in the CPython docs, and +- with the instance layout of object (or BaseException) on Python 3.10 to 3.14 + (including the free-threaded build) and on 3.15, wherever the class exists, + checked at runtime. Native subclasses were also compiled and tested. + +Names are the full names mypy uses, i.e. those of the modules where typeshed +defines the classes, which aren't always the public modules (for example, +asyncio.Protocol is asyncio.protocols.Protocol). Where the stubs define a class +in a different place on some Python version, both names are listed. + +Classes only available on Python 3.10 and 3.11 are left out. The Windows definitions +of asyncio.SelectorEventLoop and asyncio.DefaultEventLoopPolicy were added based on +the CPython sources, since their Unix equivalents are on the list. +""" + +from __future__ import annotations + +from typing import Final + +# These have a safe layout but are left out for now: they keep weak references to +# the instance, which doesn't work yet with native subclasses of interpreted classes. +# Add them back once that's supported: +# +# logging.FileHandler, logging.Handler, logging.NullHandler, logging.StreamHandler, +# logging.handlers.BufferingHandler, logging.handlers.RotatingFileHandler, +# logging.handlers.SocketHandler, multiprocessing.context.Process, +# multiprocessing.pool.Pool, multiprocessing.pool.ThreadPool, +# multiprocessing.queues.Queue, tempfile.TemporaryDirectory, threading.Thread + +SAFE_STUB_BASE_CLASSES: Final = frozenset( + [ + "abc.ABC", + "argparse.Action", + "argparse.ArgumentDefaultsHelpFormatter", + "argparse.ArgumentParser", + "argparse.ArgumentTypeError", + "argparse.FileType", + "argparse.HelpFormatter", + "argparse.MetavarTypeHelpFormatter", + "argparse.Namespace", + "argparse.RawDescriptionHelpFormatter", + "argparse.RawTextHelpFormatter", + "ast.NodeTransformer", + "ast.NodeVisitor", + "asyncio.events.AbstractEventLoop", + "asyncio.exceptions.CancelledError", + "asyncio.locks.Event", + "asyncio.protocols.BaseProtocol", + "asyncio.protocols.BufferedProtocol", + "asyncio.protocols.DatagramProtocol", + "asyncio.protocols.Protocol", + "asyncio.queues.Queue", + "asyncio.unix_events._UnixDefaultEventLoopPolicy", + "asyncio.unix_events._UnixSelectorEventLoop", + "asyncio.windows_events.WindowsProactorEventLoopPolicy", + "asyncio.windows_events._WindowsProactorEventLoopPolicy", + "asyncio.windows_events._WindowsSelectorEventLoop", + "cmd.Cmd", + "code.InteractiveConsole", + "code.InteractiveInterpreter", + "codecs.Codec", + "codecs.IncrementalDecoder", + "codecs.IncrementalEncoder", + "codecs.StreamReader", + "codecs.StreamWriter", + "_collections_abc.Buffer", + "collections.ChainMap", + "collections.Counter", + "collections.UserDict", + "collections.UserList", + "collections.UserString", + "concurrent.futures._base.CancelledError", + "concurrent.futures._base.Executor", + "concurrent.futures._base.Future", + "concurrent.futures.process.ProcessPoolExecutor", + "concurrent.futures.thread.ThreadPoolExecutor", + "configparser.ConfigParser", + "configparser.RawConfigParser", + "contextlib.AbstractAsyncContextManager", + "contextlib.AbstractContextManager", + "contextlib.ContextDecorator", + "contextlib.suppress", + "csv.Dialect", + "csv.excel", + "datetime.tzinfo", + "doctest.DocTestFinder", + "doctest.DocTestParser", + "doctest.DocTestRunner", + "doctest.OutputChecker", + "email.message.EmailMessage", + "email.message.Message", + "email.policy.EmailPolicy", + "functools.cached_property", + "gettext.GNUTranslations", + "gettext.NullTranslations", + "html.parser.HTMLParser", + "http.client.HTTPConnection", + "http.client.HTTPMessage", + "http.client.HTTPSConnection", + "http.client.IncompleteRead", + "http.cookiejar.CookieJar", + "http.server.BaseHTTPRequestHandler", + "http.server.HTTPServer", + "http.server.SimpleHTTPRequestHandler", + "http.server.ThreadingHTTPServer", + "_frozen_importlib_external.SourceFileLoader", + "importlib._abc.Loader", + "importlib.abc.MetaPathFinder", + "importlib.abc.SourceLoader", + "importlib.metadata.Distribution", + "ipaddress.IPv4Network", + "ipaddress.IPv6Network", + "json.decoder.JSONDecodeError", + "json.decoder.JSONDecoder", + "json.encoder.JSONEncoder", + "logging.Filter", + "logging.Formatter", + "logging.Logger", + "logging.LoggerAdapter", + "multiprocessing.managers.BaseManager", + "multiprocessing.queues.SimpleQueue", + "numbers.Integral", + "numbers.Real", + "optparse.IndentedHelpFormatter", + "optparse.Option", + "optparse.OptionParser", + "optparse.Values", + "os.PathLike", + "pdb.Pdb", + "_pickle.PicklingError", + "pprint.PrettyPrinter", + "queue.Queue", + "reprlib.Repr", + "selectors.BaseSelector", + "socketserver.BaseRequestHandler", + "socketserver.StreamRequestHandler", + "socketserver.TCPServer", + "socketserver.ThreadingMixIn", + "socketserver.ThreadingTCPServer", + "socketserver.ThreadingUnixStreamServer", + "socketserver.UnixStreamServer", + "sqlite3.OperationalError", + "ssl.SSLObject", + "string.Formatter", + "string.Template", + "subprocess.CalledProcessError", + "subprocess.CompletedProcess", + "subprocess.Popen", + "textwrap.TextWrapper", + "threading.Barrier", + "threading.Event", + "typing.AbstractSet", + "typing.AsyncContextManager", + "typing.AsyncGenerator", + "typing.AsyncIterable", + "typing.AsyncIterator", + "typing.Awaitable", + "typing.BinaryIO", + "typing.Collection", + "typing.Container", + "typing.ContextManager", + "typing.Coroutine", + "typing.Generator", + "typing.Hashable", + "typing.IO", + "typing.Iterable", + "typing.Iterator", + "typing.Mapping", + "typing.MutableMapping", + "typing.MutableSequence", + "typing.MutableSet", + "typing.Sequence", + "typing.Sized", + "typing.SupportsInt", + "typing.TextIO", + "typing_extensions.Buffer", + "unittest.async_case.IsolatedAsyncioTestCase", + "unittest.case.SkipTest", + "unittest.case.TestCase", + "unittest.loader.TestLoader", + "unittest.main.TestProgram", + "unittest.result.TestResult", + "unittest.runner.TextTestResult", + "unittest.runner.TextTestRunner", + "unittest.suite.TestSuite", + "urllib.request.BaseHandler", + "urllib.request.HTTPDigestAuthHandler", + "urllib.request.HTTPHandler", + "urllib.request.HTTPRedirectHandler", + "urllib.request.HTTPSHandler", + "urllib.request.OpenerDirector", + "urllib.request.Request", + "venv.EnvBuilder", + "warnings.catch_warnings", + "webbrowser.MacOSXOSAScript", + "wsgiref.simple_server.WSGIRequestHandler", + "wsgiref.simple_server.WSGIServer", + "xml.sax.handler.ContentHandler", + "xml.sax.saxutils.XMLFilterBase", + "xml.sax.saxutils.XMLGenerator", + "xmlrpc.client.Transport", + "zipfile.ZipFile", + "zipimport.zipimporter", + ] +) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 0941e122c604..6c1d288d84f6 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -2148,6 +2148,70 @@ class AnyBase(Any): ... class Indirect(AnyBase): ... IndirectAlias = Indirect +[case testInheritFromDefaultDictOrOrderedDictError] +from collections import defaultdict, OrderedDict + +from mypy_extensions import mypyc_attr + +class D(defaultdict[str, int]): # E: Native classes cannot inherit from "collections.defaultdict" (defined in a stub, so its instance layout is unknown) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +class O(OrderedDict[str, int]): # E: Native classes cannot inherit from "collections.OrderedDict" (defined in a stub, so its instance layout is unknown) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +class Ok(dict[str, int]): + pass + +@mypyc_attr(native_class=False) +class NonNative(defaultdict[str, int]): + pass + +[case testInheritFromClassInCExtensionError] +from compiled import Base +from plain import Indirect, Plain + +from mypy_extensions import mypyc_attr + +class C(Base): # E: Native classes cannot inherit from "compiled.Base" (its instance layout is unknown, since it's imported from a C extension) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +class D(Indirect): # E: Native classes cannot inherit from "compiled.Base" (its instance layout is unknown, since it's imported from a C extension) \ + # N: Potential workaround: @mypy_extensions.mypyc_attr(native_class=False) \ + # N: https://mypyc.readthedocs.io/en/stable/native_classes.html#defining-non-native-classes + pass + +class Ok(Plain): + pass + +@mypyc_attr(native_class=False) +class NonNative(Base): + pass + +[file compiled.py] +class Base: + pass + +-- An extension module next to compiled.py is imported instead of it at runtime. +-- The check only looks for the file, so the contents don't matter. Create both +-- to cover the extension suffixes on all platforms. +[file compiled.so] +[file compiled.pyd] + +[file plain.py] +from compiled import Base + +class Indirect(Base): + pass + +class Plain: + pass + [case testMypycAttrNativeClassMetaError] from mypy_extensions import mypyc_attr diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index edcd8a4488ae..ee562eeff3d9 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -1546,7 +1546,9 @@ def welp() -> int: from native import welp assert welp() == 35 -[case testSubclassUnsupportedException] +[case testNonNativeSubclassOfUnsupportedBase] +from collections import defaultdict, OrderedDict + from mypy_extensions import mypyc_attr @mypyc_attr(native_class=False) @@ -1593,6 +1595,32 @@ def test_raise_non_native_exception_subclass_2() -> None: assert False assert x +@mypyc_attr(native_class=False) +class DD(defaultdict[str, int]): + def __init__(self) -> None: + super().__init__(int) + self.x = 5 + +@mypyc_attr(native_class=False) +class OD(OrderedDict[str, int]): + def __init__(self) -> None: + super().__init__() + self.x = 6 + +def test_non_native_default_dict_subclass() -> None: + d = DD() + assert d["a"] == 0 + d["b"] += 2 + assert d == {"a": 0, "b": 2} + assert d.x == 5 + +def test_non_native_ordered_dict_subclass() -> None: + d = OD() + d["b"] = 1 + d["a"] = 2 + assert list(d) == ["b", "a"] + assert d.x == 6 + [case testSubclassExternalPythonBase] # Subclass Python classes defined outside the compiled module (some are unknown during # compilation and have type Any) @@ -2346,25 +2374,24 @@ native.test() ok [case testNativeClassHiddenMetaclass] -# The stub hides the base classes' metaclass and base resolution behavior. +# The driver replaces the base classes at runtime with ones that the +# compile-time checks would reject, so that the runtime checks are exercised. from hidden import Prefix, Base, Second class C(Prefix, Base, Second): pass -[file hidden.pyi] -class Prefix: ... -class Base: ... -class Second: ... - [file hidden.py] class Prefix: pass +class Base: + pass + class Meta(type): pass -class Base(metaclass=Meta): +class MetaBase(metaclass=Meta): pass class Second: @@ -2399,6 +2426,7 @@ import sys import hidden from testutil import assertRaises +hidden.Base = hidden.MetaBase for _ in range(3): with assertRaises(TypeError, "mypyc classes can't have a metaclass"): import native diff --git a/mypyc/test/test_safe_bases.py b/mypyc/test/test_safe_bases.py new file mode 100644 index 000000000000..02cf838d9ae0 --- /dev/null +++ b/mypyc/test/test_safe_bases.py @@ -0,0 +1,66 @@ +"""Test that SAFE_STUB_BASE_CLASSES matches how mypy resolves the classes in typeshed. + +The allowlist contains the full names mypy uses for base classes, which are those of the +modules where typeshed defines the classes (e.g. asyncio.protocols.Protocol). If typeshed +moves a class, its full name changes and native subclasses would be rejected, so check +the names after semantic analysis and type checking. + +This only checks the current Python version and platform, since CI runs the tests on all +supported versions and the main platforms. +""" + +from __future__ import annotations + +import sys +import unittest + +from mypy import build +from mypy.modulefinder import BuildSource +from mypy.nodes import TypeInfo +from mypy.options import Options +from mypyc.irbuild.safe_bases import SAFE_STUB_BASE_CLASSES + + +def resolve_bases(names: list[str]) -> dict[str, str]: + """Type check a subclass of each class and return {allowlist name: base full name}. + + This targets the current Python version and platform. Classes that don't exist on + the target (e.g. added in a later Python version) are left out. + """ + lines = [] + for i, name in enumerate(names): + module, short = name.rsplit(".", 1) + # Import the class by name, since a module attribute may shadow a submodule + # (unittest.main is a function as well as a module) + lines.append(f"from {module} import {short} as _B{i}") + for i in range(len(names)): + lines.append(f"class C{i}(_B{i}): pass") + options = Options() + options.incremental = False + result = build.build([BuildSource("main.py", "main", "\n".join(lines) + "\n")], options) + tree = result.files["main"] + resolved = {} + for i, name in enumerate(names): + info = tree.names[f"C{i}"].node + assert isinstance(info, TypeInfo) + if info.fallback_to_any or not info.bases: + continue + base = info.bases[0].type.fullname + if base != "builtins.object": + resolved[name] = base + return resolved + + +class TestSafeBases(unittest.TestCase): + def test_allowlist_matches_typeshed(self) -> None: + target = f"Python {sys.version_info[0]}.{sys.version_info[1]} on {sys.platform}" + resolved = resolve_bases(sorted(SAFE_STUB_BASE_CLASSES)) + assert resolved, "No allowlisted classes were found" + for name, fullname in resolved.items(): + # A name may resolve to another entry on some Python versions (e.g. + # typing.ContextManager is contextlib.AbstractContextManager on 3.13+), but + # the result must always be in the list. + assert fullname in SAFE_STUB_BASE_CLASSES, ( + f"{name} resolves to {fullname} for {target}, which isn't in " + "SAFE_STUB_BASE_CLASSES (did typeshed move the class?)" + ) diff --git a/runtests.py b/runtests.py index f79e7efa0ccb..0a56114c2045 100755 --- a/runtests.py +++ b/runtests.py @@ -19,6 +19,7 @@ MYPYC_COMMAND_LINE = "TestCommandLine" MYPYC_SEPARATE = "TestRunSeparate" MYPYC_MULTIMODULE = "multimodule" # Subset of mypyc run tests that are slow +MYPYC_SAFE_BASES = "TestSafeBases" # Type checks many stdlib modules ERROR_STREAM = "ErrorStreamSuite" DIFF_CACHE = "DiffCacheIntegrationTests" @@ -45,10 +46,16 @@ # These must be enabled by explicitly including 'mypyc-extra' on the command line. -MYPYC_OPT_IN = [MYPYC_RUN, MYPYC_RUN_MULTI, MYPYC_SEPARATE] +MYPYC_OPT_IN = [MYPYC_RUN, MYPYC_RUN_MULTI, MYPYC_SEPARATE, MYPYC_SAFE_BASES] # These mypyc test filters cover most slow test cases -MYPYC_SLOW = [MYPYC_RUN_MULTI, MYPYC_COMMAND_LINE, MYPYC_SEPARATE, MYPYC_MULTIMODULE] +MYPYC_SLOW = [ + MYPYC_RUN_MULTI, + MYPYC_COMMAND_LINE, + MYPYC_SEPARATE, + MYPYC_MULTIMODULE, + MYPYC_SAFE_BASES, +] # We split the pytest run into three parts to improve test