diff --git a/misc/dump-ast.py b/misc/dump-ast.py index 68ea8bc0dc61..7fdf905bae0b 100755 --- a/misc/dump-ast.py +++ b/misc/dump-ast.py @@ -19,7 +19,7 @@ def dump(fname: str, python_version: tuple[int, int], quiet: bool = False) -> No options.python_version = python_version with open(fname, "rb") as f: s = f.read() - tree = parse(s, fname, None, errors=Errors(options), options=options, file_exists=True) + tree = parse(s, fname, None, errors=Errors(options), options=options) if not quiet: print(tree) diff --git a/mypy/build.py b/mypy/build.py index ef481ed8f444..c96d16de963c 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -1214,9 +1214,7 @@ def parse_file( Raise CompileError if there is a parse error. """ imports_only = False - file_exists = self.fscache.exists(path) - if self.workers and file_exists: - # Currently, we can use the native parser only for actual files. + if self.workers: imports_only = True t0 = time.time() parse_errors: list[ParseError] = [] @@ -1225,13 +1223,7 @@ def parse_file( tree = load_from_raw(path, id, raw_data, self.errors, options) else: tree, parse_errors = parse( - source, - path, - id, - self.errors, - options=options, - file_exists=file_exists, - imports_only=imports_only, + source, path, id, self.errors, options=options, imports_only=imports_only ) tree._fullname = id if self.stats_enabled: diff --git a/mypy/checkstrformat.py b/mypy/checkstrformat.py index e985aa352abd..7ea67ff534cb 100644 --- a/mypy/checkstrformat.py +++ b/mypy/checkstrformat.py @@ -582,12 +582,7 @@ def apply_field_accessors( temp_errors = Errors(self.chk.options) dummy = DUMMY_FIELD_NAME + spec.field[len(spec.key) :] temp_ast, _ = parse( - dummy, - fnam="", - module=None, - options=self.chk.options, - errors=temp_errors, - file_exists=False, + dummy, fnam="", module=None, options=self.chk.options, errors=temp_errors ) if temp_errors.is_errors(): self.msg.fail( diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index f08268cfe1ca..ccdfc3483162 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -190,7 +190,11 @@ def add_error( def native_parse( - filename: str, options: Options, skip_function_bodies: bool = False, imports_only: bool = False + filename: str, + options: Options, + source: str | bytes | None = None, + skip_function_bodies: bool = False, + imports_only: bool = False, ) -> tuple[MypyFile, list[ParseError], TypeIgnores]: """Parse a Python file using the native Rust-based parser. @@ -208,7 +212,7 @@ def native_parse( return node, [], [] b, errors, ignores, import_bytes, is_partial_package, uses_template_strings = ( - parse_to_binary_ast(filename, options, skip_function_bodies) + parse_to_binary_ast(filename, options, source, skip_function_bodies) ) data = ReadBuffer(b) n = read_int(data) @@ -253,7 +257,10 @@ def read_statements(state: State, data: ReadBuffer, n: int) -> list[Statement]: def parse_to_binary_ast( - filename: str, options: Options, skip_function_bodies: bool = False + filename: str, + options: Options, + source: str | bytes | None = None, + skip_function_bodies: bool = False, ) -> tuple[bytes, list[ParseError], TypeIgnores, bytes, bool, bool]: # This is a horrible hack to work around a mypyc bug where imported # module may be not ready in a thread sometimes. @@ -264,6 +271,7 @@ def parse_to_binary_ast( raise ImportError("Cannot import ast_serialize") ast_bytes, errors, ignores, import_bytes, ast_data = ast_serialize.parse( filename, + source, skip_function_bodies=skip_function_bodies, python_version=options.python_version, platform=options.platform, diff --git a/mypy/parse.py b/mypy/parse.py index bd8e4ad5dcd3..1955c0e921b8 100644 --- a/mypy/parse.py +++ b/mypy/parse.py @@ -17,7 +17,6 @@ def parse( module: str | None, errors: Errors, options: Options, - file_exists: bool, imports_only: bool = False, ) -> tuple[MypyFile, list[ParseError]]: """Parse a source file, without doing any semantic analysis. @@ -28,27 +27,25 @@ def parse( The python_version (major, minor) option determines the Python syntax variant. """ if options.native_parser: - # Native parser only works with actual files on disk - # Fall back to fastparse for in-memory source or non-existent files - if file_exists: - import mypy.nativeparse - - ignore_errors = options.ignore_errors or fnam in errors.ignored_files - # If errors are ignored, we can drop many function bodies to speed up type checking. - strip_function_bodies = ignore_errors and not options.preserve_asts - tree, parse_errors, type_ignores = mypy.nativeparse.native_parse( - fnam, - options, - skip_function_bodies=strip_function_bodies, - imports_only=imports_only, - ) - # Convert type ignores list to dict - tree.ignored_lines = dict(type_ignores) - # Set is_stub based on file extension - tree.is_stub = fnam.endswith(".pyi") - # Note: tree.imports is populated directly by native_parse with deserialized - # import metadata, so we don't need to collect imports via AST traversal - return tree, parse_errors + import mypy.nativeparse + + ignore_errors = options.ignore_errors or fnam in errors.ignored_files + # If errors are ignored, we can drop many function bodies to speed up type checking. + strip_function_bodies = ignore_errors and not options.preserve_asts + tree, parse_errors, type_ignores = mypy.nativeparse.native_parse( + fnam, + options, + source, + skip_function_bodies=strip_function_bodies, + imports_only=imports_only, + ) + # Convert type ignores list to dict + tree.ignored_lines = dict(type_ignores) + # Set is_stub based on file extension + tree.is_stub = fnam.endswith(".pyi") + # Note: tree.imports is populated directly by native_parse with deserialized + # import metadata, so we don't need to collect imports via AST traversal + return tree, parse_errors # Fall through to fastparse for non-existent files assert not imports_only diff --git a/mypy/stubgen.py b/mypy/stubgen.py index 38bd1f228e6e..bbe286e80cd7 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -1745,12 +1745,7 @@ def parse_source_file(mod: StubSource, mypy_options: MypyOptions) -> None: source = mypy.util.decode_python_encoding(data) errors = Errors(mypy_options) mod.ast, errs = mypy.parse.parse( - source, - fnam=mod.path, - module=mod.module, - errors=errors, - options=mypy_options, - file_exists=True, + source, fnam=mod.path, module=mod.module, errors=errors, options=mypy_options ) mod.ast._fullname = mod.module for err in errs: diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 94be60e328b7..abd8f63f0260 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -212,7 +212,7 @@ def format_reachable_imports(node: MypyFile) -> list[str]: @unittest.skipUnless(has_nativeparse, "nativeparse not available") class TestNativeParserBinaryFormat(unittest.TestCase): - def test_trivial_binary_data(self) -> None: + def _assert_trivial_binary_data(self, b: bytes) -> None: # A quick sanity check to ensure the serialized data looks as expected. Only covers # a few AST nodes. @@ -228,9 +228,9 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> int_enc(end_column - start_column), ] - with temp_source("print('hello')") as fnam: - b, _, _, _, _, _ = parse_to_binary_ast(fnam, Options()) - assert list(b) == ( + self.assertEqual( + list(b), + ( [LITERAL_INT, 22, nodes.EXPR_STMT, nodes.CALL_EXPR] + [nodes.NAME_EXPR, LITERAL_STR] + [int_enc(5)] @@ -247,7 +247,21 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> + [LIST_GEN, 22, LITERAL_NONE] + locs(1, 0, 1, 14) + [END_TAG, END_TAG] - ) + ), + ) + + def test_trivial_binary_data_from_file(self) -> None: + with temp_source("print('hello')") as fnam: + b, _, _, _, _, _ = parse_to_binary_ast(fnam, Options()) + self._assert_trivial_binary_data(b) + + def test_trivial_binary_data_from_string_source(self) -> None: + b, _, _, _, _, _ = parse_to_binary_ast("", Options(), "print('hello')") + self._assert_trivial_binary_data(b) + + def test_trivial_binary_data_from_bytes_source(self) -> None: + b, _, _, _, _, _ = parse_to_binary_ast("", Options(), b"print('hello')") + self._assert_trivial_binary_data(b) @contextlib.contextmanager diff --git a/mypy/test/testparse.py b/mypy/test/testparse.py index 09177126426d..22d4b88a523a 100644 --- a/mypy/test/testparse.py +++ b/mypy/test/testparse.py @@ -61,12 +61,7 @@ def test_parser(testcase: DataDrivenTestCase) -> None: try: errors = Errors(options) n, _ = parse( - bytes(source, "ascii"), - fnam="main", - module="__main__", - errors=errors, - options=options, - file_exists=False, + bytes(source, "ascii"), fnam="main", module="__main__", errors=errors, options=options ) if errors.is_errors(): errors.raise_error() @@ -107,7 +102,6 @@ def test_parse_error(testcase: DataDrivenTestCase) -> None: "__main__", errors=errors, options=options, - file_exists=False, ) if errors.is_errors(): errors.raise_error()