Allow nativeparse to parse source code directly

misc/dump-ast.py

@@ -19,7 +19,7 @@ def dump(fname: str, python_version: tuple[int, int], quiet: bool = False) -> No
     options.python_version = python_version
     with open(fname, "rb") as f:
         s = f.read()
-        tree = parse(s, fname, None, errors=Errors(options), options=options, file_exists=True)
+        tree = parse(s, fname, None, errors=Errors(options), options=options)
         if not quiet:
             print(tree)
 

mypy/build.py

@@ -1214,9 +1214,7 @@ def parse_file(
         Raise CompileError if there is a parse error.
         """
         imports_only = False
-        file_exists = self.fscache.exists(path)
-        if self.workers and file_exists:
-            # Currently, we can use the native parser only for actual files.
+        if self.workers:
             imports_only = True
         t0 = time.time()
         parse_errors: list[ParseError] = []
@@ -1225,13 +1223,7 @@ def parse_file(
             tree = load_from_raw(path, id, raw_data, self.errors, options)
         else:
             tree, parse_errors = parse(
-                source,
-                path,
-                id,
-                self.errors,
-                options=options,
-                file_exists=file_exists,
-                imports_only=imports_only,
+                source, path, id, self.errors, options=options, imports_only=imports_only
             )
         tree._fullname = id
         if self.stats_enabled:

mypy/checkstrformat.py

@@ -582,12 +582,7 @@ def apply_field_accessors(
         temp_errors = Errors(self.chk.options)
         dummy = DUMMY_FIELD_NAME + spec.field[len(spec.key) :]
         temp_ast, _ = parse(
-            dummy,
-            fnam="<format>",
-            module=None,
-            options=self.chk.options,
-            errors=temp_errors,
-            file_exists=False,
+            dummy, fnam="<format>", module=None, options=self.chk.options, errors=temp_errors
         )
         if temp_errors.is_errors():
             self.msg.fail(

mypy/nativeparse.py

@@ -190,7 +190,11 @@ def add_error(
 
 
 def native_parse(
-    filename: str, options: Options, skip_function_bodies: bool = False, imports_only: bool = False
+    filename: str,
+    options: Options,
+    source: str | bytes | None = None,
+    skip_function_bodies: bool = False,
+    imports_only: bool = False,
 ) -> tuple[MypyFile, list[ParseError], TypeIgnores]:
     """Parse a Python file using the native Rust-based parser.
 
@@ -208,7 +212,7 @@ def native_parse(
         return node, [], []
 
     b, errors, ignores, import_bytes, is_partial_package, uses_template_strings = (
-        parse_to_binary_ast(filename, options, skip_function_bodies)
+        parse_to_binary_ast(filename, options, source, skip_function_bodies)
     )
     data = ReadBuffer(b)
     n = read_int(data)
@@ -253,7 +257,10 @@ def read_statements(state: State, data: ReadBuffer, n: int) -> list[Statement]:
 
 
 def parse_to_binary_ast(
-    filename: str, options: Options, skip_function_bodies: bool = False
+    filename: str,
+    options: Options,
+    source: str | bytes | None = None,
+    skip_function_bodies: bool = False,
 ) -> tuple[bytes, list[ParseError], TypeIgnores, bytes, bool, bool]:
     # This is a horrible hack to work around a mypyc bug where imported
     # module may be not ready in a thread sometimes.
@@ -264,6 +271,7 @@ def parse_to_binary_ast(
             raise ImportError("Cannot import ast_serialize")
     ast_bytes, errors, ignores, import_bytes, ast_data = ast_serialize.parse(
         filename,
+        source,
         skip_function_bodies=skip_function_bodies,
         python_version=options.python_version,
         platform=options.platform,

mypy/parse.py

@@ -17,7 +17,6 @@ def parse(
     module: str | None,
     errors: Errors,
     options: Options,
-    file_exists: bool,
     imports_only: bool = False,
 ) -> tuple[MypyFile, list[ParseError]]:
     """Parse a source file, without doing any semantic analysis.
@@ -28,27 +27,25 @@ def parse(
     The python_version (major, minor) option determines the Python syntax variant.
     """
     if options.native_parser:
-        # Native parser only works with actual files on disk
-        # Fall back to fastparse for in-memory source or non-existent files
-        if file_exists:
-            import mypy.nativeparse
-
-            ignore_errors = options.ignore_errors or fnam in errors.ignored_files
-            # If errors are ignored, we can drop many function bodies to speed up type checking.
-            strip_function_bodies = ignore_errors and not options.preserve_asts
-            tree, parse_errors, type_ignores = mypy.nativeparse.native_parse(
-                fnam,
-                options,
-                skip_function_bodies=strip_function_bodies,
-                imports_only=imports_only,
-            )
-            # Convert type ignores list to dict
-            tree.ignored_lines = dict(type_ignores)
-            # Set is_stub based on file extension
-            tree.is_stub = fnam.endswith(".pyi")
-            # Note: tree.imports is populated directly by native_parse with deserialized
-            # import metadata, so we don't need to collect imports via AST traversal
-            return tree, parse_errors
+        import mypy.nativeparse
+
+        ignore_errors = options.ignore_errors or fnam in errors.ignored_files
+        # If errors are ignored, we can drop many function bodies to speed up type checking.
+        strip_function_bodies = ignore_errors and not options.preserve_asts
+        tree, parse_errors, type_ignores = mypy.nativeparse.native_parse(
+            fnam,
+            options,
+            source,
+            skip_function_bodies=strip_function_bodies,
+            imports_only=imports_only,
+        )
+        # Convert type ignores list to dict
+        tree.ignored_lines = dict(type_ignores)
+        # Set is_stub based on file extension
+        tree.is_stub = fnam.endswith(".pyi")
+        # Note: tree.imports is populated directly by native_parse with deserialized
+        # import metadata, so we don't need to collect imports via AST traversal
+        return tree, parse_errors
         # Fall through to fastparse for non-existent files
 
     assert not imports_only

mypy/stubgen.py

@@ -1745,12 +1745,7 @@ def parse_source_file(mod: StubSource, mypy_options: MypyOptions) -> None:
     source = mypy.util.decode_python_encoding(data)
     errors = Errors(mypy_options)
     mod.ast, errs = mypy.parse.parse(
-        source,
-        fnam=mod.path,
-        module=mod.module,
-        errors=errors,
-        options=mypy_options,
-        file_exists=True,
+        source, fnam=mod.path, module=mod.module, errors=errors, options=mypy_options
     )
     mod.ast._fullname = mod.module
     for err in errs:

mypy/test/test_nativeparse.py

@@ -212,7 +212,7 @@ def format_reachable_imports(node: MypyFile) -> list[str]:
 
 @unittest.skipUnless(has_nativeparse, "nativeparse not available")
 class TestNativeParserBinaryFormat(unittest.TestCase):
-    def test_trivial_binary_data(self) -> None:
+    def _assert_trivial_binary_data(self, b: bytes) -> None:
         # A quick sanity check to ensure the serialized data looks as expected. Only covers
         # a few AST nodes.
 
@@ -228,9 +228,9 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) ->
                 int_enc(end_column - start_column),
             ]
 
-        with temp_source("print('hello')") as fnam:
-            b, _, _, _, _, _ = parse_to_binary_ast(fnam, Options())
-            assert list(b) == (
+        self.assertEqual(
+            list(b),
+            (
                 [LITERAL_INT, 22, nodes.EXPR_STMT, nodes.CALL_EXPR]
                 + [nodes.NAME_EXPR, LITERAL_STR]
                 + [int_enc(5)]
@@ -247,7 +247,21 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) ->
                 + [LIST_GEN, 22, LITERAL_NONE]
                 + locs(1, 0, 1, 14)
                 + [END_TAG, END_TAG]
-            )
+            ),
+        )
+
+    def test_trivial_binary_data_from_file(self) -> None:
+        with temp_source("print('hello')") as fnam:
+            b, _, _, _, _, _ = parse_to_binary_ast(fnam, Options())
+            self._assert_trivial_binary_data(b)
+
+    def test_trivial_binary_data_from_string_source(self) -> None:
+        b, _, _, _, _, _ = parse_to_binary_ast("", Options(), "print('hello')")
+        self._assert_trivial_binary_data(b)
+
+    def test_trivial_binary_data_from_bytes_source(self) -> None:
+        b, _, _, _, _, _ = parse_to_binary_ast("", Options(), b"print('hello')")
+        self._assert_trivial_binary_data(b)
 
 
 @contextlib.contextmanager

mypy/test/testparse.py

@@ -61,12 +61,7 @@ def test_parser(testcase: DataDrivenTestCase) -> None:
     try:
         errors = Errors(options)
         n, _ = parse(
-            bytes(source, "ascii"),
-            fnam="main",
-            module="__main__",
-            errors=errors,
-            options=options,
-            file_exists=False,
+            bytes(source, "ascii"), fnam="main", module="__main__", errors=errors, options=options
         )
         if errors.is_errors():
             errors.raise_error()
@@ -107,7 +102,6 @@ def test_parse_error(testcase: DataDrivenTestCase) -> None:
             "__main__",
             errors=errors,
             options=options,
-            file_exists=False,
         )
         if errors.is_errors():
             errors.raise_error()