Bulk-scan single-line string bodies

CHANGELOG.md

@@ -5,6 +5,9 @@
 ### Changed
 
 - Speed up membership tests (`key in ...`) on `Container`, `Table` and `InlineTable` with native `__contains__` implementations, avoiding the inherited `MutableMapping` round-trip through `__getitem__` (which resolves the value and builds an exception on every absent key). ([#483](https://github.com/python-poetry/tomlkit/issues/483))
+- Speed up parsing by making `Source` index-based: it now tracks an integer position over the input string instead of materializing a list of `(index, char)` tuples up front, so construction is O(1) and state save/restore no longer copies an iterator. ([#489](https://github.com/python-poetry/tomlkit/pull/489))
+- Speed up parsing by scanning character runs in bulk: `Source.advance_while`/`advance_until` consume a whole run of whitespace, bare-key or number characters in a single pass over the input string instead of one `inc()` call per character. ([#490](https://github.com/python-poetry/tomlkit/pull/490))
+- Speed up parsing of single-line strings by bulk-appending the run of ordinary characters up to the next delimiter, backslash or control character in one pass, instead of one character at a time. ([#491](https://github.com/python-poetry/tomlkit/pull/491))
 
 ### Fixed
 

tomlkit/parser.py

@@ -57,6 +57,22 @@
 CTRL_CHAR_LIMIT = 0x1F
 CHR_DEL = 0x7F
 
+# Character sets for Source.advance_while / advance_until bulk run scans
+# (replace per-character `while self._current.is_*() and self.inc()` loops with
+# a single underlying-string scan).
+_SPACES_SET = frozenset(TOMLChar.SPACES)
+_BARE_KEY_OR_SPACE = frozenset(TOMLChar.BARE + TOMLChar.SPACES)
+_NUM_STOP = frozenset(" \t\n\r#,]}")
+_DATE_TAIL_STOP = frozenset("\t\n\r#,]}")
+# Control chars invalid inside a single-line string (DEL + everything <= 0x1F
+# except tab) — exactly the set that raises InvalidControlChar in the per-char
+# string loop. The single-line string-body fast-path stops its bulk scan at the
+# first delimiter / backslash / control char, then the main loop handles that
+# char with its existing branch (raising InvalidControlChar where needed).
+_CTRL_SINGLE = frozenset(chr(c) for c in range(0x20) if c != CTRL_I) | {chr(CHR_DEL)}
+_SINGLE_LITERAL_STOP = _CTRL_SINGLE | {"'"}  # literal: only the closing quote
+_SINGLE_BASIC_STOP = _CTRL_SINGLE | {'"', "\\"}  # basic: quote or escape
+
 
 class Parser:
     """
@@ -304,8 +320,7 @@ def _parse_comment_trail(self, parse_trail: bool = True) -> tuple[str, str, str]
 
         trail = ""
         if parse_trail:
-            while self._current.is_spaces() and self.inc():
-                pass
+            self._src.advance_while(_SPACES_SET)
 
             if self._current == "\r":
                 with self._state(restore=True):
@@ -325,8 +340,7 @@ def _parse_key_value(self, parse_comment: bool = False) -> tuple[Key, Item]:
         # Leading indent
         self.mark()
 
-        while self._current.is_spaces() and self.inc():
-            pass
+        self._src.advance_while(_SPACES_SET)
 
         indent = self.extract()
 
@@ -374,9 +388,8 @@ def _parse_key(self) -> Key:
         WS before the key must be exhausted first at the callsite.
         """
         self.mark()
-        while self._current.is_spaces() and self.inc():
-            # Skip any leading whitespace
-            pass
+        # Skip any leading whitespace (bulk scan)
+        self._src.advance_while(_SPACES_SET)
         if self._current in "\"'":
             return self._parse_quoted_key()
         else:
@@ -401,8 +414,7 @@ def _parse_quoted_key(self) -> Key:
             raise self.parse_error(UnexpectedCharError, key_str._t.value)
         original += key_str.as_string()
         self.mark()
-        while self._current.is_spaces() and self.inc():
-            pass
+        self._src.advance_while(_SPACES_SET)
         original += self.extract()
         result: Key = SingleKey(str(key_str), t=key_type, sep="", original=original)
         if self._current == ".":
@@ -415,10 +427,7 @@ def _parse_bare_key(self) -> Key:
         """
         Parses a bare key.
         """
-        while (
-            self._current.is_bare_key_char() or self._current.is_spaces()
-        ) and self.inc():
-            pass
+        self._src.advance_while(_BARE_KEY_OR_SPACE)
 
         original = self.extract()
         key_s = original.strip()
@@ -467,8 +476,7 @@ def _parse_value(self) -> Item:
             "nan",
         }:
             # Number
-            while self._current not in " \t\n\r#,]}" and self.inc():
-                pass
+            self._src.advance_until(_NUM_STOP)
 
             raw = self.extract()
 
@@ -479,8 +487,7 @@ def _parse_value(self) -> Item:
             raise self.parse_error(InvalidNumberError)
         elif c in string.digits:
             # Integer, Float, Date, Time or DateTime
-            while self._current not in " \t\n\r#,]}" and self.inc():
-                pass
+            self._src.advance_until(_NUM_STOP)
 
             raw = self.extract()
 
@@ -512,8 +519,7 @@ def _parse_value(self) -> Item:
                         assert isinstance(dt, datetime.date)
                         date = Date(dt.year, dt.month, dt.day, trivia, raw)
                         self.mark()
-                        while self._current not in "\t\n\r#,]}" and self.inc():
-                            pass
+                        self._src.advance_until(_DATE_TAIL_STOP)
 
                         time_raw = self.extract()
                         time_part = time_raw.rstrip()
@@ -838,6 +844,16 @@ def _parse_string(self, delim: StringType) -> String:
                 if cur == "\r\n":
                     self.inc_n(2, exception=UnexpectedEofError)
 
+        # PERF: stop-set for the single-line string-body bulk fast-path (None for
+        # multiline, which keeps the per-char loop because of \r\n handling).
+        src = self._src
+        EOF = src.EOF
+        single_stop = None
+        if delim.is_singleline():
+            single_stop = (
+                _SINGLE_BASIC_STOP if delim.is_basic() else _SINGLE_LITERAL_STOP
+            )
+
         escaped = False  # whether the previous key was ESCAPE
         while True:
             code = ord(self._current)
@@ -913,10 +929,24 @@ def _parse_string(self, delim: StringType) -> String:
             else:
                 # this is either a literal string where we keep everything as is,
                 # or this is not a special escaped char in a basic string
-                value += self._current
+                if single_stop is not None:
+                    # PERF fast-path: bulk-append the run of ordinary characters
+                    # up to the next delimiter / backslash / control char, instead
+                    # of one `value += cur; inc()` iteration per character. The
+                    # stop char is then handled by the branches above on the next
+                    # iteration (single-line only; multiline keeps the per-char
+                    # loop for CRLF handling).
+                    run_start = src._idx
+                    src.advance_until(single_stop)
+                    if src._current is EOF:
+                        # mid-string EOF — same error as the per-char inc()
+                        raise self.parse_error(UnexpectedEofError)
+                    value += src[run_start : src._idx]
+                else:
+                    value += self._current
 
-                # consume this char, EOF here is an issue (middle of string)
-                self.inc(exception=UnexpectedEofError)
+                    # consume this char, EOF here is an issue (middle of string)
+                    self.inc(exception=UnexpectedEofError)
 
     def _parse_table(
         self, parent_name: Key | None = None, parent: Table | None = None

tomlkit/source.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-from copy import copy
 from typing import Any
 
 from tomlkit.exceptions import ParseError
@@ -21,7 +20,9 @@ def __init__(
 
     def __enter__(self) -> _State:
         # Entering this context manager - save the state
-        self._chars = copy(self._source._chars)
+        # PERF: snapshot only the integer index + current char + marker.
+        # We no longer carry an iterator (`_chars`) so there's no `copy(...)`
+        # to do here — saving 3 attribute reads vs the original iter copy.
         self._idx = self._source._idx
         self._current = self._source._current
         self._marker = self._source._marker
@@ -36,7 +37,6 @@ def __exit__(
     ) -> None:
         # Exiting this context manager - restore the prior state
         if self.restore or exception_type:
-            self._source._chars = self._chars
             self._source._idx = self._idx
             self._source._current = self._current
             if self._save_marker:
@@ -80,12 +80,14 @@ class Source(str):
     def __init__(self, _: str) -> None:
         super().__init__()
 
-        # Collection of TOMLChars
-        self._chars = iter([(i, TOMLChar(c)) for i, c in enumerate(self)])
-
-        self._idx = 0
+        # PERF: previously built `iter([(i, TOMLChar(c)) for i, c in enumerate(self)])`
+        # which materialized N tuples + N TOMLChars at init time (~584 k allocations
+        # per 150-parse benchmark). Switching to an integer index over the underlying
+        # str makes init O(1) and lets `inc()` just bump the index and slice the str.
+        # The TOMLChar cache (toml_char.py) absorbs the per-character cost.
+        self._idx = -1  # pre-start sentinel; first inc() will land on 0
         self._marker = 0
-        self._current = TOMLChar("")
+        self._current: TOMLChar = TOMLChar("")
 
         self._state = _StateHandler(self)
 
@@ -125,17 +127,63 @@ def inc(self, exception: type[ParseError] | None = None) -> bool:
         Increments the parser if the end of the input has not been reached.
         Returns whether or not it was able to advance.
         """
-        try:
-            self._idx, self._current = next(self._chars)
+        # PERF: integer increment + cached TOMLChar lookup, no iterator/next()/
+        # StopIteration triage. After the first char of each kind has been seen,
+        # `TOMLChar(self[i])` is a dict.get cache hit.
+        next_idx = self._idx + 1
+        if next_idx < len(self):
+            self._idx = next_idx
+            self._current = TOMLChar(self[next_idx])
+            return True
 
+        # Past end : pin to len, switch current to EOF, raise if asked.
+        self._idx = len(self)
+        self._current = self.EOF
+        if exception:
+            raise self.parse_error(exception) from None
+        return False
+
+    def advance_while(self, charset: frozenset) -> bool:
+        """Advance while the current character is in ``charset``.
+
+        Equivalent to ``while self.current in charset and self.inc(): pass`` but
+        it scans the underlying string in a single pass and updates the index
+        and current character only once, instead of paying a per-character
+        ``inc()`` call. On return ``current`` is the first character NOT in
+        ``charset`` (or EOF). Returns ``True`` if it stopped on a real
+        character, ``False`` at EOF — the same value contract as the loop.
+        """
+        i = self._idx
+        n = len(self)
+        while i < n and self[i] in charset:
+            i += 1
+        if i < n:
+            self._idx = i
+            self._current = TOMLChar(self[i])
             return True
-        except StopIteration:
-            self._idx = len(self)
-            self._current = self.EOF
-            if exception:
-                raise self.parse_error(exception) from None
+        self._idx = n
+        self._current = self.EOF
+        return False
 
-            return False
+    def advance_until(self, stopset: frozenset) -> bool:
+        """Advance while the current character is NOT in ``stopset``.
+
+        The mirror of :meth:`advance_while`: equivalent to
+        ``while self.current not in stopset and self.inc(): pass`` in a single
+        scan. On return ``current`` is the first character IN ``stopset`` (or
+        EOF), with the same return-value contract.
+        """
+        i = self._idx
+        n = len(self)
+        while i < n and self[i] not in stopset:
+            i += 1
+        if i < n:
+            self._idx = i
+            self._current = TOMLChar(self[i])
+            return True
+        self._idx = n
+        self._current = self.EOF
+        return False
 
     def inc_n(self, n: int, exception: type[ParseError] | None = None) -> bool:
         """