gh-161199: Speed up finding line endings in TextIOWrapper

With newline='', the mode the csv documentation recommends, readline()
and iteration searched for '\r' or '\n' one character at a time.  So did
the newline decoder, both when updating the newlines attribute with
newline='' and when translating '\r\n' and '\r' to '\n' with the default
newline=None, which it does for every chunk that contains '\r'.

For 1-byte strings, these loops now first skip ahead one size_t word at
a time.  A cheap test skips words with no byte below 0x0E, and an exact
test finds '\r' and '\n' in the rest, so tabs do not stop the search.
On little-endian machines, ctz() gives the position, as in
find_first_nonascii() in unicodeobject.c.  The translation copies whole
words as it goes.  Each 1-byte loop is the old loop with the skip in
front of it, so line endings are handled as before.  If either of the
first two characters is '\r' or below, the old loop runs directly,
since it is fastest for short lines.

2- and 4-byte strings run the same code as before, as does
_PyIO_find_line_ending(), which StringIO also uses.

Author: ptweezy

Lib/test/test_io/test_textio.py

@@ -459,6 +459,28 @@ def test_newlines_input(self):
             txt.seek(0)
             self.assertEqual(txt.read(), "".join(expected))
 
+    def test_newlines_input_long_lines(self):
+        # Lines from a few characters to many machine words long, with and
+        # without control or non-ASCII characters in them, read with and
+        # without newline translation.
+        lengths = [*range(20), *range(248, 272), 600]
+        for end in ("\n", "\r", "\r\n"):
+            for ctrl in ("", "\x00", "\t", "\x0c", "\x0e", "\xe9"):
+                lines = ["a" * n + ctrl + "b" * (n % 9) + end
+                         for n in lengths]
+                data = "".join(lines).encode("latin-1")
+                for newline in ("", None):
+                    if newline is None:
+                        expected = [line[:-len(end)] + "\n" for line in lines]
+                    else:
+                        expected = lines
+                    with self.subTest(end=end, ctrl=ctrl, newline=newline):
+                        txt = self.TextIOWrapper(self.BytesIO(data),
+                                                 encoding="latin-1",
+                                                 newline=newline)
+                        self.assertEqual(txt.readlines(), expected)
+                        self.assertEqual(txt.newlines, end)
+
     def test_newlines_output(self):
         testdict = {
             "": b"AAA\nBBB\nCCC\nX\rY\r\nZ",

Misc/NEWS.d/next/Library/2026-11-02-12-43-50.gh-issue-161199.0eS0KY.rst

@@ -0,0 +1,4 @@
+Speed up :class:`io.TextIOWrapper` for text with no characters above U+00FF
+when reading lines with ``newline=''`` (the mode recommended for :mod:`csv`)
+and when translating ``\r\n`` and ``\r`` line endings with the default
+``newline=None``. Patch by Parker Loflin.

Modules/_io/textio.c

@@ -324,6 +324,194 @@ check_decoded(PyObject *decoded)
 #define SEEN_CRLF 4
 #define SEEN_ALL (SEEN_CR | SEEN_LF | SEEN_CRLF)
 
+/* Count trailing zero bits; v must not be 0.  As in unicodeobject.c. */
+#if defined(__clang__) || defined(__GNUC__)
+#  define HAVE_CTZ 1
+static inline unsigned int
+ctz(size_t v)
+{
+    return __builtin_ctzll((unsigned long long)v);
+}
+#elif defined(_MSC_VER)
+#  define HAVE_CTZ 1
+static inline unsigned int
+ctz(size_t v)
+{
+    unsigned long pos;
+#  if SIZEOF_SIZE_T == 4
+    _BitScanForward(&pos, v);
+#  else
+    _BitScanForward64(&pos, v);
+#  endif
+    return pos;
+}
+#else
+#  define HAVE_CTZ 0
+#endif
+
+/* Skip a size_t word at a time towards the first '\r' or '\n' in the
+   NUL-terminated UCS1 buffer [s, end).  Return a pointer to it, or to an
+   earlier character from which the caller's own loop finds it.
+
+   If out is not NULL, also copy the skipped characters to out.  out must
+   be no further into its buffer than s is into the input, and its buffer
+   must have room for the whole input.  Whole words are copied, so up to
+   sizeof(size_t) bytes past the skipped characters may also be written;
+   the caller overwrites them or leaves them past the end of its result.
+
+   Lines are often short, and the caller's loop is fastest for them, so if
+   either of the first two characters is '\r' or below, return a pointer
+   to it at once.  Most words of text have no byte below 0x0E and cost one
+   cheap test; the rest are checked for '\r' and '\n' exactly, so tabs and
+   other control characters do not stop the search.
+
+   (x - ones * n) & ~x & high flags the bytes of x below n.  The cheap test
+   uses n = 0x0E; the exact test uses n = 1 on x = w ^ (ones * '\n') and
+   x = w ^ (ones * '\r'), whose zero bytes are the matches.  Borrows can
+   flag extra bytes, but only above a byte below n, so the lowest flagged
+   byte is always correct: on little-endian machines, ctz() gives the
+   first match; elsewhere, the caller finds it in the word. */
+static inline const char *
+skip_to_newline_ucs1(const char *s, const char *end, char *out)
+{
+    const size_t ones = (size_t)-1 / 0xFF;  /* 0x01 in every byte */
+    const size_t high = ones * 0x80;
+
+    /* s[1] is only read when s[0] is not the terminating NUL. */
+    if ((unsigned char)s[0] <= '\r') {
+        return s;
+    }
+    if ((unsigned char)s[1] <= '\r') {
+        if (out != NULL) {
+            out[0] = s[0];
+        }
+        return s + 1;
+    }
+    while (end - s >= (Py_ssize_t)sizeof(size_t)) {
+        size_t w;
+        memcpy(&w, s, sizeof(w));
+        if (out != NULL) {
+            memcpy(out, &w, sizeof(w));
+            out += sizeof(w);
+        }
+        if ((w - ones * 0x0E) & ~w & high) {
+            size_t lf = w ^ (ones * '\n');
+            size_t cr = w ^ (ones * '\r');
+            size_t m = (((lf - ones) & ~lf) | ((cr - ones) & ~cr)) & high;
+            if (m != 0) {
+#if PY_LITTLE_ENDIAN && HAVE_CTZ
+                return s + ctz(m) / 8;
+#else
+                return s;
+#endif
+            }
+        }
+        s += sizeof(w);
+    }
+    return s;
+}
+
+/* The universal newlines search of _PyIO_find_line_ending(), for the
+   NUL-terminated 1-byte string [start, end). */
+static Py_ssize_t
+find_line_ending_ucs1(const char *start, const char *end,
+                      Py_ssize_t *consumed)
+{
+    const char *s = start;
+    for (;;) {
+        unsigned char ch;
+        s = skip_to_newline_ucs1(s, end, NULL);
+        while ((unsigned char)*s > '\r') {
+            s++;
+        }
+        if (s >= end) {
+            *consumed = end - start;
+            return -1;
+        }
+        ch = (unsigned char)*s++;
+        if (ch == '\n') {
+            return s - start;
+        }
+        if (ch == '\r') {
+            return *s == '\n' ? s - start + 1 : s - start;
+        }
+    }
+}
+
+/* The newlines scan of _PyIncrementalNewlineDecoder_decode() below, for
+   the NUL-terminated 1-byte string [s, s + len), len > 0. */
+static Py_NO_INLINE int
+scan_newlines_ucs1(const char *s, Py_ssize_t len, int seennl)
+{
+    const char *end = s + len;
+    for (;;) {
+        unsigned char c;
+        s = skip_to_newline_ucs1(s, end, NULL);
+        while ((unsigned char)*s > '\r') {
+            s++;
+        }
+        c = (unsigned char)*s++;
+        if (c == '\n') {
+            seennl |= SEEN_LF;
+        }
+        else if (c == '\r') {
+            if (*s == '\n') {
+                seennl |= SEEN_CRLF;
+                s++;
+            }
+            else {
+                seennl |= SEEN_CR;
+            }
+        }
+        if (s >= end || seennl == SEEN_ALL) {
+            return seennl;
+        }
+    }
+}
+
+/* The translation loop of _PyIncrementalNewlineDecoder_decode() below,
+   for the NUL-terminated 1-byte string [in, in + len).  Write the result
+   to out, which has room for len characters, and return its length. */
+static Py_NO_INLINE Py_ssize_t
+translate_newlines_ucs1(const char *in, Py_ssize_t len, char *out,
+                        int *seennl)
+{
+    const char *s = in, *end = in + len;
+    char *o = out;
+    int seen = *seennl;
+    for (;;) {
+        unsigned char c;
+        const char *p = skip_to_newline_ucs1(s, end, o);
+        o += p - s;
+        s = p;
+        while ((c = (unsigned char)*s++) > '\r') {
+            *o++ = (char)c;
+        }
+        if (c == '\n') {
+            *o++ = '\n';
+            seen |= SEEN_LF;
+            continue;
+        }
+        if (c == '\r') {
+            if (*s == '\n') {
+                s++;
+                seen |= SEEN_CRLF;
+            }
+            else {
+                seen |= SEEN_CR;
+            }
+            *o++ = '\n';
+            continue;
+        }
+        if (s > end) {
+            break;
+        }
+        *o++ = (char)c;
+    }
+    *seennl = seen;
+    return o - out;
+}
+
 PyObject *
 _PyIncrementalNewlineDecoder_decode(PyObject *myself,
                                     PyObject *input, int final)
@@ -440,6 +628,10 @@ _PyIncrementalNewlineDecoder_decode(PyObject *myself,
             /* We have already seen all newline types, no need to scan again */
             if (seennl == SEEN_ALL)
                 goto endscan;
+            if (kind == PyUnicode_1BYTE_KIND) {
+                seennl = scan_newlines_ucs1(in_str, len, seennl);
+                goto endscan;
+            }
             for (;;) {
                 Py_UCS4 c;
                 /* Fast loop for non-control characters */
@@ -481,6 +673,11 @@ _PyIncrementalNewlineDecoder_decode(PyObject *myself,
                 goto error;
             }
             in = out = 0;
+            if (kind == PyUnicode_1BYTE_KIND) {
+                out = translate_newlines_ucs1(in_str, len, translated,
+                                              &seennl);
+                goto translated_all;
+            }
             for (;;) {
                 Py_UCS4 c;
                 /* Fast loop for non-control characters */
@@ -505,6 +702,7 @@ _PyIncrementalNewlineDecoder_decode(PyObject *myself,
                     break;
                 PyUnicode_WRITE(kind, translated, out++, c);
             }
+        translated_all:
             Py_DECREF(output);
             output = PyUnicode_FromKindAndData(kind, translated, out);
             PyMem_Free(translated);
@@ -2385,12 +2583,21 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit)
         line_len = PyUnicode_GET_LENGTH(line);
         kind = PyUnicode_KIND(line);
 
-        endpos = _PyIO_find_line_ending(
-            self->readtranslate, self->readuniversal, self->readnl,
-            kind,
-            ptr + kind * start,
-            ptr + kind * line_len,
-            &consumed);
+        if (kind == PyUnicode_1BYTE_KIND
+            && self->readuniversal && !self->readtranslate)
+        {
+            /* newline='' */
+            endpos = find_line_ending_ucs1(ptr + start, ptr + line_len,
+                                           &consumed);
+        }
+        else {
+            endpos = _PyIO_find_line_ending(
+                self->readtranslate, self->readuniversal, self->readnl,
+                kind,
+                ptr + kind * start,
+                ptr + kind * line_len,
+                &consumed);
+        }
         if (endpos >= 0) {
             endpos += start;
             if (limit >= 0 && (endpos - start) + chunked >= limit)