Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions Lib/_pyio.py
Original file line number Diff line number Diff line change
Expand Up @@ -2458,9 +2458,11 @@ def tell(self):
# be O(1) in most situations (common decoders, sensible input).
# Actually, it will be exactly 1 for fixed-size codecs (all
# 8-bit codecs, also UTF-16 and UTF-32).
skip_bytes = int(self._b2cratio * chars_to_skip)
# _b2cratio is only a hint and may not have been computed from
# this snapshot, so never guess past the end of next_input.
skip_bytes = min(int(self._b2cratio * chars_to_skip),
len(next_input))
skip_back = 1
assert skip_bytes <= len(next_input)
while skip_bytes > 0:
decoder.setstate((b'', dec_flags))
# Decode up to temptative start point
Expand Down
26 changes: 26 additions & 0 deletions Lib/test/test_io/test_textio.py
Original file line number Diff line number Diff line change
Expand Up @@ -705,6 +705,32 @@ def test_tell_after_readline_with_cr(self):
remaining = f.read()
self.assertEqual(remaining, "")

def test_tell_after_seek_with_chars_to_skip(self):
# tell() after seek() to a position inside the output of one decoder
# step, when the chunk read before had many bytes per character.
for encoding, errors, data, decoded in [
("big5hkscs", "ignore", b"\x88\x62" + b"\xff" * 8190,
"\xca\u0304"),
("utf-7", "strict", b"+AGEAYg-+" + b"A" * 8183, "ab"),
]:
with self.subTest(encoding=encoding):
f = self.TextIOWrapper(self.BytesIO(data),
encoding=encoding, errors=errors)
self.assertEqual(f.read(1), decoded[0])
pos = f.tell()
f.seek(pos)
self.assertEqual(f.tell(), pos)
self.assertEqual(f.read(1), decoded[1])

def test_tell_after_seek_after_lone_cr(self):
data = b"\xff" * 8000 + "a\r\u3042\n".encode()
f = self.TextIOWrapper(self.BytesIO(data),
encoding="utf-8", errors="ignore")
self.assertEqual(f.readline(), "a\n")
pos = f.tell()
f.seek(pos)
self.assertEqual(f.tell(), pos)
self.assertEqual(f.read(), "\u3042\n")

def test_seek_with_encoder_state(self):
f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix :meth:`io.TextIOWrapper.tell` reading past the end of an internal buffer
when called after :meth:`~io.TextIOWrapper.seek`, which could make it raise a
spurious :exc:`UnicodeDecodeError`.
7 changes: 4 additions & 3 deletions Modules/_io/textio.c
Original file line number Diff line number Diff line change
Expand Up @@ -2981,10 +2981,11 @@ _io_TextIOWrapper_tell_impl(textio *self)
} while (0)

/* Fast search for an acceptable start point, close to our
current pos */
skip_bytes = (Py_ssize_t) (self->b2cratio * chars_to_skip);
current pos. b2cratio is only a hint and may not have been computed
from this snapshot, so never guess past the end of next_input. */
skip_bytes = (Py_ssize_t) Py_MIN(self->b2cratio * chars_to_skip,
(double) PyBytes_GET_SIZE(next_input));
skip_back = 1;
assert(skip_bytes <= PyBytes_GET_SIZE(next_input));
input = PyBytes_AS_STRING(next_input);
while (skip_bytes > 0) {
/* Decode up to temptative start point */
Expand Down
Loading