Skip to content

Commit fa6f019

Browse files
kidozkidoz
committed
Decode bytes with codec aliases, CPython error handlers, and byte order mark handling and move bytes representation into shared text helpers
1 parent 9256ee0 commit fa6f019

9 files changed

Lines changed: 1142 additions & 58 deletions

File tree

‎src/DotPython.Runtime.Managed/Execution/PythonBuiltinMethods.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,13 @@ internal static class PythonBuiltinMethods
176176
),
177177
};
178178

179+
private static readonly Dictionary<string, PythonProtocolFunctionValue> BytesMethods = new(
180+
StringComparer.Ordinal
181+
)
182+
{
183+
["decode"] = PythonBytesText.DecodeMethod,
184+
};
185+
179186
private static readonly Dictionary<string, PythonProtocolFunctionValue> ListMethods = new(
180187
StringComparer.Ordinal
181188
)
@@ -560,6 +567,7 @@ out PythonProtocolFunctionValue method
560567
var table = target switch
561568
{
562569
PythonTextValue => TextMethods,
570+
PythonByteSequenceValue => BytesMethods,
563571
PythonListValue => ListMethods,
564572
PythonDictionaryValue => DictionaryMethods,
565573
PythonTupleValue => TupleMethods,
@@ -764,6 +772,7 @@ internal static void MergeInto(
764772
internal static bool SupportsMethods(PythonValue target) =>
765773
target
766774
is PythonTextValue
775+
or PythonByteSequenceValue
767776
or PythonListValue
768777
or PythonDictionaryValue
769778
or PythonTupleValue

‎src/DotPython.Runtime.Managed/Execution/PythonBytesDecoding.cs‎

Lines changed: 260 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,260 @@
1+
using System.Text;
2+
using DotPython.Language.Text;
3+
4+
namespace DotPython.Runtime.Managed.Execution;
5+
6+
/// <summary>
7+
/// Decodes the runtime's qualified byte codecs without CLR fallback grouping.
8+
/// Error boundaries and BOM behavior follow CPython 3.14.7 unicodeobject.c.
9+
/// </summary>
10+
internal static class PythonBytesDecoding
11+
{
12+
private const int MaximumTextLength = 10_000_000;
13+
14+
internal static string Decode(
15+
byte[] bytes,
16+
int codePage,
17+
bool detectByteOrderMark,
18+
string errors,
19+
TextSpan span
20+
)
21+
{
22+
UserObjectProtocols.Dispatcher?.CheckIterationWork(span);
23+
return new Decoder(bytes, codePage, detectByteOrderMark, errors, span).Decode();
24+
}
25+
26+
private sealed class Decoder(
27+
byte[] bytes,
28+
int codePage,
29+
bool detectByteOrderMark,
30+
string errors,
31+
TextSpan span
32+
)
33+
{
34+
private readonly StringBuilder _text = new(Math.Min(bytes.Length, MaximumTextLength));
35+
private int _position;
36+
private int _nextWork;
37+
private string _encoding = string.Empty;
38+
private bool _bigEndian;
39+
40+
internal string Decode()
41+
{
42+
_encoding = codePage switch
43+
{
44+
65001 => "utf-8",
45+
20127 => "ascii",
46+
28591 => "iso8859-1",
47+
1200 => detectByteOrderMark && !BitConverter.IsLittleEndian
48+
? "utf-16-be"
49+
: "utf-16-le",
50+
1201 => "utf-16-be",
51+
_ => throw new ArgumentOutOfRangeException(nameof(codePage)),
52+
};
53+
_bigEndian =
54+
codePage == 1201
55+
|| codePage == 1200 && detectByteOrderMark && !BitConverter.IsLittleEndian;
56+
if (codePage == 1200 && detectByteOrderMark && bytes.Length >= 2)
57+
{
58+
if (bytes[0] == 0xff && bytes[1] == 0xfe)
59+
{
60+
_bigEndian = false;
61+
_position = 2;
62+
}
63+
else if (bytes[0] == 0xfe && bytes[1] == 0xff)
64+
{
65+
_bigEndian = true;
66+
_position = 2;
67+
}
68+
_encoding = _bigEndian ? "utf-16-be" : "utf-16-le";
69+
}
70+
71+
while (_position < bytes.Length)
72+
{
73+
CheckProgress();
74+
switch (codePage)
75+
{
76+
case 65001:
77+
DecodeUtf8();
78+
break;
79+
case 1200:
80+
case 1201:
81+
DecodeUtf16();
82+
break;
83+
default:
84+
if (codePage == 20127 && bytes[_position] >= 128)
85+
HandleError(_position + 1, "ordinal not in range(128)");
86+
else
87+
_text.Append((char)bytes[_position++]);
88+
break;
89+
}
90+
CheckLength();
91+
}
92+
UserObjectProtocols.Dispatcher?.CheckIterationWork(span);
93+
return _text.ToString();
94+
}
95+
96+
private void CheckLength()
97+
{
98+
if (_text.Length > MaximumTextLength)
99+
throw ManagedObjectProtocols.Fault(
100+
"DPY4003",
101+
"The decoded text exceeds the supported size.",
102+
span,
103+
"OverflowError"
104+
);
105+
}
106+
107+
private void CheckProgress()
108+
{
109+
if (_position < _nextWork)
110+
return;
111+
UserObjectProtocols.Dispatcher?.CheckIterationWork(span);
112+
// A scalar or malformed prefix consumes at most four bytes. Leave
113+
// room for it so checkpoints are never more than 256 bytes apart.
114+
_nextWork = _position > int.MaxValue - 252 ? int.MaxValue : _position + 252;
115+
}
116+
117+
private void DecodeUtf8()
118+
{
119+
var first = bytes[_position];
120+
if (first < 128)
121+
{
122+
_text.Append((char)first);
123+
++_position;
124+
return;
125+
}
126+
var length = first switch
127+
{
128+
>= 0xc2 and <= 0xdf => 2,
129+
>= 0xe0 and <= 0xef => 3,
130+
>= 0xf0 and <= 0xf4 => 4,
131+
_ => 0,
132+
};
133+
if (length == 0)
134+
{
135+
HandleError(_position + 1, "invalid start byte");
136+
return;
137+
}
138+
139+
var scalar = first & (0x7f >> length);
140+
for (var offset = 1; offset < length; ++offset)
141+
{
142+
if (offset >= bytes.Length - _position)
143+
{
144+
HandleError(bytes.Length, "unexpected end of data");
145+
return;
146+
}
147+
var next = bytes[_position + offset];
148+
if (
149+
next is < 0x80 or > 0xbf
150+
|| offset == 1
151+
&& (
152+
first == 0xe0 && next < 0xa0
153+
|| first == 0xed && next >= 0xa0
154+
|| first == 0xf0 && next < 0x90
155+
|| first == 0xf4 && next >= 0x90
156+
)
157+
)
158+
{
159+
HandleError(_position + offset, "invalid continuation byte");
160+
return;
161+
}
162+
scalar = (scalar << 6) | (next & 0x3f);
163+
}
164+
_text.Append(char.ConvertFromUtf32(scalar));
165+
_position += length;
166+
}
167+
168+
private void DecodeUtf16()
169+
{
170+
if (bytes.Length - _position < 2)
171+
{
172+
HandleError(bytes.Length, "truncated data");
173+
return;
174+
}
175+
var first = ReadUtf16(_position);
176+
if (first is < 0xd800 or > 0xdfff)
177+
{
178+
_text.Append((char)first);
179+
_position += 2;
180+
return;
181+
}
182+
if (first >= 0xdc00)
183+
{
184+
HandleError(_position + 2, "illegal encoding");
185+
return;
186+
}
187+
if (bytes.Length - _position < 4)
188+
{
189+
HandleError(bytes.Length, "unexpected end of data");
190+
return;
191+
}
192+
var second = ReadUtf16(_position + 2);
193+
if (second is < 0xdc00 or > 0xdfff)
194+
{
195+
HandleError(_position + 2, "illegal UTF-16 surrogate");
196+
return;
197+
}
198+
_text.Append((char)first);
199+
_text.Append((char)second);
200+
_position += 4;
201+
}
202+
203+
private int ReadUtf16(int index) =>
204+
_bigEndian
205+
? (bytes[index] << 8) | bytes[index + 1]
206+
: bytes[index] | (bytes[index + 1] << 8);
207+
208+
private void HandleError(int end, string reason)
209+
{
210+
switch (errors)
211+
{
212+
case "ignore":
213+
break;
214+
case "replace":
215+
_text.Append('\ufffd');
216+
break;
217+
case "backslashreplace":
218+
for (var index = _position; index < end; ++index)
219+
{
220+
_text.Append("\\x");
221+
_text.Append(
222+
bytes[index]
223+
.ToString("x2", System.Globalization.CultureInfo.InvariantCulture)
224+
);
225+
CheckLength();
226+
}
227+
break;
228+
case "strict":
229+
var location =
230+
end == _position + 1
231+
? $"byte 0x{bytes[_position]:x2} in position {_position}"
232+
: $"bytes in position {_position}-{end - 1}";
233+
throw ManagedObjectProtocols.Fault(
234+
"DPY4003",
235+
$"'{_encoding}' codec can't decode {location}: {reason}",
236+
span,
237+
"UnicodeDecodeError"
238+
);
239+
case "xmlcharrefreplace":
240+
case "namereplace":
241+
throw ManagedObjectProtocols.Fault(
242+
"DPY4003",
243+
"don't know how to handle UnicodeDecodeError in error callback",
244+
span,
245+
"TypeError"
246+
);
247+
default:
248+
// CPython looks handlers up only when malformed input actually
249+
// needs one. Valid data accepts even an unregistered name.
250+
throw ManagedObjectProtocols.Fault(
251+
"DPY4003",
252+
$"unknown error handler name '{errors}'",
253+
span,
254+
"LookupError"
255+
);
256+
}
257+
_position = end;
258+
}
259+
}
260+
}

0 commit comments

Comments
 (0)