From 5be274f96e805acda36b9b6a70a9c50fb8079654 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 12:53:48 +0200 Subject: [PATCH 1/8] Keep response headers case-insensitive build_response copied the headers into a plain dict, which replaced the CaseInsensitiveDict that Response starts with, so response.headers ["content-type"] raised KeyError. Build the CaseInsensitiveDict directly. Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_response.py | 8 +++++--- tls_client/response.py | 7 +++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_response.py b/tests/unit/test_response.py index 3212844..5b61e98 100644 --- a/tests/unit/test_response.py +++ b/tests/unit/test_response.py @@ -40,8 +40,10 @@ def test_null_headers_become_an_empty_mapping(): assert dict(response.headers) == {} -@pytest.mark.xfail(reason="build_response replaces the CaseInsensitiveDict with a plain dict") -def test_header_lookup_ignores_case(): +@pytest.mark.parametrize("name", ["Content-Type", "content-type", "CONTENT-TYPE"]) +def test_header_lookup_ignores_case(name): response = build_response(library_reply(headers={"Content-Type": ["text/html"]}), cookiejar_from_dict({})) - assert response.headers["content-type"] == "text/html" + assert response.headers[name] == "text/html" + assert response.headers.get(name) == "text/html" + assert name in response.headers diff --git a/tls_client/response.py b/tls_client/response.py index f57904b..e674157 100644 --- a/tls_client/response.py +++ b/tls_client/response.py @@ -1,5 +1,4 @@ import json -from collections.abc import MutableMapping from typing import Any from .cookies import RequestsCookieJar, cookiejar_from_dict @@ -19,8 +18,8 @@ def __init__(self) -> None: # String of responded HTTP Body. self.text: str = "" - # Dictionary of Response Headers. Headers sent more than once map to a list of values. - self.headers: MutableMapping[str, str | list[str]] = CaseInsensitiveDict() + # Case-insensitive dictionary of Response Headers. Headers sent more than once map to a list of values. + self.headers: CaseInsensitiveDict[str | list[str]] = CaseInsensitiveDict() # A CookieJar of Cookies the server sent back. self.cookies = cookiejar_from_dict({}) @@ -51,7 +50,7 @@ def build_response(res: dict[str, Any], res_cookies: RequestsCookieJar) -> Respo # Add status code response.status_code = res["status"] # Add headers - response_headers: dict[str, str | list[str]] = {} + response_headers: CaseInsensitiveDict[str | list[str]] = CaseInsensitiveDict() if res["headers"] is not None: for header_key, header_value in res["headers"].items(): if len(header_value) == 1: From 8c770ba0e84b19a0f57b743422e25e1d4955127a Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 12:58:58 +0200 Subject: [PATCH 2/8] Return binary response bodies unchanged The library converted every body to UTF-8 text, and Response.content was that text encoded again, so images and other binary downloads came back corrupted. Requests now set isByteResponse, which makes the library send the raw bytes as a base64 data URI, and build_response decodes them into .content. .text is now a property that decodes .content like requests does: with the charset from the Content-Type header, else UTF-8, replacing bytes that don't decode. The charset is exposed as .encoding and can be set to decode with another one. The data URI's own mime type is ignored because the library sniffs it from the bytes. The httpbin test for this asked /base64 for PNG bytes, which httpbin can only return as UTF-8 text; it now downloads /image/png. A stdlib server covers exact bytes, empty and large bodies, and charsets. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_httpbin.py | 9 +- tests/integration/test_local_server.py | 117 +++++++++++++++++++++++++ tests/unit/test_response.py | 76 +++++++++++++++- tests/unit/test_sessions.py | 5 ++ tls_client/response.py | 45 ++++++++-- tls_client/sessions.py | 2 + 6 files changed, 241 insertions(+), 13 deletions(-) create mode 100644 tests/integration/test_local_server.py diff --git a/tests/integration/test_httpbin.py b/tests/integration/test_httpbin.py index 975d91a..3ac4d67 100644 --- a/tests/integration/test_httpbin.py +++ b/tests/integration/test_httpbin.py @@ -3,6 +3,7 @@ from collections.abc import Iterator import pytest +from httpbin.core import resource as httpbin_resource from tls_client import Session from tls_client.exceptions import TLSClientException @@ -78,13 +79,11 @@ def test_utf8_body_is_decoded(self, httpbin, session): assert "∮ E⋅da = Q" in response.text - @pytest.mark.xfail(reason="the library hands the body over as text, so bytes that are not UTF-8 get mangled") def test_binary_body_arrives_unchanged(self, httpbin, session): - payload = b"\x89PNG\r\n\x1a\n" + response = session.get(f"{httpbin.url}/image/png") - response = session.get(f"{httpbin.url}/base64/{base64.urlsafe_b64encode(payload).decode()}") - - assert response.content == payload + # httpbin serves this file from its package data. + assert response.content == httpbin_resource("images/pig_icon.png") class TestRedirects: diff --git a/tests/integration/test_local_server.py b/tests/integration/test_local_server.py new file mode 100644 index 0000000..0be2383 --- /dev/null +++ b/tests/integration/test_local_server.py @@ -0,0 +1,117 @@ +"""Real requests through the native library against a stdlib HTTP server, for bodies and headers httpbin can't +produce byte for byte.""" + +import base64 +import json +import os +import threading +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from tls_client import Session + +pytestmark = pytest.mark.integration + +# A 1x1 PNG, the kind of download that used to arrive mangled. +PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" +) +LARGE = os.urandom(8 * 1024 * 1024) + +# Path -> (Content-Type, body) +BODIES = { + "/bytes": ("application/octet-stream", bytes(range(256))), + "/png": ("image/png", PNG), + "/large": ("application/octet-stream", LARGE), + "/empty": ("text/plain", b""), + "/json": ("application/json", json.dumps({"name": "tls", "ids": [1, 2]}).encode()), + "/utf8": ("text/plain", "∮ E⋅da = Q".encode()), + "/latin-1": ("text/plain; charset=iso-8859-1", "café".encode("latin-1")), +} + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self) -> None: + if self.path == "/headers": + self.echo_headers() + else: + content_type, body = BODIES[self.path] + self.reply(content_type, body, {"x-MiXeD-case": "yes"}) + + def do_POST(self) -> None: + self.rfile.read(int(self.headers.get("Content-Length", 0))) + self.echo_headers() + + def echo_headers(self) -> None: + received = {name.lower(): value for name, value in self.headers.items()} + self.reply("application/json", json.dumps(received).encode()) + + def reply(self, content_type: str, body: bytes, extra_headers: dict[str, str] | None = None) -> None: + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + for name, value in (extra_headers or {}).items(): + self.send_header(name, value) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + pass + + +@pytest.fixture(scope="module") +def server_url() -> Iterator[str]: + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{server.server_port}" + server.shutdown() + server.server_close() + + +@pytest.fixture +def session() -> Iterator[Session]: + with Session() as s: + yield s + + +class TestBody: + @pytest.mark.parametrize("path", ["/bytes", "/png", "/large"]) + def test_binary_body_arrives_byte_for_byte(self, server_url, session, path): + response = session.get(f"{server_url}{path}") + + assert response.status_code == 200 + assert response.content == BODIES[path][1] + + def test_empty_body(self, server_url, session): + response = session.get(f"{server_url}/empty") + + assert response.content == b"" + assert response.text == "" + + def test_json_body(self, server_url, session): + assert session.get(f"{server_url}/json").json() == {"name": "tls", "ids": [1, 2]} + + def test_text_defaults_to_utf8(self, server_url, session): + assert session.get(f"{server_url}/utf8").text == "∮ E⋅da = Q" + + def test_text_uses_the_charset_from_the_content_type_header(self, server_url, session): + response = session.get(f"{server_url}/latin-1") + + assert response.content == "café".encode("latin-1") + assert response.encoding == "iso-8859-1" + assert response.text == "café" + + +class TestHeaders: + @pytest.mark.parametrize("name", ["X-Mixed-Case", "x-mixed-case", "X-MIXED-CASE"]) + def test_response_header_lookup_ignores_case(self, server_url, session, name): + response = session.get(f"{server_url}/bytes") + + assert response.headers[name] == "yes" + assert response.headers["content-type"] == "application/octet-stream" diff --git a/tests/unit/test_response.py b/tests/unit/test_response.py index 5b61e98..9f135ee 100644 --- a/tests/unit/test_response.py +++ b/tests/unit/test_response.py @@ -1,3 +1,4 @@ +import base64 from typing import Any import pytest @@ -6,16 +7,28 @@ from tls_client.response import build_response +def data_uri(body: bytes, mime_type: str = "text/plain; charset=utf-8") -> str: + """A body the way the Go library returns it for isByteResponse requests; it sniffs the mime type itself.""" + return f"data:{mime_type};base64,{base64.b64encode(body).decode()}" + + def library_reply(**fields: Any) -> dict[str, Any]: """A reply shaped like the one the Go library returns.""" - return {"id": "response-1", "status": 200, "target": "https://example.com/", "body": "", "headers": {}, **fields} + return { + "id": "response-1", + "status": 200, + "target": "https://example.com/", + "body": data_uri(b""), + "headers": {}, + **fields, + } def test_reply_fields_are_mapped_onto_the_response(): cookies = cookiejar_from_dict({"sid": "abc"}) response = build_response( - library_reply(status=404, target="https://example.com/final", body='{"error": "missing"}'), + library_reply(status=404, target="https://example.com/final", body=data_uri(b'{"error": "missing"}')), cookies, ) @@ -47,3 +60,62 @@ def test_header_lookup_ignores_case(name): assert response.headers[name] == "text/html" assert response.headers.get(name) == "text/html" assert name in response.headers + + +class TestBody: + def test_binary_body_is_decoded_byte_for_byte(self): + body = bytes(range(256)) + + response = build_response( + library_reply(body=data_uri(body, "application/octet-stream")), cookiejar_from_dict({}) + ) + + assert response.content == body + + def test_empty_body(self): + response = build_response(library_reply(), cookiejar_from_dict({})) + + assert response.content == b"" + assert response.text == "" + + def test_text_uses_the_charset_from_the_content_type_header(self): + # The library labels the data URI with a sniffed type; only the header names the real charset. + response = build_response( + library_reply( + body=data_uri("café".encode("latin-1")), + headers={"content-type": ['text/plain; Charset="ISO-8859-1"']}, + ), + cookiejar_from_dict({}), + ) + + assert response.encoding == "ISO-8859-1" + assert response.text == "café" + + def test_text_defaults_to_utf8_and_replaces_undecodable_bytes(self): + response = build_response( + library_reply(body=data_uri("∮ ok ".encode() + b"\xff"), headers={"Content-Type": ["text/plain"]}), + cookiejar_from_dict({}), + ) + + assert response.encoding is None + assert response.text == "∮ ok �" + + def test_unknown_charset_falls_back_to_utf8(self): + response = build_response( + library_reply(body=data_uri("é".encode()), headers={"Content-Type": ["text/plain; charset=made-up"]}), + cookiejar_from_dict({}), + ) + + assert response.text == "é" + + def test_text_follows_an_encoding_set_by_the_caller(self): + response = build_response(library_reply(body=data_uri("café".encode("cp1252"))), cookiejar_from_dict({})) + + response.encoding = "cp1252" + + assert response.text == "café" + + def test_body_that_is_not_a_data_uri_is_taken_as_text(self): + response = build_response(library_reply(body='{"plain": true}'), cookiejar_from_dict({})) + + assert response.json() == {"plain": True} diff --git a/tests/unit/test_sessions.py b/tests/unit/test_sessions.py index 083c696..9c8997c 100644 --- a/tests/unit/test_sessions.py +++ b/tests/unit/test_sessions.py @@ -73,6 +73,11 @@ def test_bytes_are_base64_encoded_and_flagged_for_the_library(self, session, fak assert fake_lib.sent["isByteRequest"] is True assert base64.b64decode(fake_lib.sent["requestBody"]) == body + def test_response_body_is_requested_as_bytes(self, session, fake_lib): + session.get(URL) + + assert fake_lib.sent["isByteResponse"] is True + def test_data_takes_precedence_over_json(self, session, fake_lib): session.post(URL, data={"from": "data"}, json={"from": "json"}) diff --git a/tls_client/response.py b/tls_client/response.py index e674157..a3c16f7 100644 --- a/tls_client/response.py +++ b/tls_client/response.py @@ -1,3 +1,4 @@ +import binascii import json from typing import Any @@ -15,15 +16,17 @@ def __init__(self) -> None: # Integer Code of responded HTTP Status, e.g. 404 or 200. self.status_code: int = 0 - # String of responded HTTP Body. - self.text: str = "" - # Case-insensitive dictionary of Response Headers. Headers sent more than once map to a list of values. self.headers: CaseInsensitiveDict[str | list[str]] = CaseInsensitiveDict() # A CookieJar of Cookies the server sent back. self.cookies = cookiejar_from_dict({}) + # Encoding used to decode .text, taken from the charset of the Content-Type header. Set it to decode the body + # with another encoding; None means UTF-8. + self.encoding: str | None = None + + # Bytes of responded HTTP Body. self._content: bytes = b"" def __enter__(self) -> "Response": @@ -41,6 +44,25 @@ def content(self) -> bytes: """Content of the response, in bytes.""" return self._content + @property + def text(self) -> str: + """Content of the response, decoded with .encoding (UTF-8 if unset). Undecodable bytes become U+FFFD.""" + # Decoded on every access rather than stored, so a large body isn't kept twice. + try: + return str(self._content, self.encoding or "utf-8", errors="replace") + except LookupError: + # The server named a charset Python doesn't know. + return str(self._content, "utf-8", errors="replace") + + +def _charset(content_type: str) -> str | None: + """Charset parameter of a Content-Type header value, e.g. "ISO-8859-1" for "text/html; charset=ISO-8859-1".""" + for param in content_type.split(";")[1:]: + key, _, value = param.partition("=") + if key.strip().lower() == "charset": + return value.strip().strip("\"'") or None + return None + def build_response(res: dict[str, Any], res_cookies: RequestsCookieJar) -> Response: """Builds a Response object""" @@ -58,10 +80,21 @@ def build_response(res: dict[str, Any], res_cookies: RequestsCookieJar) -> Respo else: response_headers[header_key] = header_value response.headers = response_headers + # Decode .text with the charset the server declared, if any + content_type = response_headers.get("Content-Type") + if content_type: + response.encoding = _charset(content_type if isinstance(content_type, str) else content_type[0]) # Add cookies response.cookies = res_cookies - # Add response body - response.text = res["body"] # Add response content (bytes) - response._content = res["body"].encode() + # With isByteResponse the library sends "data:;base64,". It sniffs that mime type from the + # bytes, which is why .encoding comes from the Content-Type header instead. + body: str = res["body"] + prefix, _, payload = body.partition(",") + if prefix.startswith("data:") and prefix.endswith(";base64"): + # a2b_base64 reads an ASCII str in place, without encoding a copy first. + response._content = binascii.a2b_base64(payload) + else: + # Not a data URI: the body is plain text. + response._content = body.encode() return response diff --git a/tls_client/sessions.py b/tls_client/sessions.py index c9197e0..2fb4c81 100644 --- a/tls_client/sessions.py +++ b/tls_client/sessions.py @@ -439,6 +439,8 @@ def execute_request( "headerOrder": request_header_order, "insecureSkipVerify": insecure_skip_verify, "isByteRequest": is_byte_request, + # Return the body as base64 so binary bodies survive; build_response decodes it. + "isByteResponse": True, "additionalDecode": self.additional_decode, "proxyUrl": proxy_url, "requestUrl": url, From 2343ae9abbf02d9007e58e3f137e25e0ef6ea658 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 12:59:46 +0200 Subject: [PATCH 3/8] Free copies of large response bodies early A response used to keep its body twice (.text and .content), and while it was built the raw reply, its decoded string, the parsed body, .text and .content were all alive at once. With a 36 MB JSON body, tracemalloc showed a 192 MB peak and 72 MB retained. .content is now the only stored copy (.text is decoded on access since the previous commit), the reply is decoded and parsed without binding the intermediate copies, and build_response pops the data URI from the reply and drops it before decoding. Measured the same way, on Python 3.10 and 3.12: 95 MB peak, 36 MB retained. Memory the Go library holds is not affected. Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_response.py | 19 +++++++++++++++++++ tls_client/response.py | 6 ++++-- tls_client/sessions.py | 11 +++-------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_response.py b/tests/unit/test_response.py index 9f135ee..68e5190 100644 --- a/tests/unit/test_response.py +++ b/tests/unit/test_response.py @@ -1,4 +1,6 @@ import base64 +import os +import tracemalloc from typing import Any import pytest @@ -119,3 +121,20 @@ def test_body_that_is_not_a_data_uri_is_taken_as_text(self): response = build_response(library_reply(body='{"plain": true}'), cookiejar_from_dict({})) assert response.json() == {"plain": True} + + def test_large_body_is_not_held_in_several_copies_at_once(self): + body = os.urandom(8_000_000) + tracemalloc.start() + try: + reply = library_reply(body=data_uri(body)) + tracemalloc.reset_peak() + response = build_response(reply, cookiejar_from_dict({})) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert response.content == body + assert "body" not in reply + # The data URI and its payload are each 4/3 of the body. Freeing the data URI before decoding keeps the + # peak near 8/3 of the body; holding it, the payload and the decoded bytes together would reach 11/3. + assert peak < 3 * len(body) diff --git a/tls_client/response.py b/tls_client/response.py index a3c16f7..e79420b 100644 --- a/tls_client/response.py +++ b/tls_client/response.py @@ -65,7 +65,7 @@ def _charset(content_type: str) -> str | None: def build_response(res: dict[str, Any], res_cookies: RequestsCookieJar) -> Response: - """Builds a Response object""" + """Builds a Response object. Pops the body from res, so a large body can be freed as soon as it's decoded.""" response = Response() # Add target / url response.url = res["target"] @@ -89,9 +89,11 @@ def build_response(res: dict[str, Any], res_cookies: RequestsCookieJar) -> Respo # Add response content (bytes) # With isByteResponse the library sends "data:;base64,". It sniffs that mime type from the # bytes, which is why .encoding comes from the Content-Type header instead. - body: str = res["body"] + body: str = res.pop("body") prefix, _, payload = body.partition(",") if prefix.startswith("data:") and prefix.endswith(";base64"): + # Free the data URI before decoding, so it isn't held next to both the payload and the bytes. + del body # a2b_base64 reads an ASCII str in place, without encoding a copy first. response._content = binascii.a2b_base64(payload) else: diff --git a/tls_client/sessions.py b/tls_client/sessions.py index 2fb4c81..5b998ee 100644 --- a/tls_client/sessions.py +++ b/tls_client/sessions.py @@ -474,14 +474,9 @@ def execute_request( request_payload["tlsClientIdentifier"] = self.client_identifier request_payload["withRandomTLSExtensionOrder"] = self.random_tls_extension_order - # this is a pointer to the response - response = request(dumps(request_payload).encode("utf-8")) - # dereference the pointer to a byte array - response_bytes = ctypes.string_at(response) - # convert our byte array to a string (tls client returns json) - response_string = response_bytes.decode("utf-8") - # convert response string to json - response_object = loads(response_string) + # dereference the pointer to the response, decode it and parse it (tls client returns json). None of the steps + # is bound to a name, so each copy of a large reply is freed as soon as the next one exists. + response_object = loads(ctypes.string_at(request(dumps(request_payload).encode("utf-8"))).decode("utf-8")) # free the memory freeMemory(response_object["id"].encode("utf-8")) # --- Response ------------------------------------------------------------------------------------------------- From 700d49ab07528d19e9c424c3b2728d236198d8b1 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 13:00:35 +0200 Subject: [PATCH 4/8] Stop the inferred Content-Type from sticking to the session execute_request wrote the Content-Type it inferred from json= or data= into self.headers. After one json= POST, every later request on the session claimed to be application/json, form posts included. The label now goes into the per-request copy of the headers, and only if neither the session nor the request set a Content-Type (in any case). Setting it to None for a request sends none. Request headers are always merged into a copy, so None values in session headers are dropped too. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_local_server.py | 21 +++++++++++++++++ tests/unit/test_sessions.py | 10 ++++++-- tls_client/sessions.py | 32 +++++++++++--------------- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/tests/integration/test_local_server.py b/tests/integration/test_local_server.py index 0be2383..3e01ef6 100644 --- a/tests/integration/test_local_server.py +++ b/tests/integration/test_local_server.py @@ -115,3 +115,24 @@ def test_response_header_lookup_ignores_case(self, server_url, session, name): assert response.headers[name] == "yes" assert response.headers["content-type"] == "application/octet-stream" + + def test_inferred_content_type_does_not_leak_into_later_requests(self, server_url, session): + json_post = session.post(f"{server_url}/headers", json={"a": 1}).json() + form_post = session.post(f"{server_url}/headers", data={"a": "1"}).json() + get = session.get(f"{server_url}/headers").json() + + assert json_post["content-type"] == "application/json" + assert form_post["content-type"] == "application/x-www-form-urlencoded" + assert "content-type" not in get + assert "Content-Type" not in session.headers + + def test_content_type_from_the_session_or_the_request_is_kept(self, server_url, session): + session.headers["content-type"] = "application/vnd.api+json" + + from_session = session.post(f"{server_url}/headers", json={"a": 1}).json() + from_request = session.post( + f"{server_url}/headers", json={"a": 1}, headers={"CONTENT-TYPE": "text/plain"} + ).json() + + assert from_session["content-type"] == "application/vnd.api+json" + assert from_request["content-type"] == "text/plain" diff --git a/tests/unit/test_sessions.py b/tests/unit/test_sessions.py index 9c8997c..be80697 100644 --- a/tests/unit/test_sessions.py +++ b/tests/unit/test_sessions.py @@ -95,14 +95,20 @@ def test_content_type_passed_with_the_request_wins(self, session, fake_lib): assert header_values(fake_lib.sent["headers"], "Content-Type") == ["text/plain"] - @pytest.mark.xfail(reason="the inferred Content-Type is written into session.headers and sticks to later requests") def test_inferred_content_type_applies_to_one_request_only(self, session, fake_lib): session.post(URL, json={"a": 1}) session.post(URL, data={"a": "1"}) - assert header_values(fake_lib.sent["headers"], "Content-Type") == ["application/x-www-form-urlencoded"] + + session.post(URL, data="raw") + assert header_values(fake_lib.sent["headers"], "Content-Type") == [] assert "Content-Type" not in session.headers + def test_content_type_set_to_none_suppresses_the_inferred_one(self, session, fake_lib): + session.post(URL, json={"a": 1}, headers={"Content-Type": None}) + + assert header_values(fake_lib.sent["headers"], "Content-Type") == [] + @pytest.mark.parametrize("method", ["post", "put", "patch"]) def test_body_verbs_take_data_as_second_positional_argument(self, session, fake_lib, method): getattr(session, method)(URL, {"k": "v"}) diff --git a/tls_client/sessions.py b/tls_client/sessions.py index 5b998ee..0e65c29 100644 --- a/tls_client/sessions.py +++ b/tls_client/sessions.py @@ -364,26 +364,20 @@ def execute_request( else: request_body = urllib.parse.urlencode(data, doseq=True) content_type = "application/x-www-form-urlencoded" - # set content type if it isn't set - if content_type is not None and "content-type" not in self.headers: - self.headers["Content-Type"] = content_type - # --- Headers -------------------------------------------------------------------------------------------------- - request_headers: Mapping[str, str | None] - if self.headers is None: - request_headers = CaseInsensitiveDict(headers) - elif headers is None: - request_headers = self.headers - else: - merged_headers: CaseInsensitiveDict[str | None] = CaseInsensitiveDict(self.headers) - merged_headers.update(headers) - - # Remove items, where the key or value is set to None. - none_keys = [k for (k, v) in merged_headers.items() if v is None or k is None] - for key in none_keys: - del merged_headers[key] - - request_headers = merged_headers + # Merge into a copy, so nothing set for this request sticks to self.headers. + request_headers: CaseInsensitiveDict[str | None] = CaseInsensitiveDict(self.headers) + if headers is not None: + request_headers.update(headers) + + # set content type if neither the session nor the request set one (setting it to None sends none) + if content_type is not None and "content-type" not in request_headers: + request_headers["Content-Type"] = content_type + + # Remove items, where the key or value is set to None. + none_keys = [k for (k, v) in request_headers.items() if v is None or k is None] + for key in none_keys: + del request_headers[key] # Ability to overwrite self.header_order for flexibility request_header_order = self.header_order From c8d981cec3b39b0cded60f47dd178d77ccd6c4d5 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 13:02:08 +0200 Subject: [PATCH 5/8] Add a requests-style timeout argument Requests take timeout= now, in seconds as an int or float, next to the existing timeout_seconds=. Passing both raises ValueError instead of picking one silently. session.timeout_seconds can be a float as well. The timeout goes to the library as timeoutMilliseconds, so fractions of a second work; timeoutSeconds only takes whole seconds and the library refuses requests that set both, so it is no longer sent. The library still fixes a session's timeout on its first request (the existing xfail test, now run for both names). The docstring and README say so. Co-Authored-By: Claude Opus 5.5 --- README.md | 7 ++++++ tests/integration/test_httpbin.py | 5 +++-- tests/integration/test_local_server.py | 31 ++++++++++++++++++++++++-- tests/unit/test_sessions.py | 22 ++++++++++++++---- tls_client/sessions.py | 24 +++++++++++++++----- 5 files changed, 75 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index bf13fc9..1fd90d6 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,13 @@ res = session.post( ) ``` +Timeouts: +```python +res = session.get("https://www.example.com/", timeout=2.5) # seconds, int or float +``` +`timeout` works like in requests. `timeout_seconds` is the older name for it; passing both raises `ValueError`. Without either, `session.timeout_seconds` (30 by default) applies. +The native library sets a session's timeout when the session sends its first request and keeps it, so a different timeout on a later request of the same session has no effect. Use a new `Session` for that. + # Pyinstaller / Pyarmor **If you want to pack the library with Pyinstaller or Pyarmor, make sure to add this to your command:** diff --git a/tests/integration/test_httpbin.py b/tests/integration/test_httpbin.py index 3ac4d67..367cacc 100644 --- a/tests/integration/test_httpbin.py +++ b/tests/integration/test_httpbin.py @@ -129,11 +129,12 @@ def test_session_timeout_raises(self, httpbin, session): session.get(f"{httpbin.url}/delay/2") @pytest.mark.xfail(reason="the native session keeps the timeout of its first request") - def test_request_timeout_applies_after_the_first_request(self, httpbin, session): + @pytest.mark.parametrize("kwarg", ["timeout", "timeout_seconds"]) + def test_request_timeout_applies_after_the_first_request(self, httpbin, session, kwarg): session.get(f"{httpbin.url}/get") with pytest.raises(TLSClientException, match="Timeout exceeded"): - session.get(f"{httpbin.url}/delay/2", timeout_seconds=1) + session.get(f"{httpbin.url}/delay/2", **{kwarg: 1}) class TestCertificateVerification: diff --git a/tests/integration/test_local_server.py b/tests/integration/test_local_server.py index 3e01ef6..86755a0 100644 --- a/tests/integration/test_local_server.py +++ b/tests/integration/test_local_server.py @@ -5,12 +5,14 @@ import json import os import threading +import time from collections.abc import Iterator from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import pytest from tls_client import Session +from tls_client.exceptions import TLSClientException pytestmark = pytest.mark.integration @@ -19,6 +21,8 @@ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" ) LARGE = os.urandom(8 * 1024 * 1024) +# How long /slow takes to answer. +SLOW_SECONDS = 0.6 # Path -> (Content-Type, body) BODIES = { @@ -38,6 +42,9 @@ class Handler(BaseHTTPRequestHandler): def do_GET(self) -> None: if self.path == "/headers": self.echo_headers() + elif self.path == "/slow": + time.sleep(SLOW_SECONDS) + self.reply("text/plain", b"done") else: content_type, body = BODIES[self.path] self.reply(content_type, body, {"x-MiXeD-case": "yes"}) @@ -56,8 +63,12 @@ def reply(self, content_type: str, body: bytes, extra_headers: dict[str, str] | self.send_header("Content-Length", str(len(body))) for name, value in (extra_headers or {}).items(): self.send_header(name, value) - self.end_headers() - self.wfile.write(body) + try: + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + # The client gave up waiting, as the timeout tests make it do. + pass def log_message(self, format: str, *args: object) -> None: pass @@ -136,3 +147,19 @@ def test_content_type_from_the_session_or_the_request_is_kept(self, server_url, assert from_session["content-type"] == "application/vnd.api+json" assert from_request["content-type"] == "text/plain" + + +class TestTimeout: + @pytest.mark.parametrize("kwarg", ["timeout", "timeout_seconds"]) + def test_fractional_timeout_is_applied(self, server_url, session, kwarg): + with pytest.raises(TLSClientException, match="Timeout exceeded"): + session.get(f"{server_url}/slow", **{kwarg: SLOW_SECONDS / 3}) + + def test_response_within_the_timeout_arrives(self, server_url, session): + assert session.get(f"{server_url}/slow", timeout=SLOW_SECONDS * 4).text == "done" + + def test_fractional_session_timeout_is_applied(self, server_url, session): + session.timeout_seconds = SLOW_SECONDS / 3 + + with pytest.raises(TLSClientException, match="Timeout exceeded"): + session.get(f"{server_url}/slow") diff --git a/tests/unit/test_sessions.py b/tests/unit/test_sessions.py index be80697..bb94082 100644 --- a/tests/unit/test_sessions.py +++ b/tests/unit/test_sessions.py @@ -220,14 +220,28 @@ def test_request_proxy_wins_over_session_proxy(self, session, fake_lib): assert fake_lib.sent["proxyUrl"] == "http://request-proxy:8080" - def test_request_timeout_wins_over_session_timeout(self, session, fake_lib): + @pytest.mark.parametrize("kwarg", ["timeout", "timeout_seconds"]) + def test_request_timeout_wins_over_session_timeout(self, session, fake_lib, kwarg): session.timeout_seconds = 12 session.get(URL) - assert fake_lib.sent["timeoutSeconds"] == 12 + assert fake_lib.sent["timeoutMilliseconds"] == 12_000 - session.get(URL, timeout_seconds=3) - assert fake_lib.sent["timeoutSeconds"] == 3 + session.get(URL, **{kwarg: 3}) + assert fake_lib.sent["timeoutMilliseconds"] == 3_000 + + def test_timeout_is_sent_in_milliseconds_only(self, session, fake_lib): + session.get(URL, timeout=2.5) + + assert fake_lib.sent["timeoutMilliseconds"] == 2_500 + # The library refuses a request that sets both. + assert "timeoutSeconds" not in fake_lib.sent + + def test_timeout_and_timeout_seconds_together_are_rejected(self, session, fake_lib): + with pytest.raises(ValueError, match="not both"): + session.get(URL, timeout=1, timeout_seconds=2) + + assert fake_lib.payloads == [] @pytest.mark.parametrize( ("session_kwargs", "request_kwargs", "payload_key"), diff --git a/tls_client/sessions.py b/tls_client/sessions.py index 0e65c29..8a59b27 100644 --- a/tls_client/sessions.py +++ b/tls_client/sessions.py @@ -87,8 +87,8 @@ def __init__( # CookieJar containing all currently outstanding cookies set on this session self.cookies = cookiejar_from_dict({}) - # Timeout - self.timeout_seconds = 30 + # Timeout in seconds (int or float) for requests that don't pass timeout + self.timeout_seconds: float = 30 # Certificate pinning self.certificate_pinning = certificate_pinning @@ -342,9 +342,18 @@ def execute_request( json: _Json | None = None, allow_redirects: bool = False, insecure_skip_verify: bool = False, - timeout_seconds: int | None = None, + timeout_seconds: float | None = None, proxy: _Proxy | None = None, + timeout: float | None = None, ) -> Response: + """Sends a request and returns its Response. + + timeout is how many seconds (int or float) to wait for the response, like in requests. timeout_seconds is + the older name for it; passing both raises ValueError. Without either, self.timeout_seconds applies. + + The native library sets a session's timeout when the session sends its first request and keeps it, so a + different timeout on a later request of the same session has no effect. Use a new Session for that. + """ # --- URL ------------------------------------------------------------------------------------------------------ # Prepare URL - add params to url if params is not None: @@ -412,9 +421,11 @@ def execute_request( proxy_url = "" # --- Timeout -------------------------------------------------------------------------------------------------- - # maximum time to wait for a response + # maximum time to wait for a response. timeout is the requests-style name for timeout_seconds. + if timeout is not None and timeout_seconds is not None: + raise ValueError("Pass either timeout or timeout_seconds, not both") - timeout_seconds = timeout_seconds or self.timeout_seconds + request_timeout = timeout or timeout_seconds or self.timeout_seconds # --- Certificate pinning -------------------------------------------------------------------------------------- # pins a certificate so that it restricts which certificates are considered valid @@ -443,7 +454,8 @@ def execute_request( if isinstance(request_body, (bytes, bytearray)) else request_body, "requestCookies": request_cookies, - "timeoutSeconds": timeout_seconds, + # Milliseconds, so fractions of a second work. The library rejects timeoutSeconds alongside it. + "timeoutMilliseconds": round(request_timeout * 1000), } if certificate_pinning: request_payload["certificatePinningHosts"] = certificate_pinning From 7a5cd00875bcf8021e13f67af6b72dd5bbcffe58 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 13:06:05 +0200 Subject: [PATCH 6/8] Ignore every kind of client disconnect in the test server Windows reports a client that gave up as ConnectionAbortedError, which the handler didn't catch. Catch ConnectionError, the base of all three, so the timeout tests don't print tracebacks from the server thread. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_local_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_local_server.py b/tests/integration/test_local_server.py index 86755a0..2222f44 100644 --- a/tests/integration/test_local_server.py +++ b/tests/integration/test_local_server.py @@ -66,7 +66,7 @@ def reply(self, content_type: str, body: bytes, extra_headers: dict[str, str] | try: self.end_headers() self.wfile.write(body) - except (BrokenPipeError, ConnectionResetError): + except ConnectionError: # The client gave up waiting, as the timeout tests make it do. pass From 8655c89332cd29b622465f417b34aa5e291e4bd3 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 13:08:14 +0200 Subject: [PATCH 7/8] Decode .text with a charset declared in a meta tag or BOM The library used to pick the charset of text responses itself, including from tags. Since .text is now decoded in Python, pages that only declare GBK or Shift_JIS in the document would turn into mojibake. Sniff BOMs, and declarations in the first 1024 bytes when the Content-Type header names no charset, and decode GB2312 as GBK like browsers do. Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_response.py | 52 +++++++++++++++++++++++++++++++++++++ tls_client/response.py | 47 ++++++++++++++++++++++++++++----- 2 files changed, 93 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_response.py b/tests/unit/test_response.py index 68e5190..d25d452 100644 --- a/tests/unit/test_response.py +++ b/tests/unit/test_response.py @@ -117,6 +117,58 @@ def test_text_follows_an_encoding_set_by_the_caller(self): assert response.text == "café" + @pytest.mark.parametrize( + "html", + [ + '{}', + "{}", + "{}", + ], + ) + def test_text_uses_the_charset_declared_in_a_meta_tag(self, html): + response = build_response( + library_reply(body=data_uri(html.format("中文").encode("gbk")), headers={"Content-Type": ["text/html"]}), + cookiejar_from_dict({}), + ) + + assert response.encoding.lower() == "gbk" + assert "中文" in response.text + + def test_text_uses_the_encoding_of_an_xml_declaration(self): + body = '日本'.encode("shift_jis") + + response = build_response( + library_reply(body=data_uri(body), headers={"Content-Type": ["application/xml"]}), cookiejar_from_dict({}) + ) + + assert response.text.endswith("日本") + + def test_header_charset_wins_over_a_meta_tag(self): + body = 'café'.encode("latin-1") + + response = build_response( + library_reply(body=data_uri(body), headers={"Content-Type": ["text/html; charset=ISO-8859-1"]}), + cookiejar_from_dict({}), + ) + + assert response.encoding == "ISO-8859-1" + assert response.text.endswith("café") + + def test_meta_tags_are_ignored_outside_html_and_xml(self): + body = b'{"html": ""}' + + response = build_response( + library_reply(body=data_uri(body), headers={"Content-Type": ["application/json"]}), cookiejar_from_dict({}) + ) + + assert response.encoding is None + + @pytest.mark.parametrize("encoding", ["utf-8-sig", "utf-16"]) + def test_text_uses_and_drops_a_byte_order_mark(self, encoding): + response = build_response(library_reply(body=data_uri("héllo".encode(encoding))), cookiejar_from_dict({})) + + assert response.text == "héllo" + def test_body_that_is_not_a_data_uri_is_taken_as_text(self): response = build_response(library_reply(body='{"plain": true}'), cookiejar_from_dict({})) diff --git a/tls_client/response.py b/tls_client/response.py index e79420b..c721ca6 100644 --- a/tls_client/response.py +++ b/tls_client/response.py @@ -1,5 +1,6 @@ import binascii import json +import re from typing import Any from .cookies import RequestsCookieJar, cookiejar_from_dict @@ -22,8 +23,8 @@ def __init__(self) -> None: # A CookieJar of Cookies the server sent back. self.cookies = cookiejar_from_dict({}) - # Encoding used to decode .text, taken from the charset of the Content-Type header. Set it to decode the body - # with another encoding; None means UTF-8. + # Encoding used to decode .text: the charset of the Content-Type header, else a BOM or the charset the document + # declares in a tag. Set it to decode the body with another encoding; None means UTF-8. self.encoding: str | None = None # Bytes of responded HTTP Body. @@ -64,6 +65,40 @@ def _charset(content_type: str) -> str | None: return None +# Byte order marks and the codec that decodes them (and drops the mark). +_BOMS = ((b"\xef\xbb\xbf", "utf-8-sig"), (b"\xff\xfe", "utf-16"), (b"\xfe\xff", "utf-16")) + +# and , and +_DECLARED_CHARSET = re.compile( + rb"""]*?charset\s*=\s*["']?\s*([\w.:+-]+)|^\s*<\?xml[^>]*?encoding\s*=\s*["']([\w.:+-]+)""", + re.IGNORECASE, +) + +# Labels that browsers decode with a superset codec; GB2312 pages routinely contain GBK characters. +_CODEC_ALIASES = {"gb2312": "gbk", "gb_2312-80": "gbk"} + + +def _sniff_encoding(content: bytes, content_type: str) -> str | None: + """Encoding from a BOM, or from a charset the document declares in its first 1024 bytes, like browsers do. + + The library used to do this before .text was decoded in Python, so pages that only name their charset in a + tag keep decoding the same way. + """ + for bom, codec in _BOMS: + if content.startswith(bom): + return codec + if content_type and not any(kind in content_type.lower() for kind in ("html", "xml")): + return None + match = _DECLARED_CHARSET.search(content[:1024]) + if match is None: + return None + return (match.group(1) or match.group(2)).decode("ascii") + + +def _codec(label: str | None) -> str | None: + return _CODEC_ALIASES.get(label.lower(), label) if label else None + + def build_response(res: dict[str, Any], res_cookies: RequestsCookieJar) -> Response: """Builds a Response object. Pops the body from res, so a large body can be freed as soon as it's decoded.""" response = Response() @@ -80,10 +115,8 @@ def build_response(res: dict[str, Any], res_cookies: RequestsCookieJar) -> Respo else: response_headers[header_key] = header_value response.headers = response_headers - # Decode .text with the charset the server declared, if any - content_type = response_headers.get("Content-Type") - if content_type: - response.encoding = _charset(content_type if isinstance(content_type, str) else content_type[0]) + content_type_header = response_headers.get("Content-Type") or "" + content_type = content_type_header if isinstance(content_type_header, str) else content_type_header[0] # Add cookies response.cookies = res_cookies # Add response content (bytes) @@ -99,4 +132,6 @@ def build_response(res: dict[str, Any], res_cookies: RequestsCookieJar) -> Respo else: # Not a data URI: the body is plain text. response._content = body.encode() + # Decode .text with the charset the server declared, else the one the document declares, if any + response.encoding = _codec(_charset(content_type) or _sniff_encoding(response._content, content_type)) return response From 64e2f7a4b7e4db2f0947f42914be729761439fa9 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 13:19:11 +0200 Subject: [PATCH 8/8] Drop the README changes Co-Authored-By: Claude Opus 5.5 --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index 1fd90d6..bf13fc9 100644 --- a/README.md +++ b/README.md @@ -92,13 +92,6 @@ res = session.post( ) ``` -Timeouts: -```python -res = session.get("https://www.example.com/", timeout=2.5) # seconds, int or float -``` -`timeout` works like in requests. `timeout_seconds` is the older name for it; passing both raises `ValueError`. Without either, `session.timeout_seconds` (30 by default) applies. -The native library sets a session's timeout when the session sends its first request and keeps it, so a different timeout on a later request of the same session has no effect. Use a new `Session` for that. - # Pyinstaller / Pyarmor **If you want to pack the library with Pyinstaller or Pyarmor, make sure to add this to your command:**