diff --git a/tests/integration/test_httpbin.py b/tests/integration/test_httpbin.py
index 975d91a..367cacc 100644
--- a/tests/integration/test_httpbin.py
+++ b/tests/integration/test_httpbin.py
@@ -3,6 +3,7 @@
from collections.abc import Iterator
import pytest
+from httpbin.core import resource as httpbin_resource
from tls_client import Session
from tls_client.exceptions import TLSClientException
@@ -78,13 +79,11 @@ def test_utf8_body_is_decoded(self, httpbin, session):
assert "∮ E⋅da = Q" in response.text
- @pytest.mark.xfail(reason="the library hands the body over as text, so bytes that are not UTF-8 get mangled")
def test_binary_body_arrives_unchanged(self, httpbin, session):
- payload = b"\x89PNG\r\n\x1a\n"
+ response = session.get(f"{httpbin.url}/image/png")
- response = session.get(f"{httpbin.url}/base64/{base64.urlsafe_b64encode(payload).decode()}")
-
- assert response.content == payload
+ # httpbin serves this file from its package data.
+ assert response.content == httpbin_resource("images/pig_icon.png")
class TestRedirects:
@@ -130,11 +129,12 @@ def test_session_timeout_raises(self, httpbin, session):
session.get(f"{httpbin.url}/delay/2")
@pytest.mark.xfail(reason="the native session keeps the timeout of its first request")
- def test_request_timeout_applies_after_the_first_request(self, httpbin, session):
+ @pytest.mark.parametrize("kwarg", ["timeout", "timeout_seconds"])
+ def test_request_timeout_applies_after_the_first_request(self, httpbin, session, kwarg):
session.get(f"{httpbin.url}/get")
with pytest.raises(TLSClientException, match="Timeout exceeded"):
- session.get(f"{httpbin.url}/delay/2", timeout_seconds=1)
+ session.get(f"{httpbin.url}/delay/2", **{kwarg: 1})
class TestCertificateVerification:
diff --git a/tests/integration/test_local_server.py b/tests/integration/test_local_server.py
new file mode 100644
index 0000000..2222f44
--- /dev/null
+++ b/tests/integration/test_local_server.py
@@ -0,0 +1,165 @@
+"""Real requests through the native library against a stdlib HTTP server, for bodies and headers httpbin can't
+produce byte for byte."""
+
+import base64
+import json
+import os
+import threading
+import time
+from collections.abc import Iterator
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import pytest
+
+from tls_client import Session
+from tls_client.exceptions import TLSClientException
+
+pytestmark = pytest.mark.integration
+
+# A 1x1 PNG, the kind of download that used to arrive mangled.
+PNG = base64.b64decode(
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+)
+LARGE = os.urandom(8 * 1024 * 1024)
+# How long /slow takes to answer.
+SLOW_SECONDS = 0.6
+
+# Path -> (Content-Type, body)
+BODIES = {
+ "/bytes": ("application/octet-stream", bytes(range(256))),
+ "/png": ("image/png", PNG),
+ "/large": ("application/octet-stream", LARGE),
+ "/empty": ("text/plain", b""),
+ "/json": ("application/json", json.dumps({"name": "tls", "ids": [1, 2]}).encode()),
+ "/utf8": ("text/plain", "∮ E⋅da = Q".encode()),
+ "/latin-1": ("text/plain; charset=iso-8859-1", "café".encode("latin-1")),
+}
+
+
+class Handler(BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.1"
+
+ def do_GET(self) -> None:
+ if self.path == "/headers":
+ self.echo_headers()
+ elif self.path == "/slow":
+ time.sleep(SLOW_SECONDS)
+ self.reply("text/plain", b"done")
+ else:
+ content_type, body = BODIES[self.path]
+ self.reply(content_type, body, {"x-MiXeD-case": "yes"})
+
+ def do_POST(self) -> None:
+ self.rfile.read(int(self.headers.get("Content-Length", 0)))
+ self.echo_headers()
+
+ def echo_headers(self) -> None:
+ received = {name.lower(): value for name, value in self.headers.items()}
+ self.reply("application/json", json.dumps(received).encode())
+
+ def reply(self, content_type: str, body: bytes, extra_headers: dict[str, str] | None = None) -> None:
+ self.send_response(200)
+ self.send_header("Content-Type", content_type)
+ self.send_header("Content-Length", str(len(body)))
+ for name, value in (extra_headers or {}).items():
+ self.send_header(name, value)
+ try:
+ self.end_headers()
+ self.wfile.write(body)
+ except ConnectionError:
+ # The client gave up waiting, as the timeout tests make it do.
+ pass
+
+ def log_message(self, format: str, *args: object) -> None:
+ pass
+
+
+@pytest.fixture(scope="module")
+def server_url() -> Iterator[str]:
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ server.daemon_threads = True
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ yield f"http://127.0.0.1:{server.server_port}"
+ server.shutdown()
+ server.server_close()
+
+
+@pytest.fixture
+def session() -> Iterator[Session]:
+ with Session() as s:
+ yield s
+
+
+class TestBody:
+ @pytest.mark.parametrize("path", ["/bytes", "/png", "/large"])
+ def test_binary_body_arrives_byte_for_byte(self, server_url, session, path):
+ response = session.get(f"{server_url}{path}")
+
+ assert response.status_code == 200
+ assert response.content == BODIES[path][1]
+
+ def test_empty_body(self, server_url, session):
+ response = session.get(f"{server_url}/empty")
+
+ assert response.content == b""
+ assert response.text == ""
+
+ def test_json_body(self, server_url, session):
+ assert session.get(f"{server_url}/json").json() == {"name": "tls", "ids": [1, 2]}
+
+ def test_text_defaults_to_utf8(self, server_url, session):
+ assert session.get(f"{server_url}/utf8").text == "∮ E⋅da = Q"
+
+ def test_text_uses_the_charset_from_the_content_type_header(self, server_url, session):
+ response = session.get(f"{server_url}/latin-1")
+
+ assert response.content == "café".encode("latin-1")
+ assert response.encoding == "iso-8859-1"
+ assert response.text == "café"
+
+
+class TestHeaders:
+ @pytest.mark.parametrize("name", ["X-Mixed-Case", "x-mixed-case", "X-MIXED-CASE"])
+ def test_response_header_lookup_ignores_case(self, server_url, session, name):
+ response = session.get(f"{server_url}/bytes")
+
+ assert response.headers[name] == "yes"
+ assert response.headers["content-type"] == "application/octet-stream"
+
+ def test_inferred_content_type_does_not_leak_into_later_requests(self, server_url, session):
+ json_post = session.post(f"{server_url}/headers", json={"a": 1}).json()
+ form_post = session.post(f"{server_url}/headers", data={"a": "1"}).json()
+ get = session.get(f"{server_url}/headers").json()
+
+ assert json_post["content-type"] == "application/json"
+ assert form_post["content-type"] == "application/x-www-form-urlencoded"
+ assert "content-type" not in get
+ assert "Content-Type" not in session.headers
+
+ def test_content_type_from_the_session_or_the_request_is_kept(self, server_url, session):
+ session.headers["content-type"] = "application/vnd.api+json"
+
+ from_session = session.post(f"{server_url}/headers", json={"a": 1}).json()
+ from_request = session.post(
+ f"{server_url}/headers", json={"a": 1}, headers={"CONTENT-TYPE": "text/plain"}
+ ).json()
+
+ assert from_session["content-type"] == "application/vnd.api+json"
+ assert from_request["content-type"] == "text/plain"
+
+
+class TestTimeout:
+ @pytest.mark.parametrize("kwarg", ["timeout", "timeout_seconds"])
+ def test_fractional_timeout_is_applied(self, server_url, session, kwarg):
+ with pytest.raises(TLSClientException, match="Timeout exceeded"):
+ session.get(f"{server_url}/slow", **{kwarg: SLOW_SECONDS / 3})
+
+ def test_response_within_the_timeout_arrives(self, server_url, session):
+ assert session.get(f"{server_url}/slow", timeout=SLOW_SECONDS * 4).text == "done"
+
+ def test_fractional_session_timeout_is_applied(self, server_url, session):
+ session.timeout_seconds = SLOW_SECONDS / 3
+
+ with pytest.raises(TLSClientException, match="Timeout exceeded"):
+ session.get(f"{server_url}/slow")
diff --git a/tests/unit/test_response.py b/tests/unit/test_response.py
index 3212844..d25d452 100644
--- a/tests/unit/test_response.py
+++ b/tests/unit/test_response.py
@@ -1,3 +1,6 @@
+import base64
+import os
+import tracemalloc
from typing import Any
import pytest
@@ -6,16 +9,28 @@
from tls_client.response import build_response
+def data_uri(body: bytes, mime_type: str = "text/plain; charset=utf-8") -> str:
+ """A body the way the Go library returns it for isByteResponse requests; it sniffs the mime type itself."""
+ return f"data:{mime_type};base64,{base64.b64encode(body).decode()}"
+
+
def library_reply(**fields: Any) -> dict[str, Any]:
"""A reply shaped like the one the Go library returns."""
- return {"id": "response-1", "status": 200, "target": "https://example.com/", "body": "", "headers": {}, **fields}
+ return {
+ "id": "response-1",
+ "status": 200,
+ "target": "https://example.com/",
+ "body": data_uri(b""),
+ "headers": {},
+ **fields,
+ }
def test_reply_fields_are_mapped_onto_the_response():
cookies = cookiejar_from_dict({"sid": "abc"})
response = build_response(
- library_reply(status=404, target="https://example.com/final", body='{"error": "missing"}'),
+ library_reply(status=404, target="https://example.com/final", body=data_uri(b'{"error": "missing"}')),
cookies,
)
@@ -40,8 +55,138 @@ def test_null_headers_become_an_empty_mapping():
assert dict(response.headers) == {}
-@pytest.mark.xfail(reason="build_response replaces the CaseInsensitiveDict with a plain dict")
-def test_header_lookup_ignores_case():
+@pytest.mark.parametrize("name", ["Content-Type", "content-type", "CONTENT-TYPE"])
+def test_header_lookup_ignores_case(name):
response = build_response(library_reply(headers={"Content-Type": ["text/html"]}), cookiejar_from_dict({}))
- assert response.headers["content-type"] == "text/html"
+ assert response.headers[name] == "text/html"
+ assert response.headers.get(name) == "text/html"
+ assert name in response.headers
+
+
+class TestBody:
+ def test_binary_body_is_decoded_byte_for_byte(self):
+ body = bytes(range(256))
+
+ response = build_response(
+ library_reply(body=data_uri(body, "application/octet-stream")), cookiejar_from_dict({})
+ )
+
+ assert response.content == body
+
+ def test_empty_body(self):
+ response = build_response(library_reply(), cookiejar_from_dict({}))
+
+ assert response.content == b""
+ assert response.text == ""
+
+ def test_text_uses_the_charset_from_the_content_type_header(self):
+ # The library labels the data URI with a sniffed type; only the header names the real charset.
+ response = build_response(
+ library_reply(
+ body=data_uri("café".encode("latin-1")),
+ headers={"content-type": ['text/plain; Charset="ISO-8859-1"']},
+ ),
+ cookiejar_from_dict({}),
+ )
+
+ assert response.encoding == "ISO-8859-1"
+ assert response.text == "café"
+
+ def test_text_defaults_to_utf8_and_replaces_undecodable_bytes(self):
+ response = build_response(
+ library_reply(body=data_uri("∮ ok ".encode() + b"\xff"), headers={"Content-Type": ["text/plain"]}),
+ cookiejar_from_dict({}),
+ )
+
+ assert response.encoding is None
+ assert response.text == "∮ ok �"
+
+ def test_unknown_charset_falls_back_to_utf8(self):
+ response = build_response(
+ library_reply(body=data_uri("é".encode()), headers={"Content-Type": ["text/plain; charset=made-up"]}),
+ cookiejar_from_dict({}),
+ )
+
+ assert response.text == "é"
+
+ def test_text_follows_an_encoding_set_by_the_caller(self):
+ response = build_response(library_reply(body=data_uri("café".encode("cp1252"))), cookiejar_from_dict({}))
+
+ response.encoding = "cp1252"
+
+ assert response.text == "café"
+
+ @pytest.mark.parametrize(
+ "html",
+ [
+ '
{}',
+ "{}",
+ "{}",
+ ],
+ )
+ def test_text_uses_the_charset_declared_in_a_meta_tag(self, html):
+ response = build_response(
+ library_reply(body=data_uri(html.format("中文").encode("gbk")), headers={"Content-Type": ["text/html"]}),
+ cookiejar_from_dict({}),
+ )
+
+ assert response.encoding.lower() == "gbk"
+ assert "中文" in response.text
+
+ def test_text_uses_the_encoding_of_an_xml_declaration(self):
+ body = '日本'.encode("shift_jis")
+
+ response = build_response(
+ library_reply(body=data_uri(body), headers={"Content-Type": ["application/xml"]}), cookiejar_from_dict({})
+ )
+
+ assert response.text.endswith("日本")
+
+ def test_header_charset_wins_over_a_meta_tag(self):
+ body = 'café'.encode("latin-1")
+
+ response = build_response(
+ library_reply(body=data_uri(body), headers={"Content-Type": ["text/html; charset=ISO-8859-1"]}),
+ cookiejar_from_dict({}),
+ )
+
+ assert response.encoding == "ISO-8859-1"
+ assert response.text.endswith("café")
+
+ def test_meta_tags_are_ignored_outside_html_and_xml(self):
+ body = b'{"html": ""}'
+
+ response = build_response(
+ library_reply(body=data_uri(body), headers={"Content-Type": ["application/json"]}), cookiejar_from_dict({})
+ )
+
+ assert response.encoding is None
+
+ @pytest.mark.parametrize("encoding", ["utf-8-sig", "utf-16"])
+ def test_text_uses_and_drops_a_byte_order_mark(self, encoding):
+ response = build_response(library_reply(body=data_uri("héllo".encode(encoding))), cookiejar_from_dict({}))
+
+ assert response.text == "héllo"
+
+ def test_body_that_is_not_a_data_uri_is_taken_as_text(self):
+ response = build_response(library_reply(body='{"plain": true}'), cookiejar_from_dict({}))
+
+ assert response.json() == {"plain": True}
+
+ def test_large_body_is_not_held_in_several_copies_at_once(self):
+ body = os.urandom(8_000_000)
+ tracemalloc.start()
+ try:
+ reply = library_reply(body=data_uri(body))
+ tracemalloc.reset_peak()
+ response = build_response(reply, cookiejar_from_dict({}))
+ _, peak = tracemalloc.get_traced_memory()
+ finally:
+ tracemalloc.stop()
+
+ assert response.content == body
+ assert "body" not in reply
+ # The data URI and its payload are each 4/3 of the body. Freeing the data URI before decoding keeps the
+ # peak near 8/3 of the body; holding it, the payload and the decoded bytes together would reach 11/3.
+ assert peak < 3 * len(body)
diff --git a/tests/unit/test_sessions.py b/tests/unit/test_sessions.py
index 083c696..bb94082 100644
--- a/tests/unit/test_sessions.py
+++ b/tests/unit/test_sessions.py
@@ -73,6 +73,11 @@ def test_bytes_are_base64_encoded_and_flagged_for_the_library(self, session, fak
assert fake_lib.sent["isByteRequest"] is True
assert base64.b64decode(fake_lib.sent["requestBody"]) == body
+ def test_response_body_is_requested_as_bytes(self, session, fake_lib):
+ session.get(URL)
+
+ assert fake_lib.sent["isByteResponse"] is True
+
def test_data_takes_precedence_over_json(self, session, fake_lib):
session.post(URL, data={"from": "data"}, json={"from": "json"})
@@ -90,14 +95,20 @@ def test_content_type_passed_with_the_request_wins(self, session, fake_lib):
assert header_values(fake_lib.sent["headers"], "Content-Type") == ["text/plain"]
- @pytest.mark.xfail(reason="the inferred Content-Type is written into session.headers and sticks to later requests")
def test_inferred_content_type_applies_to_one_request_only(self, session, fake_lib):
session.post(URL, json={"a": 1})
session.post(URL, data={"a": "1"})
-
assert header_values(fake_lib.sent["headers"], "Content-Type") == ["application/x-www-form-urlencoded"]
+
+ session.post(URL, data="raw")
+ assert header_values(fake_lib.sent["headers"], "Content-Type") == []
assert "Content-Type" not in session.headers
+ def test_content_type_set_to_none_suppresses_the_inferred_one(self, session, fake_lib):
+ session.post(URL, json={"a": 1}, headers={"Content-Type": None})
+
+ assert header_values(fake_lib.sent["headers"], "Content-Type") == []
+
@pytest.mark.parametrize("method", ["post", "put", "patch"])
def test_body_verbs_take_data_as_second_positional_argument(self, session, fake_lib, method):
getattr(session, method)(URL, {"k": "v"})
@@ -209,14 +220,28 @@ def test_request_proxy_wins_over_session_proxy(self, session, fake_lib):
assert fake_lib.sent["proxyUrl"] == "http://request-proxy:8080"
- def test_request_timeout_wins_over_session_timeout(self, session, fake_lib):
+ @pytest.mark.parametrize("kwarg", ["timeout", "timeout_seconds"])
+ def test_request_timeout_wins_over_session_timeout(self, session, fake_lib, kwarg):
session.timeout_seconds = 12
session.get(URL)
- assert fake_lib.sent["timeoutSeconds"] == 12
+ assert fake_lib.sent["timeoutMilliseconds"] == 12_000
+
+ session.get(URL, **{kwarg: 3})
+ assert fake_lib.sent["timeoutMilliseconds"] == 3_000
+
+ def test_timeout_is_sent_in_milliseconds_only(self, session, fake_lib):
+ session.get(URL, timeout=2.5)
+
+ assert fake_lib.sent["timeoutMilliseconds"] == 2_500
+ # The library refuses a request that sets both.
+ assert "timeoutSeconds" not in fake_lib.sent
+
+ def test_timeout_and_timeout_seconds_together_are_rejected(self, session, fake_lib):
+ with pytest.raises(ValueError, match="not both"):
+ session.get(URL, timeout=1, timeout_seconds=2)
- session.get(URL, timeout_seconds=3)
- assert fake_lib.sent["timeoutSeconds"] == 3
+ assert fake_lib.payloads == []
@pytest.mark.parametrize(
("session_kwargs", "request_kwargs", "payload_key"),
diff --git a/tls_client/response.py b/tls_client/response.py
index f57904b..c721ca6 100644
--- a/tls_client/response.py
+++ b/tls_client/response.py
@@ -1,5 +1,6 @@
+import binascii
import json
-from collections.abc import MutableMapping
+import re
from typing import Any
from .cookies import RequestsCookieJar, cookiejar_from_dict
@@ -16,15 +17,17 @@ def __init__(self) -> None:
# Integer Code of responded HTTP Status, e.g. 404 or 200.
self.status_code: int = 0
- # String of responded HTTP Body.
- self.text: str = ""
-
- # Dictionary of Response Headers. Headers sent more than once map to a list of values.
- self.headers: MutableMapping[str, str | list[str]] = CaseInsensitiveDict()
+ # Case-insensitive dictionary of Response Headers. Headers sent more than once map to a list of values.
+ self.headers: CaseInsensitiveDict[str | list[str]] = CaseInsensitiveDict()
# A CookieJar of Cookies the server sent back.
self.cookies = cookiejar_from_dict({})
+ # Encoding used to decode .text: the charset of the Content-Type header, else a BOM or the charset the document
+ # declares in a tag. Set it to decode the body with another encoding; None means UTF-8.
+ self.encoding: str | None = None
+
+ # Bytes of responded HTTP Body.
self._content: bytes = b""
def __enter__(self) -> "Response":
@@ -42,16 +45,69 @@ def content(self) -> bytes:
"""Content of the response, in bytes."""
return self._content
+ @property
+ def text(self) -> str:
+ """Content of the response, decoded with .encoding (UTF-8 if unset). Undecodable bytes become U+FFFD."""
+ # Decoded on every access rather than stored, so a large body isn't kept twice.
+ try:
+ return str(self._content, self.encoding or "utf-8", errors="replace")
+ except LookupError:
+ # The server named a charset Python doesn't know.
+ return str(self._content, "utf-8", errors="replace")
+
+
+def _charset(content_type: str) -> str | None:
+ """Charset parameter of a Content-Type header value, e.g. "ISO-8859-1" for "text/html; charset=ISO-8859-1"."""
+ for param in content_type.split(";")[1:]:
+ key, _, value = param.partition("=")
+ if key.strip().lower() == "charset":
+ return value.strip().strip("\"'") or None
+ return None
+
+
+# Byte order marks and the codec that decodes them (and drops the mark).
+_BOMS = ((b"\xef\xbb\xbf", "utf-8-sig"), (b"\xff\xfe", "utf-16"), (b"\xfe\xff", "utf-16"))
+
+# and , and
+_DECLARED_CHARSET = re.compile(
+ rb"""]*?charset\s*=\s*["']?\s*([\w.:+-]+)|^\s*<\?xml[^>]*?encoding\s*=\s*["']([\w.:+-]+)""",
+ re.IGNORECASE,
+)
+
+# Labels that browsers decode with a superset codec; GB2312 pages routinely contain GBK characters.
+_CODEC_ALIASES = {"gb2312": "gbk", "gb_2312-80": "gbk"}
+
+
+def _sniff_encoding(content: bytes, content_type: str) -> str | None:
+ """Encoding from a BOM, or from a charset the document declares in its first 1024 bytes, like browsers do.
+
+ The library used to do this before .text was decoded in Python, so pages that only name their charset in a
+ tag keep decoding the same way.
+ """
+ for bom, codec in _BOMS:
+ if content.startswith(bom):
+ return codec
+ if content_type and not any(kind in content_type.lower() for kind in ("html", "xml")):
+ return None
+ match = _DECLARED_CHARSET.search(content[:1024])
+ if match is None:
+ return None
+ return (match.group(1) or match.group(2)).decode("ascii")
+
+
+def _codec(label: str | None) -> str | None:
+ return _CODEC_ALIASES.get(label.lower(), label) if label else None
+
def build_response(res: dict[str, Any], res_cookies: RequestsCookieJar) -> Response:
- """Builds a Response object"""
+ """Builds a Response object. Pops the body from res, so a large body can be freed as soon as it's decoded."""
response = Response()
# Add target / url
response.url = res["target"]
# Add status code
response.status_code = res["status"]
# Add headers
- response_headers: dict[str, str | list[str]] = {}
+ response_headers: CaseInsensitiveDict[str | list[str]] = CaseInsensitiveDict()
if res["headers"] is not None:
for header_key, header_value in res["headers"].items():
if len(header_value) == 1:
@@ -59,10 +115,23 @@ def build_response(res: dict[str, Any], res_cookies: RequestsCookieJar) -> Respo
else:
response_headers[header_key] = header_value
response.headers = response_headers
+ content_type_header = response_headers.get("Content-Type") or ""
+ content_type = content_type_header if isinstance(content_type_header, str) else content_type_header[0]
# Add cookies
response.cookies = res_cookies
- # Add response body
- response.text = res["body"]
# Add response content (bytes)
- response._content = res["body"].encode()
+ # With isByteResponse the library sends "data:;base64,". It sniffs that mime type from the
+ # bytes, which is why .encoding comes from the Content-Type header instead.
+ body: str = res.pop("body")
+ prefix, _, payload = body.partition(",")
+ if prefix.startswith("data:") and prefix.endswith(";base64"):
+ # Free the data URI before decoding, so it isn't held next to both the payload and the bytes.
+ del body
+ # a2b_base64 reads an ASCII str in place, without encoding a copy first.
+ response._content = binascii.a2b_base64(payload)
+ else:
+ # Not a data URI: the body is plain text.
+ response._content = body.encode()
+ # Decode .text with the charset the server declared, else the one the document declares, if any
+ response.encoding = _codec(_charset(content_type) or _sniff_encoding(response._content, content_type))
return response
diff --git a/tls_client/sessions.py b/tls_client/sessions.py
index c9197e0..8a59b27 100644
--- a/tls_client/sessions.py
+++ b/tls_client/sessions.py
@@ -87,8 +87,8 @@ def __init__(
# CookieJar containing all currently outstanding cookies set on this session
self.cookies = cookiejar_from_dict({})
- # Timeout
- self.timeout_seconds = 30
+ # Timeout in seconds (int or float) for requests that don't pass timeout
+ self.timeout_seconds: float = 30
# Certificate pinning
self.certificate_pinning = certificate_pinning
@@ -342,9 +342,18 @@ def execute_request(
json: _Json | None = None,
allow_redirects: bool = False,
insecure_skip_verify: bool = False,
- timeout_seconds: int | None = None,
+ timeout_seconds: float | None = None,
proxy: _Proxy | None = None,
+ timeout: float | None = None,
) -> Response:
+ """Sends a request and returns its Response.
+
+ timeout is how many seconds (int or float) to wait for the response, like in requests. timeout_seconds is
+ the older name for it; passing both raises ValueError. Without either, self.timeout_seconds applies.
+
+ The native library sets a session's timeout when the session sends its first request and keeps it, so a
+ different timeout on a later request of the same session has no effect. Use a new Session for that.
+ """
# --- URL ------------------------------------------------------------------------------------------------------
# Prepare URL - add params to url
if params is not None:
@@ -364,26 +373,20 @@ def execute_request(
else:
request_body = urllib.parse.urlencode(data, doseq=True)
content_type = "application/x-www-form-urlencoded"
- # set content type if it isn't set
- if content_type is not None and "content-type" not in self.headers:
- self.headers["Content-Type"] = content_type
-
# --- Headers --------------------------------------------------------------------------------------------------
- request_headers: Mapping[str, str | None]
- if self.headers is None:
- request_headers = CaseInsensitiveDict(headers)
- elif headers is None:
- request_headers = self.headers
- else:
- merged_headers: CaseInsensitiveDict[str | None] = CaseInsensitiveDict(self.headers)
- merged_headers.update(headers)
+ # Merge into a copy, so nothing set for this request sticks to self.headers.
+ request_headers: CaseInsensitiveDict[str | None] = CaseInsensitiveDict(self.headers)
+ if headers is not None:
+ request_headers.update(headers)
- # Remove items, where the key or value is set to None.
- none_keys = [k for (k, v) in merged_headers.items() if v is None or k is None]
- for key in none_keys:
- del merged_headers[key]
+ # set content type if neither the session nor the request set one (setting it to None sends none)
+ if content_type is not None and "content-type" not in request_headers:
+ request_headers["Content-Type"] = content_type
- request_headers = merged_headers
+ # Remove items, where the key or value is set to None.
+ none_keys = [k for (k, v) in request_headers.items() if v is None or k is None]
+ for key in none_keys:
+ del request_headers[key]
# Ability to overwrite self.header_order for flexibility
request_header_order = self.header_order
@@ -418,9 +421,11 @@ def execute_request(
proxy_url = ""
# --- Timeout --------------------------------------------------------------------------------------------------
- # maximum time to wait for a response
+ # maximum time to wait for a response. timeout is the requests-style name for timeout_seconds.
+ if timeout is not None and timeout_seconds is not None:
+ raise ValueError("Pass either timeout or timeout_seconds, not both")
- timeout_seconds = timeout_seconds or self.timeout_seconds
+ request_timeout = timeout or timeout_seconds or self.timeout_seconds
# --- Certificate pinning --------------------------------------------------------------------------------------
# pins a certificate so that it restricts which certificates are considered valid
@@ -439,6 +444,8 @@ def execute_request(
"headerOrder": request_header_order,
"insecureSkipVerify": insecure_skip_verify,
"isByteRequest": is_byte_request,
+ # Return the body as base64 so binary bodies survive; build_response decodes it.
+ "isByteResponse": True,
"additionalDecode": self.additional_decode,
"proxyUrl": proxy_url,
"requestUrl": url,
@@ -447,7 +454,8 @@ def execute_request(
if isinstance(request_body, (bytes, bytearray))
else request_body,
"requestCookies": request_cookies,
- "timeoutSeconds": timeout_seconds,
+ # Milliseconds, so fractions of a second work. The library rejects timeoutSeconds alongside it.
+ "timeoutMilliseconds": round(request_timeout * 1000),
}
if certificate_pinning:
request_payload["certificatePinningHosts"] = certificate_pinning
@@ -472,14 +480,9 @@ def execute_request(
request_payload["tlsClientIdentifier"] = self.client_identifier
request_payload["withRandomTLSExtensionOrder"] = self.random_tls_extension_order
- # this is a pointer to the response
- response = request(dumps(request_payload).encode("utf-8"))
- # dereference the pointer to a byte array
- response_bytes = ctypes.string_at(response)
- # convert our byte array to a string (tls client returns json)
- response_string = response_bytes.decode("utf-8")
- # convert response string to json
- response_object = loads(response_string)
+ # dereference the pointer to the response, decode it and parse it (tls client returns json). None of the steps
+ # is bound to a name, so each copy of a large reply is freed as soon as the next one exists.
+ response_object = loads(ctypes.string_at(request(dumps(request_payload).encode("utf-8"))).decode("utf-8"))
# free the memory
freeMemory(response_object["id"].encode("utf-8"))
# --- Response -------------------------------------------------------------------------------------------------