diff --git a/tests/integration/test_httpbin.py b/tests/integration/test_httpbin.py index 975d91a..367cacc 100644 --- a/tests/integration/test_httpbin.py +++ b/tests/integration/test_httpbin.py @@ -3,6 +3,7 @@ from collections.abc import Iterator import pytest +from httpbin.core import resource as httpbin_resource from tls_client import Session from tls_client.exceptions import TLSClientException @@ -78,13 +79,11 @@ def test_utf8_body_is_decoded(self, httpbin, session): assert "∮ E⋅da = Q" in response.text - @pytest.mark.xfail(reason="the library hands the body over as text, so bytes that are not UTF-8 get mangled") def test_binary_body_arrives_unchanged(self, httpbin, session): - payload = b"\x89PNG\r\n\x1a\n" + response = session.get(f"{httpbin.url}/image/png") - response = session.get(f"{httpbin.url}/base64/{base64.urlsafe_b64encode(payload).decode()}") - - assert response.content == payload + # httpbin serves this file from its package data. + assert response.content == httpbin_resource("images/pig_icon.png") class TestRedirects: @@ -130,11 +129,12 @@ def test_session_timeout_raises(self, httpbin, session): session.get(f"{httpbin.url}/delay/2") @pytest.mark.xfail(reason="the native session keeps the timeout of its first request") - def test_request_timeout_applies_after_the_first_request(self, httpbin, session): + @pytest.mark.parametrize("kwarg", ["timeout", "timeout_seconds"]) + def test_request_timeout_applies_after_the_first_request(self, httpbin, session, kwarg): session.get(f"{httpbin.url}/get") with pytest.raises(TLSClientException, match="Timeout exceeded"): - session.get(f"{httpbin.url}/delay/2", timeout_seconds=1) + session.get(f"{httpbin.url}/delay/2", **{kwarg: 1}) class TestCertificateVerification: diff --git a/tests/integration/test_local_server.py b/tests/integration/test_local_server.py new file mode 100644 index 0000000..2222f44 --- /dev/null +++ b/tests/integration/test_local_server.py @@ -0,0 +1,165 @@ +"""Real requests through the native library against a stdlib HTTP server, for bodies and headers httpbin can't +produce byte for byte.""" + +import base64 +import json +import os +import threading +import time +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest + +from tls_client import Session +from tls_client.exceptions import TLSClientException + +pytestmark = pytest.mark.integration + +# A 1x1 PNG, the kind of download that used to arrive mangled. +PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" +) +LARGE = os.urandom(8 * 1024 * 1024) +# How long /slow takes to answer. +SLOW_SECONDS = 0.6 + +# Path -> (Content-Type, body) +BODIES = { + "/bytes": ("application/octet-stream", bytes(range(256))), + "/png": ("image/png", PNG), + "/large": ("application/octet-stream", LARGE), + "/empty": ("text/plain", b""), + "/json": ("application/json", json.dumps({"name": "tls", "ids": [1, 2]}).encode()), + "/utf8": ("text/plain", "∮ E⋅da = Q".encode()), + "/latin-1": ("text/plain; charset=iso-8859-1", "café".encode("latin-1")), +} + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self) -> None: + if self.path == "/headers": + self.echo_headers() + elif self.path == "/slow": + time.sleep(SLOW_SECONDS) + self.reply("text/plain", b"done") + else: + content_type, body = BODIES[self.path] + self.reply(content_type, body, {"x-MiXeD-case": "yes"}) + + def do_POST(self) -> None: + self.rfile.read(int(self.headers.get("Content-Length", 0))) + self.echo_headers() + + def echo_headers(self) -> None: + received = {name.lower(): value for name, value in self.headers.items()} + self.reply("application/json", json.dumps(received).encode()) + + def reply(self, content_type: str, body: bytes, extra_headers: dict[str, str] | None = None) -> None: + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + for name, value in (extra_headers or {}).items(): + self.send_header(name, value) + try: + self.end_headers() + self.wfile.write(body) + except ConnectionError: + # The client gave up waiting, as the timeout tests make it do. + pass + + def log_message(self, format: str, *args: object) -> None: + pass + + +@pytest.fixture(scope="module") +def server_url() -> Iterator[str]: + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{server.server_port}" + server.shutdown() + server.server_close() + + +@pytest.fixture +def session() -> Iterator[Session]: + with Session() as s: + yield s + + +class TestBody: + @pytest.mark.parametrize("path", ["/bytes", "/png", "/large"]) + def test_binary_body_arrives_byte_for_byte(self, server_url, session, path): + response = session.get(f"{server_url}{path}") + + assert response.status_code == 200 + assert response.content == BODIES[path][1] + + def test_empty_body(self, server_url, session): + response = session.get(f"{server_url}/empty") + + assert response.content == b"" + assert response.text == "" + + def test_json_body(self, server_url, session): + assert session.get(f"{server_url}/json").json() == {"name": "tls", "ids": [1, 2]} + + def test_text_defaults_to_utf8(self, server_url, session): + assert session.get(f"{server_url}/utf8").text == "∮ E⋅da = Q" + + def test_text_uses_the_charset_from_the_content_type_header(self, server_url, session): + response = session.get(f"{server_url}/latin-1") + + assert response.content == "café".encode("latin-1") + assert response.encoding == "iso-8859-1" + assert response.text == "café" + + +class TestHeaders: + @pytest.mark.parametrize("name", ["X-Mixed-Case", "x-mixed-case", "X-MIXED-CASE"]) + def test_response_header_lookup_ignores_case(self, server_url, session, name): + response = session.get(f"{server_url}/bytes") + + assert response.headers[name] == "yes" + assert response.headers["content-type"] == "application/octet-stream" + + def test_inferred_content_type_does_not_leak_into_later_requests(self, server_url, session): + json_post = session.post(f"{server_url}/headers", json={"a": 1}).json() + form_post = session.post(f"{server_url}/headers", data={"a": "1"}).json() + get = session.get(f"{server_url}/headers").json() + + assert json_post["content-type"] == "application/json" + assert form_post["content-type"] == "application/x-www-form-urlencoded" + assert "content-type" not in get + assert "Content-Type" not in session.headers + + def test_content_type_from_the_session_or_the_request_is_kept(self, server_url, session): + session.headers["content-type"] = "application/vnd.api+json" + + from_session = session.post(f"{server_url}/headers", json={"a": 1}).json() + from_request = session.post( + f"{server_url}/headers", json={"a": 1}, headers={"CONTENT-TYPE": "text/plain"} + ).json() + + assert from_session["content-type"] == "application/vnd.api+json" + assert from_request["content-type"] == "text/plain" + + +class TestTimeout: + @pytest.mark.parametrize("kwarg", ["timeout", "timeout_seconds"]) + def test_fractional_timeout_is_applied(self, server_url, session, kwarg): + with pytest.raises(TLSClientException, match="Timeout exceeded"): + session.get(f"{server_url}/slow", **{kwarg: SLOW_SECONDS / 3}) + + def test_response_within_the_timeout_arrives(self, server_url, session): + assert session.get(f"{server_url}/slow", timeout=SLOW_SECONDS * 4).text == "done" + + def test_fractional_session_timeout_is_applied(self, server_url, session): + session.timeout_seconds = SLOW_SECONDS / 3 + + with pytest.raises(TLSClientException, match="Timeout exceeded"): + session.get(f"{server_url}/slow") diff --git a/tests/unit/test_response.py b/tests/unit/test_response.py index 3212844..d25d452 100644 --- a/tests/unit/test_response.py +++ b/tests/unit/test_response.py @@ -1,3 +1,6 @@ +import base64 +import os +import tracemalloc from typing import Any import pytest @@ -6,16 +9,28 @@ from tls_client.response import build_response +def data_uri(body: bytes, mime_type: str = "text/plain; charset=utf-8") -> str: + """A body the way the Go library returns it for isByteResponse requests; it sniffs the mime type itself.""" + return f"data:{mime_type};base64,{base64.b64encode(body).decode()}" + + def library_reply(**fields: Any) -> dict[str, Any]: """A reply shaped like the one the Go library returns.""" - return {"id": "response-1", "status": 200, "target": "https://example.com/", "body": "", "headers": {}, **fields} + return { + "id": "response-1", + "status": 200, + "target": "https://example.com/", + "body": data_uri(b""), + "headers": {}, + **fields, + } def test_reply_fields_are_mapped_onto_the_response(): cookies = cookiejar_from_dict({"sid": "abc"}) response = build_response( - library_reply(status=404, target="https://example.com/final", body='{"error": "missing"}'), + library_reply(status=404, target="https://example.com/final", body=data_uri(b'{"error": "missing"}')), cookies, ) @@ -40,8 +55,138 @@ def test_null_headers_become_an_empty_mapping(): assert dict(response.headers) == {} -@pytest.mark.xfail(reason="build_response replaces the CaseInsensitiveDict with a plain dict") -def test_header_lookup_ignores_case(): +@pytest.mark.parametrize("name", ["Content-Type", "content-type", "CONTENT-TYPE"]) +def test_header_lookup_ignores_case(name): response = build_response(library_reply(headers={"Content-Type": ["text/html"]}), cookiejar_from_dict({})) - assert response.headers["content-type"] == "text/html" + assert response.headers[name] == "text/html" + assert response.headers.get(name) == "text/html" + assert name in response.headers + + +class TestBody: + def test_binary_body_is_decoded_byte_for_byte(self): + body = bytes(range(256)) + + response = build_response( + library_reply(body=data_uri(body, "application/octet-stream")), cookiejar_from_dict({}) + ) + + assert response.content == body + + def test_empty_body(self): + response = build_response(library_reply(), cookiejar_from_dict({})) + + assert response.content == b"" + assert response.text == "" + + def test_text_uses_the_charset_from_the_content_type_header(self): + # The library labels the data URI with a sniffed type; only the header names the real charset. + response = build_response( + library_reply( + body=data_uri("café".encode("latin-1")), + headers={"content-type": ['text/plain; Charset="ISO-8859-1"']}, + ), + cookiejar_from_dict({}), + ) + + assert response.encoding == "ISO-8859-1" + assert response.text == "café" + + def test_text_defaults_to_utf8_and_replaces_undecodable_bytes(self): + response = build_response( + library_reply(body=data_uri("∮ ok ".encode() + b"\xff"), headers={"Content-Type": ["text/plain"]}), + cookiejar_from_dict({}), + ) + + assert response.encoding is None + assert response.text == "∮ ok �" + + def test_unknown_charset_falls_back_to_utf8(self): + response = build_response( + library_reply(body=data_uri("é".encode()), headers={"Content-Type": ["text/plain; charset=made-up"]}), + cookiejar_from_dict({}), + ) + + assert response.text == "é" + + def test_text_follows_an_encoding_set_by_the_caller(self): + response = build_response(library_reply(body=data_uri("café".encode("cp1252"))), cookiejar_from_dict({})) + + response.encoding = "cp1252" + + assert response.text == "café" + + @pytest.mark.parametrize( + "html", + [ + '{}', + "{}", + "{}", + ], + ) + def test_text_uses_the_charset_declared_in_a_meta_tag(self, html): + response = build_response( + library_reply(body=data_uri(html.format("中文").encode("gbk")), headers={"Content-Type": ["text/html"]}), + cookiejar_from_dict({}), + ) + + assert response.encoding.lower() == "gbk" + assert "中文" in response.text + + def test_text_uses_the_encoding_of_an_xml_declaration(self): + body = '日本'.encode("shift_jis") + + response = build_response( + library_reply(body=data_uri(body), headers={"Content-Type": ["application/xml"]}), cookiejar_from_dict({}) + ) + + assert response.text.endswith("日本") + + def test_header_charset_wins_over_a_meta_tag(self): + body = 'café'.encode("latin-1") + + response = build_response( + library_reply(body=data_uri(body), headers={"Content-Type": ["text/html; charset=ISO-8859-1"]}), + cookiejar_from_dict({}), + ) + + assert response.encoding == "ISO-8859-1" + assert response.text.endswith("café") + + def test_meta_tags_are_ignored_outside_html_and_xml(self): + body = b'{"html": ""}' + + response = build_response( + library_reply(body=data_uri(body), headers={"Content-Type": ["application/json"]}), cookiejar_from_dict({}) + ) + + assert response.encoding is None + + @pytest.mark.parametrize("encoding", ["utf-8-sig", "utf-16"]) + def test_text_uses_and_drops_a_byte_order_mark(self, encoding): + response = build_response(library_reply(body=data_uri("héllo".encode(encoding))), cookiejar_from_dict({})) + + assert response.text == "héllo" + + def test_body_that_is_not_a_data_uri_is_taken_as_text(self): + response = build_response(library_reply(body='{"plain": true}'), cookiejar_from_dict({})) + + assert response.json() == {"plain": True} + + def test_large_body_is_not_held_in_several_copies_at_once(self): + body = os.urandom(8_000_000) + tracemalloc.start() + try: + reply = library_reply(body=data_uri(body)) + tracemalloc.reset_peak() + response = build_response(reply, cookiejar_from_dict({})) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + + assert response.content == body + assert "body" not in reply + # The data URI and its payload are each 4/3 of the body. Freeing the data URI before decoding keeps the + # peak near 8/3 of the body; holding it, the payload and the decoded bytes together would reach 11/3. + assert peak < 3 * len(body) diff --git a/tests/unit/test_sessions.py b/tests/unit/test_sessions.py index 083c696..bb94082 100644 --- a/tests/unit/test_sessions.py +++ b/tests/unit/test_sessions.py @@ -73,6 +73,11 @@ def test_bytes_are_base64_encoded_and_flagged_for_the_library(self, session, fak assert fake_lib.sent["isByteRequest"] is True assert base64.b64decode(fake_lib.sent["requestBody"]) == body + def test_response_body_is_requested_as_bytes(self, session, fake_lib): + session.get(URL) + + assert fake_lib.sent["isByteResponse"] is True + def test_data_takes_precedence_over_json(self, session, fake_lib): session.post(URL, data={"from": "data"}, json={"from": "json"}) @@ -90,14 +95,20 @@ def test_content_type_passed_with_the_request_wins(self, session, fake_lib): assert header_values(fake_lib.sent["headers"], "Content-Type") == ["text/plain"] - @pytest.mark.xfail(reason="the inferred Content-Type is written into session.headers and sticks to later requests") def test_inferred_content_type_applies_to_one_request_only(self, session, fake_lib): session.post(URL, json={"a": 1}) session.post(URL, data={"a": "1"}) - assert header_values(fake_lib.sent["headers"], "Content-Type") == ["application/x-www-form-urlencoded"] + + session.post(URL, data="raw") + assert header_values(fake_lib.sent["headers"], "Content-Type") == [] assert "Content-Type" not in session.headers + def test_content_type_set_to_none_suppresses_the_inferred_one(self, session, fake_lib): + session.post(URL, json={"a": 1}, headers={"Content-Type": None}) + + assert header_values(fake_lib.sent["headers"], "Content-Type") == [] + @pytest.mark.parametrize("method", ["post", "put", "patch"]) def test_body_verbs_take_data_as_second_positional_argument(self, session, fake_lib, method): getattr(session, method)(URL, {"k": "v"}) @@ -209,14 +220,28 @@ def test_request_proxy_wins_over_session_proxy(self, session, fake_lib): assert fake_lib.sent["proxyUrl"] == "http://request-proxy:8080" - def test_request_timeout_wins_over_session_timeout(self, session, fake_lib): + @pytest.mark.parametrize("kwarg", ["timeout", "timeout_seconds"]) + def test_request_timeout_wins_over_session_timeout(self, session, fake_lib, kwarg): session.timeout_seconds = 12 session.get(URL) - assert fake_lib.sent["timeoutSeconds"] == 12 + assert fake_lib.sent["timeoutMilliseconds"] == 12_000 + + session.get(URL, **{kwarg: 3}) + assert fake_lib.sent["timeoutMilliseconds"] == 3_000 + + def test_timeout_is_sent_in_milliseconds_only(self, session, fake_lib): + session.get(URL, timeout=2.5) + + assert fake_lib.sent["timeoutMilliseconds"] == 2_500 + # The library refuses a request that sets both. + assert "timeoutSeconds" not in fake_lib.sent + + def test_timeout_and_timeout_seconds_together_are_rejected(self, session, fake_lib): + with pytest.raises(ValueError, match="not both"): + session.get(URL, timeout=1, timeout_seconds=2) - session.get(URL, timeout_seconds=3) - assert fake_lib.sent["timeoutSeconds"] == 3 + assert fake_lib.payloads == [] @pytest.mark.parametrize( ("session_kwargs", "request_kwargs", "payload_key"), diff --git a/tls_client/response.py b/tls_client/response.py index f57904b..c721ca6 100644 --- a/tls_client/response.py +++ b/tls_client/response.py @@ -1,5 +1,6 @@ +import binascii import json -from collections.abc import MutableMapping +import re from typing import Any from .cookies import RequestsCookieJar, cookiejar_from_dict @@ -16,15 +17,17 @@ def __init__(self) -> None: # Integer Code of responded HTTP Status, e.g. 404 or 200. self.status_code: int = 0 - # String of responded HTTP Body. - self.text: str = "" - - # Dictionary of Response Headers. Headers sent more than once map to a list of values. - self.headers: MutableMapping[str, str | list[str]] = CaseInsensitiveDict() + # Case-insensitive dictionary of Response Headers. Headers sent more than once map to a list of values. + self.headers: CaseInsensitiveDict[str | list[str]] = CaseInsensitiveDict() # A CookieJar of Cookies the server sent back. self.cookies = cookiejar_from_dict({}) + # Encoding used to decode .text: the charset of the Content-Type header, else a BOM or the charset the document + # declares in a tag. Set it to decode the body with another encoding; None means UTF-8. + self.encoding: str | None = None + + # Bytes of responded HTTP Body. self._content: bytes = b"" def __enter__(self) -> "Response": @@ -42,16 +45,69 @@ def content(self) -> bytes: """Content of the response, in bytes.""" return self._content + @property + def text(self) -> str: + """Content of the response, decoded with .encoding (UTF-8 if unset). Undecodable bytes become U+FFFD.""" + # Decoded on every access rather than stored, so a large body isn't kept twice. + try: + return str(self._content, self.encoding or "utf-8", errors="replace") + except LookupError: + # The server named a charset Python doesn't know. + return str(self._content, "utf-8", errors="replace") + + +def _charset(content_type: str) -> str | None: + """Charset parameter of a Content-Type header value, e.g. "ISO-8859-1" for "text/html; charset=ISO-8859-1".""" + for param in content_type.split(";")[1:]: + key, _, value = param.partition("=") + if key.strip().lower() == "charset": + return value.strip().strip("\"'") or None + return None + + +# Byte order marks and the codec that decodes them (and drops the mark). +_BOMS = ((b"\xef\xbb\xbf", "utf-8-sig"), (b"\xff\xfe", "utf-16"), (b"\xfe\xff", "utf-16")) + +# and , and +_DECLARED_CHARSET = re.compile( + rb"""]*?charset\s*=\s*["']?\s*([\w.:+-]+)|^\s*<\?xml[^>]*?encoding\s*=\s*["']([\w.:+-]+)""", + re.IGNORECASE, +) + +# Labels that browsers decode with a superset codec; GB2312 pages routinely contain GBK characters. +_CODEC_ALIASES = {"gb2312": "gbk", "gb_2312-80": "gbk"} + + +def _sniff_encoding(content: bytes, content_type: str) -> str | None: + """Encoding from a BOM, or from a charset the document declares in its first 1024 bytes, like browsers do. + + The library used to do this before .text was decoded in Python, so pages that only name their charset in a + tag keep decoding the same way. + """ + for bom, codec in _BOMS: + if content.startswith(bom): + return codec + if content_type and not any(kind in content_type.lower() for kind in ("html", "xml")): + return None + match = _DECLARED_CHARSET.search(content[:1024]) + if match is None: + return None + return (match.group(1) or match.group(2)).decode("ascii") + + +def _codec(label: str | None) -> str | None: + return _CODEC_ALIASES.get(label.lower(), label) if label else None + def build_response(res: dict[str, Any], res_cookies: RequestsCookieJar) -> Response: - """Builds a Response object""" + """Builds a Response object. Pops the body from res, so a large body can be freed as soon as it's decoded.""" response = Response() # Add target / url response.url = res["target"] # Add status code response.status_code = res["status"] # Add headers - response_headers: dict[str, str | list[str]] = {} + response_headers: CaseInsensitiveDict[str | list[str]] = CaseInsensitiveDict() if res["headers"] is not None: for header_key, header_value in res["headers"].items(): if len(header_value) == 1: @@ -59,10 +115,23 @@ def build_response(res: dict[str, Any], res_cookies: RequestsCookieJar) -> Respo else: response_headers[header_key] = header_value response.headers = response_headers + content_type_header = response_headers.get("Content-Type") or "" + content_type = content_type_header if isinstance(content_type_header, str) else content_type_header[0] # Add cookies response.cookies = res_cookies - # Add response body - response.text = res["body"] # Add response content (bytes) - response._content = res["body"].encode() + # With isByteResponse the library sends "data:;base64,". It sniffs that mime type from the + # bytes, which is why .encoding comes from the Content-Type header instead. + body: str = res.pop("body") + prefix, _, payload = body.partition(",") + if prefix.startswith("data:") and prefix.endswith(";base64"): + # Free the data URI before decoding, so it isn't held next to both the payload and the bytes. + del body + # a2b_base64 reads an ASCII str in place, without encoding a copy first. + response._content = binascii.a2b_base64(payload) + else: + # Not a data URI: the body is plain text. + response._content = body.encode() + # Decode .text with the charset the server declared, else the one the document declares, if any + response.encoding = _codec(_charset(content_type) or _sniff_encoding(response._content, content_type)) return response diff --git a/tls_client/sessions.py b/tls_client/sessions.py index c9197e0..8a59b27 100644 --- a/tls_client/sessions.py +++ b/tls_client/sessions.py @@ -87,8 +87,8 @@ def __init__( # CookieJar containing all currently outstanding cookies set on this session self.cookies = cookiejar_from_dict({}) - # Timeout - self.timeout_seconds = 30 + # Timeout in seconds (int or float) for requests that don't pass timeout + self.timeout_seconds: float = 30 # Certificate pinning self.certificate_pinning = certificate_pinning @@ -342,9 +342,18 @@ def execute_request( json: _Json | None = None, allow_redirects: bool = False, insecure_skip_verify: bool = False, - timeout_seconds: int | None = None, + timeout_seconds: float | None = None, proxy: _Proxy | None = None, + timeout: float | None = None, ) -> Response: + """Sends a request and returns its Response. + + timeout is how many seconds (int or float) to wait for the response, like in requests. timeout_seconds is + the older name for it; passing both raises ValueError. Without either, self.timeout_seconds applies. + + The native library sets a session's timeout when the session sends its first request and keeps it, so a + different timeout on a later request of the same session has no effect. Use a new Session for that. + """ # --- URL ------------------------------------------------------------------------------------------------------ # Prepare URL - add params to url if params is not None: @@ -364,26 +373,20 @@ def execute_request( else: request_body = urllib.parse.urlencode(data, doseq=True) content_type = "application/x-www-form-urlencoded" - # set content type if it isn't set - if content_type is not None and "content-type" not in self.headers: - self.headers["Content-Type"] = content_type - # --- Headers -------------------------------------------------------------------------------------------------- - request_headers: Mapping[str, str | None] - if self.headers is None: - request_headers = CaseInsensitiveDict(headers) - elif headers is None: - request_headers = self.headers - else: - merged_headers: CaseInsensitiveDict[str | None] = CaseInsensitiveDict(self.headers) - merged_headers.update(headers) + # Merge into a copy, so nothing set for this request sticks to self.headers. + request_headers: CaseInsensitiveDict[str | None] = CaseInsensitiveDict(self.headers) + if headers is not None: + request_headers.update(headers) - # Remove items, where the key or value is set to None. - none_keys = [k for (k, v) in merged_headers.items() if v is None or k is None] - for key in none_keys: - del merged_headers[key] + # set content type if neither the session nor the request set one (setting it to None sends none) + if content_type is not None and "content-type" not in request_headers: + request_headers["Content-Type"] = content_type - request_headers = merged_headers + # Remove items, where the key or value is set to None. + none_keys = [k for (k, v) in request_headers.items() if v is None or k is None] + for key in none_keys: + del request_headers[key] # Ability to overwrite self.header_order for flexibility request_header_order = self.header_order @@ -418,9 +421,11 @@ def execute_request( proxy_url = "" # --- Timeout -------------------------------------------------------------------------------------------------- - # maximum time to wait for a response + # maximum time to wait for a response. timeout is the requests-style name for timeout_seconds. + if timeout is not None and timeout_seconds is not None: + raise ValueError("Pass either timeout or timeout_seconds, not both") - timeout_seconds = timeout_seconds or self.timeout_seconds + request_timeout = timeout or timeout_seconds or self.timeout_seconds # --- Certificate pinning -------------------------------------------------------------------------------------- # pins a certificate so that it restricts which certificates are considered valid @@ -439,6 +444,8 @@ def execute_request( "headerOrder": request_header_order, "insecureSkipVerify": insecure_skip_verify, "isByteRequest": is_byte_request, + # Return the body as base64 so binary bodies survive; build_response decodes it. + "isByteResponse": True, "additionalDecode": self.additional_decode, "proxyUrl": proxy_url, "requestUrl": url, @@ -447,7 +454,8 @@ def execute_request( if isinstance(request_body, (bytes, bytearray)) else request_body, "requestCookies": request_cookies, - "timeoutSeconds": timeout_seconds, + # Milliseconds, so fractions of a second work. The library rejects timeoutSeconds alongside it. + "timeoutMilliseconds": round(request_timeout * 1000), } if certificate_pinning: request_payload["certificatePinningHosts"] = certificate_pinning @@ -472,14 +480,9 @@ def execute_request( request_payload["tlsClientIdentifier"] = self.client_identifier request_payload["withRandomTLSExtensionOrder"] = self.random_tls_extension_order - # this is a pointer to the response - response = request(dumps(request_payload).encode("utf-8")) - # dereference the pointer to a byte array - response_bytes = ctypes.string_at(response) - # convert our byte array to a string (tls client returns json) - response_string = response_bytes.decode("utf-8") - # convert response string to json - response_object = loads(response_string) + # dereference the pointer to the response, decode it and parse it (tls client returns json). None of the steps + # is bound to a name, so each copy of a large reply is freed as soon as the next one exists. + response_object = loads(ctypes.string_at(request(dumps(request_payload).encode("utf-8"))).decode("utf-8")) # free the memory freeMemory(response_object["id"].encode("utf-8")) # --- Response -------------------------------------------------------------------------------------------------