diff --git a/tests/integration/test_httpbin.py b/tests/integration/test_httpbin.py index 4487d7e..422f7e2 100644 --- a/tests/integration/test_httpbin.py +++ b/tests/integration/test_httpbin.py @@ -244,7 +244,6 @@ def test_session_timeout_raises(self, httpbin, session): with pytest.raises(TLSClientException, match="Timeout exceeded"): session.get(f"{httpbin.url}/delay/2") - @pytest.mark.xfail(reason="the native session keeps the timeout of its first request") @pytest.mark.parametrize("kwarg", ["timeout", "timeout_seconds"]) def test_request_timeout_applies_after_the_first_request(self, httpbin, session, kwarg): session.get(f"{httpbin.url}/get") @@ -262,3 +261,11 @@ def test_insecure_skip_verify_accepts_self_signed_certificate(self, httpbin_secu response = session.get(f"{httpbin_secure.url}/get", insecure_skip_verify=True) assert response.status_code == 200 + + def test_insecure_skip_verify_applies_after_the_first_request(self, httpbin_secure, session): + with pytest.raises(TLSClientException, match="certificate"): + session.get(f"{httpbin_secure.url}/get") + + assert session.get(f"{httpbin_secure.url}/get", insecure_skip_verify=True).status_code == 200 + with pytest.raises(TLSClientException, match="certificate"): + session.get(f"{httpbin_secure.url}/get") diff --git a/tests/unit/test_cookies.py b/tests/unit/test_cookies.py index ddd6f38..f9cee5e 100644 --- a/tests/unit/test_cookies.py +++ b/tests/unit/test_cookies.py @@ -53,7 +53,6 @@ def test_same_name_on_two_paths_must_be_disambiguated(self): jar["id"] assert jar.get("id", path="/api") == "api" - @pytest.mark.xfail(reason="_find_no_duplicates treats an empty value as a missing cookie") def test_cookie_with_empty_value_is_readable(self): jar = cookiejar_from_dict({"consent": ""}) @@ -79,7 +78,6 @@ def test_list_domains_and_paths_are_deduplicated(self): assert sorted(jar.list_domains()) == ["a.example", "b.example"] assert sorted(jar.list_paths()) == ["/", "/api"] - @pytest.mark.xfail(reason="multiple_domains() looks for a repeated domain instead of a second distinct one") def test_multiple_domains(self): jar = RequestsCookieJar() jar.set("a", "1", domain="a.example") diff --git a/tests/unit/test_sessions.py b/tests/unit/test_sessions.py index db90e52..8a93084 100644 --- a/tests/unit/test_sessions.py +++ b/tests/unit/test_sessions.py @@ -27,13 +27,21 @@ def test_params_are_encoded_into_the_query_string(self, session, fake_lib): assert fake_lib.sent["requestUrl"] == f"{URL}?q=a+b&tag=x&tag=y" - @pytest.mark.xfail(reason="params are appended after a second '?' instead of extending the existing query") def test_params_extend_an_existing_query_string(self, session, fake_lib): session.get(f"{URL}?page=2", params={"q": "x"}) assert fake_lib.sent["requestUrl"] == f"{URL}?page=2&q=x" - @pytest.mark.xfail(reason="session.params is never read when building a request") + def test_params_go_before_the_fragment(self, session, fake_lib): + session.get(f"{URL}#top", params={"q": "x"}) + + assert fake_lib.sent["requestUrl"] == f"{URL}?q=x#top" + + def test_empty_params_leave_the_url_alone(self, session, fake_lib): + session.get(URL, params={}) + + assert fake_lib.sent["requestUrl"] == URL + def test_session_params_are_sent_with_every_request(self, session, fake_lib): session.params = {"api_key": "k"} @@ -41,6 +49,13 @@ def test_session_params_are_sent_with_every_request(self, session, fake_lib): assert fake_lib.sent["requestUrl"] == f"{URL}?api_key=k" + def test_request_params_replace_session_params_of_the_same_name(self, session, fake_lib): + session.params = {"api_key": "k", "lang": "en"} + + session.get(URL, params={"lang": "de", "q": "x"}) + + assert fake_lib.sent["requestUrl"] == f"{URL}?api_key=k&lang=de&q=x" + class TestBody: def test_json_is_serialized_and_labelled(self, session, fake_lib): @@ -449,6 +464,7 @@ class TestTransportOptions: [ ("http://user:pass@proxy:8080", "http://user:pass@proxy:8080"), ({"http": "http://proxy:8080", "https": "http://ignored:8080"}, "http://proxy:8080"), + ({"https": "http://proxy:8080"}, "http://proxy:8080"), ({}, ""), ], ) @@ -595,6 +611,23 @@ def test_requests_share_the_session_id_of_their_session_only(self, fake_lib): assert ids[0] == ids[1] assert ids[1] != ids[2] + @pytest.mark.parametrize( + "kwargs", [{"timeout": 5}, {"timeout_seconds": 5}, {"insecure_skip_verify": True}], ids=lambda k: next(iter(k)) + ) + def test_request_with_other_transport_settings_gets_a_native_session_of_its_own(self, fake_lib, kwargs): + # The library keeps the timeout and certificate verification of a native session's first request + with Session() as session: + session.get(URL) + session.get(URL, **kwargs) + session.get(URL, **kwargs) + session.get(URL) + + ids = [payload["sessionId"] for payload in fake_lib.payloads] + assert ids[0] != ids[1] + assert ids[1] == ids[2] + assert ids[3] == ids[0] + assert sorted(fake_lib.destroyed_session_ids) == sorted({ids[0], ids[1]}) + def test_library_error_is_raised_as_tls_client_exception(self, session, fake_lib): fake_lib.reply_with(status=0, body="failed to do request: dial tcp: connection refused") diff --git a/tls_client/cookies.py b/tls_client/cookies.py index d78d696..2df2968 100644 --- a/tls_client/cookies.py +++ b/tls_client/cookies.py @@ -219,12 +219,7 @@ def multiple_domains(self) -> bool: :rtype: bool """ - domains: list[str] = [] - for cookie in iter(self): - if cookie.domain is not None and cookie.domain in domains: - return True - domains.append(cookie.domain) - return False # there is only one domain in jar + return len({cookie.domain for cookie in iter(self)}) > 1 def get_dict(self, domain: str | None = None, path: str | None = None) -> dict[str, str | None]: """Takes as an argument an optional domain and path and returns a plain @@ -314,6 +309,7 @@ def _find_no_duplicates(self, name: str, domain: str | None = None, path: str | that match name and optionally domain and path :return: cookie.value """ + found = False toReturn = None for cookie in iter(self): if ( @@ -321,13 +317,15 @@ def _find_no_duplicates(self, name: str, domain: str | None = None, path: str | and (domain is None or cookie.domain == domain) and (path is None or cookie.path == path) ): - if toReturn is not None: + if found: # if there are multiple cookies that meet passed in criteria raise CookieConflictError(f"There are multiple cookies with name, {name!r}") # we will eventually return this as long as no cookie conflict + found = True toReturn = cookie.value - if toReturn: + # An empty value is a value, only a cookie without one (Set-Cookie: name) counts as missing + if toReturn is not None: return toReturn raise KeyError(f"name={name!r}, domain={domain!r}, path={path!r}") diff --git a/tls_client/sessions.py b/tls_client/sessions.py index 089925d..4ccf38f 100644 --- a/tls_client/sessions.py +++ b/tls_client/sessions.py @@ -21,7 +21,7 @@ _Data: TypeAlias = str | bytes | Mapping[str, Any] | list[tuple[str, Any]] # JSON body: dicts and lists are serialized, a str is sent as-is. _Json: TypeAlias = dict[str, Any] | list[Any] | str -# Either a proxy URL, or a dict whose "http" entry is used. +# Either a proxy URL, or a dict whose "http" entry is used, or its "https" entry if it has no "http" one. _Proxy: TypeAlias = dict[str, str] | str # Statuses followed with allow_redirects, if the response has a Location header @@ -91,6 +91,10 @@ def __init__( ) self._session_id = str(uuid.uuid4()) + # The library sets a native session's timeout and certificate verification when the session sends its first + # request and keeps them. So each combination of the two gets a native session of its own, whose ID is made + # from self._session_id and the two values. These are the IDs sent so far, which close() destroys. + self._native_session_ids: set[str] = set() # --- Standard Settings ---------------------------------------------------------------------------------------- # Case-insensitive dictionary of headers, send on each request @@ -346,17 +350,20 @@ def __exit__(self, *args: object) -> None: self.close() def close(self) -> str: - destroy_session_payload = {"sessionId": self._session_id} + """Destroys the native sessions of this session and returns the library's reply for the last one.""" + destroy_session_response_string = "" + for session_id in sorted(self._native_session_ids) or [self._session_id]: + destroy_session_payload = {"sessionId": session_id} - destroy_session_response = destroySession(dumps(destroy_session_payload).encode("utf-8")) - # we dereference the pointer to a byte array - destroy_session_response_bytes = ctypes.string_at(destroy_session_response) - # convert our byte array to a string (tls client returns json) - destroy_session_response_string = destroy_session_response_bytes.decode("utf-8") - # convert response string to json - destroy_session_response_object = loads(destroy_session_response_string) + destroy_session_response = destroySession(dumps(destroy_session_payload).encode("utf-8")) + # we dereference the pointer to a byte array + destroy_session_response_bytes = ctypes.string_at(destroy_session_response) + # convert our byte array to a string (tls client returns json) + destroy_session_response_string = destroy_session_response_bytes.decode("utf-8") + # convert response string to json + destroy_session_response_object = loads(destroy_session_response_string) - freeMemory(destroy_session_response_object["id"].encode("utf-8")) + freeMemory(destroy_session_response_object["id"].encode("utf-8")) return destroy_session_response_string @@ -379,16 +386,17 @@ def execute_request( ) -> Response: """Sends a request and returns its Response. + params are added to the query string after self.params, and after any query the URL already has. + timeout is how many seconds (int or float) to wait for the response, like in requests. timeout_seconds is the older name for it; passing both raises ValueError. Without either, self.timeout_seconds applies. - - The native library sets a session's timeout when the session sends its first request and keeps it, so a - different timeout on a later request of the same session has no effect. Use a new Session for that. """ # --- URL ------------------------------------------------------------------------------------------------------ - # Prepare URL - add params to url - if params is not None: - url = f"{url}?{urllib.parse.urlencode(params, doseq=True)}" + # Prepare URL - add params to url. A parameter passed in replaces the session parameter of the same name. + query = urllib.parse.urlencode({**self.params, **(params or {})}, doseq=True) + if query: + parts = urllib.parse.urlsplit(url) + url = urllib.parse.urlunsplit(parts._replace(query=f"{parts.query}&{query}" if parts.query else query)) # --- Request Body --------------------------------------------------------------------------------------------- # Prepare request body - build request body @@ -442,8 +450,9 @@ def execute_request( # --- Proxy ---------------------------------------------------------------------------------------------------- proxy = proxy or self.proxies - if isinstance(proxy, dict) and "http" in proxy: - proxy_url = proxy["http"] + if isinstance(proxy, dict): + # The "http" entry is used for every URL, https ones included + proxy_url = proxy.get("http") or proxy.get("https") or "" elif isinstance(proxy, str): proxy_url = proxy else: @@ -455,6 +464,11 @@ def execute_request( raise ValueError("Pass either timeout or timeout_seconds, not both") request_timeout = timeout or timeout_seconds or self.timeout_seconds + timeout_milliseconds = round(request_timeout * 1000) + + # The native session for this timeout and certificate verification, see self._native_session_ids + session_id = f"{self._session_id}-{timeout_milliseconds}ms{'-insecure' if insecure_skip_verify else ''}" + self._native_session_ids.add(session_id) # --- Certificate pinning -------------------------------------------------------------------------------------- # pins a certificate so that it restricts which certificates are considered valid @@ -464,7 +478,7 @@ def execute_request( # --- Request -------------------------------------------------------------------------------------------------- # Settings shared by every request of a redirect chain. The URL, method, headers and body are set per request. request_payload: dict[str, Any] = { - "sessionId": self._session_id, + "sessionId": session_id, # Redirects are followed below, so that the cookies set on every hop end up in self.cookies. "followRedirects": False, # Cookies come from self.cookies only. Without this, the library also keeps the cookies of every response @@ -480,7 +494,7 @@ def execute_request( "additionalDecode": self.additional_decode, "proxyUrl": proxy_url, # Milliseconds, so fractions of a second work. The library rejects timeoutSeconds alongside it. - "timeoutMilliseconds": round(request_timeout * 1000), + "timeoutMilliseconds": timeout_milliseconds, } if certificate_pinning: request_payload["certificatePinningHosts"] = certificate_pinning