diff --git a/packages/smithy-http/src/smithy_http/aio/crt.py b/packages/smithy-http/src/smithy_http/aio/crt.py index 7b0422d7e..09ad39a1f 100644 --- a/packages/smithy-http/src/smithy_http/aio/crt.py +++ b/packages/smithy-http/src/smithy_http/aio/crt.py @@ -132,10 +132,13 @@ class AWSCRTHTTPClientConfig(http_interfaces.HTTPClientConfiguration): :param trust_store_path: Path to a PEM CA bundle that overrides the default system trust store when verifying TLS peers. Enables connecting to a server whose certificate chains to a private CA. + :param proxy: The HTTP proxy to send requests through. When not set, the client + reads HTTP_PROXY, HTTPS_PROXY and NO_PROXY from the environment """ read_buffer_size: int = DEFAULT_READ_BUFFER_SIZE trust_store_path: str | None = None + proxy: http_interfaces.HTTPProxyConfig | None = None def __post_init__(self) -> None: _assert_crt() @@ -164,7 +167,9 @@ def __init__(self, client_config: AWSCRTHTTPClientConfig | None = None) -> None: client. """ _assert_crt() - self._init(_AWSCRTEventLoop(), client_config) + config = client_config or AWSCRTHTTPClientConfig() + proxy = config.proxy or http_interfaces.HTTPProxyConfig.from_environment() + self._init(_AWSCRTEventLoop(), config, proxy) async def __aenter__(self) -> Self: if self._closed: @@ -177,7 +182,7 @@ async def __aexit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None def __deepcopy__(self, memo: Any) -> "AWSCRTHTTPClient": # Config is copied per operation, so copies share the event loop's thread. copied = AWSCRTHTTPClient.__new__(AWSCRTHTTPClient) - copied._init(self._eventloop, deepcopy(self._config)) + copied._init(self._eventloop, deepcopy(self._config), proxy=self._proxy) return copied async def send( @@ -239,7 +244,8 @@ async def close(self) -> None: def _init( self, eventloop: _AWSCRTEventLoop, - client_config: AWSCRTHTTPClientConfig | None, + client_config: AWSCRTHTTPClientConfig, + proxy: http_interfaces.HTTPProxyConfig, ) -> None: self._config = client_config or AWSCRTHTTPClientConfig() self._eventloop = eventloop @@ -250,6 +256,7 @@ def _init( None, self._config.trust_store_path ) self._tls_ctx = crt_io.ClientTlsContext(tls_ctx_options) + self._proxy = proxy self._socket_options = crt_io.SocketOptions() self._connections: ConnectionPoolDict = {} self._closed = False @@ -313,12 +320,45 @@ async def _build_new_connection( if url.port is not None: port = url.port + proxy = self._proxy.proxy_for(url) return await AIOHttpClientConnectionUnified.new( bootstrap=self._client_bootstrap, host_name=url.host, port=port, socket_options=self._socket_options, tls_connection_options=tls_connection_options, + proxy_options=( + self._build_proxy_options(proxy, url) if proxy is not None else None + ), + ) + + def _build_proxy_options( + self, proxy: http_interfaces.ProxyURI, url: core_interfaces.URI + ) -> crt_http.HttpProxyOptions: + proxy_tls_options = None + if proxy.scheme == "https": + proxy_tls_options = self._tls_ctx.new_connection_options() + proxy_tls_options.set_server_name(proxy.host) + + auth_username = auth_password = None + auth_type = crt_http.HttpProxyAuthenticationType.Nothing + if proxy.username is not None: + auth_type = crt_http.HttpProxyAuthenticationType.Basic + auth_username = proxy.username + auth_password = proxy.password or "" + + connection_type = crt_http.HttpProxyConnectionType.Forwarding + if url.scheme == "https": + connection_type = crt_http.HttpProxyConnectionType.Tunneling + + return crt_http.HttpProxyOptions( + host_name=proxy.host, + port=proxy.port, + tls_connection_options=proxy_tls_options, + auth_type=auth_type, + auth_username=auth_username, + auth_password=auth_password, + connection_type=connection_type, ) async def _validate_connection( diff --git a/packages/smithy-http/src/smithy_http/interfaces/__init__.py b/packages/smithy-http/src/smithy_http/interfaces/__init__.py index 65f914454..01ec0b78c 100644 --- a/packages/smithy-http/src/smithy_http/interfaces/__init__.py +++ b/packages/smithy-http/src/smithy_http/interfaces/__init__.py @@ -1,14 +1,20 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from collections.abc import Iterator -from dataclasses import dataclass -from typing import Any, Literal, Protocol - -from smithy_core.interfaces import Request, Response +import logging +import os +from collections.abc import Iterator, Mapping +from dataclasses import dataclass, field +from typing import Any, Literal, Protocol, Self +from urllib.parse import unquote, urlsplit + +from smithy_core import URI as _URI +from smithy_core.interfaces import Request, Response, URI from smithy_core.schemas import APIOperation from smithy_core.shapes import ShapeID from smithy_core.utils import read_streaming_blob +from ..exceptions import SmithyHTTPError as _SmithyHTTPError + __all__ = [ "Field", "FieldPosition", @@ -16,9 +22,11 @@ "HTTPClient", "HTTPClientConfiguration", "HTTPErrorIdentifier", + "HTTPProxyConfig", "HTTPRequest", "HTTPRequestConfiguration", "HTTPResponse", + "ProxyURI", "QueryParamsList", ] @@ -183,6 +191,162 @@ def identify( QueryParamsList = list[tuple[str, str]] +_LOGGER = logging.getLogger(__name__) + + +@dataclass(kw_only=True, frozen=True, eq=False) +class ProxyURI(_URI): + """The port to connect to the proxy on. + + Port is filled in from the scheme (80 for 'http', 443 for 'https') + field() makes port required: a bare annotation would inherit URI's None default + pyright flags a required override, but kw_only=True makes it valid + """ + + port: int = field() # pyright: ignore[reportGeneralTypeIssues] + + +@dataclass(kw_only=True, frozen=True, repr=False) +class HTTPProxyConfig: + """Client-level HTTP proxy configuration. + + Proxy URLs parsed as 'scheme://[user[:password]@]host[:port]'. + Proxying disabled when URLs are not set, + + :param http_proxy: Proxy URL for requests to 'http' destinations + :param https_proxy: Proxy URL for requests to 'https' destinations + :param no_proxy: Comma-separated hosts that bypass the proxy + '*' bypass every host. + Entry with a port like 'example.com:8443' matches only that port. + IPv6 entries need brackets '[::1]'. + """ + + http_proxy: str | None = None + https_proxy: str | None = None + no_proxy: str | None = None + + _http_proxy_uri: ProxyURI | None = field( + default=None, init=False, repr=False, compare=False + ) + _https_proxy_uri: ProxyURI | None = field( + default=None, init=False, repr=False, compare=False + ) + + def __post_init__(self) -> None: + """Parses URL and sets proxy once so proxy_for doesn't parse again on each call""" + if self.http_proxy: + object.__setattr__( + self, "_http_proxy_uri", self._parse_url(self.http_proxy) + ) + if self.https_proxy: + object.__setattr__( + self, "_https_proxy_uri", self._parse_url(self.https_proxy) + ) + + def __repr__(self) -> str: + """Omit passwords in urls""" + return ( + f"HTTPProxyConfig(http_proxy={self._http_proxy_uri!r}, " + f"https_proxy={self._https_proxy_uri!r}, " + f"no_proxy={self.no_proxy!r})" + ) + + @classmethod + def from_environment(cls, environ: Mapping[str, str] | None = None) -> Self: + """Build a configuration from the HTTP_PROXY, HTTPS_PROXY and + NO_PROXY environment variables. + + :param environ: The variables to read. Defaults to os.environ. + """ + envvars = os.environ if environ is None else environ + + def get(name: str) -> str | None: + return envvars.get(name.lower()) or envvars.get(name.upper()) or None + + def get_proxy_url(name: str) -> str | None: + if (value := get(name.upper())) is None: + return None + try: + cls._parse_url(value) + except _SmithyHTTPError as e: + _LOGGER.warning("Ignoring the %s environment variable: %s", name, e) + return None + return value + + return cls( + http_proxy=get_proxy_url("http_proxy"), + https_proxy=get_proxy_url("https_proxy"), + no_proxy=get("no_proxy"), + ) + + def proxy_for(self, destination: URI) -> ProxyURI | None: + """Return the proxy to use for 'destination', or None to connect directly. + + :param destination: The URI a request is sent to. + """ + match destination.scheme: + case "http": + proxy = self._http_proxy_uri + case "https": + proxy = self._https_proxy_uri + case _: + return None + if proxy is None or self._bypass_addresses(destination): + return None + return proxy + + def _bypass_addresses(self, destination: URI) -> bool: + """Bypasses addresses listed in NO_PROXY + Leading dot is ignored: '.example.com' matches example.com + Entry without "port" matches the host on any port + Entry with port like "example.com:8443", matches only that port. + """ + if not self.no_proxy: + return False + if self.no_proxy.strip() == "*": + return True + host = destination.host.lower() + if ":" in host: + host = f"[{host}]" + port = destination.port or (443 if destination.scheme == "https" else 80) + host_port = f"{host}:{port}" + for raw_entry in self.no_proxy.split(","): + if not (entry := raw_entry.strip().lstrip(".").lower()): + continue + for candidate in (host, host_port): + if candidate == entry or candidate.endswith(f".{entry}"): + return True + return False + + @staticmethod + def _parse_url(value: str) -> ProxyURI: + url = value.strip() + if "://" not in url: + url = f"http://{url}" + parts = urlsplit(url) + scheme = parts.scheme.lower() + if scheme not in ("http", "https"): + raise _SmithyHTTPError( + f"Unsupported proxy URL scheme {scheme!r}: expected 'http' or 'https'." + ) + try: + port = parts.port + except ValueError as e: + raise _SmithyHTTPError("Proxy URL has an invalid port.") from e + if not parts.hostname: + raise _SmithyHTTPError("Proxy URL has no host.") + return ProxyURI( + scheme=scheme, + host=parts.hostname, + port=port if port is not None else (443 if scheme == "https" else 80), + username=( + unquote(parts.username or "") + if parts.username or parts.password is not None + else None + ), + password=unquote(parts.password) if parts.password is not None else None, + ) + @dataclass(kw_only=True) class HTTPClientConfiguration: diff --git a/packages/smithy-http/tests/unit/aio/test_crt.py b/packages/smithy-http/tests/unit/aio/test_crt.py index 38b448a01..1f64dc46e 100644 --- a/packages/smithy-http/tests/unit/aio/test_crt.py +++ b/packages/smithy-http/tests/unit/aio/test_crt.py @@ -2,9 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # pyright: reportPrivateUsage=false import asyncio -from collections.abc import AsyncIterator +import os +from collections.abc import AsyncIterator, Iterator, Mapping from copy import deepcopy from io import BytesIO +from typing import Any from unittest.mock import AsyncMock, Mock, patch import pytest @@ -15,8 +17,26 @@ from smithy_http.aio.crt import ( AsyncAWSCRTHTTPResponse, AWSCRTHTTPClient, + AWSCRTHTTPClientConfig, ) from smithy_http.exceptions import SmithyHTTPError +from smithy_http.interfaces import HTTPProxyConfig + +_PROXY_ENV_VARS = ("http_proxy", "https_proxy", "no_proxy") + + +def _proxy_env(**values: str) -> Any: + """Replace the proxy variables in 'os.environ' with 'values'""" + names = {n for name in _PROXY_ENV_VARS for n in (name, name.upper())} + environ = {k: v for k, v in os.environ.items() if k not in names} + return patch.dict(os.environ, {**environ, **values}, clear=True) + + +@pytest.fixture(autouse=True) +def _clear_proxy_env() -> Iterator[None]: # pyright: ignore[reportUnusedFunction] + """keep external proxy settings out of tests.""" + with _proxy_env(): + yield def test_deepcopy_client() -> None: @@ -594,3 +614,110 @@ def test_response_properties() -> None: assert response.status == 404 assert response.fields == fields assert response.reason is None + + +async def _captured_connection_kwargs( + client: AWSCRTHTTPClient, url: URI, *, require_http2: bool = False +) -> Mapping[str, Any]: + with patch( + "smithy_http.aio.crt.AIOHttpClientConnectionUnified.new", AsyncMock() + ) as mock_new: + await client._build_new_connection(url, require_http2=require_http2) + return mock_new.call_args.kwargs + + +def _proxied_client(**proxy: str) -> AWSCRTHTTPClient: + return AWSCRTHTTPClient(AWSCRTHTTPClientConfig(proxy=HTTPProxyConfig(**proxy))) + + +async def test_no_proxy_by_default() -> None: + kwargs = await _captured_connection_kwargs( + AWSCRTHTTPClient(), URI(scheme="https", host="example.com") + ) + assert kwargs["proxy_options"] is None + + +async def test_https_proxy_destination() -> None: + client = _proxied_client(https_proxy="http://proxy.local:3128") + url = URI(scheme="https", host="bedrock-runtime.us-west-2.amazonaws.com") + + kwargs = await _captured_connection_kwargs(client, url, require_http2=True) + + assert kwargs["host_name"] == "bedrock-runtime.us-west-2.amazonaws.com" + assert kwargs["port"] == 443 + assert kwargs["tls_connection_options"] is not None + proxy_options = kwargs["proxy_options"] + assert isinstance(proxy_options, crt_http.HttpProxyOptions) + assert proxy_options.host_name == "proxy.local" + assert proxy_options.port == 3128 + assert proxy_options.tls_connection_options is None + assert proxy_options.connection_type == crt_http.HttpProxyConnectionType.Tunneling + assert proxy_options.auth_type == crt_http.HttpProxyAuthenticationType.Nothing + + +async def test_http_proxy_destination_forwarded() -> None: + client = _proxied_client(http_proxy="proxy.local:3128") + + kwargs = await _captured_connection_kwargs( + client, URI(scheme="http", host="example.com") + ) + + proxy_options = kwargs["proxy_options"] + assert isinstance(proxy_options, crt_http.HttpProxyOptions) + assert proxy_options.connection_type == crt_http.HttpProxyConnectionType.Forwarding + + +async def test_proxy_has_proper_scheme() -> None: + client = _proxied_client(http_proxy="http://proxy.local:3128") + + kwargs = await _captured_connection_kwargs( + client, URI(scheme="https", host="example.com") + ) + + assert kwargs["proxy_options"] is None + + +async def test_proxy_basic_auth() -> None: + client = _proxied_client(https_proxy="http://user:p%40ss@proxy.local:3128") + + kwargs = await _captured_connection_kwargs( + client, URI(scheme="https", host="example.com") + ) + + proxy_options = kwargs["proxy_options"] + assert isinstance(proxy_options, crt_http.HttpProxyOptions) + assert proxy_options.auth_type == crt_http.HttpProxyAuthenticationType.Basic + assert proxy_options.auth_username == "user" + assert proxy_options.auth_password == "p@ss" + + +async def test_no_proxy_bypass() -> None: + client = _proxied_client( + https_proxy="http://proxy.local:3128", no_proxy=".amazonaws.com" + ) + + kwargs = await _captured_connection_kwargs( + client, URI(scheme="https", host="sts.us-west-2.amazonaws.com") + ) + + assert kwargs["proxy_options"] is None + + +async def test_proxy_from_environment() -> None: + with _proxy_env(HTTPS_PROXY="http://proxy.local:3128"): + client = AWSCRTHTTPClient() + + kwargs = await _captured_connection_kwargs( + client, URI(scheme="https", host="example.com") + ) + + proxy_options = kwargs["proxy_options"] + assert isinstance(proxy_options, crt_http.HttpProxyOptions) + assert proxy_options.host_name == "proxy.local" + + +def test_unusable_environment_proxy_does_not_stop_client() -> None: + with _proxy_env(HTTPS_PROXY="socks5h://127.0.0.1:1080"): + client = AWSCRTHTTPClient() + + assert client._proxy == HTTPProxyConfig() diff --git a/packages/smithy-http/tests/unit/test_proxy.py b/packages/smithy-http/tests/unit/test_proxy.py new file mode 100644 index 000000000..07fac4566 --- /dev/null +++ b/packages/smithy-http/tests/unit/test_proxy.py @@ -0,0 +1,215 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +import os +from unittest.mock import patch + +import pytest +from smithy_core import URI +from smithy_http.exceptions import SmithyHTTPError +from smithy_http.interfaces import HTTPProxyConfig + + +@pytest.mark.parametrize( + "proxy_url, expected", + [ + ("proxy.local ", URI(scheme="http", host="proxy.local", port=80)), + ( + "http://proxy.local:3128", + URI(scheme="http", host="proxy.local", port=3128), + ), + ("proxy.local:3128", URI(scheme="http", host="proxy.local", port=3128)), + ("http://proxy.local", URI(scheme="http", host="proxy.local", port=80)), + ("https://proxy.local", URI(scheme="https", host="proxy.local", port=443)), + ("HTTP://Proxy.Local:8080/", URI(scheme="http", host="proxy.local", port=8080)), + ("http://[::1]:3128", URI(scheme="http", host="::1", port=3128)), + ( + "http://user:passw0rd@proxy.local:3128", + URI( + scheme="http", + host="proxy.local", + port=3128, + username="user", + password="passw0rd", + ), + ), + ( + "http://user@proxy.local:3128", + URI(scheme="http", host="proxy.local", port=3128, username="user"), + ), + ( + "http://:token@proxy.local:3128", + URI( + scheme="http", + host="proxy.local", + port=3128, + username="", + password="token", + ), + ), + ("http://@proxy.local:3128", URI(scheme="http", host="proxy.local", port=3128)), + ], +) +def test_proxy_url_parsing(proxy_url: str, expected: URI) -> None: + proxy = HTTPProxyConfig._parse_url(proxy_url) # pyright: ignore[reportPrivateUsage] + assert proxy == expected + assert proxy.username == expected.username + assert proxy.password == expected.password + + +def test_proxy_by_destination_scheme() -> None: + config = HTTPProxyConfig( + http_proxy="http://plain.local:8080", https_proxy="http://secure.local:8081" + ) + http_proxy = config.proxy_for(URI(scheme="http", host="example.com")) + https_proxy = config.proxy_for(URI(scheme="https", host="example.com")) + assert http_proxy is not None and http_proxy.host == "plain.local" + assert https_proxy is not None and https_proxy.host == "secure.local" + + +def test_no_proxy_unknown_scheme() -> None: + config = HTTPProxyConfig(http_proxy="http://proxy:1", https_proxy="http://proxy:1") + assert config.proxy_for(URI(scheme="ws", host="example.com")) is None + + +def test_empty_config_disables_proxy() -> None: + assert HTTPProxyConfig().proxy_for(URI(host="example.com")) is None + + +@pytest.mark.parametrize( + "proxy_url, message", + [ + ("socks5://proxy.local:1080", "Unsupported proxy URL scheme"), + ("http://proxy.local:noport", "invalid port"), + ("http://:8080", "no host"), + ], +) +def test_invalid_proxy_fails(proxy_url: str, message: str) -> None: + with pytest.raises(SmithyHTTPError, match=message): + HTTPProxyConfig(https_proxy=proxy_url) + + +def test_invalid_proxy_error_omits_credentials() -> None: + with pytest.raises(SmithyHTTPError) as exc_info: + HTTPProxyConfig(https_proxy="http://user:passw0rd@proxy.local:bad") + assert "passw0rd" not in str(exc_info.value) + + +@pytest.mark.parametrize( + "no_proxy, host, port, bypassed", + [ + # should be bypassed + (".example.com", "example.com", None, True), + (".example.com", "api.example.com", None, True), + ("EXAMPLE.com", "Api.Example.COM", None, True), + ("other.com, example.com", "example.com", None, True), + ("example.com:8443", "example.com", 8443, True), + ("example.com:443", "example.com", None, True), + ("169.254.169.254", "169.254.169.254", None, True), + ("[::1]", "::1", None, True), + ("[::1]:8443", "::1", 8443, True), + ("example.com:abc, other.com", "other.com", None, True), + # should not be bypassed + ("[::1]:8443", "::1", None, False), + # IPv6 entries need brackets + ("::1", "::1", None, False), + ("::1", "fd00::1", None, False), + ("example.com", "notexample.com", None, False), + (" , ,", "example.com", None, False), + ("example.com:8443", "example.com", None, False), + ("example.com:abc", "example.com", None, False), + ("example.com:", "example.com", None, False), + ], +) +def test_bypass_matching( + no_proxy: str, host: str, port: int | None, bypassed: bool +) -> None: + config = HTTPProxyConfig(https_proxy="http://proxy.local:3128", no_proxy=no_proxy) + proxy = config.proxy_for(URI(scheme="https", host=host, port=port)) + assert (proxy is None) is bypassed + + +def test_from_environment() -> None: + environ = { + "HTTP_PROXY": "http://plain.local:1", + "HTTPS_PROXY": "http://secure.local:2", + "NO_PROXY": "somedomain", + } + assert HTTPProxyConfig.from_environment(environ) == HTTPProxyConfig( + http_proxy="http://plain.local:1", + https_proxy="http://secure.local:2", + no_proxy="somedomain", + ) + + +@pytest.mark.parametrize( + "environ", + [ + {}, + {"HTTPS_PROXY": ""}, + {"https_proxy": "", "HTTPS_PROXY": ""}, + ], +) +def test_proxy_unset(environ: dict[str, str]) -> None: + assert HTTPProxyConfig.from_environment(environ) == HTTPProxyConfig() + + +def test_empty_lowercase_falls_back_to_uppercase() -> None: + environ = {"https_proxy": "", "HTTPS_PROXY": "http://upper.local:2"} + assert HTTPProxyConfig.from_environment(environ).https_proxy == ( + "http://upper.local:2" + ) + + +@pytest.mark.parametrize( + "name, value", + [ + ("HTTPS_PROXY", "socks5h://127.0.0.1:1080"), + ("https_proxy", "http://user:passw0rd@proxy.local:noport"), + ("HTTP_PROXY", "http://:3128"), + ], +) +def test_ignores_unusable_proxy_urls( + name: str, value: str, caplog: pytest.LogCaptureFixture +) -> None: + config = HTTPProxyConfig.from_environment({name: value, "NO_PROXY": "somedomain"}) + + assert config == HTTPProxyConfig(no_proxy="somedomain") + assert f"Ignoring the {name.lower()} environment variable" in caplog.text + assert "passw0rd" not in caplog.text + + +def test_keeps_usable_proxy_when_other_is_unusable() -> None: + environ = { + "HTTP_PROXY": "socks5://proxy.local:1080", + "HTTPS_PROXY": "http://proxy.local:3128", + } + assert HTTPProxyConfig.from_environment(environ) == HTTPProxyConfig( + https_proxy="http://proxy.local:3128" + ) + + +def test_defaults_to_os_environ() -> None: + with patch.dict(os.environ, {"HTTPS_PROXY": "http://secure.local:2"}, clear=True): + assert HTTPProxyConfig.from_environment() == HTTPProxyConfig( + https_proxy="http://secure.local:2" + ) + + +@pytest.mark.parametrize( + "proxy_url", + [ + "http://user:passw0rd@proxy.local:3128", + "user:passw0rd@proxy.local:3128", + "http://:passw0rd@proxy.local:3128", + "http://user@proxy.local:3128", + "http://proxy.local:3128", + ], +) +def test_repr_omits_password(proxy_url: str) -> None: + config = HTTPProxyConfig(http_proxy=proxy_url, https_proxy=proxy_url) + rendered = repr(config) + assert "passw0rd" not in rendered + assert str(config) == rendered + expected = HTTPProxyConfig._parse_url(proxy_url) # pyright: ignore[reportPrivateUsage] + assert f"http_proxy={expected!r}" in rendered + assert f"https_proxy={expected!r}" in rendered