diff --git a/.agents/skills/python-proxy-headers/SKILL.md b/.agents/skills/python-proxy-headers/SKILL.md index 588131e..809aeab 100644 --- a/.agents/skills/python-proxy-headers/SKILL.md +++ b/.agents/skills/python-proxy-headers/SKILL.md @@ -45,6 +45,8 @@ with ProxySession(proxy_headers={'X-ProxyMesh-Country': 'US'}) as session: ### httpx +Requires `httpx>=0.28,<1` (`pip install 'python-proxy-headers[httpx]'`). httpx 1.0 drops httpcore and proxy support. Importing `httpx_proxy` still succeeds and warns; calling it raises `UnsupportedHttpxVersion`. + ```python from python_proxy_headers.httpx_proxy import get diff --git a/.github/workflows/client_compatibility.yml b/.github/workflows/client_compatibility.yml new file mode 100644 index 0000000..91c2626 --- /dev/null +++ b/.github/workflows/client_compatibility.yml @@ -0,0 +1,75 @@ +# Newest stable releases and pre-releases of every supported client. +# Not triggered on pull_request, so a red run is visible in Actions and via +# workflow_dispatch, and it is not a required PR check. +# Pre-release failures must stay failing (no continue-on-error). +# Runners are ubuntu-24.04. Do not switch these jobs to ubuntu-latest. + +name: Client compatibility + +on: + schedule: + - cron: "0 15 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + clients: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.14"] + channel: [stable, pre] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies (pycurl) + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + + - name: Install package + run: | + python -m pip install --upgrade pip + pip install -e . + + - name: Install ${{ matrix.channel }} clients + run: | + set -euo pipefail + packages=(urllib3 requests aiohttp httpx pycurl cloudscraper autoscraper) + if [ "${{ matrix.channel }}" = "pre" ]; then + pip install --pre --upgrade "${packages[@]}" + else + pip install --upgrade "${packages[@]}" + fi + python - <<'PY' + import importlib.metadata as metadata + names = ( + "urllib3", + "requests", + "aiohttp", + "httpx", + "httpcore", + "pycurl", + "cloudscraper", + "autoscraper", + ) + for name in names: + try: + print(f"{name}=={metadata.version(name)}") + except metadata.PackageNotFoundError: + print(f"{name}==not-installed") + PY + + - name: Run header security unit tests + run: python test_header_security.py + + - name: Run local mock-proxy internal tests + run: python test_client_internals.py diff --git a/.github/workflows/proxy_integration_tests.yml b/.github/workflows/proxy_integration_tests.yml index 68ca318..c44cbab 100644 --- a/.github/workflows/proxy_integration_tests.yml +++ b/.github/workflows/proxy_integration_tests.yml @@ -1,7 +1,13 @@ -# Integration tests against a real proxy (PROXY_URL). Add repository secret PROXY_URL -# under Settings → Secrets and variables → Actions, then mark this workflow as a required -# status check under branch protection (pull_request events only receive secrets for PRs -# from the same repository, not from forks). +# Pull-request checks. +# - unit: Python matrix, compatible client extras, header tests, and a local +# CONNECT mock (no PROXY_URL). +# - integration: real proxy via the PROXY_URL secret. pull_request events only +# receive secrets for PRs from this repository, not from forks. +# +# Pre-release clients are not installed here. The weekly Client compatibility +# workflow runs `pip install --pre` and is not a pull_request check, so a +# pre-release failure cannot block a PR. +# Runners are ubuntu-24.04. Do not switch these jobs to ubuntu-latest. name: Proxy integration tests @@ -14,6 +20,10 @@ permissions: jobs: unit: runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -22,20 +32,22 @@ jobs: - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: "3.x" + python-version: ${{ matrix.python-version }} - name: Install system dependencies (pycurl) run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - - name: Install package and test dependencies + - name: Install package and compatible client extras run: | python -m pip install --upgrade pip - pip install -e . - pip install urllib3 requests aiohttp httpx pycurl + pip install -e ".[urllib3,requests,httpx,aiohttp,pycurl,cloudscraper,autoscraper]" - name: Run header security unit tests run: python test_header_security.py + - name: Run local mock-proxy internal tests + run: python test_client_internals.py + integration: runs-on: ubuntu-24.04 @@ -47,16 +59,15 @@ jobs: - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: "3.x" + python-version: "3.12" - name: Install system dependencies (pycurl) run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - - name: Install package and test dependencies + - name: Install package and compatible client extras run: | python -m pip install --upgrade pip - pip install -e . - pip install urllib3 requests aiohttp httpx cloudscraper autoscraper pycurl + pip install -e ".[urllib3,requests,httpx,aiohttp,pycurl,cloudscraper,autoscraper]" - name: Require PROXY_URL Actions secret env: diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8694e2f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,27 @@ +# Changelog + +## Unreleased + +### httpx 1.0 + +- Importing `python_proxy_headers.httpx_proxy` no longer raises `ImportError: cannot import name AsyncHTTPTransport` when httpx 1.x is installed. +- httpx 1.0.dev6 (2026-08-31) is a rewrite without httpcore, without `HTTPTransport` / `AsyncHTTPTransport`, and without proxy support ([encode/httpx#947](https://github.com/encode/httpx/issues/947)). The module warns at import time and raises `UnsupportedHttpxVersion` if you call it. +- The `[httpx]` extra is `httpx>=0.28,<1`. There is no 1.0 adapter in this change. Options are documented in `docs/httpx.rst`: stay on 0.28, wait for a public CONNECT hook, or add a small backend after `NetworkBackend` can take an existing tunnel. Vendoring httpcore is not one of them. + +### Tests + +- Contract checks and a local CONNECT mock (`tests/mock_connect_proxy.py`, no proxy credentials) exercise the private hooks in `urllib3_proxy_manager.py` (`_prepare_proxy`, `_tunnel`, `http.client._read_headers`), `httpx_proxy` (copied httpcore `handle_request` / `handle_async_request`), and `aiohttp_proxy.py` (copied `_create_proxy_connection`). They name the upstream symbol when it moves. + +### Packaging + +- Keywords, trove classifiers, and optional extras: `[urllib3]`, `[requests]`, `[httpx]`, `[aiohttp]`, `[pycurl]`, `[cloudscraper]`, `[autoscraper]`. +- `requires-python` is `>=3.10`. urllib3 2.8, requests 2.34, and aiohttp 3.14 all require 3.10+, and the httpx adapter already uses `str | None`. +- The changelog URL points at this file. + +## 0.2.3 + +CONNECT header injection hardening and safer merging of proxy response headers. See the [GitHub release](https://github.com/proxymesh/python-proxy-headers/releases/tag/v0.2.3). + +## Earlier + +[GitHub Releases](https://github.com/proxymesh/python-proxy-headers/releases). diff --git a/README.md b/README.md index dc08423..a9f7c6d 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,21 @@ This library solves both problems for popular Python HTTP libraries. pip install python-proxy-headers ``` -Then install the HTTP library you want to use (e.g., `pip install requests`). +Or install the client extra you need. Each extra pins a range that CI tests: -> **Note:** This package has no dependencies by default - install only what you need. +```bash +pip install 'python-proxy-headers[requests]' +pip install 'python-proxy-headers[urllib3]' +pip install 'python-proxy-headers[httpx]' +pip install 'python-proxy-headers[aiohttp]' +pip install 'python-proxy-headers[pycurl]' +pip install 'python-proxy-headers[cloudscraper]' +pip install 'python-proxy-headers[autoscraper]' +``` + +The `httpx` extra is `httpx>=0.28,<1`. httpx 1.0 drops proxy support. Importing `python_proxy_headers.httpx_proxy` still succeeds, and using it raises `UnsupportedHttpxVersion`. See the [httpx docs](https://python-proxy-headers.readthedocs.io/en/latest/httpx.html). + +> **Note:** This package has no required dependencies - install only the extra you need. ## Quick Start @@ -122,6 +134,10 @@ response = scraper.get('https://example.com') A test harness is included to verify proxy header functionality: ```bash +# Header checks and a local CONNECT mock (no proxy credentials) +python test_header_security.py +python test_client_internals.py + # Set your proxy export PROXY_URL='http://user:pass@proxy.example.com:8080' @@ -142,8 +158,10 @@ For detailed documentation, API reference, and more examples: - **Full Documentation:** [python-proxy-headers.readthedocs.io](https://python-proxy-headers.readthedocs.io/en/latest/) - **Example Code:** [proxy-examples for Python](https://github.com/proxymesh/proxy-examples/tree/main/python) -## Related Projects +## Other languages +- **[javascript-proxy-headers](https://github.com/proxymesh/javascript-proxy-headers)** - Send and read custom proxy headers on HTTPS CONNECT for Node.js HTTP clients +- **[ruby-proxy-headers](https://github.com/proxymesh/ruby-proxy-headers)** - Send and read custom proxy headers on HTTPS CONNECT for Ruby HTTP clients - **[scrapy-proxy-headers](https://github.com/proxymesh/scrapy-proxy-headers)** - Proxy header support for Scrapy ## About diff --git a/docs/httpx.rst b/docs/httpx.rst index e33076d..6a0ca88 100644 --- a/docs/httpx.rst +++ b/docs/httpx.rst @@ -3,6 +3,30 @@ httpx `HTTPX `_ is a fully featured HTTP client for Python 3, which provides sync and async APIs, and support for both HTTP/1.1 and HTTP/2. This page describes how to use httpx with proxies and how to send and receive custom proxy headers. +httpx 1.0 +--------- + +httpx 1.0 (pre-release ``1.0.dev6``, 2026-08-31) is a rewrite. It no longer depends on httpcore, it does not provide ``HTTPTransport`` or ``AsyncHTTPTransport``, and it has no proxy or CONNECT support (`encode/httpx#947 `_). + +Installing that release does not crash ``import python_proxy_headers.httpx_proxy``. Import warns, and calling ``get``, ``HTTPProxyTransport``, or ``AsyncHTTPProxyTransport`` raises ``UnsupportedHttpxVersion`` instead of ``ImportError``. + +The ``httpx`` extra stays on the 0.28 line:: + + pip install 'python-proxy-headers[httpx]' + +which installs ``httpx>=0.28,<1``. This release does not include a 1.0 adapter. + +Support options +~~~~~~~~~~~~~~~ + +1. **Stay on httpx 0.28.x.** This is the supported path. The adapter copies httpcore's private ``handle_request`` and cannot load on the 1.0 rewrite. + +2. **Wait for a public proxy API.** httpx 1.0's ``NetworkBackend.connect`` / ``connect_tls`` and ``Transport.send`` open a direct connection. They cannot send CONNECT headers or return the proxy's response headers. When `issue 947 `_ grows a public hook, implement against that instead of another private copy. + +3. **A small CONNECT backend, later.** A future adapter could open the proxy socket, write CONNECT plus the custom headers, read the proxy response, wrap TLS, and pass the stream to ``httpx.Connection``. ``NetworkBackend`` does not accept an existing socket today, and the 1.0 API is still early, so this is not in this release. + +4. **Do not vendor httpcore** to pretend 1.0 works. That would ignore the client the user installed. + Getting Started --------------- @@ -14,7 +38,7 @@ This section shows you how to quickly get up and running with proxy headers in h .. code-block:: bash - pip install python-proxy-headers httpx + pip install 'python-proxy-headers[httpx]' 2. Import the module: diff --git a/docs/index.rst b/docs/index.rst index a8cca37..de3f8e0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,20 +25,27 @@ None of these modules provide good support for parsing custom response headers f If you are looking for `Scrapy `_ support, please see our `scrapy-proxy-headers `_ project. +The same CONNECT header support in other languages: + +* `javascript-proxy-headers `_ +* `ruby-proxy-headers `_ + Installation ------------ To use these extension modules, you must first do the following: -1. Install the package: +1. Install the package, optionally with the client you use: .. code-block:: bash pip install python-proxy-headers + pip install 'python-proxy-headers[requests]' + pip install 'python-proxy-headers[httpx]' -2. Install the appropriate package based on the Python module you want to use. +2. Or install the HTTP library yourself. Extras pin the ranges we test. The ``httpx`` extra is ``httpx>=0.28,<1``. httpx 1.0 is not supported yet; see :doc:`httpx`. -This package does not have any dependencies because we don't know which module you want to use. +This package does not have any required dependencies because we don't know which module you want to use. You can also find more example code in our `proxy-examples for python `_. diff --git a/pyproject.toml b/pyproject.toml index a1d2869..52d27ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,19 +10,56 @@ authors = [ ] description = "Handle custom proxy headers for http & https requests in various python libraries" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.10" +keywords = [ + "proxy", + "proxy-headers", + "https-proxy", + "connect", + "tunnel", + "requests", + "urllib3", + "httpx", + "aiohttp", + "web-scraping", + "proxymesh", +] classifiers = [ + "Development Status :: 4 - Beta", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Operating System :: OS Independent", - "License :: OSI Approved :: MIT License", - "Intended Audience :: Developers", - "Topic :: Internet :: WWW/HTTP", + "License :: OSI Approved :: MIT License", + "Intended Audience :: Developers", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Internet :: Proxy Servers", "Topic :: Software Development :: Libraries :: Python Modules", + "Framework :: AsyncIO", ] +# Newest releases checked on 2026-10-23: urllib3 2.8.0, requests 2.34.2, +# httpx 0.28.1 (httpcore 1.0.9), aiohttp 3.14.5, pycurl 7.48.0, +# cloudscraper 1.2.71, autoscraper 1.1.14. +# Lower bounds are the oldest minor whose private hooks still match the +# adapters. CI installs the newest release inside each range. httpx 1.x is +# excluded: 1.0.dev6 drops httpcore and proxy support (encode/httpx#947). +[project.optional-dependencies] +urllib3 = ["urllib3>=2.2,<3"] +requests = ["requests>=2.31,<3"] +httpx = ["httpx>=0.28,<1"] +aiohttp = ["aiohttp>=3.10,<4"] +pycurl = ["pycurl>=7.45,<8"] +cloudscraper = ["cloudscraper>=1.2.71,<2"] +autoscraper = ["autoscraper>=1.1.14,<2"] + [project.urls] Homepage = "https://github.com/proxymesh/python-proxy-headers" -Changelog = "https://github.com/proxymesh/python-proxy-headers/commits/main/" +Changelog = "https://github.com/proxymesh/python-proxy-headers/blob/main/CHANGELOG.md" Issues = "https://github.com/proxymesh/python-proxy-headers/issues" Documentation = "https://python-proxy-headers.readthedocs.io/en/latest/" ProxyMesh = "https://proxymesh.com" diff --git a/python_proxy_headers/_httpx_httpcore.py b/python_proxy_headers/_httpx_httpcore.py new file mode 100644 index 0000000..e47d89d --- /dev/null +++ b/python_proxy_headers/_httpx_httpcore.py @@ -0,0 +1,397 @@ +"""httpcore backend for httpx 0.x. + +``httpx_proxy`` imports this module only when the installed httpx is the 0.28 +line. httpx 1.0 drops httpcore and these private proxy types, so this file is +not imported there. The tunnel classes are the existing copies of httpcore's +``handle_request``; they are not a 1.0 adapter. +""" + +from contextlib import contextmanager +from httpcore._sync.http_proxy import HTTPProxy, TunnelHTTPConnection, merge_headers, logger +from httpcore._sync.http11 import HTTP11Connection +from httpcore._async.http_proxy import AsyncHTTPProxy, AsyncTunnelHTTPConnection +from httpcore._async.http11 import AsyncHTTP11Connection +from httpcore._models import URL, Request +from httpcore._exceptions import ProxyError +from httpcore._ssl import default_ssl_context +from httpcore._trace import Trace +from httpx import AsyncHTTPTransport, HTTPTransport, Client +from httpx._config import DEFAULT_LIMITS, DEFAULT_TIMEOUT_CONFIG, Proxy, create_ssl_context +from .header_utils import filter_connect_headers + +class ProxyTunnelHTTPConnection(TunnelHTTPConnection): + # Unfortunately the only way to get connect_response.headers into the Response + # is to override this whole method + def handle_request(self, request): + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("connect", None) + + with self._connect_lock: + if not self._connected: + target = b"%b:%d" % (self._remote_origin.host, self._remote_origin.port) + + connect_url = URL( + scheme=self._proxy_origin.scheme, + host=self._proxy_origin.host, + port=self._proxy_origin.port, + target=target, + ) + connect_headers = merge_headers( + [(b"Host", target), (b"Accept", b"*/*")], self._proxy_headers + ) + connect_request = Request( + method=b"CONNECT", + url=connect_url, + headers=connect_headers, + extensions=request.extensions, + ) + connect_response = self._connection.handle_request( + connect_request + ) + + if connect_response.status < 200 or connect_response.status > 299: + reason_bytes = connect_response.extensions.get("reason_phrase", b"") + reason_str = reason_bytes.decode("ascii", errors="ignore") + msg = "%d %s" % (connect_response.status, reason_str) + self._connection.close() + raise ProxyError(msg) + + stream = connect_response.extensions["network_stream"] + + # Upgrade the stream to SSL + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": self._remote_origin.host.decode("ascii"), + "timeout": timeout, + } + with Trace("start_tls", logger, request, kwargs) as trace: + stream = stream.start_tls(**kwargs) + trace.return_value = stream + + # Determine if we should be using HTTP/1.1 or HTTP/2 + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) + + # Create the HTTP/1.1 or HTTP/2 connection + if http2_negotiated or (self._http2 and not self._http1): + from httpcore._sync.http2 import HTTP2Connection + + self._connection = HTTP2Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + else: + self._connection = HTTP11Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + + self._connect_response_headers = connect_response.headers + self._connected = True + # Merge safe CONNECT headers without overwriting origin values + response = self._connection.handle_request(request) + response.headers = merge_headers( + response.headers, + filter_connect_headers( + response.headers, getattr(self, "_connect_response_headers", ()) + ), + ) + return response + +class AsyncProxyTunnelHTTPConnection(AsyncTunnelHTTPConnection): + async def handle_async_request(self, request): + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("connect", None) + + async with self._connect_lock: + if not self._connected: + target = b"%b:%d" % (self._remote_origin.host, self._remote_origin.port) + + connect_url = URL( + scheme=self._proxy_origin.scheme, + host=self._proxy_origin.host, + port=self._proxy_origin.port, + target=target, + ) + connect_headers = merge_headers( + [(b"Host", target), (b"Accept", b"*/*")], self._proxy_headers + ) + connect_request = Request( + method=b"CONNECT", + url=connect_url, + headers=connect_headers, + extensions=request.extensions, + ) + connect_response = await self._connection.handle_async_request( + connect_request + ) + + if connect_response.status < 200 or connect_response.status > 299: + reason_bytes = connect_response.extensions.get("reason_phrase", b"") + reason_str = reason_bytes.decode("ascii", errors="ignore") + msg = "%d %s" % (connect_response.status, reason_str) + await self._connection.aclose() + raise ProxyError(msg) + + stream = connect_response.extensions["network_stream"] + + # Upgrade the stream to SSL + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": self._remote_origin.host.decode("ascii"), + "timeout": timeout, + } + async with Trace("start_tls", logger, request, kwargs) as trace: + stream = await stream.start_tls(**kwargs) + trace.return_value = stream + + # Determine if we should be using HTTP/1.1 or HTTP/2 + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) + + # Create the HTTP/1.1 or HTTP/2 connection + if http2_negotiated or (self._http2 and not self._http1): + from httpcore._async.http2 import AsyncHTTP2Connection + + self._connection = AsyncHTTP2Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + else: + self._connection = AsyncHTTP11Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + + self._connect_response_headers = connect_response.headers + self._connected = True + # Merge safe CONNECT headers without overwriting origin values + response = await self._connection.handle_async_request(request) + response.headers = merge_headers( + response.headers, + filter_connect_headers( + response.headers, getattr(self, "_connect_response_headers", ()) + ), + ) + return response + +class HTTPProxyHeaders(HTTPProxy): + def create_connection(self, origin): + if origin.scheme == b"http": + return super().create_connection(origin) + return ProxyTunnelHTTPConnection( + proxy_origin=self._proxy_url.origin, + proxy_headers=self._proxy_headers, + remote_origin=origin, + ssl_context=self._ssl_context, + proxy_ssl_context=self._proxy_ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + network_backend=self._network_backend, + ) + +class AsyncHTTPProxyHeaders(AsyncHTTPProxy): + def create_connection(self, origin): + if origin.scheme == b"http": + return super().create_connection(origin) + return AsyncProxyTunnelHTTPConnection( + proxy_origin=self._proxy_url.origin, + proxy_headers=self._proxy_headers, + remote_origin=origin, + ssl_context=self._ssl_context, + proxy_ssl_context=self._proxy_ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + network_backend=self._network_backend, + ) + +# class ProxyConnectionPool(ConnectionPool): +# def create_connection(self, origin): +# if self._proxy is not None: +# if self._proxy.url.scheme in (b"socks5", b"socks5h"): +# return super().create_connection(origin) +# elif origin.scheme == b"http": +# return super().create_connection(origin) + +# return ProxyTunnelHTTPConnection( +# proxy_origin=self._proxy.url.origin, +# proxy_headers=self._proxy.headers, +# proxy_ssl_context=self._proxy.ssl_context, +# remote_origin=origin, +# ssl_context=self._ssl_context, +# keepalive_expiry=self._keepalive_expiry, +# http1=self._http1, +# http2=self._http2, +# network_backend=self._network_backend, +# ) + +# return super().create_connection(origin) + +class HTTPProxyTransport(HTTPTransport): + def __init__( + self, + verify = True, + cert = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits = DEFAULT_LIMITS, + proxy = None, + uds: str | None = None, + local_address: str | None = None, + retries: int = 0, + socket_options = None, + ) -> None: + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + if proxy and proxy.url.scheme in ("http", "https"): + self._pool = HTTPProxyHeaders( + proxy_url=URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + proxy_headers=proxy.headers.raw, + ssl_context=ssl_context, + proxy_ssl_context=proxy.ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + socket_options=socket_options, + ) + else: + super().__init__(verify, cert, trust_env, http1, http2, limits, proxy, uds, local_address, retries, socket_options) + +class AsyncHTTPProxyTransport(AsyncHTTPTransport): + def __init__( + self, + verify = True, + cert = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits = DEFAULT_LIMITS, + proxy = None, + uds: str | None = None, + local_address: str | None = None, + retries: int = 0, + socket_options = None, + ) -> None: + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + if proxy and proxy.url.scheme in ("http", "https"): + self._pool = AsyncHTTPProxyHeaders( + proxy_url=URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + proxy_headers=proxy.headers.raw, + proxy_ssl_context=proxy.ssl_context, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + socket_options=socket_options, + ) + else: + super().__init__(verify, cert, trust_env, http1, http2, limits, proxy, uds, local_address, retries, socket_options) + +def request(method: str, + url: URL | str, + *, + cookies = None, + proxy = None, + timeout = DEFAULT_TIMEOUT_CONFIG, + verify = True, + trust_env: bool = True, + **kwargs): + transport = HTTPProxyTransport(proxy=proxy) + with Client( + cookies=cookies, + verify=verify, + timeout=timeout, + trust_env=trust_env, + mounts={'http://': transport, 'https://': transport} + ) as client: + return client.request(method=method, url=url, **kwargs) + +def get(*args, **kwargs): + return request('GET', *args, **kwargs) + +def options(*args, **kwargs): + return request('OPTIONS', *args, **kwargs) + +def head(*args, **kwargs): + return request('HEAD', *args, **kwargs) + +def post(*args, **kwargs): + return request('POST', *args, **kwargs) + +def put(*args, **kwargs): + return request('PUT', *args, **kwargs) + +def patch(*args, **kwargs): + return request('PATCH', *args, **kwargs) + +def delete(*args, **kwargs): + return request('DELETE', *args, **kwargs) + +@contextmanager +def stream(method: str, + url: URL | str, + *, + cookies = None, + proxy = None, + timeout = DEFAULT_TIMEOUT_CONFIG, + verify = True, + trust_env: bool = True, + **kwargs): + transport = HTTPProxyTransport(proxy=proxy) + with Client( + cookies=cookies, + verify=verify, + timeout=timeout, + trust_env=trust_env, + mounts={'http://': transport, 'https://': transport} + ) as client: + with client.stream(method=method, url=url, **kwargs) as response: + yield response \ No newline at end of file diff --git a/python_proxy_headers/httpx_proxy.py b/python_proxy_headers/httpx_proxy.py index 430b5e3..796ebdf 100644 --- a/python_proxy_headers/httpx_proxy.py +++ b/python_proxy_headers/httpx_proxy.py @@ -1,389 +1,73 @@ -from contextlib import contextmanager -from httpcore._sync.http_proxy import HTTPProxy, TunnelHTTPConnection, merge_headers, logger -from httpcore._sync.http11 import HTTP11Connection -from httpcore._async.http_proxy import AsyncHTTPProxy, AsyncTunnelHTTPConnection -from httpcore._async.http11 import AsyncHTTP11Connection -from httpcore._models import URL, Request -from httpcore._exceptions import ProxyError -from httpcore._ssl import default_ssl_context -from httpcore._trace import Trace -from httpx import AsyncHTTPTransport, HTTPTransport, Client -from httpx._config import DEFAULT_LIMITS, DEFAULT_TIMEOUT_CONFIG, Proxy, create_ssl_context -from .header_utils import filter_connect_headers - -class ProxyTunnelHTTPConnection(TunnelHTTPConnection): - # Unfortunately the only way to get connect_response.headers into the Response - # is to override this whole method - def handle_request(self, request): - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("connect", None) - - with self._connect_lock: - if not self._connected: - target = b"%b:%d" % (self._remote_origin.host, self._remote_origin.port) - - connect_url = URL( - scheme=self._proxy_origin.scheme, - host=self._proxy_origin.host, - port=self._proxy_origin.port, - target=target, - ) - connect_headers = merge_headers( - [(b"Host", target), (b"Accept", b"*/*")], self._proxy_headers - ) - connect_request = Request( - method=b"CONNECT", - url=connect_url, - headers=connect_headers, - extensions=request.extensions, - ) - connect_response = self._connection.handle_request( - connect_request - ) - - if connect_response.status < 200 or connect_response.status > 299: - reason_bytes = connect_response.extensions.get("reason_phrase", b"") - reason_str = reason_bytes.decode("ascii", errors="ignore") - msg = "%d %s" % (connect_response.status, reason_str) - self._connection.close() - raise ProxyError(msg) - - stream = connect_response.extensions["network_stream"] - - # Upgrade the stream to SSL - ssl_context = ( - default_ssl_context() - if self._ssl_context is None - else self._ssl_context - ) - alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] - ssl_context.set_alpn_protocols(alpn_protocols) - - kwargs = { - "ssl_context": ssl_context, - "server_hostname": self._remote_origin.host.decode("ascii"), - "timeout": timeout, - } - with Trace("start_tls", logger, request, kwargs) as trace: - stream = stream.start_tls(**kwargs) - trace.return_value = stream - - # Determine if we should be using HTTP/1.1 or HTTP/2 - ssl_object = stream.get_extra_info("ssl_object") - http2_negotiated = ( - ssl_object is not None - and ssl_object.selected_alpn_protocol() == "h2" - ) - - # Create the HTTP/1.1 or HTTP/2 connection - if http2_negotiated or (self._http2 and not self._http1): - from httpcore._sync.http2 import HTTP2Connection - - self._connection = HTTP2Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - else: - self._connection = HTTP11Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - - self._connect_response_headers = connect_response.headers - self._connected = True - # Merge safe CONNECT headers without overwriting origin values - response = self._connection.handle_request(request) - response.headers = merge_headers( - response.headers, - filter_connect_headers( - response.headers, getattr(self, "_connect_response_headers", ()) - ), - ) - return response - -class AsyncProxyTunnelHTTPConnection(AsyncTunnelHTTPConnection): - async def handle_async_request(self, request): - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("connect", None) - - async with self._connect_lock: - if not self._connected: - target = b"%b:%d" % (self._remote_origin.host, self._remote_origin.port) - - connect_url = URL( - scheme=self._proxy_origin.scheme, - host=self._proxy_origin.host, - port=self._proxy_origin.port, - target=target, - ) - connect_headers = merge_headers( - [(b"Host", target), (b"Accept", b"*/*")], self._proxy_headers - ) - connect_request = Request( - method=b"CONNECT", - url=connect_url, - headers=connect_headers, - extensions=request.extensions, - ) - connect_response = await self._connection.handle_async_request( - connect_request - ) - - if connect_response.status < 200 or connect_response.status > 299: - reason_bytes = connect_response.extensions.get("reason_phrase", b"") - reason_str = reason_bytes.decode("ascii", errors="ignore") - msg = "%d %s" % (connect_response.status, reason_str) - await self._connection.aclose() - raise ProxyError(msg) - - stream = connect_response.extensions["network_stream"] - - # Upgrade the stream to SSL - ssl_context = ( - default_ssl_context() - if self._ssl_context is None - else self._ssl_context - ) - alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] - ssl_context.set_alpn_protocols(alpn_protocols) - - kwargs = { - "ssl_context": ssl_context, - "server_hostname": self._remote_origin.host.decode("ascii"), - "timeout": timeout, - } - async with Trace("start_tls", logger, request, kwargs) as trace: - stream = await stream.start_tls(**kwargs) - trace.return_value = stream - - # Determine if we should be using HTTP/1.1 or HTTP/2 - ssl_object = stream.get_extra_info("ssl_object") - http2_negotiated = ( - ssl_object is not None - and ssl_object.selected_alpn_protocol() == "h2" - ) - - # Create the HTTP/1.1 or HTTP/2 connection - if http2_negotiated or (self._http2 and not self._http1): - from httpcore._async.http2 import AsyncHTTP2Connection - - self._connection = AsyncHTTP2Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - else: - self._connection = AsyncHTTP11Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - - self._connect_response_headers = connect_response.headers - self._connected = True - # Merge safe CONNECT headers without overwriting origin values - response = await self._connection.handle_async_request(request) - response.headers = merge_headers( - response.headers, - filter_connect_headers( - response.headers, getattr(self, "_connect_response_headers", ()) - ), - ) - return response - -class HTTPProxyHeaders(HTTPProxy): - def create_connection(self, origin): - if origin.scheme == b"http": - return super().create_connection(origin) - return ProxyTunnelHTTPConnection( - proxy_origin=self._proxy_url.origin, - proxy_headers=self._proxy_headers, - remote_origin=origin, - ssl_context=self._ssl_context, - proxy_ssl_context=self._proxy_ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - network_backend=self._network_backend, - ) - -class AsyncHTTPProxyHeaders(AsyncHTTPProxy): - def create_connection(self, origin): - if origin.scheme == b"http": - return super().create_connection(origin) - return AsyncProxyTunnelHTTPConnection( - proxy_origin=self._proxy_url.origin, - proxy_headers=self._proxy_headers, - remote_origin=origin, - ssl_context=self._ssl_context, - proxy_ssl_context=self._proxy_ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - network_backend=self._network_backend, - ) - -# class ProxyConnectionPool(ConnectionPool): -# def create_connection(self, origin): -# if self._proxy is not None: -# if self._proxy.url.scheme in (b"socks5", b"socks5h"): -# return super().create_connection(origin) -# elif origin.scheme == b"http": -# return super().create_connection(origin) - -# return ProxyTunnelHTTPConnection( -# proxy_origin=self._proxy.url.origin, -# proxy_headers=self._proxy.headers, -# proxy_ssl_context=self._proxy.ssl_context, -# remote_origin=origin, -# ssl_context=self._ssl_context, -# keepalive_expiry=self._keepalive_expiry, -# http1=self._http1, -# http2=self._http2, -# network_backend=self._network_backend, -# ) - -# return super().create_connection(origin) - -class HTTPProxyTransport(HTTPTransport): - def __init__( - self, - verify = True, - cert = None, - trust_env: bool = True, - http1: bool = True, - http2: bool = False, - limits = DEFAULT_LIMITS, - proxy = None, - uds: str | None = None, - local_address: str | None = None, - retries: int = 0, - socket_options = None, - ) -> None: - proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy - ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) - - if proxy and proxy.url.scheme in ("http", "https"): - self._pool = HTTPProxyHeaders( - proxy_url=URL( - scheme=proxy.url.raw_scheme, - host=proxy.url.raw_host, - port=proxy.url.port, - target=proxy.url.raw_path, - ), - proxy_auth=proxy.raw_auth, - proxy_headers=proxy.headers.raw, - ssl_context=ssl_context, - proxy_ssl_context=proxy.ssl_context, - max_connections=limits.max_connections, - max_keepalive_connections=limits.max_keepalive_connections, - keepalive_expiry=limits.keepalive_expiry, - http1=http1, - http2=http2, - socket_options=socket_options, - ) - else: - super().__init__(verify, cert, trust_env, http1, http2, limits, proxy, uds, local_address, retries, socket_options) - -class AsyncHTTPProxyTransport(AsyncHTTPTransport): - def __init__( - self, - verify = True, - cert = None, - trust_env: bool = True, - http1: bool = True, - http2: bool = False, - limits = DEFAULT_LIMITS, - proxy = None, - uds: str | None = None, - local_address: str | None = None, - retries: int = 0, - socket_options = None, - ) -> None: - proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy - ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) - - if proxy and proxy.url.scheme in ("http", "https"): - self._pool = AsyncHTTPProxyHeaders( - proxy_url=URL( - scheme=proxy.url.raw_scheme, - host=proxy.url.raw_host, - port=proxy.url.port, - target=proxy.url.raw_path, - ), - proxy_auth=proxy.raw_auth, - proxy_headers=proxy.headers.raw, - proxy_ssl_context=proxy.ssl_context, - ssl_context=ssl_context, - max_connections=limits.max_connections, - max_keepalive_connections=limits.max_keepalive_connections, - keepalive_expiry=limits.keepalive_expiry, - http1=http1, - http2=http2, - socket_options=socket_options, - ) - else: - super().__init__(verify, cert, trust_env, http1, http2, limits, proxy, uds, local_address, retries, socket_options) - -def request(method: str, - url: URL | str, - *, - cookies = None, - proxy = None, - timeout = DEFAULT_TIMEOUT_CONFIG, - verify = True, - trust_env: bool = True, - **kwargs): - transport = HTTPProxyTransport(proxy=proxy) - with Client( - cookies=cookies, - verify=verify, - timeout=timeout, - trust_env=trust_env, - mounts={'http://': transport, 'https://': transport} - ) as client: - return client.request(method=method, url=url, **kwargs) - -def get(*args, **kwargs): - return request('GET', *args, **kwargs) - -def options(*args, **kwargs): - return request('OPTIONS', *args, **kwargs) - -def head(*args, **kwargs): - return request('HEAD', *args, **kwargs) - -def post(*args, **kwargs): - return request('POST', *args, **kwargs) - -def put(*args, **kwargs): - return request('PUT', *args, **kwargs) - -def patch(*args, **kwargs): - return request('PATCH', *args, **kwargs) - -def delete(*args, **kwargs): - return request('DELETE', *args, **kwargs) - -@contextmanager -def stream(method: str, - url: URL | str, - *, - cookies = None, - proxy = None, - timeout = DEFAULT_TIMEOUT_CONFIG, - verify = True, - trust_env: bool = True, - **kwargs): - transport = HTTPProxyTransport(proxy=proxy) - with Client( - cookies=cookies, - verify=verify, - timeout=timeout, - trust_env=trust_env, - mounts={'http://': transport, 'https://': transport} - ) as client: - with client.stream(method=method, url=url, **kwargs) as response: - yield response \ No newline at end of file +"""Send and read CONNECT proxy headers with httpx 0.28.x. + +httpx 1.0 removes httpcore and proxy support, so this module does not import +httpcore when httpx>=1 is installed. Importing still succeeds: it warns, and +the helpers raise :class:`UnsupportedHttpxVersion`. + +The ``[httpx]`` extra is ``httpx>=0.28,<1``. +""" + +from __future__ import annotations + +import warnings +from typing import Any, Dict + +from .httpx_support import ( + HttpxStatus, + UnsupportedHttpxVersion, + classify_httpx, + install_message, + load_backend, + unavailable_namespace, + BACKEND_EXPORTS, +) + +__all__ = [ + "AsyncHTTPProxyHeaders", + "AsyncHTTPProxyTransport", + "AsyncProxyTunnelHTTPConnection", + "HTTPProxyHeaders", + "HTTPProxyTransport", + "HTTPX_STATUS", + "HttpxStatus", + "ProxyTunnelHTTPConnection", + "UnsupportedHttpxVersion", + "delete", + "get", + "head", + "options", + "patch", + "post", + "put", + "request", + "stream", +] + + +def _export(namespace: Dict[str, Any]) -> None: + for name in BACKEND_EXPORTS: + globals()[name] = namespace[name] + + +def _activate() -> HttpxStatus: + """Bind the real backend or the raising stand-ins. Never raises ImportError.""" + status = classify_httpx() + if status.kind == "supported": + try: + namespace = load_backend() + except ImportError as exc: + status = HttpxStatus("broken", status.version, detail=str(exc)) + message = install_message(status) + warnings.warn(message, UserWarning, stacklevel=3) + namespace = unavailable_namespace(message) + _export(namespace) + return status + + message = install_message(status) + if status.kind == "unsupported": + warnings.warn(message, UserWarning, stacklevel=3) + _export(unavailable_namespace(message)) + return status + + +HTTPX_STATUS = _activate() diff --git a/python_proxy_headers/httpx_support.py b/python_proxy_headers/httpx_support.py new file mode 100644 index 0000000..2f19a21 --- /dev/null +++ b/python_proxy_headers/httpx_support.py @@ -0,0 +1,213 @@ +"""Detect whether the installed httpx can load the CONNECT adapter. + +httpx 0.28.x (httpcore) is supported. httpx 1.0 is a rewrite without httpcore, +without ``HTTPTransport`` / ``AsyncHTTPTransport``, and without proxy support +(https://github.com/encode/httpx/issues/947). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Dict, Optional + +# Kept in sync with [project.optional-dependencies] httpx in pyproject.toml. +SUPPORTED_HTTPX_SPEC = "httpx>=0.28,<1" +HTTPX_PROXY_ISSUE = "https://github.com/encode/httpx/issues/947" + +BACKEND_EXPORTS = ( + "ProxyTunnelHTTPConnection", + "AsyncProxyTunnelHTTPConnection", + "HTTPProxyHeaders", + "AsyncHTTPProxyHeaders", + "HTTPProxyTransport", + "AsyncHTTPProxyTransport", + "request", + "get", + "options", + "head", + "post", + "put", + "patch", + "delete", + "stream", +) + +_MISSING = object() + + +class UnsupportedHttpxVersion(RuntimeError): + """Raised when httpx cannot send or read CONNECT proxy headers. + + httpx 1.x is installed, httpx is missing, or the httpcore backend failed + to import. The message names the installed version and the supported range. + """ + + +@dataclass(frozen=True) +class HttpxStatus: + """What ``classify_httpx`` decided about the environment. + + ``kind`` is ``supported`` (httpx 0.x), ``unsupported`` (httpx 1 or newer), + ``missing`` (httpx is not installed), or ``broken`` (version looks like + 0.x but the httpcore backend did not import). + """ + + kind: str + version: Optional[str] = None + detail: Optional[str] = None + + +def httpx_major(version: Optional[str]) -> Optional[int]: + """Return the numeric major of an httpx version string. + + ``1.0.dev6`` is major 1 and ``0.28.1`` is major 0. Returns None when + ``version`` is empty or does not start with a number. + """ + if not version: + return None + token = version.strip().split(".", 1)[0] + digits = [] + for char in token: + if char.isdigit(): + digits.append(char) + else: + break + if not digits: + return None + return int("".join(digits)) + + +def _imported_version() -> Optional[str]: + try: + import httpx + except ImportError: + return None + version = getattr(httpx, "__version__", None) + if isinstance(version, str) and version: + return version + return None + + +def _metadata_version() -> Optional[str]: + try: + from importlib.metadata import PackageNotFoundError, version + except ImportError: # pragma: no cover - importlib.metadata is stdlib on 3.10+ + return None + try: + return version("httpx") + except PackageNotFoundError: + return None + + +def probe_installed_version() -> Optional[str]: + """Return the httpx version that ``import httpx`` will actually use. + + Distribution metadata can disagree with a module earlier on ``sys.path`` + (for example a shadowed 1.0 build while metadata still says 0.28). When + the majors differ, the imported module wins. + """ + imported = _imported_version() + meta = _metadata_version() + if imported and meta and httpx_major(imported) != httpx_major(meta): + return imported + return imported or meta + + +def classify_httpx(version: Any = _MISSING, *, probe: bool = True) -> HttpxStatus: + """Classify an httpx version string, or the version installed in this process. + + Pass ``probe=False`` and no version to classify a missing install without + touching the environment. + """ + if version is _MISSING: + version = probe_installed_version() if probe else None + if version is None: + return HttpxStatus("missing", None) + if not isinstance(version, str): + version = str(version) + major = httpx_major(version) + if major == 0: + return HttpxStatus("supported", version) + return HttpxStatus("unsupported", version) + + +def install_message(status: HttpxStatus) -> str: + """Explain why the httpx helpers cannot run, including the 1.0 plan pointer.""" + spec = SUPPORTED_HTTPX_SPEC + if status.kind == "missing": + return ( + "httpx is not installed. python-proxy-headers supports the httpx " + f"0.28 line only ({spec}). Install it with " + "`pip install 'python-proxy-headers[httpx]'`." + ) + if status.kind == "broken": + detail = f" ({status.detail})" if status.detail else "" + found = f"httpx {status.version}" if status.version else "httpx 0.x" + return ( + f"{found} looks like a supported 0.x release, but the httpcore " + f"CONNECT adapter failed to import{detail}. Reinstall with " + "`pip install 'python-proxy-headers[httpx]'` " + f"({spec})." + ) + found = f"httpx {status.version}" if status.version else "httpx 1.x" + return ( + f"{found} is installed, but python-proxy-headers only supports the " + f"httpx 0.28 line ({spec}). httpx 1.0 is a rewrite that drops httpcore " + "and does not support proxies " + f"({HTTPX_PROXY_ISSUE}); AsyncHTTPTransport and HTTPTransport are gone, " + "so the 0.x adapter is not loaded. Importing this module succeeds, and " + "calling the helpers raises UnsupportedHttpxVersion. " + "Install a compatible release with " + "`pip install 'python-proxy-headers[httpx]'`. " + "A 1.0 adapter is not included; see the httpx documentation for the " + "support options (stay on 0.28, wait for a public CONNECT hook, or add " + "a small backend once NetworkBackend can take an existing tunnel)." + ) + + +def load_backend() -> Dict[str, Any]: + """Import the httpcore implementation. + + Raises: + ImportError: httpx 0.x symbols or httpcore are not importable. + """ + from httpx import AsyncHTTPTransport, HTTPTransport # noqa: F401 + import httpcore # noqa: F401 + from . import _httpx_httpcore as backend + + namespace: Dict[str, Any] = {} + for name in BACKEND_EXPORTS: + namespace[name] = getattr(backend, name) + return namespace + + +def _make_raiser(message: str) -> Callable[..., Any]: + def _raise(*_args: Any, **_kwargs: Any) -> Any: + raise UnsupportedHttpxVersion(message) + + return _raise + + +def unavailable_namespace(message: str) -> Dict[str, Any]: + """Build stand-ins that raise ``UnsupportedHttpxVersion`` when used. + + The public names still import. Nothing here touches httpcore. + """ + namespace: Dict[str, Any] = {} + for name in BACKEND_EXPORTS: + if name[:1].isupper(): + raiser = _make_raiser(message) + + def _init(self: Any, *args: Any, _raiser: Callable[..., Any] = raiser, **kwargs: Any) -> None: + _raiser(*args, **kwargs) + + cls = type(name, (), {"__init__": _init, "__doc__": message}) + cls.__module__ = "python_proxy_headers.httpx_proxy" + namespace[name] = cls + else: + fn = _make_raiser(message) + fn.__name__ = name + fn.__doc__ = message + fn.__module__ = "python_proxy_headers.httpx_proxy" + namespace[name] = fn + return namespace diff --git a/test_client_internals.py b/test_client_internals.py new file mode 100644 index 0000000..4b5765f --- /dev/null +++ b/test_client_internals.py @@ -0,0 +1,753 @@ +#!/usr/bin/env python3 +"""Exercise private client hooks against a local CONNECT proxy. + +urllib3, httpx/httpcore, and aiohttp adapters override or copy private +methods. These tests call those paths and fail with the upstream symbol name +when a release moves them. No PROXY_URL or network access is required. + +httpx 1.x has no adapter. Importing ``httpx_proxy`` must not raise, and using +it must raise UnsupportedHttpxVersion. +""" + +from __future__ import annotations + +import asyncio +import importlib +import inspect +import os +import socket +import ssl +import subprocess +import sys +import tempfile +import textwrap +import threading +import unittest +from pathlib import Path +from typing import Any, Dict, List, Optional + +ROOT = Path(__file__).resolve().parent +MOCK_PROXY = ROOT / "tests" / "mock_connect_proxy.py" +FAKE_IP = "203.0.113.7" +SEND_HEADER = {"X-ProxyMesh-Country": "US"} + + +def _runtime() -> str: + """Describe the interpreters and client versions under test.""" + parts = [f"Python {sys.version.split()[0]}"] + for name in ("urllib3", "requests", "httpx", "httpcore", "aiohttp"): + try: + module = importlib.import_module(name) + except ImportError: + parts.append(f"{name} not-installed") + else: + parts.append(f"{name} {getattr(module, '__version__', '?')}") + return ", ".join(parts) + + +def _param_names(func: Any) -> List[str]: + return list(inspect.signature(func).parameters) + + +def _header(headers: Any, name: str) -> Optional[str]: + if headers is None: + return None + value = headers.get(name) + if value is None: + value = headers.get(name.lower()) + if isinstance(value, bytes): + return value.decode("latin-1") + if value is None: + return None + return str(value) + + +class HttpxGuardTests(unittest.TestCase): + """httpx 1.x must not crash the import, and the extra must stay on 0.28.""" + + def test_version_major(self) -> None: + from python_proxy_headers.httpx_support import httpx_major + + self.assertEqual(httpx_major("0.28.1"), 0) + self.assertEqual(httpx_major("1.0.dev6"), 1) + self.assertEqual(httpx_major("1.0.0rc1"), 1) + self.assertIsNone(httpx_major(None)) + self.assertIsNone(httpx_major("")) + + def test_classify_without_probing(self) -> None: + from python_proxy_headers.httpx_support import classify_httpx + + self.assertEqual(classify_httpx("0.28.1").kind, "supported") + self.assertEqual(classify_httpx("1.0.dev6").kind, "unsupported") + self.assertEqual(classify_httpx(probe=False).kind, "missing") + + def test_imported_version_wins_when_major_disagrees(self) -> None: + from unittest import mock + + from python_proxy_headers.httpx_support import classify_httpx + + with mock.patch( + "python_proxy_headers.httpx_support._imported_version", + return_value="1.0.dev6", + ), mock.patch( + "python_proxy_headers.httpx_support._metadata_version", + return_value="0.28.1", + ): + status = classify_httpx() + self.assertEqual(status.kind, "unsupported") + self.assertEqual(status.version, "1.0.dev6") + + def test_unsupported_message_names_issue_and_extra(self) -> None: + from python_proxy_headers.httpx_support import ( + HttpxStatus, + install_message, + ) + + text = install_message(HttpxStatus("unsupported", "1.0.dev6")) + self.assertIn("1.0.dev6", text) + self.assertIn("947", text) + self.assertIn("httpx>=0.28,<1", text) + self.assertIn("AsyncHTTPTransport", text) + missing = install_message(HttpxStatus("missing", None)) + self.assertIn("python-proxy-headers[httpx]", missing) + + def test_pyproject_pins_httpx_extra_and_changelog(self) -> None: + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + self.assertIn('httpx = ["httpx>=0.28,<1"]', text) + self.assertIn( + 'Changelog = "https://github.com/proxymesh/python-proxy-headers/blob/main/CHANGELOG.md"', + text, + ) + self.assertNotIn("commits/main", text) + for keyword in ("proxy-headers", "https-proxy", "proxymesh"): + self.assertIn(f'"{keyword}"', text) + self.assertIn("Topic :: Internet :: Proxy Servers", text) + self.assertIn('requires-python = ">=3.10"', text) + + def test_import_matches_installed_httpx(self) -> None: + """Importing httpx_proxy succeeds for both 0.28.x and 1.x.""" + code = textwrap.dedent( + """ + import warnings + warnings.simplefilter("always") + import python_proxy_headers.httpx_proxy as proxy + + print("STATUS", proxy.HTTPX_STATUS.kind, proxy.HTTPX_STATUS.version or "-") + if proxy.HTTPX_STATUS.kind == "supported": + print("READY", callable(proxy.get)) + else: + try: + proxy.get("https://example.invalid", proxy="http://127.0.0.1:9") + print("NO_RAISE") + except proxy.UnsupportedHttpxVersion as exc: + text = str(exc) + print("RAISED") + print("HAS947", "947" in text) + print("HAS_EXTRA", "python-proxy-headers[httpx]" in text) + """ + ) + completed = _run_python(code, extra_pythonpath=[]) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertNotIn("ImportError", completed.stderr) + kind = _status_kind(completed.stdout) + if kind == "supported": + self.assertIn("READY True", completed.stdout) + self.assertNotIn("drops httpcore", completed.stderr) + elif kind == "unsupported": + self.assertIn("RAISED", completed.stdout) + self.assertIn("HAS947 True", completed.stdout) + self.assertIn("947", completed.stderr) + self.assertIn("UserWarning", completed.stderr) + elif kind == "missing": + self.assertIn("RAISED", completed.stdout) + self.assertIn("HAS_EXTRA True", completed.stdout) + self.assertNotIn("UserWarning", completed.stderr) + else: + self.fail(f"unexpected httpx status {kind}\n{completed.stdout}\n{completed.stderr}") + + def test_shadowed_httpx_1_dev6_does_not_crash_on_import(self) -> None: + """A 1.0.dev6 module earlier on sys.path must not raise ImportError.""" + fake = Path(tempfile.mkdtemp()) + package = fake / "httpx" + package.mkdir() + (package / "__init__.py").write_text('__version__ = "1.0.dev6"\n', encoding="utf-8") + code = textwrap.dedent( + """ + import warnings + warnings.simplefilter("always") + import python_proxy_headers.httpx_proxy as proxy + + print("STATUS", proxy.HTTPX_STATUS.kind, proxy.HTTPX_STATUS.version or "-") + for label, call in ( + ("get", lambda: proxy.get("https://example.invalid", proxy="http://127.0.0.1:9")), + ("transport", lambda: proxy.HTTPProxyTransport(proxy="http://127.0.0.1:9")), + ("async-transport", lambda: proxy.AsyncHTTPProxyTransport(proxy="http://127.0.0.1:9")), + ("stream", lambda: proxy.stream("GET", "https://example.invalid", proxy="http://127.0.0.1:9")), + ): + try: + call() + print("NO_RAISE", label) + except proxy.UnsupportedHttpxVersion: + print("RAISED", label) + """ + ) + completed = _run_python(code, extra_pythonpath=[fake]) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertIn("STATUS unsupported 1.0.dev6", completed.stdout) + for label in ("get", "transport", "async-transport", "stream"): + self.assertIn(f"RAISED {label}", completed.stdout) + self.assertNotIn("NO_RAISE", completed.stdout) + self.assertNotIn("ImportError", completed.stderr) + self.assertIn("1.0.dev6", completed.stderr) + self.assertIn("947", completed.stderr) + + +class Urllib3ContractTests(unittest.TestCase): + def test_prepare_proxy_signature_still_matches(self) -> None: + import urllib3 + from urllib3.connectionpool import HTTPSConnectionPool + + from python_proxy_headers.urllib3_proxy_manager import HTTPSProxyConnectionPool + + base = _param_names(HTTPSConnectionPool._prepare_proxy) + ours = _param_names(HTTPSProxyConnectionPool._prepare_proxy) + self.assertIsNot( + HTTPSProxyConnectionPool._prepare_proxy, + HTTPSConnectionPool._prepare_proxy, + ) + if base != ours: + self.fail( + "urllib3 HTTPSConnectionPool._prepare_proxy signature changed. " + f"python_proxy_headers override parameters are {ours}; " + f"upstream parameters are now {base}. urllib3 {urllib3.__version__}. " + "HTTPSProxyConnectionPool._prepare_proxy in urllib3_proxy_manager.py " + "needs an update before CONNECT header capture keeps working." + ) + + def test_tunnel_and_read_headers_hooks(self) -> None: + import http.client + import urllib3 + from python_proxy_headers.urllib3_proxy_manager import HTTPSProxyConnection + + if not hasattr(http.client, "_read_headers"): + self.fail( + "http.client._read_headers is gone. urllib3_proxy_manager.py " + "imports it at module level (and the <3.12 _tunnel copy calls it). " + f"{_runtime()}." + ) + if sys.version_info < (3, 12): + if "_tunnel" not in HTTPSProxyConnection.__dict__: + self.fail( + "HTTPSProxyConnection._tunnel is not defined on Python " + f"{sys.version.split()[0]}. The <3.12 branch copies " + "HTTPConnection._tunnel and reads headers with " + f"http.client._read_headers. urllib3 {urllib3.__version__}." + ) + if "get_proxy_response_headers" not in HTTPSProxyConnection.__dict__: + self.fail( + "HTTPSProxyConnection.get_proxy_response_headers is missing " + "on Python < 3.12, so _prepare_proxy cannot read CONNECT headers." + ) + else: + if not hasattr(http.client.HTTPConnection, "get_proxy_response_headers"): + self.fail( + "HTTPConnection.get_proxy_response_headers is missing. On " + "Python 3.12+, HTTPSProxyConnectionPool._prepare_proxy reads " + "CONNECT headers through that stdlib method. " + f"{_runtime()}." + ) + if not hasattr(http.client.HTTPConnection, "_tunnel"): + self.fail( + "HTTPConnection._tunnel is missing. urllib3 still opens the " + f"CONNECT tunnel through it. {_runtime()}." + ) + + +class HttpxContractTests(unittest.TestCase): + def test_copied_handle_request_still_matches_httpcore(self) -> None: + self._require_httpx_0() + import httpcore + import httpx + from httpcore._async.http_proxy import AsyncHTTPProxy, AsyncTunnelHTTPConnection + from httpcore._sync.http_proxy import HTTPProxy, TunnelHTTPConnection, merge_headers + + from python_proxy_headers.httpx_proxy import ( + AsyncHTTPProxyHeaders, + AsyncProxyTunnelHTTPConnection, + HTTPProxyHeaders, + ProxyTunnelHTTPConnection, + ) + + required = ( + "proxy_origin", + "remote_origin", + "ssl_context", + "proxy_ssl_context", + "proxy_headers", + "keepalive_expiry", + "http1", + "http2", + "network_backend", + ) + params = inspect.signature(TunnelHTTPConnection.__init__).parameters + missing = [name for name in required if name not in params] + if missing or not hasattr(TunnelHTTPConnection, "handle_request"): + self.fail( + "httpcore TunnelHTTPConnection no longer matches the copied " + "ProxyTunnelHTTPConnection.handle_request. Missing " + f"{missing or ['handle_request']}. httpcore {httpcore.__version__}, " + f"httpx {httpx.__version__}." + ) + if not hasattr(AsyncTunnelHTTPConnection, "handle_async_request"): + self.fail( + "httpcore AsyncTunnelHTTPConnection.handle_async_request is gone. " + f"httpcore {httpcore.__version__}." + ) + self.assertTrue(callable(merge_headers)) + self.assertTrue(hasattr(HTTPProxy, "create_connection")) + self.assertTrue(hasattr(AsyncHTTPProxy, "create_connection")) + self.assertIsNot( + ProxyTunnelHTTPConnection.handle_request, + TunnelHTTPConnection.handle_request, + ) + self.assertIsNot( + AsyncProxyTunnelHTTPConnection.handle_async_request, + AsyncTunnelHTTPConnection.handle_async_request, + ) + self.assertIsNot(HTTPProxyHeaders.create_connection, HTTPProxy.create_connection) + self.assertIsNot( + AsyncHTTPProxyHeaders.create_connection, + AsyncHTTPProxy.create_connection, + ) + + def _require_httpx_0(self) -> None: + from python_proxy_headers.httpx_proxy import HTTPX_STATUS + + if HTTPX_STATUS.kind != "supported": + self.skipTest( + "httpcore handle_request checks need httpx 0.28.x; " + f"found {HTTPX_STATUS.kind} {HTTPX_STATUS.version}. " + "httpx 1.x has no CONNECT adapter (encode/httpx#947)." + ) + + +class AiohttpContractTests(unittest.TestCase): + def test_create_proxy_connection_signature_and_helpers(self) -> None: + import aiohttp + from aiohttp.connector import TCPConnector + + from python_proxy_headers.aiohttp_proxy import ProxyTCPConnector + + base = _param_names(TCPConnector._create_proxy_connection) + ours = _param_names(ProxyTCPConnector._create_proxy_connection) + if base != ours: + self.fail( + "aiohttp TCPConnector._create_proxy_connection signature changed. " + f"ProxyTCPConnector copy parameters are {ours}; upstream parameters " + f"are now {base}. aiohttp {aiohttp.__version__}. The copy in " + "aiohttp_proxy.py has to be updated or CONNECT headers will not " + "be captured." + ) + required = ( + "_fail_on_no_start_tls", + "_loop_supports_start_tls", + "_create_direct_connection", + "_warn_about_tls_in_tls", + "_get_ssl_context", + "_wrap_existing_connection", + "_start_tls_connection", + ) + missing = [name for name in required if not hasattr(TCPConnector, name)] + if missing: + self.fail( + "aiohttp TCPConnector dropped helpers used by the copied " + f"_create_proxy_connection: {', '.join(missing)}. " + f"aiohttp {aiohttp.__version__}." + ) + + async def _instance_attrs() -> List[str]: + connector = ProxyTCPConnector() + try: + return [ + name + for name in ("_timeout_ceil_threshold", "_factory") + if not hasattr(connector, name) + ] + finally: + await connector.close() + + absent = asyncio.run(_instance_attrs()) + if absent: + self.fail( + "ProxyTCPConnector instances no longer have " + f"{', '.join(absent)}, which the copied _create_proxy_connection " + f"reads. aiohttp {aiohttp.__version__}." + ) + + +class _LocalTunnel: + """TLS origin plus tests/mock_connect_proxy.py on 127.0.0.1.""" + + def __init__(self) -> None: + self.proxy_port = 0 + self.origin_port = 0 + self.logs: List[str] = [] + self._proc: Optional[subprocess.Popen[str]] = None + self._origin: Optional[socket.socket] = None + self._stop = threading.Event() + self._tmpdir = tempfile.mkdtemp() + + @property + def proxy_url(self) -> str: + return f"http://127.0.0.1:{self.proxy_port}" + + @property + def origin_url(self) -> str: + return f"https://127.0.0.1:{self.origin_port}/" + + def start(self) -> None: + key = os.path.join(self._tmpdir, "key.pem") + cert = os.path.join(self._tmpdir, "cert.pem") + subprocess.check_call( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-keyout", + key, + "-out", + cert, + "-days", + "1", + "-nodes", + "-subj", + "/CN=127.0.0.1", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert, key) + origin = socket.socket() + origin.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + origin.bind(("127.0.0.1", 0)) + origin.listen(16) + self._origin = origin + self.origin_port = origin.getsockname()[1] + threading.Thread( + target=self._serve_origin, args=(context,), daemon=True + ).start() + + self._proc = subprocess.Popen( + [sys.executable, str(MOCK_PROXY), "0"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert self._proc.stdout is not None + assert self._proc.stderr is not None + threading.Thread(target=self._read_logs, args=(self._proc.stderr,), daemon=True).start() + line = self._proc.stdout.readline().strip() + if not line.startswith("PORT "): + self.stop() + raise RuntimeError( + "mock_connect_proxy did not report a port " + f"({line!r}). logs={''.join(self.logs)!r}" + ) + self.proxy_port = int(line.split()[1]) + + def stop(self) -> None: + self._stop.set() + if self._origin is not None: + try: + self._origin.close() + except OSError: + pass + if self._proc is not None and self._proc.poll() is None: + self._proc.terminate() + try: + self._proc.wait(timeout=3) + except subprocess.TimeoutExpired: + self._proc.kill() + + def _read_logs(self, stream: Any) -> None: + for line in stream: + self.logs.append(line) + + def _serve_origin(self, context: ssl.SSLContext) -> None: + assert self._origin is not None + self._origin.settimeout(0.5) + while not self._stop.is_set(): + try: + conn, _addr = self._origin.accept() + except socket.timeout: + continue + except OSError: + return + threading.Thread( + target=self._handle_origin, args=(context, conn), daemon=True + ).start() + + def _handle_origin(self, context: ssl.SSLContext, conn: socket.socket) -> None: + try: + tls = context.wrap_socket(conn, server_side=True) + tls.settimeout(5) + data = b"" + while b"\r\n\r\n" not in data: + chunk = tls.recv(4096) + if not chunk: + break + data += chunk + body = b"origin-body" + tls.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: " + str(len(body)).encode() + b"\r\n" + b"Connection: close\r\n\r\n" + body + ) + try: + tls.shutdown(socket.SHUT_RDWR) + except OSError: + pass + tls.close() + except Exception: + try: + conn.close() + except OSError: + pass + + +class MockProxyFunctionalTests(unittest.TestCase): + """End-to-end CONNECT through the private hooks. Fails if they drift.""" + + tunnel: _LocalTunnel + + @classmethod + def setUpClass(cls) -> None: + import urllib3 + + urllib3.disable_warnings() + cls.tunnel = _LocalTunnel() + cls.tunnel.start() + + @classmethod + def tearDownClass(cls) -> None: + tunnel = getattr(cls, "tunnel", None) + if tunnel is not None: + tunnel.stop() + + def test_urllib3_prepare_proxy_reads_connect_headers(self) -> None: + import urllib3 + from python_proxy_headers.urllib3_proxy_manager import proxy_from_url + + context = ssl._create_unverified_context() + try: + with proxy_from_url( + self.tunnel.proxy_url, + proxy_headers=SEND_HEADER, + ssl_context=context, + retries=False, + ) as manager: + response = manager.request("GET", self.tunnel.origin_url, timeout=8.0) + except Exception as exc: + self.fail(self._hook("urllib3", "HTTPSProxyConnectionPool._prepare_proxy and HTTPConnection._tunnel / http.client._read_headers", exc)) + self.assertEqual(response.status, 200, _runtime()) + self.assertEqual(response.data, b"origin-body") + self._assert_merged( + response.headers, + "urllib3", + "HTTPSProxyConnectionPool._prepare_proxy", + urllib3.__version__, + ) + + def test_httpx_handle_request_merges_connect_headers(self) -> None: + self._require_httpx_0() + import httpcore + import httpx + from python_proxy_headers.httpx_proxy import ( + AsyncHTTPProxyTransport, + HTTPProxyTransport, + ) + + proxy = httpx.Proxy(self.tunnel.proxy_url, headers=SEND_HEADER) + try: + transport = HTTPProxyTransport(proxy=proxy, verify=False) + with httpx.Client( + mounts={"https://": transport}, + verify=False, + timeout=8.0, + ) as client: + response = client.get(self.tunnel.origin_url) + except Exception as exc: + self.fail( + self._hook( + "httpx", + "ProxyTunnelHTTPConnection.handle_request (copy of httpcore TunnelHTTPConnection.handle_request)", + exc, + ) + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.text, "origin-body") + self._assert_merged( + response.headers, + "httpx", + "ProxyTunnelHTTPConnection.handle_request", + f"httpx {httpx.__version__} / httpcore {httpcore.__version__}", + ) + + async def _async_get() -> Any: + async_proxy = httpx.Proxy(self.tunnel.proxy_url, headers=SEND_HEADER) + async_transport = AsyncHTTPProxyTransport(proxy=async_proxy, verify=False) + async with httpx.AsyncClient( + mounts={"https://": async_transport}, + verify=False, + timeout=8.0, + ) as client: + return await client.get(self.tunnel.origin_url) + + try: + async_response = asyncio.run(_async_get()) + except Exception as exc: + self.fail( + self._hook( + "httpx", + "AsyncProxyTunnelHTTPConnection.handle_async_request", + exc, + ) + ) + self._assert_merged( + async_response.headers, + "httpx", + "AsyncProxyTunnelHTTPConnection.handle_async_request", + f"httpx {httpx.__version__} / httpcore {httpcore.__version__}", + ) + + def test_aiohttp_create_proxy_connection_reads_connect_headers(self) -> None: + import aiohttp + from python_proxy_headers.aiohttp_proxy import ProxyClientSession + + async def _get() -> Dict[str, Optional[str]]: + timeout = aiohttp.ClientTimeout(total=8) + async with ProxyClientSession(timeout=timeout) as session: + async with session.get( + self.tunnel.origin_url, + proxy=self.tunnel.proxy_url, + proxy_headers=SEND_HEADER, + ssl=False, + ) as response: + body = await response.text() + return { + "status": str(response.status), + "body": body, + "ip": _header(response.headers, "X-ProxyMesh-IP"), + "echo": _header(response.headers, "X-ProxyMesh-Country-Echo"), + } + + try: + result = asyncio.run(_get()) + except Exception as exc: + self.fail( + self._hook( + "aiohttp", + "ProxyTCPConnector._create_proxy_connection (copy of TCPConnector._create_proxy_connection)", + exc, + ) + ) + self.assertEqual(result["status"], "200") + self.assertEqual(result["body"], "origin-body") + self.assertEqual( + result["ip"], + FAKE_IP, + "aiohttp copied _create_proxy_connection did not surface " + f"X-ProxyMesh-IP. aiohttp {aiohttp.__version__}. {_runtime()}. " + f"proxy log: {''.join(self.tunnel.logs)!r}", + ) + self.assertEqual(result["echo"], "US") + + def _require_httpx_0(self) -> None: + from python_proxy_headers.httpx_proxy import HTTPX_STATUS + + if HTTPX_STATUS.kind != "supported": + self.skipTest( + "httpx CONNECT checks need httpx 0.28.x; " + f"found {HTTPX_STATUS.kind} {HTTPX_STATUS.version}." + ) + + def _hook(self, library: str, hook: str, exc: BaseException) -> str: + return ( + f"{library} private hook failed ({hook}). A method this package " + f"overrides or copies no longer matches the installed release: " + f"{type(exc).__name__}: {exc}. {_runtime()}. " + f"proxy log: {''.join(self.tunnel.logs)!r}" + ) + + def _assert_merged(self, headers: Any, library: str, hook: str, version: str) -> None: + ip = _header(headers, "X-ProxyMesh-IP") + echo = _header(headers, "X-ProxyMesh-Country-Echo") + self.assertEqual( + ip, + FAKE_IP, + f"{library} {hook} did not merge X-ProxyMesh-IP from the CONNECT " + f"response ({version}). {_runtime()}. " + f"proxy log: {''.join(self.tunnel.logs)!r}", + ) + self.assertEqual( + echo, + "US", + f"{library} {hook} did not echo X-ProxyMesh-Country onto the " + f"CONNECT response ({version}). The request header never reached " + f"the proxy, or the response header was dropped. {_runtime()}.", + ) + + +class OptionalClientImportTests(unittest.TestCase): + """Import adapters for clients that are installed. Skip when they are not.""" + + def test_pycurl_module_imports(self) -> None: + self._import_pair("pycurl", "python_proxy_headers.pycurl_proxy") + + def test_cloudscraper_module_imports(self) -> None: + self._import_pair("cloudscraper", "python_proxy_headers.cloudscraper_proxy") + + def test_autoscraper_module_imports(self) -> None: + self._import_pair("autoscraper", "python_proxy_headers.autoscraper_proxy") + + def _import_pair(self, distribution: str, module_name: str) -> None: + try: + importlib.import_module(distribution) + except ImportError: + self.skipTest(f"{distribution} is not installed") + importlib.import_module(module_name) + + +def _run_python(code: str, extra_pythonpath: List[Path]) -> subprocess.CompletedProcess[str]: + script = Path(tempfile.mkdtemp()) / "check_httpx_import.py" + script.write_text(code, encoding="utf-8") + parts = [str(path) for path in extra_pythonpath] + parts.append(str(ROOT)) + existing = os.environ.get("PYTHONPATH") + if existing: + parts.append(existing) + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join(parts) + return subprocess.run( + [sys.executable, str(script)], + env=env, + text=True, + capture_output=True, + timeout=30, + check=False, + ) + + +def _status_kind(stdout: str) -> str: + for line in stdout.splitlines(): + if line.startswith("STATUS "): + return line.split()[1] + return "" + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/mock_connect_proxy.py b/tests/mock_connect_proxy.py new file mode 100644 index 0000000..56ef6bd --- /dev/null +++ b/tests/mock_connect_proxy.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Minimal CONNECT proxy for local adapter tests. No credentials. + +Replies to CONNECT with ``HTTP/1.1 200 Connection established``, +``X-ProxyMesh-IP``, and an echo of each ``X-ProxyMesh-*`` request header, +then pipes bytes. Plain HTTP absolute-URI requests are forwarded with +``X-ProxyMesh-IP`` added. + +Usage: + python3 tests/mock_connect_proxy.py [port] + +``port`` defaults to 18080. Use ``0`` to bind an ephemeral port. The chosen +port is written to stdout as ``PORT `` before the accept loop starts. +""" + +from __future__ import annotations + +import select +import socket +import sys +import threading + +FAKE_IP = "203.0.113.7" + +def pipe(a, b, label=""): + socks = [a, b] + n = [0, 0] + try: + while True: + r, _, _ = select.select(socks, [], [], 60) + if not r: + return + for s in r: + data = s.recv(65536) + if not data: + return + n[0 if s is a else 1] += len(data) + (b if s is a else a).sendall(data) + except OSError: + pass + finally: + sys.stderr.write(f"[proxy] tunnel {label} closed: client->up {n[0]}B, up->client {n[1]}B\n") + a.close(); b.close() + +def read_head(c): + buf = b"" + while b"\r\n\r\n" not in buf: + d = c.recv(4096) + if not d: + break + buf += d + head, _, rest = buf.partition(b"\r\n\r\n") + return head, rest + +def handle(c): + try: + head, rest = read_head(c) + lines = head.decode("latin-1").split("\r\n") + method, target, _ = lines[0].split(" ", 2) + hdrs = [l.split(":", 1) for l in lines[1:] if ":" in l] + echo = "".join(f"{k.strip()}-Echo: {v.strip()}\r\n" for k, v in hdrs if k.lower().startswith("x-proxymesh")) + sys.stderr.write(f"[proxy] {method} {target} hdrs={[k for k,_ in hdrs]}\n") + if method == "CONNECT": + host, port = target.rsplit(":", 1) + up = socket.create_connection((host, int(port)), timeout=20) + c.sendall(("HTTP/1.1 200 Connection established\r\n" + f"X-ProxyMesh-IP: {FAKE_IP}\r\n{echo}\r\n").encode()) + if rest: + up.sendall(rest) + pipe(c, up, target) + else: + from urllib.parse import urlsplit + u = urlsplit(target) + up = socket.create_connection((u.hostname, u.port or 80), timeout=20) + path = (u.path or "/") + (("?" + u.query) if u.query else "") + out = [f"{method} {path} HTTP/1.1"] + [f"{k}:{v}" for k, v in hdrs if not k.lower().startswith("proxy-") and k.lower() != "connection"] + ["Connection: close"] + up.sendall(("\r\n".join(out) + "\r\n\r\n").encode("latin-1") + rest) + resp = b"" + while True: + d = up.recv(65536) + if not d: break + resp += d + h, _, body = resp.partition(b"\r\n\r\n") + c.sendall(h + f"\r\nX-ProxyMesh-IP: {FAKE_IP}".encode() + b"\r\n\r\n" + body) + c.close(); up.close() + except Exception as e: + sys.stderr.write(f"[proxy] error {e!r}\n") + try: c.close() + except OSError: pass + +def serve(port: int = 18080) -> None: + """Bind ``127.0.0.1:port`` and serve until the process is killed.""" + listener = socket.socket() + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", port)) + bound = listener.getsockname()[1] + listener.listen(128) + sys.stdout.write(f"PORT {bound}\n") + sys.stdout.flush() + sys.stderr.write(f"[proxy] listening on 127.0.0.1:{bound}\n") + while True: + client, _addr = listener.accept() + threading.Thread(target=handle, args=(client,), daemon=True).start() + + +def main(argv: list[str] | None = None) -> None: + """Parse ``argv`` and run :func:`serve`.""" + args = list(sys.argv[1:] if argv is None else argv) + port = int(args[0]) if args else 18080 + serve(port) + + +if __name__ == "__main__": + main()