From c810d6cd72a7e523794eff11d799dba24045489d Mon Sep 17 00:00:00 2001 From: Cursor Date: Fri, 23 Oct 2026 03:57:49 +0000 Subject: [PATCH] Guard httpx 1.x imports and test private client hooks httpx 1.0.dev6 drops httpcore and AsyncHTTPTransport. Importing the httpx adapter now warns and raises UnsupportedHttpxVersion instead of ImportError. Optional extras pin tested client ranges, and CI runs a Python matrix plus a local CONNECT mock. A weekly workflow checks latest and pre-release clients without gating pull requests. Co-authored-by: ProxyMesh AI --- .agents/skills/python-proxy-headers/SKILL.md | 2 + .github/workflows/client_compatibility.yml | 75 ++ .github/workflows/proxy_integration_tests.yml | 35 +- CHANGELOG.md | 27 + README.md | 24 +- docs/httpx.rst | 26 +- docs/index.rst | 13 +- pyproject.toml | 47 +- python_proxy_headers/_httpx_httpcore.py | 397 +++++++++ python_proxy_headers/httpx_proxy.py | 462 ++--------- python_proxy_headers/httpx_support.py | 213 +++++ test_client_internals.py | 753 ++++++++++++++++++ tests/mock_connect_proxy.py | 114 +++ 13 files changed, 1775 insertions(+), 413 deletions(-) create mode 100644 .github/workflows/client_compatibility.yml create mode 100644 CHANGELOG.md create mode 100644 python_proxy_headers/_httpx_httpcore.py create mode 100644 python_proxy_headers/httpx_support.py create mode 100644 test_client_internals.py create mode 100644 tests/mock_connect_proxy.py diff --git a/.agents/skills/python-proxy-headers/SKILL.md b/.agents/skills/python-proxy-headers/SKILL.md index 588131e..809aeab 100644 --- a/.agents/skills/python-proxy-headers/SKILL.md +++ b/.agents/skills/python-proxy-headers/SKILL.md @@ -45,6 +45,8 @@ with ProxySession(proxy_headers={'X-ProxyMesh-Country': 'US'}) as session: ### httpx +Requires `httpx>=0.28,<1` (`pip install 'python-proxy-headers[httpx]'`). httpx 1.0 drops httpcore and proxy support. Importing `httpx_proxy` still succeeds and warns; calling it raises `UnsupportedHttpxVersion`. + ```python from python_proxy_headers.httpx_proxy import get diff --git a/.github/workflows/client_compatibility.yml b/.github/workflows/client_compatibility.yml new file mode 100644 index 0000000..91c2626 --- /dev/null +++ b/.github/workflows/client_compatibility.yml @@ -0,0 +1,75 @@ +# Newest stable releases and pre-releases of every supported client. +# Not triggered on pull_request, so a red run is visible in Actions and via +# workflow_dispatch, and it is not a required PR check. +# Pre-release failures must stay failing (no continue-on-error). +# Runners are ubuntu-24.04. Do not switch these jobs to ubuntu-latest. + +name: Client compatibility + +on: + schedule: + - cron: "0 15 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + clients: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + python-version: ["3.12", "3.14"] + channel: [stable, pre] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies (pycurl) + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + + - name: Install package + run: | + python -m pip install --upgrade pip + pip install -e . + + - name: Install ${{ matrix.channel }} clients + run: | + set -euo pipefail + packages=(urllib3 requests aiohttp httpx pycurl cloudscraper autoscraper) + if [ "${{ matrix.channel }}" = "pre" ]; then + pip install --pre --upgrade "${packages[@]}" + else + pip install --upgrade "${packages[@]}" + fi + python - <<'PY' + import importlib.metadata as metadata + names = ( + "urllib3", + "requests", + "aiohttp", + "httpx", + "httpcore", + "pycurl", + "cloudscraper", + "autoscraper", + ) + for name in names: + try: + print(f"{name}=={metadata.version(name)}") + except metadata.PackageNotFoundError: + print(f"{name}==not-installed") + PY + + - name: Run header security unit tests + run: python test_header_security.py + + - name: Run local mock-proxy internal tests + run: python test_client_internals.py diff --git a/.github/workflows/proxy_integration_tests.yml b/.github/workflows/proxy_integration_tests.yml index 68ca318..c44cbab 100644 --- a/.github/workflows/proxy_integration_tests.yml +++ b/.github/workflows/proxy_integration_tests.yml @@ -1,7 +1,13 @@ -# Integration tests against a real proxy (PROXY_URL). Add repository secret PROXY_URL -# under Settings → Secrets and variables → Actions, then mark this workflow as a required -# status check under branch protection (pull_request events only receive secrets for PRs -# from the same repository, not from forks). +# Pull-request checks. +# - unit: Python matrix, compatible client extras, header tests, and a local +# CONNECT mock (no PROXY_URL). +# - integration: real proxy via the PROXY_URL secret. pull_request events only +# receive secrets for PRs from this repository, not from forks. +# +# Pre-release clients are not installed here. The weekly Client compatibility +# workflow runs `pip install --pre` and is not a pull_request check, so a +# pre-release failure cannot block a PR. +# Runners are ubuntu-24.04. Do not switch these jobs to ubuntu-latest. name: Proxy integration tests @@ -14,6 +20,10 @@ permissions: jobs: unit: runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -22,20 +32,22 @@ jobs: - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: "3.x" + python-version: ${{ matrix.python-version }} - name: Install system dependencies (pycurl) run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - - name: Install package and test dependencies + - name: Install package and compatible client extras run: | python -m pip install --upgrade pip - pip install -e . - pip install urllib3 requests aiohttp httpx pycurl + pip install -e ".[urllib3,requests,httpx,aiohttp,pycurl,cloudscraper,autoscraper]" - name: Run header security unit tests run: python test_header_security.py + - name: Run local mock-proxy internal tests + run: python test_client_internals.py + integration: runs-on: ubuntu-24.04 @@ -47,16 +59,15 @@ jobs: - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: "3.x" + python-version: "3.12" - name: Install system dependencies (pycurl) run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - - name: Install package and test dependencies + - name: Install package and compatible client extras run: | python -m pip install --upgrade pip - pip install -e . - pip install urllib3 requests aiohttp httpx cloudscraper autoscraper pycurl + pip install -e ".[urllib3,requests,httpx,aiohttp,pycurl,cloudscraper,autoscraper]" - name: Require PROXY_URL Actions secret env: diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8694e2f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,27 @@ +# Changelog + +## Unreleased + +### httpx 1.0 + +- Importing `python_proxy_headers.httpx_proxy` no longer raises `ImportError: cannot import name AsyncHTTPTransport` when httpx 1.x is installed. +- httpx 1.0.dev6 (2026-08-31) is a rewrite without httpcore, without `HTTPTransport` / `AsyncHTTPTransport`, and without proxy support ([encode/httpx#947](https://github.com/encode/httpx/issues/947)). The module warns at import time and raises `UnsupportedHttpxVersion` if you call it. +- The `[httpx]` extra is `httpx>=0.28,<1`. There is no 1.0 adapter in this change. Options are documented in `docs/httpx.rst`: stay on 0.28, wait for a public CONNECT hook, or add a small backend after `NetworkBackend` can take an existing tunnel. Vendoring httpcore is not one of them. + +### Tests + +- Contract checks and a local CONNECT mock (`tests/mock_connect_proxy.py`, no proxy credentials) exercise the private hooks in `urllib3_proxy_manager.py` (`_prepare_proxy`, `_tunnel`, `http.client._read_headers`), `httpx_proxy` (copied httpcore `handle_request` / `handle_async_request`), and `aiohttp_proxy.py` (copied `_create_proxy_connection`). They name the upstream symbol when it moves. + +### Packaging + +- Keywords, trove classifiers, and optional extras: `[urllib3]`, `[requests]`, `[httpx]`, `[aiohttp]`, `[pycurl]`, `[cloudscraper]`, `[autoscraper]`. +- `requires-python` is `>=3.10`. urllib3 2.8, requests 2.34, and aiohttp 3.14 all require 3.10+, and the httpx adapter already uses `str | None`. +- The changelog URL points at this file. + +## 0.2.3 + +CONNECT header injection hardening and safer merging of proxy response headers. See the [GitHub release](https://github.com/proxymesh/python-proxy-headers/releases/tag/v0.2.3). + +## Earlier + +[GitHub Releases](https://github.com/proxymesh/python-proxy-headers/releases). diff --git a/README.md b/README.md index dc08423..a9f7c6d 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,21 @@ This library solves both problems for popular Python HTTP libraries. pip install python-proxy-headers ``` -Then install the HTTP library you want to use (e.g., `pip install requests`). +Or install the client extra you need. Each extra pins a range that CI tests: -> **Note:** This package has no dependencies by default - install only what you need. +```bash +pip install 'python-proxy-headers[requests]' +pip install 'python-proxy-headers[urllib3]' +pip install 'python-proxy-headers[httpx]' +pip install 'python-proxy-headers[aiohttp]' +pip install 'python-proxy-headers[pycurl]' +pip install 'python-proxy-headers[cloudscraper]' +pip install 'python-proxy-headers[autoscraper]' +``` + +The `httpx` extra is `httpx>=0.28,<1`. httpx 1.0 drops proxy support. Importing `python_proxy_headers.httpx_proxy` still succeeds, and using it raises `UnsupportedHttpxVersion`. See the [httpx docs](https://python-proxy-headers.readthedocs.io/en/latest/httpx.html). + +> **Note:** This package has no required dependencies - install only the extra you need. ## Quick Start @@ -122,6 +134,10 @@ response = scraper.get('https://example.com') A test harness is included to verify proxy header functionality: ```bash +# Header checks and a local CONNECT mock (no proxy credentials) +python test_header_security.py +python test_client_internals.py + # Set your proxy export PROXY_URL='http://user:pass@proxy.example.com:8080' @@ -142,8 +158,10 @@ For detailed documentation, API reference, and more examples: - **Full Documentation:** [python-proxy-headers.readthedocs.io](https://python-proxy-headers.readthedocs.io/en/latest/) - **Example Code:** [proxy-examples for Python](https://github.com/proxymesh/proxy-examples/tree/main/python) -## Related Projects +## Other languages +- **[javascript-proxy-headers](https://github.com/proxymesh/javascript-proxy-headers)** - Send and read custom proxy headers on HTTPS CONNECT for Node.js HTTP clients +- **[ruby-proxy-headers](https://github.com/proxymesh/ruby-proxy-headers)** - Send and read custom proxy headers on HTTPS CONNECT for Ruby HTTP clients - **[scrapy-proxy-headers](https://github.com/proxymesh/scrapy-proxy-headers)** - Proxy header support for Scrapy ## About diff --git a/docs/httpx.rst b/docs/httpx.rst index e33076d..6a0ca88 100644 --- a/docs/httpx.rst +++ b/docs/httpx.rst @@ -3,6 +3,30 @@ httpx `HTTPX `_ is a fully featured HTTP client for Python 3, which provides sync and async APIs, and support for both HTTP/1.1 and HTTP/2. This page describes how to use httpx with proxies and how to send and receive custom proxy headers. +httpx 1.0 +--------- + +httpx 1.0 (pre-release ``1.0.dev6``, 2026-08-31) is a rewrite. It no longer depends on httpcore, it does not provide ``HTTPTransport`` or ``AsyncHTTPTransport``, and it has no proxy or CONNECT support (`encode/httpx#947 `_). + +Installing that release does not crash ``import python_proxy_headers.httpx_proxy``. Import warns, and calling ``get``, ``HTTPProxyTransport``, or ``AsyncHTTPProxyTransport`` raises ``UnsupportedHttpxVersion`` instead of ``ImportError``. + +The ``httpx`` extra stays on the 0.28 line:: + + pip install 'python-proxy-headers[httpx]' + +which installs ``httpx>=0.28,<1``. This release does not include a 1.0 adapter. + +Support options +~~~~~~~~~~~~~~~ + +1. **Stay on httpx 0.28.x.** This is the supported path. The adapter copies httpcore's private ``handle_request`` and cannot load on the 1.0 rewrite. + +2. **Wait for a public proxy API.** httpx 1.0's ``NetworkBackend.connect`` / ``connect_tls`` and ``Transport.send`` open a direct connection. They cannot send CONNECT headers or return the proxy's response headers. When `issue 947 `_ grows a public hook, implement against that instead of another private copy. + +3. **A small CONNECT backend, later.** A future adapter could open the proxy socket, write CONNECT plus the custom headers, read the proxy response, wrap TLS, and pass the stream to ``httpx.Connection``. ``NetworkBackend`` does not accept an existing socket today, and the 1.0 API is still early, so this is not in this release. + +4. **Do not vendor httpcore** to pretend 1.0 works. That would ignore the client the user installed. + Getting Started --------------- @@ -14,7 +38,7 @@ This section shows you how to quickly get up and running with proxy headers in h .. code-block:: bash - pip install python-proxy-headers httpx + pip install 'python-proxy-headers[httpx]' 2. Import the module: diff --git a/docs/index.rst b/docs/index.rst index a8cca37..de3f8e0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,20 +25,27 @@ None of these modules provide good support for parsing custom response headers f If you are looking for `Scrapy `_ support, please see our `scrapy-proxy-headers `_ project. +The same CONNECT header support in other languages: + +* `javascript-proxy-headers `_ +* `ruby-proxy-headers `_ + Installation ------------ To use these extension modules, you must first do the following: -1. Install the package: +1. Install the package, optionally with the client you use: .. code-block:: bash pip install python-proxy-headers + pip install 'python-proxy-headers[requests]' + pip install 'python-proxy-headers[httpx]' -2. Install the appropriate package based on the Python module you want to use. +2. Or install the HTTP library yourself. Extras pin the ranges we test. The ``httpx`` extra is ``httpx>=0.28,<1``. httpx 1.0 is not supported yet; see :doc:`httpx`. -This package does not have any dependencies because we don't know which module you want to use. +This package does not have any required dependencies because we don't know which module you want to use. You can also find more example code in our `proxy-examples for python `_. diff --git a/pyproject.toml b/pyproject.toml index a1d2869..52d27ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,19 +10,56 @@ authors = [ ] description = "Handle custom proxy headers for http & https requests in various python libraries" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.10" +keywords = [ + "proxy", + "proxy-headers", + "https-proxy", + "connect", + "tunnel", + "requests", + "urllib3", + "httpx", + "aiohttp", + "web-scraping", + "proxymesh", +] classifiers = [ + "Development Status :: 4 - Beta", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Operating System :: OS Independent", - "License :: OSI Approved :: MIT License", - "Intended Audience :: Developers", - "Topic :: Internet :: WWW/HTTP", + "License :: OSI Approved :: MIT License", + "Intended Audience :: Developers", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Internet :: Proxy Servers", "Topic :: Software Development :: Libraries :: Python Modules", + "Framework :: AsyncIO", ] +# Newest releases checked on 2026-10-23: urllib3 2.8.0, requests 2.34.2, +# httpx 0.28.1 (httpcore 1.0.9), aiohttp 3.14.5, pycurl 7.48.0, +# cloudscraper 1.2.71, autoscraper 1.1.14. +# Lower bounds are the oldest minor whose private hooks still match the +# adapters. CI installs the newest release inside each range. httpx 1.x is +# excluded: 1.0.dev6 drops httpcore and proxy support (encode/httpx#947). +[project.optional-dependencies] +urllib3 = ["urllib3>=2.2,<3"] +requests = ["requests>=2.31,<3"] +httpx = ["httpx>=0.28,<1"] +aiohttp = ["aiohttp>=3.10,<4"] +pycurl = ["pycurl>=7.45,<8"] +cloudscraper = ["cloudscraper>=1.2.71,<2"] +autoscraper = ["autoscraper>=1.1.14,<2"] + [project.urls] Homepage = "https://github.com/proxymesh/python-proxy-headers" -Changelog = "https://github.com/proxymesh/python-proxy-headers/commits/main/" +Changelog = "https://github.com/proxymesh/python-proxy-headers/blob/main/CHANGELOG.md" Issues = "https://github.com/proxymesh/python-proxy-headers/issues" Documentation = "https://python-proxy-headers.readthedocs.io/en/latest/" ProxyMesh = "https://proxymesh.com" diff --git a/python_proxy_headers/_httpx_httpcore.py b/python_proxy_headers/_httpx_httpcore.py new file mode 100644 index 0000000..e47d89d --- /dev/null +++ b/python_proxy_headers/_httpx_httpcore.py @@ -0,0 +1,397 @@ +"""httpcore backend for httpx 0.x. + +``httpx_proxy`` imports this module only when the installed httpx is the 0.28 +line. httpx 1.0 drops httpcore and these private proxy types, so this file is +not imported there. The tunnel classes are the existing copies of httpcore's +``handle_request``; they are not a 1.0 adapter. +""" + +from contextlib import contextmanager +from httpcore._sync.http_proxy import HTTPProxy, TunnelHTTPConnection, merge_headers, logger +from httpcore._sync.http11 import HTTP11Connection +from httpcore._async.http_proxy import AsyncHTTPProxy, AsyncTunnelHTTPConnection +from httpcore._async.http11 import AsyncHTTP11Connection +from httpcore._models import URL, Request +from httpcore._exceptions import ProxyError +from httpcore._ssl import default_ssl_context +from httpcore._trace import Trace +from httpx import AsyncHTTPTransport, HTTPTransport, Client +from httpx._config import DEFAULT_LIMITS, DEFAULT_TIMEOUT_CONFIG, Proxy, create_ssl_context +from .header_utils import filter_connect_headers + +class ProxyTunnelHTTPConnection(TunnelHTTPConnection): + # Unfortunately the only way to get connect_response.headers into the Response + # is to override this whole method + def handle_request(self, request): + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("connect", None) + + with self._connect_lock: + if not self._connected: + target = b"%b:%d" % (self._remote_origin.host, self._remote_origin.port) + + connect_url = URL( + scheme=self._proxy_origin.scheme, + host=self._proxy_origin.host, + port=self._proxy_origin.port, + target=target, + ) + connect_headers = merge_headers( + [(b"Host", target), (b"Accept", b"*/*")], self._proxy_headers + ) + connect_request = Request( + method=b"CONNECT", + url=connect_url, + headers=connect_headers, + extensions=request.extensions, + ) + connect_response = self._connection.handle_request( + connect_request + ) + + if connect_response.status < 200 or connect_response.status > 299: + reason_bytes = connect_response.extensions.get("reason_phrase", b"") + reason_str = reason_bytes.decode("ascii", errors="ignore") + msg = "%d %s" % (connect_response.status, reason_str) + self._connection.close() + raise ProxyError(msg) + + stream = connect_response.extensions["network_stream"] + + # Upgrade the stream to SSL + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": self._remote_origin.host.decode("ascii"), + "timeout": timeout, + } + with Trace("start_tls", logger, request, kwargs) as trace: + stream = stream.start_tls(**kwargs) + trace.return_value = stream + + # Determine if we should be using HTTP/1.1 or HTTP/2 + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) + + # Create the HTTP/1.1 or HTTP/2 connection + if http2_negotiated or (self._http2 and not self._http1): + from httpcore._sync.http2 import HTTP2Connection + + self._connection = HTTP2Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + else: + self._connection = HTTP11Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + + self._connect_response_headers = connect_response.headers + self._connected = True + # Merge safe CONNECT headers without overwriting origin values + response = self._connection.handle_request(request) + response.headers = merge_headers( + response.headers, + filter_connect_headers( + response.headers, getattr(self, "_connect_response_headers", ()) + ), + ) + return response + +class AsyncProxyTunnelHTTPConnection(AsyncTunnelHTTPConnection): + async def handle_async_request(self, request): + timeouts = request.extensions.get("timeout", {}) + timeout = timeouts.get("connect", None) + + async with self._connect_lock: + if not self._connected: + target = b"%b:%d" % (self._remote_origin.host, self._remote_origin.port) + + connect_url = URL( + scheme=self._proxy_origin.scheme, + host=self._proxy_origin.host, + port=self._proxy_origin.port, + target=target, + ) + connect_headers = merge_headers( + [(b"Host", target), (b"Accept", b"*/*")], self._proxy_headers + ) + connect_request = Request( + method=b"CONNECT", + url=connect_url, + headers=connect_headers, + extensions=request.extensions, + ) + connect_response = await self._connection.handle_async_request( + connect_request + ) + + if connect_response.status < 200 or connect_response.status > 299: + reason_bytes = connect_response.extensions.get("reason_phrase", b"") + reason_str = reason_bytes.decode("ascii", errors="ignore") + msg = "%d %s" % (connect_response.status, reason_str) + await self._connection.aclose() + raise ProxyError(msg) + + stream = connect_response.extensions["network_stream"] + + # Upgrade the stream to SSL + ssl_context = ( + default_ssl_context() + if self._ssl_context is None + else self._ssl_context + ) + alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] + ssl_context.set_alpn_protocols(alpn_protocols) + + kwargs = { + "ssl_context": ssl_context, + "server_hostname": self._remote_origin.host.decode("ascii"), + "timeout": timeout, + } + async with Trace("start_tls", logger, request, kwargs) as trace: + stream = await stream.start_tls(**kwargs) + trace.return_value = stream + + # Determine if we should be using HTTP/1.1 or HTTP/2 + ssl_object = stream.get_extra_info("ssl_object") + http2_negotiated = ( + ssl_object is not None + and ssl_object.selected_alpn_protocol() == "h2" + ) + + # Create the HTTP/1.1 or HTTP/2 connection + if http2_negotiated or (self._http2 and not self._http1): + from httpcore._async.http2 import AsyncHTTP2Connection + + self._connection = AsyncHTTP2Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + else: + self._connection = AsyncHTTP11Connection( + origin=self._remote_origin, + stream=stream, + keepalive_expiry=self._keepalive_expiry, + ) + + self._connect_response_headers = connect_response.headers + self._connected = True + # Merge safe CONNECT headers without overwriting origin values + response = await self._connection.handle_async_request(request) + response.headers = merge_headers( + response.headers, + filter_connect_headers( + response.headers, getattr(self, "_connect_response_headers", ()) + ), + ) + return response + +class HTTPProxyHeaders(HTTPProxy): + def create_connection(self, origin): + if origin.scheme == b"http": + return super().create_connection(origin) + return ProxyTunnelHTTPConnection( + proxy_origin=self._proxy_url.origin, + proxy_headers=self._proxy_headers, + remote_origin=origin, + ssl_context=self._ssl_context, + proxy_ssl_context=self._proxy_ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + network_backend=self._network_backend, + ) + +class AsyncHTTPProxyHeaders(AsyncHTTPProxy): + def create_connection(self, origin): + if origin.scheme == b"http": + return super().create_connection(origin) + return AsyncProxyTunnelHTTPConnection( + proxy_origin=self._proxy_url.origin, + proxy_headers=self._proxy_headers, + remote_origin=origin, + ssl_context=self._ssl_context, + proxy_ssl_context=self._proxy_ssl_context, + keepalive_expiry=self._keepalive_expiry, + http1=self._http1, + http2=self._http2, + network_backend=self._network_backend, + ) + +# class ProxyConnectionPool(ConnectionPool): +# def create_connection(self, origin): +# if self._proxy is not None: +# if self._proxy.url.scheme in (b"socks5", b"socks5h"): +# return super().create_connection(origin) +# elif origin.scheme == b"http": +# return super().create_connection(origin) + +# return ProxyTunnelHTTPConnection( +# proxy_origin=self._proxy.url.origin, +# proxy_headers=self._proxy.headers, +# proxy_ssl_context=self._proxy.ssl_context, +# remote_origin=origin, +# ssl_context=self._ssl_context, +# keepalive_expiry=self._keepalive_expiry, +# http1=self._http1, +# http2=self._http2, +# network_backend=self._network_backend, +# ) + +# return super().create_connection(origin) + +class HTTPProxyTransport(HTTPTransport): + def __init__( + self, + verify = True, + cert = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits = DEFAULT_LIMITS, + proxy = None, + uds: str | None = None, + local_address: str | None = None, + retries: int = 0, + socket_options = None, + ) -> None: + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + if proxy and proxy.url.scheme in ("http", "https"): + self._pool = HTTPProxyHeaders( + proxy_url=URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + proxy_headers=proxy.headers.raw, + ssl_context=ssl_context, + proxy_ssl_context=proxy.ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + socket_options=socket_options, + ) + else: + super().__init__(verify, cert, trust_env, http1, http2, limits, proxy, uds, local_address, retries, socket_options) + +class AsyncHTTPProxyTransport(AsyncHTTPTransport): + def __init__( + self, + verify = True, + cert = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits = DEFAULT_LIMITS, + proxy = None, + uds: str | None = None, + local_address: str | None = None, + retries: int = 0, + socket_options = None, + ) -> None: + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + if proxy and proxy.url.scheme in ("http", "https"): + self._pool = AsyncHTTPProxyHeaders( + proxy_url=URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + proxy_headers=proxy.headers.raw, + proxy_ssl_context=proxy.ssl_context, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + socket_options=socket_options, + ) + else: + super().__init__(verify, cert, trust_env, http1, http2, limits, proxy, uds, local_address, retries, socket_options) + +def request(method: str, + url: URL | str, + *, + cookies = None, + proxy = None, + timeout = DEFAULT_TIMEOUT_CONFIG, + verify = True, + trust_env: bool = True, + **kwargs): + transport = HTTPProxyTransport(proxy=proxy) + with Client( + cookies=cookies, + verify=verify, + timeout=timeout, + trust_env=trust_env, + mounts={'http://': transport, 'https://': transport} + ) as client: + return client.request(method=method, url=url, **kwargs) + +def get(*args, **kwargs): + return request('GET', *args, **kwargs) + +def options(*args, **kwargs): + return request('OPTIONS', *args, **kwargs) + +def head(*args, **kwargs): + return request('HEAD', *args, **kwargs) + +def post(*args, **kwargs): + return request('POST', *args, **kwargs) + +def put(*args, **kwargs): + return request('PUT', *args, **kwargs) + +def patch(*args, **kwargs): + return request('PATCH', *args, **kwargs) + +def delete(*args, **kwargs): + return request('DELETE', *args, **kwargs) + +@contextmanager +def stream(method: str, + url: URL | str, + *, + cookies = None, + proxy = None, + timeout = DEFAULT_TIMEOUT_CONFIG, + verify = True, + trust_env: bool = True, + **kwargs): + transport = HTTPProxyTransport(proxy=proxy) + with Client( + cookies=cookies, + verify=verify, + timeout=timeout, + trust_env=trust_env, + mounts={'http://': transport, 'https://': transport} + ) as client: + with client.stream(method=method, url=url, **kwargs) as response: + yield response \ No newline at end of file diff --git a/python_proxy_headers/httpx_proxy.py b/python_proxy_headers/httpx_proxy.py index 430b5e3..796ebdf 100644 --- a/python_proxy_headers/httpx_proxy.py +++ b/python_proxy_headers/httpx_proxy.py @@ -1,389 +1,73 @@ -from contextlib import contextmanager -from httpcore._sync.http_proxy import HTTPProxy, TunnelHTTPConnection, merge_headers, logger -from httpcore._sync.http11 import HTTP11Connection -from httpcore._async.http_proxy import AsyncHTTPProxy, AsyncTunnelHTTPConnection -from httpcore._async.http11 import AsyncHTTP11Connection -from httpcore._models import URL, Request -from httpcore._exceptions import ProxyError -from httpcore._ssl import default_ssl_context -from httpcore._trace import Trace -from httpx import AsyncHTTPTransport, HTTPTransport, Client -from httpx._config import DEFAULT_LIMITS, DEFAULT_TIMEOUT_CONFIG, Proxy, create_ssl_context -from .header_utils import filter_connect_headers - -class ProxyTunnelHTTPConnection(TunnelHTTPConnection): - # Unfortunately the only way to get connect_response.headers into the Response - # is to override this whole method - def handle_request(self, request): - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("connect", None) - - with self._connect_lock: - if not self._connected: - target = b"%b:%d" % (self._remote_origin.host, self._remote_origin.port) - - connect_url = URL( - scheme=self._proxy_origin.scheme, - host=self._proxy_origin.host, - port=self._proxy_origin.port, - target=target, - ) - connect_headers = merge_headers( - [(b"Host", target), (b"Accept", b"*/*")], self._proxy_headers - ) - connect_request = Request( - method=b"CONNECT", - url=connect_url, - headers=connect_headers, - extensions=request.extensions, - ) - connect_response = self._connection.handle_request( - connect_request - ) - - if connect_response.status < 200 or connect_response.status > 299: - reason_bytes = connect_response.extensions.get("reason_phrase", b"") - reason_str = reason_bytes.decode("ascii", errors="ignore") - msg = "%d %s" % (connect_response.status, reason_str) - self._connection.close() - raise ProxyError(msg) - - stream = connect_response.extensions["network_stream"] - - # Upgrade the stream to SSL - ssl_context = ( - default_ssl_context() - if self._ssl_context is None - else self._ssl_context - ) - alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] - ssl_context.set_alpn_protocols(alpn_protocols) - - kwargs = { - "ssl_context": ssl_context, - "server_hostname": self._remote_origin.host.decode("ascii"), - "timeout": timeout, - } - with Trace("start_tls", logger, request, kwargs) as trace: - stream = stream.start_tls(**kwargs) - trace.return_value = stream - - # Determine if we should be using HTTP/1.1 or HTTP/2 - ssl_object = stream.get_extra_info("ssl_object") - http2_negotiated = ( - ssl_object is not None - and ssl_object.selected_alpn_protocol() == "h2" - ) - - # Create the HTTP/1.1 or HTTP/2 connection - if http2_negotiated or (self._http2 and not self._http1): - from httpcore._sync.http2 import HTTP2Connection - - self._connection = HTTP2Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - else: - self._connection = HTTP11Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - - self._connect_response_headers = connect_response.headers - self._connected = True - # Merge safe CONNECT headers without overwriting origin values - response = self._connection.handle_request(request) - response.headers = merge_headers( - response.headers, - filter_connect_headers( - response.headers, getattr(self, "_connect_response_headers", ()) - ), - ) - return response - -class AsyncProxyTunnelHTTPConnection(AsyncTunnelHTTPConnection): - async def handle_async_request(self, request): - timeouts = request.extensions.get("timeout", {}) - timeout = timeouts.get("connect", None) - - async with self._connect_lock: - if not self._connected: - target = b"%b:%d" % (self._remote_origin.host, self._remote_origin.port) - - connect_url = URL( - scheme=self._proxy_origin.scheme, - host=self._proxy_origin.host, - port=self._proxy_origin.port, - target=target, - ) - connect_headers = merge_headers( - [(b"Host", target), (b"Accept", b"*/*")], self._proxy_headers - ) - connect_request = Request( - method=b"CONNECT", - url=connect_url, - headers=connect_headers, - extensions=request.extensions, - ) - connect_response = await self._connection.handle_async_request( - connect_request - ) - - if connect_response.status < 200 or connect_response.status > 299: - reason_bytes = connect_response.extensions.get("reason_phrase", b"") - reason_str = reason_bytes.decode("ascii", errors="ignore") - msg = "%d %s" % (connect_response.status, reason_str) - await self._connection.aclose() - raise ProxyError(msg) - - stream = connect_response.extensions["network_stream"] - - # Upgrade the stream to SSL - ssl_context = ( - default_ssl_context() - if self._ssl_context is None - else self._ssl_context - ) - alpn_protocols = ["http/1.1", "h2"] if self._http2 else ["http/1.1"] - ssl_context.set_alpn_protocols(alpn_protocols) - - kwargs = { - "ssl_context": ssl_context, - "server_hostname": self._remote_origin.host.decode("ascii"), - "timeout": timeout, - } - async with Trace("start_tls", logger, request, kwargs) as trace: - stream = await stream.start_tls(**kwargs) - trace.return_value = stream - - # Determine if we should be using HTTP/1.1 or HTTP/2 - ssl_object = stream.get_extra_info("ssl_object") - http2_negotiated = ( - ssl_object is not None - and ssl_object.selected_alpn_protocol() == "h2" - ) - - # Create the HTTP/1.1 or HTTP/2 connection - if http2_negotiated or (self._http2 and not self._http1): - from httpcore._async.http2 import AsyncHTTP2Connection - - self._connection = AsyncHTTP2Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - else: - self._connection = AsyncHTTP11Connection( - origin=self._remote_origin, - stream=stream, - keepalive_expiry=self._keepalive_expiry, - ) - - self._connect_response_headers = connect_response.headers - self._connected = True - # Merge safe CONNECT headers without overwriting origin values - response = await self._connection.handle_async_request(request) - response.headers = merge_headers( - response.headers, - filter_connect_headers( - response.headers, getattr(self, "_connect_response_headers", ()) - ), - ) - return response - -class HTTPProxyHeaders(HTTPProxy): - def create_connection(self, origin): - if origin.scheme == b"http": - return super().create_connection(origin) - return ProxyTunnelHTTPConnection( - proxy_origin=self._proxy_url.origin, - proxy_headers=self._proxy_headers, - remote_origin=origin, - ssl_context=self._ssl_context, - proxy_ssl_context=self._proxy_ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - network_backend=self._network_backend, - ) - -class AsyncHTTPProxyHeaders(AsyncHTTPProxy): - def create_connection(self, origin): - if origin.scheme == b"http": - return super().create_connection(origin) - return AsyncProxyTunnelHTTPConnection( - proxy_origin=self._proxy_url.origin, - proxy_headers=self._proxy_headers, - remote_origin=origin, - ssl_context=self._ssl_context, - proxy_ssl_context=self._proxy_ssl_context, - keepalive_expiry=self._keepalive_expiry, - http1=self._http1, - http2=self._http2, - network_backend=self._network_backend, - ) - -# class ProxyConnectionPool(ConnectionPool): -# def create_connection(self, origin): -# if self._proxy is not None: -# if self._proxy.url.scheme in (b"socks5", b"socks5h"): -# return super().create_connection(origin) -# elif origin.scheme == b"http": -# return super().create_connection(origin) - -# return ProxyTunnelHTTPConnection( -# proxy_origin=self._proxy.url.origin, -# proxy_headers=self._proxy.headers, -# proxy_ssl_context=self._proxy.ssl_context, -# remote_origin=origin, -# ssl_context=self._ssl_context, -# keepalive_expiry=self._keepalive_expiry, -# http1=self._http1, -# http2=self._http2, -# network_backend=self._network_backend, -# ) - -# return super().create_connection(origin) - -class HTTPProxyTransport(HTTPTransport): - def __init__( - self, - verify = True, - cert = None, - trust_env: bool = True, - http1: bool = True, - http2: bool = False, - limits = DEFAULT_LIMITS, - proxy = None, - uds: str | None = None, - local_address: str | None = None, - retries: int = 0, - socket_options = None, - ) -> None: - proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy - ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) - - if proxy and proxy.url.scheme in ("http", "https"): - self._pool = HTTPProxyHeaders( - proxy_url=URL( - scheme=proxy.url.raw_scheme, - host=proxy.url.raw_host, - port=proxy.url.port, - target=proxy.url.raw_path, - ), - proxy_auth=proxy.raw_auth, - proxy_headers=proxy.headers.raw, - ssl_context=ssl_context, - proxy_ssl_context=proxy.ssl_context, - max_connections=limits.max_connections, - max_keepalive_connections=limits.max_keepalive_connections, - keepalive_expiry=limits.keepalive_expiry, - http1=http1, - http2=http2, - socket_options=socket_options, - ) - else: - super().__init__(verify, cert, trust_env, http1, http2, limits, proxy, uds, local_address, retries, socket_options) - -class AsyncHTTPProxyTransport(AsyncHTTPTransport): - def __init__( - self, - verify = True, - cert = None, - trust_env: bool = True, - http1: bool = True, - http2: bool = False, - limits = DEFAULT_LIMITS, - proxy = None, - uds: str | None = None, - local_address: str | None = None, - retries: int = 0, - socket_options = None, - ) -> None: - proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy - ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) - - if proxy and proxy.url.scheme in ("http", "https"): - self._pool = AsyncHTTPProxyHeaders( - proxy_url=URL( - scheme=proxy.url.raw_scheme, - host=proxy.url.raw_host, - port=proxy.url.port, - target=proxy.url.raw_path, - ), - proxy_auth=proxy.raw_auth, - proxy_headers=proxy.headers.raw, - proxy_ssl_context=proxy.ssl_context, - ssl_context=ssl_context, - max_connections=limits.max_connections, - max_keepalive_connections=limits.max_keepalive_connections, - keepalive_expiry=limits.keepalive_expiry, - http1=http1, - http2=http2, - socket_options=socket_options, - ) - else: - super().__init__(verify, cert, trust_env, http1, http2, limits, proxy, uds, local_address, retries, socket_options) - -def request(method: str, - url: URL | str, - *, - cookies = None, - proxy = None, - timeout = DEFAULT_TIMEOUT_CONFIG, - verify = True, - trust_env: bool = True, - **kwargs): - transport = HTTPProxyTransport(proxy=proxy) - with Client( - cookies=cookies, - verify=verify, - timeout=timeout, - trust_env=trust_env, - mounts={'http://': transport, 'https://': transport} - ) as client: - return client.request(method=method, url=url, **kwargs) - -def get(*args, **kwargs): - return request('GET', *args, **kwargs) - -def options(*args, **kwargs): - return request('OPTIONS', *args, **kwargs) - -def head(*args, **kwargs): - return request('HEAD', *args, **kwargs) - -def post(*args, **kwargs): - return request('POST', *args, **kwargs) - -def put(*args, **kwargs): - return request('PUT', *args, **kwargs) - -def patch(*args, **kwargs): - return request('PATCH', *args, **kwargs) - -def delete(*args, **kwargs): - return request('DELETE', *args, **kwargs) - -@contextmanager -def stream(method: str, - url: URL | str, - *, - cookies = None, - proxy = None, - timeout = DEFAULT_TIMEOUT_CONFIG, - verify = True, - trust_env: bool = True, - **kwargs): - transport = HTTPProxyTransport(proxy=proxy) - with Client( - cookies=cookies, - verify=verify, - timeout=timeout, - trust_env=trust_env, - mounts={'http://': transport, 'https://': transport} - ) as client: - with client.stream(method=method, url=url, **kwargs) as response: - yield response \ No newline at end of file +"""Send and read CONNECT proxy headers with httpx 0.28.x. + +httpx 1.0 removes httpcore and proxy support, so this module does not import +httpcore when httpx>=1 is installed. Importing still succeeds: it warns, and +the helpers raise :class:`UnsupportedHttpxVersion`. + +The ``[httpx]`` extra is ``httpx>=0.28,<1``. +""" + +from __future__ import annotations + +import warnings +from typing import Any, Dict + +from .httpx_support import ( + HttpxStatus, + UnsupportedHttpxVersion, + classify_httpx, + install_message, + load_backend, + unavailable_namespace, + BACKEND_EXPORTS, +) + +__all__ = [ + "AsyncHTTPProxyHeaders", + "AsyncHTTPProxyTransport", + "AsyncProxyTunnelHTTPConnection", + "HTTPProxyHeaders", + "HTTPProxyTransport", + "HTTPX_STATUS", + "HttpxStatus", + "ProxyTunnelHTTPConnection", + "UnsupportedHttpxVersion", + "delete", + "get", + "head", + "options", + "patch", + "post", + "put", + "request", + "stream", +] + + +def _export(namespace: Dict[str, Any]) -> None: + for name in BACKEND_EXPORTS: + globals()[name] = namespace[name] + + +def _activate() -> HttpxStatus: + """Bind the real backend or the raising stand-ins. Never raises ImportError.""" + status = classify_httpx() + if status.kind == "supported": + try: + namespace = load_backend() + except ImportError as exc: + status = HttpxStatus("broken", status.version, detail=str(exc)) + message = install_message(status) + warnings.warn(message, UserWarning, stacklevel=3) + namespace = unavailable_namespace(message) + _export(namespace) + return status + + message = install_message(status) + if status.kind == "unsupported": + warnings.warn(message, UserWarning, stacklevel=3) + _export(unavailable_namespace(message)) + return status + + +HTTPX_STATUS = _activate() diff --git a/python_proxy_headers/httpx_support.py b/python_proxy_headers/httpx_support.py new file mode 100644 index 0000000..2f19a21 --- /dev/null +++ b/python_proxy_headers/httpx_support.py @@ -0,0 +1,213 @@ +"""Detect whether the installed httpx can load the CONNECT adapter. + +httpx 0.28.x (httpcore) is supported. httpx 1.0 is a rewrite without httpcore, +without ``HTTPTransport`` / ``AsyncHTTPTransport``, and without proxy support +(https://github.com/encode/httpx/issues/947). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable, Dict, Optional + +# Kept in sync with [project.optional-dependencies] httpx in pyproject.toml. +SUPPORTED_HTTPX_SPEC = "httpx>=0.28,<1" +HTTPX_PROXY_ISSUE = "https://github.com/encode/httpx/issues/947" + +BACKEND_EXPORTS = ( + "ProxyTunnelHTTPConnection", + "AsyncProxyTunnelHTTPConnection", + "HTTPProxyHeaders", + "AsyncHTTPProxyHeaders", + "HTTPProxyTransport", + "AsyncHTTPProxyTransport", + "request", + "get", + "options", + "head", + "post", + "put", + "patch", + "delete", + "stream", +) + +_MISSING = object() + + +class UnsupportedHttpxVersion(RuntimeError): + """Raised when httpx cannot send or read CONNECT proxy headers. + + httpx 1.x is installed, httpx is missing, or the httpcore backend failed + to import. The message names the installed version and the supported range. + """ + + +@dataclass(frozen=True) +class HttpxStatus: + """What ``classify_httpx`` decided about the environment. + + ``kind`` is ``supported`` (httpx 0.x), ``unsupported`` (httpx 1 or newer), + ``missing`` (httpx is not installed), or ``broken`` (version looks like + 0.x but the httpcore backend did not import). + """ + + kind: str + version: Optional[str] = None + detail: Optional[str] = None + + +def httpx_major(version: Optional[str]) -> Optional[int]: + """Return the numeric major of an httpx version string. + + ``1.0.dev6`` is major 1 and ``0.28.1`` is major 0. Returns None when + ``version`` is empty or does not start with a number. + """ + if not version: + return None + token = version.strip().split(".", 1)[0] + digits = [] + for char in token: + if char.isdigit(): + digits.append(char) + else: + break + if not digits: + return None + return int("".join(digits)) + + +def _imported_version() -> Optional[str]: + try: + import httpx + except ImportError: + return None + version = getattr(httpx, "__version__", None) + if isinstance(version, str) and version: + return version + return None + + +def _metadata_version() -> Optional[str]: + try: + from importlib.metadata import PackageNotFoundError, version + except ImportError: # pragma: no cover - importlib.metadata is stdlib on 3.10+ + return None + try: + return version("httpx") + except PackageNotFoundError: + return None + + +def probe_installed_version() -> Optional[str]: + """Return the httpx version that ``import httpx`` will actually use. + + Distribution metadata can disagree with a module earlier on ``sys.path`` + (for example a shadowed 1.0 build while metadata still says 0.28). When + the majors differ, the imported module wins. + """ + imported = _imported_version() + meta = _metadata_version() + if imported and meta and httpx_major(imported) != httpx_major(meta): + return imported + return imported or meta + + +def classify_httpx(version: Any = _MISSING, *, probe: bool = True) -> HttpxStatus: + """Classify an httpx version string, or the version installed in this process. + + Pass ``probe=False`` and no version to classify a missing install without + touching the environment. + """ + if version is _MISSING: + version = probe_installed_version() if probe else None + if version is None: + return HttpxStatus("missing", None) + if not isinstance(version, str): + version = str(version) + major = httpx_major(version) + if major == 0: + return HttpxStatus("supported", version) + return HttpxStatus("unsupported", version) + + +def install_message(status: HttpxStatus) -> str: + """Explain why the httpx helpers cannot run, including the 1.0 plan pointer.""" + spec = SUPPORTED_HTTPX_SPEC + if status.kind == "missing": + return ( + "httpx is not installed. python-proxy-headers supports the httpx " + f"0.28 line only ({spec}). Install it with " + "`pip install 'python-proxy-headers[httpx]'`." + ) + if status.kind == "broken": + detail = f" ({status.detail})" if status.detail else "" + found = f"httpx {status.version}" if status.version else "httpx 0.x" + return ( + f"{found} looks like a supported 0.x release, but the httpcore " + f"CONNECT adapter failed to import{detail}. Reinstall with " + "`pip install 'python-proxy-headers[httpx]'` " + f"({spec})." + ) + found = f"httpx {status.version}" if status.version else "httpx 1.x" + return ( + f"{found} is installed, but python-proxy-headers only supports the " + f"httpx 0.28 line ({spec}). httpx 1.0 is a rewrite that drops httpcore " + "and does not support proxies " + f"({HTTPX_PROXY_ISSUE}); AsyncHTTPTransport and HTTPTransport are gone, " + "so the 0.x adapter is not loaded. Importing this module succeeds, and " + "calling the helpers raises UnsupportedHttpxVersion. " + "Install a compatible release with " + "`pip install 'python-proxy-headers[httpx]'`. " + "A 1.0 adapter is not included; see the httpx documentation for the " + "support options (stay on 0.28, wait for a public CONNECT hook, or add " + "a small backend once NetworkBackend can take an existing tunnel)." + ) + + +def load_backend() -> Dict[str, Any]: + """Import the httpcore implementation. + + Raises: + ImportError: httpx 0.x symbols or httpcore are not importable. + """ + from httpx import AsyncHTTPTransport, HTTPTransport # noqa: F401 + import httpcore # noqa: F401 + from . import _httpx_httpcore as backend + + namespace: Dict[str, Any] = {} + for name in BACKEND_EXPORTS: + namespace[name] = getattr(backend, name) + return namespace + + +def _make_raiser(message: str) -> Callable[..., Any]: + def _raise(*_args: Any, **_kwargs: Any) -> Any: + raise UnsupportedHttpxVersion(message) + + return _raise + + +def unavailable_namespace(message: str) -> Dict[str, Any]: + """Build stand-ins that raise ``UnsupportedHttpxVersion`` when used. + + The public names still import. Nothing here touches httpcore. + """ + namespace: Dict[str, Any] = {} + for name in BACKEND_EXPORTS: + if name[:1].isupper(): + raiser = _make_raiser(message) + + def _init(self: Any, *args: Any, _raiser: Callable[..., Any] = raiser, **kwargs: Any) -> None: + _raiser(*args, **kwargs) + + cls = type(name, (), {"__init__": _init, "__doc__": message}) + cls.__module__ = "python_proxy_headers.httpx_proxy" + namespace[name] = cls + else: + fn = _make_raiser(message) + fn.__name__ = name + fn.__doc__ = message + fn.__module__ = "python_proxy_headers.httpx_proxy" + namespace[name] = fn + return namespace diff --git a/test_client_internals.py b/test_client_internals.py new file mode 100644 index 0000000..4b5765f --- /dev/null +++ b/test_client_internals.py @@ -0,0 +1,753 @@ +#!/usr/bin/env python3 +"""Exercise private client hooks against a local CONNECT proxy. + +urllib3, httpx/httpcore, and aiohttp adapters override or copy private +methods. These tests call those paths and fail with the upstream symbol name +when a release moves them. No PROXY_URL or network access is required. + +httpx 1.x has no adapter. Importing ``httpx_proxy`` must not raise, and using +it must raise UnsupportedHttpxVersion. +""" + +from __future__ import annotations + +import asyncio +import importlib +import inspect +import os +import socket +import ssl +import subprocess +import sys +import tempfile +import textwrap +import threading +import unittest +from pathlib import Path +from typing import Any, Dict, List, Optional + +ROOT = Path(__file__).resolve().parent +MOCK_PROXY = ROOT / "tests" / "mock_connect_proxy.py" +FAKE_IP = "203.0.113.7" +SEND_HEADER = {"X-ProxyMesh-Country": "US"} + + +def _runtime() -> str: + """Describe the interpreters and client versions under test.""" + parts = [f"Python {sys.version.split()[0]}"] + for name in ("urllib3", "requests", "httpx", "httpcore", "aiohttp"): + try: + module = importlib.import_module(name) + except ImportError: + parts.append(f"{name} not-installed") + else: + parts.append(f"{name} {getattr(module, '__version__', '?')}") + return ", ".join(parts) + + +def _param_names(func: Any) -> List[str]: + return list(inspect.signature(func).parameters) + + +def _header(headers: Any, name: str) -> Optional[str]: + if headers is None: + return None + value = headers.get(name) + if value is None: + value = headers.get(name.lower()) + if isinstance(value, bytes): + return value.decode("latin-1") + if value is None: + return None + return str(value) + + +class HttpxGuardTests(unittest.TestCase): + """httpx 1.x must not crash the import, and the extra must stay on 0.28.""" + + def test_version_major(self) -> None: + from python_proxy_headers.httpx_support import httpx_major + + self.assertEqual(httpx_major("0.28.1"), 0) + self.assertEqual(httpx_major("1.0.dev6"), 1) + self.assertEqual(httpx_major("1.0.0rc1"), 1) + self.assertIsNone(httpx_major(None)) + self.assertIsNone(httpx_major("")) + + def test_classify_without_probing(self) -> None: + from python_proxy_headers.httpx_support import classify_httpx + + self.assertEqual(classify_httpx("0.28.1").kind, "supported") + self.assertEqual(classify_httpx("1.0.dev6").kind, "unsupported") + self.assertEqual(classify_httpx(probe=False).kind, "missing") + + def test_imported_version_wins_when_major_disagrees(self) -> None: + from unittest import mock + + from python_proxy_headers.httpx_support import classify_httpx + + with mock.patch( + "python_proxy_headers.httpx_support._imported_version", + return_value="1.0.dev6", + ), mock.patch( + "python_proxy_headers.httpx_support._metadata_version", + return_value="0.28.1", + ): + status = classify_httpx() + self.assertEqual(status.kind, "unsupported") + self.assertEqual(status.version, "1.0.dev6") + + def test_unsupported_message_names_issue_and_extra(self) -> None: + from python_proxy_headers.httpx_support import ( + HttpxStatus, + install_message, + ) + + text = install_message(HttpxStatus("unsupported", "1.0.dev6")) + self.assertIn("1.0.dev6", text) + self.assertIn("947", text) + self.assertIn("httpx>=0.28,<1", text) + self.assertIn("AsyncHTTPTransport", text) + missing = install_message(HttpxStatus("missing", None)) + self.assertIn("python-proxy-headers[httpx]", missing) + + def test_pyproject_pins_httpx_extra_and_changelog(self) -> None: + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + self.assertIn('httpx = ["httpx>=0.28,<1"]', text) + self.assertIn( + 'Changelog = "https://github.com/proxymesh/python-proxy-headers/blob/main/CHANGELOG.md"', + text, + ) + self.assertNotIn("commits/main", text) + for keyword in ("proxy-headers", "https-proxy", "proxymesh"): + self.assertIn(f'"{keyword}"', text) + self.assertIn("Topic :: Internet :: Proxy Servers", text) + self.assertIn('requires-python = ">=3.10"', text) + + def test_import_matches_installed_httpx(self) -> None: + """Importing httpx_proxy succeeds for both 0.28.x and 1.x.""" + code = textwrap.dedent( + """ + import warnings + warnings.simplefilter("always") + import python_proxy_headers.httpx_proxy as proxy + + print("STATUS", proxy.HTTPX_STATUS.kind, proxy.HTTPX_STATUS.version or "-") + if proxy.HTTPX_STATUS.kind == "supported": + print("READY", callable(proxy.get)) + else: + try: + proxy.get("https://example.invalid", proxy="http://127.0.0.1:9") + print("NO_RAISE") + except proxy.UnsupportedHttpxVersion as exc: + text = str(exc) + print("RAISED") + print("HAS947", "947" in text) + print("HAS_EXTRA", "python-proxy-headers[httpx]" in text) + """ + ) + completed = _run_python(code, extra_pythonpath=[]) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertNotIn("ImportError", completed.stderr) + kind = _status_kind(completed.stdout) + if kind == "supported": + self.assertIn("READY True", completed.stdout) + self.assertNotIn("drops httpcore", completed.stderr) + elif kind == "unsupported": + self.assertIn("RAISED", completed.stdout) + self.assertIn("HAS947 True", completed.stdout) + self.assertIn("947", completed.stderr) + self.assertIn("UserWarning", completed.stderr) + elif kind == "missing": + self.assertIn("RAISED", completed.stdout) + self.assertIn("HAS_EXTRA True", completed.stdout) + self.assertNotIn("UserWarning", completed.stderr) + else: + self.fail(f"unexpected httpx status {kind}\n{completed.stdout}\n{completed.stderr}") + + def test_shadowed_httpx_1_dev6_does_not_crash_on_import(self) -> None: + """A 1.0.dev6 module earlier on sys.path must not raise ImportError.""" + fake = Path(tempfile.mkdtemp()) + package = fake / "httpx" + package.mkdir() + (package / "__init__.py").write_text('__version__ = "1.0.dev6"\n', encoding="utf-8") + code = textwrap.dedent( + """ + import warnings + warnings.simplefilter("always") + import python_proxy_headers.httpx_proxy as proxy + + print("STATUS", proxy.HTTPX_STATUS.kind, proxy.HTTPX_STATUS.version or "-") + for label, call in ( + ("get", lambda: proxy.get("https://example.invalid", proxy="http://127.0.0.1:9")), + ("transport", lambda: proxy.HTTPProxyTransport(proxy="http://127.0.0.1:9")), + ("async-transport", lambda: proxy.AsyncHTTPProxyTransport(proxy="http://127.0.0.1:9")), + ("stream", lambda: proxy.stream("GET", "https://example.invalid", proxy="http://127.0.0.1:9")), + ): + try: + call() + print("NO_RAISE", label) + except proxy.UnsupportedHttpxVersion: + print("RAISED", label) + """ + ) + completed = _run_python(code, extra_pythonpath=[fake]) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertIn("STATUS unsupported 1.0.dev6", completed.stdout) + for label in ("get", "transport", "async-transport", "stream"): + self.assertIn(f"RAISED {label}", completed.stdout) + self.assertNotIn("NO_RAISE", completed.stdout) + self.assertNotIn("ImportError", completed.stderr) + self.assertIn("1.0.dev6", completed.stderr) + self.assertIn("947", completed.stderr) + + +class Urllib3ContractTests(unittest.TestCase): + def test_prepare_proxy_signature_still_matches(self) -> None: + import urllib3 + from urllib3.connectionpool import HTTPSConnectionPool + + from python_proxy_headers.urllib3_proxy_manager import HTTPSProxyConnectionPool + + base = _param_names(HTTPSConnectionPool._prepare_proxy) + ours = _param_names(HTTPSProxyConnectionPool._prepare_proxy) + self.assertIsNot( + HTTPSProxyConnectionPool._prepare_proxy, + HTTPSConnectionPool._prepare_proxy, + ) + if base != ours: + self.fail( + "urllib3 HTTPSConnectionPool._prepare_proxy signature changed. " + f"python_proxy_headers override parameters are {ours}; " + f"upstream parameters are now {base}. urllib3 {urllib3.__version__}. " + "HTTPSProxyConnectionPool._prepare_proxy in urllib3_proxy_manager.py " + "needs an update before CONNECT header capture keeps working." + ) + + def test_tunnel_and_read_headers_hooks(self) -> None: + import http.client + import urllib3 + from python_proxy_headers.urllib3_proxy_manager import HTTPSProxyConnection + + if not hasattr(http.client, "_read_headers"): + self.fail( + "http.client._read_headers is gone. urllib3_proxy_manager.py " + "imports it at module level (and the <3.12 _tunnel copy calls it). " + f"{_runtime()}." + ) + if sys.version_info < (3, 12): + if "_tunnel" not in HTTPSProxyConnection.__dict__: + self.fail( + "HTTPSProxyConnection._tunnel is not defined on Python " + f"{sys.version.split()[0]}. The <3.12 branch copies " + "HTTPConnection._tunnel and reads headers with " + f"http.client._read_headers. urllib3 {urllib3.__version__}." + ) + if "get_proxy_response_headers" not in HTTPSProxyConnection.__dict__: + self.fail( + "HTTPSProxyConnection.get_proxy_response_headers is missing " + "on Python < 3.12, so _prepare_proxy cannot read CONNECT headers." + ) + else: + if not hasattr(http.client.HTTPConnection, "get_proxy_response_headers"): + self.fail( + "HTTPConnection.get_proxy_response_headers is missing. On " + "Python 3.12+, HTTPSProxyConnectionPool._prepare_proxy reads " + "CONNECT headers through that stdlib method. " + f"{_runtime()}." + ) + if not hasattr(http.client.HTTPConnection, "_tunnel"): + self.fail( + "HTTPConnection._tunnel is missing. urllib3 still opens the " + f"CONNECT tunnel through it. {_runtime()}." + ) + + +class HttpxContractTests(unittest.TestCase): + def test_copied_handle_request_still_matches_httpcore(self) -> None: + self._require_httpx_0() + import httpcore + import httpx + from httpcore._async.http_proxy import AsyncHTTPProxy, AsyncTunnelHTTPConnection + from httpcore._sync.http_proxy import HTTPProxy, TunnelHTTPConnection, merge_headers + + from python_proxy_headers.httpx_proxy import ( + AsyncHTTPProxyHeaders, + AsyncProxyTunnelHTTPConnection, + HTTPProxyHeaders, + ProxyTunnelHTTPConnection, + ) + + required = ( + "proxy_origin", + "remote_origin", + "ssl_context", + "proxy_ssl_context", + "proxy_headers", + "keepalive_expiry", + "http1", + "http2", + "network_backend", + ) + params = inspect.signature(TunnelHTTPConnection.__init__).parameters + missing = [name for name in required if name not in params] + if missing or not hasattr(TunnelHTTPConnection, "handle_request"): + self.fail( + "httpcore TunnelHTTPConnection no longer matches the copied " + "ProxyTunnelHTTPConnection.handle_request. Missing " + f"{missing or ['handle_request']}. httpcore {httpcore.__version__}, " + f"httpx {httpx.__version__}." + ) + if not hasattr(AsyncTunnelHTTPConnection, "handle_async_request"): + self.fail( + "httpcore AsyncTunnelHTTPConnection.handle_async_request is gone. " + f"httpcore {httpcore.__version__}." + ) + self.assertTrue(callable(merge_headers)) + self.assertTrue(hasattr(HTTPProxy, "create_connection")) + self.assertTrue(hasattr(AsyncHTTPProxy, "create_connection")) + self.assertIsNot( + ProxyTunnelHTTPConnection.handle_request, + TunnelHTTPConnection.handle_request, + ) + self.assertIsNot( + AsyncProxyTunnelHTTPConnection.handle_async_request, + AsyncTunnelHTTPConnection.handle_async_request, + ) + self.assertIsNot(HTTPProxyHeaders.create_connection, HTTPProxy.create_connection) + self.assertIsNot( + AsyncHTTPProxyHeaders.create_connection, + AsyncHTTPProxy.create_connection, + ) + + def _require_httpx_0(self) -> None: + from python_proxy_headers.httpx_proxy import HTTPX_STATUS + + if HTTPX_STATUS.kind != "supported": + self.skipTest( + "httpcore handle_request checks need httpx 0.28.x; " + f"found {HTTPX_STATUS.kind} {HTTPX_STATUS.version}. " + "httpx 1.x has no CONNECT adapter (encode/httpx#947)." + ) + + +class AiohttpContractTests(unittest.TestCase): + def test_create_proxy_connection_signature_and_helpers(self) -> None: + import aiohttp + from aiohttp.connector import TCPConnector + + from python_proxy_headers.aiohttp_proxy import ProxyTCPConnector + + base = _param_names(TCPConnector._create_proxy_connection) + ours = _param_names(ProxyTCPConnector._create_proxy_connection) + if base != ours: + self.fail( + "aiohttp TCPConnector._create_proxy_connection signature changed. " + f"ProxyTCPConnector copy parameters are {ours}; upstream parameters " + f"are now {base}. aiohttp {aiohttp.__version__}. The copy in " + "aiohttp_proxy.py has to be updated or CONNECT headers will not " + "be captured." + ) + required = ( + "_fail_on_no_start_tls", + "_loop_supports_start_tls", + "_create_direct_connection", + "_warn_about_tls_in_tls", + "_get_ssl_context", + "_wrap_existing_connection", + "_start_tls_connection", + ) + missing = [name for name in required if not hasattr(TCPConnector, name)] + if missing: + self.fail( + "aiohttp TCPConnector dropped helpers used by the copied " + f"_create_proxy_connection: {', '.join(missing)}. " + f"aiohttp {aiohttp.__version__}." + ) + + async def _instance_attrs() -> List[str]: + connector = ProxyTCPConnector() + try: + return [ + name + for name in ("_timeout_ceil_threshold", "_factory") + if not hasattr(connector, name) + ] + finally: + await connector.close() + + absent = asyncio.run(_instance_attrs()) + if absent: + self.fail( + "ProxyTCPConnector instances no longer have " + f"{', '.join(absent)}, which the copied _create_proxy_connection " + f"reads. aiohttp {aiohttp.__version__}." + ) + + +class _LocalTunnel: + """TLS origin plus tests/mock_connect_proxy.py on 127.0.0.1.""" + + def __init__(self) -> None: + self.proxy_port = 0 + self.origin_port = 0 + self.logs: List[str] = [] + self._proc: Optional[subprocess.Popen[str]] = None + self._origin: Optional[socket.socket] = None + self._stop = threading.Event() + self._tmpdir = tempfile.mkdtemp() + + @property + def proxy_url(self) -> str: + return f"http://127.0.0.1:{self.proxy_port}" + + @property + def origin_url(self) -> str: + return f"https://127.0.0.1:{self.origin_port}/" + + def start(self) -> None: + key = os.path.join(self._tmpdir, "key.pem") + cert = os.path.join(self._tmpdir, "cert.pem") + subprocess.check_call( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-keyout", + key, + "-out", + cert, + "-days", + "1", + "-nodes", + "-subj", + "/CN=127.0.0.1", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert, key) + origin = socket.socket() + origin.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + origin.bind(("127.0.0.1", 0)) + origin.listen(16) + self._origin = origin + self.origin_port = origin.getsockname()[1] + threading.Thread( + target=self._serve_origin, args=(context,), daemon=True + ).start() + + self._proc = subprocess.Popen( + [sys.executable, str(MOCK_PROXY), "0"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert self._proc.stdout is not None + assert self._proc.stderr is not None + threading.Thread(target=self._read_logs, args=(self._proc.stderr,), daemon=True).start() + line = self._proc.stdout.readline().strip() + if not line.startswith("PORT "): + self.stop() + raise RuntimeError( + "mock_connect_proxy did not report a port " + f"({line!r}). logs={''.join(self.logs)!r}" + ) + self.proxy_port = int(line.split()[1]) + + def stop(self) -> None: + self._stop.set() + if self._origin is not None: + try: + self._origin.close() + except OSError: + pass + if self._proc is not None and self._proc.poll() is None: + self._proc.terminate() + try: + self._proc.wait(timeout=3) + except subprocess.TimeoutExpired: + self._proc.kill() + + def _read_logs(self, stream: Any) -> None: + for line in stream: + self.logs.append(line) + + def _serve_origin(self, context: ssl.SSLContext) -> None: + assert self._origin is not None + self._origin.settimeout(0.5) + while not self._stop.is_set(): + try: + conn, _addr = self._origin.accept() + except socket.timeout: + continue + except OSError: + return + threading.Thread( + target=self._handle_origin, args=(context, conn), daemon=True + ).start() + + def _handle_origin(self, context: ssl.SSLContext, conn: socket.socket) -> None: + try: + tls = context.wrap_socket(conn, server_side=True) + tls.settimeout(5) + data = b"" + while b"\r\n\r\n" not in data: + chunk = tls.recv(4096) + if not chunk: + break + data += chunk + body = b"origin-body" + tls.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Content-Length: " + str(len(body)).encode() + b"\r\n" + b"Connection: close\r\n\r\n" + body + ) + try: + tls.shutdown(socket.SHUT_RDWR) + except OSError: + pass + tls.close() + except Exception: + try: + conn.close() + except OSError: + pass + + +class MockProxyFunctionalTests(unittest.TestCase): + """End-to-end CONNECT through the private hooks. Fails if they drift.""" + + tunnel: _LocalTunnel + + @classmethod + def setUpClass(cls) -> None: + import urllib3 + + urllib3.disable_warnings() + cls.tunnel = _LocalTunnel() + cls.tunnel.start() + + @classmethod + def tearDownClass(cls) -> None: + tunnel = getattr(cls, "tunnel", None) + if tunnel is not None: + tunnel.stop() + + def test_urllib3_prepare_proxy_reads_connect_headers(self) -> None: + import urllib3 + from python_proxy_headers.urllib3_proxy_manager import proxy_from_url + + context = ssl._create_unverified_context() + try: + with proxy_from_url( + self.tunnel.proxy_url, + proxy_headers=SEND_HEADER, + ssl_context=context, + retries=False, + ) as manager: + response = manager.request("GET", self.tunnel.origin_url, timeout=8.0) + except Exception as exc: + self.fail(self._hook("urllib3", "HTTPSProxyConnectionPool._prepare_proxy and HTTPConnection._tunnel / http.client._read_headers", exc)) + self.assertEqual(response.status, 200, _runtime()) + self.assertEqual(response.data, b"origin-body") + self._assert_merged( + response.headers, + "urllib3", + "HTTPSProxyConnectionPool._prepare_proxy", + urllib3.__version__, + ) + + def test_httpx_handle_request_merges_connect_headers(self) -> None: + self._require_httpx_0() + import httpcore + import httpx + from python_proxy_headers.httpx_proxy import ( + AsyncHTTPProxyTransport, + HTTPProxyTransport, + ) + + proxy = httpx.Proxy(self.tunnel.proxy_url, headers=SEND_HEADER) + try: + transport = HTTPProxyTransport(proxy=proxy, verify=False) + with httpx.Client( + mounts={"https://": transport}, + verify=False, + timeout=8.0, + ) as client: + response = client.get(self.tunnel.origin_url) + except Exception as exc: + self.fail( + self._hook( + "httpx", + "ProxyTunnelHTTPConnection.handle_request (copy of httpcore TunnelHTTPConnection.handle_request)", + exc, + ) + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.text, "origin-body") + self._assert_merged( + response.headers, + "httpx", + "ProxyTunnelHTTPConnection.handle_request", + f"httpx {httpx.__version__} / httpcore {httpcore.__version__}", + ) + + async def _async_get() -> Any: + async_proxy = httpx.Proxy(self.tunnel.proxy_url, headers=SEND_HEADER) + async_transport = AsyncHTTPProxyTransport(proxy=async_proxy, verify=False) + async with httpx.AsyncClient( + mounts={"https://": async_transport}, + verify=False, + timeout=8.0, + ) as client: + return await client.get(self.tunnel.origin_url) + + try: + async_response = asyncio.run(_async_get()) + except Exception as exc: + self.fail( + self._hook( + "httpx", + "AsyncProxyTunnelHTTPConnection.handle_async_request", + exc, + ) + ) + self._assert_merged( + async_response.headers, + "httpx", + "AsyncProxyTunnelHTTPConnection.handle_async_request", + f"httpx {httpx.__version__} / httpcore {httpcore.__version__}", + ) + + def test_aiohttp_create_proxy_connection_reads_connect_headers(self) -> None: + import aiohttp + from python_proxy_headers.aiohttp_proxy import ProxyClientSession + + async def _get() -> Dict[str, Optional[str]]: + timeout = aiohttp.ClientTimeout(total=8) + async with ProxyClientSession(timeout=timeout) as session: + async with session.get( + self.tunnel.origin_url, + proxy=self.tunnel.proxy_url, + proxy_headers=SEND_HEADER, + ssl=False, + ) as response: + body = await response.text() + return { + "status": str(response.status), + "body": body, + "ip": _header(response.headers, "X-ProxyMesh-IP"), + "echo": _header(response.headers, "X-ProxyMesh-Country-Echo"), + } + + try: + result = asyncio.run(_get()) + except Exception as exc: + self.fail( + self._hook( + "aiohttp", + "ProxyTCPConnector._create_proxy_connection (copy of TCPConnector._create_proxy_connection)", + exc, + ) + ) + self.assertEqual(result["status"], "200") + self.assertEqual(result["body"], "origin-body") + self.assertEqual( + result["ip"], + FAKE_IP, + "aiohttp copied _create_proxy_connection did not surface " + f"X-ProxyMesh-IP. aiohttp {aiohttp.__version__}. {_runtime()}. " + f"proxy log: {''.join(self.tunnel.logs)!r}", + ) + self.assertEqual(result["echo"], "US") + + def _require_httpx_0(self) -> None: + from python_proxy_headers.httpx_proxy import HTTPX_STATUS + + if HTTPX_STATUS.kind != "supported": + self.skipTest( + "httpx CONNECT checks need httpx 0.28.x; " + f"found {HTTPX_STATUS.kind} {HTTPX_STATUS.version}." + ) + + def _hook(self, library: str, hook: str, exc: BaseException) -> str: + return ( + f"{library} private hook failed ({hook}). A method this package " + f"overrides or copies no longer matches the installed release: " + f"{type(exc).__name__}: {exc}. {_runtime()}. " + f"proxy log: {''.join(self.tunnel.logs)!r}" + ) + + def _assert_merged(self, headers: Any, library: str, hook: str, version: str) -> None: + ip = _header(headers, "X-ProxyMesh-IP") + echo = _header(headers, "X-ProxyMesh-Country-Echo") + self.assertEqual( + ip, + FAKE_IP, + f"{library} {hook} did not merge X-ProxyMesh-IP from the CONNECT " + f"response ({version}). {_runtime()}. " + f"proxy log: {''.join(self.tunnel.logs)!r}", + ) + self.assertEqual( + echo, + "US", + f"{library} {hook} did not echo X-ProxyMesh-Country onto the " + f"CONNECT response ({version}). The request header never reached " + f"the proxy, or the response header was dropped. {_runtime()}.", + ) + + +class OptionalClientImportTests(unittest.TestCase): + """Import adapters for clients that are installed. Skip when they are not.""" + + def test_pycurl_module_imports(self) -> None: + self._import_pair("pycurl", "python_proxy_headers.pycurl_proxy") + + def test_cloudscraper_module_imports(self) -> None: + self._import_pair("cloudscraper", "python_proxy_headers.cloudscraper_proxy") + + def test_autoscraper_module_imports(self) -> None: + self._import_pair("autoscraper", "python_proxy_headers.autoscraper_proxy") + + def _import_pair(self, distribution: str, module_name: str) -> None: + try: + importlib.import_module(distribution) + except ImportError: + self.skipTest(f"{distribution} is not installed") + importlib.import_module(module_name) + + +def _run_python(code: str, extra_pythonpath: List[Path]) -> subprocess.CompletedProcess[str]: + script = Path(tempfile.mkdtemp()) / "check_httpx_import.py" + script.write_text(code, encoding="utf-8") + parts = [str(path) for path in extra_pythonpath] + parts.append(str(ROOT)) + existing = os.environ.get("PYTHONPATH") + if existing: + parts.append(existing) + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join(parts) + return subprocess.run( + [sys.executable, str(script)], + env=env, + text=True, + capture_output=True, + timeout=30, + check=False, + ) + + +def _status_kind(stdout: str) -> str: + for line in stdout.splitlines(): + if line.startswith("STATUS "): + return line.split()[1] + return "" + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/mock_connect_proxy.py b/tests/mock_connect_proxy.py new file mode 100644 index 0000000..56ef6bd --- /dev/null +++ b/tests/mock_connect_proxy.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Minimal CONNECT proxy for local adapter tests. No credentials. + +Replies to CONNECT with ``HTTP/1.1 200 Connection established``, +``X-ProxyMesh-IP``, and an echo of each ``X-ProxyMesh-*`` request header, +then pipes bytes. Plain HTTP absolute-URI requests are forwarded with +``X-ProxyMesh-IP`` added. + +Usage: + python3 tests/mock_connect_proxy.py [port] + +``port`` defaults to 18080. Use ``0`` to bind an ephemeral port. The chosen +port is written to stdout as ``PORT `` before the accept loop starts. +""" + +from __future__ import annotations + +import select +import socket +import sys +import threading + +FAKE_IP = "203.0.113.7" + +def pipe(a, b, label=""): + socks = [a, b] + n = [0, 0] + try: + while True: + r, _, _ = select.select(socks, [], [], 60) + if not r: + return + for s in r: + data = s.recv(65536) + if not data: + return + n[0 if s is a else 1] += len(data) + (b if s is a else a).sendall(data) + except OSError: + pass + finally: + sys.stderr.write(f"[proxy] tunnel {label} closed: client->up {n[0]}B, up->client {n[1]}B\n") + a.close(); b.close() + +def read_head(c): + buf = b"" + while b"\r\n\r\n" not in buf: + d = c.recv(4096) + if not d: + break + buf += d + head, _, rest = buf.partition(b"\r\n\r\n") + return head, rest + +def handle(c): + try: + head, rest = read_head(c) + lines = head.decode("latin-1").split("\r\n") + method, target, _ = lines[0].split(" ", 2) + hdrs = [l.split(":", 1) for l in lines[1:] if ":" in l] + echo = "".join(f"{k.strip()}-Echo: {v.strip()}\r\n" for k, v in hdrs if k.lower().startswith("x-proxymesh")) + sys.stderr.write(f"[proxy] {method} {target} hdrs={[k for k,_ in hdrs]}\n") + if method == "CONNECT": + host, port = target.rsplit(":", 1) + up = socket.create_connection((host, int(port)), timeout=20) + c.sendall(("HTTP/1.1 200 Connection established\r\n" + f"X-ProxyMesh-IP: {FAKE_IP}\r\n{echo}\r\n").encode()) + if rest: + up.sendall(rest) + pipe(c, up, target) + else: + from urllib.parse import urlsplit + u = urlsplit(target) + up = socket.create_connection((u.hostname, u.port or 80), timeout=20) + path = (u.path or "/") + (("?" + u.query) if u.query else "") + out = [f"{method} {path} HTTP/1.1"] + [f"{k}:{v}" for k, v in hdrs if not k.lower().startswith("proxy-") and k.lower() != "connection"] + ["Connection: close"] + up.sendall(("\r\n".join(out) + "\r\n\r\n").encode("latin-1") + rest) + resp = b"" + while True: + d = up.recv(65536) + if not d: break + resp += d + h, _, body = resp.partition(b"\r\n\r\n") + c.sendall(h + f"\r\nX-ProxyMesh-IP: {FAKE_IP}".encode() + b"\r\n\r\n" + body) + c.close(); up.close() + except Exception as e: + sys.stderr.write(f"[proxy] error {e!r}\n") + try: c.close() + except OSError: pass + +def serve(port: int = 18080) -> None: + """Bind ``127.0.0.1:port`` and serve until the process is killed.""" + listener = socket.socket() + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", port)) + bound = listener.getsockname()[1] + listener.listen(128) + sys.stdout.write(f"PORT {bound}\n") + sys.stdout.flush() + sys.stderr.write(f"[proxy] listening on 127.0.0.1:{bound}\n") + while True: + client, _addr = listener.accept() + threading.Thread(target=handle, args=(client,), daemon=True).start() + + +def main(argv: list[str] | None = None) -> None: + """Parse ``argv`` and run :func:`serve`.""" + args = list(sys.argv[1:] if argv is None else argv) + port = int(args[0]) if args else 18080 + serve(port) + + +if __name__ == "__main__": + main()