Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .agents/skills/python-proxy-headers/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ with ProxySession(proxy_headers={'X-ProxyMesh-Country': 'US'}) as session:

### httpx

Requires `httpx>=0.28,<1` (`pip install 'python-proxy-headers[httpx]'`). httpx 1.0 drops httpcore and proxy support. Importing `httpx_proxy` still succeeds and warns; calling it raises `UnsupportedHttpxVersion`.

```python
from python_proxy_headers.httpx_proxy import get

Expand Down
75 changes: 75 additions & 0 deletions .github/workflows/client_compatibility.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# Newest stable releases and pre-releases of every supported client.
# Not triggered on pull_request, so a red run is visible in Actions and via
# workflow_dispatch, and it is not a required PR check.
# Pre-release failures must stay failing (no continue-on-error).
# Runners are ubuntu-24.04. Do not switch these jobs to ubuntu-latest.

name: Client compatibility

on:
schedule:
- cron: "0 15 * * 1"
workflow_dispatch:

permissions:
contents: read

jobs:
clients:
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
python-version: ["3.12", "3.14"]
channel: [stable, pre]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}

- name: Install system dependencies (pycurl)
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev

- name: Install package
run: |
python -m pip install --upgrade pip
pip install -e .

- name: Install ${{ matrix.channel }} clients
run: |
set -euo pipefail
packages=(urllib3 requests aiohttp httpx pycurl cloudscraper autoscraper)
if [ "${{ matrix.channel }}" = "pre" ]; then
pip install --pre --upgrade "${packages[@]}"
else
pip install --upgrade "${packages[@]}"
fi
python - <<'PY'
import importlib.metadata as metadata
names = (
"urllib3",
"requests",
"aiohttp",
"httpx",
"httpcore",
"pycurl",
"cloudscraper",
"autoscraper",
)
for name in names:
try:
print(f"{name}=={metadata.version(name)}")
except metadata.PackageNotFoundError:
print(f"{name}==not-installed")
PY

- name: Run header security unit tests
run: python test_header_security.py

- name: Run local mock-proxy internal tests
run: python test_client_internals.py
35 changes: 23 additions & 12 deletions .github/workflows/proxy_integration_tests.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,13 @@
# Integration tests against a real proxy (PROXY_URL). Add repository secret PROXY_URL
# under Settings → Secrets and variables → Actions, then mark this workflow as a required
# status check under branch protection (pull_request events only receive secrets for PRs
# from the same repository, not from forks).
# Pull-request checks.
# - unit: Python matrix, compatible client extras, header tests, and a local
# CONNECT mock (no PROXY_URL).
# - integration: real proxy via the PROXY_URL secret. pull_request events only
# receive secrets for PRs from this repository, not from forks.
#
# Pre-release clients are not installed here. The weekly Client compatibility
# workflow runs `pip install --pre` and is not a pull_request check, so a
# pre-release failure cannot block a PR.
# Runners are ubuntu-24.04. Do not switch these jobs to ubuntu-latest.

name: Proxy integration tests

Expand All @@ -14,6 +20,10 @@ permissions:
jobs:
unit:
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.12", "3.13", "3.14"]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
Expand All @@ -22,20 +32,22 @@ jobs:
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.x"
python-version: ${{ matrix.python-version }}

- name: Install system dependencies (pycurl)
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev

- name: Install package and test dependencies
- name: Install package and compatible client extras
run: |
python -m pip install --upgrade pip
pip install -e .
pip install urllib3 requests aiohttp httpx pycurl
pip install -e ".[urllib3,requests,httpx,aiohttp,pycurl,cloudscraper,autoscraper]"

- name: Run header security unit tests
run: python test_header_security.py

- name: Run local mock-proxy internal tests
run: python test_client_internals.py

integration:
runs-on: ubuntu-24.04

Expand All @@ -47,16 +59,15 @@ jobs:
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.x"
python-version: "3.12"

- name: Install system dependencies (pycurl)
run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev

- name: Install package and test dependencies
- name: Install package and compatible client extras
run: |
python -m pip install --upgrade pip
pip install -e .
pip install urllib3 requests aiohttp httpx cloudscraper autoscraper pycurl
pip install -e ".[urllib3,requests,httpx,aiohttp,pycurl,cloudscraper,autoscraper]"

- name: Require PROXY_URL Actions secret
env:
Expand Down
27 changes: 27 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Changelog

## Unreleased

### httpx 1.0

- Importing `python_proxy_headers.httpx_proxy` no longer raises `ImportError: cannot import name AsyncHTTPTransport` when httpx 1.x is installed.
- httpx 1.0.dev6 (2026-08-31) is a rewrite without httpcore, without `HTTPTransport` / `AsyncHTTPTransport`, and without proxy support ([encode/httpx#947](https://github.com/encode/httpx/issues/947)). The module warns at import time and raises `UnsupportedHttpxVersion` if you call it.
- The `[httpx]` extra is `httpx>=0.28,<1`. There is no 1.0 adapter in this change. Options are documented in `docs/httpx.rst`: stay on 0.28, wait for a public CONNECT hook, or add a small backend after `NetworkBackend` can take an existing tunnel. Vendoring httpcore is not one of them.

### Tests

- Contract checks and a local CONNECT mock (`tests/mock_connect_proxy.py`, no proxy credentials) exercise the private hooks in `urllib3_proxy_manager.py` (`_prepare_proxy`, `_tunnel`, `http.client._read_headers`), `httpx_proxy` (copied httpcore `handle_request` / `handle_async_request`), and `aiohttp_proxy.py` (copied `_create_proxy_connection`). They name the upstream symbol when it moves.

### Packaging

- Keywords, trove classifiers, and optional extras: `[urllib3]`, `[requests]`, `[httpx]`, `[aiohttp]`, `[pycurl]`, `[cloudscraper]`, `[autoscraper]`.
- `requires-python` is `>=3.10`. urllib3 2.8, requests 2.34, and aiohttp 3.14 all require 3.10+, and the httpx adapter already uses `str | None`.
- The changelog URL points at this file.

## 0.2.3

CONNECT header injection hardening and safer merging of proxy response headers. See the [GitHub release](https://github.com/proxymesh/python-proxy-headers/releases/tag/v0.2.3).

## Earlier

[GitHub Releases](https://github.com/proxymesh/python-proxy-headers/releases).
24 changes: 21 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,21 @@ This library solves both problems for popular Python HTTP libraries.
pip install python-proxy-headers
```

Then install the HTTP library you want to use (e.g., `pip install requests`).
Or install the client extra you need. Each extra pins a range that CI tests:

> **Note:** This package has no dependencies by default - install only what you need.
```bash
pip install 'python-proxy-headers[requests]'
pip install 'python-proxy-headers[urllib3]'
pip install 'python-proxy-headers[httpx]'
pip install 'python-proxy-headers[aiohttp]'
pip install 'python-proxy-headers[pycurl]'
pip install 'python-proxy-headers[cloudscraper]'
pip install 'python-proxy-headers[autoscraper]'
```

The `httpx` extra is `httpx>=0.28,<1`. httpx 1.0 drops proxy support. Importing `python_proxy_headers.httpx_proxy` still succeeds, and using it raises `UnsupportedHttpxVersion`. See the [httpx docs](https://python-proxy-headers.readthedocs.io/en/latest/httpx.html).

> **Note:** This package has no required dependencies - install only the extra you need.

## Quick Start

Expand Down Expand Up @@ -122,6 +134,10 @@ response = scraper.get('https://example.com')
A test harness is included to verify proxy header functionality:

```bash
# Header checks and a local CONNECT mock (no proxy credentials)
python test_header_security.py
python test_client_internals.py

# Set your proxy
export PROXY_URL='http://user:pass@proxy.example.com:8080'

Expand All @@ -142,8 +158,10 @@ For detailed documentation, API reference, and more examples:
- **Full Documentation:** [python-proxy-headers.readthedocs.io](https://python-proxy-headers.readthedocs.io/en/latest/)
- **Example Code:** [proxy-examples for Python](https://github.com/proxymesh/proxy-examples/tree/main/python)

## Related Projects
## Other languages

- **[javascript-proxy-headers](https://github.com/proxymesh/javascript-proxy-headers)** - Send and read custom proxy headers on HTTPS CONNECT for Node.js HTTP clients
- **[ruby-proxy-headers](https://github.com/proxymesh/ruby-proxy-headers)** - Send and read custom proxy headers on HTTPS CONNECT for Ruby HTTP clients
- **[scrapy-proxy-headers](https://github.com/proxymesh/scrapy-proxy-headers)** - Proxy header support for Scrapy

## About
Expand Down
26 changes: 25 additions & 1 deletion docs/httpx.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,30 @@ httpx

`HTTPX <https://www.python-httpx.org/>`_ is a fully featured HTTP client for Python 3, which provides sync and async APIs, and support for both HTTP/1.1 and HTTP/2. This page describes how to use httpx with proxies and how to send and receive custom proxy headers.

httpx 1.0
---------

httpx 1.0 (pre-release ``1.0.dev6``, 2026-08-31) is a rewrite. It no longer depends on httpcore, it does not provide ``HTTPTransport`` or ``AsyncHTTPTransport``, and it has no proxy or CONNECT support (`encode/httpx#947 <https://github.com/encode/httpx/issues/947>`_).

Installing that release does not crash ``import python_proxy_headers.httpx_proxy``. Import warns, and calling ``get``, ``HTTPProxyTransport``, or ``AsyncHTTPProxyTransport`` raises ``UnsupportedHttpxVersion`` instead of ``ImportError``.

The ``httpx`` extra stays on the 0.28 line::

pip install 'python-proxy-headers[httpx]'

which installs ``httpx>=0.28,<1``. This release does not include a 1.0 adapter.

Support options
~~~~~~~~~~~~~~~

1. **Stay on httpx 0.28.x.** This is the supported path. The adapter copies httpcore's private ``handle_request`` and cannot load on the 1.0 rewrite.

2. **Wait for a public proxy API.** httpx 1.0's ``NetworkBackend.connect`` / ``connect_tls`` and ``Transport.send`` open a direct connection. They cannot send CONNECT headers or return the proxy's response headers. When `issue 947 <https://github.com/encode/httpx/issues/947>`_ grows a public hook, implement against that instead of another private copy.

3. **A small CONNECT backend, later.** A future adapter could open the proxy socket, write CONNECT plus the custom headers, read the proxy response, wrap TLS, and pass the stream to ``httpx.Connection``. ``NetworkBackend`` does not accept an existing socket today, and the 1.0 API is still early, so this is not in this release.

4. **Do not vendor httpcore** to pretend 1.0 works. That would ignore the client the user installed.

Getting Started
---------------

Expand All @@ -14,7 +38,7 @@ This section shows you how to quickly get up and running with proxy headers in h

.. code-block:: bash

pip install python-proxy-headers httpx
pip install 'python-proxy-headers[httpx]'

2. Import the module:

Expand Down
13 changes: 10 additions & 3 deletions docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,27 @@ None of these modules provide good support for parsing custom response headers f

If you are looking for `Scrapy <https://scrapy.org/>`_ support, please see our `scrapy-proxy-headers <https://github.com/proxymesh/scrapy-proxy-headers>`_ project.

The same CONNECT header support in other languages:

* `javascript-proxy-headers <https://github.com/proxymesh/javascript-proxy-headers>`_
* `ruby-proxy-headers <https://github.com/proxymesh/ruby-proxy-headers>`_

Installation
------------

To use these extension modules, you must first do the following:

1. Install the package:
1. Install the package, optionally with the client you use:

.. code-block:: bash

pip install python-proxy-headers
pip install 'python-proxy-headers[requests]'
pip install 'python-proxy-headers[httpx]'

2. Install the appropriate package based on the Python module you want to use.
2. Or install the HTTP library yourself. Extras pin the ranges we test. The ``httpx`` extra is ``httpx>=0.28,<1``. httpx 1.0 is not supported yet; see :doc:`httpx`.

This package does not have any dependencies because we don't know which module you want to use.
This package does not have any required dependencies because we don't know which module you want to use.

You can also find more example code in our `proxy-examples for python <https://github.com/proxymesh/proxy-examples/tree/main/python>`_.

Expand Down
47 changes: 42 additions & 5 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,56 @@ authors = [
]
description = "Handle custom proxy headers for http & https requests in various python libraries"
readme = "README.md"
requires-python = ">=3.8"
requires-python = ">=3.10"
keywords = [
"proxy",
"proxy-headers",
"https-proxy",
"connect",
"tunnel",
"requests",
"urllib3",
"httpx",
"aiohttp",
"web-scraping",
"proxymesh",
]
classifiers = [
"Development Status :: 4 - Beta",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Operating System :: OS Independent",
"License :: OSI Approved :: MIT License",
"Intended Audience :: Developers",
"Topic :: Internet :: WWW/HTTP",
"License :: OSI Approved :: MIT License",
"Intended Audience :: Developers",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Internet :: Proxy Servers",
"Topic :: Software Development :: Libraries :: Python Modules",
"Framework :: AsyncIO",
]

# Newest releases checked on 2026-10-23: urllib3 2.8.0, requests 2.34.2,
# httpx 0.28.1 (httpcore 1.0.9), aiohttp 3.14.5, pycurl 7.48.0,
# cloudscraper 1.2.71, autoscraper 1.1.14.
# Lower bounds are the oldest minor whose private hooks still match the
# adapters. CI installs the newest release inside each range. httpx 1.x is
# excluded: 1.0.dev6 drops httpcore and proxy support (encode/httpx#947).
[project.optional-dependencies]
urllib3 = ["urllib3>=2.2,<3"]
requests = ["requests>=2.31,<3"]
httpx = ["httpx>=0.28,<1"]
aiohttp = ["aiohttp>=3.10,<4"]
pycurl = ["pycurl>=7.45,<8"]
cloudscraper = ["cloudscraper>=1.2.71,<2"]
autoscraper = ["autoscraper>=1.1.14,<2"]

[project.urls]
Homepage = "https://github.com/proxymesh/python-proxy-headers"
Changelog = "https://github.com/proxymesh/python-proxy-headers/commits/main/"
Changelog = "https://github.com/proxymesh/python-proxy-headers/blob/main/CHANGELOG.md"
Issues = "https://github.com/proxymesh/python-proxy-headers/issues"
Documentation = "https://python-proxy-headers.readthedocs.io/en/latest/"
ProxyMesh = "https://proxymesh.com"
Loading