diff --git a/.agents/skills/ruby-proxy-headers/SKILL.md b/.agents/skills/ruby-proxy-headers/SKILL.md index 08cb09d..a442c7d 100644 --- a/.agents/skills/ruby-proxy-headers/SKILL.md +++ b/.agents/skills/ruby-proxy-headers/SKILL.md @@ -2,7 +2,7 @@ name: ruby-proxy-headers description: >- Send and receive custom headers during HTTPS CONNECT tunneling in Ruby. - Use when integrating proxy headers with Net::HTTP, Faraday, HTTParty, or Excon. + Use when integrating proxy headers with Net::HTTP, Faraday, HTTParty, Excon, HTTP.rb, Typhoeus, or RestClient. --- # ruby-proxy-headers @@ -31,6 +31,9 @@ Install faraday, httparty, or excon as needed. | Faraday | `ruby_proxy_headers/faraday` | `FaradayIntegration.connection` | | HTTParty | `ruby_proxy_headers/httparty` | `ProxyHeadersConnectionAdapter` | | Excon | `ruby_proxy_headers/excon` | `ExconIntegration.get` (send-only) | +| HTTP.rb | `ruby_proxy_headers/http_gem` | `HTTPGem.get` | +| Typhoeus | `ruby_proxy_headers/typhoeus` | `Typhoeus.get` (HTTPS via Net::HTTP) | +| RestClient | `ruby_proxy_headers/rest_client` | `RestClient.get` | ## Usage Patterns diff --git a/.github/workflows/latest_client_gems.yml b/.github/workflows/latest_client_gems.yml new file mode 100644 index 0000000..5e24436 --- /dev/null +++ b/.github/workflows/latest_client_gems.yml @@ -0,0 +1,43 @@ +# Weekly check that the adapters still work with the newest client gems allowed +# by the gemspec. No lockfile: bundle update resolves the latest versions. +# Local mock-proxy specs do not need the PROXY_URL secret. + +name: Latest client gems + +on: + schedule: + - cron: '0 6 * * 1' + workflow_dispatch: + +permissions: + contents: read + +jobs: + latest: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + ruby-version: ['3.2', '3.3', '3.4'] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install libcurl and Python + run: sudo apt-get update && sudo apt-get install -y libcurl4 python3 + + - name: Set up Ruby ${{ matrix.ruby-version }} + uses: ruby/setup-ruby@v1 # kept unpinned per maintainer guidance — pinning freezes available Ruby versions + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: false + + - name: Install latest client gems + run: | + rm -f Gemfile.lock + bundle update + + - name: Run specs and mock-proxy integration tests + run: bundle exec rspec diff --git a/.github/workflows/proxy_integration_tests.yml b/.github/workflows/proxy_integration_tests.yml index a94d23d..cfc0a79 100644 --- a/.github/workflows/proxy_integration_tests.yml +++ b/.github/workflows/proxy_integration_tests.yml @@ -20,6 +20,9 @@ jobs: with: persist-credentials: false + - name: Install libcurl + run: sudo apt-get update && sudo apt-get install -y libcurl4 + - name: Set up Ruby uses: ruby/setup-ruby@v1 # kept unpinned per maintainer guidance — pinning freezes available Ruby versions with: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 6bf1db3..2cfa8f6 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -12,13 +12,16 @@ jobs: strategy: matrix: - ruby-version: ['3.2', '3.3'] + ruby-version: ['3.2', '3.3', '3.4'] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Install libcurl and Python + run: sudo apt-get update && sudo apt-get install -y libcurl4 python3 + - name: Set up Ruby ${{ matrix.ruby-version }} uses: ruby/setup-ruby@v1 # kept unpinned per maintainer guidance — pinning freezes available Ruby versions with: @@ -27,3 +30,49 @@ jobs: - name: Run RSpec run: bundle exec rspec + + # Plaintext CONNECT on every supported net-http, plus HTTPS-proxy / timeouted_connect + # where that release has them. Kept off the main Gemfile so faraday-net_http's + # net-http ~> 0.5 constraint does not hide older default gems. + net-http: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - ruby-version: '3.2' + net-http-version: '0.3.2' + - ruby-version: '3.3' + net-http-version: '0.4.1' + - ruby-version: '3.3' + net-http-version: '0.5.0' + - ruby-version: '3.4' + net-http-version: '0.6.0' + - ruby-version: '3.3' + net-http-version: '0.9.1' + - ruby-version: '3.4' + net-http-version: '0.10.0' + + env: + BUNDLE_GEMFILE: gemfiles/net_http_matrix.gemfile + NET_HTTP_VERSION: ${{ matrix.net-http-version }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install Python + run: sudo apt-get update && sudo apt-get install -y python3 + + - name: Set up Ruby ${{ matrix.ruby-version }} + uses: ruby/setup-ruby@v1 # kept unpinned per maintainer guidance — pinning freezes available Ruby versions + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: false + + - name: Install net-http ${{ matrix.net-http-version }} + run: bundle install + + - name: Run net-http compatibility specs + run: bundle exec rspec spec/net_http_compat_spec.rb spec/connect_header_injection_spec.rb diff --git a/.gitignore b/.gitignore index f01362c..ff29e08 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ *.swp .env .ruby-version +spec/examples.txt +gemfiles/*.lock diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e112f2..d48fed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- HTTP.rb, RestClient, and Typhoeus no longer raise `NoMethodError` on HTTPS requests. They send custom CONNECT headers through `RubyProxyHeaders::NetHTTP.request` and expose the proxy's CONNECT response headers. +- The Net::HTTP patch no longer replaces `Net::HTTP#connect` with a stale copy. Socket setup, `timeouted_connect`, and HTTPS proxies (`proxy_use_ssl`) stay in the installed net-http. +- The Faraday adapter no longer overrides faraday-net_http's private `request_with_wrapped_block`. CONNECT response headers are merged from the public adapter hooks. + +### Changed + +- Gem metadata now includes `homepage_uri`, `changelog_uri`, and `bug_tracker_uri`. +- The published gem no longer ships `IMPLEMENTATION_PRIORITY.md`, `LIBRARY_RESEARCH.md`, or `DEFERRED.md`. Per-library guides under `docs/` are included instead. +- Faraday, HTTParty, and Excon guides describe `FaradayIntegration`, `ProxyHeadersConnectionAdapter`, and `ExconIntegration` instead of middleware and helpers that were never implemented. + ## [0.1.0] - 2024-02-28 ### Added @@ -23,5 +35,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Comprehensive documentation - Test suite with RSpec -[Unreleased]: https://github.com/proxymeshai/ruby-proxy-headers/compare/v0.1.0...HEAD -[0.1.0]: https://github.com/proxymeshai/ruby-proxy-headers/releases/tag/v0.1.0 +[Unreleased]: https://github.com/proxymesh/ruby-proxy-headers/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/proxymesh/ruby-proxy-headers/releases/tag/v0.1.0 diff --git a/DEFERRED.md b/DEFERRED.md index c2babae..cfda260 100644 --- a/DEFERRED.md +++ b/DEFERRED.md @@ -1,12 +1,10 @@ # Deferred / not fully supported -## Typhoeus / Ethon +## Typhoeus / Ethon native proxy headers -**Status:** Not implemented in this gem. +**Status:** HTTPS works by delegating to patched Net::HTTP. Native libcurl is not used for CONNECT headers. -[Ethon](https://github.com/typhoeus/ethon) does not expose `CURLOPT_PROXYHEADER` (or equivalent) in its `Ethon::Easy` option mapping as of ethon 0.18.x. Adding CONNECT response header capture would require Ethon/libcurl changes or a custom C extension. - -**Workaround:** Use `Net::HTTP` + {RubyProxyHeaders::NetHTTP.patch!} or Faraday with `ruby_proxy_headers_net_http` adapter. +[Ethon](https://github.com/typhoeus/ethon) does not expose `CURLOPT_PROXYHEADER` (or equivalent) in its `Ethon::Easy` option mapping as of ethon 0.18.x. `RubyProxyHeaders::Typhoeus` therefore sends HTTPS requests that need proxy headers through `RubyProxyHeaders::NetHTTP.request`, which can both set CONNECT headers and read the proxy's CONNECT response. A future version may call libcurl directly. --- diff --git a/Gemfile.lock b/Gemfile.lock index 2043da1..534271b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -39,6 +39,10 @@ GEM ffi (1.17.4-x86_64-linux-gnu) ffi (1.17.4-x86_64-linux-musl) hashdiff (1.2.1) + http (6.0.4) + http-cookie (~> 1.0) + llhttp (~> 0.6.1) + http-accept (1.7.0) http-cookie (1.1.0) domain_name (~> 0.5) httparty (0.24.3) @@ -48,6 +52,7 @@ GEM json (3.1.0) language_server-protocol (3.17.0.6) lint_roller (1.1.0) + llhttp (0.6.2) logger (1.7.0) mechanize (2.14.1) addressable (~> 2.8) @@ -77,6 +82,7 @@ GEM connection_pool (>= 2.2.4, < 4) net-protocol (0.4.0) timeout + netrc (0.11.0) nkf (0.2.0) nokogiri (1.19.4) mini_portile2 (~> 2.8.2) @@ -107,6 +113,11 @@ GEM rainbow (3.1.1) rake (13.4.2) regexp_parser (2.13.1) + rest-client (2.1.0) + http-accept (>= 1.7.0, < 2.0) + http-cookie (>= 1.0.2, < 2.0) + mime-types (>= 1.16, < 4.0) + netrc (~> 0.8) rexml (3.4.4) rspec (3.13.2) rspec-core (~> 3.13.0) @@ -169,9 +180,11 @@ DEPENDENCIES excon (~> 1.4) faraday (~> 2.14) faraday-net_http (~> 3.4) + http (~> 6.0) httparty (~> 0.24) mechanize (~> 2.14) rake (~> 13.0) + rest-client (~> 2.1) rspec (~> 3.12) rubocop (~> 1.50) ruby-proxy-headers! @@ -203,11 +216,14 @@ CHECKSUMS ffi (1.17.4-x86_64-linux-gnu) sha256=9d3db14c2eae074b382fa9c083fe95aec6e0a1451da249eab096c34002bc752d ffi (1.17.4-x86_64-linux-musl) sha256=3fdf9888483de005f8ef8d1cf2d3b20d86626af206cbf780f6a6a12439a9c49e hashdiff (1.2.1) sha256=9c079dbc513dfc8833ab59c0c2d8f230fa28499cc5efb4b8dd276cf931457cd1 + http (6.0.4) sha256=3768f902489ea8309424f1881c668af8d3ca73675648a755dfe2cb0030d42d26 + http-accept (1.7.0) sha256=c626860682bfbb3b46462f8c39cd470fd7b0584f61b3cc9df5b2e9eb9972a126 http-cookie (1.1.0) sha256=38a5e60d1527eebc396831b8c4b9455440509881219273a6c99943d29eadbb19 httparty (0.24.3) sha256=5c95cf29ca70d5bfd834e56140ddfca6e94166c3b870cde9c25d157cdb5a00d2 json (3.1.0) sha256=3ce7ccf8fde58960177a3a756c31f292a4c1e1dedee884547096a5b0604e3bd8 language_server-protocol (3.17.0.6) sha256=5ef2c0c138f8267e1bc631d3328347d354f96724b0af22f2c79516120443b7f0 lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87 + llhttp (0.6.2) sha256=3c3c59aafb1e1594ab2f45293478f697458f5a91eb0f0db4b27f61a064024982 logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 mechanize (2.14.1) sha256=b66205133a01d3dc3bc0cbc5e99888e5476818398d8d315193429f21488abafd mime-types (3.7.0) sha256=dcebf61c246f08e15a4de34e386ebe8233791e868564a470c3fe77c00eed5e56 @@ -219,6 +235,7 @@ CHECKSUMS net-http-digest_auth (1.4.1) sha256=4b8ad50ed8d180a58db5d6c49449b987dd0466fe01e24037945bc007562a08db net-http-persistent (4.0.8) sha256=ef3de8319d691537b329053fae3a33195f8b070bbbfae8bf1a58c796081960e6 net-protocol (0.4.0) sha256=f9f90c8c6f64794a45160bf422177a336db12809de3beeeaa26b88fcd2e8c853 + netrc (0.11.0) sha256=de1ce33da8c99ab1d97871726cba75151113f117146becbe45aa85cb3dabee3f nkf (0.2.0) sha256=fbc151bda025451f627fafdfcb3f4f13d0b22ae11f58c6d3a2939c76c5f5f126 nokogiri (1.19.4) sha256=50c951611c92bca05c51411aef45f1cbc50f2821c4802758c5c6d34696533ab5 nokogiri (1.19.4-aarch64-linux-gnu) sha256=1269fb644a6de405057a53dd5c762b1209b43ca7424f839454d3dbc677c31a8f @@ -237,6 +254,7 @@ CHECKSUMS rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701 regexp_parser (2.13.1) sha256=5aedb6b7c35688f51e86eef17a15374dfd287c83f4ca644b2c5f3ec50a33b44b + rest-client (2.1.0) sha256=35a6400bdb14fae28596618e312776c158f7ebbb0ccad752ff4fa142bf2747e3 rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587 rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d diff --git a/MARKETING_PLAN.md b/MARKETING_PLAN.md index 62c7f26..e2b15dc 100644 --- a/MARKETING_PLAN.md +++ b/MARKETING_PLAN.md @@ -43,7 +43,7 @@ Keep the messaging accurate: this project should market implemented integrations ## 1) Package and Registry Presence -- Publish and maintain on RubyGems (`ruby_proxy_headers`). +- Publish and maintain on RubyGems (`ruby-proxy-headers`). - Ensure gem metadata is complete: homepage, source code URI, changelog URI, bug tracker URI. - Keep release cadence visible with changelog updates and GitHub releases. diff --git a/README.md b/README.md index 97cb3c6..7252c7f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ -[![Gem Version](https://badge.fury.io/rb/ruby-proxy-headers.svg)](https://badge.fury.io/rb/ruby-proxy-headers) # Ruby Proxy Headers +[![Gem Version](https://badge.fury.io/rb/ruby-proxy-headers.svg)](https://badge.fury.io/rb/ruby-proxy-headers) + Send and receive custom proxy headers during HTTPS `CONNECT` tunneling in modern Ruby HTTP workflows (for example [ProxyMesh](https://proxymesh.com) `X-ProxyMesh-IP` / `X-ProxyMesh-Country`). ## The problem @@ -10,7 +11,7 @@ Most Ruby HTTP clients use `Net::HTTP`, Faraday, or libcurl without exposing: 1. Extra headers on the `CONNECT` request to the proxy. 2. Headers from the proxy’s `CONNECT` response (often discarded after the tunnel is established). -This library adds opt-in support for **Net::HTTP**, **Faraday** (2.x via `faraday-net_http`), **HTTParty**, and documents **Excon**’s built-in `:ssl_proxy_headers` for sends. +This library adds opt-in support for **Net::HTTP**, **Faraday** (2.x via `faraday-net_http`), **HTTParty**, **HTTP.rb**, **Typhoeus**, and **RestClient**, and documents **Excon**’s built-in `:ssl_proxy_headers` for sends. ## Why teams use this @@ -30,7 +31,7 @@ Or add to your `Gemfile`: gem 'ruby-proxy-headers' ``` -The `Net::HTTP` patch is pure Ruby. Install **faraday**, **faraday-net_http**, **httparty**, and/or **excon** when you use those integrations. +The `Net::HTTP` patch is pure Ruby. Install **faraday**, **faraday-net_http**, **httparty**, **http**, **typhoeus**, **rest-client**, and/or **excon** when you use those integrations. ## Net::HTTP @@ -94,9 +95,55 @@ HTTParty.get( puts RubyProxyHeaders.proxy_connect_response_headers['X-ProxyMesh-IP'] ``` +## HTTP.rb + +HTTPS requests go through patched `Net::HTTP`, because HTTP.rb does not expose CONNECT headers. Plain HTTP keeps using the `http` gem. + +```ruby +require 'ruby_proxy_headers/http_gem' + +response = RubyProxyHeaders::HTTPGem.get( + 'https://api.ipify.org?format=json', + proxy: ENV.fetch('PROXY_URL'), + proxy_headers: { 'X-ProxyMesh-Country' => 'US' } +) +puts response.body +puts response.proxy_response_headers['X-ProxyMesh-IP'] +``` + +## Typhoeus + +Ethon does not expose libcurl's `CURLOPT_PROXYHEADER`. HTTPS requests that send proxy headers use patched `Net::HTTP`, so the proxy's CONNECT response headers are readable. + +```ruby +require 'ruby_proxy_headers/typhoeus' + +response = RubyProxyHeaders::Typhoeus.get( + 'https://api.ipify.org?format=json', + proxy: ENV.fetch('PROXY_URL'), + proxy_headers: { 'X-ProxyMesh-Country' => 'US' } +) +puts response.body +puts response.proxy_response_headers['X-ProxyMesh-IP'] +``` + +## RestClient + +```ruby +require 'ruby_proxy_headers/rest_client' + +response = RubyProxyHeaders::RestClient.get( + 'https://api.ipify.org?format=json', + proxy: ENV.fetch('PROXY_URL'), + proxy_headers: { 'X-ProxyMesh-Country' => 'US' } +) +puts response.body +puts response.proxy_response_headers['X-ProxyMesh-IP'] +``` + ## Excon (send-only; CONNECT response headers) -Excon supports **sending** extra headers on CONNECT with `:ssl_proxy_headers`. Reading `X-ProxyMesh-IP` from the CONNECT response is **not** exposed on the origin response object — see [DEFERRED.md](DEFERRED.md). +Excon supports **sending** extra headers on CONNECT with `:ssl_proxy_headers`. Reading `X-ProxyMesh-IP` from the CONNECT response is **not** exposed on the origin response object — see [DEFERRED.md](https://github.com/proxymesh/ruby-proxy-headers/blob/main/DEFERRED.md). ```ruby require 'ruby_proxy_headers/excon' @@ -127,8 +174,11 @@ export PROXY_URL=http://user:pass@proxyhost:port bundle exec ruby test/test_proxy_headers.rb -v ``` -## Related +`bundle exec rspec` does not need a live proxy. It includes specs that start [test/mock_connect_proxy.py](https://github.com/proxymesh/ruby-proxy-headers/blob/main/test/mock_connect_proxy.py) on `127.0.0.1` and check that the custom header arrives on `CONNECT` and that the proxy response headers can be read. + +## Other languages - [python-proxy-headers](https://github.com/proxymesh/python-proxy-headers) - [javascript-proxy-headers](https://github.com/proxymesh/javascript-proxy-headers) +- [scrapy-proxy-headers](https://github.com/proxymesh/scrapy-proxy-headers) - [proxy-examples (Ruby)](https://github.com/proxymesh/proxy-examples/tree/main/ruby) diff --git a/docs/excon.md b/docs/excon.md index ba51702..2398f4f 100644 --- a/docs/excon.md +++ b/docs/excon.md @@ -1,84 +1,29 @@ # Excon -[Excon](https://github.com/excon/excon) is a fast, simple HTTP(S) client. The `RubyProxyHeaders::Excon` module adds proxy header support. +[Excon](https://github.com/excon/excon) already sends extra CONNECT headers through `:ssl_proxy_headers`. `RubyProxyHeaders::ExconIntegration.get` maps a proxy URL and header hash onto that option. + +Excon does not expose the CONNECT response on the origin response, so `X-ProxyMesh-IP` is not readable from this helper. ## Installation ```ruby gem 'excon' -gem 'ruby_proxy_headers' +gem 'ruby-proxy-headers' ``` -## Basic Usage +## Basic usage ```ruby -require 'ruby_proxy_headers' +require 'ruby_proxy_headers/excon' -response = RubyProxyHeaders::Excon.get( +response = RubyProxyHeaders::ExconIntegration.get( 'https://api.ipify.org?format=json', - proxy: 'http://user:pass@proxy.example.com:8080', - proxy_headers: { 'X-ProxyMesh-Country' => 'US' } -) - -puts "Status: #{response.code}" -puts "Body: #{response.body}" -puts "Proxy IP: #{response.proxy_response_headers['x-proxymesh-ip']}" -``` - -## API Reference - -### `RubyProxyHeaders::Excon.get(url, options)` - -Make a GET request. - -**Parameters:** - -- `url` (String) - Target URL -- `options` (Hash): - - `:proxy` (String) - Proxy URL (required) - - `:proxy_headers` (Hash) - Headers to send to proxy - - `:headers` (Hash) - Request headers - -**Returns:** `ProxyResponse` - -### `RubyProxyHeaders::Excon.post(url, options)` - -Make a POST request. - -**Parameters:** - -- Same as `get`, plus: - - `:body` (String) - Request body - -### `RubyProxyHeaders::Excon.request(method, url, options)` - -Make a request with any HTTP method. - -## Response Object - -```ruby -response.status # HTTP status code (alias: code) -response.body # Response body -response.headers # Response headers -response.proxy_response_headers # Proxy CONNECT response headers -``` - -## Example: AWS SDK Compatible - -Excon is commonly used with AWS SDKs. Here's how to use proxy headers with AWS: - -```ruby -require 'ruby_proxy_headers' - -response = RubyProxyHeaders::Excon.get( - 'https://sts.amazonaws.com/?Action=GetCallerIdentity&Version=2011-06-15', - proxy: ENV['PROXY_URL'], - proxy_headers: { 'X-ProxyMesh-Country' => 'US' }, - headers: { - 'Authorization' => aws_signature, - 'X-Amz-Date' => Time.now.utc.strftime('%Y%m%dT%H%M%SZ') - } + proxy_url: 'http://user:pass@proxy.example.com:8080', + proxy_connect_headers: { 'X-ProxyMesh-Country' => 'US' } ) +puts response.status puts response.body ``` + +`proxy_connect_headers` is sent as Excon's `:ssl_proxy_headers`. Extra keyword arguments are merged into the `Excon.get` options. A proxy URL without a scheme is treated as `http://`. diff --git a/docs/faraday.md b/docs/faraday.md index d688688..13e2e1d 100644 --- a/docs/faraday.md +++ b/docs/faraday.md @@ -1,122 +1,44 @@ # Faraday -[Faraday](https://lostisland.github.io/faraday/) is a popular HTTP client library with a middleware architecture. The `RubyProxyHeaders::Faraday` module provides middleware for proxy header support. +[Faraday](https://lostisland.github.io/faraday/) 2.x talks to `Net::HTTP` through `faraday-net_http`. This gem registers the adapter `:ruby_proxy_headers_net_http`, which uses Faraday's public `net_http_connection` and `call` hooks. It does not override the private `request_with_wrapped_block` method. ## Installation -Add Faraday to your Gemfile: - ```ruby gem 'faraday' -gem 'ruby_proxy_headers' -``` - -## Basic Usage - -### Using Middleware - -```ruby -require 'faraday' -require 'ruby_proxy_headers' - -conn = Faraday.new(url: 'https://api.ipify.org') do |f| - f.use RubyProxyHeaders::Faraday::Middleware, - proxy: 'http://user:pass@proxy.example.com:8080', - proxy_headers: { 'X-ProxyMesh-Country' => 'US' } - f.adapter Faraday.default_adapter -end - -response = conn.get('/?format=json') - -puts "Status: #{response.status}" -puts "Body: #{response.body}" -puts "Proxy IP: #{response.env[:proxy_response_headers]['x-proxymesh-ip']}" +gem 'faraday-net_http' +gem 'ruby-proxy-headers' ``` -### Using Helper Method +## Basic usage ```ruby -require 'ruby_proxy_headers' +require 'ruby_proxy_headers/faraday' -conn = RubyProxyHeaders::Faraday.create_connection( - 'https://api.ipify.org', - proxy: 'http://user:pass@proxy:8080', - proxy_headers: { 'X-ProxyMesh-Country' => 'US' } +conn = RubyProxyHeaders::FaradayIntegration.connection( + proxy: 'http://user:pass@proxy.example.com:8080', + proxy_connect_headers: { 'X-ProxyMesh-Country' => 'US' } # optional ) +response = conn.get('https://api.ipify.org?format=json') -response = conn.get('/?format=json') +puts response.status puts response.body +puts response.headers['X-ProxyMesh-IP'] ``` -## Middleware Options - -### `RubyProxyHeaders::Faraday::Middleware` - -**Options:** +`FaradayIntegration.connection` calls `RubyProxyHeaders::NetHTTP.patch!`, builds a Faraday connection with the registered adapter, and sets `proxy_connect_request_headers` on the `Net::HTTP` object. CONNECT response headers are merged into `response.headers` (existing response headers are left in place). -- `:proxy` (String) - Proxy URL (required) -- `:proxy_headers` (Hash) - Headers to send to proxy during CONNECT -- `:on_proxy_connect` (Proc) - Callback when proxy connection is established - -### Callback Example +Pass a block to configure the builder, including TLS verification for a lab proxy: ```ruby -conn = Faraday.new do |f| - f.use RubyProxyHeaders::Faraday::Middleware, - proxy: 'http://proxy:8080', - proxy_headers: { 'X-Country' => 'US' }, - on_proxy_connect: ->(headers) { - puts "Connected via: #{headers['x-proxymesh-ip']}" - } - f.adapter Faraday.default_adapter +conn = RubyProxyHeaders::FaradayIntegration.connection( + proxy: ENV.fetch('PROXY_URL'), + proxy_connect_headers: { 'X-ProxyMesh-Country' => 'US' } +) do |f| + f.ssl.verify = false end ``` -## Accessing Proxy Headers - -Proxy response headers are stored in the Faraday environment: - -```ruby -response = conn.get('/') -proxy_headers = response.env[:proxy_response_headers] - -puts proxy_headers['x-proxymesh-ip'] -puts proxy_headers['x-proxymesh-country'] -``` - -## API Reference - -### `RubyProxyHeaders::Faraday.create_connection(url, options)` - -Create a Faraday connection with proxy header support. - -**Parameters:** - -- `url` (String) - Base URL -- `options` (Hash): - - `:proxy` (String) - Proxy URL (required) - - `:proxy_headers` (Hash) - Headers to send to proxy - - Additional Faraday options - -**Returns:** `Faraday::Connection` +## HTTPS proxies -## Example: Session Persistence - -```ruby -require 'faraday' -require 'ruby_proxy_headers' - -session_id = "session-#{SecureRandom.hex(8)}" - -conn = Faraday.new do |f| - f.use RubyProxyHeaders::Faraday::Middleware, - proxy: 'http://user:pass@proxyhost:port', - proxy_headers: { 'X-ProxyMesh-Session' => session_id } - f.adapter Faraday.default_adapter -end - -5.times do |i| - response = conn.get('https://api.ipify.org?format=json') - puts "Request #{i + 1}: #{response.body}" -end -``` +An `https://` proxy URL is passed through to Faraday. `faraday-net_http` sets `proxy_use_ssl` on `Net::HTTP` itself, and the patch leaves that path on the installed net-http. diff --git a/docs/getting-started.md b/docs/getting-started.md index 252153c..ad295d6 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -9,7 +9,7 @@ This guide will help you get started with ruby-proxy-headers. Add to your `Gemfile`: ```ruby -gem 'ruby_proxy_headers' +gem 'ruby-proxy-headers' ``` Then run: @@ -21,7 +21,7 @@ bundle install ### Direct Installation ```bash -gem install ruby_proxy_headers +gem install ruby-proxy-headers ``` ## Basic Usage @@ -49,34 +49,39 @@ Choose the module that matches your HTTP library: === "Faraday" ```ruby - require 'faraday' - - conn = Faraday.new do |f| - f.use RubyProxyHeaders::Faraday::Middleware, - proxy: 'http://user:pass@proxy:8080', - proxy_headers: { 'X-Custom-Header' => 'value' } - f.adapter Faraday.default_adapter - end - + require 'ruby_proxy_headers/faraday' + + conn = RubyProxyHeaders::FaradayIntegration.connection( + proxy: 'http://user:pass@proxy:8080', + proxy_connect_headers: { 'X-Custom-Header' => 'value' } + ) response = conn.get('https://example.com') + puts response.headers['X-ProxyMesh-IP'] ``` === "HTTParty" ```ruby - response = RubyProxyHeaders::HTTParty.get( + require 'httparty' + require 'ruby_proxy_headers/httparty' + + RubyProxyHeaders::NetHTTP.patch! + proxy = URI('http://user:pass@proxy:8080') + response = HTTParty.get( 'https://example.com', - proxy: 'http://user:pass@proxy:8080', - proxy_headers: { 'X-Custom-Header' => 'value' } + http_proxyaddr: proxy.host, + http_proxyport: proxy.port, + http_proxyuser: proxy.user, + http_proxypass: proxy.password, + proxy_connect_request_headers: { 'X-Custom-Header' => 'value' }, + connection_adapter: RubyProxyHeaders::ProxyHeadersConnectionAdapter ) + puts RubyProxyHeaders.proxy_connect_response_headers['X-ProxyMesh-IP'] ``` ### 3. Access Proxy Response Headers -```ruby -# All modules provide proxy_response_headers -puts response.proxy_response_headers['x-proxymesh-ip'] -``` +`NetHTTP.get` returns `response.proxy_response_headers`. Faraday merges the same headers into `response.headers`. HTTParty leaves them on `RubyProxyHeaders.proxy_connect_response_headers`. Lookup is case-insensitive. ## Configuration Options @@ -102,31 +107,10 @@ proxy_headers: { ## Environment Variables -You can configure the proxy via environment variables: +The live proxy harness in `test/test_proxy_headers.rb` reads `PROXY_URL`. The library methods take the proxy URL as an argument; they do not fall back to `HTTPS_PROXY`. ```bash export PROXY_URL='http://user:pass@proxy.example.com:8080' -export HTTPS_PROXY='http://proxy.example.com:8080' -``` - -The libraries will use `HTTPS_PROXY` or `HTTP_PROXY` as fallback if no proxy is specified. - -## Error Handling - -```ruby -begin - response = RubyProxyHeaders::NetHTTP.get( - 'https://example.com', - proxy: 'http://proxy:8080', - proxy_headers: { 'X-Custom' => 'value' } - ) -rescue RubyProxyHeaders::ProxyAuthenticationError => e - puts "Proxy auth failed: #{e.message}" -rescue RubyProxyHeaders::ConnectError => e - puts "Proxy connection failed: #{e.message}" -rescue StandardError => e - puts "Error: #{e.message}" -end ``` ## Next Steps diff --git a/docs/http-gem.md b/docs/http-gem.md index e2285d4..7c60d73 100644 --- a/docs/http-gem.md +++ b/docs/http-gem.md @@ -6,7 +6,7 @@ ```ruby gem 'http' -gem 'ruby_proxy_headers' +gem 'ruby-proxy-headers' ``` ## Basic Usage diff --git a/docs/httparty.md b/docs/httparty.md index bb3f1e8..768ead4 100644 --- a/docs/httparty.md +++ b/docs/httparty.md @@ -1,136 +1,36 @@ # HTTParty -[HTTParty](https://github.com/jnunemaker/httparty) makes HTTP fun! The `RubyProxyHeaders::HTTParty` module provides both standalone functions and a mixin for class-based usage. +[HTTParty](https://github.com/jnunemaker/httparty) uses `Net::HTTP`. `RubyProxyHeaders::ProxyHeadersConnectionAdapter` copies CONNECT request headers onto that connection. Read the proxy's CONNECT response from the thread-local `RubyProxyHeaders.proxy_connect_response_headers`. ## Installation ```ruby gem 'httparty' -gem 'ruby_proxy_headers' +gem 'ruby-proxy-headers' ``` -## Basic Usage - -### Standalone Functions +## Basic usage ```ruby -require 'ruby_proxy_headers' +require 'httparty' +require 'ruby_proxy_headers/httparty' + +RubyProxyHeaders::NetHTTP.patch! -response = RubyProxyHeaders::HTTParty.get( +proxy = URI('http://user:pass@proxy.example.com:8080') +response = HTTParty.get( 'https://api.ipify.org?format=json', - proxy: 'http://user:pass@proxy.example.com:8080', - proxy_headers: { 'X-ProxyMesh-Country' => 'US' } + http_proxyaddr: proxy.host, + http_proxyport: proxy.port, + http_proxyuser: proxy.user, + http_proxypass: proxy.password, + proxy_connect_request_headers: { 'X-ProxyMesh-Country' => 'US' }, + connection_adapter: RubyProxyHeaders::ProxyHeadersConnectionAdapter ) -puts "Status: #{response.code}" -puts "Body: #{response.body}" -puts "Proxy IP: #{response.proxy_response_headers['x-proxymesh-ip']}" -``` - -### Class-Based Usage - -```ruby -require 'httparty' -require 'ruby_proxy_headers' - -class ProxyClient - include HTTParty - include RubyProxyHeaders::HTTParty - - base_uri 'https://api.ipify.org' - http_proxy 'proxy.example.com', 8080, 'user', 'pass' - proxy_headers 'X-ProxyMesh-Country' => 'US' -end - -response = ProxyClient.get('/?format=json') +puts response.code puts response.body -puts response.proxy_response_headers['x-proxymesh-ip'] +puts RubyProxyHeaders.proxy_connect_response_headers['X-ProxyMesh-IP'] ``` -## API Reference - -### Standalone Methods - -#### `RubyProxyHeaders::HTTParty.get(url, options)` - -Make a GET request. - -**Parameters:** - -- `url` (String) - Target URL -- `options` (Hash): - - `:proxy` (String) - Proxy URL (required) - - `:proxy_headers` (Hash) - Headers to send to proxy - - `:headers` (Hash) - Request headers - -**Returns:** Response with `proxy_response_headers` accessor - -#### `RubyProxyHeaders::HTTParty.post(url, options)` - -Make a POST request. - -**Parameters:** - -- Same as `get`, plus: - - `:body` (String/Hash) - Request body - -### Class Methods (when including module) - -#### `proxy_headers(headers)` - -Set default proxy headers for all requests. - -```ruby -class MyClient - include HTTParty - include RubyProxyHeaders::HTTParty - - proxy_headers 'X-Country' => 'US', 'X-Session' => 'abc' -end -``` - -## Example: Multiple Countries - -```ruby -require 'ruby_proxy_headers' - -COUNTRIES = %w[US UK DE FR JP] - -COUNTRIES.each do |country| - response = RubyProxyHeaders::HTTParty.get( - 'https://api.ipify.org?format=json', - proxy: ENV['PROXY_URL'], - proxy_headers: { 'X-ProxyMesh-Country' => country } - ) - - puts "#{country}: #{response.body}" -end -``` - -## Example: Persistent Client - -```ruby -require 'httparty' -require 'ruby_proxy_headers' - -class IPChecker - include HTTParty - include RubyProxyHeaders::HTTParty - - base_uri 'https://api.ipify.org' - format :json - - http_proxy 'proxyhost', 1234, ENV['PROXY_USER'], ENV['PROXY_PASS'] - proxy_headers 'X-ProxyMesh-Session' => 'persistent' - - def self.check_ip - response = get('/?format=json') - { - ip: response['ip'], - proxy_ip: response.proxy_response_headers['x-proxymesh-ip'] - } - end -end - -puts IPChecker.check_ip -``` +`proxy_connect_headers` is accepted as an alias of `proxy_connect_request_headers`. Call `NetHTTP.patch!` once before the first request. The thread-local headers are whatever the most recent CONNECT on this thread returned. diff --git a/docs/index.md b/docs/index.md index dd26ae8..cee3276 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,15 +29,15 @@ Standard Ruby HTTP libraries support basic proxy authentication but **do not** a ## Supported Libraries -| Library | Module | Status | -|---------|--------|--------| -| Net::HTTP | `RubyProxyHeaders::NetHTTP` | ✅ | -| Faraday | `RubyProxyHeaders::Faraday` | ✅ | -| HTTParty | `RubyProxyHeaders::HTTParty` | ✅ | -| HTTP.rb | `RubyProxyHeaders::HTTPGem` | ✅ | -| Typhoeus | `RubyProxyHeaders::Typhoeus` | ✅ | -| Excon | `RubyProxyHeaders::Excon` | ✅ | -| RestClient | `RubyProxyHeaders::RestClient` | ✅ | +| Library | Entry point | Status | +|---------|------------|--------| +| Net::HTTP | `RubyProxyHeaders::NetHTTP` | Send and read | +| Faraday | `RubyProxyHeaders::FaradayIntegration` | Send and read | +| HTTParty | `RubyProxyHeaders::ProxyHeadersConnectionAdapter` | Send and read | +| HTTP.rb | `RubyProxyHeaders::HTTPGem` | Send and read (HTTPS via Net::HTTP) | +| Typhoeus | `RubyProxyHeaders::Typhoeus` | Send and read (HTTPS via Net::HTTP) | +| RestClient | `RubyProxyHeaders::RestClient` | Send and read (HTTPS via Net::HTTP) | +| Excon | `RubyProxyHeaders::ExconIntegration` | Send only (`:ssl_proxy_headers`) | ## Quick Start @@ -79,7 +79,7 @@ Many proxy services use custom headers for: Add to your Gemfile: ```ruby -gem 'ruby_proxy_headers' +gem 'ruby-proxy-headers' ``` Then run: @@ -91,15 +91,15 @@ bundle install Or install directly: ```bash -gem install ruby_proxy_headers +gem install ruby-proxy-headers ``` ## Requirements -- Ruby >= 2.7 +- Ruby >= 3.1 - OpenSSL (included in standard Ruby) ## Related Projects - [python-proxy-headers](https://github.com/proxymesh/python-proxy-headers) - Python version -- [javascript-proxy-headers](https://github.com/proxymeshai/javascript-proxy-headers) - JavaScript/Node.js version +- [javascript-proxy-headers](https://github.com/proxymesh/javascript-proxy-headers) - JavaScript/Node.js version diff --git a/docs/rest-client.md b/docs/rest-client.md index 4d183a4..71ae45b 100644 --- a/docs/rest-client.md +++ b/docs/rest-client.md @@ -6,7 +6,7 @@ ```ruby gem 'rest-client' -gem 'ruby_proxy_headers' +gem 'ruby-proxy-headers' ``` ## Basic Usage diff --git a/docs/testing.md b/docs/testing.md index 51a3c3e..d582729 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -119,27 +119,29 @@ puts "Proxy Headers: #{response.proxy_response_headers}" ### GitHub Actions ```yaml -# .github/workflows/test.yml -name: Tests +# .github/workflows/unit_tests.yml +name: Unit tests -on: [push, pull_request] +on: [pull_request] jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: matrix: - ruby: ['2.7', '3.0', '3.1', '3.2', '3.3'] - + ruby: ['3.2', '3.3', '3.4'] + steps: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: ruby-version: ${{ matrix.ruby }} bundler-cache: true - - run: bundle exec rake spec + - run: bundle exec rspec ``` +`bundle exec rspec` includes the local mock-proxy specs in `spec/mock_proxy_integration_spec.rb`. Those start `test/mock_connect_proxy.py` on 127.0.0.1 and do not need `PROXY_URL`. The live script below still talks to a real proxy. + ## Troubleshooting ### Connection Timeouts diff --git a/docs/typhoeus.md b/docs/typhoeus.md index 60b9a23..4fb6c48 100644 --- a/docs/typhoeus.md +++ b/docs/typhoeus.md @@ -6,7 +6,7 @@ ```ruby gem 'typhoeus' -gem 'ruby_proxy_headers' +gem 'ruby-proxy-headers' ``` **Note:** Typhoeus requires libcurl to be installed on your system. diff --git a/gemfiles/net_http_matrix.gemfile b/gemfiles/net_http_matrix.gemfile new file mode 100644 index 0000000..7dfc2c6 --- /dev/null +++ b/gemfiles/net_http_matrix.gemfile @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# Isolated bundle for spec/net_http_compat_spec.rb. The main gemspec pulls +# faraday-net_http, which requires net-http ~> 0.5 and would refuse older +# releases. Set NET_HTTP_VERSION to the gem version under test. +source 'https://rubygems.org' + +gem 'net-http', ENV.fetch('NET_HTTP_VERSION', '0.10.0') +gem 'rspec', '~> 3.12' diff --git a/lib/ruby_proxy_headers/faraday.rb b/lib/ruby_proxy_headers/faraday.rb index 2e089bc..4b0b956 100644 --- a/lib/ruby_proxy_headers/faraday.rb +++ b/lib/ruby_proxy_headers/faraday.rb @@ -14,27 +14,51 @@ module RubyProxyHeaders # Faraday adapter that merges proxy CONNECT response headers into Faraday response headers. # Use with {RubyProxyHeaders::NetHTTP.patch!}. + # + # faraday-net_http 3.4+ implements the request on the private method + # +request_with_wrapped_block+. This adapter does not override that method. + # It uses the public +net_http_connection+ and +call+ hooks so a rename of the + # private method still leaves CONNECT response headers on the Faraday response. module FaradayAdapter class NetHttp < ::Faraday::Adapter::NetHttp - def request_with_wrapped_block(http, env, &block) - res = super + def net_http_connection(env) + super.tap { |http| faraday_http_stack << http } + end + + def call(env) + depth = faraday_http_stack.size + response = super + http = faraday_http_stack.last if faraday_http_stack.size > depth merge_proxy_headers(env, http) - res + response + ensure + pop_faraday_http_stack(depth) end private + def faraday_http_stack + Thread.current[:ruby_proxy_headers_faraday_stack] ||= [] + end + + def pop_faraday_http_stack(depth) + return if depth.nil? + + stack = faraday_http_stack + stack.pop while stack.size > depth + end + def merge_proxy_headers(env, http) - return unless env[:response_headers] + return unless env.respond_to?(:[]) && env[:response_headers] return unless http.respond_to?(:last_proxy_connect_response_headers) - ph = http.last_proxy_connect_response_headers - return unless ph.is_a?(Hash) + proxy_headers = http.last_proxy_connect_response_headers + return unless proxy_headers.respond_to?(:each) - ph.each do |k, v| - next if v.nil? + proxy_headers.each do |key, value| + next if value.nil? - env[:response_headers][k] ||= v + env[:response_headers][key] ||= value end end end diff --git a/lib/ruby_proxy_headers/http_gem.rb b/lib/ruby_proxy_headers/http_gem.rb index 0a1eb2b..f6142d4 100644 --- a/lib/ruby_proxy_headers/http_gem.rb +++ b/lib/ruby_proxy_headers/http_gem.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative '../ruby_proxy_headers' + module RubyProxyHeaders # HTTP.rb (http gem) integration for proxy headers support. # @@ -64,7 +66,8 @@ def delete(url, **options) def request(method, url, **options) uri = URI.parse(url) - # Use our core connection for HTTPS + # HTTPS CONNECT headers go through patched Net::HTTP. HTTP.rb does not + # expose the proxy CONNECT request or its response headers. if uri.scheme == 'https' response = RubyProxyHeaders::NetHTTP.request( method, @@ -72,22 +75,28 @@ def request(method, url, **options) proxy: @proxy, proxy_headers: @proxy_headers, headers: options[:headers], - body: options[:body] + body: options[:body], + verify_ssl: options.fetch(:verify_ssl, true), + open_timeout: options[:open_timeout], + read_timeout: options[:read_timeout] ) @last_proxy_response_headers = response.proxy_response_headers return response end - # For HTTP, use standard http gem require 'http' proxy_uri = URI.parse(@proxy) + passthrough = options.dup + passthrough.delete(:verify_ssl) + passthrough.delete(:open_timeout) + passthrough.delete(:read_timeout) ::HTTP.via( proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password - ).send(method, url, **options) + ).send(method, url, **passthrough) end end end diff --git a/lib/ruby_proxy_headers/net_http.rb b/lib/ruby_proxy_headers/net_http.rb index 23a2f27..34ad1f8 100644 --- a/lib/ruby_proxy_headers/net_http.rb +++ b/lib/ruby_proxy_headers/net_http.rb @@ -1,16 +1,55 @@ # frozen_string_literal: true require 'net/http' -require 'net/protocol' require 'openssl' -require 'resolv' -require 'timeout' +require 'uri' + +# Loaded for header validation. Safe if this file is required first: the parent +# file requires us back and Ruby drops the cycle. +require_relative '../ruby_proxy_headers' module RubyProxyHeaders - # Patches Net::HTTP#connect for HTTPS-over-proxy so extra headers can be sent on - # the CONNECT request and response headers can be read (e.g. ProxyMesh - # X-ProxyMesh-IP). Based on the same tunnel flow as Python's http.client / urllib3 - # extensions in python-proxy-headers. + # Case-insensitive view of HTTP headers. Keys are stored downcased, which is + # how Net::HTTP yields them, and looked up regardless of case. + class HeaderHash < Hash + def [](key) + return nil if key.nil? + + super(normalize(key)) + end + + def []=(key, value) + super(normalize(key), value) + end + + def store(key, value) + self[key] = value + end + + def fetch(key, *args, &block) + super(normalize(key), *args, &block) + end + + def key?(key) + super(normalize(key)) + end + alias has_key? key? + alias include? key? + alias member? key? + + private + + def normalize(key) + key.to_s.downcase + end + end + + # Patches Net::HTTP so HTTPS-through-proxy can send extra CONNECT headers and + # keep the proxy's CONNECT response headers. + # + # The patch does not copy +Net::HTTP#connect+. Upstream owns socket setup, + # +timeouted_connect+, and HTTPS-to-proxy (+@proxy_use_ssl+). We only wrap + # that method and adjust the CONNECT bytes it writes. # # @example # require 'ruby_proxy_headers/net_http' @@ -22,158 +61,286 @@ module RubyProxyHeaders # res = http.request(Net::HTTP::Get.new(uri)) # p http.last_proxy_connect_response_headers module NetHTTP - unless const_defined?(:ORIGINAL_CONNECT, false) - ORIGINAL_CONNECT = ::Net::HTTP.instance_method(:connect) + REQUEST_CLASSES = { + get: ::Net::HTTP::Get, + post: ::Net::HTTP::Post, + put: ::Net::HTTP::Put, + delete: ::Net::HTTP::Delete, + head: ::Net::HTTP::Head, + patch: ::Net::HTTP::Patch, + options: ::Net::HTTP::Options + }.freeze + + # Inserts +headers+ before the terminating blank line of a CONNECT request. + def self.inject_connect_headers(buf, headers) + return buf if headers.nil? || headers.empty? + + validate_request_headers!(headers) + unless buf.end_with?("\r\n\r\n") + raise ArgumentError, + 'upstream CONNECT request has no header terminator; ' \ + 'ruby-proxy-headers cannot add proxy headers' + end + + extra = String.new(encoding: buf.encoding) + headers.each do |key, value| + extra << "#{key}: #{value}\r\n".encode(buf.encoding) + end + "#{buf.sub(/\r\n\r\n\z/, "\r\n")}#{extra}\r\n" end - module Extension - attr_accessor :proxy_connect_request_headers - attr_reader :last_proxy_connect_response_headers + def self.validate_request_headers!(headers) + return if headers.nil? - def connect - if use_ssl? && proxy? - connect_with_proxy_tunnel - else - RubyProxyHeaders::NetHTTP::ORIGINAL_CONNECT.bind_call(self) - end + headers.each do |key, value| + RubyProxyHeaders.validate_header!(key, value) end + end - private + def self.headers_from(response) + headers = HeaderHash.new + response.each_header { |key, value| headers[key] = value } + headers + end - # Duplicates Net::HTTP#connect (MRI 3.2) for the HTTPS + proxy branch, with - # optional extra CONNECT headers and capture of the CONNECT response headers. - def connect_with_proxy_tunnel - s = nil - if use_ssl? - @ssl_context = OpenSSL::SSL::SSLContext.new - end + # HTTPS CONNECT through a proxy, with optional extra CONNECT headers. + # + # @param method [Symbol, String] + # @param url [String] + # @param proxy [String] proxy URL (http:// or https://) + # @param proxy_headers [Hash, nil] + # @param headers [Hash, nil] origin request headers + # @param body [String, nil] + # @param verify_ssl [Boolean] + # @return [ProxyResponse] + def self.request(method, url, proxy:, proxy_headers: nil, headers: nil, body: nil, + verify_ssl: true, open_timeout: nil, read_timeout: nil) + patch! + uri = URI(url) + proxy_uri = parse_proxy(proxy) + http = build_http( + uri, proxy_uri, + verify_ssl: verify_ssl, + open_timeout: open_timeout, + read_timeout: read_timeout + ) + http.proxy_connect_request_headers = proxy_headers if proxy_headers&.any? - if use_ssl? && proxy? - # Fail before any socket I/O: CONNECT interpolates these into a raw request line. - RubyProxyHeaders.validate_connect_target!(conn_address, @port) - RubyProxyHeaders.validate_connect_target!(@address, @port) - end + req = request_class(method).new(uri.request_uri) + headers&.each { |key, value| req[key.to_s] = value } + req.body = body if !body.nil? && req.request_body_permitted? - if proxy? - conn_addr = proxy_address - conn_port = proxy_port - else - conn_addr = conn_address - conn_port = port + response = http.request(req) + ProxyResponse.new(response, http.last_proxy_connect_response_headers) + end + + def self.get(url, **options) + request(:get, url, **options) + end + + def self.post(url, body = nil, **options) + body = options.delete(:body) if body.nil? && options.key?(:body) + request(:post, url, body: body, **options) + end + + def self.put(url, body = nil, **options) + body = options.delete(:body) if body.nil? && options.key?(:body) + request(:put, url, body: body, **options) + end + + def self.delete(url, **options) + request(:delete, url, **options) + end + + def self.parse_proxy(proxy) + value = proxy.to_s.strip + value = "http://#{value}" unless value.match?(%r{\A[a-z][a-z0-9+\-.]*://}i) + URI.parse(value) + end + + def self.build_http(uri, proxy_uri, verify_ssl:, open_timeout:, read_timeout:) + http = ::Net::HTTP.new( + uri.host, + uri.port, + proxy_uri.host, + proxy_uri.port, + decode_component(proxy_uri.user), + decode_component(proxy_uri.password) + ) + if proxy_uri.scheme == 'https' + unless http.respond_to?(:proxy_use_ssl=) + version = defined?(::Net::HTTP::VERSION) ? ::Net::HTTP::VERSION : 'unknown' + raise ArgumentError, + "HTTPS proxies require net-http >= 0.5 (proxy_use_ssl); loaded net-http #{version}" end + http.proxy_use_ssl = true + end + http.use_ssl = uri.scheme == 'https' + unless verify_ssl + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + http.verify_hostname = false if http.respond_to?(:verify_hostname=) + end + http.open_timeout = open_timeout if open_timeout + http.read_timeout = read_timeout if read_timeout + http + end + + def self.decode_component(value) + return nil if value.nil? - debug "opening connection to #{conn_addr}:#{conn_port}..." - s = Timeout.timeout(@open_timeout, Net::OpenTimeout) do - begin - TCPSocket.open(conn_addr, conn_port, @local_host, @local_port) - rescue StandardError => e - raise e, "Failed to open TCP connection to " \ - "#{conn_addr}:#{conn_port} (#{e.message})" + URI.decode_www_form_component(value) + rescue ArgumentError + value + end + + def self.request_class(method) + REQUEST_CLASSES.fetch(method.to_s.downcase.to_sym) do + raise ArgumentError, "unsupported HTTP method: #{method.inspect}" + end + end + + # Rewrites the CONNECT request while a patched connection is opening. + module ConnectRequestWrite + def write(*strs) + http = Thread.current[:ruby_proxy_headers_active_http] + if http && strs.all? { |part| part.is_a?(String) } + joined = strs.join + if joined.start_with?('CONNECT ') + strs = [ + RubyProxyHeaders::NetHTTP.inject_connect_headers( + joined, + http.proxy_connect_request_headers + ) + ] end end - s.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) - debug 'opened' - - if use_ssl? - if proxy? - plain_sock = Net::BufferedIO.new(s, read_timeout: @read_timeout, - write_timeout: @write_timeout, - continue_timeout: @continue_timeout, - debug_output: @debug_output) - buf = +"CONNECT #{conn_address}:#{@port} HTTP/#{::Net::HTTP::HTTPVersion}\r\n" \ - "Host: #{@address}:#{@port}\r\n" - if proxy_user - credential = ["#{proxy_user}:#{proxy_pass}"].pack('m0') - buf << "Proxy-Authorization: Basic #{credential}\r\n" - end - if proxy_connect_request_headers&.any? - proxy_connect_request_headers.each do |k, v| - RubyProxyHeaders.validate_header!(k, v) - buf << "#{k}: #{v}\r\n" - end - end - buf << "\r\n" - plain_sock.write(buf) - - proxy_res = ::Net::HTTPResponse.read_new(plain_sock) - @last_proxy_connect_response_headers = {} - proxy_res.each_header do |k, v| - @last_proxy_connect_response_headers[k] = v - end - Thread.current[:ruby_proxy_headers_connect_headers] = @last_proxy_connect_response_headers.dup - proxy_res.value - end + super(*strs) + end + end - ssl_parameters = {} - iv_list = instance_variables - Net::HTTP::SSL_IVNAMES.each_with_index do |ivname, i| - if iv_list.include?(ivname) - value = instance_variable_get(ivname) - unless value.nil? - ssl_parameters[Net::HTTP::SSL_ATTRIBUTES[i]] = value - end - end - end - @ssl_context.set_params(ssl_parameters) - unless @ssl_context.session_cache_mode.nil? - @ssl_context.session_cache_mode = - OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | - OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE - end - if @ssl_context.respond_to?(:session_new_cb) - @ssl_context.session_new_cb = proc { |sock, sess| @ssl_session = sess } - end + # Records CONNECT response headers. Active only while #connect is running, + # so the later origin response is left alone. + module ConnectResponseRead + def read_new(sock) + response = super + http = Thread.current[:ruby_proxy_headers_active_http] + if http + headers = RubyProxyHeaders::NetHTTP.headers_from(response) + http.instance_variable_set(:@last_proxy_connect_response_headers, headers) + Thread.current[:ruby_proxy_headers_connect_headers] = headers.dup + end + response + end + end - verify_hostname = @ssl_context.verify_hostname + module Extension + attr_accessor :proxy_connect_request_headers + attr_reader :last_proxy_connect_response_headers - case @address - when Resolv::IPv4::Regex, Resolv::IPv6::Regex - @ssl_context.verify_hostname = false - else - ssl_host_address = @address - end + def connect + return super unless use_ssl? && proxy? - debug "starting SSL for #{conn_addr}:#{conn_port}..." - s = OpenSSL::SSL::SSLSocket.new(s, @ssl_context) - s.sync_close = true - s.hostname = ssl_host_address if s.respond_to?(:hostname=) && ssl_host_address + # Reject smuggled targets and header values before any socket I/O. + # `super` is upstream Net::HTTP#connect (timeouted_connect, HTTPS proxies). + RubyProxyHeaders.validate_connect_target!(conn_address, @port) + RubyProxyHeaders.validate_connect_target!(@address, @port) + RubyProxyHeaders::NetHTTP.validate_request_headers!(proxy_connect_request_headers) - if @ssl_session && - Process.clock_gettime(Process::CLOCK_REALTIME) < @ssl_session.time.to_f + @ssl_session.timeout - s.session = @ssl_session - end - ssl_socket_connect(s, @open_timeout) - if (@ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE) && verify_hostname - s.post_connection_check(@address) - end - debug "SSL established, protocol: #{s.ssl_version}, cipher: #{s.cipher[0]}" + @last_proxy_connect_response_headers = nil + previous = Thread.current[:ruby_proxy_headers_active_http] + begin + Thread.current[:ruby_proxy_headers_active_http] = self + super + ensure + Thread.current[:ruby_proxy_headers_active_http] = previous end - @socket = Net::BufferedIO.new(s, read_timeout: @read_timeout, - write_timeout: @write_timeout, - continue_timeout: @continue_timeout, - debug_output: @debug_output) - @last_communicated = nil - on_connect - rescue StandardError => e - if s - debug "Conn close because of connect error #{e}" - s.close + end + end + + class ProxyResponse + attr_reader :proxy_response_headers, :body, :headers, :response + + def initialize(response, proxy_headers) + @response = response + @body = response.body + @headers = RubyProxyHeaders::NetHTTP.headers_from(response) + @proxy_response_headers = proxy_headers || HeaderHash.new + end + + def code + @response.code.to_i + end + + def success? + (200..299).cover?(code) + end + + def method_missing(name, *args, &block) + if @response.respond_to?(name) + @response.public_send(name, *args, &block) + else + super end - raise + end + + def respond_to_missing?(name, include_private = false) + @response.respond_to?(name, include_private) || super + end + end + + # Applies {Extension} to subclasses created after the parent was patched. + # WebMock's Net::HTTP stand-in is created before +patch!+, so it is patched + # explicitly as well. + module SubclassInstaller + def inherited(subclass) + super + RubyProxyHeaders::NetHTTP.send(:install_extension, subclass) end end class << self def patch! - return if @patched - - ::Net::HTTP.prepend(Extension) - @patched = true + @patch_mutex ||= Mutex.new + @patch_mutex.synchronize do + install_extension(::Net::HTTP) + install_webmock_classes + install_io_hooks + @patched = true + end end def patched? @patched == true end + + def install_extension(klass) + return if klass.nil? + return if klass.method_defined?(:proxy_connect_request_headers=) + + klass.prepend(Extension) + singleton = klass.singleton_class + singleton.prepend(SubclassInstaller) unless singleton.ancestors.include?(SubclassInstaller) + end + + def install_webmock_classes + return unless defined?(::WebMock::HttpLibAdapters::NetHttpAdapter) + + adapter = ::WebMock::HttpLibAdapters::NetHttpAdapter + install_extension(adapter::OriginalNetHTTP) if adapter.const_defined?(:OriginalNetHTTP, false) + install_extension(adapter.instance_variable_get(:@webMockNetHTTP)) + end + + def install_io_hooks + unless ::Net::BufferedIO.ancestors.include?(ConnectRequestWrite) + ::Net::BufferedIO.prepend(ConnectRequestWrite) + end + response_singleton = ::Net::HTTPResponse.singleton_class + unless response_singleton.ancestors.include?(ConnectResponseRead) + response_singleton.prepend(ConnectResponseRead) + end + end end + private_class_method :build_http, :decode_component, :request_class, :parse_proxy, + :install_extension, :install_webmock_classes, :install_io_hooks end end diff --git a/lib/ruby_proxy_headers/rest_client.rb b/lib/ruby_proxy_headers/rest_client.rb index 845f019..79953ec 100644 --- a/lib/ruby_proxy_headers/rest_client.rb +++ b/lib/ruby_proxy_headers/rest_client.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative '../ruby_proxy_headers' + module RubyProxyHeaders # RestClient integration for proxy headers support. # @@ -34,26 +36,32 @@ def self.delete(url, proxy:, proxy_headers: {}, **options) # Make a request with proxy headers. def self.request(method, url, proxy:, proxy_headers: {}, **options) - require 'rest-client' - uri = URI.parse(url) - # For HTTPS with proxy headers, use our core connection - if uri.scheme == 'https' && !proxy_headers.empty? - response = RubyProxyHeaders::NetHTTP.request( + # RestClient cannot attach CONNECT headers or read the CONNECT response. + # Every HTTPS request through the proxy uses patched Net::HTTP. + if uri.scheme == 'https' + return RubyProxyHeaders::NetHTTP.request( method, url, proxy: proxy, proxy_headers: proxy_headers, headers: options[:headers], - body: options[:body] + body: options[:body] || options[:payload], + verify_ssl: options.fetch(:verify_ssl, true), + open_timeout: options[:open_timeout], + read_timeout: options[:read_timeout] ) - return response end - # For HTTP or no proxy headers, use standard RestClient + require 'rest-client' + ::RestClient.proxy = proxy - response = ::RestClient.send(method, url, options) + rest_options = options.dup + rest_options.delete(:verify_ssl) + rest_options.delete(:open_timeout) + rest_options.delete(:read_timeout) + response = ::RestClient.send(method, url, rest_options) ProxyResponse.new(response, {}) end diff --git a/lib/ruby_proxy_headers/typhoeus.rb b/lib/ruby_proxy_headers/typhoeus.rb index 3810086..9f4157b 100644 --- a/lib/ruby_proxy_headers/typhoeus.rb +++ b/lib/ruby_proxy_headers/typhoeus.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative '../ruby_proxy_headers' + module RubyProxyHeaders # Typhoeus/Ethon integration for proxy headers support. # Typhoeus wraps libcurl, which has native support for CURLOPT_PROXYHEADER. @@ -36,30 +38,33 @@ def self.post(url, proxy:, proxy_headers: {}, body: nil, **options) # @param options [Hash] Additional Typhoeus options # @return [ProxyResponse] def self.request(method, url, proxy:, proxy_headers: {}, **options) - require 'typhoeus' - uri = URI.parse(url) - # For HTTPS, we need custom handling since Typhoeus doesn't expose CURLOPT_PROXYHEADER - if uri.scheme == 'https' && !proxy_headers.empty? - # Use our core connection for now - # TODO: Extend Ethon to expose CURLOPT_PROXYHEADER for native support - response = RubyProxyHeaders::NetHTTP.request( + # Ethon 0.18 does not expose CURLOPT_PROXYHEADER. Every HTTPS request + # through the proxy uses patched Net::HTTP so CONNECT headers and the + # proxy response stay available. + if uri.scheme == 'https' + return RubyProxyHeaders::NetHTTP.request( method, url, proxy: proxy, proxy_headers: proxy_headers, headers: options[:headers], - body: options[:body] + body: options[:body], + verify_ssl: options.fetch(:verify_ssl, true), + open_timeout: options[:open_timeout], + read_timeout: options[:read_timeout] ) - return response end - # For HTTP or no proxy headers, use standard Typhoeus - typhoeus_options = options.merge( - proxy: proxy, - method: method - ) + require 'typhoeus' + + typhoeus_options = options.dup + typhoeus_options.delete(:verify_ssl) + typhoeus_options.delete(:open_timeout) + typhoeus_options.delete(:read_timeout) + typhoeus_options[:proxy] = proxy + typhoeus_options[:method] = method response = ::Typhoeus::Request.new(url, typhoeus_options).run ProxyResponse.new(response, {}) diff --git a/mkdocs.yml b/mkdocs.yml index 35d8733..d2d336e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,8 +3,8 @@ site_description: Send and receive custom proxy headers during HTTPS CONNECT tun site_author: ProxyMesh site_url: https://ruby-proxy-headers.readthedocs.io/ -repo_name: proxymeshai/ruby-proxy-headers -repo_url: https://github.com/proxymeshai/ruby-proxy-headers +repo_name: proxymesh/ruby-proxy-headers +repo_url: https://github.com/proxymesh/ruby-proxy-headers theme: name: material @@ -54,4 +54,4 @@ nav: extra: social: - icon: fontawesome/brands/github - link: https://github.com/proxymeshai/ruby-proxy-headers + link: https://github.com/proxymesh/ruby-proxy-headers diff --git a/ruby-proxy-headers.gemspec b/ruby-proxy-headers.gemspec index e69bf9b..73b6090 100644 --- a/ruby-proxy-headers.gemspec +++ b/ruby-proxy-headers.gemspec @@ -18,8 +18,12 @@ Gem::Specification.new do |s| s.metadata['source_code_uri'] = 'https://github.com/proxymesh/ruby-proxy-headers' s.metadata['documentation_uri'] = 'https://rubydoc.info/gems/ruby-proxy-headers' + s.metadata['homepage_uri'] = 'https://github.com/proxymesh/ruby-proxy-headers' + s.metadata['changelog_uri'] = 'https://github.com/proxymesh/ruby-proxy-headers/blob/main/CHANGELOG.md' + s.metadata['bug_tracker_uri'] = 'https://github.com/proxymesh/ruby-proxy-headers/issues' + s.metadata['rubygems_mfa_required'] = 'true' - s.files = Dir['lib/**/*', 'LICENSE', 'README.md', 'IMPLEMENTATION_PRIORITY.md', 'LIBRARY_RESEARCH.md', 'DEFERRED.md', '.yardopts'] + s.files = Dir['lib/**/*', 'LICENSE', 'README.md', 'CHANGELOG.md', 'docs/**/*.md', '.yardopts'] s.require_paths = ['lib'] s.add_development_dependency 'bundler', '>= 2.4' @@ -33,5 +37,7 @@ Gem::Specification.new do |s| s.add_development_dependency 'httparty', '~> 0.24' s.add_development_dependency 'mechanize', '~> 2.14' s.add_development_dependency 'typhoeus', '~> 1.6' + s.add_development_dependency 'http', '~> 6.0' + s.add_development_dependency 'rest-client', '~> 2.1' s.add_development_dependency 'webmock', '~> 3.18' end diff --git a/spec/connect_header_injection_spec.rb b/spec/connect_header_injection_spec.rb new file mode 100644 index 0000000..8e52533 --- /dev/null +++ b/spec/connect_header_injection_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +root = File.expand_path('..', __dir__) +$LOAD_PATH.unshift(File.join(root, 'lib')) unless $LOAD_PATH.include?(File.join(root, 'lib')) +require 'ruby_proxy_headers' + +RSpec.describe 'RubyProxyHeaders::NetHTTP.request' do + it 'is defined for the HTTP.rb, RestClient, and Typhoeus adapters' do + expect(RubyProxyHeaders::NetHTTP).to respond_to(:request) + expect(RubyProxyHeaders::NetHTTP).to respond_to(:get) + end + + it 'inserts custom headers before the CONNECT terminator' do + raw = "CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n" + injected = RubyProxyHeaders::NetHTTP.inject_connect_headers( + raw, + { 'X-ProxyMesh-Country' => 'US' } + ) + + expect(injected).to include("X-ProxyMesh-Country: US\r\n") + expect(injected).to end_with("\r\n\r\n") + expect(injected.index('X-ProxyMesh-Country')).to be < injected.rindex("\r\n\r\n") + expect(injected).to include('Proxy-Authorization').or include('Host: example.com:443') + end + + it 'rejects CR/LF in a CONNECT header before returning a modified request' do + raw = "CONNECT example.com:443 HTTP/1.1\r\nHost: example.com:443\r\n\r\n" + expect do + RubyProxyHeaders::NetHTTP.inject_connect_headers(raw, { 'X-Bad' => "no\r\nInjected: yes" }) + end.to raise_error(ArgumentError, /header value/) + end +end + +RSpec.describe RubyProxyHeaders::HeaderHash do + it 'reads header names without caring about case' do + headers = described_class.new + headers['X-ProxyMesh-IP'] = '203.0.113.7' + expect(headers['x-proxymesh-ip']).to eq('203.0.113.7') + expect(headers['X-ProxyMesh-IP']).to eq('203.0.113.7') + end +end diff --git a/spec/faraday_adapter_spec.rb b/spec/faraday_adapter_spec.rb new file mode 100644 index 0000000..5521288 --- /dev/null +++ b/spec/faraday_adapter_spec.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'ruby_proxy_headers/faraday' + +RSpec.describe RubyProxyHeaders::FaradayAdapter::NetHttp do + it 'uses public Faraday hooks instead of the private request method' do + expect(described_class.instance_methods(false)).to include(:call, :net_http_connection) + expect(described_class.private_instance_methods(false)).not_to include(:request_with_wrapped_block) + expect(described_class.private_instance_methods).to include(:request_with_wrapped_block) + end +end diff --git a/spec/mock_proxy_integration_spec.rb b/spec/mock_proxy_integration_spec.rb new file mode 100644 index 0000000..435a1bb --- /dev/null +++ b/spec/mock_proxy_integration_spec.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'json' +require 'http' +require 'rest-client' +require 'typhoeus' +require 'ruby_proxy_headers/faraday' +require 'ruby_proxy_headers/http_gem' +require 'ruby_proxy_headers/httparty' +require 'ruby_proxy_headers/rest_client' +require 'ruby_proxy_headers/typhoeus' +require_relative 'support/connect_lab' + +RSpec.describe 'local mock proxy CONNECT headers' do + before do + WebMock.disable! if defined?(WebMock) + @origin = ConnectLab::HttpsOrigin.new + @proxy = ConnectLab::MockProxy.start + end + + after do + @proxy&.stop + @origin&.close + if defined?(WebMock) + WebMock.enable! + WebMock.disable_net_connect! + end + end + + let(:proxy_url) { "http://127.0.0.1:#{@proxy.port}" } + let(:target_url) { "https://127.0.0.1:#{@origin.port}/resource" } + let(:proxy_headers) { { 'X-ProxyMesh-Country' => 'US' } } + + def expect_tunnel(proxy_response_headers, body) + expect(body).to include('local-https') + expect(ConnectLab.header(proxy_response_headers, 'X-ProxyMesh-IP')).to eq('203.0.113.7') + expect(ConnectLab.header(proxy_response_headers, 'X-ProxyMesh-Country-Echo')).to eq('US') + origin_request = Timeout.timeout(2) { @origin.requests.pop } + expect(origin_request).not_to include('X-ProxyMesh-Country') + end + + it 'sends the custom header on CONNECT for Net::HTTP and reads the proxy response' do + response = RubyProxyHeaders::NetHTTP.get( + target_url, + proxy: proxy_url, + proxy_headers: proxy_headers, + verify_ssl: false, + open_timeout: 5, + read_timeout: 5 + ) + + expect(response.code).to eq(200) + expect_tunnel(response.proxy_response_headers, response.body) + end + + it 'sends the custom header on CONNECT for HTTP.rb and reads the proxy response' do + client = RubyProxyHeaders::HTTPGem.create_client( + proxy: proxy_url, + proxy_headers: proxy_headers + ) + response = client.get(target_url, verify_ssl: false, open_timeout: 5, read_timeout: 5) + + expect(response.code).to eq(200) + expect(client.last_proxy_response_headers['X-ProxyMesh-IP']).to eq('203.0.113.7') + expect_tunnel(response.proxy_response_headers, response.body) + end + + it 'sends the custom header on CONNECT for RestClient and reads the proxy response' do + response = RubyProxyHeaders::RestClient.get( + target_url, + proxy: proxy_url, + proxy_headers: proxy_headers, + verify_ssl: false, + open_timeout: 5, + read_timeout: 5 + ) + + expect(response.code).to eq(200) + expect_tunnel(response.proxy_response_headers, response.body) + end + + it 'sends the custom header on CONNECT for Typhoeus and reads the proxy response' do + response = RubyProxyHeaders::Typhoeus.get( + target_url, + proxy: proxy_url, + proxy_headers: proxy_headers, + verify_ssl: false, + open_timeout: 5, + read_timeout: 5 + ) + + expect(response).to be_success + expect_tunnel(response.proxy_response_headers, response.body) + end + + it 'sends the custom header on CONNECT for Faraday and exposes proxy response headers' do + connection = RubyProxyHeaders::FaradayIntegration.connection( + proxy: proxy_url, + proxy_connect_headers: proxy_headers + ) do |builder| + builder.ssl.verify = false + end + response = connection.get(target_url) + + expect(response.status).to eq(200) + expect(response.headers['X-ProxyMesh-IP']).to eq('203.0.113.7') + expect(response.headers['X-ProxyMesh-Country-Echo']).to eq('US') + origin_request = Timeout.timeout(2) { @origin.requests.pop } + expect(origin_request).not_to include('X-ProxyMesh-Country') + expect(response.body).to include('local-https') + end + + it 'sends the custom header on CONNECT for HTTParty and reads the proxy response' do + RubyProxyHeaders::NetHTTP.patch! + proxy = URI(proxy_url) + response = HTTParty.get( + target_url, + http_proxyaddr: proxy.host, + http_proxyport: proxy.port, + verify: false, + proxy_connect_request_headers: proxy_headers, + connection_adapter: RubyProxyHeaders::ProxyHeadersConnectionAdapter, + timeout: 5 + ) + + expect(response.code).to eq(200) + headers = RubyProxyHeaders.proxy_connect_response_headers + expect(ConnectLab.header(headers, 'X-ProxyMesh-IP')).to eq('203.0.113.7') + expect(ConnectLab.header(headers, 'X-ProxyMesh-Country-Echo')).to eq('US') + end +end diff --git a/spec/net_http_compat_spec.rb b/spec/net_http_compat_spec.rb new file mode 100644 index 0000000..ac7671c --- /dev/null +++ b/spec/net_http_compat_spec.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +require 'socket' +require 'timeout' + +root = File.expand_path('..', __dir__) +$LOAD_PATH.unshift(File.join(root, 'lib')) unless $LOAD_PATH.include?(File.join(root, 'lib')) +require 'ruby_proxy_headers' +require_relative 'support/connect_lab' + +# Exercises the installed net-http, including versions older than the lockfile. +# Run across versions with gemfiles/net_http_matrix.gemfile. +RSpec.describe 'net-http CONNECT compatibility' do + before do + WebMock.disable! if defined?(WebMock) + RubyProxyHeaders::NetHTTP.patch! + end + + after do + if defined?(WebMock) + WebMock.enable! + WebMock.disable_net_connect! + end + end + + def net_http_version + return ::Net::HTTP::VERSION if defined?(::Net::HTTP::VERSION) + + spec = Gem.loaded_specs['net-http'] + spec ? spec.version.to_s : 'default' + end + + it "round-trips a custom CONNECT header on net-http #{defined?(::Net::HTTP::VERSION) ? ::Net::HTTP::VERSION : 'loaded'}" do + origin = ConnectLab::HttpsOrigin.new + proxy = ConnectLab::MockProxy.start + response = RubyProxyHeaders::NetHTTP.get( + "https://127.0.0.1:#{origin.port}/", + proxy: "http://127.0.0.1:#{proxy.port}", + proxy_headers: { 'X-ProxyMesh-Country' => 'US' }, + verify_ssl: false, + open_timeout: 5, + read_timeout: 5 + ) + + expect(response.code).to eq(200) + expect(response.body).to include('local-https') + expect(response.proxy_response_headers['X-ProxyMesh-IP']).to eq('203.0.113.7') + expect(response.proxy_response_headers['x-proxymesh-country-echo']).to eq('US') + ensure + proxy&.stop + origin&.close + end + + it 'does not write CONNECT bytes when the target host contains CR' do + server = TCPServer.new('127.0.0.1', 0) + received = Queue.new + thread = Thread.new do + sock = Timeout.timeout(0.4) { server.accept } + received << sock.readpartial(4096) + sock.close + rescue Timeout::Error, EOFError, Errno::ECONNRESET + received << :none + end + + http = Net::HTTP.new("evil.com\r\nX-Injected: smuggled", 443, '127.0.0.1', server.addr[1]) + http.use_ssl = true + http.open_timeout = 1 + http.read_timeout = 1 + http.proxy_connect_request_headers = { 'X-ProxyMesh-Country' => 'US' } + + expect { http.send(:connect) }.to raise_error(ArgumentError, /target host/i) + payload = Timeout.timeout(0.5) { received.pop } + expect(payload).to eq(:none) + ensure + thread&.kill + server&.close + end + + it 'calls upstream timeouted_connect when that method exists' do + skip "net-http #{net_http_version} opens the socket inline" unless Net::HTTP.private_instance_methods.include?(:timeouted_connect) + + proxy = ConnectLab::MockProxy.start + http = Net::HTTP.new('127.0.0.1', 9, '127.0.0.1', proxy.port) + http.use_ssl = true + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + http.open_timeout = 2 + http.read_timeout = 2 + called = [] + http.singleton_class.prepend(Module.new do + define_method(:timeouted_connect) do |*args| + called << true + super(*args) + end + end) + + begin + http.send(:connect) + rescue StandardError + # Port 9 refuses the tunneled target. The assertion is that upstream ran. + nil + end + expect(called).not_to be_empty + ensure + proxy&.stop + end + + it 'keeps TLS-to-proxy (proxy_use_ssl) instead of writing a cleartext CONNECT' do + sample = Net::HTTP.new('127.0.0.1', 443, '127.0.0.1', 9) + skip "net-http #{net_http_version} has no HTTPS proxy support" unless sample.respond_to?(:proxy_use_ssl=) + + origin = ConnectLab::HttpsOrigin.new + proxy = ConnectLab::TlsProxy.new + response = RubyProxyHeaders::NetHTTP.get( + "https://127.0.0.1:#{origin.port}/", + proxy: "https://127.0.0.1:#{proxy.port}", + proxy_headers: { 'X-ProxyMesh-Country' => 'US' }, + verify_ssl: false, + open_timeout: 5, + read_timeout: 5 + ) + + expect(response.code).to eq(200) + expect(response.body).to include('local-https') + expect(response.proxy_response_headers['X-ProxyMesh-IP']).to eq('203.0.113.9') + first = Timeout.timeout(2) { proxy.first_bytes.pop } + # TLS handshake record, not a cleartext "CONNECT". + expect(first.bytes.first).to eq(0x16) + head = Timeout.timeout(2) { proxy.connect_heads.pop } + expect(head).to start_with('CONNECT ') + expect(head).to include('X-ProxyMesh-Country: US') + ensure + proxy&.close + origin&.close + end +end diff --git a/spec/net_http_connect_target_spec.rb b/spec/net_http_connect_target_spec.rb index 2a6b93f..2cc47c7 100644 --- a/spec/net_http_connect_target_spec.rb +++ b/spec/net_http_connect_target_spec.rb @@ -27,6 +27,24 @@ def connect!(http) http.send(:connect) end + it 'still defines CONNECT header writers after WebMock swaps Net::HTTP' do + WebMock.enable! + RubyProxyHeaders::NetHTTP.patch! + WebMock.disable! + RubyProxyHeaders::NetHTTP.patch! + + disabled = Net::HTTP.new('example.com', 443, '127.0.0.1', 9) + expect(disabled).to respond_to(:proxy_connect_request_headers=) + expect(disabled).to respond_to(:last_proxy_connect_response_headers) + + WebMock.enable! + enabled = Net::HTTP.new('example.com', 443, '127.0.0.1', 9) + expect(enabled).to respond_to(:proxy_connect_request_headers=) + ensure + WebMock.enable! + WebMock.disable_net_connect! + end + it 'rejects CONNECT host containing CR' do expect { connect!(proxied_https("evil.com\r\nX-Injected: smuggled", 443)) } .to raise_error(ArgumentError, /target host.*invalid/i) diff --git a/spec/support/connect_lab.rb b/spec/support/connect_lab.rb new file mode 100644 index 0000000..943d77d --- /dev/null +++ b/spec/support/connect_lab.rb @@ -0,0 +1,225 @@ +# frozen_string_literal: true + +require 'openssl' +require 'socket' +require 'tempfile' +require 'timeout' + +module ConnectLab + module_function + + def self_signed_cert + key = OpenSSL::PKey::EC.generate('prime256v1') + cert = OpenSSL::X509::Certificate.new + cert.version = 2 + cert.serial = 1 + cert.subject = cert.issuer = OpenSSL::X509::Name.parse('CN=127.0.0.1') + cert.not_before = Time.now - 3600 + cert.not_after = Time.now + 86_400 + cert.public_key = key + factory = OpenSSL::X509::ExtensionFactory.new + factory.subject_certificate = cert + factory.issuer_certificate = cert + cert.add_extension(factory.create_extension('basicConstraints', 'CA:FALSE', true)) + cert.add_extension(factory.create_extension('subjectAltName', 'IP:127.0.0.1,DNS:localhost', false)) + cert.sign(key, OpenSSL::Digest::SHA256.new) + [cert, key] + end + + def server_context + cert, key = self_signed_cert + ctx = OpenSSL::SSL::SSLContext.new + ctx.cert = cert + ctx.key = key + ctx + end + + # Minimal HTTPS origin. Records the cleartext HTTP request that arrived + # through the tunnel and answers with a fixed body. + class HttpsOrigin + attr_reader :port, :requests + + def initialize + @requests = Queue.new + @ctx = ConnectLab.server_context + @tcp = TCPServer.new('127.0.0.1', 0) + @port = @tcp.addr[1] + @accept = Thread.new { accept_loop } + @accept.abort_on_exception = false + end + + def close + @tcp.close rescue nil + @accept.kill rescue nil + end + + private + + def accept_loop + loop do + client = @tcp.accept + Thread.new { handle(client) } + end + rescue IOError, Errno::EBADF + nil + end + + def handle(client) + ssl = OpenSSL::SSL::SSLSocket.new(client, @ctx) + ssl.sync_close = true + ssl.accept + request = +'' + while (line = ssl.gets) + request << line + break if line == "\r\n" || line == "\n" + end + @requests << request + body = '{"via":"local-https"}' + ssl.write( + "HTTP/1.1 200 OK\r\n" \ + "Content-Type: application/json\r\n" \ + "Content-Length: #{body.bytesize}\r\n" \ + "Connection: close\r\n\r\n#{body}" + ) + ssl.close + rescue IOError, Errno::ECONNRESET, Errno::EPIPE, OpenSSL::SSL::SSLError + nil + end + end + + # HTTPS proxy: TLS first, then CONNECT, then a raw byte pipe to the target. + # + # net-http's +proxy_use_ssl+ speaks TLS only for the proxy handshake and the + # CONNECT request. The origin handshake is started on the underlying TCP + # socket (ruby/net-http#212), so bytes after the CONNECT response must be + # forwarded undecrypted. + class TlsProxy + attr_reader :port, :connect_heads, :first_bytes + + def initialize + @connect_heads = Queue.new + @first_bytes = Queue.new + @ctx = ConnectLab.server_context + @tcp = TCPServer.new('127.0.0.1', 0) + @port = @tcp.addr[1] + @accept = Thread.new { accept_loop } + end + + def close + @tcp.close rescue nil + @accept.kill rescue nil + end + + private + + def accept_loop + loop do + client = @tcp.accept + Thread.new { handle(client) } + end + rescue IOError, Errno::EBADF + nil + end + + def handle(client) + client.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) + peeked = client.recv(5, Socket::MSG_PEEK) + @first_bytes << peeked + ssl = OpenSSL::SSL::SSLSocket.new(client, @ctx) + ssl.sync_close = false + ssl.sync = true + ssl.accept + head = +'' + while (line = ssl.gets) + head << line + break if line == "\r\n" || line == "\n" + end + @connect_heads << head + target = head.lines.first.to_s.split[1] + host, port = target.rpartition(':').values_at(0, 2) + upstream = TCPSocket.new(host, port.to_i) + upstream.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) + ssl.write( + "HTTP/1.1 200 Connection established\r\n" \ + "X-ProxyMesh-IP: 203.0.113.9\r\n\r\n" + ) + pump(client, upstream) + rescue StandardError + client.close rescue nil + end + + def pump(left, right) + [left, right].permutation(2).each do |src, dst| + Thread.new do + loop { dst.write(src.readpartial(16_384)) } + rescue EOFError, IOError, Errno::ECONNRESET, Errno::EPIPE, OpenSSL::SSL::SSLError + begin + dst.close_write + rescue IOError, Errno::ENOTCONN + nil + end + end + end + end + end + + class MockProxy + attr_reader :port, :pid + + def self.start + script = File.expand_path('../../test/mock_connect_proxy.py', __dir__) + log = Tempfile.new('mock-connect-proxy') + port = reserve_port + pid = Process.spawn('python3', script, port.to_s, out: log.path, err: log.path, pgroup: true) + proxy = new(port, pid, log) + proxy.wait_until_listening + proxy + end + + def initialize(port, pid, log) + @port = port + @pid = pid + @log = log + end + + def stop + Process.kill('TERM', -pid) rescue nil + Process.wait(pid) rescue nil + @log.close rescue nil + end + + def log_text + File.read(@log.path) + rescue Errno::ENOENT + '' + end + + def wait_until_listening + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 5 + loop do + TCPSocket.new('127.0.0.1', port).close + return + rescue Errno::ECONNREFUSED, Errno::ECONNRESET + raise "mock proxy did not listen on #{port}: #{log_text}" if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline + + sleep 0.05 + end + end + + def self.reserve_port + server = TCPServer.new('127.0.0.1', 0) + server.addr[1] + ensure + server&.close + end + end + + def header(headers, name) + return nil unless headers.respond_to?(:[]) + + return headers[name] if headers[name] + + key = headers.keys.find { |candidate| candidate.to_s.casecmp(name).zero? } + key ? headers[key] : nil + end +end diff --git a/test/mock_connect_proxy.py b/test/mock_connect_proxy.py new file mode 100644 index 0000000..5db87dc --- /dev/null +++ b/test/mock_connect_proxy.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Minimal CONNECT proxy for offline-ish adapter testing. +Replies to CONNECT with `HTTP/1.1 200 Connection established` + X-ProxyMesh-IP, +echoing any X-ProxyMesh-* request header back as a response header, then pipes bytes. +Plain-HTTP (absolute-URI) requests are forwarded with X-ProxyMesh-IP added. +Usage: python3 mock_connect_proxy.py [port] (default 18080)""" +import socket, sys, threading, select + +PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 18080 +FAKE_IP = "203.0.113.7" + +def pipe(a, b, label=""): + socks = [a, b] + n = [0, 0] + try: + while True: + r, _, _ = select.select(socks, [], [], 60) + if not r: + return + for s in r: + data = s.recv(65536) + if not data: + return + n[0 if s is a else 1] += len(data) + (b if s is a else a).sendall(data) + except OSError: + pass + finally: + sys.stderr.write(f"[proxy] tunnel {label} closed: client->up {n[0]}B, up->client {n[1]}B\n") + a.close(); b.close() + +def read_head(c): + buf = b"" + while b"\r\n\r\n" not in buf: + d = c.recv(4096) + if not d: + break + buf += d + head, _, rest = buf.partition(b"\r\n\r\n") + return head, rest + +def handle(c): + try: + head, rest = read_head(c) + lines = head.decode("latin-1").split("\r\n") + method, target, _ = lines[0].split(" ", 2) + hdrs = [l.split(":", 1) for l in lines[1:] if ":" in l] + echo = "".join(f"{k.strip()}-Echo: {v.strip()}\r\n" for k, v in hdrs if k.lower().startswith("x-proxymesh")) + sys.stderr.write(f"[proxy] {method} {target} hdrs={[k for k,_ in hdrs]}\n") + if method == "CONNECT": + host, port = target.rsplit(":", 1) + up = socket.create_connection((host, int(port)), timeout=20) + c.sendall(("HTTP/1.1 200 Connection established\r\n" + f"X-ProxyMesh-IP: {FAKE_IP}\r\n{echo}\r\n").encode()) + if rest: + up.sendall(rest) + pipe(c, up, target) + else: + from urllib.parse import urlsplit + u = urlsplit(target) + up = socket.create_connection((u.hostname, u.port or 80), timeout=20) + path = (u.path or "/") + (("?" + u.query) if u.query else "") + out = [f"{method} {path} HTTP/1.1"] + [f"{k}:{v}" for k, v in hdrs if not k.lower().startswith("proxy-") and k.lower() != "connection"] + ["Connection: close"] + up.sendall(("\r\n".join(out) + "\r\n\r\n").encode("latin-1") + rest) + resp = b"" + while True: + d = up.recv(65536) + if not d: break + resp += d + h, _, body = resp.partition(b"\r\n\r\n") + c.sendall(h + f"\r\nX-ProxyMesh-IP: {FAKE_IP}".encode() + b"\r\n\r\n" + body) + c.close(); up.close() + except Exception as e: + sys.stderr.write(f"[proxy] error {e!r}\n") + try: c.close() + except OSError: pass + +s = socket.socket(); s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +s.bind(("127.0.0.1", PORT)); s.listen(128) +sys.stderr.write(f"[proxy] listening on 127.0.0.1:{PORT}\n") +while True: + c, _ = s.accept() + threading.Thread(target=handle, args=(c,), daemon=True).start() diff --git a/test/test_proxy_headers.rb b/test/test_proxy_headers.rb index ec8e9c5..6554a55 100644 --- a/test/test_proxy_headers.rb +++ b/test/test_proxy_headers.rb @@ -139,6 +139,73 @@ def test_httparty(verbose:) { ok: false, error: e.message } end +def test_http_gem(verbose:) + require 'ruby_proxy_headers/http_gem' + + proxy_url = env_proxy_url + raise 'Set PROXY_URL' unless proxy_url + + test_url = ENV.fetch('TEST_URL', 'https://api.ipify.org?format=json') + proxy_header = ENV.fetch('PROXY_HEADER', 'X-ProxyMesh-IP') + h = send_headers_from_env + + response = RubyProxyHeaders::HTTPGem.get( + test_url, + proxy: normalized_proxy_url(proxy_url), + proxy_headers: h + ) + return { ok: false, error: "HTTP #{response.code}" } unless response.success? + + val = find_header(response.proxy_response_headers, proxy_header) + assert_proxy_header(val, proxy_header, verbose: verbose, label: 'http_gem') +rescue StandardError => e + { ok: false, error: e.message } +end + +def test_typhoeus(verbose:) + require 'ruby_proxy_headers/typhoeus' + + proxy_url = env_proxy_url + raise 'Set PROXY_URL' unless proxy_url + + test_url = ENV.fetch('TEST_URL', 'https://api.ipify.org?format=json') + proxy_header = ENV.fetch('PROXY_HEADER', 'X-ProxyMesh-IP') + h = send_headers_from_env + response = RubyProxyHeaders::Typhoeus.get( + test_url, + proxy: normalized_proxy_url(proxy_url), + proxy_headers: h + ) + return { ok: false, error: "HTTP #{response.code}" } unless response.success? + + val = find_header(response.proxy_response_headers, proxy_header) + assert_proxy_header(val, proxy_header, verbose: verbose, label: 'typhoeus') +rescue StandardError => e + { ok: false, error: e.message } +end + +def test_rest_client(verbose:) + require 'ruby_proxy_headers/rest_client' + + proxy_url = env_proxy_url + raise 'Set PROXY_URL' unless proxy_url + + test_url = ENV.fetch('TEST_URL', 'https://api.ipify.org?format=json') + proxy_header = ENV.fetch('PROXY_HEADER', 'X-ProxyMesh-IP') + h = send_headers_from_env + response = RubyProxyHeaders::RestClient.get( + test_url, + proxy: normalized_proxy_url(proxy_url), + proxy_headers: h + ) + return { ok: false, error: "HTTP #{response.code}" } unless response.success? + + val = find_header(response.proxy_response_headers, proxy_header) + assert_proxy_header(val, proxy_header, verbose: verbose, label: 'rest_client') +rescue StandardError => e + { ok: false, error: e.message } +end + def test_excon(verbose:) require 'ruby_proxy_headers/excon' @@ -173,7 +240,10 @@ def test_excon(verbose:) 'net_http' => method(:test_net_http), 'faraday' => method(:test_faraday), 'httparty' => method(:test_httparty), - 'excon' => method(:test_excon) + 'excon' => method(:test_excon), + 'http_gem' => method(:test_http_gem), + 'typhoeus' => method(:test_typhoeus), + 'rest_client' => method(:test_rest_client) }.freeze def print_help