From 2ae95257f73306c66859ce3f75a82cd46be074fe Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 23 Oct 2026 03:59:39 +0000 Subject: [PATCH] Fix the undici proxy bypass and align clients, types, and CI. HTTPS requests passed an open tunnel socket as undici connect.socket, which the default connector replaces with a direct dial, leaking the client IP. Use a custom connector on ProxyHeadersAgent and fail tests unless the mock proxy sees CONNECT and tunneled bytes. Also accept ky 2.2 Responses, widen peer ranges through the versions we test, require Node >= 22.19, and add unit, type, and latest-client workflows. Co-authored-by: ProxyMesh AI --- .../skills/javascript-proxy-headers/SKILL.md | 4 +- .github/workflows/ci.yml | 88 +++++++ .github/workflows/latest-clients.yml | 59 +++++ IMPLEMENTATION_PRIORITY.md | 4 +- LIBRARY_RESEARCH.md | 2 +- README.md | 23 +- docs/index.md | 3 +- docs/ky.md | 4 +- docs/undici.md | 5 +- lib/axios-proxy.js | 4 +- lib/got-proxy.js | 4 +- lib/ky-proxy.js | 133 ++++++++++- lib/node-fetch-proxy.js | 6 +- lib/superagent-proxy.js | 4 +- lib/undici-proxy.js | 221 ++++++++---------- package-lock.json | 18 +- package.json | 26 ++- test/local-mock-integration.js | 68 ++++++ test/mock-connect-proxy.js | 183 +++++++++++++++ test/test_proxy_traversal.js | 197 ++++++++++++++++ tsconfig.json | 2 - types/ky.d.ts | 2 + types/undici.d.ts | 4 + 23 files changed, 891 insertions(+), 173 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/latest-clients.yml create mode 100644 test/local-mock-integration.js create mode 100644 test/mock-connect-proxy.js create mode 100644 test/test_proxy_traversal.js diff --git a/.agents/skills/javascript-proxy-headers/SKILL.md b/.agents/skills/javascript-proxy-headers/SKILL.md index 07dcf79..2b7e70c 100644 --- a/.agents/skills/javascript-proxy-headers/SKILL.md +++ b/.agents/skills/javascript-proxy-headers/SKILL.md @@ -40,7 +40,7 @@ Install your HTTP client as a peer dependency (axios, got, node-fetch, etc.). ```javascript import { createProxyAxios } from 'javascript-proxy-headers/axios'; -const client = createProxyAxios({ +const client = await createProxyAxios({ proxy: 'http://user:pass@proxy.example.com:8080', proxyHeaders: { 'X-ProxyMesh-Country': 'US' } }); @@ -67,7 +67,7 @@ console.log(response.proxyHeaders.get('x-proxymesh-ip')); ```javascript import { createProxyGot } from 'javascript-proxy-headers/got'; -const client = createProxyGot({ +const client = await createProxyGot({ proxy: 'http://user:pass@proxy.example.com:8080', proxyHeaders: { 'X-ProxyMesh-Country': 'US' } }); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..53f5e8b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,88 @@ +# Unit tests, typecheck, and local mock-proxy integration. +# Real-proxy integration stays in proxy_integration_tests.yml (needs PROXY_URL). + +name: CI + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + # Required for setup-node npm cache save under restricted GITHUB_TOKEN scopes. + actions: write + +jobs: + unit: + name: unit (Node ${{ matrix.node }}) + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + node: ["22", "24"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.node }} + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Unit tests + run: npm run test:unit + + types: + name: types (Node ${{ matrix.node }}) + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + node: ["22", "24"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.node }} + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Typecheck + run: npm run test:types + + local-mock-proxy: + name: local mock proxy (Node ${{ matrix.node }}) + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + node: ["22", "24"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.node }} + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Adapter tests through a local CONNECT proxy + run: npm run test:local-proxy diff --git a/.github/workflows/latest-clients.yml b/.github/workflows/latest-clients.yml new file mode 100644 index 0000000..a7aee99 --- /dev/null +++ b/.github/workflows/latest-clients.yml @@ -0,0 +1,59 @@ +# Weekly (and manual) run against the latest published versions of every +# supported HTTP client. Ignores the lockfile for those clients. Does not +# change Dependabot auto-merge. + +name: Latest clients + +on: + schedule: + # Monday 12:00 UTC, after the weekly Dependabot window. + - cron: "0 12 * * 1" + workflow_dispatch: + +permissions: + contents: read + actions: write + +jobs: + latest: + name: latest clients (Node ${{ matrix.node }}) + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + node: ["22", "24"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.node }} + cache: npm + + - name: Install lockfile dependencies + run: npm ci + + - name: Install latest supported clients (ignore the lockfile) + run: | + set -euo pipefail + npm install --no-save --no-package-lock \ + axios@latest \ + got@latest \ + ky@latest \ + make-fetch-happen@latest \ + needle@latest \ + node-fetch@latest \ + superagent@latest \ + typed-rest-client@latest \ + undici@latest \ + wretch@latest + npm ls --depth=0 axios got ky make-fetch-happen needle node-fetch superagent typed-rest-client undici wretch + + - name: Unit tests + run: npm run test:unit + + - name: Local mock-proxy integration (includes proxy-traversal assertion) + run: npm run test:local-proxy diff --git a/IMPLEMENTATION_PRIORITY.md b/IMPLEMENTATION_PRIORITY.md index b176041..7b74347 100644 --- a/IMPLEMENTATION_PRIORITY.md +++ b/IMPLEMENTATION_PRIORITY.md @@ -125,7 +125,7 @@ console.log(response.proxyHeaders.get('x-proxymesh-ip')); ```javascript import { createProxyAxios } from 'javascript-proxy-headers/axios'; -const client = createProxyAxios({ +const client = await createProxyAxios({ proxy: 'http://proxy:8080', proxyHeaders: { 'X-ProxyMesh-Country': 'US' } }); @@ -151,7 +151,7 @@ console.log(response.headers['x-proxymesh-ip']); // Merged into response ```javascript import { createProxyGot } from 'javascript-proxy-headers/got'; -const client = createProxyGot({ +const client = await createProxyGot({ proxy: 'http://proxy:8080', proxyHeaders: { 'X-ProxyMesh-Country': 'US' } }); diff --git a/LIBRARY_RESEARCH.md b/LIBRARY_RESEARCH.md index 6c0372c..4cab3c3 100644 --- a/LIBRARY_RESEARCH.md +++ b/LIBRARY_RESEARCH.md @@ -120,7 +120,7 @@ console.log(response.proxyHeaders.get('X-ProxyMesh-IP')); ```javascript import { createProxyGot } from 'javascript-proxy-headers/got'; -const proxyGot = createProxyGot({ +const proxyGot = await createProxyGot({ proxy: 'http://proxy:8080', proxyHeaders: { 'X-ProxyMesh-Country': 'US' } }); diff --git a/README.md b/README.md index 8f062c6..a23329d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ -[![npm version](https://badge.fury.io/js/javascript-proxy-headers.svg)](https://badge.fury.io/js/javascript-proxy-headers) # JavaScript Proxy Headers -Extensions for JavaScript HTTP libraries to support **sending and receiving custom proxy headers** during HTTPS CONNECT tunneling. +[![npm version](https://badge.fury.io/js/javascript-proxy-headers.svg)](https://badge.fury.io/js/javascript-proxy-headers) + +Send and read custom proxy headers on HTTPS CONNECT for axios, got, node-fetch, undici, ky, and more. ## The Problem @@ -28,7 +29,7 @@ This library solves both problems for popular JavaScript HTTP libraries. | [needle](https://github.com/tomas/needle) | `javascript-proxy-headers/needle` | Lean HTTP client | | [typed-rest-client](https://github.com/microsoft/typed-rest-client) | `javascript-proxy-headers/typed-rest-client` | Azure / DevOps–style REST client | -**urllib** is not integrated yet: it expects an [undici](https://undici.nodejs.org/) `Dispatcher`, not a Node `Agent`. See `notes/urllib-integration-deferred.md` in this repo for a possible approach. +**urllib** is not integrated yet: it expects an [undici](https://undici.nodejs.org/) `Dispatcher`, not a Node `Agent`. See [notes/urllib-integration-deferred.md](https://github.com/proxymesh/javascript-proxy-headers/blob/main/notes/urllib-integration-deferred.md) for a possible approach. ## Installation @@ -61,7 +62,7 @@ response.proxyHeaders.get('x-proxymesh-ip') ```javascript import { createProxyAxios } from 'javascript-proxy-headers/axios'; -const client = createProxyAxios({ +const client = await createProxyAxios({ proxy: 'http://user:pass@proxy.example.com:8080', proxyHeaders: { 'X-ProxyMesh-Country': 'US' } }); @@ -90,7 +91,7 @@ console.log(response.proxyHeaders.get('x-proxymesh-ip')); ```javascript import { createProxyGot } from 'javascript-proxy-headers/got'; -const client = createProxyGot({ +const client = await createProxyGot({ proxy: 'http://user:pass@proxy.example.com:8080', proxyHeaders: { 'X-ProxyMesh-Country': 'US' } }); @@ -234,13 +235,15 @@ Verbose (`-v`) prints captured header values. See `test/test_proxy_headers.js -- ## Requirements -- Node.js >= 18.0.0 +- Node.js >= 22.19.0 (undici 8, got 16, and ky 2 require current Node 22) - One or more supported HTTP libraries -## Related Projects +## Other languages -- **[python-proxy-headers](https://github.com/proxymesh/python-proxy-headers)** - Same functionality for Python -- **[proxy-examples](https://github.com/proxymesh/proxy-examples)** - Example code for using proxies +- **[python-proxy-headers](https://github.com/proxymesh/python-proxy-headers)** — requests, urllib3, httpx, and aiohttp +- **[ruby-proxy-headers](https://github.com/proxymesh/ruby-proxy-headers)** — Net::HTTP, Faraday, HTTParty, Excon, and Typhoeus +- **[scrapy-proxy-headers](https://github.com/proxymesh/scrapy-proxy-headers)** — Scrapy download handler +- **[proxy-examples](https://github.com/proxymesh/proxy-examples)** — example proxy code ## About @@ -248,4 +251,4 @@ Created by [ProxyMesh](https://proxymesh.com) to help our customers use custom h ## License -MIT License +[MIT](https://github.com/proxymesh/javascript-proxy-headers/blob/main/LICENSE) diff --git a/docs/index.md b/docs/index.md index 19b69bf..c8d2983 100644 --- a/docs/index.md +++ b/docs/index.md @@ -78,12 +78,13 @@ We at [ProxyMesh](https://proxymesh.com) created these extension modules to supp ## Related Projects - **[python-proxy-headers](https://github.com/proxymesh/python-proxy-headers)** - Same functionality for Python +- **[ruby-proxy-headers](https://github.com/proxymesh/ruby-proxy-headers)** - Same functionality for Ruby - **[proxy-examples](https://github.com/proxymesh/proxy-examples)** - Example code for using proxies - **[scrapy-proxy-headers](https://github.com/proxymesh/scrapy-proxy-headers)** - Proxy header support for Scrapy ## Requirements -- Node.js >= 18.0.0 +- Node.js >= 22.19.0 - One or more supported HTTP libraries ## License diff --git a/docs/ky.md b/docs/ky.md index 6400976..726dc1b 100644 --- a/docs/ky.md +++ b/docs/ky.md @@ -2,6 +2,8 @@ [ky](https://github.com/sindresorhus/ky) is a small HTTP client built on `fetch`. This package wires ky to a custom `fetch` implemented with [node-fetch](node-fetch.md) and `ProxyHeadersAgent`, so CONNECT proxy headers work the same way as in the node-fetch adapter. +ky 1 receives that node-fetch response directly. ky 2 (including 2.2+, which throws `The fetch option must resolve with a Response` for a node-fetch `Response`) receives a global `Response` with the same `proxyHeaders` Map attached. `node-fetch` is still required. + ## Getting Started ### Prerequisites @@ -71,4 +73,4 @@ const res = await api.get('v1/status').json(); ## Accessing Proxy Headers -Responses are [`ProxyResponse`](node-fetch.md) instances: use `response.proxyHeaders.get('x-proxymesh-ip')` (and the usual `response.json()`, `response.text()`, etc.). Proxy CONNECT failures use the same errors as the underlying fetch stack; see [Core API](core-api.md) for `ConnectError` on the agent. +Use `response.proxyHeaders.get('x-proxymesh-ip')` (and the usual `response.json()`, `response.text()`, etc.). On ky 2, `response` is a global `Response`; `proxyHeaders` is an own property and is not copied by `response.clone()`. Proxy CONNECT failures use the same errors as the underlying fetch stack; see [Core API](core-api.md) for `ConnectError` on the agent. diff --git a/docs/undici.md b/docs/undici.md index 27d46e2..2ae4c29 100644 --- a/docs/undici.md +++ b/docs/undici.md @@ -2,6 +2,8 @@ [undici](https://undici.nodejs.org/) is a fast HTTP/1.1 client from the Node.js core team. It powers the native `fetch` in Node.js 18+. This page describes how to use undici with proxies and how to send and receive custom proxy headers. +HTTPS requests use a custom undici connector. The connector opens the proxy tunnel with `ProxyHeadersAgent` (custom CONNECT headers in, CONNECT response headers out) and hands undici the TLS socket only after CONNECT succeeds. Do not pass a pre-opened socket as `connect: { socket }`: undici's default connector ignores that option and dials the origin directly, which bypasses the proxy. Plain HTTP is forwarded with undici `ProxyAgent` and its `headers` option (undici PR #1877); that path does not expose a separate CONNECT response. Pass `requestTls` for origin TLS options such as `{ rejectUnauthorized: false }`. undici 8 requires Node.js >= 22.19.0. + ## Getting Started ### Prerequisites @@ -45,7 +47,8 @@ Make an HTTP request through a proxy with custom headers. |------|------|-------------| | `url` | `string \| URL` | Target URL | | `options.proxy` | `string` | Proxy URL (required) | -| `options.proxyHeaders` | `Object` | Headers to send to the proxy | +| `options.proxyHeaders` | `Object` | Headers to send to the proxy on CONNECT | +| `options.requestTls` | `Object` | TLS options for the origin connection | | `options.method` | `string` | HTTP method (default: 'GET') | | `options.headers` | `Object` | Request headers | | `options.body` | `*` | Request body | diff --git a/lib/axios-proxy.js b/lib/axios-proxy.js index f73a7f8..b93e179 100644 --- a/lib/axios-proxy.js +++ b/lib/axios-proxy.js @@ -16,10 +16,10 @@ import { getProxyHeaders } from './core/proxy-headers-store.js'; * @param {Object} options.proxyHeaders - Headers to send to the proxy * @param {Function} options.onProxyConnect - Callback when CONNECT completes * @param {Object} options.axiosOptions - Additional axios instance options - * @returns {import('axios').AxiosInstance} + * @returns {Promise} * * @example - * const client = createProxyAxios({ + * const client = await createProxyAxios({ * proxy: 'http://user:pass@proxy:8080', * proxyHeaders: { 'X-ProxyMesh-Country': 'US' } * }); diff --git a/lib/got-proxy.js b/lib/got-proxy.js index 4dbf53c..cf59e68 100644 --- a/lib/got-proxy.js +++ b/lib/got-proxy.js @@ -16,10 +16,10 @@ import { getProxyHeaders } from './core/proxy-headers-store.js'; * @param {Object} options.proxyHeaders - Headers to send to the proxy * @param {Function} options.onProxyConnect - Callback when CONNECT completes * @param {Object} options.gotOptions - Additional got instance options - * @returns {import('got').Got} + * @returns {Promise} * * @example - * const client = createProxyGot({ + * const client = await createProxyGot({ * proxy: 'http://user:pass@proxy:8080', * proxyHeaders: { 'X-ProxyMesh-Country': 'US' } * }); diff --git a/lib/ky-proxy.js b/lib/ky-proxy.js index bb2ffcb..61a0724 100644 --- a/lib/ky-proxy.js +++ b/lib/ky-proxy.js @@ -2,10 +2,122 @@ * ky extension for proxy header support. * * Uses a custom `fetch` backed by node-fetch + ProxyHeadersAgent. + * ky 2.2+ requires that `fetch` resolve with a global `Response` + * (`instanceof Response` or `[object Response]`). node-fetch's Response and + * the ProxyResponse wrapper fail that check, so ky 2 gets a native Response + * with `proxyHeaders` attached. ky 1 still receives the node-fetch wrapper, + * which that line accepts. */ +import { Readable } from 'node:stream'; +import { createRequire } from 'node:module'; +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; import { createProxyFetch } from './node-fetch-proxy.js'; +const require = createRequire(import.meta.url); + +const NULL_BODY_STATUS = new Set([101, 204, 205, 304]); +const DROP_HEADERS = new Set([ + 'connection', + 'keep-alive', + 'proxy-connection', + 'transfer-encoding', + 'upgrade', + 'trailer', + 'te', + // node-fetch already decompresses when Content-Encoding is set, but leaves + // the header in place. Copying it makes the native Response decode twice. + 'content-encoding', + 'content-length', +]); + +/** + * @returns {number} ky major version, or 2 when it cannot be read + */ +function kyMajor() { + try { + const entry = require.resolve('ky'); + let dir = dirname(entry); + for (let i = 0; i < 6; i++) { + try { + const pkg = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')); + if (pkg && pkg.name === 'ky') { + return Number(String(pkg.version).split('.')[0]) || 2; + } + } catch { + // Keep walking toward the package root. + } + const parent = dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } + } catch { + // Fall through to the ky 2 behavior (native Response). + } + return 2; +} + +/** + * @param {import('./core/proxy-response.js').ProxyResponse} proxyResponse + * @returns {Response} + */ +function toNativeResponse(proxyResponse) { + if (typeof Response !== 'undefined' && proxyResponse instanceof Response) { + if (!proxyResponse.proxyHeaders) { + proxyResponse.proxyHeaders = new Map(); + } + return proxyResponse; + } + + const status = Number(proxyResponse.status ?? 200); + const nullBody = NULL_BODY_STATUS.has(status); + let body = null; + if (!nullBody && proxyResponse.body) { + const stream = proxyResponse.body; + body = typeof stream.getReader === 'function' + ? stream + : Readable.toWeb(stream); + } + + const native = new Response(body, { + status, + statusText: proxyResponse.statusText || '', + headers: copyHeaders(proxyResponse.headers), + }); + native.proxyHeaders = proxyResponse.proxyHeaders || new Map(); + return native; +} + +/** + * @param {{ raw?: () => Record, entries?: () => Iterable<[string, string]> }|undefined} source + * @returns {Headers} + */ +function copyHeaders(source) { + const headers = new Headers(); + if (!source) { + return headers; + } + const raw = typeof source.raw === 'function' ? source.raw() : null; + const pairs = raw + ? Object.entries(raw).flatMap(([name, values]) => + [].concat(values).map((value) => [name, value])) + : [...source.entries()]; + for (const [name, value] of pairs) { + if (value == null || DROP_HEADERS.has(String(name).toLowerCase())) { + continue; + } + try { + headers.append(name, String(value)); + } catch { + // The Fetch implementation rejects some header names. + } + } + return headers; +} + /** * Create a ky instance with proxy header support. * @@ -13,11 +125,19 @@ import { createProxyFetch } from './node-fetch-proxy.js'; * @param {string} options.proxy - Proxy URL * @param {Object} [options.proxyHeaders] - Headers to send on CONNECT * @param {Function} [options.onProxyConnect] - CONNECT callback + * @param {Object} [options.tlsOptions] - TLS options for the origin connection * @param {Object} [options.kyOptions] - Options passed to ky.create() * @returns {Promise} */ export async function createProxyKy(options) { - const { proxy, proxyHeaders = {}, onProxyConnect, proxyTlsOptions, kyOptions = {} } = options; + const { + proxy, + proxyHeaders = {}, + onProxyConnect, + proxyTlsOptions, + tlsOptions, + kyOptions = {}, + } = options; if (!proxy) { throw new Error('proxy option is required'); @@ -30,7 +150,16 @@ export async function createProxyKy(options) { throw new Error('ky is required. Install it with: npm install ky'); } - const fetch = createProxyFetch({ proxy, proxyHeaders, onProxyConnect, proxyTlsOptions }); + const fetchImpl = createProxyFetch({ + proxy, + proxyHeaders, + onProxyConnect, + proxyTlsOptions, + tlsOptions, + }); + const fetch = kyMajor() >= 2 + ? async (input, init) => toNativeResponse(await fetchImpl(input, init)) + : fetchImpl; return ky.create({ ...kyOptions, diff --git a/lib/node-fetch-proxy.js b/lib/node-fetch-proxy.js index 4541ae0..527fe38 100644 --- a/lib/node-fetch-proxy.js +++ b/lib/node-fetch-proxy.js @@ -29,7 +29,7 @@ import { getProxyHeaders, runWithProxyHeadersContext, takeProxyHeadersContext } * const data = await response.json(); */ export async function proxyFetch(url, options = {}) { - const { proxy, proxyHeaders = {}, onProxyConnect, proxyTlsOptions, ...fetchOptions } = options; + const { proxy, proxyHeaders = {}, onProxyConnect, proxyTlsOptions, tlsOptions, ...fetchOptions } = options; if (!proxy) { throw new Error('proxy option is required'); @@ -46,6 +46,7 @@ export async function proxyFetch(url, options = {}) { proxyHeaders, onProxyConnect, proxyTlsOptions, + tlsOptions, }); let requestUrl = url; @@ -89,7 +90,7 @@ export async function proxyFetch(url, options = {}) { * const response = await fetch('https://httpbin.org/ip'); */ export function createProxyFetch(options) { - const { proxy, proxyHeaders = {}, onProxyConnect, proxyTlsOptions } = options; + const { proxy, proxyHeaders = {}, onProxyConnect, proxyTlsOptions, tlsOptions } = options; return (url, fetchOptions = {}) => { return proxyFetch(url, { @@ -97,6 +98,7 @@ export function createProxyFetch(options) { proxyHeaders: { ...proxyHeaders, ...fetchOptions.proxyHeaders }, onProxyConnect, proxyTlsOptions, + tlsOptions, ...fetchOptions, }); }; diff --git a/lib/superagent-proxy.js b/lib/superagent-proxy.js index b4f6428..7ce6286 100644 --- a/lib/superagent-proxy.js +++ b/lib/superagent-proxy.js @@ -68,10 +68,10 @@ export function proxyPlugin(options) { * Create a superagent instance bound to a specific proxy configuration. * * @param {Object} options - Proxy configuration - * @returns {Object} - Object with get, post, etc. methods + * @returns {Promise} - Object with get, post, etc. methods * * @example - * const client = createProxySuperagent({ + * const client = await createProxySuperagent({ * proxy: 'http://proxy:8080', * proxyHeaders: { 'X-ProxyMesh-Country': 'US' } * }); diff --git a/lib/undici-proxy.js b/lib/undici-proxy.js index 45434ef..3737eaf 100644 --- a/lib/undici-proxy.js +++ b/lib/undici-proxy.js @@ -1,112 +1,18 @@ /** * Undici extension for proxy header support. * - * Provides wrappers around undici's request function that support - * sending custom headers to proxies and receiving proxy response headers. + * HTTPS requests use a custom undici connector. Passing an already-open + * tunnel as `connect: { socket }` does not work: undici's default connector + * (`lib/core/connect.js`) only upgrades `httpSocket` and otherwise dials + * `host` itself, which bypasses the proxy and leaks the client IP. + * The connector below never dials the origin. It reuses ProxyHeadersAgent, + * which sends CONNECT (including custom headers) and returns the TLS socket + * only after the proxy tunnel is up. CONNECT response headers are read from + * that socket. */ -import { parseProxyUrl, buildConnectRequest, createProxySocket, proxyReadyEvent } from './core/utils.js'; -import { parseConnectResponse, hasCompleteHeaders, ConnectError } from './core/connect-parser.js'; -import { attachProxyHeaders } from './core/proxy-headers-store.js'; - -/** - * Create a tunnel through the proxy with custom headers. - * - * @param {Object} options - Tunnel options - * @param {string} options.proxy - Proxy URL - * @param {string} options.targetHost - Target hostname - * @param {number} options.targetPort - Target port - * @param {Object} options.proxyHeaders - Headers to send to proxy - * @param {Object} [options.proxyTlsOptions] - TLS options for an https:// proxy - * @param {number} options.timeout - Timeout in ms - * @returns {Promise<{ socket: import('node:net').Socket, proxyHeaders: Map }>} - */ -async function createProxyTunnel(options) { - const { proxy, targetHost, targetPort, proxyHeaders = {}, proxyTlsOptions = {}, timeout = 30000 } = options; - - const proxyInfo = parseProxyUrl(proxy); - - return new Promise((resolve, reject) => { - const socket = createProxySocket(proxyInfo, proxyTlsOptions); - - let buffer = Buffer.alloc(0); - let timeoutId = null; - - const cleanup = () => { - if (timeoutId) { - clearTimeout(timeoutId); - timeoutId = null; - } - socket.removeAllListeners('data'); - socket.removeAllListeners('error'); - socket.removeAllListeners('close'); - }; - - const handleError = (err) => { - cleanup(); - socket.destroy(); - reject(err); - }; - - timeoutId = setTimeout(() => { - handleError(new Error(`Proxy CONNECT timeout after ${timeout}ms`)); - }, timeout); - - socket.on('error', handleError); - - socket.on('close', () => { - handleError(new Error('Proxy connection closed unexpectedly')); - }); - - socket.on(proxyReadyEvent(proxyInfo.protocol), () => { - try { - const connectRequest = buildConnectRequest( - targetHost, - targetPort, - proxyInfo.auth, - proxyHeaders - ); - socket.write(connectRequest); - } catch (err) { - handleError(err); - } - }); - - socket.on('data', (data) => { - buffer = Buffer.concat([buffer, data]); - - if (!hasCompleteHeaders(buffer)) { - return; - } - - const response = parseConnectResponse(buffer); - - if (!response) { - handleError(new Error('Invalid CONNECT response from proxy')); - return; - } - - cleanup(); - attachProxyHeaders(socket, response.headers); - - if (response.statusCode !== 200) { - socket.destroy(); - reject(new ConnectError( - `Proxy CONNECT failed: ${response.statusCode} ${response.statusMessage}`, - response.statusCode, - response.statusMessage, - response.headers - )); - return; - } - - resolve({ - socket, - proxyHeaders: response.headers, - }); - }); - }); -} +import { ProxyHeadersAgent } from './core/proxy-headers-agent.js'; +import { getProxyHeadersFromSocket } from './core/proxy-headers-store.js'; /** * Make an HTTP request through a proxy with custom headers. @@ -114,10 +20,12 @@ async function createProxyTunnel(options) { * @param {string|URL} url - Target URL * @param {Object} options - Request options * @param {string} options.proxy - Proxy URL (e.g., 'http://user:pass@proxy:8080') - * @param {Object} options.proxyHeaders - Headers to send to the proxy - * @param {string} options.method - HTTP method (default: 'GET') - * @param {Object} options.headers - Request headers - * @param {*} options.body - Request body + * @param {Object} [options.proxyHeaders] - Headers to send to the proxy + * @param {Object} [options.proxyTlsOptions] - TLS options for an https:// proxy + * @param {Object} [options.requestTls] - TLS options for the origin (https only) + * @param {string} [options.method] - HTTP method (default: 'GET') + * @param {Object} [options.headers] - Request headers + * @param {*} [options.body] - Request body * @returns {Promise<{ statusCode: number, headers: Object, body: *, proxyHeaders: Map }>} * * @example @@ -132,7 +40,14 @@ async function createProxyTunnel(options) { * console.log(proxyHeaders.get('x-proxymesh-ip')); */ export async function request(url, options = {}) { - const { proxy, proxyHeaders = {}, proxyTlsOptions = {}, ...requestOptions } = options; + const { + proxy, + proxyHeaders = {}, + proxyTlsOptions = {}, + requestTls = {}, + tlsOptions, + ...requestOptions + } = options; if (!proxy) { throw new Error('proxy option is required'); @@ -149,7 +64,7 @@ export async function request(url, options = {}) { const isHttps = targetUrl.protocol === 'https:'; if (!isHttps) { - const proxyAgent = new undici.ProxyAgent(proxy); + const proxyAgent = createHttpProxyAgent(undici, proxy, proxyHeaders, proxyTlsOptions); try { const response = await undici.request(url, { @@ -161,6 +76,8 @@ export async function request(url, options = {}) { statusCode: response.statusCode, headers: response.headers, body: response.body, + // Plain HTTP is forwarded, not CONNECT-tunneled, so there is + // no separate CONNECT response to capture. proxyHeaders: new Map(), }; } finally { @@ -168,26 +85,50 @@ export async function request(url, options = {}) { } } - const targetHost = targetUrl.hostname; + const origin = `${targetUrl.protocol}//${targetUrl.host}`; const targetPort = parseInt(targetUrl.port, 10) || 443; - - const tunnel = await createProxyTunnel({ - proxy, - targetHost, - targetPort, + const agent = new ProxyHeadersAgent(proxy, { proxyHeaders, proxyTlsOptions, + tlsOptions: { ...requestTls, ...tlsOptions }, }); - const client = new undici.Client(`https://${targetHost}:${targetPort}`, { - connect: { - socket: tunnel.socket, - }, - }); + /** @type {Map|null} */ + let connectHeaders = null; + let client; try { + client = new undici.Client(origin, { + connect(opts, callback) { + let settled = false; + const done = (err, socket) => { + if (settled) { + return; + } + settled = true; + if (err) { + callback(err); + return; + } + connectHeaders = getProxyHeadersFromSocket(socket) + || agent.lastProxyHeaders + || new Map(); + callback(null, socket); + }; + + try { + agent.createConnection({ + host: opts.hostname, + port: opts.port || targetPort, + }, done); + } catch (err) { + done(err); + } + }, + }); + const response = await client.request({ - path: targetUrl.pathname + targetUrl.search, + path: `${targetUrl.pathname}${targetUrl.search}`, method: requestOptions.method || 'GET', headers: requestOptions.headers, body: requestOptions.body, @@ -197,10 +138,40 @@ export async function request(url, options = {}) { statusCode: response.statusCode, headers: response.headers, body: response.body, - proxyHeaders: tunnel.proxyHeaders, + proxyHeaders: connectHeaders || new Map(), }; } finally { - await client.close(); + // Don't await close before the caller can read `body` (close waits + // for the in-flight response). The tunnel socket is owned by undici, + // not by the agent's pool. + if (client) { + client.close().catch(() => {}); + } + agent.destroy(); + } +} + +/** + * ProxyAgent for plain HTTP. `headers` is undici's CONNECT/proxy header + * option (undici PR #1877). Older ProxyAgent builds that reject the option + * fall back to a URL-only agent. + * @param {typeof import('undici')} undici + * @param {string} proxy + * @param {Record} proxyHeaders + * @param {object} proxyTlsOptions + */ +function createHttpProxyAgent(undici, proxy, proxyHeaders, proxyTlsOptions) { + const proxyOptions = { uri: proxy }; + if (proxyTlsOptions && Object.keys(proxyTlsOptions).length > 0) { + proxyOptions.proxyTls = proxyTlsOptions; + } + if (proxyHeaders && Object.keys(proxyHeaders).length > 0) { + proxyOptions.headers = proxyHeaders; + } + try { + return new undici.ProxyAgent(proxyOptions); + } catch { + return new undici.ProxyAgent(proxy); } } @@ -218,4 +189,4 @@ export async function post(url, options) { return request(url, { ...options, method: 'POST' }); } -export { ConnectError }; +export { ConnectError } from './core/connect-parser.js'; diff --git a/package-lock.json b/package-lock.json index a297f01..b935ac0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "wretch": "^3.0.7" }, "engines": { - "node": ">=18.0.0" + "node": ">=22.19.0" }, "funding": { "type": "individual", @@ -32,14 +32,14 @@ }, "peerDependencies": { "axios": "^1.0.0", - "got": "^12.0.0 || ^13.0.0 || ^14.0.0", - "ky": "^2.0.0", - "make-fetch-happen": "^14.0.0", + "got": "^12.0.0 || ^13.0.0 || ^14.0.0 || ^15.0.0 || ^16.0.0", + "ky": "^1.0.0 || ^2.0.0", + "make-fetch-happen": "^14.0.0 || ^15.0.0 || ^16.0.0", "needle": "^3.0.0", "node-fetch": "^3.0.0", "superagent": "^8.0.0 || ^9.0.0 || ^10.0.0", - "typed-rest-client": "^2.0.0", - "undici": "^5.0.0 || ^6.0.0 || ^7.0.0", + "typed-rest-client": "^2.0.0 || ^3.0.0", + "undici": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0", "wretch": "^2.0.0 || ^3.0.0" }, "peerDependenciesMeta": { @@ -2376,9 +2376,9 @@ } }, "node_modules/ky": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/ky/-/ky-2.1.0.tgz", - "integrity": "sha512-nIKwelw+7qVqKOlX4Eht6GXfEXlDHcdIjyLWV3s119kYt5s+70ayTnYS+0fRNmtgGRRAfqdeWQbh8YizqkS0ng==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ky/-/ky-2.2.0.tgz", + "integrity": "sha512-4+B63kyZt+Y8snQiUGydK1vgY9xYQJZEh9K/nIAIEts+gYJzy1EiAw3BbK8xpWRuWUpN70SNCMjqRGTm75xxIA==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index f379b4f..19c3b80 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "javascript-proxy-headers", "version": "0.3.0", - "description": "Extensions for JavaScript HTTP libraries to support sending and receiving custom proxy headers during HTTPS CONNECT tunneling", + "description": "Send and read custom proxy headers on HTTPS CONNECT for axios, got, node-fetch, undici, ky and more", "type": "module", "main": "index.js", "types": "types/index.d.ts", @@ -60,7 +60,8 @@ ], "scripts": { "release:pr": "node scripts/create-release-pr.mjs", - "test:unit": "node --test test/test_connect_security.js", + "test:unit": "node --test test/test_connect_security.js test/test_proxy_traversal.js", + "test:local-proxy": "node test/local-mock-integration.js", "test": "node test/test_proxy_headers.js core axios node-fetch got undici superagent ky wretch make-fetch-happen needle typed-rest-client", "test:run": "node run_tests.js", "test:verbose": "node test/test_proxy_headers.js -v", @@ -72,24 +73,31 @@ }, "keywords": [ "proxy", + "proxy-headers", "http", "https", + "https-proxy", + "https-proxy-agent", + "proxy-agent", "connect", "tunnel", "headers", "axios", "fetch", + "node-fetch", "got", "undici", "superagent", "proxymesh", + "rotating-proxy", "web-scraping", "http-client", "ky", "wretch", "make-fetch-happen", "needle", - "typed-rest-client" + "typed-rest-client", + "x-proxymesh-ip" ], "author": { "name": "ProxyMesh", @@ -109,18 +117,18 @@ "url": "https://proxymesh.com" }, "engines": { - "node": ">=18.0.0" + "node": ">=22.19.0" }, "peerDependencies": { "axios": "^1.0.0", - "got": "^12.0.0 || ^13.0.0 || ^14.0.0", - "ky": "^2.0.0", - "make-fetch-happen": "^14.0.0", + "got": "^12.0.0 || ^13.0.0 || ^14.0.0 || ^15.0.0 || ^16.0.0", + "ky": "^1.0.0 || ^2.0.0", + "make-fetch-happen": "^14.0.0 || ^15.0.0 || ^16.0.0", "needle": "^3.0.0", "node-fetch": "^3.0.0", "superagent": "^8.0.0 || ^9.0.0 || ^10.0.0", - "typed-rest-client": "^2.0.0", - "undici": "^5.0.0 || ^6.0.0 || ^7.0.0", + "typed-rest-client": "^2.0.0 || ^3.0.0", + "undici": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0", "wretch": "^2.0.0 || ^3.0.0" }, "peerDependenciesMeta": { diff --git a/test/local-mock-integration.js b/test/local-mock-integration.js new file mode 100644 index 0000000..0f55d2f --- /dev/null +++ b/test/local-mock-integration.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node +/** + * Run the adapter integration harness against a local CONNECT proxy. + * Fails unless the mock proxy observed CONNECT and tunneled bytes. + * + * Usage: node test/local-mock-integration.js [harness args...] + */ + +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +import { makeSelfSignedCert, startConnectProxy, startOrigin } from './mock-connect-proxy.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const harness = join(__dirname, 'test_proxy_headers.js'); + +const cert = makeSelfSignedCert(); +const origin = await startOrigin(cert); +const proxy = await startConnectProxy(); + +const child = spawn(process.execPath, [harness, ...process.argv.slice(2)], { + stdio: 'inherit', + env: { + ...process.env, + PROXY_URL: proxy.url, + TEST_URL: origin.url, + PROXY_HEADER: 'X-ProxyMesh-IP', + NODE_EXTRA_CA_CERTS: cert.certPath, + // The origin cert is a local test CA via NODE_EXTRA_CA_CERTS. + // Do not inherit a caller-supplied reject-unauthorized override. + NODE_TLS_REJECT_UNAUTHORIZED: process.env.NODE_TLS_REJECT_UNAUTHORIZED || '1', + }, +}); + +const code = await new Promise((resolve, reject) => { + child.on('error', reject); + child.on('exit', (status) => resolve(status ?? 1)); +}); + +origin.server.close(); +proxy.server.close(); +cert.cleanup(); + +if (code !== 0) { + process.exit(code); +} + +const failures = []; +if (proxy.stats.connects < 1) { + failures.push('mock proxy saw no CONNECT'); +} +if (proxy.stats.clientToUp < 1) { + failures.push('mock proxy tunneled 0 client bytes'); +} +if (proxy.stats.upToClient < 1) { + failures.push('mock proxy tunneled 0 origin bytes'); +} +if (failures.length) { + console.error(`[FAIL] proxy traversal: ${failures.join('; ')}`); + console.error(proxy.stats); + process.exit(1); +} + +console.log( + `[PASS] proxy traversal: ${proxy.stats.connects} CONNECT(s), ` + + `${proxy.stats.clientToUp}B client->origin, ${proxy.stats.upToClient}B origin->client`, +); diff --git a/test/mock-connect-proxy.js b/test/mock-connect-proxy.js new file mode 100644 index 0000000..a8f1277 --- /dev/null +++ b/test/mock-connect-proxy.js @@ -0,0 +1,183 @@ +/** + * Local CONNECT proxy and HTTPS origin for adapter tests. + * The proxy records CONNECT requests and tunneled byte counts so a test can + * fail when a client dials the origin directly. + */ + +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import net from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import tls from 'node:tls'; + +export const FAKE_PROXY_IP = '203.0.113.7'; + +export function makeSelfSignedCert() { + const dir = mkdtempSync(join(tmpdir(), 'jph-tls-')); + const keyPath = join(dir, 'key.pem'); + const certPath = join(dir, 'cert.pem'); + const configPath = join(dir, 'openssl.cnf'); + writeFileSync(configPath, [ + '[req]', + 'distinguished_name=req_distinguished_name', + 'x509_extensions=v3_req', + 'prompt=no', + '[req_distinguished_name]', + 'CN=127.0.0.1', + '[v3_req]', + 'subjectAltName=IP:127.0.0.1,DNS:localhost', + 'basicConstraints=CA:FALSE', + '', + ].join('\n')); + execFileSync('openssl', [ + 'req', '-x509', '-newkey', 'rsa:2048', + '-keyout', keyPath, + '-out', certPath, + '-days', '1', + '-nodes', + '-config', configPath, + '-extensions', 'v3_req', + ], { stdio: 'pipe' }); + return { + key: readFileSync(keyPath), + cert: readFileSync(certPath), + certPath, + cleanup: () => rmSync(dir, { recursive: true, force: true }), + }; +} + +/** + * HTTPS origin that answers one JSON body per request. + * @param {{ cert: Buffer, key: Buffer }} tlsOptions + */ +export async function startOrigin(tlsOptions) { + const server = tls.createServer({ cert: tlsOptions.cert, key: tlsOptions.key }, (socket) => { + let buf = Buffer.alloc(0); + socket.on('data', (chunk) => { + buf = Buffer.concat([buf, chunk]); + const sep = buf.indexOf('\r\n\r\n'); + if (sep === -1) { + return; + } + const head = buf.subarray(0, sep).toString('latin1'); + const body = JSON.stringify({ + ok: true, + via: 'origin', + requestLine: head.split('\r\n')[0], + }); + socket.end( + 'HTTP/1.1 200 OK\r\n' + + 'Content-Type: application/json\r\n' + + 'Connection: close\r\n' + + `Content-Length: ${Buffer.byteLength(body)}\r\n` + + '\r\n' + + body, + ); + }); + socket.on('error', () => {}); + }); + await listen(server); + const { port } = server.address(); + return { + server, + port, + url: `https://127.0.0.1:${port}/`, + }; +} + +/** + * HTTP CONNECT proxy. Echoes X-ProxyMesh-* request headers and always + * sends X-ProxyMesh-IP. Counts bytes piped after the CONNECT response. + */ +export async function startConnectProxy() { + const stats = { + connects: 0, + clientToUp: 0, + upToClient: 0, + lastConnectHead: '', + }; + const server = net.createServer((sock) => { + let buf = Buffer.alloc(0); + const onClientData = (chunk) => { + buf = Buffer.concat([buf, chunk]); + const sep = buf.indexOf('\r\n\r\n'); + if (sep === -1) { + return; + } + sock.off('data', onClientData); + const head = buf.subarray(0, sep).toString('latin1'); + const rest = buf.subarray(sep + 4); + stats.connects += 1; + stats.lastConnectHead = head; + const lines = head.split('\r\n'); + const requestLine = lines[0] || ''; + const target = requestLine.split(' ')[1] || ''; + const colon = target.lastIndexOf(':'); + let host = colon === -1 ? target : target.slice(0, colon); + const port = colon === -1 ? 443 : Number(target.slice(colon + 1)); + if (host.startsWith('[') && host.endsWith(']')) { + host = host.slice(1, -1); + } + const echo = lines.slice(1) + .filter((line) => line.toLowerCase().startsWith('x-proxymesh')) + .map((line) => { + const idx = line.indexOf(':'); + const name = line.slice(0, idx).trim(); + const value = line.slice(idx + 1).trim(); + return `${name}-Echo: ${value}\r\n`; + }) + .join(''); + + const up = net.connect({ host, port }); + const fail = () => { + sock.destroy(); + up.destroy(); + }; + up.on('error', fail); + sock.on('error', fail); + up.once('connect', () => { + sock.write( + 'HTTP/1.1 200 Connection established\r\n' + + `X-ProxyMesh-IP: ${FAKE_PROXY_IP}\r\n` + + echo + + '\r\n', + ); + if (rest.length) { + stats.clientToUp += rest.length; + up.write(rest); + } + sock.on('data', (chunk) => { + stats.clientToUp += chunk.length; + up.write(chunk); + }); + up.on('data', (chunk) => { + stats.upToClient += chunk.length; + sock.write(chunk); + }); + sock.on('end', () => up.end()); + up.on('end', () => sock.end()); + }); + }; + sock.on('data', onClientData); + sock.on('error', () => {}); + }); + await listen(server); + const { port } = server.address(); + return { + server, + port, + url: `http://127.0.0.1:${port}`, + stats, + }; +} + +function listen(server) { + return new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + server.off('error', reject); + resolve(); + }); + }); +} diff --git a/test/test_proxy_traversal.js b/test/test_proxy_traversal.js new file mode 100644 index 0000000..977b4c4 --- /dev/null +++ b/test/test_proxy_traversal.js @@ -0,0 +1,197 @@ +#!/usr/bin/env node +/** + * Proxy-traversal tests. These fail when a client dials the origin directly + * instead of sending CONNECT through the local mock proxy. + */ + +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import net from 'node:net'; +import { dirname, join } from 'node:path'; +import { test } from 'node:test'; + +import { createProxyKy } from '../lib/ky-proxy.js'; +import { createProxyRestClient } from '../lib/typed-rest-client-proxy.js'; +import { request as undiciRequest } from '../lib/undici-proxy.js'; +import { + FAKE_PROXY_IP, + makeSelfSignedCert, + startConnectProxy, + startOrigin, +} from './mock-connect-proxy.js'; + +const require = createRequire(import.meta.url); + +function packageVersion(name) { + const entry = require.resolve(name); + let dir = dirname(entry); + for (let i = 0; i < 6; i++) { + try { + const pkg = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')); + if (pkg.name === name) { + return String(pkg.version); + } + } catch { + // Walk up to the package root. + } + const parent = dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } + throw new Error(`Could not read version for ${name}`); +} + +async function withTunnel(fn) { + const cert = makeSelfSignedCert(); + const origin = await startOrigin(cert); + const proxy = await startConnectProxy(); + try { + await fn({ cert, origin, proxy }); + } finally { + origin.server.close(); + proxy.server.close(); + cert.cleanup(); + } +} + +function assertTraversed(proxy, originPort) { + assert.ok(proxy.stats.connects >= 1, 'mock proxy saw no CONNECT (traffic bypassed the proxy)'); + assert.match( + proxy.stats.lastConnectHead, + new RegExp(`^CONNECT 127\\.0\\.0\\.1:${originPort} HTTP/1\\.1`), + ); + assert.match(proxy.stats.lastConnectHead, /X-ProxyMesh-Country: US/); + assert.ok( + proxy.stats.clientToUp > 0, + 'mock proxy tunneled 0 bytes from the client (CONNECT was not used as a tunnel)', + ); + assert.ok( + proxy.stats.upToClient > 0, + 'mock proxy tunneled 0 bytes from the origin', + ); +} + +test('undici HTTPS traffic goes through the proxy tunnel', async () => { + await withTunnel(async ({ origin, proxy }) => { + const response = await undiciRequest(origin.url, { + proxy: proxy.url, + proxyHeaders: { 'X-ProxyMesh-Country': 'US' }, + requestTls: { rejectUnauthorized: false }, + }); + const body = await response.body.json(); + assert.equal(response.statusCode, 200); + assert.equal(body.ok, true); + assert.equal(body.via, 'origin'); + assert.ok(response.proxyHeaders instanceof Map); + assert.equal(response.proxyHeaders.get('x-proxymesh-ip'), FAKE_PROXY_IP); + assert.equal(response.proxyHeaders.get('x-proxymesh-country-echo'), 'US'); + assertTraversed(proxy, origin.port); + }); +}); + +test('undici plain HTTP sends proxy headers through ProxyAgent', async () => { + const origin = net.createServer((sock) => { + sock.once('data', () => { + const body = 'origin-ok'; + sock.end(`HTTP/1.1 200 OK\r\nContent-Length: ${body.length}\r\nConnection: close\r\n\r\n${body}`); + }); + sock.on('error', () => {}); + }); + await listen(origin); + const originPort = origin.address().port; + let sawProxyHeader = false; + const proxy = net.createServer((sock) => { + let buf = Buffer.alloc(0); + const onData = (chunk) => { + buf = Buffer.concat([buf, chunk]); + if (!buf.includes('\r\n\r\n')) { + return; + } + sock.off('data', onData); + const head = buf.toString('latin1'); + sawProxyHeader = /x-proxymesh-country:\s*US/i.test(head); + const up = net.connect(originPort, '127.0.0.1', () => { + up.write(buf); + }); + sock.pipe(up); + up.pipe(sock); + up.on('error', () => sock.destroy()); + sock.on('error', () => up.destroy()); + }; + sock.on('data', onData); + }); + await listen(proxy); + try { + const response = await undiciRequest(`http://127.0.0.1:${originPort}/`, { + proxy: `http://127.0.0.1:${proxy.address().port}`, + proxyHeaders: { 'X-ProxyMesh-Country': 'US' }, + }); + assert.equal(response.statusCode, 200); + assert.equal(await response.body.text(), 'origin-ok'); + assert.equal(sawProxyHeader, true); + assert.equal(response.proxyHeaders.size, 0); + } finally { + origin.close(); + proxy.close(); + } +}); + +function listen(server) { + return new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + server.off('error', reject); + resolve(); + }); + }); +} + +test('ky resolves fetch with a Response and traverses the proxy', async () => { + await withTunnel(async ({ origin, proxy }) => { + const api = await createProxyKy({ + proxy: proxy.url, + proxyHeaders: { 'X-ProxyMesh-Country': 'US' }, + tlsOptions: { rejectUnauthorized: false }, + kyOptions: { retry: 0, timeout: 10000 }, + }); + const response = await api(origin.url); + const kyMajor = Number(packageVersion('ky').split('.')[0]); + if (kyMajor >= 2) { + const isResponse = response instanceof Response + || Object.prototype.toString.call(response) === '[object Response]'; + assert.equal(isResponse, true, 'ky 2 fetch must resolve with a Response'); + } + assert.equal(response.proxyHeaders.get('x-proxymesh-ip'), FAKE_PROXY_IP); + const body = await response.json(); + assert.equal(body.ok, true); + assertTraversed(proxy, origin.port); + }); +}); + +test('typed-rest-client _getAgent override tunnels HTTPS on v2 and v3', async () => { + const trcVersion = packageVersion('typed-rest-client'); + const major = Number(trcVersion.split('.')[0]); + assert.ok(major === 2 || major === 3, `unexpected typed-rest-client ${trcVersion}`); + + await withTunnel(async ({ origin, proxy }) => { + const client = createProxyRestClient({ + userAgent: 'javascript-proxy-headers-test', + proxy: proxy.url, + proxyHeaders: { 'X-ProxyMesh-Country': 'US' }, + }); + assert.equal( + client.client._getAgent({ protocol: 'https:' }), + client.proxyAgent, + 'HTTPS must use ProxyHeadersAgent via the _getAgent override', + ); + client.proxyAgent.tlsOptions = { rejectUnauthorized: false }; + const result = await client.get(origin.url); + assert.equal(result.statusCode, 200); + assert.equal(result.result.ok, true); + assert.equal(result.proxyHeaders.get('x-proxymesh-ip'), FAKE_PROXY_IP); + assertTraversed(proxy, origin.port); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index 3eae74c..6ed5249 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,6 +1,5 @@ { "compilerOptions": { - "ignoreDeprecations": "6.0", "target": "ES2020", "module": "ESNext", "moduleResolution": "Bundler", @@ -9,7 +8,6 @@ "forceConsistentCasingInFileNames": true, "esModuleInterop": true, "lib": ["ES2020", "DOM"], - "baseUrl": ".", "paths": { "javascript-proxy-headers": ["./types/index.d.ts"], "javascript-proxy-headers/axios": ["./types/axios.d.ts"], diff --git a/types/ky.d.ts b/types/ky.d.ts index faa18e5..a0115b2 100644 --- a/types/ky.d.ts +++ b/types/ky.d.ts @@ -5,6 +5,8 @@ export interface CreateProxyKyOptions { proxyHeaders?: Record; onProxyConnect?: (headers: Map) => void; proxyTlsOptions?: object; + /** TLS options for the connection to the origin */ + tlsOptions?: object; kyOptions?: KyOptions; } diff --git a/types/undici.d.ts b/types/undici.d.ts index 34ed9fb..4135ac0 100644 --- a/types/undici.d.ts +++ b/types/undici.d.ts @@ -5,6 +5,10 @@ export interface UndiciRequestOptions { proxyHeaders?: Record; /** TLS options for an https:// proxy */ proxyTlsOptions?: object; + /** TLS options for the connection to the origin (https only) */ + requestTls?: object; + /** Alias of requestTls */ + tlsOptions?: object; /** HTTP method */ method?: string; /** Request headers */