From 1b49c9117a108d1b3abe058f9ca94e395dc00fdb Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 17 Mar 2026 18:49:12 +0200 Subject: [PATCH 1/6] make async `prepareRequestOptions` --- lib/request.js | 3 +- lib/utils.js | 256 +++++++++++++++++++++++++------------------------ 2 files changed, 135 insertions(+), 124 deletions(-) diff --git a/lib/request.js b/lib/request.js index 15b4f40c8..d7448b2d4 100644 --- a/lib/request.js +++ b/lib/request.js @@ -162,7 +162,8 @@ export default function(options, iframely_options, callback) { url: options.uri }); - fetchData(utils.prepareRequestOptions(options, iframely_options)) + utils.prepareRequestOptions(options, iframely_options) + .then(request_options => fetchData(request_options)) .then(result => { finish(null, result); }) diff --git a/lib/utils.js b/lib/utils.js index 418d98624..0e4a78f86 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -82,99 +82,104 @@ export function getMaxCacheTTL(url, options, default_min_ttl) { }; export function prepareRequestOptions(request_options, options) { + return new Promise((resolve, reject) => { + try { + if (request_options.url && !request_options.uri) { + request_options.uri = request_options.url; + } + var uri = request_options.uri; + delete request_options.url; - if (request_options.url && !request_options.uri) { - request_options.uri = request_options.url; - } - var uri = request_options.uri; - delete request_options.url; + var disable_language = false; - var disable_language = false; + function setPrerender(prerender_option) { + if (options?.allowPrerender && CONFIG.PRERENDER_URL) { + request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri); + + // Use string `proxy.prerender` as additional prerender param. + if (typeof prerender_option === 'string') { + request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + `prerender=${prerender_option}`; + } + } + } - function setPrerender(prerender_option) { - if (options?.allowPrerender && CONFIG.PRERENDER_URL) { - request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri); - - // Use string `proxy.prerender` as additional prerender param. - if (typeof prerender_option === 'string') { - request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + `prerender=${prerender_option}`; + var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender'); + if (enable_domain_prerender) { + setPrerender(enable_domain_prerender); } - } - } - var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender'); - if (enable_domain_prerender) { - setPrerender(enable_domain_prerender); - } + if (CONFIG.PROXY || (options && options.proxy)) { - if (CONFIG.PROXY || (options && options.proxy)) { + var proxy = (options && options.proxy) || getCustomProxyForUri(uri, options); + if (proxy) { + if (!enable_domain_prerender && proxy.prerender && options?.allowPrerender) { + setPrerender(proxy.prerender); - var proxy = (options && options.proxy) || getCustomProxyForUri(uri, options); - if (proxy) { - if (!enable_domain_prerender && proxy.prerender && options?.allowPrerender) { - setPrerender(proxy.prerender); - - } else if (proxy.proxy && CONFIG.PROXY_URL) { - request_options.uri = /{url}/.test(CONFIG.PROXY_URL) - ? CONFIG.PROXY_URL.replace(/{url}/, encodeURIComponent(uri)) - : CONFIG.PROXY_URL + encodeURIComponent(uri); - } else if (proxy.proxy_url && /{url}/.test(proxy.proxy_url)) { - request_options.uri = proxy.proxy_url.replace(/{url}/, encodeURIComponent(uri)); - } - if (proxy.user_agent) { - request_options.headers = request_options.headers || {}; - request_options.headers['User-Agent'] = proxy.user_agent; - } - if (proxy.headers) { - request_options.headers = request_options.headers || {}; - Object.assign(request_options.headers, proxy.headers) - } - if (proxy.request_options) { - Object.assign(request_options, proxy.request_options); - } - if (proxy.disable_http2) { - request_options.disable_http2 = true; - } - if (proxy.disable_language) { - disable_language = true; - } - if (options && proxy.maxredirects && (!options.redirectsHistory || options.redirectsHistory.length === 0)) { - options.maxRedirects = proxy.maxredirects; - } - if (proxy && proxy.timeout > 0) { - request_options.timeout = proxy.timeout > 100 ? proxy.timeout : proxy.timeout * 1000; + } else if (proxy.proxy && CONFIG.PROXY_URL) { + request_options.uri = /{url}/.test(CONFIG.PROXY_URL) + ? CONFIG.PROXY_URL.replace(/{url}/, encodeURIComponent(uri)) + : CONFIG.PROXY_URL + encodeURIComponent(uri); + } else if (proxy.proxy_url && /{url}/.test(proxy.proxy_url)) { + request_options.uri = proxy.proxy_url.replace(/{url}/, encodeURIComponent(uri)); + } + if (proxy.user_agent) { + request_options.headers = request_options.headers || {}; + request_options.headers['User-Agent'] = proxy.user_agent; + } + if (proxy.headers) { + request_options.headers = request_options.headers || {}; + Object.assign(request_options.headers, proxy.headers) + } + if (proxy.request_options) { + Object.assign(request_options, proxy.request_options); + } + if (proxy.disable_http2) { + request_options.disable_http2 = true; + } + if (proxy.disable_language) { + disable_language = true; + } + if (options && proxy.maxredirects && (!options.redirectsHistory || options.redirectsHistory.length === 0)) { + options.maxRedirects = proxy.maxredirects; + } + if (proxy && proxy.timeout > 0) { + request_options.timeout = proxy.timeout > 100 ? proxy.timeout : proxy.timeout * 1000; + } + } } - } - } - if (options && options.getProviderOptions - && (options.getProviderOptions('app.name') || options.getProviderOptions('app.ua_extension')) - && request_options.headers - && request_options.headers['User-Agent'] === CONFIG.USER_AGENT) { - - var ext = options.getProviderOptions('app.ua_extension', options.getProviderOptions('app.name')); - if ( /^[a-zA-Z0-9\.\s_-]+$/.test(ext) && ext.length > 1) { - ext = ext[0].toUpperCase() + ext.slice(1) - request_options.headers['User-Agent'] += ' ' + ext; - } - } - - var lang = options?.getProviderOptions && options.getProviderOptions('locale') || ''; - if (!disable_language) { - request_options.headers = request_options.headers || {}; - request_options.headers['Accept-Language'] = - lang /* !== '' */ - ? lang.replace('_', '-') - + (/\-/.test(lang) ? `,${lang.replace(/\-.*$/, '')};q=0.9` : '') - + ',en;q=0.7,*;q=0.5' - : 'en,*'; - } + if (options && options.getProviderOptions + && (options.getProviderOptions('app.name') || options.getProviderOptions('app.ua_extension')) + && request_options.headers + && request_options.headers['User-Agent'] === CONFIG.USER_AGENT) { + + var ext = options.getProviderOptions('app.ua_extension', options.getProviderOptions('app.name')); + if ( /^[a-zA-Z0-9\.\s_-]+$/.test(ext) && ext.length > 1) { + ext = ext[0].toUpperCase() + ext.slice(1) + request_options.headers['User-Agent'] += ' ' + ext; + } + } - prepareEncodedUri(request_options, 'uri'); + var lang = options?.getProviderOptions && options.getProviderOptions('locale') || ''; + if (!disable_language) { + request_options.headers = request_options.headers || {}; + request_options.headers['Accept-Language'] = + lang /* !== '' */ + ? lang.replace('_', '-') + + (/\-/.test(lang) ? `,${lang.replace(/\-.*$/, '')};q=0.9` : '') + + ',en;q=0.7,*;q=0.5' + : 'en,*'; + } - setCookieFromJar(uri, request_options.headers, options?.jar) + prepareEncodedUri(request_options, 'uri'); - return request_options; + setCookieFromJar(uri, request_options.headers, options?.jar) + + resolve(request_options); + } catch(ex) { + reject(ex); + } + }); }; /** @@ -207,21 +212,24 @@ export function prepareRequestOptions(request_options, options) { follow = 0; } - var request_options = prepareRequestOptions({ - // Reviewed. - uri: url, - method: 'GET', - headers: { - 'User-Agent': options.user_agent || CONFIG.USER_AGENT, - 'Accept': '*/*' - }, - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, - redirect: redirect, - follow: follow - }, options); - try { - fetchStreamKeepAlive(request_options) + var request_options; + prepareRequestOptions({ + // Reviewed. + uri: url, + method: 'GET', + headers: { + 'User-Agent': options.user_agent || CONFIG.USER_AGENT, + 'Accept': '*/*' + }, + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, + redirect: redirect, + follow: follow + }, options) + .then(_request_options => { + request_options = _request_options; + return fetchStreamKeepAlive(request_options) + }) .then(stream => { // Custom inner redirect logic with cookies. @@ -281,23 +289,26 @@ var getHead = function(url, options, callbacks) { follow = 0; } - var request_options = prepareRequestOptions({ - // jar: jar, - - // Reviewed. - uri: url, - method: 'HEAD', - headers: { - 'User-Agent': CONFIG.USER_AGENT - }, - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, - redirect: redirect, - follow: follow - // No abort controller for head. - }, options); - try { - fetchStreamAuthorized(request_options) + var request_options; + prepareRequestOptions({ + // jar: jar, + + // Reviewed. + uri: url, + method: 'HEAD', + headers: { + 'User-Agent': CONFIG.USER_AGENT + }, + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, + redirect: redirect, + follow: follow + // No abort controller for head. + }, options) + .then(_request_options => { + request_options = _request_options; + return fetchStreamAuthorized(request_options); + }) .then(response => { callbacks.onResponse && callbacks.onResponse(response, request_options); }) @@ -1237,19 +1248,18 @@ export function generateLinksHtml(data, options) { var getUriStatusPrivate = function(uri, options, cb) { - var request_options = prepareRequestOptions({ - // Reviewed. - uri: uri, - method: 'GET', - headers: { - 'User-Agent': CONFIG.USER_AGENT - }, - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, - follow: CONFIG.MAX_REDIRECTS - }); - try { - fetchStream(request_options) + prepareRequestOptions({ + // Reviewed. + uri: uri, + method: 'GET', + headers: { + 'User-Agent': CONFIG.USER_AGENT + }, + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, + follow: CONFIG.MAX_REDIRECTS + }) + .then(request_options => fetchStream(request_options)) .then(res => { // TODO: may cause AbortError before cb. res.abortController.abort(); From 964dc1c973bc9714f61f32375e39d5c6ea3fa993 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 17 Mar 2026 19:10:03 +0200 Subject: [PATCH 2/6] get custom request headers with `options.getSigHeaders` --- lib/utils.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/utils.js b/lib/utils.js index 0e4a78f86..32826a37d 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -88,9 +88,11 @@ export function prepareRequestOptions(request_options, options) { request_options.uri = request_options.url; } var uri = request_options.uri; + var original_uri = uri; delete request_options.url; var disable_language = false; + var request_sig = false; function setPrerender(prerender_option) { if (options?.allowPrerender && CONFIG.PRERENDER_URL) { @@ -145,6 +147,9 @@ export function prepareRequestOptions(request_options, options) { if (proxy && proxy.timeout > 0) { request_options.timeout = proxy.timeout > 100 ? proxy.timeout : proxy.timeout * 1000; } + if (proxy.sig && options.getSigHeaders) { + request_sig = true; + } } } @@ -174,8 +179,22 @@ export function prepareRequestOptions(request_options, options) { prepareEncodedUri(request_options, 'uri'); setCookieFromJar(uri, request_options.headers, options?.jar) + + if (request_sig) { + options.getSigHeaders(original_uri, function(error, headers) { + if (error) { + // Skip error. + resolve(request_options); + } else { + request_options.headers = request_options.headers || {}; + Object.assign(request_options.headers, headers) + resolve(request_options); + } + }); + } else { + resolve(request_options); + } - resolve(request_options); } catch(ex) { reject(ex); } From abeffaa16c16f15fd1d1c58df5ab31c384c08c58 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 17 Mar 2026 19:15:57 +0200 Subject: [PATCH 3/6] request custom headers when `options.sig` --- lib/utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index 32826a37d..d859cdd09 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -147,7 +147,7 @@ export function prepareRequestOptions(request_options, options) { if (proxy && proxy.timeout > 0) { request_options.timeout = proxy.timeout > 100 ? proxy.timeout : proxy.timeout * 1000; } - if (proxy.sig && options.getSigHeaders) { + if (proxy.sig) { request_sig = true; } } @@ -180,7 +180,7 @@ export function prepareRequestOptions(request_options, options) { setCookieFromJar(uri, request_options.headers, options?.jar) - if (request_sig) { + if ((options.sig || request_sig) && options.getSigHeaders) { options.getSigHeaders(original_uri, function(error, headers) { if (error) { // Skip error. From 11f906a4adcedf52df82895f875450be906c8b7f Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 17 Mar 2026 19:21:12 +0200 Subject: [PATCH 4/6] catch error in `getSigHeaders` --- lib/utils.js | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index d859cdd09..3137dc9d8 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -181,16 +181,21 @@ export function prepareRequestOptions(request_options, options) { setCookieFromJar(uri, request_options.headers, options?.jar) if ((options.sig || request_sig) && options.getSigHeaders) { - options.getSigHeaders(original_uri, function(error, headers) { - if (error) { - // Skip error. - resolve(request_options); - } else { - request_options.headers = request_options.headers || {}; - Object.assign(request_options.headers, headers) - resolve(request_options); - } - }); + try { + options.getSigHeaders(original_uri, function(error, headers) { + if (error) { + // Skip error. + resolve(request_options); + } else { + request_options.headers = request_options.headers || {}; + Object.assign(request_options.headers, headers); + resolve(request_options); + } + }); + } catch(ex) { + console.error('options.getSigHeaders exception', ex); + resolve(request_options); + } } else { resolve(request_options); } From 2359c819b1d93e6ddda79444b97d7443b535ca20 Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Tue, 17 Mar 2026 20:25:36 +0200 Subject: [PATCH 5/6] Update audit.log --- audit.log | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/audit.log b/audit.log index a0658338e..1e5a70c05 100644 --- a/audit.log +++ b/audit.log @@ -55,5 +55,89 @@ ├─────────────────────┼────────────────────────────────────────────────────────┤ │ More info │ https://github.com/advisories/GHSA-qpx9-hpmf-5gmw │ └─────────────────────┴────────────────────────────────────────────────────────┘ -4 vulnerabilities found -Severity: 4 high +┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ high │ Undici: Malicious WebSocket 64-bit length overflows │ +│ │ parser and crashes the client │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=7.0.0 <7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>cheerio>undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-f269-vfmq-vjvj │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ high │ Undici has Unbounded Memory Consumption in WebSocket │ +│ │ permessage-deflate Decompression │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=7.0.0 <7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>cheerio>undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-vrm6-8vpv-qv8q │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ high │ Undici has Unhandled Exception in WebSocket Client Due │ +│ │ to Invalid server_max_window_bits Validation │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=7.0.0 <7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>cheerio>undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-v9p9-hfj2-hcw8 │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ moderate │ Undici has an HTTP Request/Response Smuggling issue │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=7.0.0 <7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>cheerio>undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-2mjp-6q6p-2qxm │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ moderate │ Undici has CRLF Injection in undici via `upgrade` │ +│ │ option │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=7.0.0 <7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>cheerio>undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-4992-7rv2-5pvq │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ moderate │ Undici has Unbounded Memory Consumption in its │ +│ │ DeduplicationHandler via Response Buffering that leads │ +│ │ to DoS │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=7.17.0 <7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>cheerio>undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-phc3-fgpg-7m6h │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +10 vulnerabilities found +Severity: 3 moderate | 7 high From 4f3e7ac3df9eb64f59fbce6ac117b1e908e5b6cd Mon Sep 17 00:00:00 2001 From: Nazar Leush Date: Wed, 18 Mar 2026 21:26:16 +0200 Subject: [PATCH 6/6] log getSigHeaders error --- lib/utils.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/utils.js b/lib/utils.js index 3137dc9d8..5ca62405c 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -185,6 +185,7 @@ export function prepareRequestOptions(request_options, options) { options.getSigHeaders(original_uri, function(error, headers) { if (error) { // Skip error. + console.error('getSigHeaders error', error); resolve(request_options); } else { request_options.headers = request_options.headers || {};