diff --git a/audit.log b/audit.log index a0658338e..1e5a70c05 100644 --- a/audit.log +++ b/audit.log @@ -55,5 +55,89 @@ ├─────────────────────┼────────────────────────────────────────────────────────┤ │ More info │ https://github.com/advisories/GHSA-qpx9-hpmf-5gmw │ └─────────────────────┴────────────────────────────────────────────────────────┘ -4 vulnerabilities found -Severity: 4 high +┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ high │ Undici: Malicious WebSocket 64-bit length overflows │ +│ │ parser and crashes the client │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=7.0.0 <7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>cheerio>undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-f269-vfmq-vjvj │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ high │ Undici has Unbounded Memory Consumption in WebSocket │ +│ │ permessage-deflate Decompression │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=7.0.0 <7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>cheerio>undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-vrm6-8vpv-qv8q │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ high │ Undici has Unhandled Exception in WebSocket Client Due │ +│ │ to Invalid server_max_window_bits Validation │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=7.0.0 <7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>cheerio>undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-v9p9-hfj2-hcw8 │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ moderate │ Undici has an HTTP Request/Response Smuggling issue │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=7.0.0 <7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>cheerio>undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-2mjp-6q6p-2qxm │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ moderate │ Undici has CRLF Injection in undici via `upgrade` │ +│ │ option │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=7.0.0 <7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>cheerio>undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-4992-7rv2-5pvq │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +┌─────────────────────┬────────────────────────────────────────────────────────┐ +│ moderate │ Undici has Unbounded Memory Consumption in its │ +│ │ DeduplicationHandler via Response Buffering that leads │ +│ │ to DoS │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Package │ undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Vulnerable versions │ >=7.17.0 <7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Patched versions │ >=7.24.0 │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ Paths │ .>cheerio>undici │ +├─────────────────────┼────────────────────────────────────────────────────────┤ +│ More info │ https://github.com/advisories/GHSA-phc3-fgpg-7m6h │ +└─────────────────────┴────────────────────────────────────────────────────────┘ +10 vulnerabilities found +Severity: 3 moderate | 7 high diff --git a/lib/request.js b/lib/request.js index 15b4f40c8..d7448b2d4 100644 --- a/lib/request.js +++ b/lib/request.js @@ -162,7 +162,8 @@ export default function(options, iframely_options, callback) { url: options.uri }); - fetchData(utils.prepareRequestOptions(options, iframely_options)) + utils.prepareRequestOptions(options, iframely_options) + .then(request_options => fetchData(request_options)) .then(result => { finish(null, result); }) diff --git a/lib/utils.js b/lib/utils.js index 418d98624..5ca62405c 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -82,99 +82,129 @@ export function getMaxCacheTTL(url, options, default_min_ttl) { }; export function prepareRequestOptions(request_options, options) { + return new Promise((resolve, reject) => { + try { + if (request_options.url && !request_options.uri) { + request_options.uri = request_options.url; + } + var uri = request_options.uri; + var original_uri = uri; + delete request_options.url; - if (request_options.url && !request_options.uri) { - request_options.uri = request_options.url; - } - var uri = request_options.uri; - delete request_options.url; + var disable_language = false; + var request_sig = false; - var disable_language = false; + function setPrerender(prerender_option) { + if (options?.allowPrerender && CONFIG.PRERENDER_URL) { + request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri); + + // Use string `proxy.prerender` as additional prerender param. + if (typeof prerender_option === 'string') { + request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + `prerender=${prerender_option}`; + } + } + } - function setPrerender(prerender_option) { - if (options?.allowPrerender && CONFIG.PRERENDER_URL) { - request_options.uri = CONFIG.PRERENDER_URL + encodeURIComponent(uri); - - // Use string `proxy.prerender` as additional prerender param. - if (typeof prerender_option === 'string') { - request_options.uri += (request_options.uri.indexOf('?') > -1 ? '&' : '?' ) + `prerender=${prerender_option}`; + var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender'); + if (enable_domain_prerender) { + setPrerender(enable_domain_prerender); } - } - } - var enable_domain_prerender = options?.getDomainOptions && options.getDomainOptions('meta.prerender'); - if (enable_domain_prerender) { - setPrerender(enable_domain_prerender); - } + if (CONFIG.PROXY || (options && options.proxy)) { - if (CONFIG.PROXY || (options && options.proxy)) { + var proxy = (options && options.proxy) || getCustomProxyForUri(uri, options); + if (proxy) { + if (!enable_domain_prerender && proxy.prerender && options?.allowPrerender) { + setPrerender(proxy.prerender); - var proxy = (options && options.proxy) || getCustomProxyForUri(uri, options); - if (proxy) { - if (!enable_domain_prerender && proxy.prerender && options?.allowPrerender) { - setPrerender(proxy.prerender); - - } else if (proxy.proxy && CONFIG.PROXY_URL) { - request_options.uri = /{url}/.test(CONFIG.PROXY_URL) - ? CONFIG.PROXY_URL.replace(/{url}/, encodeURIComponent(uri)) - : CONFIG.PROXY_URL + encodeURIComponent(uri); - } else if (proxy.proxy_url && /{url}/.test(proxy.proxy_url)) { - request_options.uri = proxy.proxy_url.replace(/{url}/, encodeURIComponent(uri)); - } - if (proxy.user_agent) { - request_options.headers = request_options.headers || {}; - request_options.headers['User-Agent'] = proxy.user_agent; + } else if (proxy.proxy && CONFIG.PROXY_URL) { + request_options.uri = /{url}/.test(CONFIG.PROXY_URL) + ? CONFIG.PROXY_URL.replace(/{url}/, encodeURIComponent(uri)) + : CONFIG.PROXY_URL + encodeURIComponent(uri); + } else if (proxy.proxy_url && /{url}/.test(proxy.proxy_url)) { + request_options.uri = proxy.proxy_url.replace(/{url}/, encodeURIComponent(uri)); + } + if (proxy.user_agent) { + request_options.headers = request_options.headers || {}; + request_options.headers['User-Agent'] = proxy.user_agent; + } + if (proxy.headers) { + request_options.headers = request_options.headers || {}; + Object.assign(request_options.headers, proxy.headers) + } + if (proxy.request_options) { + Object.assign(request_options, proxy.request_options); + } + if (proxy.disable_http2) { + request_options.disable_http2 = true; + } + if (proxy.disable_language) { + disable_language = true; + } + if (options && proxy.maxredirects && (!options.redirectsHistory || options.redirectsHistory.length === 0)) { + options.maxRedirects = proxy.maxredirects; + } + if (proxy && proxy.timeout > 0) { + request_options.timeout = proxy.timeout > 100 ? proxy.timeout : proxy.timeout * 1000; + } + if (proxy.sig) { + request_sig = true; + } + } } - if (proxy.headers) { + + if (options && options.getProviderOptions + && (options.getProviderOptions('app.name') || options.getProviderOptions('app.ua_extension')) + && request_options.headers + && request_options.headers['User-Agent'] === CONFIG.USER_AGENT) { + + var ext = options.getProviderOptions('app.ua_extension', options.getProviderOptions('app.name')); + if ( /^[a-zA-Z0-9\.\s_-]+$/.test(ext) && ext.length > 1) { + ext = ext[0].toUpperCase() + ext.slice(1) + request_options.headers['User-Agent'] += ' ' + ext; + } + } + + var lang = options?.getProviderOptions && options.getProviderOptions('locale') || ''; + if (!disable_language) { request_options.headers = request_options.headers || {}; - Object.assign(request_options.headers, proxy.headers) - } - if (proxy.request_options) { - Object.assign(request_options, proxy.request_options); - } - if (proxy.disable_http2) { - request_options.disable_http2 = true; - } - if (proxy.disable_language) { - disable_language = true; + request_options.headers['Accept-Language'] = + lang /* !== '' */ + ? lang.replace('_', '-') + + (/\-/.test(lang) ? `,${lang.replace(/\-.*$/, '')};q=0.9` : '') + + ',en;q=0.7,*;q=0.5' + : 'en,*'; } - if (options && proxy.maxredirects && (!options.redirectsHistory || options.redirectsHistory.length === 0)) { - options.maxRedirects = proxy.maxredirects; - } - if (proxy && proxy.timeout > 0) { - request_options.timeout = proxy.timeout > 100 ? proxy.timeout : proxy.timeout * 1000; - } - } - } - if (options && options.getProviderOptions - && (options.getProviderOptions('app.name') || options.getProviderOptions('app.ua_extension')) - && request_options.headers - && request_options.headers['User-Agent'] === CONFIG.USER_AGENT) { - - var ext = options.getProviderOptions('app.ua_extension', options.getProviderOptions('app.name')); - if ( /^[a-zA-Z0-9\.\s_-]+$/.test(ext) && ext.length > 1) { - ext = ext[0].toUpperCase() + ext.slice(1) - request_options.headers['User-Agent'] += ' ' + ext; - } - } - - var lang = options?.getProviderOptions && options.getProviderOptions('locale') || ''; - if (!disable_language) { - request_options.headers = request_options.headers || {}; - request_options.headers['Accept-Language'] = - lang /* !== '' */ - ? lang.replace('_', '-') - + (/\-/.test(lang) ? `,${lang.replace(/\-.*$/, '')};q=0.9` : '') - + ',en;q=0.7,*;q=0.5' - : 'en,*'; - } - - prepareEncodedUri(request_options, 'uri'); + prepareEncodedUri(request_options, 'uri'); - setCookieFromJar(uri, request_options.headers, options?.jar) + setCookieFromJar(uri, request_options.headers, options?.jar) - return request_options; + if ((options.sig || request_sig) && options.getSigHeaders) { + try { + options.getSigHeaders(original_uri, function(error, headers) { + if (error) { + // Skip error. + console.error('getSigHeaders error', error); + resolve(request_options); + } else { + request_options.headers = request_options.headers || {}; + Object.assign(request_options.headers, headers); + resolve(request_options); + } + }); + } catch(ex) { + console.error('options.getSigHeaders exception', ex); + resolve(request_options); + } + } else { + resolve(request_options); + } + + } catch(ex) { + reject(ex); + } + }); }; /** @@ -207,21 +237,24 @@ export function prepareRequestOptions(request_options, options) { follow = 0; } - var request_options = prepareRequestOptions({ - // Reviewed. - uri: url, - method: 'GET', - headers: { - 'User-Agent': options.user_agent || CONFIG.USER_AGENT, - 'Accept': '*/*' - }, - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, - redirect: redirect, - follow: follow - }, options); - try { - fetchStreamKeepAlive(request_options) + var request_options; + prepareRequestOptions({ + // Reviewed. + uri: url, + method: 'GET', + headers: { + 'User-Agent': options.user_agent || CONFIG.USER_AGENT, + 'Accept': '*/*' + }, + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, + redirect: redirect, + follow: follow + }, options) + .then(_request_options => { + request_options = _request_options; + return fetchStreamKeepAlive(request_options) + }) .then(stream => { // Custom inner redirect logic with cookies. @@ -281,23 +314,26 @@ var getHead = function(url, options, callbacks) { follow = 0; } - var request_options = prepareRequestOptions({ - // jar: jar, - - // Reviewed. - uri: url, - method: 'HEAD', - headers: { - 'User-Agent': CONFIG.USER_AGENT - }, - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, - redirect: redirect, - follow: follow - // No abort controller for head. - }, options); - try { - fetchStreamAuthorized(request_options) + var request_options; + prepareRequestOptions({ + // jar: jar, + + // Reviewed. + uri: url, + method: 'HEAD', + headers: { + 'User-Agent': CONFIG.USER_AGENT + }, + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, + redirect: redirect, + follow: follow + // No abort controller for head. + }, options) + .then(_request_options => { + request_options = _request_options; + return fetchStreamAuthorized(request_options); + }) .then(response => { callbacks.onResponse && callbacks.onResponse(response, request_options); }) @@ -1237,19 +1273,18 @@ export function generateLinksHtml(data, options) { var getUriStatusPrivate = function(uri, options, cb) { - var request_options = prepareRequestOptions({ - // Reviewed. - uri: uri, - method: 'GET', - headers: { - 'User-Agent': CONFIG.USER_AGENT - }, - timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, - follow: CONFIG.MAX_REDIRECTS - }); - try { - fetchStream(request_options) + prepareRequestOptions({ + // Reviewed. + uri: uri, + method: 'GET', + headers: { + 'User-Agent': CONFIG.USER_AGENT + }, + timeout: options.timeout || CONFIG.RESPONSE_TIMEOUT, + follow: CONFIG.MAX_REDIRECTS + }) + .then(request_options => fetchStream(request_options)) .then(res => { // TODO: may cause AbortError before cb. res.abortController.abort();