diff --git a/packages/core/src/crawl.js b/packages/core/src/crawl.js index f70c6ac..cd5d390 100644 --- a/packages/core/src/crawl.js +++ b/packages/core/src/crawl.js @@ -54,6 +54,24 @@ const BROKEN_CODES = new Set([ const TLS_CODE = /^(ERR_TLS_|ERR_SSL_|CERT_|UNABLE_TO_|DEPTH_ZERO_SELF_SIGNED_CERT|SELF_SIGNED_CERT_IN_CHAIN|HOSTNAME_MISMATCH)/; +/** + * Our own services and the vendors behind them (rule 4 of the crawler rule): + * a house site, a pit.world host, our Supabase projects. A failure talking to + * one of these is ours to fix, not a slow third party, so it keeps the short + * retry instead of 30 days. Matched on the host the request went to. + */ +export const OWN_HOSTS = + /(^|\.)(pit\.world|profullstack\.com|nichedb\.dev|bittorrented\.com|ch4nnels\.com|c0upons\.com|outreachgraph\.com|smshub\.dev|rssamplifier\.com|p0dcasters\.com|logicsrc\.com|c0mpute\.com|crawlproof\.com|r4ck\.dev|weedforcrypto\.com|supabase\.co|coinpayportal\.com)$/i; + +/** The host a failed request went to: makeHttp records it; otherwise the first one the message names. */ +export function failedHost(err) { + const raw = err?.upstreamUrl ?? String(err?.message ?? ''); + const m = + String(raw).match(/https?:\/\/([^/\s:?#]+)/i) ?? + String(raw).match(/\b(?:ENOTFOUND|EAI_AGAIN)\s+([a-z0-9.-]+\.[a-z]{2,})/i); + return m ? m[1].toLowerCase() : null; +} + function codesOf(err) { const out = []; for (let e = err, depth = 0; e && depth < 4; e = e.cause, depth++) { @@ -76,6 +94,8 @@ export function failureClass(err) { if (!err || (typeof err !== 'object' && typeof err !== 'function')) return null; if ('flag' in err && (err.flag === 'slow' || err.flag === 'broken' || err.flag === null)) return err.flag; + const host = failedHost(err); + if (host && OWN_HOSTS.test(host)) return null; const codes = codesOf(err); if (codes.includes('EAI_AGAIN')) return null; // Our own database failing is never the upstream's fault: Bun SQL errors diff --git a/packages/core/src/http.js b/packages/core/src/http.js index c9d1be0..c889a1e 100644 --- a/packages/core/src/http.js +++ b/packages/core/src/http.js @@ -23,7 +23,7 @@ export function makeHttp({ userAgent, log = () => {} }) { try { return await send(url, opts); } catch (err) { - throw markUpstream(err); + throw markUpstream(err, url); } } @@ -186,14 +186,16 @@ async function httpError(res, url) { return Object.assign(new Error(`${res.status} from ${String(url).slice(0, 400)}${detail}`), { status: res.status, upstream: true, + upstreamUrl: String(url), }); } /** Mark a fetch failure as the upstream's, keeping the error itself (its name, code and cause). */ -function markUpstream(err) { +function markUpstream(err, url) { if (err && typeof err === 'object') { try { err.upstream = true; + err.upstreamUrl ??= String(url); } catch { // A frozen error stays as it is. } diff --git a/test/crawl-rule.test.js b/test/crawl-rule.test.js index 1779c55..6ef460b 100644 --- a/test/crawl-rule.test.js +++ b/test/crawl-rule.test.js @@ -90,6 +90,33 @@ describe('failureClass', () => { }); }); +describe('our own services keep the short retry (rule 4)', () => { + test('a house site, a pit.world host or our Supabase is never flagged', () => { + expect(failureClass(sysErr('ENOTFOUND', 'getaddrinfo ENOTFOUND abcxyz.supabase.co'))).toBe( + null, + ); + expect( + failureClass( + Object.assign(new Error('502 from https://ch4nnels.com/api/v1/seedboxes'), { status: 502 }), + ), + ).toBe(null); + expect( + failureClass( + Object.assign(new Error('The operation timed out.'), { + name: 'TimeoutError', + upstreamUrl: 'https://built.indie.pit.world/api/v1/products', + }), + ), + ).toBe(null); + // A third party on a lookalike name is still a third party. + expect( + failureClass( + Object.assign(new Error('502 from https://notpit.world.example/x'), { status: 502 }), + ), + ).toBe('broken'); + }); +}); + describe('runSource: a slow or broken upstream is flagged for 30 days', () => { const fail = (err) => ({ name: 'fake', pull: async () => Promise.reject(err) }); const run = (adapter) => {