diff --git a/scripts/ad-networks/fetch-policy.mjs b/scripts/ad-networks/fetch-policy.mjs index 51802b84..5d25f498 100644 --- a/scripts/ad-networks/fetch-policy.mjs +++ b/scripts/ad-networks/fetch-policy.mjs @@ -318,13 +318,17 @@ export function homepageChallenge(html) { return titles.some(value => /^\s*(?:just a moment|one moment,? please|attention required|access denied)[.!…\s]*(?:\|\s*(?:cloudflare|sucuri))?\s*$/i.test(value)); } +export const HOMEPAGE_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + export async function fetchHomepage(domain, dependencies = {}) { const target = `https://${domain}/`; let base = { checkedUrl: target, outcome: "inconclusive" }; try { if (!getDomain(domain, { allowPrivateDomains: true })) return { ...base, reason: "invalid_domain" }; const { response, finalUrl } = await safeFetch(target, { - headers: { Accept: "text/html,application/xhtml+xml" }, + // A standard browser Accept header. Some site firewalls answer 403 to the + // bare "text/html,application/xhtml+xml" list that browsers never send. + headers: { Accept: HOMEPAGE_ACCEPT }, timeoutMs: 15_000, }, dependencies); base = { ...base, finalUrl, httpStatus: response.status, contentType: response.headers.get("content-type") }; diff --git a/tests/pages/ad-networks/homepage-misses.test.mjs b/tests/pages/ad-networks/homepage-misses.test.mjs index c1ed8293..ed018a19 100644 --- a/tests/pages/ad-networks/homepage-misses.test.mjs +++ b/tests/pages/ad-networks/homepage-misses.test.mjs @@ -84,6 +84,12 @@ test('fetcher records HTTP and challenge outcomes; normal Cloudflare analytics r } assert.equal(homepageChallenge('
Access denied errors
'),false); }); +test('homepage requests send a standard browser Accept header',async()=>{ + const deps={lookup:async()=>[{address:'93.184.216.34',family:4}],sharedHostSlot:async()=>()=>{}}; + let accept; + await fetchHomepage('example.com',{...deps,fetch:async(_url,init)=>{accept=new Headers(init.headers).get('accept');return new Response('