diff --git a/scripts/ad-networks/fetch-policy.mjs b/scripts/ad-networks/fetch-policy.mjs index 51802b84..5d25f498 100644 --- a/scripts/ad-networks/fetch-policy.mjs +++ b/scripts/ad-networks/fetch-policy.mjs @@ -318,13 +318,17 @@ export function homepageChallenge(html) { return titles.some(value => /^\s*(?:just a moment|one moment,? please|attention required|access denied)[.!…\s]*(?:\|\s*(?:cloudflare|sucuri))?\s*$/i.test(value)); } +export const HOMEPAGE_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + export async function fetchHomepage(domain, dependencies = {}) { const target = `https://${domain}/`; let base = { checkedUrl: target, outcome: "inconclusive" }; try { if (!getDomain(domain, { allowPrivateDomains: true })) return { ...base, reason: "invalid_domain" }; const { response, finalUrl } = await safeFetch(target, { - headers: { Accept: "text/html,application/xhtml+xml" }, + // A standard browser Accept header. Some site firewalls answer 403 to the + // bare "text/html,application/xhtml+xml" list that browsers never send. + headers: { Accept: HOMEPAGE_ACCEPT }, timeoutMs: 15_000, }, dependencies); base = { ...base, finalUrl, httpStatus: response.status, contentType: response.headers.get("content-type") }; diff --git a/tests/pages/ad-networks/homepage-misses.test.mjs b/tests/pages/ad-networks/homepage-misses.test.mjs index c1ed8293..ed018a19 100644 --- a/tests/pages/ad-networks/homepage-misses.test.mjs +++ b/tests/pages/ad-networks/homepage-misses.test.mjs @@ -84,6 +84,12 @@ test('fetcher records HTTP and challenge outcomes; normal Cloudflare analytics r } assert.equal(homepageChallenge('How Cloudflare works

Access denied errors

'),false); }); +test('homepage requests send a standard browser Accept header',async()=>{ + const deps={lookup:async()=>[{address:'93.184.216.34',family:4}],sharedHostSlot:async()=>()=>{}}; + let accept; + await fetchHomepage('example.com',{...deps,fetch:async(_url,init)=>{accept=new Headers(init.headers).get('accept');return new Response('Home',{status:200,headers:{'content-type':'text/html'}});}}); + assert.equal(accept,'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'); +}); test('PostgreSQL writer does not update an inconclusive relationship',async()=>{ const queries=[],relationship={ad_network_id:'32',publisher_id:'1',status:'stale',consecutive_misses:2}; const client={query:async(sql,args)=>{queries.push({sql,args});return{rows:sql.includes('for update')?[relationship]:[]};},release(){}};