From 2039041ca8140058e36fa63e3ae85ecbb7cba197 Mon Sep 17 00:00:00 2001
From: RaviTejaKNTS <ravitejaknts@gmail.com>
Date: Fri, 13 Nov 2026 22:09:51 +0530
Subject: [PATCH] Send a standard browser Accept header for homepage checks

Some site firewalls answer 403 to the bare
"text/html,application/xhtml+xml" list, which browsers never send.
From OVH, with the same stack127bot user agent, 350cc.com and
1teachmore.lk return 403 to it and the full page to the standard
browser value. About 40% of live correction checks were inconclusive,
and 14 of 15 sampled were HTTP status failures. The user agent and the
HTML content-type check are unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 scripts/ad-networks/fetch-policy.mjs             | 6 +++++-
 tests/pages/ad-networks/homepage-misses.test.mjs | 6 ++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/scripts/ad-networks/fetch-policy.mjs b/scripts/ad-networks/fetch-policy.mjs
index 51802b84..5d25f498 100644
--- a/scripts/ad-networks/fetch-policy.mjs
+++ b/scripts/ad-networks/fetch-policy.mjs
@@ -318,13 +318,17 @@ export function homepageChallenge(html) {
   return titles.some(value => /^\s*(?:just a moment|one moment,? please|attention required|access denied)[.!…\s]*(?:\|\s*(?:cloudflare|sucuri))?\s*$/i.test(value));
 }
 
+export const HOMEPAGE_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
+
 export async function fetchHomepage(domain, dependencies = {}) {
   const target = `https://${domain}/`;
   let base = { checkedUrl: target, outcome: "inconclusive" };
   try {
     if (!getDomain(domain, { allowPrivateDomains: true })) return { ...base, reason: "invalid_domain" };
     const { response, finalUrl } = await safeFetch(target, {
-      headers: { Accept: "text/html,application/xhtml+xml" },
+      // A standard browser Accept header. Some site firewalls answer 403 to the
+      // bare "text/html,application/xhtml+xml" list that browsers never send.
+      headers: { Accept: HOMEPAGE_ACCEPT },
       timeoutMs: 15_000,
     }, dependencies);
     base = { ...base, finalUrl, httpStatus: response.status, contentType: response.headers.get("content-type") };
diff --git a/tests/pages/ad-networks/homepage-misses.test.mjs b/tests/pages/ad-networks/homepage-misses.test.mjs
index c1ed8293..ed018a19 100644
--- a/tests/pages/ad-networks/homepage-misses.test.mjs
+++ b/tests/pages/ad-networks/homepage-misses.test.mjs
@@ -84,6 +84,12 @@ test('fetcher records HTTP and challenge outcomes; normal Cloudflare analytics r
   }
   assert.equal(homepageChallenge('<title>How Cloudflare works</title><p>Access denied errors</p>'),false);
 });
+test('homepage requests send a standard browser Accept header',async()=>{
+  const deps={lookup:async()=>[{address:'93.184.216.34',family:4}],sharedHostSlot:async()=>()=>{}};
+  let accept;
+  await fetchHomepage('example.com',{...deps,fetch:async(_url,init)=>{accept=new Headers(init.headers).get('accept');return new Response('<title>Home</title>',{status:200,headers:{'content-type':'text/html'}});}});
+  assert.equal(accept,'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8');
+});
 test('PostgreSQL writer does not update an inconclusive relationship',async()=>{
   const queries=[],relationship={ad_network_id:'32',publisher_id:'1',status:'stale',consecutive_misses:2};
   const client={query:async(sql,args)=>{queries.push({sql,args});return{rows:sql.includes('for update')?[relationship]:[]};},release(){}};