Recognize ad code that speed and consent plugins delay

package.json

@@ -112,6 +112,7 @@
     "@namefi/dnssec-audit": "^2.1.1",
     "@next/mdx": "16.2.9",
     "@types/pg": "^8.20.0",
+    "acorn": "^8.19.0",
     "dns-packet": "^5.6.1",
     "echarts": "^6.1.0",
     "lucide-react": "^1.18.0",

scripts/ad-networks/local-verifier-store.mjs

@@ -143,7 +143,8 @@ function ageMissingRelationships(database, existing, positiveIds, checkedAt, evi
       checkedAt,
       checkedAt,
       checkedAt,
-      JSON.stringify({ checked_at: checkedAt, homepage: evidence?.homepage ?? null, ads_txt_sha256: evidence?.ads_txt_sha256 ?? null }),
+      JSON.stringify({ checked_at: checkedAt, homepage: evidence?.homepage ?? null, ads_txt_sha256: evidence?.ads_txt_sha256 ?? null,
+        matcher_version: evidence?.matcher_version ?? null }),
       relationship.ad_network_id,
       relationship.publisher_id,
     );

scripts/ad-networks/matcher.mjs

@@ -1,4 +1,5 @@
 import { parse } from 'parse5';
+import { parse as parseJavaScript } from 'acorn';
 
 function sourceNetworkSlugs(discoverySources) {
   const aliases = new Map([
@@ -32,17 +33,102 @@ function evidenceForManager(variable, fetchEvidence, parserVersion) {
   };
 }
 
-export const MATCHER_VERSION = "ad-network-1.4.1";
+export const MATCHER_VERSION = "ad-network-1.5.2";
 
 const javascriptTypes = new Set(['text/javascript', 'application/javascript',
   'text/ecmascript', 'application/ecmascript', 'application/x-javascript',
   'application/x-ecmascript', 'text/jscript', 'text/livescript', 'text/x-javascript',
   'text/x-ecmascript', ...Array.from({ length: 6 }, (_, i) => `text/javascript1.${i}`)]);
 
+// Each rule lists only sources its loader actually activates. The source-code
+// audit and measured site counts are in content_docs/ad-networks/maintenance.md.
+const deferredSources = new Map([
+  ['litespeed/javascript', { sources: ['data-src', 'src'] }],
+  ...['rocketlazyloadscript', 'text/rocketlazyloadscript'].map(type => [type,
+    { sources: ['data-rocket-src', 'src'], savedType: 'data-rocket-type' }]),
+  ['pmdelayedscript', { sources: ['data-src', 'src'], savedType: 'data-perfmatters-type' }],
+  ['pencilazyscript', { sources: ['src'], savedType: 'data-pencilazy-type' }],
+  ['o/js-lzl', { sources: ['data-src', 'src'], savedType: 'data-type' }],
+  ['starcache/delay', { sources: ['data-starcache-src', 'src'], savedType: 'data-starcache-type' }],
+]);
+const categories = new Set(['preferences', 'statistics', 'marketing']);
+const activationSources = ['src', 'data-src', 'data-rocket-src', 'data-wpmeteor-src',
+  'consent-original-src-_', 'data-starcache-src', 'data-md-delayed-src',
+  'data-borlabs-script-blocker-src', 'data-cmplz-src'];
+const value = (attributes, name) => attributes.get(name)?.trim() ?? '';
+
+const isJavaScript = type => type === '' || type === 'module' || javascriptTypes.has(type.split(';')[0].trim());
+
+function activation(type, attributes) {
+  const held = (sources, savedType) => {
+    const restored = savedType ? value(attributes, savedType).toLowerCase() : '';
+    // Only this measured chain has an audited second activation step.
+    const liteSpeedChain = normalized === 'pmdelayedscript' && restored === 'litespeed/javascript';
+    if (savedType && !isJavaScript(restored) && !liteSpeedChain) return null;
+    return { sources, held: true, module: savedType ? restored === 'module' : normalized.endsWith('-module') || normalized === 'module' };
+  };
+  const normalized = type.trim().toLowerCase();
+  if (normalized === 'pmdelayedscript' && value(attributes, 'data-perfmatters-type').toLowerCase() !== 'litespeed/javascript') {
+    return held(['src'], 'data-perfmatters-type');
+  }
+  if (deferredSources.has(normalized)) {
+    const { sources, savedType } = deferredSources.get(normalized);
+    return held(sources, savedType);
+  }
+  if (/^[a-f0-9]{24}-(?:text\/javascript|application\/javascript|module)$/.test(normalized)) return held(['src']);
+  if (normalized === 'javascript/blocked' && value(attributes, 'data-wpmeteor-type')
+    && isJavaScript(value(attributes, 'data-wpmeteor-type').toLowerCase())) return held(['data-wpmeteor-src'], 'data-wpmeteor-type');
+  if (normalized === 'application/consent' && value(attributes, 'consent-required') && value(attributes, 'consent-original-type-_')
+    && isJavaScript(value(attributes, 'consent-original-type-_').toLowerCase())) return held(['consent-original-src-_', 'src'], 'consent-original-type-_');
+  if (normalized === 'didomi/javascript'
+    && ['data-vendor', 'data-vendor-raw', 'data-purposes'].some(name => value(attributes, name))) return held(['src']);
+  if (normalized === 'text/template' && value(attributes, 'data-borlabs-script-blocker-id')) {
+    return held(['data-borlabs-script-blocker-src', 'src']);
+  }
+  if (normalized === 'text/plain') {
+    if (value(attributes, 'data-md-delayed-src')) return held(['data-md-delayed-src']);
+    if (value(attributes, 'class').split(/\s+/).some(token => /^optanon-category-C\d+(?:-C\d+)*$/.test(token))) return held(['src']);
+    const cookiebot = value(attributes, 'data-cookieconsent').split(',').map(item => item.trim());
+    if (cookiebot.length && cookiebot.every(item => ['preferences', 'statistics', 'marketing'].includes(item))) return held(['src']);
+    // Complianz selects these exact categories; data-service alone is metadata,
+    // not enough to identify a held script. Its external source is cmplz-src.
+    if (categories.has(value(attributes, 'data-category'))) return held(['data-cmplz-src']);
+    if (value(attributes, 'data-cookiescript') === 'accepted') return held(['src']);
+    // Orest Bida CookieConsent v2, measured with advertising consent.
+    if (['necessary', 'analytics', 'advertising', 'targeting'].includes(value(attributes, 'data-cookiecategory'))) return held(['data-src', 'src'], 'data-type');
+    if (['necessary', 'functional', 'analytics', 'performance', 'advertisement', 'marketing'].includes(value(attributes, 'data-faz-category'))) return held(['src'], 'data-faz-original-type');
+    return null;
+  }
+  if (isJavaScript(normalized)) {
+    return value(attributes, 'data-type') === 'lazy' ? held(['data-src'])
+      : { sources: ['src'], held: false, module: normalized === 'module' };
+  }
+  return null;
+}
+
+function inlinePayload(body, module = false) {
+  const comments = [];
+  try {
+    // Parsing never executes page code or proves control flow. Strip comments
+    // without damaging URLs/regexes; preserve all ordinary inline string matches.
+    parseJavaScript(body, { ecmaVersion: 'latest', sourceType: module ? 'module' : 'script',
+      onComment: (_, __, start, end) => comments.push([start, end]) });
+    const chunks = [];
+    let offset = 0;
+    for (const [start, end] of comments) {
+      chunks.push(body.slice(offset, start), ' ');
+      offset = end;
+    }
+    chunks.push(body.slice(offset));
+    return { text: chunks.join(''), parseFailed: false };
+  } catch { return { text: body, parseFailed: true }; }
+}
+const escapeAttribute = text => text.replaceAll('&', '&amp;').replaceAll('"', '&quot;').replaceAll('<', '&lt;');
+
 function pageSignals(html) {
   // HTML parsing keeps script-looking text in textarea/style/noscript inert.
   const pending = [parse(html, { scriptingEnabled: true, sourceCodeLocationInfo: true })];
-  const scripts = [], accountElements = [];
+  const scripts = [], accountElements = [], uncertainPayloads = [];
   let hasEthicalPublisher = false;
   const inert = new Set(['template','textarea','style','title','xmp','iframe','noembed','noframes','noscript','plaintext']);
   while (pending.length) {
@@ -54,22 +140,49 @@ function pageSignals(html) {
         const language = attributes.get('language')?.trim().toLowerCase();
         const type = attributes.has('type') ? attributes.get('type').trim()
           : language ? `text/${language}` : '';
-        const executable = type === '' || type === 'module' || javascriptTypes.has(type.toLowerCase().split(';')[0].trim());
-        if (executable && node.sourceCodeLocation?.endTag) {
-          scripts.push(html.slice(node.sourceCodeLocation.startOffset, node.sourceCodeLocation.endOffset));
+        const loader = activation(type, attributes);
+        if (loader && node.sourceCodeLocation?.endTag) {
+          // Attribute-copy order differs between loaders. Never choose a winner
+          // when known activation attributes (including plain src) disagree.
+          const candidates = (loader.held ? activationSources : loader.sources)
+            .filter(name => attributes.has(name)).map(name => value(attributes, name));
+          if (new Set(candidates).size > 1) {
+            uncertainPayloads.push(...candidates);
+            if (candidates.some(source => /^(?:https?:)?\/\/pagead2\.googlesyndication\.com\/pagead\/js\/adsbygoogle\.js(?:[?#]|$)/i.test(source))) {
+              uncertainPayloads.push(value(attributes, 'data-ad-client'));
+            }
+            continue;
+          }
+          // A source-backed script ignores its inline body. An unused attribute
+          // (original-src, check-url, etc.) must never supply a match.
+          const source = loader.sources.map(name => value(attributes, name)).find(Boolean);
+          if (source) {
+            scripts.push(`<script src="${escapeAttribute(source)}"></script>`);
+            // AdSense consumes this one account option on its actual loader.
+            if (/^(?:https?:)?\/\/pagead2\.googlesyndication\.com\/pagead\/js\/adsbygoogle\.js(?:[?#]|$)/i.test(source)) {
+              accountElements.push(`adsbygoogle ${value(attributes, 'data-ad-client')}`);
+            }
+          } else {
+            const body = inlinePayload(html.slice(node.sourceCodeLocation.startTag.endOffset,
+              node.sourceCodeLocation.endTag.startOffset), loader.module);
+            // Held code never confirms: its strings may be settings, examples,
+            // or unreachable calls. Parse failures also cannot prove absence.
+            if (loader.held || body.parseFailed) uncertainPayloads.push(body.text);
+            else if (body.text) scripts.push(`<script>${body.text}</script>`);
+          }
         }
       } else {
         if (attributes.get('data-ea-publisher')?.trim()) hasEthicalPublisher = true;
         const accountMeta = node.tagName === 'meta' && attributes.get('name')?.toLowerCase() === 'google-adsense-account';
         const accountSlot = node.tagName === 'ins' && attributes.has('data-ad-client')
           && attributes.get('class')?.split(/\s+/).includes('adsbygoogle');
         const location = node.sourceCodeLocation?.startTag ?? node.sourceCodeLocation;
-        if ((accountMeta || accountSlot) && location) accountElements.push(html.slice(location.startOffset, location.endOffset));
+        if ((accountMeta || accountSlot) && location) accountElements.push(`google-adsense-account ${value(attributes, accountMeta ? 'content' : 'data-ad-client')}`);
       }
     }
     for (let i = (node.childNodes?.length ?? 0) - 1; i >= 0; i--) pending.push(node.childNodes[i]);
   }
-  return { scripts:scripts.join('\n'), accountMarkup:[...scripts,...accountElements].join('\n'), hasEthicalPublisher };
+  return { scripts:scripts.join('\n'), accountMarkup:[...scripts,...accountElements].join('\n'), uncertainPayloads: uncertainPayloads.join('\n').toLowerCase(), hasEthicalPublisher };
 }
 
 export function scriptMarkup(html) { return pageSignals(html).scripts; }
@@ -132,6 +245,12 @@ export function needsHomepage(parsed, networks, discoverySources = []) {
 export function matchNetworks({ parsed, networks, fetchEvidence, homepage, discoverySources }) {
   const confirmed = new Map();
   const review = new Map();
+  const inconclusive = new Set();
+  const confirm = match => {
+    const previous = confirmed.get(match.networkId);
+    match.signals = [...(previous?.signals ?? (previous ? [previous.evidence] : [])), match.evidence];
+    confirmed.set(match.networkId, match);
+  };
   const managers = new Set(parsed.managerDomains.map((item) => item.value));
   const directSystems = directAdSystems(parsed);
   const sourceSlugs = sourceNetworkSlugs(discoverySources);
@@ -140,7 +259,7 @@ export function matchNetworks({ parsed, networks, fetchEvidence, homepage, disco
   for (const variable of parsed.managerDomains) {
     for (const network of networks) {
       if (!(network.manager_domains ?? []).includes(variable.value)) continue;
-      confirmed.set(network.id, {
+      confirm({
         networkId: network.id,
         slug: network.slug,
         status: "confirmed",
@@ -155,40 +274,43 @@ export function matchNetworks({ parsed, networks, fetchEvidence, homepage, disco
     const signals = pageSignals(homepage.html);
     const normalizedScripts = signals.scripts.toLowerCase();
     for (const network of networks) {
-      if (confirmed.has(network.id)) continue;
+      if (confirmed.get(network.id)?.verificationMethod === "ads_txt_managerdomain") continue;
       for (const signature of network.program_signatures ?? []) {
         if (signature.kind === "adsense_account_match") {
           const sellerIds = directAdsenseSellerIds(parsed, signature);
           const pageIds = pageAdsenseSellerIds(signals.accountMarkup);
-          const matchedId = [...sellerIds].find((sellerId) => pageIds.has(sellerId));
-          if (!matchedId) continue;
-          confirmed.set(network.id, {
-            networkId: network.id,
-            slug: network.slug,
-            status: "confirmed",
-            relationshipType: signature.relationship_type ?? "active_network_use",
-            verificationMethod: "ads_txt_and_adsense_account",
-            evidence: {
-              checked_url: fetchEvidence.finalUrl ?? fetchEvidence.checkedUrl,
-              requested_url: fetchEvidence.checkedUrl,
-              checked_at: fetchEvidence.checkedAt,
-              matched_signal: `${signature.ad_system ?? "google.com"},${matchedId},${signature.seller_relationship ?? "DIRECT"} + ca-${matchedId}`,
-              seller_id: matchedId,
-              page_url: homepage.finalUrl,
-              identity_source: signature.source_url ?? null,
-              ads_txt_sha256: fetchEvidence.sha256,
-              parser_version: parsed.parserVersion,
-              matcher_version: MATCHER_VERSION,
-            },
-          });
+          if ([...sellerIds].some(id => signals.uncertainPayloads.includes(`ca-${id}`))) inconclusive.add(network.id);
+          for (const matchedId of sellerIds) {
+            if (!pageIds.has(matchedId)) continue;
+            confirm({
+              networkId: network.id,
+              slug: network.slug,
+              status: "confirmed",
+              relationshipType: signature.relationship_type ?? "active_network_use",
+              verificationMethod: "ads_txt_and_adsense_account",
+              evidence: {
+                checked_url: fetchEvidence.finalUrl ?? fetchEvidence.checkedUrl,
+                requested_url: fetchEvidence.checkedUrl,
+                checked_at: fetchEvidence.checkedAt,
+                matched_signal: `${signature.ad_system ?? "google.com"},${matchedId},${signature.seller_relationship ?? "DIRECT"} + ca-${matchedId}`,
+                seller_id: matchedId,
+                page_url: homepage.finalUrl,
+                identity_source: signature.source_url ?? null,
+                ads_txt_sha256: fetchEvidence.sha256,
+                parser_version: parsed.parserVersion,
+                matcher_version: MATCHER_VERSION,
+              },
+            });
+          }
           continue;
         }
         if (signature.kind !== "html_substring" || typeof signature.value !== "string") continue;
         if (!signatureRequirementsMatch(signature, network, { managers, directSystems, sellerSystems, sourceSlugs })) continue;
+        if (signals.uncertainPayloads.includes(signature.value.toLowerCase())) inconclusive.add(network.id);
         const present = signature.allow_without_ads_txt && signature.value === 'data-ea-publisher='
           ? signals.hasEthicalPublisher : normalizedScripts.includes(signature.value.toLowerCase());
         if (!present) continue;
-        confirmed.set(network.id, {
+        confirm({
           networkId: network.id,
           slug: network.slug,
           status: "confirmed",
@@ -237,5 +359,6 @@ export function matchNetworks({ parsed, networks, fetchEvidence, homepage, disco
     });
   }
 
-  return { confirmed: [...confirmed.values()], review: [...review.values()] };
+  return { confirmed: [...confirmed.values()], review: [...review.values()],
+    inconclusiveNetworkIds: [...inconclusive].filter(id => !confirmed.has(id)) };
 }

scripts/ad-networks/review-rules.mjs

@@ -1,4 +1,4 @@
-import { matchNetworks,scriptMarkup } from './matcher.mjs';
+import { matchNetworks } from './matcher.mjs';
 
 export const REVIEW_METHOD='discovery_hint_and_ads_txt_system';
 export function reviewLinks(db,id) {
@@ -22,7 +22,7 @@ export function gateReviewMatches({matches,relationships=[],networks,parsed,fetc
     else if(!review.some(row=>row.networkId===match.networkId))review.push({...match,status:'needs_review',relationshipType:'unknown',
       verificationMethod:REVIEW_METHOD,evidence:{...relationship.evidence,last_review:{checked_at:checkedAt,reason:result.reason}}});
   }
-  return {confirmed,review};
+  return {...matches,confirmed,review};
 }
 export function reviewDecision({relationship,network,parsed,fetchResult,homepage,checkedAt,discoverySources=[]}) {
   const keep=reason=>({decision:'kept',reason});
@@ -43,16 +43,15 @@ export function reviewDecision({relationship,network,parsed,fetchResult,homepage
   }
   if(managers.length)return {...keep('different_manager'),present:true};
   if(!homepage?.html)return {...keep('homepage_unavailable'),present:true};
-  // Only configured, network-specific tags/configuration in executable script markup qualify.
+  // Only configured tags/configuration in JavaScript or recognized loader scripts qualify.
   // Plain article text, commented-out tags, seller rows and shared wrappers cannot match.
-  const scripts=scriptMarkup(homepage.html);
   const signatures=(network.program_signatures??[]).filter(signature=>signature.kind==='html_substring'
     &&typeof signature.value==='string'&&signature.value.trim().length>=8
     &&signature.distinctive!==false&&signature.shared_wrapper!==true
     &&!/(?:googletagmanager\.com|googlesyndication\.com|doubleclick\.net|prebid\.js|gpt\.js)/i.test(signature.value));
   if(!signatures.length)return {...keep('no_distinctive_configured_signature'),present:true};
   const matches=matchNetworks({parsed,networks:[{...network,program_signatures:signatures}],
-    fetchEvidence:{...fetchResult,checkedAt},homepage:{...homepage,html:scripts},discoverySources});
+    fetchEvidence:{...fetchResult,checkedAt},homepage,discoverySources});
   const match=matches.confirmed.find(row=>row.networkId===network.id&&row.verificationMethod!=='ads_txt_managerdomain');
   if(!match)return {...keep('no_current_distinctive_tag'),present:true};
   return {decision:'confirmed',reason:'exact_line_and_current_distinctive_tag',present:true,evidence:{...match.evidence,

scripts/ad-networks/verifier-store.mjs

@@ -170,7 +170,8 @@ async function ageMissingRelationships(client, existing, positiveIds, checkedAt,
         first_missed_at = coalesce(first_missed_at, $2), last_missed_at = $2,
         evidence = evidence || jsonb_build_object('last_absence', $7::jsonb)
       where ad_network_id = $1 and publisher_id = $6
-    `, [relationship.ad_network_id, checkedAt, status, endedAt, misses, relationship.publisher_id, JSON.stringify({ checked_at: checkedAt, homepage: evidence?.homepage ?? null, ads_txt_sha256: evidence?.ads_txt_sha256 ?? null })]);
+    `, [relationship.ad_network_id, checkedAt, status, endedAt, misses, relationship.publisher_id, JSON.stringify({ checked_at: checkedAt, homepage: evidence?.homepage ?? null, ads_txt_sha256: evidence?.ads_txt_sha256 ?? null,
+      matcher_version: evidence?.matcher_version ?? null })]);
   }
 }