diff --git a/content_docs/ad-networks/maintenance.md b/content_docs/ad-networks/maintenance.md index 5c1ce97c..aeb7909a 100644 --- a/content_docs/ad-networks/maintenance.md +++ b/content_docs/ad-networks/maintenance.md @@ -183,6 +183,74 @@ OVH cron claims bounded verification batches with four concurrent checks. A stop Every confirmed relationship stores the checked URL, time, matched signal, ads.txt checksum, parser version, and matcher version. Public pages continue to query only `confirmed` relationships. +Matcher `ad-network-1.5.2` reads real script elements, including ad code held until interaction or consent. The same payload rules apply to AdSense account IDs and `html_substring` signatures. A held tag proves an installed integration; it does not prove that an ad ran before consent. + +Only a loader's activation source can confirm a held/deferred script. If a tag has different values in two known activation source attributes, or in an activation attribute and plain `src`, signals from every candidate are inconclusive for their networks. Attribute order never chooses a winner. An empty source conflicts with a nonempty source too. Several identical values may confirm only when this loader activates that source; unsupported fallback attributes cannot confirm alone. A separate qualifying tag can still confirm the network. Plain `src` is accepted only where the loader's own activation code supports it, as audited below. Unused attributes cannot supply an account or signature. An external script's inline body is ignored. AdSense's `data-ad-client` is read only from an actual `adsbygoogle.js` loader, an AdSense `ins` element, or the account meta element's `content`. + +A recognized held/deferred **inline body never confirms** a network. If it contains that network's signal or the exact authorized AdSense ID, the result is inconclusive for that network. An unused URL, a URL followed by unrelated code, and a loader inside `if(false)` receive the same treatment. The matcher does not try to prove execution. The verifier retries without a confirmation or miss; the existing 30-day rule can make the relationship stale, never ended from these checks. A separate qualifying external source or account element can still confirm it. + +Acorn parses inline code in its original classic/module mode and strips comments with one chunk pass and one join. Any parse failure containing a network signal makes that network inconclusive. Ordinary executable inline strings retain their existing matching behavior, including unused variables and unreachable calls; this rule change concerns held/deferred code. Comment-only copies cannot confirm. Parse failures conservatively retain raw signals because comments cannot be removed reliably from malformed code. + +A saved original type must be empty, an ordinary executable JavaScript type, or `module`. JSON/JSON-LD, import maps, unknown types, and other held types are ignored entirely: they neither confirm nor make the network inconclusive. The only audited chain is Perfmatters restoring `litespeed/javascript`, followed by LiteSpeed promoting `data-src`. Other saved loader types are not generalized into chains. + +Real scripts inside `pre`, `code`, and `blockquote` use the normal matching rules. Those elements do not stop script execution. Documentation examples must escape the markup (`<script>`). Comments, `template`, `textarea`, scripting-enabled `noscript`, `xmp`, and iframe text/srcdoc remain inert. + +The following counts come from the 274 captured November 13 homepages. Each number counts sites containing the original signal in that format, not all installations of that product. Counts overlap. Zero means a documented fixture only. Do not generalize from a similar attribute or a type containing “JavaScript”, “consent”, or “lazy”. + +| Loader / CMP | Verified held format | Activation source | Measured sites | Reference | +| --- | --- | --- | ---: | --- | +| LiteSpeed | `litespeed/javascript` | `data-src` | 60 | [Official rewrite](https://github.com/litespeedtech/lscache_wp/blob/master/src/optimize.cls.php) | +| WP Rocket | `rocketlazyloadscript`, `text/rocketlazyloadscript`; saved type must be executable | `data-rocket-src` | 13 + 7 | [Official rewrite](https://github.com/wp-media/wp-rocket/blob/develop/inc/Engine/Optimization/DelayJS/HTML.php) | +| Cloudflare Rocket Loader | 24 hexadecimal characters followed by `-text/javascript`, `-application/javascript`, or `-module` | `src` | 28 | [Rocket Loader](https://developers.cloudflare.com/speed/optimization/content/rocket-loader/); prefix measured in captured tags | +| Perfmatters | `pmdelayedscript` | `src`; `data-src` only when saved type is `litespeed/javascript` | 3 | [Official delay guide](https://perfmatters.io/docs/delay-javascript/); exact source measured in captured tags | +| Penci | `PenciLazyScript`; saved type must be executable | `src` | 2 | [Measured publisher markup](https://10travelingfeet.com/) | +| WP Meteor | `javascript/blocked` and a saved JavaScript `data-wpmeteor-type` | `data-wpmeteor-src` | 2 | [Official distribution](https://plugins.svn.wordpress.org/wp-meteor/trunk/assets/js/public/); [rewrite](https://plugins.svn.wordpress.org/wp-meteor/trunk/frontend/Rewrite.php) | +| Seraphinite | `o/js-lzl`; saved type must be executable | `data-src`, otherwise `src` | 1 | [Measured publisher markup](https://persiapage.com/) | +| Starcache | `starcache/delay`; saved type must be executable | `data-starcache-src` | 1 | [Publisher's own inline loader](https://relife.sk/) copies this attribute to `src` | +| MD delayed loader | `text/plain` with `data-md-delayed-src` | `data-md-delayed-src` | 1 | [Publisher's own inline loader](https://mojadelicja.pl/) creates a script from this attribute | +| Flying Scripts | JavaScript with `data-type="lazy"` | `data-src` | 0 | [Official rewrite](https://plugins.svn.wordpress.org/flying-scripts/trunk/html-rewrite.php); [loader](https://plugins.svn.wordpress.org/flying-scripts/trunk/source.js) | +| Real Cookie Banner | `application/consent`, nonempty `consent-required`, saved JavaScript `consent-original-type-_` | `consent-original-src-_` | 2 | [Vendor's blocker documentation](https://docs.devowl.io/real-cookie-banner/php/classes/DevOwl-HeadlessContentBlocker-plugins-AdditionalAttributesBlocker.html); exact markup measured on [neuesland.at](https://neuesland.at/) | +| Didomi | `didomi/javascript` with `data-vendor`, `data-vendor-raw`, or `data-purposes` | `src` | 1 | [Vendor's tag documentation](https://developers.didomi.io/cmp/web-sdk/third-parties/no-tag-manager.md) | +| Borlabs | `text/template` with a blocker ID | `data-borlabs-script-blocker-src` | 1 | [Vendor's blocker description](https://borlabs.io/borlabs-cookie/features/library/); exact paired attributes measured on [kornkompetenz.de](https://kornkompetenz.de/) | +| OneTrust | `text/plain`, class `optanon-category-C0002` or multiple numeric categories such as `optanon-category-C0002-C0004` | `src` | 1 | [Vendor's held-script example](https://developer.onetrust.com/onetrust/docs/custom-common-html) | +| Complianz | `text/plain`, `data-category` exactly `preferences`, `statistics`, or `marketing` | `data-cmplz-src` | 1 | [Vendor's activation code](https://raw.githubusercontent.com/Complianz/complianz-gdpr/master/cookiebanner/js/complianz.js); `functional` is skipped by that code | +| Orest Bida CookieConsent v2 | `text/plain`, `data-cookiecategory` exactly `necessary`, `analytics`, `advertising`, or `targeting` | `data-src` | 1 | [Vendor's v2 activation code](https://raw.githubusercontent.com/orestbida/cookieconsent/v2.9.2/dist/cookieconsent.js) | +| FAZ Cookie Manager | `text/plain`, `data-faz-category` exactly `necessary`, `functional`, `analytics`, `performance`, `advertisement`, or `marketing` | `src` | 1 | [Vendor's manual blocking instructions](https://github.com/fabiodalez-dev/FAZ-Cookie-Manager/blob/main/readme.txt); source restoration also present in captured page runtime | +| Cookiebot | `text/plain`, `data-cookieconsent` containing only `preferences`, `statistics`, `marketing` (comma separated allowed) | `src` | 0 | [Vendor's manual blocking instructions](https://support.cookiebot.com/hc/en-us/articles/4405978132242-Manual-cookie-blocking) | +| CookieScript | `text/plain`, `data-cookiescript="accepted"` | `src` | 0 | [Vendor's script examples](https://help.cookie-script.com/en/articles/37185-blocking-third-party-cookies-set-with-javascript) | + +### Activation source audit + +Checked November 13, 2026. Line numbers below refer to the linked source file as fetched that day, before formatting. A minified file can have its whole activation function on one line. Publisher inline references identify that page's captured HTML line. These are installation signals, not proof of execution before consent. The source column lists audited activation attributes, not precedence. Any conflicting known source values make the affected signals inconclusive, even when a loader has no plain-`src` fallback. Plain `src` alone can confirm only where explicitly supported. Every saved type in the table must pass the executable-type rule above; missing/empty saved types use classic JavaScript unless the format requires an explicit saved type. + +| Loader / CMP | Audited activation sources | Plain `src` | Saved original type | Activation code checked | +| --- | --- | --- | --- | --- | +| LiteSpeed | `data-src`, `src`; differing values are inconclusive | Supported by attribute copy | None; forces classic JS | [`assets/js/js_delay.js`, lines 65–75](https://github.com/litespeedtech/lscache_wp/blob/master/assets/js/js_delay.js#L65): copies attributes in document order and renames `data-src` to `src`. The last assignment wins at runtime; the matcher never resolves conflicting values. | +| WP Rocket (both MIME forms) | `data-rocket-src`, `src` | Supported by CSP fallback's attribute copy | `data-rocket-type`: executable JS/`module` only; absent/empty = classic | [`assets/js/lazyload-scripts1.2.6.min.js`, line 1, `_transformScript`](https://raw.githubusercontent.com/wp-media/wp-rocket/develop/assets/js/lazyload-scripts1.2.6.min.js); also [`lazyload-scripts.min.js`, line 1, `G`](https://raw.githubusercontent.com/wp-media/wp-rocket/develop/assets/js/lazyload-scripts.min.js): fallback copies attributes, renames source/type. [Rewrite lines 250–256](https://github.com/wp-media/wp-rocket/blob/develop/inc/Engine/Optimization/DelayJS/HTML.php#L250). | +| Cloudflare Rocket Loader | `src` | Primary source | Original type follows 24-hex prefix | [`rocket-loader.min.js`, line 1, `i`, `p`, `removeNonce`](https://ajax.cloudflare.com/cdn-cgi/scripts/7d0fa10a/cloudflare-static/rocket-loader.min.js): reads `src`, copies attributes, strips type prefix. | +| Perfmatters | `src`; with saved LiteSpeed type: `data-src`, `src` | Primary source | `data-perfmatters-type`: executable JS/`module`, or the audited LiteSpeed chain; absent/empty = classic | [Captured publisher HTML line 150, `l` transformation](https://1stelectricians.co.uk/): copies attributes except held type and restores `data-perfmatters-type`; only `src` loads directly. The measured `data-src` form restores LiteSpeed type, whose audited loader then promotes it. | +| Penci | `src` | Primary source | `data-pencilazy-type`: executable JS/`module` only; absent/empty = classic | [Captured publisher HTML line 487, `SoledadLazyLoad._transformScript`](https://10travelingfeet.com/): copies `src`, renames saved type. | +| WP Meteor | `data-wpmeteor-src` | No fallback accepted | `data-wpmeteor-type` | [`assets/js/public/public-debug.js`, lines 740–758, `unblock`](https://plugins.svn.wordpress.org/wp-meteor/trunk/assets/js/public/public-debug.js): explicitly restores saved source/type; does not read plain `src` as a fallback. | +| Seraphinite | `data-src`, `src` | Supported by attribute copy | `data-type`: executable JS/`module` only; absent/empty = classic | [Captured publisher HTML line 287, inline `F` / `n` transformation](https://persiapage.com/): renames `data-src`/`data-type`, copies other attributes. | +| Starcache | `data-starcache-src`, `src` | Supported by attribute copy | `data-starcache-type`: executable JS/`module` only; absent/empty = classic | [Captured publisher HTML line 860, `load`](https://relife.sk/): copies `src`, then applies saved source/type. | +| MD delayed loader | `data-md-delayed-src` | No fallback | None; new classic script | [Captured publisher HTML lines 723–732, `load`](https://mojadelicja.pl/): unconditionally assigns the saved source to the new script. | +| Flying Scripts | `data-src` | No fallback; overwritten from `data-src` | None; original executable type retained | [`source.js`, lines 21–23, `loadScripts`](https://plugins.svn.wordpress.org/flying-scripts/trunk/source.js); [`html-rewrite.php`, lines 52–58](https://plugins.svn.wordpress.org/flying-scripts/trunk/html-rewrite.php). Only `data-type="lazy"` is selected. Unverified legacy `type="lazy"` is no longer accepted. | +| Real Cookie Banner | `consent-original-src-_`, `src` | Supported by cloned attributes | `consent-original-type-_` | [Publisher runtime `723519805.js`, line 1, `z`](https://neuesland.at/wp-content/uploads/ae6ce1ff82/dist/723519805.js): clones the script, restores `consent-original-*-_` attributes, then reinserts markup. [Prefix constants, `2090049149.js`, line 1](https://neuesland.at/wp-content/uploads/ae6ce1ff82/dist/2090049149.js). | +| Didomi | `src` | Primary source | None; forces classic JS | [`sdk.06e49314862e41f31e84eec3e7b6ece61247d21d.js`, line 2, `replaceTags`](https://sdk.privacy-center.org/sdk/06e49314862e41f31e84eec3e7b6ece61247d21d/modern/sdk.06e49314862e41f31e84eec3e7b6ece61247d21d.js): copies every attribute except `type` to the new script. | +| Borlabs | `data-borlabs-script-blocker-src`, `src` | Supported by attribute copy | None; forces classic JS | [Publisher's `borlabs-cookie.min.js`, line 1, `oo`](https://kornkompetenz.de/wp-content/plugins/borlabs-cookie/javascript/borlabs-cookie.min.js): copies non-Borlabs/non-type attributes, then applies blocker source when present. | +| OneTrust | `src` | Primary source | None; forces classic JS | [`202405.1.0/otBannerSdk.js`, line 7, `reactivateScriptTag`](https://cdn.cookielaw.org/scripttemplates/202405.1.0/otBannerSdk.js): copies attributes and replaces held type with classic JS. | +| Complianz | `data-cmplz-src` | **Unsupported; removed** | None; creates classic JS | [`cookiebanner/js/complianz.js`, lines 889–917](https://github.com/Complianz/complianz-gdpr/blob/master/cookiebanner/js/complianz.js#L889): uses `data-cmplz-src` or inline text. Plain `src` is never read in that branch. | +| Orest Bida CookieConsent v2 | `data-src`, `src` | Supported by attribute copy | `data-type`: executable JS/`module` only; absent/empty = classic | [`v2.9.2/dist/cookieconsent.js`, line 7, `bn` / `o` activation loop](https://raw.githubusercontent.com/orestbida/cookieconsent/v2.9.2/dist/cookieconsent.js): restores saved type, copies attributes, uses `data-src` when present, otherwise original `src`. | +| FAZ Cookie Manager | `src` | Primary source | `data-faz-original-type`: executable JS/`module` only; absent/empty = classic | [`frontend/js/script.js`, lines 7306–7353, `_fazBuildRestoredScript`](https://github.com/fabiodalez-dev/FAZ-Cookie-Manager/blob/main/frontend/js/script.js#L7306): reads script `src` and original type. `data-faz-src` is a resource/iframe parking attribute, not this script's source. | +| Cookiebot | `src` | Primary source | None; forces classic JS | [`uc.js`, line 2, `runScripts`, `runScriptTags`, `cloneScriptTag`](https://consent.cookiebot.com/uc.js): checks held scripts' `src`, clones external scripts, and assigns classic type. `data-cookieblock-src` support for iframes/images is not generalized to scripts. | +| CookieScript | `src` | Primary source | None; forces classic JS | [Vendor-served script, line 14, `et` accepted-script loop](https://cdn.cookie-script.com/s/5c21561bbdd60ecf2d50f10b54b299cb.js): copies script attributes and sets classic type. Its non-script resource branch restores `data-src`; its script branch does not. | + +The recognized formats can hold inline JavaScript, which remains inconclusive when it contains a network signal. Ordinary deferred JavaScript, including Autoptimize, uses the regular executable path. Bare `text/plain`, arbitrary `data-cmp-*`, `data-service` alone, CookieYes markers without a verified format, unknown categories, JSON/JSON-LD script types, import maps, resource hints, and inert examples cannot confirm. Unmeasured Usercentrics, Klaro, iubenda, and other CMP formats are not added. + +Fresh results retain all qualifying signals per network. Normal verification still chooses one relationship. Correction can find the original account/signature even when a later configured signature also matches. Review-promoted rows still need both their original seller line and page signature. + +The [homepage correction procedure](production-runbook.md#correct-inconclusive-homepage-misses-november-2026) has two paths. A fresh exact match can reconfirm current use after any fault-window absence, regardless of matcher version. A historical departure is erased only with saved proof that the miss had no usable homepage (a 304 or recorded inconclusive result). The stores saved no full-fetch homepage markup at these old misses. Version numbers, unchanged ads.txt, and today's delayed tag cannot prove the old page contained it. Those full-fetch histories remain intact, and a fresh return is recorded as `joined`. + ### Fetch policy The fetcher uses the Public Suffix List through `tldts`, resolves every request and redirect target, rejects local and reserved addresses, permits only HTTP and HTTPS on their normal ports, and caps redirects and response size. Publisher verification reads the public `/ads.txt` file and, only when a network rule needs it, the public homepage without consulting `robots.txt`. It sends the stack127 crawler identity, prefers HTTPS, and uses a limited HTTP fallback only when HTTPS cannot be reached. A second redirect outside the publisher's registrable domain is rejected. Discovery source collectors keep their separate robots policy. diff --git a/content_docs/ad-networks/production-runbook.md b/content_docs/ad-networks/production-runbook.md index f1481b5f..0ae78629 100644 --- a/content_docs/ad-networks/production-runbook.md +++ b/content_docs/ad-networks/production-runbook.md @@ -151,29 +151,64 @@ Read-only probes on November 13 used the same crawler identity and 15-second tim ### Selection and limits -`scripts/ad-networks/correct-homepage-misses.mjs` defaults to a dry run. It repairs only a previously confirmed homepage-method relationship with misses inside the fault window. The latest check must be the miss itself, and either a 304 or a recorded inconclusive homepage must prove that check did not obtain absence evidence. A recorded valid `last_absence` excludes the row. +`scripts/ad-networks/correct-homepage-misses.mjs` defaults to a dry run. It rechecks only a previously confirmed homepage-method relationship with misses inside the fault window. The latest check must be the miss itself. A 304 or recorded inconclusive homepage is no longer required: full-fetch misses also qualify because the old matcher ignored real deferred or consent-held scripts. This includes the 4,499 rows previously skipped as `no_proof_of_inconclusive_miss` in the November 13 source audit. Eligibility does not establish that a miss was wrong. -A fresh full check must confirm the exact original signal. For a review-promoted seller relationship, both the exact ads.txt seller line and the saved `page_signature` must match current evidence. If the stored original ads.txt hash exists, the current hash must match too. Changed signals, changed files, blocked checks, actual absences, incomplete evidence, and old full-fetch misses without homepage diagnostics remain untouched. Counts report each reason. This intentionally leaves unresolved damage for later evidence-based review. +A fresh full check must confirm the exact original signal. For a review-promoted seller relationship, both the exact ads.txt seller line and the saved `page_signature` must match current evidence. If the stored original ads.txt hash exists, the current hash must match too. The fresh result must come from the current matcher (`ad-network-1.5.2` in this release). Changed signals, changed files, blocked checks, actual absences, incomplete evidence, and checks after the saved miss remain untouched. Counts report each reason. -The script preserves both original confirmation dates. It clears misses and the end date, restores `confirmed`, and queues priority 100 re-verification. It stores the original miss window and end date under `homepage_miss_correction`. Verification and compact sync preserve that marker. +There are two separate correction paths: + +- **Current reconfirmation:** a fresh exact original signal restores current `confirmed` status, clears misses/end date, and records the fresh confirmation time. The first confirmation date stays. Full-fetch misses and recorded absences can take this path regardless of matcher version. Their historical `left` event and old snapshots stay. An ended relationship gets a `joined` return at the fresh confirmation time during PostgreSQL finalization. +- **History correction:** a 304 or a saved inconclusive homepage from the miss itself permits removal of its exact departure, conversion of its apparent return to a baseline, and invalidation of affected historical snapshots. This path preserves the original confirmation dates. A stored old matcher version is not proof. Full-fetch history correction would require markup saved at the miss showing the original signal in an ignored held script. These old stores did not save that markup, so no old full-fetch row qualifies on that basis. + +The latest-check/miss equality guard applies to both. A recorded `last_absence` is retained for audit; it cannot prove that an old departure was wrong. A fresh confirmation cannot prove historical continuity. Legacy correction markers without an explicit action must still pass the stored-miss proof check before any history is erased. + +The matcher confirms held/deferred scripts only from activation source attributes in the explicit formats in [the maintenance guide](maintenance.md#evidence-rules). Conflicting source attributes, held inline signals, and inline parse failures are inconclusive for the affected network: they cannot confirm it or add a miss. They are retried and can become stale after 30 days. Normal executable inline strings keep their existing matching behavior. Saved non-JavaScript types are ignored, apart from the audited Perfmatters-to-LiteSpeed chain. Inline comments, JSON script types, unused attributes, and bare `text/plain` cannot confirm a network. Fresh matches retain every qualifying account/signature, so a newly recognized signature cannot hide the unchanged original. + +Both paths queue priority 100 re-verification and save the original miss/end times under `homepage_miss_correction`. The marker also records `history_action` (`repair` or `preserve`), `history_proof`, original confirmation dates, and the exact fresh confirmation evidence. Verification and compact sync preserve it. PostgreSQL finalization uses its durable checkpoint for both paths. It inserts a current-only return once, and never deletes history or rewrites old observations for that path. Re-running source correction, sync, or finalization is safe. + +The November 13 read-only estimate found **5,774 history candidates** (988 ended, 4,786 stale) and **4,499 current-only candidates** (926 ended, 2,852 stale, 721 confirmed with misses). These are candidate limits, not promises of fresh confirmation. All 10,273 source check records lack stored homepage markup/outcomes and `last_absence` records. The 5,774 history candidates qualify through their saved 304 result. The full-fetch cohort has **zero stored proofs** of an ignored historical signal. + +If every eligible history candidate freshly reconfirms, the remaining deletion limit is **147 daily snapshot rows across 8 networks**, down from 196 across 13 when current-only rows were included. Current-only rows add **zero** historical deletions. By network: Amazon Publisher Services 19, AdSense 1, Infolinks 22, MGID 21, Revcontent 20, SHE Media 23, Taboola 21, Teads 20. The PostgreSQL estimate permits up to 988 exact departure removals; there are currently no joined arrivals to neutralize in those windows. The one earlier completed repair is excluded. Actual deletion can be lower after fresh checks and must be previewed again before apply. + +The 274 captured requests cover 271 distinct domains: three domains have two sampled network relationships. Pair captures by their saved order, not the first row with the same domain. Replaying the 250 larger-sample relationships through `ad-network-1.5.2` gives **80/125 history-path confirmations** and **45/125 current-only confirmations**, compared with 93/125 and 49/125 before this review. All 17 lost confirmations are now inconclusive, not absence evidence. Four previously hash-excluded checks also become inconclusive. The separate 24-row batch still has 20 confirmations, one blocked check, and three absences. The four follow-up fixes in 1.5.2 leave these sampled counts unchanged from 1.5.1. Against base `6a90331d`, the same captures yield 7, 33, and 17 confirmations for batch1, the larger history cohort, and the current-only cohort respectively. Isolating ordinary executable inline bodies on the 218 obtained pages with parsed ads.txt gives identical confirmed network IDs under base and 1.5.2. This body-only check excludes unused attributes and held scripts; those follow the stricter rules above. + +Using the population's actual method mix gives approximately **4,067 history-path restorations** among the remaining 5,750 candidates, **1,774 current-only reconfirmations**, plus the separate batch's 20 confirmations. That is about **5,861 total**, roughly 436 fewer than the paired prior-matcher replay. These are planning estimates, not production results: sample uncertainty is hundreds of rows, captures age, live checks may differ, and the 721 confirmed-with-misses current-only rows were not sampled. Neither fresh markup nor these ratios prove historical continuity. `--limit` bounds distinct domains checked, default 100. `nextAfterDomain` supplies the next batch's `--after-domain` value. Use the same cursor for a preview and its apply. Advance only after that batch has applied. Failed or excluded rows remain unchanged; start again without a cursor to retry them later. History work has its own `--history-limit` domain bound (defaults to `--limit`). Both restoration and history selection honor `--after-domain`. Stats-only output uses `nextAfterDomain` for history progress. PostgreSQL records completed `(network, publisher, correction time, version)` repairs in `ad_network_homepage_miss_repairs`, in the same transaction as the history changes. This target-side checkpoint survives a later source sync. Interrupted source repair, sync and history stages can each be repeated without advancing their cursors. Completed history is skipped even when starting again without a cursor. Each store applies its batch in one write transaction. Network fetches happen before that transaction. Changed rows abort and roll back the batch. ### Release and apply order +**Keep the hourly verification job paused until every source correction, sync, PostgreSQL finalization, and public check has finished.** Installing the matcher is not a reason to resume it. Verification can clear the miss fields and replace the evidence needed to select remaining rows. A read-only `ssh ovh 'crontab -l'` check on November 13, 2026 confirmed the current state: the lead resumed the eight other jobs (snapshot, platform, domain-age, ads-txt-summary, discovery, publisher-names, traffic, and backup) that day. Only verification remains commented, with `# paused-until-matcher-fix`. Preserve that pause through the whole sequence, including between batches. Do not change the runner configuration during implementation. + The lead runs this only after review and release authorization. No production step below was run during implementation. Start with **25 domains**, verify the public result, then use **100-domain** batches. A dry run fetches fresh evidence; apply fetches again and may restore fewer rows if a page changes or challenges it. #### 1. Open a maintenance window Pausing collectors alone is insufficient. OVH's `stack127-cache-worker.timer` is enabled and dispatches `.github/workflows/stack127-frontend-refresh.yml` from queue changes. A sync commit can otherwise start a build before history repair finishes. The static builder correctly rejects a changing source revision. Leave database triggers enabled and keep their queue entries. -From the operator's local checkout, pause frontend dispatch and save the existing OVH crontab. These commands preserve unrelated cron entries. Use the same saved filename when resuming. +From the operator's local checkout, pause frontend dispatch and save the existing OVH crontab. These commands preserve unrelated cron entries. Keep `ADN_WINDOW` in the local shell: its unique prefix identifies only lines paused by this window. The saved file is a backup, never the source for resume. ```bash # Local shell. The implementation agent must not run these production mutations. export ADN_WINDOW=$(date -u +%Y%m%dT%H%M%SZ) -ssh ovh "crontab -l > /srv/stack127-data/state/crontab-before-homepage-$ADN_WINDOW.txt" -ssh ovh "sed '/^[^#].*scripts\/ad-networks\/ovh\/run-job.sh/s/^/# homepage-maintenance /' /srv/stack127-data/state/crontab-before-homepage-$ADN_WINDOW.txt | crontab -" +ssh ovh python3 - "$ADN_WINDOW" <<'PYTHON' +import pathlib +import subprocess +import sys + +window = sys.argv[1] +assert window and all(c in '0123456789TZ' for c in window), 'Invalid maintenance window' +current = subprocess.check_output(['crontab', '-l']) +with pathlib.Path('/srv/stack127-data/state/crontab-before-homepage-' + window + '.txt').open('xb') as saved: + saved.write(current) +prefix = ('# homepage-maintenance-' + window + ' ').encode() +paused = b''.join(prefix + line if line.strip() and not line.lstrip().startswith(b'#') + and b'/srv/stack127/scripts/ad-networks/ovh/run-job.sh ' in line else line + for line in current.splitlines(keepends=True)) +assert subprocess.check_output(['crontab', '-l']) == current, 'Crontab changed; inspect before retrying' +subprocess.run(['crontab', '-'], input=paused, check=True) +assert subprocess.check_output(['crontab', '-l']) == paused, 'Installed crontab differs' +PYTHON ssh ovh 'sudo -n systemctl stop stack127-cache-worker.timer' # Let a currently executing worker finish; do not kill it mid-dispatch. ssh ovh 'while systemctl is-active --quiet stack127-cache-worker.service; do sleep 2; done' @@ -292,19 +327,75 @@ node scripts/ad-networks/correct-homepage-misses.mjs --environment production -- Repeat sync and history repair after each source batch. Keep dispatch paused during mutations. Repeat step 5 once all repairs are finished and wait for that refresh before resuming writers. To inspect any intermediate 100-domain batch publicly, finish its entire sync/history sequence, perform one refresh, and wait for completion before the next mutation. -Once public acceptance passes, restore the saved crontab from the local operator shell. If the niche timer was active before this window, the operator resumes it with `ssh homelab 'systemctl --user start stack127-ad-network-niche.timer'`. Preserve any unrelated cron edits made during the window; compare the saved and current crontabs before restoring. +Only once all correction batches, syncs, history/return finalizations, and public acceptance pass may the lead resume writers. Read the **current** crontab; never reinstall the saved file. That would replace unrelated edits. + +If step 1 opened a maintenance window, first remove its exact `# homepage-maintenance- ` prefix from **every non-verification line carrying it**. Keep each line's remaining bytes, including any edits made during the window. Leave verification paused for the last step. A different window's prefix and unrelated comments stay untouched. ```bash -ssh ovh "crontab /srv/stack127-data/state/crontab-before-homepage-$ADN_WINDOW.txt" +# Local shell, using the ADN_WINDOW from step 1. +: "${ADN_WINDOW:?Use the same maintenance window ID as step 1}" +ssh ovh python3 - "$ADN_WINDOW" <<'PYTHON' +import subprocess +import sys + +window = sys.argv[1] +assert window and all(c in '0123456789TZ' for c in window), 'Invalid maintenance window' +prefix = ('# homepage-maintenance-' + window + ' ').encode() +verification = b'/srv/stack127/scripts/ad-networks/ovh/run-job.sh verification ' +current = subprocess.check_output(['crontab', '-l']) +resumed = b''.join(line[len(prefix):] if line.startswith(prefix) and verification not in line + else line for line in current.splitlines(keepends=True)) +assert subprocess.check_output(['crontab', '-l']) == current, 'Crontab changed; rerun against current contents' +subprocess.run(['crontab', '-'], input=resumed, check=True) +assert subprocess.check_output(['crontab', '-l']) == resumed, 'Installed crontab differs' +print("This window's non-verification jobs resumed; verification stays paused") +PYTHON ssh ovh 'sudo -n systemctl start stack127-cache-worker.timer' ssh ovh 'systemctl is-active stack127-cache-worker.timer' ``` +If the niche timer was active before this window, resume it with `ssh homelab 'systemctl --user start stack127-ad-network-niche.timer'`. Then enable verification **last**, only after acceptance, with the next command. It removes either this window's exact prefix or the pre-existing `# paused-until-matcher-fix ` from exactly the reviewed verification line. It stops on a missing, duplicated, active, or edited verification command; inspect such a change instead of replacing it. + +**From the current November 13 OVH state:** the eight other jobs are already active. If no new step-1 window has been opened, skip the maintenance-prefix resume and timer commands above. After all correction work and acceptance, run `unset ADN_WINDOW` and then run only this verification block and its crontab check. No saved crontab is required. If a new window was opened, keep its `ADN_WINDOW` and run both resume stages in order. + +```bash +ADN_RESUMED_AT=$(ssh ovh 'date -u +%FT%TZ') +ssh ovh python3 - "${ADN_WINDOW:-}" <<'PYTHON' +import subprocess +import sys + +window = sys.argv[1] if len(sys.argv) > 1 else "" # ssh drops an empty argument +assert not window or all(c in '0123456789TZ' for c in window), 'Invalid maintenance window' +current = subprocess.check_output(['crontab', '-l']) +active = b'40 * * * * /srv/stack127/scripts/ad-networks/ovh/run-job.sh verification --limit 1000 --time-budget-minutes 10' +prefixes = [b'# paused-until-matcher-fix '] +if window: + prefixes.append(('# homepage-maintenance-' + window + ' ').encode()) +paused = {prefix + active for prefix in prefixes} +lines = current.splitlines(keepends=True) +assert sum(line.rstrip(b'\r\n') in paused for line in lines) == 1, 'Expected exactly one reviewed paused verification line' +commands = [line for line in lines if not line.lstrip().startswith(b'#') + and b'/srv/stack127/scripts/ad-networks/ovh/run-job.sh verification ' in line] +assert not commands, 'Verification is already active' +resumed = b''.join(line[len(next(prefix for prefix in prefixes if line.startswith(prefix))):] + if line.rstrip(b'\r\n') in paused else line for line in lines) +if window: + assert not any(line.startswith(prefixes[-1]) for line in resumed.splitlines()), "Resume this window's other jobs first" +assert subprocess.check_output(['crontab', '-l']) == current, 'Crontab changed; rerun against current contents' +subprocess.run(['crontab', '-'], input=resumed, check=True) +assert subprocess.check_output(['crontab', '-l']) == resumed, 'Installed crontab differs' +print('Exactly one verification line enabled; every other line preserved') +PYTHON +ssh ovh 'crontab -l' +``` + +After the next hourly `:40` run finishes, inspect `ssh ovh 'tail -n 100 /srv/stack127-data/logs/verification.log'`. Require a `start verification` timestamp after `$ADN_RESUMED_AT` and its following `end verification exit 0`. An old exit line or an active crontab alone does not pass. If that next run fails, keep the release incomplete and investigate before claiming acceptance. + Check the durable `ad_networks/all` queue event drains after a successful refresh and the next hourly verification exits successfully. Do not remove the backups until acceptance. The whole false-loss spike **will not disappear**: blocked checks, changed signals and historical misses without proof remain unresolved. Missing old snapshots mean unknown history, not zero losses. -For each restored relationship, history repair deletes only its exact invalid `left` event. An apparent return within that correction window becomes `baseline`. It adds the restored network to the existing observation so the next check cannot manufacture a join. +For **proven history repairs only**, finalization deletes the exact invalid `left` event and turns an apparent return in that window into `baseline`. It adds the network to existing observations through the correction time so the next check cannot manufacture a join. Old per-network daily snapshots in those repair windows are removed; today's counts are rebuilt. Individual stale-transition times were never stored, so missing old days stay unknown. -Old per-network daily snapshots inside the affected miss window are removed. Individual stale-transition times were never stored, so their confirmed/stale split cannot be reconstructed honestly. Today's counts are rebuilt from corrected relationships. Missing old days remain unknown. The stats page reads net change and switches from events, and history/coverage from daily counts. This correction changes both sources consistently, without invented history. +For **current-only reconfirmations**, the old `left` and all old snapshots/observations stay. An ended relationship gets exactly one `joined` return at `corrected_at`. Repeated sync/finalization must not create a second return. Stale or confirmed relationships without a departure get no artificial return. These separate rules keep the stats page's movement and coverage records consistent with what is actually known. The existing relationship, event, and daily-count triggers update the source revision, touch published ad-network pages, and queue the `ad_networks/all` cache refresh. Keep those triggers enabled. No new job or cron schedule is needed. The monitoring descriptions mention the new inconclusive rule. @@ -330,9 +421,34 @@ select count(*) as remaining_invalid_departures from public.ad_network_publisher_relationships r join public.ad_network_relationship_events e using(ad_network_id,publisher_id) where r.evidence->'homepage_miss_correction'->>'version'='homepage-misses-v1' +and r.evidence->'homepage_miss_correction'->>'history_action'='repair' +and r.evidence->'homepage_miss_correction'->>'history_proof' in ('ads_txt_304','homepage_inconclusive') and e.event='left' and e.observed_at=(r.evidence->'homepage_miss_correction'->>'prior_ended_at')::timestamptz; +-- Must be zero: ended current-only rows retain their original departure +-- and have exactly one return, at the fresh confirmation time. +with current_only as ( + select r.ad_network_id,r.publisher_id,r.evidence->'homepage_miss_correction' as marker + from public.ad_network_publisher_relationships r + where r.evidence->'homepage_miss_correction'->>'version'='homepage-misses-v1' + and r.evidence->'homepage_miss_correction'->>'history_action'='preserve' + and r.evidence->'homepage_miss_correction'->>'prior_ended_at' is not null +) +select count(*) as broken_current_only_returns +from current_only r +where (select count(*) from public.ad_network_relationship_events e + where e.ad_network_id=r.ad_network_id and e.publisher_id=r.publisher_id + and e.event='left' and e.observed_at=(r.marker->>'prior_ended_at')::timestamptz)<>1 + or (select count(*) from public.ad_network_relationship_events e + where e.ad_network_id=r.ad_network_id and e.publisher_id=r.publisher_id + and e.event='joined' + and e.observed_at>(r.marker->>'prior_ended_at')::timestamptz + and e.observed_at<=(r.marker->>'corrected_at')::timestamptz)<>1 + or not exists(select 1 from public.ad_network_relationship_events e + where e.ad_network_id=r.ad_network_id and e.publisher_id=r.publisher_id + and e.event='joined' and e.observed_at=(r.marker->>'corrected_at')::timestamptz); + -- Must be zero immediately after correction; later real checks can change it. select count(*) as broken_restorations from public.ad_network_publisher_relationships r diff --git a/package-lock.json b/package-lock.json index 9377b856..e5e60907 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@namefi/dnssec-audit": "^2.1.1", "@next/mdx": "16.2.9", "@types/pg": "^8.20.0", + "acorn": "^8.19.0", "dns-packet": "^5.6.1", "echarts": "^6.1.0", "lucide-react": "^1.18.0", diff --git a/package.json b/package.json index c6d38bf4..a98909b2 100644 --- a/package.json +++ b/package.json @@ -112,6 +112,7 @@ "@namefi/dnssec-audit": "^2.1.1", "@next/mdx": "16.2.9", "@types/pg": "^8.20.0", + "acorn": "^8.19.0", "dns-packet": "^5.6.1", "echarts": "^6.1.0", "lucide-react": "^1.18.0", diff --git a/scripts/ad-networks/correct-homepage-misses.mjs b/scripts/ad-networks/correct-homepage-misses.mjs index 98359924..f6c75037 100644 --- a/scripts/ad-networks/correct-homepage-misses.mjs +++ b/scripts/ad-networks/correct-homepage-misses.mjs @@ -8,6 +8,7 @@ import { targetPool } from './runtime-db.mjs'; import { loadLocalNetworks } from './local-db.mjs'; import { assertLocalCatalog } from './identity-catalog.mjs'; import { verifyPublisher } from './verify-publisher.mjs'; +import { MATCHER_VERSION } from './matcher.mjs'; import { snapshotCounts } from './snapshot.mjs'; // The code predates the production runner. The earliest affected homepage-method @@ -18,31 +19,52 @@ const methods = new Set(['ads_txt_and_adsense_account', 'ads_txt_and_page_signat const json = value => typeof value === 'string' ? JSON.parse(value) : value ?? {}; const iso = value => value == null ? null : new Date(value).toISOString(); +// A fresh positive proves today's state only. The old stores did not save +// response HTML at a miss, so a full fetch/version alone cannot repair history. +export function historyCorrectionProof(row) { + const check = json(row.last_check_evidence); + if (!row.last_missed_at || Date.parse(row.last_checked_at) !== Date.parse(row.last_missed_at)) return null; + if (row.last_check_outcome === 'not_modified') return 'ads_txt_304'; + if (check.homepage?.outcome === 'inconclusive') return 'homepage_inconclusive'; + return null; +} + +function repairsHistory(row, evidence = row.correction_evidence ?? json(row.evidence)) { + const marker = evidence.homepage_miss_correction; + // Legacy v1 markers have no action/proof. Recheck the stored miss itself; + // never grant history repair just because an older correction restored it. + return marker?.history_action === 'repair' && ['ads_txt_304', 'homepage_inconclusive'].includes(marker.history_proof) + || (!marker?.history_action && Boolean(historyCorrectionProof({ ...row, last_missed_at: marker?.last_missed_at }))); +} + export function correctionEligibility(row, since = FAULT_SINCE) { - const evidence = json(row.evidence), check = json(row.last_check_evidence); + const evidence = json(row.evidence); if (evidence.homepage_miss_correction?.version === CORRECTION) return row.history_repaired_at ? 'history_complete' : 'already_restored'; if (!methods.has(row.verification_method)) return 'not_homepage_dependent'; if (!row.first_confirmed_at || !row.last_confirmed_at || row.relationship_type === 'unknown' || !['checked_url','checked_at','matched_signal','parser_version'].every(key => Object.hasOwn(evidence,key))) return 'no_original_confirmation'; - if (evidence.homepage_miss_correction?.version === CORRECTION && row.status === 'confirmed' - && !row.consecutive_misses) return 'already_restored'; if (!['confirmed','stale','ended'].includes(row.status) || !row.consecutive_misses) return 'not_damaged'; - if (!row.first_missed_at || new Date(row.first_missed_at).getTime() < Date.parse(since)) return 'before_fault_window'; - // A new verifier records obtained evidence of absence. Never undo that. - if (evidence.last_absence) return 'recorded_absence'; - if (new Date(row.last_checked_at).getTime() !== new Date(row.last_missed_at).getTime()) return 'later_or_unknown_check'; - // Historical full-fetch misses have no homepage outcome. They cannot be - // distinguished from genuine absences. A 304 proves this miss had no page. - if (row.last_check_outcome !== 'not_modified' && check.homepage?.outcome !== 'inconclusive') { - return 'no_proof_of_inconclusive_miss'; - } + if (!Number.isFinite(Date.parse(row.first_missed_at)) || Date.parse(row.first_missed_at) < Date.parse(since)) return 'before_fault_window'; + // Any fault-window candidate may reconfirm current use. Even a reliable + // absence can be followed by a real reinstall; its departure is preserved. + if (Date.parse(row.last_checked_at) !== Date.parse(row.last_missed_at)) return 'later_or_unknown_check'; return 'needs_live_confirmation'; } +function matchingConfirmation(row, result) { + const original = json(row.evidence), signal = original.page_signature ?? original.matched_signal; + return result.matches?.confirmed.filter(match => String(match.networkId) === String(row.ad_network_id)) + .flatMap(match => [match.evidence, ...(match.signals ?? [])]) + .find(evidence => evidence.matched_signal === signal); +} + export function correctionConfirmation(row, result) { + if (result.evidence?.matcher_version !== MATCHER_VERSION) return 'live_matcher_not_current'; const match = result.matches?.confirmed.find(match => String(match.networkId) === String(row.ad_network_id)); - if (!match) return result.inconclusiveNetworkIds?.length ? 'live_check_inconclusive' : 'live_network_absent'; + if (!match) return result.inconclusiveNetworkIds?.some(id => String(id) === String(row.ad_network_id)) ? 'live_check_inconclusive' : 'live_network_absent'; const original = json(row.evidence); + const confirmation = matchingConfirmation(row, result); + if (!confirmation) return 'live_signal_changed'; if (original.page_signature) { // Review-promoted evidence requires BOTH the exact authorized seller and // its configured live tag. A matching generic tag alone is insufficient. @@ -50,8 +72,8 @@ export function correctionConfirmation(row, result) { const current = result.evidence?.ads_txt_cache?.parsed?.sellers ?? []; if (!seller || seller.length !== 3 || !current.some(item => item.adSystem === seller[0] && item.sellerId === seller[1] && item.relationship === seller[2]) - || match.evidence.matched_signal !== original.page_signature) return 'live_signal_changed'; - } else if (match.evidence.matched_signal !== original.matched_signal) return 'live_signal_changed'; + || confirmation.matched_signal !== original.page_signature) return 'live_signal_changed'; + } const oldHash = json(row.evidence).ads_txt_sha256; if (oldHash && oldHash !== result.sha256) return 'ads_txt_changed_since_confirmation'; return 'restore'; @@ -60,11 +82,17 @@ export function correctionConfirmation(row, result) { export function restoration(row, checkedAt, confirmation = null) { return { ...json(row.evidence), + ...(!historyCorrectionProof(row) && confirmation ? { + checked_at: checkedAt, matcher_version: MATCHER_VERSION, page_url: confirmation.page_url, + } : {}), homepage_miss_correction: { version: CORRECTION, corrected_at: checkedAt, confirmation, + history_action: historyCorrectionProof(row) ? 'repair' : 'preserve', + history_proof: historyCorrectionProof(row), prior_status: row.status, prior_ended_at: iso(row.ended_at), first_missed_at: iso(row.first_missed_at), last_missed_at: iso(row.last_missed_at), original_first_confirmed_at: iso(row.first_confirmed_at), + original_last_confirmed_at: iso(row.last_confirmed_at), }, }; } @@ -82,10 +110,11 @@ export async function planCorrections(rows, networks, { since = FAULT_SINCE, lim if (reason === 'already_restored') { if (!historyDomains.has(row.normalized_domain) && historyDomains.size >= historyLimit) { bump('history_limit'); continue; } historyDomains.add(row.normalized_domain); nextHistoryAfterDomain = row.normalized_domain; - repaired.push(row); bump(reason); continue; + repaired.push(row); bump(reason); + bump(repairsHistory(row) ? 'history_correction' : 'current_reconfirmation'); continue; } if (reason !== 'needs_live_confirmation') { bump(reason); continue; } - if (statsOnly) { bump('stats_only'); continue; } + if (statsOnly) { bump('stats_only'); bump(historyCorrectionProof(row) ? 'history_candidate' : 'current_only_candidate'); continue; } if (!checkedDomains.has(row.normalized_domain) && checked >= limit) { bump('live_check_limit'); continue; } let result = checkedDomains.get(row.normalized_domain); if (!result) { @@ -99,13 +128,16 @@ export async function planCorrections(rows, networks, { since = FAULT_SINCE, lim } const decision = correctionConfirmation(row, result); bump(decision); - if (decision === 'restore') restores.push({ ...row, correction_evidence: restoration(row, result.checkedAt, result.matches.confirmed.find(match => String(match.networkId) === String(row.ad_network_id)).evidence) }); + if (decision === 'restore') { + const restored = { ...row, correction_evidence: restoration(row, result.checkedAt, matchingConfirmation(row, result)) }; + restores.push(restored); bump(repairsHistory(restored) ? 'history_correction' : 'current_reconfirmation'); + } } return { counts, checked, restores, repaired, nextAfterDomain: statsOnly ? nextHistoryAfterDomain : nextAfterDomain, nextHistoryAfterDomain }; } export async function previewPostgresCorrection(client, plan) { - const repairs = [...plan.restores,...plan.repaired].map(row => { + const repairs = [...plan.restores,...plan.repaired].filter(row => repairsHistory(row)).map(row => { const marker = (row.correction_evidence ?? json(row.evidence)).homepage_miss_correction; return { ad_network_id:row.ad_network_id,publisher_id:row.publisher_id, ended_at:marker.prior_ended_at,first_missed_at:marker.first_missed_at,corrected_at:marker.corrected_at }; @@ -125,7 +157,8 @@ export async function previewPostgresCorrection(client, plan) { } export async function applyPostgresCorrection(client, plan) { - let completed = 0, restored = 0, queued = 0, removedEvents = 0, neutralizedArrivals = 0, invalidatedSnapshots = 0, observations = 0; + let completed = 0, historyRepaired = 0, currentReconfirmed = 0, returnsRecorded = 0, + restored = 0, queued = 0, removedEvents = 0, neutralizedArrivals = 0, invalidatedSnapshots = 0, observations = 0; const rows = [...plan.restores, ...plan.repaired]; for (const row of rows) { const evidence = row.correction_evidence ?? json(row.evidence); @@ -135,10 +168,11 @@ export async function applyPostgresCorrection(client, plan) { [row.ad_network_id,row.publisher_id,marker.corrected_at,CORRECTION])).rowCount) continue; if (row.correction_evidence) { const update = await client.query(`update public.ad_network_publisher_relationships - set status='confirmed',ended_at=null,consecutive_misses=0,first_missed_at=null,last_missed_at=null,evidence=$3::jsonb + set status='confirmed',ended_at=null,consecutive_misses=0,first_missed_at=null,last_missed_at=null,evidence=$3::jsonb, + last_confirmed_at=case when $8::boolean then last_confirmed_at else $9::timestamptz end where ad_network_id=$1 and publisher_id=$2 and status=$4 and last_missed_at=$5 and first_confirmed_at=$6 and evidence=$7::jsonb`, - [row.ad_network_id,row.publisher_id,JSON.stringify(evidence),row.status,row.last_missed_at,row.first_confirmed_at,JSON.stringify(json(row.evidence))]); + [row.ad_network_id,row.publisher_id,JSON.stringify(evidence),row.status,row.last_missed_at,row.first_confirmed_at,JSON.stringify(json(row.evidence)),repairsHistory(row),marker.corrected_at]); if (update.rowCount !== 1) throw new Error('Relationship changed during live check; rerun correction'); restored += update.rowCount; } @@ -148,34 +182,47 @@ export async function applyPostgresCorrection(client, plan) { and evidence->'homepage_miss_correction'=$3::jsonb for update`,[row.ad_network_id,row.publisher_id,JSON.stringify(marker)]); if (!current.rowCount) throw new Error('Corrected relationship changed during selection; rerun correction'); } - // Exact departure only. Other departures and other networks remain intact. - removedEvents += (await client.query(`delete from public.ad_network_relationship_events - where ad_network_id=$1 and publisher_id=$2 and event='left' and observed_at=$3`, - [row.ad_network_id,row.publisher_id,marker.prior_ended_at])).rowCount; - // An apparent return after the invalid departure is baseline history. - if (marker.prior_ended_at) { - neutralizedArrivals += (await client.query(`delete from public.ad_network_relationship_events j - where j.ad_network_id=$1 and j.publisher_id=$2 and j.event='joined' and j.observed_at between $3 and $4 - and exists(select 1 from public.ad_network_relationship_events b where b.ad_network_id=j.ad_network_id - and b.publisher_id=j.publisher_id and b.event='baseline' and b.observed_at=j.observed_at)`, + if (repairsHistory(row)) { + historyRepaired++; + // Exact departure only. Other departures and other networks remain intact. + removedEvents += (await client.query(`delete from public.ad_network_relationship_events + where ad_network_id=$1 and publisher_id=$2 and event='left' and observed_at=$3`, + [row.ad_network_id,row.publisher_id,marker.prior_ended_at])).rowCount; + // An apparent return after the invalid departure is baseline history. + if (marker.prior_ended_at) { + neutralizedArrivals += (await client.query(`delete from public.ad_network_relationship_events j + where j.ad_network_id=$1 and j.publisher_id=$2 and j.event='joined' and j.observed_at between $3 and $4 + and exists(select 1 from public.ad_network_relationship_events b where b.ad_network_id=j.ad_network_id + and b.publisher_id=j.publisher_id and b.event='baseline' and b.observed_at=j.observed_at)`, + [row.ad_network_id,row.publisher_id,marker.prior_ended_at,marker.corrected_at])).rowCount; + neutralizedArrivals += (await client.query(`update public.ad_network_relationship_events set event='baseline' + where ad_network_id=$1 and publisher_id=$2 and event='joined' and observed_at between $3 and $4 + and not exists(select 1 from public.ad_network_relationship_events b where b.ad_network_id=$1 + and b.publisher_id=$2 and b.event='baseline' and b.observed_at=ad_network_relationship_events.observed_at)`, [row.ad_network_id,row.publisher_id,marker.prior_ended_at,marker.corrected_at])).rowCount; - neutralizedArrivals += (await client.query(`update public.ad_network_relationship_events set event='baseline' - where ad_network_id=$1 and publisher_id=$2 and event='joined' and observed_at between $3 and $4 - and not exists(select 1 from public.ad_network_relationship_events b where b.ad_network_id=$1 - and b.publisher_id=$2 and b.event='baseline' and b.observed_at=ad_network_relationship_events.observed_at)`, - [row.ad_network_id,row.publisher_id,marker.prior_ended_at,marker.corrected_at])).rowCount; + } + // There is no history of individual miss times. Do not invent old + // confirmed/stale totals. Remove uncertain network/day snapshots and + // replace today's snapshot from current corrected relationships below. + invalidatedSnapshots += (await client.query(`delete from public.ad_network_daily_counts + where ad_network_id=$1 and captured_at between $2 and $3 + and snapshot_on < (now() at time zone 'UTC')::date`, + [row.ad_network_id,marker.first_missed_at,marker.corrected_at])).rowCount; + observations += (await client.query(`update public.ad_network_check_observations + set confirmed_network_ids=array_append(confirmed_network_ids,$1::bigint) + where publisher_id=$2 and checked_at<=$3::timestamptz and not ($1::bigint=any(confirmed_network_ids))`, + [row.ad_network_id,row.publisher_id,marker.corrected_at])).rowCount; + } else { + currentReconfirmed++; + // A real (or unproven) departure stays. Record this fresh positive as a + // return, even when SQLite sync kept the publisher's old check time. + if (marker.prior_status === 'ended' || marker.prior_ended_at) { + returnsRecorded += (await client.query(`insert into public.ad_network_relationship_events + (ad_network_id,publisher_id,event,observed_at,relationship_type) + values($1,$2,'joined',$3,$4) on conflict do nothing`, + [row.ad_network_id,row.publisher_id,marker.corrected_at,row.relationship_type])).rowCount; + } } - // There is no history of individual miss times. Do not invent old - // confirmed/stale totals. Remove uncertain network/day snapshots and - // replace today's snapshot from current corrected relationships below. - invalidatedSnapshots += (await client.query(`delete from public.ad_network_daily_counts - where ad_network_id=$1 and captured_at between $2 and $3 - and snapshot_on < (now() at time zone 'UTC')::date`, - [row.ad_network_id,marker.first_missed_at,marker.corrected_at])).rowCount; - observations += (await client.query(`update public.ad_network_check_observations - set confirmed_network_ids=array_append(confirmed_network_ids,$1::bigint) - where publisher_id=$2 and not ($1::bigint=any(confirmed_network_ids))`, - [row.ad_network_id,row.publisher_id])).rowCount; if (row.correction_evidence) queued += (await client.query(`update public.ad_network_publishers set verification_priority=100,next_check_at=now() where id=$1 and (verification_priority<>100 or next_check_at is null or next_check_at>now())`,[row.publisher_id])).rowCount; completed++; @@ -185,16 +232,18 @@ export async function applyPostgresCorrection(client, plan) { } const snapshots = completed ? (await snapshotCounts(client)).rowCount : 0; - return { historyRepaired: completed, restored, queued, removedEvents, neutralizedArrivals, invalidatedSnapshots, observations, snapshots }; + return { historyRepaired, currentReconfirmed, returnsRecorded, restored, queued, removedEvents, neutralizedArrivals, invalidatedSnapshots, observations, snapshots }; } export function applySqliteCorrection(db, plan) { let restored = 0; for (const row of plan.restores) { const update = db.prepare(`update relationships set status='confirmed',ended_at=null,consecutive_misses=0, - first_missed_at=null,last_missed_at=null,evidence=?,updated_at=? + first_missed_at=null,last_missed_at=null,evidence=?,updated_at=?, + last_confirmed_at=case when ? then last_confirmed_at else ? end where ad_network_id=? and publisher_id=? and status=? and last_missed_at=? and evidence=?`) - .run(JSON.stringify(row.correction_evidence),new Date().toISOString(),row.ad_network_id,row.publisher_id,row.status, + .run(JSON.stringify(row.correction_evidence),new Date().toISOString(),repairsHistory(row) ? 1 : 0, + row.correction_evidence.homepage_miss_correction.corrected_at,row.ad_network_id,row.publisher_id,row.status, row.last_missed_at,typeof row.evidence === 'string' ? row.evidence : JSON.stringify(row.evidence)); if (update.changes !== 1) throw new Error('Relationship changed during live check; rerun correction'); restored += update.changes; @@ -292,7 +341,9 @@ export async function runCorrection(options) { transaction = false; return { environment: options.environment,store:options.store,applied:options.apply,since, scanned:rows.length,checked:plan.checked,counts:plan.counts,nextAfterDomain:plan.nextAfterDomain,nextHistoryAfterDomain:plan.nextHistoryAfterDomain,wouldRestore:plan.restores.length, - wouldRepairStats:plan.restores.length+plan.repaired.length,...preview,...applied }; + wouldRepairStats:[...plan.restores,...plan.repaired].filter(row => repairsHistory(row)).length, + wouldReconfirmCurrent:[...plan.restores,...plan.repaired].filter(row => !repairsHistory(row)).length, + ...preview,...applied }; } catch (error) { if (transaction) { if (db) db.exec('rollback'); else await client.query('rollback'); } throw error; diff --git a/scripts/ad-networks/local-verifier-store.mjs b/scripts/ad-networks/local-verifier-store.mjs index 65119602..8adff085 100644 --- a/scripts/ad-networks/local-verifier-store.mjs +++ b/scripts/ad-networks/local-verifier-store.mjs @@ -143,7 +143,8 @@ function ageMissingRelationships(database, existing, positiveIds, checkedAt, evi checkedAt, checkedAt, checkedAt, - JSON.stringify({ checked_at: checkedAt, homepage: evidence?.homepage ?? null, ads_txt_sha256: evidence?.ads_txt_sha256 ?? null }), + JSON.stringify({ checked_at: checkedAt, homepage: evidence?.homepage ?? null, ads_txt_sha256: evidence?.ads_txt_sha256 ?? null, + matcher_version: evidence?.matcher_version ?? null }), relationship.ad_network_id, relationship.publisher_id, ); diff --git a/scripts/ad-networks/matcher.mjs b/scripts/ad-networks/matcher.mjs index d73219d8..2b8a5dc5 100644 --- a/scripts/ad-networks/matcher.mjs +++ b/scripts/ad-networks/matcher.mjs @@ -1,4 +1,5 @@ import { parse } from 'parse5'; +import { parse as parseJavaScript } from 'acorn'; function sourceNetworkSlugs(discoverySources) { const aliases = new Map([ @@ -32,17 +33,102 @@ function evidenceForManager(variable, fetchEvidence, parserVersion) { }; } -export const MATCHER_VERSION = "ad-network-1.4.1"; +export const MATCHER_VERSION = "ad-network-1.5.2"; const javascriptTypes = new Set(['text/javascript', 'application/javascript', 'text/ecmascript', 'application/ecmascript', 'application/x-javascript', 'application/x-ecmascript', 'text/jscript', 'text/livescript', 'text/x-javascript', 'text/x-ecmascript', ...Array.from({ length: 6 }, (_, i) => `text/javascript1.${i}`)]); +// Each rule lists only sources its loader actually activates. The source-code +// audit and measured site counts are in content_docs/ad-networks/maintenance.md. +const deferredSources = new Map([ + ['litespeed/javascript', { sources: ['data-src', 'src'] }], + ...['rocketlazyloadscript', 'text/rocketlazyloadscript'].map(type => [type, + { sources: ['data-rocket-src', 'src'], savedType: 'data-rocket-type' }]), + ['pmdelayedscript', { sources: ['data-src', 'src'], savedType: 'data-perfmatters-type' }], + ['pencilazyscript', { sources: ['src'], savedType: 'data-pencilazy-type' }], + ['o/js-lzl', { sources: ['data-src', 'src'], savedType: 'data-type' }], + ['starcache/delay', { sources: ['data-starcache-src', 'src'], savedType: 'data-starcache-type' }], +]); +const categories = new Set(['preferences', 'statistics', 'marketing']); +const activationSources = ['src', 'data-src', 'data-rocket-src', 'data-wpmeteor-src', + 'consent-original-src-_', 'data-starcache-src', 'data-md-delayed-src', + 'data-borlabs-script-blocker-src', 'data-cmplz-src']; +const value = (attributes, name) => attributes.get(name)?.trim() ?? ''; + +const isJavaScript = type => type === '' || type === 'module' || javascriptTypes.has(type.split(';')[0].trim()); + +function activation(type, attributes) { + const held = (sources, savedType) => { + const restored = savedType ? value(attributes, savedType).toLowerCase() : ''; + // Only this measured chain has an audited second activation step. + const liteSpeedChain = normalized === 'pmdelayedscript' && restored === 'litespeed/javascript'; + if (savedType && !isJavaScript(restored) && !liteSpeedChain) return null; + return { sources, held: true, module: savedType ? restored === 'module' : normalized.endsWith('-module') || normalized === 'module' }; + }; + const normalized = type.trim().toLowerCase(); + if (normalized === 'pmdelayedscript' && value(attributes, 'data-perfmatters-type').toLowerCase() !== 'litespeed/javascript') { + return held(['src'], 'data-perfmatters-type'); + } + if (deferredSources.has(normalized)) { + const { sources, savedType } = deferredSources.get(normalized); + return held(sources, savedType); + } + if (/^[a-f0-9]{24}-(?:text\/javascript|application\/javascript|module)$/.test(normalized)) return held(['src']); + if (normalized === 'javascript/blocked' && value(attributes, 'data-wpmeteor-type') + && isJavaScript(value(attributes, 'data-wpmeteor-type').toLowerCase())) return held(['data-wpmeteor-src'], 'data-wpmeteor-type'); + if (normalized === 'application/consent' && value(attributes, 'consent-required') && value(attributes, 'consent-original-type-_') + && isJavaScript(value(attributes, 'consent-original-type-_').toLowerCase())) return held(['consent-original-src-_', 'src'], 'consent-original-type-_'); + if (normalized === 'didomi/javascript' + && ['data-vendor', 'data-vendor-raw', 'data-purposes'].some(name => value(attributes, name))) return held(['src']); + if (normalized === 'text/template' && value(attributes, 'data-borlabs-script-blocker-id')) { + return held(['data-borlabs-script-blocker-src', 'src']); + } + if (normalized === 'text/plain') { + if (value(attributes, 'data-md-delayed-src')) return held(['data-md-delayed-src']); + if (value(attributes, 'class').split(/\s+/).some(token => /^optanon-category-C\d+(?:-C\d+)*$/.test(token))) return held(['src']); + const cookiebot = value(attributes, 'data-cookieconsent').split(',').map(item => item.trim()); + if (cookiebot.length && cookiebot.every(item => ['preferences', 'statistics', 'marketing'].includes(item))) return held(['src']); + // Complianz selects these exact categories; data-service alone is metadata, + // not enough to identify a held script. Its external source is cmplz-src. + if (categories.has(value(attributes, 'data-category'))) return held(['data-cmplz-src']); + if (value(attributes, 'data-cookiescript') === 'accepted') return held(['src']); + // Orest Bida CookieConsent v2, measured with advertising consent. + if (['necessary', 'analytics', 'advertising', 'targeting'].includes(value(attributes, 'data-cookiecategory'))) return held(['data-src', 'src'], 'data-type'); + if (['necessary', 'functional', 'analytics', 'performance', 'advertisement', 'marketing'].includes(value(attributes, 'data-faz-category'))) return held(['src'], 'data-faz-original-type'); + return null; + } + if (isJavaScript(normalized)) { + return value(attributes, 'data-type') === 'lazy' ? held(['data-src']) + : { sources: ['src'], held: false, module: normalized === 'module' }; + } + return null; +} + +function inlinePayload(body, module = false) { + const comments = []; + try { + // Parsing never executes page code or proves control flow. Strip comments + // without damaging URLs/regexes; preserve all ordinary inline string matches. + parseJavaScript(body, { ecmaVersion: 'latest', sourceType: module ? 'module' : 'script', + onComment: (_, __, start, end) => comments.push([start, end]) }); + const chunks = []; + let offset = 0; + for (const [start, end] of comments) { + chunks.push(body.slice(offset, start), ' '); + offset = end; + } + chunks.push(body.slice(offset)); + return { text: chunks.join(''), parseFailed: false }; + } catch { return { text: body, parseFailed: true }; } +} +const escapeAttribute = text => text.replaceAll('&', '&').replaceAll('"', '"').replaceAll('<', '<'); + function pageSignals(html) { // HTML parsing keeps script-looking text in textarea/style/noscript inert. const pending = [parse(html, { scriptingEnabled: true, sourceCodeLocationInfo: true })]; - const scripts = [], accountElements = []; + const scripts = [], accountElements = [], uncertainPayloads = []; let hasEthicalPublisher = false; const inert = new Set(['template','textarea','style','title','xmp','iframe','noembed','noframes','noscript','plaintext']); while (pending.length) { @@ -54,9 +140,36 @@ function pageSignals(html) { const language = attributes.get('language')?.trim().toLowerCase(); const type = attributes.has('type') ? attributes.get('type').trim() : language ? `text/${language}` : ''; - const executable = type === '' || type === 'module' || javascriptTypes.has(type.toLowerCase().split(';')[0].trim()); - if (executable && node.sourceCodeLocation?.endTag) { - scripts.push(html.slice(node.sourceCodeLocation.startOffset, node.sourceCodeLocation.endOffset)); + const loader = activation(type, attributes); + if (loader && node.sourceCodeLocation?.endTag) { + // Attribute-copy order differs between loaders. Never choose a winner + // when known activation attributes (including plain src) disagree. + const candidates = (loader.held ? activationSources : loader.sources) + .filter(name => attributes.has(name)).map(name => value(attributes, name)); + if (new Set(candidates).size > 1) { + uncertainPayloads.push(...candidates); + if (candidates.some(source => /^(?:https?:)?\/\/pagead2\.googlesyndication\.com\/pagead\/js\/adsbygoogle\.js(?:[?#]|$)/i.test(source))) { + uncertainPayloads.push(value(attributes, 'data-ad-client')); + } + continue; + } + // A source-backed script ignores its inline body. An unused attribute + // (original-src, check-url, etc.) must never supply a match. + const source = loader.sources.map(name => value(attributes, name)).find(Boolean); + if (source) { + scripts.push(``); + // AdSense consumes this one account option on its actual loader. + if (/^(?:https?:)?\/\/pagead2\.googlesyndication\.com\/pagead\/js\/adsbygoogle\.js(?:[?#]|$)/i.test(source)) { + accountElements.push(`adsbygoogle ${value(attributes, 'data-ad-client')}`); + } + } else { + const body = inlinePayload(html.slice(node.sourceCodeLocation.startTag.endOffset, + node.sourceCodeLocation.endTag.startOffset), loader.module); + // Held code never confirms: its strings may be settings, examples, + // or unreachable calls. Parse failures also cannot prove absence. + if (loader.held || body.parseFailed) uncertainPayloads.push(body.text); + else if (body.text) scripts.push(``); + } } } else { if (attributes.get('data-ea-publisher')?.trim()) hasEthicalPublisher = true; @@ -64,12 +177,12 @@ function pageSignals(html) { const accountSlot = node.tagName === 'ins' && attributes.has('data-ad-client') && attributes.get('class')?.split(/\s+/).includes('adsbygoogle'); const location = node.sourceCodeLocation?.startTag ?? node.sourceCodeLocation; - if ((accountMeta || accountSlot) && location) accountElements.push(html.slice(location.startOffset, location.endOffset)); + if ((accountMeta || accountSlot) && location) accountElements.push(`google-adsense-account ${value(attributes, accountMeta ? 'content' : 'data-ad-client')}`); } } for (let i = (node.childNodes?.length ?? 0) - 1; i >= 0; i--) pending.push(node.childNodes[i]); } - return { scripts:scripts.join('\n'), accountMarkup:[...scripts,...accountElements].join('\n'), hasEthicalPublisher }; + return { scripts:scripts.join('\n'), accountMarkup:[...scripts,...accountElements].join('\n'), uncertainPayloads: uncertainPayloads.join('\n').toLowerCase(), hasEthicalPublisher }; } export function scriptMarkup(html) { return pageSignals(html).scripts; } @@ -132,6 +245,12 @@ export function needsHomepage(parsed, networks, discoverySources = []) { export function matchNetworks({ parsed, networks, fetchEvidence, homepage, discoverySources }) { const confirmed = new Map(); const review = new Map(); + const inconclusive = new Set(); + const confirm = match => { + const previous = confirmed.get(match.networkId); + match.signals = [...(previous?.signals ?? (previous ? [previous.evidence] : [])), match.evidence]; + confirmed.set(match.networkId, match); + }; const managers = new Set(parsed.managerDomains.map((item) => item.value)); const directSystems = directAdSystems(parsed); const sourceSlugs = sourceNetworkSlugs(discoverySources); @@ -140,7 +259,7 @@ export function matchNetworks({ parsed, networks, fetchEvidence, homepage, disco for (const variable of parsed.managerDomains) { for (const network of networks) { if (!(network.manager_domains ?? []).includes(variable.value)) continue; - confirmed.set(network.id, { + confirm({ networkId: network.id, slug: network.slug, status: "confirmed", @@ -155,40 +274,43 @@ export function matchNetworks({ parsed, networks, fetchEvidence, homepage, disco const signals = pageSignals(homepage.html); const normalizedScripts = signals.scripts.toLowerCase(); for (const network of networks) { - if (confirmed.has(network.id)) continue; + if (confirmed.get(network.id)?.verificationMethod === "ads_txt_managerdomain") continue; for (const signature of network.program_signatures ?? []) { if (signature.kind === "adsense_account_match") { const sellerIds = directAdsenseSellerIds(parsed, signature); const pageIds = pageAdsenseSellerIds(signals.accountMarkup); - const matchedId = [...sellerIds].find((sellerId) => pageIds.has(sellerId)); - if (!matchedId) continue; - confirmed.set(network.id, { - networkId: network.id, - slug: network.slug, - status: "confirmed", - relationshipType: signature.relationship_type ?? "active_network_use", - verificationMethod: "ads_txt_and_adsense_account", - evidence: { - checked_url: fetchEvidence.finalUrl ?? fetchEvidence.checkedUrl, - requested_url: fetchEvidence.checkedUrl, - checked_at: fetchEvidence.checkedAt, - matched_signal: `${signature.ad_system ?? "google.com"},${matchedId},${signature.seller_relationship ?? "DIRECT"} + ca-${matchedId}`, - seller_id: matchedId, - page_url: homepage.finalUrl, - identity_source: signature.source_url ?? null, - ads_txt_sha256: fetchEvidence.sha256, - parser_version: parsed.parserVersion, - matcher_version: MATCHER_VERSION, - }, - }); + if ([...sellerIds].some(id => signals.uncertainPayloads.includes(`ca-${id}`))) inconclusive.add(network.id); + for (const matchedId of sellerIds) { + if (!pageIds.has(matchedId)) continue; + confirm({ + networkId: network.id, + slug: network.slug, + status: "confirmed", + relationshipType: signature.relationship_type ?? "active_network_use", + verificationMethod: "ads_txt_and_adsense_account", + evidence: { + checked_url: fetchEvidence.finalUrl ?? fetchEvidence.checkedUrl, + requested_url: fetchEvidence.checkedUrl, + checked_at: fetchEvidence.checkedAt, + matched_signal: `${signature.ad_system ?? "google.com"},${matchedId},${signature.seller_relationship ?? "DIRECT"} + ca-${matchedId}`, + seller_id: matchedId, + page_url: homepage.finalUrl, + identity_source: signature.source_url ?? null, + ads_txt_sha256: fetchEvidence.sha256, + parser_version: parsed.parserVersion, + matcher_version: MATCHER_VERSION, + }, + }); + } continue; } if (signature.kind !== "html_substring" || typeof signature.value !== "string") continue; if (!signatureRequirementsMatch(signature, network, { managers, directSystems, sellerSystems, sourceSlugs })) continue; + if (signals.uncertainPayloads.includes(signature.value.toLowerCase())) inconclusive.add(network.id); const present = signature.allow_without_ads_txt && signature.value === 'data-ea-publisher=' ? signals.hasEthicalPublisher : normalizedScripts.includes(signature.value.toLowerCase()); if (!present) continue; - confirmed.set(network.id, { + confirm({ networkId: network.id, slug: network.slug, status: "confirmed", @@ -237,5 +359,6 @@ export function matchNetworks({ parsed, networks, fetchEvidence, homepage, disco }); } - return { confirmed: [...confirmed.values()], review: [...review.values()] }; + return { confirmed: [...confirmed.values()], review: [...review.values()], + inconclusiveNetworkIds: [...inconclusive].filter(id => !confirmed.has(id)) }; } diff --git a/scripts/ad-networks/review-rules.mjs b/scripts/ad-networks/review-rules.mjs index 028fba82..aed8ebdf 100644 --- a/scripts/ad-networks/review-rules.mjs +++ b/scripts/ad-networks/review-rules.mjs @@ -1,4 +1,4 @@ -import { matchNetworks,scriptMarkup } from './matcher.mjs'; +import { matchNetworks } from './matcher.mjs'; export const REVIEW_METHOD='discovery_hint_and_ads_txt_system'; export function reviewLinks(db,id) { @@ -22,7 +22,7 @@ export function gateReviewMatches({matches,relationships=[],networks,parsed,fetc else if(!review.some(row=>row.networkId===match.networkId))review.push({...match,status:'needs_review',relationshipType:'unknown', verificationMethod:REVIEW_METHOD,evidence:{...relationship.evidence,last_review:{checked_at:checkedAt,reason:result.reason}}}); } - return {confirmed,review}; + return {...matches,confirmed,review}; } export function reviewDecision({relationship,network,parsed,fetchResult,homepage,checkedAt,discoverySources=[]}) { const keep=reason=>({decision:'kept',reason}); @@ -43,16 +43,15 @@ export function reviewDecision({relationship,network,parsed,fetchResult,homepage } if(managers.length)return {...keep('different_manager'),present:true}; if(!homepage?.html)return {...keep('homepage_unavailable'),present:true}; - // Only configured, network-specific tags/configuration in executable script markup qualify. + // Only configured tags/configuration in JavaScript or recognized loader scripts qualify. // Plain article text, commented-out tags, seller rows and shared wrappers cannot match. - const scripts=scriptMarkup(homepage.html); const signatures=(network.program_signatures??[]).filter(signature=>signature.kind==='html_substring' &&typeof signature.value==='string'&&signature.value.trim().length>=8 &&signature.distinctive!==false&&signature.shared_wrapper!==true &&!/(?:googletagmanager\.com|googlesyndication\.com|doubleclick\.net|prebid\.js|gpt\.js)/i.test(signature.value)); if(!signatures.length)return {...keep('no_distinctive_configured_signature'),present:true}; const matches=matchNetworks({parsed,networks:[{...network,program_signatures:signatures}], - fetchEvidence:{...fetchResult,checkedAt},homepage:{...homepage,html:scripts},discoverySources}); + fetchEvidence:{...fetchResult,checkedAt},homepage,discoverySources}); const match=matches.confirmed.find(row=>row.networkId===network.id&&row.verificationMethod!=='ads_txt_managerdomain'); if(!match)return {...keep('no_current_distinctive_tag'),present:true}; return {decision:'confirmed',reason:'exact_line_and_current_distinctive_tag',present:true,evidence:{...match.evidence, diff --git a/scripts/ad-networks/verifier-store.mjs b/scripts/ad-networks/verifier-store.mjs index 37318179..7f45afaa 100644 --- a/scripts/ad-networks/verifier-store.mjs +++ b/scripts/ad-networks/verifier-store.mjs @@ -170,7 +170,8 @@ async function ageMissingRelationships(client, existing, positiveIds, checkedAt, first_missed_at = coalesce(first_missed_at, $2), last_missed_at = $2, evidence = evidence || jsonb_build_object('last_absence', $7::jsonb) where ad_network_id = $1 and publisher_id = $6 - `, [relationship.ad_network_id, checkedAt, status, endedAt, misses, relationship.publisher_id, JSON.stringify({ checked_at: checkedAt, homepage: evidence?.homepage ?? null, ads_txt_sha256: evidence?.ads_txt_sha256 ?? null })]); + `, [relationship.ad_network_id, checkedAt, status, endedAt, misses, relationship.publisher_id, JSON.stringify({ checked_at: checkedAt, homepage: evidence?.homepage ?? null, ads_txt_sha256: evidence?.ads_txt_sha256 ?? null, + matcher_version: evidence?.matcher_version ?? null })]); } } diff --git a/scripts/ad-networks/verify-publisher.mjs b/scripts/ad-networks/verify-publisher.mjs index 7163cb17..c810b675 100644 --- a/scripts/ad-networks/verify-publisher.mjs +++ b/scripts/ad-networks/verify-publisher.mjs @@ -41,9 +41,10 @@ async function checkHomepage(domain, fetchPage) { function homepageResult(homepage, parsed, networks, publisher, matches, forceUnknown = false) { const positive = new Set(matches?.confirmed.map(match => String(match.networkId)) ?? []); + const uncertain = new Set((matches?.inconclusiveNetworkIds ?? []).map(String)); const unknown = networks.filter(network => !positive.has(String(network.id)) - && (forceUnknown ? hasHomepageSignature(network) - : parsed && homepage?.outcome !== "fetched" && needsHomepage(parsed, [network], publisher.discovery_sources))); + && (uncertain.has(String(network.id)) || (forceUnknown ? hasHomepageSignature(network) + : parsed && homepage?.outcome !== "fetched" && needsHomepage(parsed, [network], publisher.discovery_sources)))); return { inconclusiveNetworkIds: unknown.map(network => network.id), homepageEvidence: { @@ -100,7 +101,7 @@ export async function verifyPublisher(publisher, networks, { const home = homepageResult(homepage, emptyParsed, networks, publisher, matches); return { inconclusiveNetworkIds: home.inconclusiveNetworkIds, - outcome: matches.confirmed.length ? "matched" : "no_ads_txt", + outcome: matches.confirmed.length ? "matched" : matches.inconclusiveNetworkIds?.length ? "temporary_error" : "no_ads_txt", checkedAt, matches, homepageFacts: homepage?.outcome === "fetched" ? homepageFacts(homepage.html, homepage.finalUrl) : null, @@ -189,7 +190,7 @@ export async function verifyPublisher(publisher, networks, { }),relationships:publisher.review_relationships,networks,parsed,fetchResult,homepage,checkedAt, discoverySources:publisher.discovery_sources}); const home = homepageResult(homepage, parsed, networks, publisher, matches); - const outcome = matches.confirmed.length ? "matched" : "no_match"; + const outcome = matches.confirmed.length ? "matched" : matches.inconclusiveNetworkIds?.length ? "temporary_error" : "no_match"; return { outcome, checkedAt, diff --git a/tests/pages/ad-networks/delayed-ad-scripts.test.mjs b/tests/pages/ad-networks/delayed-ad-scripts.test.mjs new file mode 100644 index 00000000..93d74f4f --- /dev/null +++ b/tests/pages/ad-networks/delayed-ad-scripts.test.mjs @@ -0,0 +1,349 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { parseAdsTxt } from '../../../scripts/ad-networks/ads-txt.mjs'; +import { verifyPublisher } from '../../../scripts/ad-networks/verify-publisher.mjs'; +import { correctionConfirmation } from '../../../scripts/ad-networks/correct-homepage-misses.mjs'; +import { matchNetworks } from '../../../scripts/ad-networks/matcher.mjs'; +import { reviewDecision } from '../../../scripts/ad-networks/review-rules.mjs'; + +const fixtures = JSON.parse(readFileSync(new URL('./fixtures/delayed-ad-scripts.json', import.meta.url))); +const id = 'ca-pub-1639051418715853'; +const url = `https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=${id}`; +const network = { id: 32, slug: 'google-adsense', program_signatures: [{ kind: 'adsense_account_match' }] }; +const taboola = { id: 33, slug: 'taboola', ad_system_domains: ['taboola.com'], + program_signatures: [{ kind: 'html_substring', value: 'cdn.taboola.com/libtrc/' }] }; +const parsed = parseAdsTxt('google.com,pub-1639051418715853,DIRECT\ntaboola.com,123,DIRECT'); +const fetchEvidence = { checkedAt: '2026-11-13T00:00:00Z', sha256: 'hash', outcome: 'parsed', httpStatus: 200 }; +function matches(html, target = network) { + return matchNetworks({ parsed, networks: [target], fetchEvidence, homepage: { html }, discoverySources: [] }).confirmed; +} +function markup(fixture, source = url) { + const attrs = { ...fixture.attrs, [fixture.source]: source }; + if (fixture.type !== undefined) attrs.type = fixture.type; + return ``; +} +for (const fixture of fixtures) { + test(`${fixture.name}: exact AdSense account and html_substring both match`, () => { + assert.equal(matches(markup(fixture))[0]?.evidence.seller_id, 'pub-1639051418715853'); + const html = markup(fixture, 'https://cdn.taboola.com/libtrc/publisher/loader.js'); + assert.equal(matches(html, taboola)[0]?.evidence.matched_signal, 'page:cdn.taboola.com/libtrc/'); + assert.equal(reviewDecision({ relationship: { evidence: { matched_signal: 'taboola.com,123,DIRECT' } }, + network: taboola, parsed, fetchResult: fetchEvidence, homepage: { html }, checkedAt: fetchEvidence.checkedAt }).decision, 'confirmed'); + assert.equal(matches(markup(fixture).replaceAll(id, 'ca-pub-1111111111111111')).length, 0); + }); + test(`${fixture.name}: plain src follows this loader's activation code`, () => { + for (const target of [network, taboola]) { + const source = target === network ? url : 'https://cdn.taboola.com/libtrc/publisher/loader.js'; + const tag = markup(fixture, source).replace(fixture.source + '=', 'src='); + assert.equal(matches(tag, target).length, fixture.plainSrc ? 1 : 0); + } + }); + test(`${fixture.name}: quoted, commented and inert copies do not match`, () => { + for (const target of [network, taboola]) { + const tag = markup(fixture, target === network ? url : 'https://cdn.taboola.com/libtrc/publisher/loader.js'); + const wrapped = ['textarea','template','style','title','xmp','iframe','noembed','noframes','noscript'] + .map(name => `<${name}>${tag}`); + const escaped = tag.replaceAll('<', '<').replaceAll('>', '>'); + for (const html of [...wrapped, ``, escaped, + ...['pre','code','blockquote'].map(name => `<${name}>${escaped}`)]) { + assert.equal(matches(html, target).length, 0, html); + } + } + }); +} + +test('1040log.com LiteSpeed regression keeps the original account and seller line', () => { + const html = ''; + assert.equal(matches(html)[0].evidence.matched_signal, + 'google.com,pub-1639051418715853,DIRECT + ca-pub-1639051418715853'); +}); + +test('inline delayed and consent-held code is inconclusive without a source attribute', () => { + for (const type of ['litespeed/javascript','rocketlazyloadscript','text/rocketlazyloadscript','pmdelayedscript','PenciLazyScript', + '38dea565cbb8e5075013007e-text/javascript','text/plain']) { + const html = ``; + assert.equal(matches(html).length, 0, type); + assert.deepEqual(matchNetworks({ parsed, networks: [network], fetchEvidence, homepage: { html } }).inconclusiveNetworkIds, [network.id]); + } +}); + +test('plain data, unknown types, incomplete tags and resource hints remain insufficient', () => { + for (const type of ['application/json','application/ld+json','importmap','speculationrules','text/template', + 'text/plain','text/html','text/x-lazy-cache','custom/javascript','module;not-a-module', + 'bad-text/javascript','1234-text/javascript','38dea565cbb8e5075013007e-application/json', + 'lazy','litespeed/javascript;example=true','javascript/blocked','application/consent','starcache/delay']) { + const html = ``; + assert.equal(matches(html).length, 0, type); + } + for (const type of ['application/json','application/ld+json','text/template']) { + assert.equal(matches(``).length, 0); + } + for (const attrs of ['data-note="example"','data-cookieconsent=""','data-cookiescript="rejected"']) { + assert.equal(matches(``).length, 0); + } + for (const html of [``, `

${url}

`, + ``]) { + assert.equal(matches(html).length, 0); + } +}); + +test('CMP metadata, vendor catalogs, disabled settings and examples cannot confirm either network', () => { + for (const target of [network, taboola]) { + const source = target === network ? url : 'https://cdn.taboola.com/libtrc/example/loader.js'; + for (const attrs of ['data-cmp-vendors="catalog"', 'data-service="ad-settings"', + 'data-cmp-vendor="755"', 'data-category="vendors"', 'data-cookieyes="cookieyes-advertisement"']) { + for (const body of [JSON.stringify({ enabled:false, adsbygoogle:id, url:source }), `window.disabledSettings = {url:'${source}', adsbygoogle:'${id}'};`]) { + assert.equal(matches(``, target).length, 0, attrs); + } + } + for (const attrs of ['data-cookieconsent="marketing"', 'data-category="marketing"', 'class="optanon-category-C0004"']) { + for (const body of [JSON.stringify({adsbygoogle:id, url:source}), JSON.stringify([{url:source}]), + `'${source} adsbygoogle ${id}';`, `// adsbygoogle ${source} ${id}\n`, `/* adsbygoogle ${source} ${id} */`]) { + assert.equal(matches(``, target).length, 0, attrs + body); + } + } + } +}); + +test('only exact activation attributes and executable inline payload supply signals', () => { + const other = url.replace(id, id.slice(0, -1) + '4'); // Exactly one digit differs. + assert.equal(matches(``).length, 0); + for (const target of [network, taboola]) { + const source = target === network ? url : 'https://cdn.taboola.com/libtrc/example/loader.js'; + for (const attrs of [`src="/ad-block-detector.js" data-check-url="${source}"`, + `src="/safe.js" data-ad-client="${id}" data-description="adsbygoogle ${source}"`, + `type="litespeed/javascript" data-src="/safe.js" data-original-src="${source}"`, + `type="rocketlazyloadscript" data-src="${source}"`, + `type="text/plain" data-cookieconsent="marketing" data-src="${source}"`, + `type="text/plain" data-category="marketing" data-lazy-src="${source}"`]) { + assert.equal(matches(``, target).length, 0, attrs); + } + for (const type of ['', 'litespeed/javascript', 'text/plain']) { + const attrs = `type="${type}" data-cookieconsent="marketing"`; + for (const body of [`/* adsbygoogle ${id} ${source} */ window.loaded=true;`, + `// adsbygoogle ${id} ${source}\nwindow.loaded=true;`]) { + assert.equal(matches(``, target).length, 0); + } + assert.equal(matches(``, target).length, 0); + assert.equal(matches(``, target).length, type === '' ? 1 : 0, type); + } + } +}); + +test('unused meta and slot attributes cannot supply another AdSense account', () => { + const other = id.slice(0, -1) + '4'; + assert.equal(matches(``).length, 0); + assert.equal(matches(``).length, 0); +}); + +const taboolaUrl = 'https://cdn.taboola.com/libtrc/publisher/loader.js'; +async function verify(html, networks = [network, taboola]) { + return verifyPublisher({ normalized_domain: 'publisher.example', discovery_sources: [] }, networks, { + fetchAdsTxt: async () => ({ ...fetchEvidence, text: 'google.com,pub-1639051418715853,DIRECT\ntaboola.com,123,DIRECT' }), + fetchHomepage: async () => ({ html, finalUrl: 'https://publisher.example/' }), + }); +} +for (const [name, body] of [ + ['unused URL variable', source => `const disabledUrl = "${source}";`], + ['literal URL followed by unrelated code', source => `"${source}"; window.unrelated = true;`], + ['unreachable loader call', source => `if (false) window.loadAd("${source}");`], +]) test(`held inline ${name} is inconclusive, never a confirmation or miss`, async () => { + for (const target of [network, taboola]) { + const source = target === network ? url : taboolaUrl; + for (const attrs of ['type="text/plain" data-category="marketing"', 'type="litespeed/javascript"']) { + const result = await verify(``); + assert.equal(result.matches.confirmed.length, 0); + assert.notEqual(result.outcome, 'no_match'); + assert.deepEqual(result.inconclusiveNetworkIds, [target.id]); + assert.equal(correctionConfirmation({ ad_network_id: target.id }, result), 'live_check_inconclusive'); + } + } +}); + +test('a held exact AdSense ID needs no marker to be inconclusive', async () => { + const result = await verify(``); + assert.deepEqual(result.inconclusiveNetworkIds, [network.id]); + assert.equal(result.matches.confirmed.length, 0); +}); + +test('Complianz activates data-cmplz-src, never plain src', async () => { + for (const target of [network, taboola]) { + const source = target === network ? url : taboolaUrl; + const attrs = 'type="text/plain" data-category="marketing"'; + assert.equal(matches(``, target).length, 0); + assert.equal(matches(``, target).length, 1); + } +}); + +test('saved module types parse imports, exports and top-level await before stripping comments', async () => { + for (const attrs of [ + 'type="rocketlazyloadscript" data-rocket-type="module"', + 'type="text/rocketlazyloadscript" data-rocket-type="module"', + 'type="38dea565cbb8e5075013007e-module"', + 'type="module" data-type="lazy"', + 'type="pmdelayedscript" data-perfmatters-type="module"', + 'type="PenciLazyScript" data-pencilazy-type="module"', + 'type="javascript/blocked" data-wpmeteor-type="module"', + 'type="application/consent" consent-required="755" consent-original-type-_="module"', + 'type="o/js-lzl" data-type="module"', + 'type="starcache/delay" data-starcache-type="module"', + 'type="text/plain" data-cookiecategory="advertising" data-type="module"', + 'type="text/plain" data-faz-category="marketing" data-faz-original-type="module"', + ]) { + const result = await verify(``); + // Parsing in classic mode would fail and preserve the comment's Taboola signal. + assert.deepEqual(result.inconclusiveNetworkIds, [network.id], attrs); + assert.equal(result.matches.confirmed.length, 0); + } +}); + +test('inline parse failures protect exactly the networks whose signals they contain', async () => { + for (const attrs of ['', 'type="module"', 'type="text/plain" data-category="marketing"']) { + const result = await verify(``); + assert.deepEqual(result.inconclusiveNetworkIds, [network.id, taboola.id]); + assert.notEqual(result.outcome, 'no_match'); + assert.equal(result.matches.confirmed.length, 0); + } +}); + +test('ordinary executable inline matching keeps the 1.4 behavior without proving execution', async () => { + for (const type of ['', 'text/javascript', 'module']) { + for (const body of [ + `const unused = "${taboolaUrl}";`, + `"${taboolaUrl}";`, + `"${taboolaUrl}"; window.unrelated = true;`, + `if (false) window.loadAd("${taboolaUrl}");`, + `adsbygoogle.push({client: "${id}"});`, + ]) { + const result = await verify(``); + assert.equal(result.matches.confirmed.length, 1); + assert.deepEqual(result.inconclusiveNetworkIds, []); + } + } +}); + +test('a separate activation source can confirm despite an inconclusive inline body', async () => { + const result = await verify(``); + assert.deepEqual(result.matches.confirmed.map(m => m.networkId), [taboola.id]); + assert.deepEqual(result.inconclusiveNetworkIds, []); +}); + +test('correction does not treat another network uncertainty as this network uncertainty', async () => { + const result = await verify(``); + assert.equal(correctionConfirmation({ ad_network_id: network.id }, result), 'live_network_absent'); +}); + + +test('Perfmatters data-src only qualifies when saved LiteSpeed type activates it afterward', () => { + assert.equal(matches(``).length, 0); + assert.equal(matches(``).length, 1); + assert.equal(matches(``).length, 1); +}); + +test('FAZ only activates script src; data-faz-src belongs to other resource types', () => { + assert.equal(matches(``).length, 0); + assert.equal(matches(``).length, 1); +}); + + +for (const fixture of fixtures.filter(item => item.savedType)) { + test(`${fixture.name}: restored non-JavaScript types are neither ad code nor inconclusive`, async () => { + for (const savedType of ['application/ld+json', 'application/json', 'importmap', 'text/plain', 'custom/javascript', + 'module;not-a-module', 'litespeed/javascript']) { + // The measured Perfmatters -> LiteSpeed chain has its own positive test. + if (fixture.name.startsWith('Perfmatters') && savedType === 'litespeed/javascript') continue; + const attrs = { ...fixture.attrs, [fixture.savedType]: savedType }; + const restored = { ...fixture, attrs }; + for (const source of [url, taboolaUrl]) { + const html = markup(restored, source); + for (const candidate of [html, html.replace('>', `>window.loadAd("${source}");`), + html.replace(`${fixture.source}="${source}"`, '') + .replace('>', `>window.loadAd("${source}");`), + html.replace('>', ' src="/safe.js">')]) { + const result = await verify(candidate); + assert.equal(result.matches.confirmed.length, 0, `${savedType}: ${candidate}`); + assert.deepEqual(result.inconclusiveNetworkIds, [], `${savedType}: ${candidate}`); + } + } + } + }); + test(`${fixture.name}: saved executable JavaScript and module types still confirm`, () => { + // data-src is activated only by the LiteSpeed step, not by restored JS. + const sourceFixture = fixture.name === 'Perfmatters with LiteSpeed' ? { ...fixture, source: 'src' } : fixture; + for (const savedType of ['', 'text/javascript', 'application/javascript; charset=utf-8', 'text/javascript1.5', 'module']) { + const restored = { ...sourceFixture, attrs: { ...fixture.attrs, [fixture.savedType]: savedType } }; + assert.equal(matches(markup(restored)).length, 1, savedType); + assert.equal(matches(markup(restored, taboolaUrl), taboola).length, 1, savedType); + } + }); +} + +for (const fixture of fixtures.filter(item => item.source !== 'src')) { + for (const reversed of [false, true]) for (const adFirst of [false, true]) { + test(`${fixture.name}: conflicting sources are inconclusive (reverse=${reversed}, adFirst=${adFirst})`, async () => { + for (const source of [url, taboolaUrl]) { + const sources = [[fixture.source, adFirst ? source : '/safe.js'], ['src', adFirst ? '/safe.js' : source]]; + if (reversed) sources.reverse(); + const attrs = { ...fixture.attrs, ...Object.fromEntries(sources) }; + const html = markup({ ...fixture, attrs }, sources.find(([key]) => key === fixture.source)[1]); + const result = await verify(html); + assert.equal(result.matches.confirmed.length, 0, html); + assert.deepEqual(result.inconclusiveNetworkIds, [source === url ? network.id : taboola.id], html); + } + }); + } + test(`${fixture.name}: identical source candidates can confirm`, () => { + const duplicate = { ...fixture, attrs: { ...fixture.attrs, src: url } }; + assert.equal(matches(markup(duplicate)).length, 1); + }); +} + +test('two activation attributes with different values are inconclusive', async () => { + const result = await verify(``); + assert.equal(result.matches.confirmed.length, 0); + assert.deepEqual(result.inconclusiveNetworkIds, [network.id, taboola.id]); +}); + +test('an empty competing source also makes a held source inconclusive', async () => { + const result = await verify(``); + assert.equal(result.matches.confirmed.length, 0); + assert.deepEqual(result.inconclusiveNetworkIds, [taboola.id]); +}); + +test('AdSense account option on a conflicting actual loader is inconclusive', async () => { + const result = await verify(``); + assert.equal(result.matches.confirmed.length, 0); + assert.deepEqual(result.inconclusiveNetworkIds, [network.id]); +}); + +test('a separate qualifying source still confirms a network with conflicting sources elsewhere', async () => { + const result = await verify(``); + assert.deepEqual(result.matches.confirmed.map(item => item.networkId), [taboola.id]); + assert.deepEqual(result.inconclusiveNetworkIds, []); +}); + +for (const wrapper of ['pre', 'code', 'blockquote']) { + test(`${wrapper}: real script elements still execute and supply ordinary or held signals`, async () => { + for (const source of [url, taboolaUrl]) { + const target = source === url ? network : taboola; + assert.equal(matches(`<${wrapper}>`, target).length, 1); + assert.equal(matches(`<${wrapper}>`, target).length, 1); + assert.equal(matches(`<${wrapper}>`, target).length, 1); + const result = await verify(`<${wrapper}>`); + assert.deepEqual(result.inconclusiveNetworkIds, [target.id]); + } + }); +} + + +for (const fixture of fixtures) { + test(`${fixture.name}: two distinct known source attributes cannot confirm`, async () => { + const secondSource = fixture.source === 'data-src' ? 'data-rocket-src' : 'data-src'; + const conflicting = { ...fixture, attrs: { ...fixture.attrs, [secondSource]: '/safe.js' } }; + const result = await verify(markup(conflicting, taboolaUrl)); + // Autoptimize is ordinary executable JS: only its actual src is a candidate. + assert.deepEqual(result.matches.confirmed.map(item => item.networkId), fixture.name === 'Autoptimize defer' ? [taboola.id] : []); + assert.deepEqual(result.inconclusiveNetworkIds, fixture.name === 'Autoptimize defer' ? [] : [taboola.id]); + }); +} diff --git a/tests/pages/ad-networks/fixtures/delayed-ad-scripts.json b/tests/pages/ad-networks/fixtures/delayed-ad-scripts.json new file mode 100644 index 00000000..7b0a2711 --- /dev/null +++ b/tests/pages/ad-networks/fixtures/delayed-ad-scripts.json @@ -0,0 +1,204 @@ +[ + { + "name": "LiteSpeed", + "type": "litespeed/javascript", + "source": "data-src", + "plainSrc": true + }, + { + "name": "WP Rocket", + "type": "rocketlazyloadscript", + "source": "data-rocket-src", + "plainSrc": true, + "savedType": "data-rocket-type" + }, + { + "name": "WP Rocket MIME variant", + "type": "text/rocketlazyloadscript", + "source": "data-rocket-src", + "plainSrc": true, + "savedType": "data-rocket-type" + }, + { + "name": "Perfmatters with LiteSpeed", + "type": "pmdelayedscript", + "source": "data-src", + "attrs": { + "data-perfmatters-type": "litespeed/javascript" + }, + "plainSrc": true, + "savedType": "data-perfmatters-type" + }, + { + "name": "Penci", + "type": "PenciLazyScript", + "source": "src", + "plainSrc": true, + "savedType": "data-pencilazy-type" + }, + { + "name": "Cloudflare Rocket Loader", + "type": "38dea565cbb8e5075013007e-text/javascript", + "source": "src", + "plainSrc": true + }, + { + "name": "Rocket Loader application MIME", + "type": "38dea565cbb8e5075013007e-application/javascript", + "source": "src", + "plainSrc": true + }, + { + "name": "Rocket Loader module", + "type": "38dea565cbb8e5075013007e-module", + "source": "src", + "plainSrc": true + }, + { + "name": "WP Meteor", + "type": "javascript/blocked", + "source": "data-wpmeteor-src", + "attrs": { + "data-wpmeteor-type": "text/javascript" + }, + "plainSrc": false + }, + { + "name": "Real Cookie Banner", + "type": "application/consent", + "source": "consent-original-src-_", + "attrs": { + "consent-required": "755,p1", + "consent-original-type-_": "application/javascript" + }, + "plainSrc": true + }, + { + "name": "Starcache", + "type": "starcache/delay", + "source": "data-starcache-src", + "plainSrc": true, + "savedType": "data-starcache-type" + }, + { + "name": "Borlabs", + "type": "text/template", + "source": "data-borlabs-script-blocker-src", + "attrs": { + "data-borlabs-script-blocker-id": "ad-sense-blocker" + }, + "plainSrc": true + }, + { + "name": "Didomi", + "type": "didomi/javascript", + "source": "src", + "attrs": { + "data-vendor": "iab:42" + }, + "plainSrc": true + }, + { + "name": "OneTrust", + "type": "text/plain", + "source": "src", + "attrs": { + "class": "optanon-category-C0002-C0004" + }, + "plainSrc": true + }, + { + "name": "Measured MD delayed loader", + "type": "text/plain", + "source": "data-md-delayed-src", + "plainSrc": false + }, + { + "name": "Flying Scripts", + "source": "data-src", + "attrs": { + "data-type": "lazy" + }, + "plainSrc": false + }, + { + "name": "Seraphinite Accelerator", + "type": "o/js-lzl", + "source": "src", + "plainSrc": true, + "savedType": "data-type" + }, + { + "name": "Autoptimize defer", + "type": "text/javascript", + "source": "src", + "attrs": { + "defer": "" + }, + "plainSrc": true + }, + { + "name": "Cookiebot", + "type": "text/plain", + "source": "src", + "attrs": { + "data-cookieconsent": "marketing" + }, + "plainSrc": true + }, + { + "name": "Complianz category", + "type": "text/plain", + "source": "data-cmplz-src", + "attrs": { + "data-category": "marketing" + }, + "plainSrc": false + }, + { + "name": "Complianz service", + "type": "text/plain", + "source": "data-cmplz-src", + "attrs": { + "data-service": "adsense", + "data-category": "marketing" + }, + "plainSrc": false + }, + { + "name": "CookieScript", + "type": "text/plain", + "source": "src", + "attrs": { + "data-cookiescript": "accepted" + }, + "plainSrc": true + }, + { + "name": "Cookie category", + "type": "text/plain", + "source": "data-src", + "attrs": { + "data-cookiecategory": "advertising" + }, + "plainSrc": true, + "savedType": "data-type" + }, + { + "name": "FAZ consent", + "type": "text/plain", + "source": "src", + "attrs": { + "data-faz-category": "marketing" + }, + "plainSrc": true, + "savedType": "data-faz-original-type" + }, + { + "name": "Perfmatters plain source", + "type": "pmdelayedscript", + "source": "src", + "plainSrc": true, + "savedType": "data-perfmatters-type" + } +] diff --git a/tests/pages/ad-networks/homepage-misses.test.mjs b/tests/pages/ad-networks/homepage-misses.test.mjs index df421113..c1ed8293 100644 --- a/tests/pages/ad-networks/homepage-misses.test.mjs +++ b/tests/pages/ad-networks/homepage-misses.test.mjs @@ -9,6 +9,7 @@ import { openLocalDatabase } from '../../../scripts/ad-networks/local-db.mjs'; import { saveLocalVerification } from '../../../scripts/ad-networks/local-verifier-store.mjs'; import { saveVerification,homepageConfirmationExpired } from '../../../scripts/ad-networks/verifier-store.mjs'; import { correctionEligibility,correctionConfirmation,planCorrections,restoration,applySqliteCorrection } from '../../../scripts/ad-networks/correct-homepage-misses.mjs'; +import { MATCHER_VERSION } from '../../../scripts/ad-networks/matcher.mjs'; const network={id:32,slug:'google-adsense',manager_domains:[],ad_system_domains:['google.com'],program_signatures:[{kind:'adsense_account_match'}]}; const manager={id:1,slug:'manager',manager_domains:['manager.com'],ad_system_domains:[],program_signatures:[]}; @@ -67,6 +68,7 @@ for(const [name,home,body] of [['fetched homepage without tag',{html:'{ @@ -96,18 +98,60 @@ const damaged={ad_network_id:32,publisher_id:1,normalized_domain:'publisher.exam last_checked_at:'2026-11-13T00:00:00Z',last_check_outcome:'not_modified', last_confirmed_at:'2026-09-26T00:00:00Z',relationship_type:'active_network_use', evidence:{matched_signal:signal,ads_txt_sha256:'hash',checked_url:'https://publisher.example/ads.txt',checked_at:'2026-09-26',parser_version:'ads-txt-1.1.0'},last_check_evidence:{}}; -test('correction excludes real absences, unknown full fetches, old misses, changed signals and hashes',async()=>{ +test('correction includes full-fetch misses but still excludes old misses, changed signals and hashes',async()=>{ assert.equal(correctionEligibility(damaged),'needs_live_confirmation'); assert.equal(correctionEligibility({...damaged,verification_method:'ads_txt_managerdomain'}),'not_homepage_dependent'); - assert.equal(correctionEligibility({...damaged,last_check_outcome:'no_match'}),'no_proof_of_inconclusive_miss'); - assert.equal(correctionEligibility({...damaged,evidence:{...damaged.evidence,last_absence:{}}}),'recorded_absence'); + assert.equal(correctionEligibility({...damaged,last_check_outcome:'no_match'}),'needs_live_confirmation'); + assert.equal(correctionEligibility({...damaged,last_check_outcome:'no_match',last_check_evidence:{homepage:{outcome:'fetched'}}}),'needs_live_confirmation'); + assert.equal(correctionEligibility({...damaged,evidence:{...damaged.evidence,last_absence:{}}}),'needs_live_confirmation'); assert.equal(correctionEligibility({...damaged,first_missed_at:'2026-09-27'}),'before_fault_window'); const positive=await verify(page); assert.equal(correctionConfirmation(damaged,positive),'restore'); assert.equal(correctionConfirmation(damaged,{...positive,sha256:'different'}),'ads_txt_changed_since_confirmation'); assert.equal(correctionConfirmation({...damaged,evidence:{matched_signal:'other'}},positive),'live_signal_changed'); const plan=await planCorrections([damaged,{...damaged,last_check_outcome:'no_match'}],[network],{verify:async()=>positive}); - assert.equal(plan.restores.length,1);assert.equal(plan.counts.no_proof_of_inconclusive_miss,1); + assert.equal(plan.restores.length,2);assert.equal(plan.checked,1); +}); + +test('absence versions do not prove a historical error or prevent current reconfirmation',async()=>{ + const row=(absence,check={})=>({...damaged,evidence:{...damaged.evidence,last_absence:absence},last_check_evidence:check}); + for(const version of ['ad-network-1.4.1','ad-network-1.4.0','ad-network-1.2.1']){ + assert.equal(correctionEligibility(row({checked_at:damaged.last_missed_at,matcher_version:version})),'needs_live_confirmation'); + assert.equal(correctionEligibility(row({checked_at:damaged.last_missed_at},{matcher_version:version})),'needs_live_confirmation'); + } + for(const version of [MATCHER_VERSION,'ad-network-1.4.2','ad-network-2.0.0','unknown',null]){ + assert.equal(correctionEligibility(row({checked_at:damaged.last_missed_at,matcher_version:version})),'needs_live_confirmation'); + } + assert.equal(correctionEligibility(row({checked_at:'2026-09-27',matcher_version:'ad-network-1.4.1'})),'needs_live_confirmation'); + assert.equal(correctionEligibility(row({matcher_version:'ad-network-1.4.1'})),'needs_live_confirmation'); + assert.equal(correctionEligibility(row({checked_at:'2026-11-12'},{matcher_version:'ad-network-1.4.1'})),'needs_live_confirmation'); + assert.equal(correctionEligibility(row({checked_at:damaged.last_missed_at})), 'needs_live_confirmation'); + const positive=await verify(page); + assert.equal(correctionConfirmation(damaged,{...positive,evidence:{...positive.evidence,matcher_version:'ad-network-1.4.1'}}),'live_matcher_not_current'); + const absent=await verify({html:'

Normal homepage without the ad

'}); + const plan=await planCorrections([row({checked_at:damaged.last_missed_at,matcher_version:'ad-network-1.4.1'})],[network],{verify:async()=>absent}); + assert.equal(plan.restores.length,0);assert.equal(plan.counts.live_network_absent,1); +}); + +test('widened full-fetch corrections require fresh exact evidence, including delayed code',async()=>{ + const positive=await verify({html:page.html.replace(''}), + }); + assert.equal(fresh.matches.confirmed[0].evidence.matched_signal,'page:static.tblcontent.com/libtrc/'); + assert.equal(fresh.matches.confirmed[0].signals.length,2); + const row={...damaged,ad_network_id:33,verification_method:'ads_txt_and_page_signature', + evidence:{...damaged.evidence,matched_signal:'page:cdn.taboola.com/libtrc/'}}; + assert.equal(correctionConfirmation(row,fresh),'restore'); + assert.equal(correctionConfirmation({...row,evidence:{...row.evidence,matched_signal:'taboola.com,123,DIRECT',page_signature:'page:cdn.taboola.com/libtrc/'}},fresh),'restore'); + assert.equal(correctionConfirmation({...row,evidence:{...row.evidence,matched_signal:'taboola.com,456,DIRECT',page_signature:'page:cdn.taboola.com/libtrc/'}},fresh),'live_signal_changed'); + assert.equal(correctionConfirmation(row,{...fresh,sha256:'changed'}),'ads_txt_changed_since_confirmation'); + const multi=await verify({html:page.html+page.html.replaceAll('1234567890123456','1234567890123457')},async()=>({...ads,text:ads.text+'google.com,pub-1234567890123457,DIRECT'})); + assert.equal(multi.matches.confirmed[0].signals.length,2); + assert.equal(correctionConfirmation(damaged,multi),'restore'); +}); + +test('legacy restored full-fetch markers cannot authorize history deletion',async()=>{ + const {previewPostgresCorrection}=await import('../../../scripts/ad-networks/correct-homepage-misses.mjs'); + const row={...damaged,status:'confirmed',last_check_outcome:'no_match',consecutive_misses:0}; + row.evidence=restoration(row,'2026-11-13T01:00:00Z'); + delete row.evidence.homepage_miss_correction.history_action; + delete row.evidence.homepage_miss_correction.history_proof; + const plan=await planCorrections([row],[],{statsOnly:true}); + assert.equal(plan.counts.current_reconfirmation,1); + assert.deepEqual(await previewPostgresCorrection({query:async()=>assert.fail('must not select destructive repairs')},plan),{ + wouldRemoveEvents:0,wouldNeutralizeArrivals:0,wouldInvalidateSnapshots:0}); +}); + + +for (const [name,html] of [ + ['held inline', ''], + ['parse failure', ''], +]) test(`${name} retries without a miss and expires only to stale after 30 days`, async t => { + const db=fixture(t); + db.prepare("update relationships set status='confirmed',consecutive_misses=0,first_missed_at=null,last_missed_at=null").run(); + const before=db.prepare('select * from relationships').get(); + const result=await verify({html}); + result.checkedAt='2026-10-20T00:00:00.000Z'; + saveLocalVerification(db,{id:1,consecutive_failures:0},result); + assert.deepEqual(db.prepare('select * from relationships').get(),before); + const stored=db.prepare('select * from publishers').get(); + assert.equal(stored.consecutive_failures,1); + assert.equal(Date.parse(stored.next_check_at)-Date.parse(result.checkedAt),6*3600000); + result.checkedAt='2026-11-01T00:00:00.000Z'; + saveLocalVerification(db,{id:1,consecutive_failures:1},result); + const expired=db.prepare('select * from relationships').get(); + assert.equal(expired.status,'stale');assert.equal(expired.consecutive_misses,0); + assert.equal(expired.ended_at,null);assert.equal(expired.last_confirmed_at,before.last_confirmed_at); + assert.equal(JSON.parse(expired.evidence).last_absence,undefined); + const queries=[],client={query:async(sql,args)=>{queries.push({sql,args});return {rows:sql.includes('for update')?[before]:[]};},release(){}}; + await saveVerification({connect:async()=>client},{id:1,consecutive_failures:0},result); + assert.equal(queries.some(q=>q.sql.includes("jsonb_build_object('last_absence'")),false); + assert.ok(queries.some(q=>q.sql.includes("set status='stale'"))); +});