From e88305833fc443bb21d298108bac80607528e61d Mon Sep 17 00:00:00 2001 From: RaviTejaKNTS Date: Fri, 13 Nov 2026 18:35:04 +0530 Subject: [PATCH] Treat blocked homepage checks as inconclusive and repair false departures A failed or challenged homepage fetch counted as "network not found" for networks that need page evidence, and unchanged ads.txt replies added further misses. Three misses ended live relationships: on 2026-11-13, 1,786 left events landed in one day, and 76% of a live sample still ran AdSense. Blocked, failed, empty and challenge pages are now inconclusive and retried with backoff. Unchanged ads.txt reuses its saved parse but no longer stands in for page evidence. Page evidence older than 30 days makes a relationship stale (not ended), and public checked dates use successful checks only. The per-domain deadline keeps the hourly job inside its budget. correct-homepage-misses.mjs restores relationships only with fresh matching evidence, removes their false left events, and repairs daily counts in bounded, checkpointed batches. The runbook covers backups, a frontend maintenance window, and 25-domain then 100-domain batches. Co-Authored-By: Claude Opus 5.5 --- .../ad-network-checker-tool.tsx | 2 +- .../ad-network-checker-view.ts | 6 +- .../ad-network-checker/content/article.mdx | 2 +- .../ad-network-checker/content/usage.mdx | 2 +- content_docs/ad-networks/maintenance.md | 12 +- .../ad-networks/production-runbook.md | 227 +++++++++- lib/ad-networks.ts | 12 +- .../ad-networks/correct-homepage-misses.mjs | 307 ++++++++++++++ scripts/ad-networks/fetch-policy.mjs | 71 +++- scripts/ad-networks/local-verifier-store.mjs | 41 +- scripts/ad-networks/matcher.mjs | 7 +- scripts/ad-networks/platform.mjs | 3 +- scripts/ad-networks/sync.mjs | 6 +- scripts/ad-networks/verification-batch.mjs | 21 + scripts/ad-networks/verifier-store.mjs | 64 ++- scripts/ad-networks/verify-publisher.mjs | 109 +++-- scripts/ad-networks/verify.mjs | 26 +- stack127_docs/monitoring.json | 4 +- ...20000_ad_network_homepage_miss_repairs.sql | 23 + .../fixtures/homepage-challenges.json | 87 ++++ .../ad-networks/homepage-misses.test.mjs | 399 ++++++++++++++++++ 21 files changed, 1340 insertions(+), 91 deletions(-) create mode 100644 scripts/ad-networks/correct-homepage-misses.mjs create mode 100644 scripts/ad-networks/verification-batch.mjs create mode 100644 supabase/migrations/20261113120000_ad_network_homepage_miss_repairs.sql create mode 100644 tests/pages/ad-networks/fixtures/homepage-challenges.json create mode 100644 tests/pages/ad-networks/homepage-misses.test.mjs diff --git a/app/(site)/(tools)/ad-network-checker/ad-network-checker-tool.tsx b/app/(site)/(tools)/ad-network-checker/ad-network-checker-tool.tsx index 5b87eb36..04990cf6 100644 --- a/app/(site)/(tools)/ad-network-checker/ad-network-checker-tool.tsx +++ b/app/(site)/(tools)/ad-network-checker/ad-network-checker-tool.tsx @@ -324,7 +324,7 @@ function HistorySection({ view }: { view: AdNetworkCheckerSiteView }) { .map((row) => ({ from: monthKey(row.first_confirmed_at!), label: row.name, - note: row.status === "stale" ? "not seen lately" : undefined, + note: row.status === "stale" ? "verification stale" : undefined, to: row.status === "ended" ? monthKey(row.last_confirmed_at ?? row.ended_at ?? row.first_confirmed_at!) : null, tone: ganttTone(view, row), })); diff --git a/app/(site)/(tools)/ad-network-checker/ad-network-checker-view.ts b/app/(site)/(tools)/ad-network-checker/ad-network-checker-view.ts index 56c3af7b..fc518ad3 100644 --- a/app/(site)/(tools)/ad-network-checker/ad-network-checker-view.ts +++ b/app/(site)/(tools)/ad-network-checker/ad-network-checker-view.ts @@ -91,6 +91,7 @@ export function evidenceOf(relationship: AdNetworkSiteRelationship): Evidence { /** What the site's ads.txt says about one network (mockup column), worded to match the evidence. */ export function saysLabel(view: AdNetworkCheckerSiteView, relationship: AdNetworkSiteRelationship) { const evidence = evidenceOf(relationship); + if (evidence === "page" && relationship.status === "stale") return "Not listed. We last found it in page ad code"; if (evidence === "page" && relationship.status !== "ended") return "Not listed. Its page ad code shows it"; if (evidence === "page" || evidence === "unknown") return NOT_LISTED; if (relationship.status === "ended") return "No longer lists it"; @@ -100,7 +101,8 @@ export function saysLabel(view: AdNetworkCheckerSiteView, relationship: AdNetwor export function statusLabel(relationship: AdNetworkSiteRelationship) { if (relationship.status === "confirmed") return "Confirmed live"; - if (relationship.status === "stale") return "Not seen lately"; + if (relationship.status === "stale") return relationship.last_confirmed_at + ? `Verification stale; last confirmed ${formatDay(relationship.last_confirmed_at)}` : "Verification stale"; return relationship.ended_at ? `Ended ${formatMonth(relationship.ended_at)}` : "Ended"; } @@ -143,7 +145,7 @@ export function leadFor(view: AdNetworkCheckerSiteView) { const who = countWord(current.length); parts.push({ text: first === last ? ` We confirmed ${who} live on ${last}.` : ` We last confirmed ${who} live between ${first} and ${last}.` }); } - if (stale.length) parts.push({ text: ` ${names(stale)} ${stale.length === 1 ? "hasn't" : "haven't"} shown up in our latest checks.` }); + if (stale.length) parts.push({ text: ` We couldn't confirm ${names(stale)} in our latest checks.` }); return parts; } diff --git a/app/(site)/(tools)/ad-network-checker/content/article.mdx b/app/(site)/(tools)/ad-network-checker/content/article.mdx index 6b7cb445..8946fa98 100644 --- a/app/(site)/(tools)/ad-network-checker/content/article.mdx +++ b/app/(site)/(tools)/ad-network-checker/content/article.mdx @@ -59,7 +59,7 @@ The `Checked` date under the site name is our latest read of its ads.txt. We rec The `Since` month is when we first confirmed a network. If that was our first check of the site, the network may have been there for years before. The changes section says when our checks of the site started. -One missed check never removes a network. A timeout or a blocked request counts as unknown. A network moves to `Not seen lately`, then `Ended`, only after repeated checks find it missing. +One missed check never removes a network. A timeout, blocked request, or bot challenge counts as unknown. After 30 days without fresh page confirmation, verification becomes stale and keeps its last confirmation date. Only repeated checks with evidence that the network is missing can mark it `Ended`. ## What does the traffic group mean? diff --git a/app/(site)/(tools)/ad-network-checker/content/usage.mdx b/app/(site)/(tools)/ad-network-checker/content/usage.mdx index a2a2cf1f..a268ad84 100644 --- a/app/(site)/(tools)/ad-network-checker/content/usage.mdx +++ b/app/(site)/(tools)/ad-network-checker/content/usage.mdx @@ -13,7 +13,7 @@ The table lists every network we've linked to the site, including ones that stop | Status | What it means | | --- | --- | | Confirmed live | Our last check found the network on the site. | -| Not seen lately | Two checks at least 48 hours apart didn't find it. It may come back. | +| Verification stale | Two checks at least 48 hours apart found no match, or page evidence is over 30 days old. The last confirmation date stays visible. | | Ended, with a month | Three checks over seven days or more didn't find it. The month is when it ended. | ## What do the other sections show? diff --git a/content_docs/ad-networks/maintenance.md b/content_docs/ad-networks/maintenance.md index f219b9f9..5c1ce97c 100644 --- a/content_docs/ad-networks/maintenance.md +++ b/content_docs/ad-networks/maintenance.md @@ -187,7 +187,7 @@ Every confirmed relationship stores the checked URL, time, matched signal, ads.t The fetcher uses the Public Suffix List through `tldts`, resolves every request and redirect target, rejects local and reserved addresses, permits only HTTP and HTTPS on their normal ports, and caps redirects and response size. Publisher verification reads the public `/ads.txt` file and, only when a network rule needs it, the public homepage without consulting `robots.txt`. It sends the stack127 crawler identity, prefers HTTPS, and uses a limited HTTP fallback only when HTTPS cannot be reached. A second redirect outside the publisher's registrable domain is rejected. Discovery source collectors keep their separate robots policy. -A positive parse requires a successful `text/plain` response. HTML, oversized bodies, unsupported content types, unsafe targets, and redirect violations cannot create a relationship. ETag, Last-Modified, and SHA-256 values allow unchanged files to be checked without reparsing them. +A positive parse requires a successful `text/plain` response. HTML, oversized bodies, unsupported content types, unsafe targets, and redirect violations cannot create a relationship. ETag, Last-Modified, and SHA-256 values identify unchanged ads.txt files. They do not say whether a homepage changed. ### Check and relationship states @@ -196,13 +196,19 @@ A positive parse requires a successful `text/plain` response. HTML, oversized bo | Confirmed match | `checked` | 7 days | | Valid file with no match | `checked` | 90 days, unless an existing relationship needs a follow-up | | Hard missing file | `checked` | 30 days, unless an existing relationship needs a follow-up | -| Unchanged file | `checked` | Uses the prior positive or negative evidence | +| Unchanged file | `checked` | Reuses ads.txt-only evidence; homepage-dependent networks need a fresh homepage | | Timeout, DNS failure, `429`, or `5xx` | `unreachable` | 6 hours, 24 hours, 3 days, then 7 days | | Invalid domain or ads.txt response | `invalid` | 14 days for the response, 365 days for the domain | Domains previously stored with `robots_blocked` were requeued when this policy changed on 2026-09-26. New verification runs do not emit that outcome. -One valid absence records a miss and keeps a confirmed relationship. A second valid absence at least 48 hours later marks it `stale`. A third valid absence spanning at least seven days marks it `ended`. Fetch failures never increase the miss counter. Positive evidence restores a stale relationship and clears its misses. A newly confirmed network is added without immediately ending another network because a site can use more than one program during a transition or by country. +One valid absence records a miss and keeps a confirmed relationship. A second valid absence at least 48 hours later marks it `stale`. A third valid absence spanning at least seven days marks it `ended`. Fetch failures never increase the miss counter. This includes homepage failures, HTTP blocks, timeouts, empty pages, oversized pages, and bot challenges returned with HTTP 200. An inconclusive homepage never adds an absence. At a check 30 days after its last successful homepage confirmation, a still-confirmed relationship becomes `stale`. It keeps its confirmation date, misses and end date. Weekly checks and the 6/24/72/168-hour retries allow several attempts within those 30 days. Only fresh positive evidence confirms it again. Independent manager evidence can still confirm a different network. Missing ads.txt authorization remains a valid absence even when a homepage cannot load. + +The verifier records the homepage outcome, status, URL, reason, and affected networks. Inconclusive checks retry after 6 hours, 24 hours, 3 days, then 7 days. They do not advance the last successful check or the arrival observation. Public checked dates use `last_successful_check_at`; the internal `last_checked_at` remains the attempt time. Public checker relationships show `stale` with the last successful `last_confirmed_at`. Job summaries include `inconclusiveChecks` and `homepageFailures` by reason. + +The verifier saves the complete parsed ads.txt, its URL, hash and validators in SQLite check evidence. It never reconstructs a file from the truncated public preview. A conditional 304 reuses that parse and fetches a fresh homepage only when the parsed authorization needs one. A missing or incompatible cache gets one unconditional retry. A second 304 stays unknown for homepage-dependent networks. Compact sync excludes the parse cache. Each domain must start before the run deadline; admitted requests finish within their existing request timeouts, then unused leases are released. The crawler identity, 15-second request timeout, three bounded transport attempts, four workers, and host throttle stay unchanged. There are no proxies or challenge bypasses. + +Valid misses now keep a bounded `last_absence` record beside the original confirmation evidence. The compact PostgreSQL sync retains this record, the homepage outcome, and correction metadata. The platform job also leaves challenge pages unavailable. Positive evidence restores a stale relationship and clears its misses. A newly confirmed network is added without immediately ending another network because a site can use more than one program during a transition or by country. ### Network identity maintenance diff --git a/content_docs/ad-networks/production-runbook.md b/content_docs/ad-networks/production-runbook.md index 577aa390..f1481b5f 100644 --- a/content_docs/ad-networks/production-runbook.md +++ b/content_docs/ad-networks/production-runbook.md @@ -50,7 +50,7 @@ Verification runs at minute 40 each hour, with four workers, a 1,000-site limit, ## What stays in production -Publisher evidence keeps `manager_domains` and `confirmed_networks`, plus four homepage fields used by niche sorting: `title`, `description`, `h1`, and `language`. Relationship evidence keeps `manager_domain`, `matched_signal`, `checked_url`, `checked_at`, and `parser_version`. The last three are required confirmation metadata. These preserve existing manager resolution, observations, and database constraints. +Publisher evidence keeps `manager_domains`, `confirmed_networks`, the bounded `homepage` check outcome and `inconclusive_networks`, plus four homepage fields used by niche sorting: `title`, `description`, `h1`, and `language`. Relationship evidence keeps `manager_domain`, `matched_signal`, `checked_url`, `checked_at`, `parser_version`, `last_absence`, and `homepage_miss_correction`. `checked_url`, `checked_at`, and `parser_version` are required confirmation metadata. These preserve existing manager resolution, observations, and database constraints. Full fetch diagnostics, redirects, seller records, parser errors, and traffic evidence stay in SQLite. `discovery_sources` keeps compact `source_key` attribution, which preserves the discovered/checker weekly chart. Source URLs, domain spellings, licenses, and collection dates stay local. Page facts, complete seller-domain summaries, bounded ads.txt previews, names, platforms, dates, traffic groups, and favicons keep their existing behavior. Existing target niches are never replaced by a collector sync. @@ -140,3 +140,228 @@ ssh ovh '/srv/stack127/scripts/ad-networks/ovh/run-job.sh rdap-import --source-s ``` The importer copies only newer RDAP checkpoints and dates, then syncs and refreshes. After its successful backup, disable homelab's old `stack127-ad-network-backup.timer`. This does not affect the niche timer or change which pages are published. + +## Correct inconclusive homepage misses (November 2026) + +A failed homepage must never count as an absent network. HTTP 200 can also be a bot challenge. The verifier now records that result as inconclusive for each network that needs the page. It preserves confirmation dates and misses. After 30 days without successful page evidence, a confirmed relationship becomes stale at its next check; failed requests never end it. Public checked dates use the last successful check, not the attempt time. It retries with backoff. A normal page without the network, or ads.txt without the required authorization, still counts as an absence. + +The fault existed before OVH. Commit `c488b9cc` (October 1) already swallowed homepage failures in `verify.mjs`. Commit `763d5a33` (October 16) extracted the same behavior into `verify-publisher.mjs`. The retained homepage-method misses start on September 28. The correction therefore defaults to `--since 2026-09-28T00:00:00.000Z`. This is a conservative search boundary, not a claim that every miss since then was wrong. OVH cron began October 19; the large November 13 loss is a later symptom. + +Read-only probes on November 13 used the same crawler identity and 15-second timeout on OVH and the Mac. Of 12 ended sites, all 12 returned normal HTTP 200 pages on the Mac. OVH returned nine normal pages, two HTTP 200 browser challenges (`tawcan.com`, `nomadicmatt.com`), and one HTTP 403 challenge (`iwandered.net`). A later four-site curl check still challenged the first two but loaded `iwandered.net` normally. Blocks vary by request. These probes establish a real failure mode; they do not establish the cause of every historical miss. The old logs and compact evidence did not retain homepage outcomes. + +### Selection and limits + +`scripts/ad-networks/correct-homepage-misses.mjs` defaults to a dry run. It repairs only a previously confirmed homepage-method relationship with misses inside the fault window. The latest check must be the miss itself, and either a 304 or a recorded inconclusive homepage must prove that check did not obtain absence evidence. A recorded valid `last_absence` excludes the row. + +A fresh full check must confirm the exact original signal. For a review-promoted seller relationship, both the exact ads.txt seller line and the saved `page_signature` must match current evidence. If the stored original ads.txt hash exists, the current hash must match too. Changed signals, changed files, blocked checks, actual absences, incomplete evidence, and old full-fetch misses without homepage diagnostics remain untouched. Counts report each reason. This intentionally leaves unresolved damage for later evidence-based review. + +The script preserves both original confirmation dates. It clears misses and the end date, restores `confirmed`, and queues priority 100 re-verification. It stores the original miss window and end date under `homepage_miss_correction`. Verification and compact sync preserve that marker. + +`--limit` bounds distinct domains checked, default 100. `nextAfterDomain` supplies the next batch's `--after-domain` value. Use the same cursor for a preview and its apply. Advance only after that batch has applied. Failed or excluded rows remain unchanged; start again without a cursor to retry them later. History work has its own `--history-limit` domain bound (defaults to `--limit`). Both restoration and history selection honor `--after-domain`. Stats-only output uses `nextAfterDomain` for history progress. PostgreSQL records completed `(network, publisher, correction time, version)` repairs in `ad_network_homepage_miss_repairs`, in the same transaction as the history changes. This target-side checkpoint survives a later source sync. Interrupted source repair, sync and history stages can each be repeated without advancing their cursors. Completed history is skipped even when starting again without a cursor. Each store applies its batch in one write transaction. Network fetches happen before that transaction. Changed rows abort and roll back the batch. + +### Release and apply order + +The lead runs this only after review and release authorization. No production step below was run during implementation. Start with **25 domains**, verify the public result, then use **100-domain** batches. A dry run fetches fresh evidence; apply fetches again and may restore fewer rows if a page changes or challenges it. + +#### 1. Open a maintenance window + +Pausing collectors alone is insufficient. OVH's `stack127-cache-worker.timer` is enabled and dispatches `.github/workflows/stack127-frontend-refresh.yml` from queue changes. A sync commit can otherwise start a build before history repair finishes. The static builder correctly rejects a changing source revision. Leave database triggers enabled and keep their queue entries. + +From the operator's local checkout, pause frontend dispatch and save the existing OVH crontab. These commands preserve unrelated cron entries. Use the same saved filename when resuming. + +```bash +# Local shell. The implementation agent must not run these production mutations. +export ADN_WINDOW=$(date -u +%Y%m%dT%H%M%SZ) +ssh ovh "crontab -l > /srv/stack127-data/state/crontab-before-homepage-$ADN_WINDOW.txt" +ssh ovh "sed '/^[^#].*scripts\/ad-networks\/ovh\/run-job.sh/s/^/# homepage-maintenance /' /srv/stack127-data/state/crontab-before-homepage-$ADN_WINDOW.txt | crontab -" +ssh ovh 'sudo -n systemctl stop stack127-cache-worker.timer' +# Let a currently executing worker finish; do not kill it mid-dispatch. +ssh ovh 'while systemctl is-active --quiet stack127-cache-worker.service; do sleep 2; done' +``` + +Coordinate other writers and releases for the same window. The authorized operator pauses homelab niche imports with `ssh homelab 'systemctl --user stop stack127-ad-network-niche.timer'` and waits until `ssh homelab 'systemctl --user is-active stack127-ad-network-niche.service'` reports inactive. Do not kill an active importer. These are release instructions, not permission for the implementation agent to operate on homelab. Wait for existing OVH collectors, syncs and retry wrappers to finish; inspect with: + +```bash +ssh ovh 'ps -eo pid,etime,args | grep -E "[s]cripts/ad-networks/(ovh/run-job.sh|refresh.mjs|verify.mjs|sync.mjs|niche-import.mjs)"' +gh run list --repo stack127labs/stack127 --status in_progress +gh run list --repo stack127labs/stack127 --status queued +``` + +Wait for all frontend refresh and production release runs (including waiting/pending runs in Actions) before changing data. For each active run, use `gh run view RUN_ID --repo stack127labs/stack127` and wait for `completed`. Do not cancel a build during its deployment step. Do not dispatch another release during this window. Stopping the timer cannot prevent a person or a separate release workflow from starting a build. If other writers cannot pause, postpone the correction. There is no global pause flag for manual or release dispatches. This is a coordinated manual window. A restart can reactivate the enabled timer, so recheck these conditions after any restart. + +#### 2. Take and verify both backups + +The nightly ad-network `backup` job at 05:00 UTC calls `backup-local.mjs`, uses SQLite's online backup API, checks `pragma integrity_check`, and retains seven daily copies under `$AD_NETWORKS_STATE_DIRECTORY/backups`. It is **not a PostgreSQL backup**. Keep a separate timestamped maintenance SQLite copy so a later nightly backup cannot replace it. + +PostgreSQL has the separate enabled `stack127-db-backup.timer`. Its on-demand service calls `/srv/stack127-db/backup.sh`. The script requires `BACKUPS_ENABLED`, writes a custom dump plus the Vault root key and password-free roles, checks the archive contents and key size, then publishes the complete bundle. An exit 0 with a missing marker is only a skip, not a usable backup. + +On OVH, in `/srv/stack127`, run: + +```bash +set -euo pipefail +umask 077 +set -a; . /srv/stack127-data/ad-networks.env; set +a +window=$(date -u +%Y%m%dT%H%M%SZ) +backup_dir=/srv/stack127-data/state/homepage-backup-$window +mkdir -m 700 "$backup_dir" +node scripts/ad-networks/backup-local.mjs --local-database "$AD_NETWORK_DB_PATH" --backup-directory "$backup_dir" --keep 7 +node --input-type=module - "$backup_dir" <<'JS' +import {DatabaseSync} from 'node:sqlite'; +import {readdirSync} from 'node:fs'; +const folder=process.argv[2], files=readdirSync(folder).filter(f=>f.endsWith('.sqlite')); +if(files.length!==1)throw Error('Expected one maintenance backup'); +const db=new DatabaseSync(folder+'/'+files[0],{readOnly:true}); +if(db.prepare('pragma integrity_check').get().integrity_check!=='ok')throw Error('Bad SQLite backup'); +console.log(db.prepare('select count(*) relationships from relationships').get());db.close(); +JS +sudo -n test -f /srv/stack127-db/BACKUPS_ENABLED +sudo -n systemctl start stack127-db-backup.service +sudo -n systemctl show stack127-db-backup.service -p Result -p ExecMainStatus +sudo -n journalctl -u stack127-db-backup.service -n 15 --no-pager +# Require a NEW 'Backup complete' line from this invocation and Result=success. +pg_backup=$(ls -t /srv/stack127-db/backups/stack127-*.dump | head -1) +test -s "$pg_backup" +test -s "$pg_backup.roles.sql" +test "$(wc -c < "$pg_backup.vault-key")" -eq 64 +sudo -n docker exec -i --user postgres stack127-db pg_restore --list < "$pg_backup" > "$backup_dir/postgres.toc" +# Decompress every affected table's archived rows, without executing them. +sudo -n docker exec -i --user postgres stack127-db pg_restore --data-only --schema=public \ + --table=ad_network_publishers --table=ad_network_publisher_relationships \ + --table=ad_network_relationship_events --table=ad_network_check_observations \ + --table=ad_network_daily_counts --file=- < "$pg_backup" > "$backup_dir/affected-tables.sql" +test -s "$backup_dir/affected-tables.sql" +chmod 600 "$backup_dir/postgres.toc" "$backup_dir/affected-tables.sql" +printf 'Keep SQLite folder %s and complete PostgreSQL bundle %s until acceptance.\n' "$backup_dir" "$pg_backup" +``` + +Verify the table list and archived rows are populated. This checks archive readability, not a fresh full restore. A full restore was tested by the existing backup workflow; if the archive check fails or its recovery evidence is no longer trusted, stop and validate a restore in an isolated database first. Never test restore into production. Keep the dump, key and roles together, with mode 600. Historical snapshot deletion is irreversible without these backups. Do not proceed with a skipped, old, empty or unreadable backup. + +#### 3. Install reviewed code and the checkpoint schema + +Install the reviewed release on OVH using the normal release procedure. Check its SHA. Apply migration `20261113120000_ad_network_homepage_miss_repairs.sql` through the normal release migration step only. Do not apply it by hand: a direct `psql` run creates the table without recording the migration, and a later migration push then fails because the table exists. If the release did not apply it, stop and fix the release. + +Confirm the table exists first with `to_regclass('public.ad_network_homepage_miss_repairs')`. The table grants only backend access. SQLite write checks require the loaded `AD_NETWORKS_RUNTIME_ENVIRONMENT=production` and the canonical `AD_NETWORK_DB_PATH` to match `--sqlite`; production also requires the confirmation flag. + +#### 4. Repair 25 domains, sync, then repair history + +Still on OVH in `/srv/stack127`, with the env loaded and the maintenance window active: + +```bash +node scripts/ad-networks/correct-homepage-misses.mjs --environment production --store sqlite --sqlite "$AD_NETWORK_DB_PATH" --limit 25 +node scripts/ad-networks/correct-homepage-misses.mjs --environment production --store sqlite --sqlite "$AD_NETWORK_DB_PATH" --limit 25 --apply --confirm stack127-production +node scripts/ad-networks/sync.mjs --environment production --confirm stack127-production --sqlite "$AD_NETWORK_DB_PATH" --state-file "$AD_NETWORKS_STATE_DIRECTORY/sync-state.sqlite" --apply +node scripts/ad-networks/correct-homepage-misses.mjs --environment production --stats-only --limit 25 --history-limit 25 +node scripts/ad-networks/correct-homepage-misses.mjs --environment production --stats-only --limit 25 --history-limit 25 --apply --confirm stack127-production +``` + +Review each preview. If history reports `history_limit`, repeat history with its `nextAfterDomain` as `--after-domain DOMAIN` until all pending corrections are repaired. Completed rows are checkpointed, so a restart without a cursor is also safe. Keep the **source apply's** `nextAfterDomain` for the next source batch; source and history cursors are separate. Never repair PostgreSQL alone: SQLite owns verification and would overwrite it. Production PostgreSQL apply therefore requires `--stats-only`. + +If interrupted: repeat source selection without advancing its cursor, sync with the same state file, then rerun stats-only. Each write stage commits atomically; the PostgreSQL repair checkpoint commits with events and snapshots. A crash before commit rolls all of that stage back. A crash after commit leaves a checkpoint that the next run skips. + +#### 5. Refresh and verify the first public result + +Run the read-only verification SQL below. Require no remaining invalid departures for repaired markers, zero broken restorations immediately after source repair, and a checkpoint for every synced correction marker. Rebuild heading counts and origin data only after history is complete: + +```bash +node scripts/ad-networks/revalidate.mjs --environment production --confirm stack127-production --apply +``` + +Keep the worker paused and collectors/imports stopped. From the local operator shell, dispatch **one** frontend refresh, then inspect its Actions run until it completes successfully: + +```bash +gh workflow run stack127-frontend-refresh.yml --repo stack127labs/stack127 --ref main -f force=true +gh run list --repo stack127labs/stack127 --workflow stack127-frontend-refresh.yml --limit 3 +gh run view RUN_ID --repo stack127labs/stack127 +curl --fail --silent --show-error https://stack127.com/frontend-release.json +``` + +Verify the completed refresh SHA matches released `main`, open `/ad-network-checker` for restored sample domains, the affected network pages, and `/ad-networks/stats`. Check restored relationships and original first-confirmed dates, last successful checked dates, movement counts and missing historical days. Check a blocked sample stays unresolved; a dry run is not a restoration. Require the workflow's public verification/acknowledgement step to succeed. Do not start the next data batch until this refresh has completed. + +#### 6. Continue in 100-domain batches, then resume + +Use `--limit 100` for source preview/apply with the prior source apply's `--after-domain DOMAIN`. Use `--history-limit 100 --limit 100` for history preview/apply. For the next batch, set `source_cursor` to the source apply's actual `nextAfterDomain` (not the history cursor), then run: + +```bash +source_cursor='DOMAIN_FROM_SOURCE_APPLY' +node scripts/ad-networks/correct-homepage-misses.mjs --environment production --store sqlite --sqlite "$AD_NETWORK_DB_PATH" --limit 100 --after-domain "$source_cursor" +node scripts/ad-networks/correct-homepage-misses.mjs --environment production --store sqlite --sqlite "$AD_NETWORK_DB_PATH" --limit 100 --after-domain "$source_cursor" --apply --confirm stack127-production +node scripts/ad-networks/sync.mjs --environment production --confirm stack127-production --sqlite "$AD_NETWORK_DB_PATH" --state-file "$AD_NETWORKS_STATE_DIRECTORY/sync-state.sqlite" --apply +node scripts/ad-networks/correct-homepage-misses.mjs --environment production --stats-only --limit 100 --history-limit 100 +node scripts/ad-networks/correct-homepage-misses.mjs --environment production --stats-only --limit 100 --history-limit 100 --apply --confirm stack127-production +``` + +Repeat sync and history repair after each source batch. Keep dispatch paused during mutations. Repeat step 5 once all repairs are finished and wait for that refresh before resuming writers. To inspect any intermediate 100-domain batch publicly, finish its entire sync/history sequence, perform one refresh, and wait for completion before the next mutation. + +Once public acceptance passes, restore the saved crontab from the local operator shell. If the niche timer was active before this window, the operator resumes it with `ssh homelab 'systemctl --user start stack127-ad-network-niche.timer'`. Preserve any unrelated cron edits made during the window; compare the saved and current crontabs before restoring. + +```bash +ssh ovh "crontab /srv/stack127-data/state/crontab-before-homepage-$ADN_WINDOW.txt" +ssh ovh 'sudo -n systemctl start stack127-cache-worker.timer' +ssh ovh 'systemctl is-active stack127-cache-worker.timer' +``` + +Check the durable `ad_networks/all` queue event drains after a successful refresh and the next hourly verification exits successfully. Do not remove the backups until acceptance. The whole false-loss spike **will not disappear**: blocked checks, changed signals and historical misses without proof remain unresolved. Missing old snapshots mean unknown history, not zero losses. + +For each restored relationship, history repair deletes only its exact invalid `left` event. An apparent return within that correction window becomes `baseline`. It adds the restored network to the existing observation so the next check cannot manufacture a join. + +Old per-network daily snapshots inside the affected miss window are removed. Individual stale-transition times were never stored, so their confirmed/stale split cannot be reconstructed honestly. Today's counts are rebuilt from corrected relationships. Missing old days remain unknown. The stats page reads net change and switches from events, and history/coverage from daily counts. This correction changes both sources consistently, without invented history. + +The existing relationship, event, and daily-count triggers update the source revision, touch published ad-network pages, and queue the `ad_networks/all` cache refresh. Keep those triggers enabled. No new job or cron schedule is needed. The monitoring descriptions mention the new inconclusive rule. + +### Implementation measurements and limits + +Read-only OVH fetches on November 13 reproduced HTTP 200 challenges from `tawcan.com` (338 ms) and `nomadicmatt.com` (181 ms), and an HTTP 403 challenge from `iwandered.net` (105 ms). The updated detector classifies all three as challenges. Separate paired ads.txt requests took 1,309 ms, 2,349 ms and 5,275 ms respectively. Tawcan's ads.txt was invalid; the other two returned parsed 6,808- and 6,806-character files. None supplied ETag or Last-Modified, so that sample could not demonstrate a 304 bandwidth reduction. Full production batch throughput, expiry counts and correction totals under the new code remain unmeasured. The ten-minute admission limit permits already-started requests to finish; it is not an immediate request cancellation deadline. + +### Read-only verification + +Run in a read-only transaction on the selected target: + +```sql +begin read only; + +select n.slug, r.status, count(*) +from public.ad_network_publisher_relationships r +join public.ad_networks n on n.id=r.ad_network_id +where r.evidence->'homepage_miss_correction'->>'version'='homepage-misses-v1' +group by n.slug,r.status order by n.slug,r.status; + +-- Must be zero: exact invalid departures still affecting movements. +select count(*) as remaining_invalid_departures +from public.ad_network_publisher_relationships r +join public.ad_network_relationship_events e using(ad_network_id,publisher_id) +where r.evidence->'homepage_miss_correction'->>'version'='homepage-misses-v1' +and e.event='left' +and e.observed_at=(r.evidence->'homepage_miss_correction'->>'prior_ended_at')::timestamptz; + +-- Must be zero immediately after correction; later real checks can change it. +select count(*) as broken_restorations +from public.ad_network_publisher_relationships r +where r.evidence->'homepage_miss_correction'->>'version'='homepage-misses-v1' +and (r.status<>'confirmed' or r.consecutive_misses<>0 or r.ended_at is not null + or r.first_confirmed_at is distinct from + (r.evidence->'homepage_miss_correction'->>'original_first_confirmed_at')::timestamptz); + +select n.slug,d.snapshot_on,d.listed_count,d.confirmed_count,d.stale_count +from public.ad_network_daily_counts d join public.ad_networks n on n.id=d.ad_network_id +where d.snapshot_on=(now() at time zone 'UTC')::date +and exists(select 1 from public.ad_network_publisher_relationships r + where r.ad_network_id=n.id and r.evidence ? 'homepage_miss_correction') +order by n.slug; + +-- Must be zero before refresh/resume: uncheckpointed synced corrections. +select count(*) as pending_history_repairs +from public.ad_network_publisher_relationships r +where r.evidence->'homepage_miss_correction'->>'version'='homepage-misses-v1' +and not exists(select 1 from public.ad_network_homepage_miss_repairs h + where h.ad_network_id=r.ad_network_id and h.publisher_id=r.publisher_id + and h.version='homepage-misses-v1' + and h.corrected_at=(r.evidence->'homepage_miss_correction'->>'corrected_at')::timestamptz); + +select revision from public.stack127_ad_network_source_revision; +select entity_type,slug,attempts,last_error +from public.stack127_revalidation_events +where entity_type='ad_networks' +order by slug; + +rollback; +``` diff --git a/lib/ad-networks.ts b/lib/ad-networks.ts index 7b6562cb..0d1bb609 100644 --- a/lib/ad-networks.ts +++ b/lib/ad-networks.ts @@ -203,7 +203,7 @@ const managerEvidenceProjection = `jsonb_build_object('manager_domains',p.last_c where r.publisher_id=p.id and r.status='confirmed' and r.relationship_type='primary_or_exclusive_manager'), '[]'::jsonb) as manager_relationships`; const siteProjection = `p.normalized_domain as domain, p.publisher_name, p.niche, p.traffic_group, - p.domain_created_on::text, p.platform, p.last_checked_at, p.favicon,${managerEvidenceProjection}, + p.domain_created_on::text, p.platform, p.last_successful_check_at as last_checked_at, p.favicon,${managerEvidenceProjection}, coalesce((select jsonb_agg(jsonb_build_object('slug',n.slug,'name',n.name) order by n.name) from public.ad_network_publisher_relationships r join public.ad_networks n on n.id=r.ad_network_id where r.publisher_id=p.id and r.status in ('confirmed','stale')),'[]'::jsonb) as networks`; @@ -275,7 +275,7 @@ export async function loadAdNetworkSitesAvailability(network: string | null, nic const coverageWhere = `exists(select 1 from public.ad_network_publisher_relationships r join public.ad_networks n on n.id=r.ad_network_id where r.publisher_id=p.id and r.status in ('confirmed','stale') and ($1::text is null or n.slug=$1)) and ($2::text is null or p.niche=$2)`; -const scopedCoverageSites = `with sites as materialized (select p.id,p.niche,p.traffic_group,p.domain_created_on,p.platform,p.publisher_name,p.last_checked_at +const scopedCoverageSites = `with sites as materialized (select p.id,p.niche,p.traffic_group,p.domain_created_on,p.platform,p.publisher_name,p.last_successful_check_at as last_checked_at from public.ad_network_publishers p where ${coverageWhere})`; const scopedManagerSites = `with sites as materialized (select p.id, jsonb_build_object('manager_domains',p.report_manager_domains) as last_check_evidence @@ -288,7 +288,7 @@ const siteSort: Record = { domain: "p.normalized_domain", traffic: "case p.traffic_group when 'top-1k' then 0 when 'top-10k' then 1 when 'top-100k' then 2 when 'top-1m' then 3 when 'not-ranked' then 4 else 5 end,p.domain_created_on asc nulls last,p.normalized_domain,p.id", oldest: "p.domain_created_on asc nulls last,p.normalized_domain", newest: "p.domain_created_on desc nulls last,p.normalized_domain", - checked: "p.last_checked_at desc nulls last,p.normalized_domain", + checked: "p.last_successful_check_at desc nulls last,p.normalized_domain", }; /** Uncached source loader for signed origin verification. */ export async function loadAdNetworkSites(query: AdNetworkSitesQuery): Promise { @@ -307,7 +307,7 @@ export async function loadAdNetworkSites(query: AdNetworkSitesQuery): Promise(`with matches as materialized ( - select p.id,p.normalized_domain,p.traffic_group,p.domain_created_on,p.last_checked_at + select p.id,p.normalized_domain,p.traffic_group,p.domain_created_on,p.last_successful_check_at from public.ad_network_publishers p where ${siteWhere} ), selected as materialized ( select p.id from matches p order by ${siteSort[effectiveSort]} limit $4 offset $5 @@ -680,7 +680,7 @@ export const getAdNetworkStatsPageData = cached("ad-network-market-report", asyn const pool={query:queryAdNetworkReport}; const observedDay=new Date(); const asOf=observedDay.toISOString().slice(0,10); - const active=`with active as materialized (select p.last_checked_at from public.ad_network_publishers p where exists( + const active=`with active as materialized (select p.last_successful_check_at as last_checked_at from public.ad_network_publishers p where exists( select 1 from public.ad_network_publisher_relationships r where r.publisher_id=p.id and r.status in ('confirmed','stale')))`; const activeAds=`with active as materialized (select p.report_ads_txt_lines as lines, p.report_ads_txt_invalid as invalid from public.ad_network_publishers p @@ -772,7 +772,7 @@ function joiningRules(networks: AdNetworkRef[]): AdNetworkJoiningRule[] { const loadReportMetadata=cached("ad-network-report-metadata",async (network: string | null) => { const [updated,methods]=await Promise.all([ - queryAdNetworkReport<{updated_at:Date|null}>(`select max(p.last_checked_at) as updated_at from public.ad_network_publishers p + queryAdNetworkReport<{updated_at:Date|null}>(`select max(p.last_successful_check_at) as updated_at from public.ad_network_publishers p where exists(select 1 from public.ad_network_publisher_relationships r join public.ad_networks n on n.id=r.ad_network_id where r.publisher_id=p.id and r.status in ('confirmed','stale') and ($1::text is null or n.slug=$1))`,[network]), queryAdNetworkReport(`select r.verification_method as key,r.verification_method as label,count(*)::int as count diff --git a/scripts/ad-networks/correct-homepage-misses.mjs b/scripts/ad-networks/correct-homepage-misses.mjs new file mode 100644 index 00000000..98359924 --- /dev/null +++ b/scripts/ad-networks/correct-homepage-misses.mjs @@ -0,0 +1,307 @@ +#!/usr/bin/env node +import { parseArgs } from 'node:util'; +import { realpathSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { DatabaseSync } from 'node:sqlite'; +import { targetPool } from './runtime-db.mjs'; +import { loadLocalNetworks } from './local-db.mjs'; +import { assertLocalCatalog } from './identity-catalog.mjs'; +import { verifyPublisher } from './verify-publisher.mjs'; +import { snapshotCounts } from './snapshot.mjs'; + +// The code predates the production runner. The earliest affected homepage-method +// miss retained in either store is 2026-09-28 (not the November loss spike). +export const FAULT_SINCE = '2026-09-28T00:00:00.000Z'; +export const CORRECTION = 'homepage-misses-v1'; +const methods = new Set(['ads_txt_and_adsense_account', 'ads_txt_and_page_signature', 'page_signature']); +const json = value => typeof value === 'string' ? JSON.parse(value) : value ?? {}; +const iso = value => value == null ? null : new Date(value).toISOString(); + +export function correctionEligibility(row, since = FAULT_SINCE) { + const evidence = json(row.evidence), check = json(row.last_check_evidence); + if (evidence.homepage_miss_correction?.version === CORRECTION) return row.history_repaired_at ? 'history_complete' : 'already_restored'; + if (!methods.has(row.verification_method)) return 'not_homepage_dependent'; + if (!row.first_confirmed_at || !row.last_confirmed_at || row.relationship_type === 'unknown' + || !['checked_url','checked_at','matched_signal','parser_version'].every(key => Object.hasOwn(evidence,key))) return 'no_original_confirmation'; + if (evidence.homepage_miss_correction?.version === CORRECTION && row.status === 'confirmed' + && !row.consecutive_misses) return 'already_restored'; + if (!['confirmed','stale','ended'].includes(row.status) || !row.consecutive_misses) return 'not_damaged'; + if (!row.first_missed_at || new Date(row.first_missed_at).getTime() < Date.parse(since)) return 'before_fault_window'; + // A new verifier records obtained evidence of absence. Never undo that. + if (evidence.last_absence) return 'recorded_absence'; + if (new Date(row.last_checked_at).getTime() !== new Date(row.last_missed_at).getTime()) return 'later_or_unknown_check'; + // Historical full-fetch misses have no homepage outcome. They cannot be + // distinguished from genuine absences. A 304 proves this miss had no page. + if (row.last_check_outcome !== 'not_modified' && check.homepage?.outcome !== 'inconclusive') { + return 'no_proof_of_inconclusive_miss'; + } + return 'needs_live_confirmation'; +} + +export function correctionConfirmation(row, result) { + const match = result.matches?.confirmed.find(match => String(match.networkId) === String(row.ad_network_id)); + if (!match) return result.inconclusiveNetworkIds?.length ? 'live_check_inconclusive' : 'live_network_absent'; + const original = json(row.evidence); + if (original.page_signature) { + // Review-promoted evidence requires BOTH the exact authorized seller and + // its configured live tag. A matching generic tag alone is insufficient. + const seller = original.matched_signal?.split(',').map(value => value.trim()); + const current = result.evidence?.ads_txt_cache?.parsed?.sellers ?? []; + if (!seller || seller.length !== 3 || !current.some(item => item.adSystem === seller[0] + && item.sellerId === seller[1] && item.relationship === seller[2]) + || match.evidence.matched_signal !== original.page_signature) return 'live_signal_changed'; + } else if (match.evidence.matched_signal !== original.matched_signal) return 'live_signal_changed'; + const oldHash = json(row.evidence).ads_txt_sha256; + if (oldHash && oldHash !== result.sha256) return 'ads_txt_changed_since_confirmation'; + return 'restore'; +} + +export function restoration(row, checkedAt, confirmation = null) { + return { + ...json(row.evidence), + homepage_miss_correction: { + version: CORRECTION, corrected_at: checkedAt, confirmation, + prior_status: row.status, prior_ended_at: iso(row.ended_at), + first_missed_at: iso(row.first_missed_at), last_missed_at: iso(row.last_missed_at), + original_first_confirmed_at: iso(row.first_confirmed_at), + }, + }; +} + +export async function planCorrections(rows, networks, { since = FAULT_SINCE, limit = 100, historyLimit = limit, afterDomain = '', statsOnly = false, verify = verifyPublisher } = {}) { + const counts = {}, restores = [], repaired = []; + const bump = reason => { counts[reason] = (counts[reason] ?? 0) + 1; }; + let checked = 0, nextAfterDomain = afterDomain; + const checkedDomains = new Map(), historyDomains = new Set(); + let nextHistoryAfterDomain = afterDomain; + // Bound work and issue requests one domain at a time; no protection bypass. + for (const row of rows) { + const reason = correctionEligibility(row, since); + if (row.normalized_domain <= afterDomain) { bump('before_cursor'); continue; } + if (reason === 'already_restored') { + if (!historyDomains.has(row.normalized_domain) && historyDomains.size >= historyLimit) { bump('history_limit'); continue; } + historyDomains.add(row.normalized_domain); nextHistoryAfterDomain = row.normalized_domain; + repaired.push(row); bump(reason); continue; + } + if (reason !== 'needs_live_confirmation') { bump(reason); continue; } + if (statsOnly) { bump('stats_only'); continue; } + if (!checkedDomains.has(row.normalized_domain) && checked >= limit) { bump('live_check_limit'); continue; } + let result = checkedDomains.get(row.normalized_domain); + if (!result) { + checked++; + nextAfterDomain = row.normalized_domain; + result = await verify({ + normalized_domain: row.normalized_domain, discovery_sources: json(row.discovery_sources || '[]'), + last_check_evidence: {}, review_relationships: [], + }, networks); + checkedDomains.set(row.normalized_domain,result); + } + const decision = correctionConfirmation(row, result); + bump(decision); + if (decision === 'restore') restores.push({ ...row, correction_evidence: restoration(row, result.checkedAt, result.matches.confirmed.find(match => String(match.networkId) === String(row.ad_network_id)).evidence) }); + } + return { counts, checked, restores, repaired, nextAfterDomain: statsOnly ? nextHistoryAfterDomain : nextAfterDomain, nextHistoryAfterDomain }; +} + +export async function previewPostgresCorrection(client, plan) { + const repairs = [...plan.restores,...plan.repaired].map(row => { + const marker = (row.correction_evidence ?? json(row.evidence)).homepage_miss_correction; + return { ad_network_id:row.ad_network_id,publisher_id:row.publisher_id, + ended_at:marker.prior_ended_at,first_missed_at:marker.first_missed_at,corrected_at:marker.corrected_at }; + }); + if (!repairs.length) return { wouldRemoveEvents:0,wouldNeutralizeArrivals:0,wouldInvalidateSnapshots:0 }; + return (await client.query(`with repairs as ( + select * from jsonb_to_recordset($1::jsonb) x(ad_network_id bigint,publisher_id bigint, + ended_at timestamptz,first_missed_at timestamptz,corrected_at timestamptz) + ) select + (select count(*)::int from public.ad_network_relationship_events e join repairs r using(ad_network_id,publisher_id) + where e.event='left' and e.observed_at=r.ended_at) as "wouldRemoveEvents", + (select count(*)::int from public.ad_network_relationship_events e join repairs r using(ad_network_id,publisher_id) + where e.event='joined' and e.observed_at between r.ended_at and r.corrected_at) as "wouldNeutralizeArrivals", + (select count(distinct (d.ad_network_id,d.snapshot_on))::int from public.ad_network_daily_counts d + join repairs r using(ad_network_id) where d.captured_at between r.first_missed_at and r.corrected_at + and d.snapshot_on < (now() at time zone 'UTC')::date) as "wouldInvalidateSnapshots"`,[JSON.stringify(repairs)])).rows[0]; +} + +export async function applyPostgresCorrection(client, plan) { + let completed = 0, restored = 0, queued = 0, removedEvents = 0, neutralizedArrivals = 0, invalidatedSnapshots = 0, observations = 0; + const rows = [...plan.restores, ...plan.repaired]; + for (const row of rows) { + const evidence = row.correction_evidence ?? json(row.evidence); + const marker = evidence.homepage_miss_correction; + if ((await client.query(`select 1 from public.ad_network_homepage_miss_repairs + where ad_network_id=$1 and publisher_id=$2 and corrected_at=$3 and version=$4`, + [row.ad_network_id,row.publisher_id,marker.corrected_at,CORRECTION])).rowCount) continue; + if (row.correction_evidence) { + const update = await client.query(`update public.ad_network_publisher_relationships + set status='confirmed',ended_at=null,consecutive_misses=0,first_missed_at=null,last_missed_at=null,evidence=$3::jsonb + where ad_network_id=$1 and publisher_id=$2 and status=$4 and last_missed_at=$5 + and first_confirmed_at=$6 and evidence=$7::jsonb`, + [row.ad_network_id,row.publisher_id,JSON.stringify(evidence),row.status,row.last_missed_at,row.first_confirmed_at,JSON.stringify(json(row.evidence))]); + if (update.rowCount !== 1) throw new Error('Relationship changed during live check; rerun correction'); + restored += update.rowCount; + } + if (!row.correction_evidence) { + const current = await client.query(`select 1 from public.ad_network_publisher_relationships + where ad_network_id=$1 and publisher_id=$2 + and evidence->'homepage_miss_correction'=$3::jsonb for update`,[row.ad_network_id,row.publisher_id,JSON.stringify(marker)]); + if (!current.rowCount) throw new Error('Corrected relationship changed during selection; rerun correction'); + } + // Exact departure only. Other departures and other networks remain intact. + removedEvents += (await client.query(`delete from public.ad_network_relationship_events + where ad_network_id=$1 and publisher_id=$2 and event='left' and observed_at=$3`, + [row.ad_network_id,row.publisher_id,marker.prior_ended_at])).rowCount; + // An apparent return after the invalid departure is baseline history. + if (marker.prior_ended_at) { + neutralizedArrivals += (await client.query(`delete from public.ad_network_relationship_events j + where j.ad_network_id=$1 and j.publisher_id=$2 and j.event='joined' and j.observed_at between $3 and $4 + and exists(select 1 from public.ad_network_relationship_events b where b.ad_network_id=j.ad_network_id + and b.publisher_id=j.publisher_id and b.event='baseline' and b.observed_at=j.observed_at)`, + [row.ad_network_id,row.publisher_id,marker.prior_ended_at,marker.corrected_at])).rowCount; + neutralizedArrivals += (await client.query(`update public.ad_network_relationship_events set event='baseline' + where ad_network_id=$1 and publisher_id=$2 and event='joined' and observed_at between $3 and $4 + and not exists(select 1 from public.ad_network_relationship_events b where b.ad_network_id=$1 + and b.publisher_id=$2 and b.event='baseline' and b.observed_at=ad_network_relationship_events.observed_at)`, + [row.ad_network_id,row.publisher_id,marker.prior_ended_at,marker.corrected_at])).rowCount; + } + // There is no history of individual miss times. Do not invent old + // confirmed/stale totals. Remove uncertain network/day snapshots and + // replace today's snapshot from current corrected relationships below. + invalidatedSnapshots += (await client.query(`delete from public.ad_network_daily_counts + where ad_network_id=$1 and captured_at between $2 and $3 + and snapshot_on < (now() at time zone 'UTC')::date`, + [row.ad_network_id,marker.first_missed_at,marker.corrected_at])).rowCount; + observations += (await client.query(`update public.ad_network_check_observations + set confirmed_network_ids=array_append(confirmed_network_ids,$1::bigint) + where publisher_id=$2 and not ($1::bigint=any(confirmed_network_ids))`, + [row.ad_network_id,row.publisher_id])).rowCount; + if (row.correction_evidence) queued += (await client.query(`update public.ad_network_publishers set verification_priority=100,next_check_at=now() + where id=$1 and (verification_priority<>100 or next_check_at is null or next_check_at>now())`,[row.publisher_id])).rowCount; + completed++; + await client.query(`insert into public.ad_network_homepage_miss_repairs + (ad_network_id,publisher_id,corrected_at,version) values($1,$2,$3,$4) on conflict do nothing`, + [row.ad_network_id,row.publisher_id,marker.corrected_at,CORRECTION]); + } + const snapshots = completed + ? (await snapshotCounts(client)).rowCount : 0; + return { historyRepaired: completed, restored, queued, removedEvents, neutralizedArrivals, invalidatedSnapshots, observations, snapshots }; +} + +export function applySqliteCorrection(db, plan) { + let restored = 0; + for (const row of plan.restores) { + const update = db.prepare(`update relationships set status='confirmed',ended_at=null,consecutive_misses=0, + first_missed_at=null,last_missed_at=null,evidence=?,updated_at=? + where ad_network_id=? and publisher_id=? and status=? and last_missed_at=? and evidence=?`) + .run(JSON.stringify(row.correction_evidence),new Date().toISOString(),row.ad_network_id,row.publisher_id,row.status, + row.last_missed_at,typeof row.evidence === 'string' ? row.evidence : JSON.stringify(row.evidence)); + if (update.changes !== 1) throw new Error('Relationship changed during live check; rerun correction'); + restored += update.changes; + db.prepare('update publishers set verification_priority=100,next_check_at=?,updated_at=? where id=?') + .run(new Date().toISOString(),new Date().toISOString(),row.publisher_id); + } + return { restored, queued: restored }; +} + +export function assertCorrectionStore(options, env = process.env) { + if (env.AD_NETWORKS_RUNTIME_ENVIRONMENT && env.AD_NETWORKS_RUNTIME_ENVIRONMENT !== options.environment) { + throw new Error('Runtime database environment mismatch'); + } + if (options.store !== 'sqlite' || !options.apply) return; + if (!env.AD_NETWORKS_RUNTIME_ENVIRONMENT || !env.AD_NETWORK_DB_PATH) { + throw new Error('SQLite writes require AD_NETWORKS_RUNTIME_ENVIRONMENT and AD_NETWORK_DB_PATH'); + } + if (!options.sqlite || realpathSync(options.sqlite) !== realpathSync(env.AD_NETWORK_DB_PATH)) { + throw new Error('SQLite source path does not match AD_NETWORK_DB_PATH'); + } + const source = realpathSync(options.sqlite); + if (options.environment !== 'production' && (source === '/srv/stack127-data/ad-networks.sqlite' + || resolve(options.sqlite) === '/srv/stack127-data/ad-networks.sqlite')) { + throw new Error('Production SQLite source requires --environment production'); + } +} + +export async function runCorrection(options) { + const since = options.since ?? FAULT_SINCE, limit = Number(options.limit ?? 100); + const historyLimit = Number(options['history-limit'] ?? limit); + if (!Number.isInteger(historyLimit) || historyLimit < 1 || historyLimit > 100000) throw new Error('Invalid --history-limit (1–100000)'); + if (!Number.isFinite(Date.parse(since)) || !Number.isInteger(limit) || limit < 1 || limit > 100000) throw new Error('Invalid --since or --limit (1–100000)'); + if (!['development','production'].includes(options.environment)) throw new Error('Invalid environment'); + if (options.apply && options.environment === 'production' && options.confirm !== 'stack127-production') throw new Error('Production writes require --confirm stack127-production'); + if (!['postgres','sqlite'].includes(options.store)) throw new Error('--store must be postgres or sqlite'); + if (options.apply && options.environment === 'production' && options.store === 'postgres' && !options['stats-only']) { + throw new Error('Production PostgreSQL correction requires --stats-only after repairing and syncing SQLite'); + } + assertCorrectionStore(options); + let pool, client, db, transaction = false; + try { + let rows, networks; + if (options.store === 'sqlite') { + if (!options.sqlite) throw new Error('--sqlite is required'); + db = new DatabaseSync(options.sqlite, { readOnly: !options.apply, timeout: 10000 }); + if (!options.apply) db.exec('pragma query_only=on'); + assertLocalCatalog(db); + // Network checks run before the write transaction; guarded updates below + // reject a source row changed by another verifier in the meantime. + rows = db.prepare(`select r.*,p.normalized_domain,p.last_checked_at,p.last_check_outcome,p.last_check_evidence, + '[]' as discovery_sources,n.slug from relationships r join publishers p on p.id=r.publisher_id + join ad_networks n on n.id=r.ad_network_id + where r.consecutive_misses>0 or json_extract(r.evidence,'$.homepage_miss_correction.version')=? + order by p.normalized_domain,n.slug`).all(CORRECTION); + for (const row of rows) row.discovery_sources = db.prepare('select source_key from publisher_sources where publisher_id=?').all(row.publisher_id); + networks = loadLocalNetworks(db); + } else { + pool = targetPool(options); client = await pool.connect(); + await client.query('begin read only'); transaction = true; + rows = (await client.query(`select r.*,p.normalized_domain,p.last_checked_at,p.last_check_outcome,p.last_check_evidence, + p.discovery_sources,n.slug,h.repaired_at as history_repaired_at from public.ad_network_publisher_relationships r + join public.ad_network_publishers p on p.id=r.publisher_id join public.ad_networks n on n.id=r.ad_network_id + left join public.ad_network_homepage_miss_repairs h on h.ad_network_id=r.ad_network_id + and h.publisher_id=r.publisher_id and h.version='homepage-misses-v1' + and h.corrected_at=(r.evidence->'homepage_miss_correction'->>'corrected_at')::timestamptz + where r.consecutive_misses>0 or r.evidence->'homepage_miss_correction'->>'version'=$1 + order by p.normalized_domain,n.slug`,[CORRECTION])).rows; + networks = (await client.query('select * from public.ad_networks order by id')).rows; + } + // Do not hold an idle transaction or row locks while fetching public sites. + if (client) { await client.query('commit'); transaction = false; } + const plan = await planCorrections(rows, networks, { since, limit, historyLimit, afterDomain: options['after-domain'] ?? '', statsOnly: options['stats-only'] }); + let applied = {}, preview = {}; + if (client && !options.apply) { + await client.query('begin read only'); transaction = true; + preview = await previewPostgresCorrection(client,plan); + await client.query('rollback'); transaction = false; + } + if (options.apply) { + if (db) db.exec('begin immediate'); + else { + await client.query('begin'); + await client.query("set local lock_timeout='5s'"); + } + transaction = true; + if (client) for (const name of ['ad-networks-runtime-heavy','ad-networks-reviewed-sync']) { + if (!(await client.query('select pg_try_advisory_xact_lock(hashtext($1)) locked',[name])).rows[0].locked) throw new Error('Another ad-network sync is running'); + } + applied = db ? applySqliteCorrection(db,plan) : await applyPostgresCorrection(client,plan); + } + if (transaction) { + if (db) db.exec(options.apply ? 'commit' : 'rollback'); + else await client.query(options.apply ? 'commit' : 'rollback'); + } + transaction = false; + return { environment: options.environment,store:options.store,applied:options.apply,since, + scanned:rows.length,checked:plan.checked,counts:plan.counts,nextAfterDomain:plan.nextAfterDomain,nextHistoryAfterDomain:plan.nextHistoryAfterDomain,wouldRestore:plan.restores.length, + wouldRepairStats:plan.restores.length+plan.repaired.length,...preview,...applied }; + } catch (error) { + if (transaction) { if (db) db.exec('rollback'); else await client.query('rollback'); } + throw error; + } finally { db?.close(); client?.release(); await pool?.end(); } +} +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const { values } = parseArgs({ options: { + environment:{type:'string',default:'development'},confirm:{type:'string'},apply:{type:'boolean',default:false}, + store:{type:'string',default:'postgres'},sqlite:{type:'string'},'after-domain':{type:'string',default:''},'stats-only':{type:'boolean',default:false},since:{type:'string',default:FAULT_SINCE},limit:{type:'string',default:'100'},'history-limit':{type:'string'}, + }}); + console.log(JSON.stringify(await runCorrection(values),null,2)); +} diff --git a/scripts/ad-networks/fetch-policy.mjs b/scripts/ad-networks/fetch-policy.mjs index 102ada56..51802b84 100644 --- a/scripts/ad-networks/fetch-policy.mjs +++ b/scripts/ad-networks/fetch-policy.mjs @@ -1,3 +1,4 @@ +import { parse } from "parse5"; import { createHash } from "node:crypto"; import dns from "node:dns/promises"; import { isIP } from "node:net"; @@ -276,14 +277,66 @@ export async function fetchAdsTxt(domain, conditional = {}, dependencies = {}) { return { ...httpsResult, candidateHostname: hostname, registrableDomain: root }; } +// Inspect active markup, never quoted examples, comments or inert HTML. +export function homepageChallenge(html) { + const pending = [parse(html)], titles = [], headings = [], scripts = [], frames = [], forms = [], elements = []; + const inert = new Set(['pre','code','blockquote','template','textarea','style','xmp','noscript','noembed','noframes']); + const text = node => (node.childNodes ?? []).map(child => child.nodeName === '#text' ? child.value : text(child)).join(' '); + while (pending.length) { + const node = pending.pop(); + if (inert.has(node.tagName) || node.nodeName === '#comment') continue; + const attrs = Object.fromEntries((node.attrs ?? []).map(({name,value}) => [name,value])); + if (Object.hasOwn(attrs,'hidden') || attrs['aria-hidden'] === 'true' || /display\s*:\s*none/i.test(attrs.style ?? '')) continue; + if (node.tagName === 'title') titles.push(text(node)); + if (/^h[1-3]$/.test(node.tagName)) headings.push(text(node)); + if (node.tagName === 'script') { + if (!attrs.type || /^(?:module|(?:text|application)\/(?:java|ecma)script)$/i.test(attrs.type)) scripts.push((attrs.src ?? '')+' '+text(node)); + continue; + } + if (node.tagName === 'iframe') { frames.push(attrs.src ?? ''); continue; } + if (node.tagName === 'form') forms.push([attrs.id,attrs.action,attrs.class].join(' ')); + if (node.tagName) elements.push([attrs.id,attrs.class,attrs['data-sitekey']].filter(Boolean).join(' ')); + pending.push(...(node.childNodes ?? [])); + } + const title = titles.join(' ').toLowerCase(), heading = headings.join(' ').toLowerCase(); + const wording = /(?:verif(?:y|ying|ication)(?:\s+that)?\s+(?:you(?:\s+are|'re)?|your|if you are)(?:\s+a)?\s+(?:human|browser)|(?:human|security|robot)\s+(?:check|verification)|checking (?:your )?browser|please (?:complete|solve).*(?:captcha|challenge))/i; + const waiting = /just a moment|one moment,? please|attention required|access denied|pardon our interruption|request (?:unsuccessful|rejected)|bot detection|enable javascript and cookies/i; + const active = scripts.join(' ').toLowerCase(), frame = frames.join(' ').toLowerCase(); + const form = forms.join(' ').toLowerCase(), markup = elements.join(' ').toLowerCase(); + const challengeWords = wording.test(title+' '+heading) || waiting.test(title); + const captcha = /(?:h-captcha|g-recaptcha|cf-turnstile)/.test(markup) + || /(?:hcaptcha\.com|recaptcha(?:\/|\.net)|google\.com\/recaptcha)/.test(active+' '+frame); + const humanHeading = headings.some(value => /^\s*(?:please\s+)?verify (?:that )?you(?: are|'re)? (?:a )?human[.!?\s]*$/i.test(value)); + if (wording.test(title) || humanHeading || (wording.test(heading) && (forms.length || captcha))) return true; + if (/challenge-form/.test(form) && (challengeWords || /(?:_cf_chl_opt|challenge-platform)/.test(active))) return true; + // Incapsula/DataDome/PerimeterX challenge frames are interstitial resources. + if (/(?:_incapsula_resource|captcha-delivery\.com|(?:^|[./])geo\.captcha-delivery\.com|px-captcha)/.test(frame)) return true; + if (challengeWords && (captcha || /(?:wsidchk|sucuri_cloudproxy|sucuri\.net|_incapsula_resource|incapsula|imperva|datadome|perimeterx|_pxappid|px-captcha|akamai|\/akam\/|\/sec-cpt\/|_cf_chl_opt|challenge-platform)/.test(active+' '+frame+' '+markup))) return true; + if (/sucuri_cloudproxy/.test(active) && /document\.cookie/.test(active) && /location/.test(active)) return true; + if (/px-captcha/.test(markup) && /(?:_pxappid|perimeterx)/.test(active)) return true; + if (/sec-cpt/.test(markup) && /(?:akamai|\/sec-cpt\/)/.test(active)) return true; + return titles.some(value => /^\s*(?:just a moment|one moment,? please|attention required|access denied)[.!…\s]*(?:\|\s*(?:cloudflare|sucuri))?\s*$/i.test(value)); +} + export async function fetchHomepage(domain, dependencies = {}) { - const root = getDomain(domain, { allowPrivateDomains: true }); - if (!root) return null; - const target = new URL(`https://${domain}/`); - const { response, finalUrl } = await safeFetch(target, { - headers: { Accept: "text/html,application/xhtml+xml" }, - timeoutMs: 15_000, - }, dependencies); - if (!response.ok || !/^text\/html(?:\s*;|$)/i.test(response.headers.get("content-type") ?? "")) { await response.body?.cancel(); return null; } - return { html: await readLimitedText(response, 2 * 1024 * 1024), finalUrl }; + const target = `https://${domain}/`; + let base = { checkedUrl: target, outcome: "inconclusive" }; + try { + if (!getDomain(domain, { allowPrivateDomains: true })) return { ...base, reason: "invalid_domain" }; + const { response, finalUrl } = await safeFetch(target, { + headers: { Accept: "text/html,application/xhtml+xml" }, + timeoutMs: 15_000, + }, dependencies); + base = { ...base, finalUrl, httpStatus: response.status, contentType: response.headers.get("content-type") }; + if (!response.ok || !/^text\/html(?:\s*;|$)|^application\/xhtml\+xml(?:\s*;|$)/i.test(base.contentType ?? "")) { + await response.body?.cancel(); + return { ...base, reason: !response.ok ? "http_status" : "content_type" }; + } + const html = await readLimitedText(response, 2 * 1024 * 1024); + if (!html.trim()) return { ...base, reason: "empty_body" }; + if (homepageChallenge(html)) return { ...base, reason: "challenge_page" }; + return { ...base, outcome: "fetched", html }; + } catch (error) { + return { ...base, reason: error.cause?.code ?? error.message }; + } } diff --git a/scripts/ad-networks/local-verifier-store.mjs b/scripts/ad-networks/local-verifier-store.mjs index 8aaa0e02..65119602 100644 --- a/scripts/ad-networks/local-verifier-store.mjs +++ b/scripts/ad-networks/local-verifier-store.mjs @@ -1,4 +1,4 @@ -import { nextCheckAt, relationshipAfterMiss } from "./verifier-store.mjs"; +import { nextCheckAt, relationshipAfterMiss, homepageConfirmationExpired } from "./verifier-store.mjs"; const SUCCESSFUL_OUTCOMES = new Set(["matched", "no_match", "no_ads_txt", "not_modified"]); @@ -91,7 +91,9 @@ function upsertMatches(database, publisherId, matches, checkedAt) { when excluded.status = 'needs_review' and relationships.status <> 'needs_review' then relationships.evidence when excluded.status = 'needs_review' and relationships.verification_method = 'discovery_hint_and_ads_txt_system' then json_set(excluded.evidence, '$.matched_signal', json_extract(relationships.evidence, '$.matched_signal')) - else excluded.evidence + else case when json_type(relationships.evidence,'$.homepage_miss_correction')='object' + then json_set(excluded.evidence,'$.homepage_miss_correction',json_extract(relationships.evidence,'$.homepage_miss_correction')) + else excluded.evidence end end, first_confirmed_at = case when excluded.status = 'confirmed' then coalesce(relationships.first_confirmed_at, excluded.first_confirmed_at) @@ -124,10 +126,11 @@ function upsertMatches(database, publisherId, matches, checkedAt) { } } -function ageMissingRelationships(database, existing, positiveIds, checkedAt) { +function ageMissingRelationships(database, existing, positiveIds, checkedAt, evidence) { const statement = database.prepare(` update relationships set status = ?, ended_at = ?, consecutive_misses = ?, - first_missed_at = coalesce(first_missed_at, ?), last_missed_at = ?, updated_at = ? + first_missed_at = coalesce(first_missed_at, ?), last_missed_at = ?, updated_at = ?, + evidence = json_set(evidence, '$.last_absence', json(?)) where ad_network_id = ? and publisher_id = ? `); for (const relationship of existing) { @@ -140,6 +143,7 @@ function ageMissingRelationships(database, existing, positiveIds, checkedAt) { checkedAt, checkedAt, checkedAt, + JSON.stringify({ checked_at: checkedAt, homepage: evidence?.homepage ?? null, ads_txt_sha256: evidence?.ads_txt_sha256 ?? null }), relationship.ad_network_id, relationship.publisher_id, ); @@ -150,16 +154,19 @@ export function saveLocalVerification(database, publisher, result) { database.exec("begin immediate"); try { const existing = database.prepare("select * from relationships where publisher_id = ?").all(publisher.id); - const failures = SUCCESSFUL_OUTCOMES.has(result.outcome) ? 0 : publisher.consecutive_failures + 1; + const conclusive = SUCCESSFUL_OUTCOMES.has(result.outcome) && !result.inconclusiveNetworkIds?.length; + const failures = conclusive ? 0 : (publisher.consecutive_failures ?? 0) + 1; const positive = result.matches?.confirmed ?? []; const review = result.matches?.review ?? []; const positiveIds = new Set([ ...positive.map((match) => String(match.networkId)), ...(result.unchangedNetworkIds ?? []).map(String), ]); + // Unknown evidence never adds misses; old confirmation expires below. + const protectedIds = new Set([...positiveIds, ...(result.inconclusiveNetworkIds ?? []).map(String)]); const missingActive = existing.filter((relationship) => new Set(["confirmed", "stale"]).has(relationship.status) - && !positiveIds.has(String(relationship.ad_network_id)) + && !protectedIds.has(String(relationship.ad_network_id)) ); if (result.outcome === "not_modified") { @@ -172,16 +179,26 @@ export function saveLocalVerification(database, publisher, result) { and ad_network_id in (${placeholders(result.unchangedNetworkIds.length)}) `).run(result.checkedAt, result.checkedAt, publisher.id, ...result.unchangedNetworkIds); } - ageMissingRelationships(database, existing, positiveIds, result.checkedAt); + ageMissingRelationships(database, existing, protectedIds, result.checkedAt, result.evidence); } else { upsertMatches(database, publisher.id, [...positive, ...review], result.checkedAt); if (SUCCESSFUL_OUTCOMES.has(result.outcome)) { - ageMissingRelationships(database, existing, positiveIds, result.checkedAt); + ageMissingRelationships(database, existing, protectedIds, result.checkedAt, result.evidence); } } - const defaultNext = nextCheckAt(result.outcome, failures, new Date(result.checkedAt)); - const relationshipNext = missingActive.length && SUCCESSFUL_OUTCOMES.has(result.outcome) + for (const relationship of existing) { + // Every check without fresh page evidence can expire an old confirmation, + // including a valid absence. The status guard keeps an ended relationship ended. + if (!positiveIds.has(String(relationship.ad_network_id)) + && homepageConfirmationExpired(relationship, result.checkedAt)) { + database.prepare("update relationships set status='stale',updated_at=? where ad_network_id=? and publisher_id=? and status='confirmed'") + .run(result.checkedAt, relationship.ad_network_id, relationship.publisher_id); + } + } + + const defaultNext = nextCheckAt(result.inconclusiveNetworkIds?.length ? "temporary_error" : result.outcome, Math.max(0, failures - 1), new Date(result.checkedAt)); + const relationshipNext = missingActive.length && conclusive ? new Date(new Date(result.checkedAt).getTime() + (missingActive.some((item) => item.consecutive_misses >= 1) ? 24 : 48) * 3_600_000) : null; database.prepare(` @@ -196,11 +213,11 @@ export function saveLocalVerification(database, publisher, result) { where id = ? `).run( result.checkedAt, - publisherStatus(result.outcome), + publisherStatus(result.inconclusiveNetworkIds?.length ? "temporary_error" : result.outcome), failures, result.outcome, JSON.stringify(result.evidence), - SUCCESSFUL_OUTCOMES.has(result.outcome) ? 1 : 0, + conclusive ? 1 : 0, result.checkedAt, result.sha256 ?? null, result.etag ?? null, diff --git a/scripts/ad-networks/matcher.mjs b/scripts/ad-networks/matcher.mjs index ffab632c..d73219d8 100644 --- a/scripts/ad-networks/matcher.mjs +++ b/scripts/ad-networks/matcher.mjs @@ -32,7 +32,7 @@ function evidenceForManager(variable, fetchEvidence, parserVersion) { }; } -export const MATCHER_VERSION = "ad-network-1.4.0"; +export const MATCHER_VERSION = "ad-network-1.4.1"; const javascriptTypes = new Set(['text/javascript', 'application/javascript', 'text/ecmascript', 'application/ecmascript', 'application/x-javascript', @@ -110,6 +110,11 @@ function signatureRequirementsMatch(signature, network, { managers, directSystem return true; } +export function hasHomepageSignature(network) { + return (network.program_signatures ?? []).some(({ kind }) => + kind === "adsense_account_match" || kind === "html_substring"); +} + export function needsHomepage(parsed, networks, discoverySources = []) { const managers = new Set(parsed.managerDomains.map((item) => item.value)); const directSystems = directAdSystems(parsed); diff --git a/scripts/ad-networks/platform.mjs b/scripts/ad-networks/platform.mjs index db040aaf..64f24be6 100644 --- a/scripts/ad-networks/platform.mjs +++ b/scripts/ad-networks/platform.mjs @@ -31,7 +31,8 @@ try { const links=selectedLinks(p.id),key=cohort(links); let page,reason; try {page=await fetchHomepage(p.normalized_domain);} catch(error){reason=error.message;} - const facts=page ? homepageFacts(page.html,page.finalUrl):null; + const facts=page?.outcome === "fetched" ? homepageFacts(page.html,page.finalUrl):null; + reason ??= page?.reason; let fetched,parsed; if(links.length) { // Unconditional fresh body: a 304 without retained parsed rows cannot establish this rule. diff --git a/scripts/ad-networks/sync.mjs b/scripts/ad-networks/sync.mjs index e521d427..756d03fb 100644 --- a/scripts/ad-networks/sync.mjs +++ b/scripts/ad-networks/sync.mjs @@ -36,12 +36,12 @@ export function compactPublisherEvidence(value, homepage = null) { const evidence = object(value ?? {}, 'last_check_evidence'); const facts = homepage ?? evidence.enrichment?.homepage; return { - ...Object.fromEntries(['manager_domains', 'confirmed_networks'].filter(key => Object.hasOwn(evidence, key)).map(key => [key, evidence[key]])), + ...Object.fromEntries(['manager_domains', 'confirmed_networks', 'homepage', 'inconclusive_networks'].filter(key => Object.hasOwn(evidence, key)).map(key => [key, evidence[key]])), ...(facts && typeof facts === 'object' ? { enrichment: { homepage: Object.fromEntries( ['title', 'description', 'h1', 'language'].filter(key => Object.hasOwn(facts, key)).map(key => [key, facts[key]])) } } : {}), }; } -const relationshipEvidenceFields = ['manager_domain','matched_signal','checked_url','checked_at','parser_version']; +const relationshipEvidenceFields = ['manager_domain','matched_signal','checked_url','checked_at','parser_version','last_absence','homepage_miss_correction']; export function compactRelationshipEvidence(value) { const evidence = object(value ?? {}, 'evidence'); return Object.fromEntries(relationshipEvidenceFields.filter(key => Object.hasOwn(evidence, key)).map(key => [key, evidence[key]])); @@ -184,6 +184,8 @@ export async function syncReviewedData(options, dependencies = {}) { const publisherSelect=fields.map(field=>field==='last_check_evidence' ? `json_object('manager_domains',json_extract(p.last_check_evidence,'$.manager_domains'), 'confirmed_networks',json_extract(p.last_check_evidence,'$.confirmed_networks'), + 'homepage',json_extract(p.last_check_evidence,'$.homepage'), + 'inconclusive_networks',json_extract(p.last_check_evidence,'$.inconclusive_networks'), 'enrichment',json_object('homepage',case when json_type(p.last_check_evidence,'$.enrichment.homepage')='object' then json_object('title',json_extract(p.last_check_evidence,'$.enrichment.homepage.title'), 'description',json_extract(p.last_check_evidence,'$.enrichment.homepage.description'), diff --git a/scripts/ad-networks/verification-batch.mjs b/scripts/ad-networks/verification-batch.mjs new file mode 100644 index 00000000..112cf2e6 --- /dev/null +++ b/scripts/ad-networks/verification-batch.mjs @@ -0,0 +1,21 @@ +// Admit each domain against the deadline. Already admitted requests finish within +// their fetch timeouts; all unstarted claims are released, including on failure. +export async function runVerificationBatch(publishers, { + concurrency, deadline, process: processPublisher, release = async () => {}, + now = Date.now, shouldStop = () => false, +}) { + let cursor = 0, failed = false; + try { + const results = await Promise.allSettled(Array.from({ length: Math.min(concurrency, publishers.length) }, async () => { + while (cursor < publishers.length && !failed && !shouldStop() && now() < deadline) { + const publisher = publishers[cursor++]; + try { await processPublisher(publisher); } + catch (error) { failed = true; throw error; } + } + })); + const failure = results.find(result => result.status === "rejected"); + if (failure) throw failure.reason; + } finally { + for (const publisher of publishers.slice(cursor)) await release(publisher.id); + } +} diff --git a/scripts/ad-networks/verifier-store.mjs b/scripts/ad-networks/verifier-store.mjs index 39c3b83c..37318179 100644 --- a/scripts/ad-networks/verifier-store.mjs +++ b/scripts/ad-networks/verifier-store.mjs @@ -1,5 +1,28 @@ const SUCCESSFUL_OUTCOMES = new Set(["matched", "no_match", "no_ads_txt", "not_modified"]); +// Weekly positive checks plus 6/24/72/168-hour retries get several chances. +// Thirty days without successful page confirmation is unknown, never a departure. +export const HOMEPAGE_CONFIRMATION_MAX_AGE_MS = 30 * 86_400_000; +// The time of the last positive page evidence. Older not-modified checks could +// move last_confirmed_at without fetching the page, so the saved evidence time +// wins when it is earlier. Without either, the confirmation counts as expired. +export function homepageEvidenceTime(relationship) { + let evidence = relationship.evidence; + if (typeof evidence === "string") { + try { evidence = JSON.parse(evidence); } catch { evidence = null; } + } + const times = [Date.parse(evidence?.checked_at ?? ""), relationship.last_confirmed_at ? new Date(relationship.last_confirmed_at).getTime() : NaN] + .filter(Number.isFinite); + return times.length ? Math.min(...times) : null; +} + +export function homepageConfirmationExpired(relationship, checkedAt) { + if (relationship.status !== "confirmed" + || !["ads_txt_and_adsense_account", "ads_txt_and_page_signature", "page_signature"].includes(relationship.verification_method)) return false; + const evidenceTime = homepageEvidenceTime(relationship); + return evidenceTime === null || Date.parse(checkedAt) - evidenceTime >= HOMEPAGE_CONFIRMATION_MAX_AGE_MS; +} + export function nextCheckAt(outcome, failures = 0, now = new Date()) { const milliseconds = { matched: 7 * 86_400_000, @@ -117,7 +140,9 @@ async function upsertMatches(client, publisherId, matches, checkedAt) { status = excluded.status, relationship_type = excluded.relationship_type, verification_method = excluded.verification_method, - evidence = excluded.evidence, + evidence = excluded.evidence || case when ad_network_publisher_relationships.evidence ? 'homepage_miss_correction' + then jsonb_build_object('homepage_miss_correction',ad_network_publisher_relationships.evidence->'homepage_miss_correction') + else '{}'::jsonb end, first_confirmed_at = case when excluded.status = 'confirmed' then coalesce(ad_network_publisher_relationships.first_confirmed_at, excluded.first_confirmed_at) else ad_network_publisher_relationships.first_confirmed_at @@ -135,16 +160,17 @@ async function upsertMatches(client, publisherId, matches, checkedAt) { } } -async function ageMissingRelationships(client, existing, positiveIds, checkedAt) { +async function ageMissingRelationships(client, existing, positiveIds, checkedAt, evidence) { for (const relationship of existing) { if (!new Set(["confirmed", "stale"]).has(relationship.status) || positiveIds.has(String(relationship.ad_network_id))) continue; const { misses, status, endedAt } = relationshipAfterMiss(relationship, checkedAt); await client.query(` update public.ad_network_publisher_relationships set status = $3, ended_at = $4, consecutive_misses = $5, - first_missed_at = coalesce(first_missed_at, $2), last_missed_at = $2 + first_missed_at = coalesce(first_missed_at, $2), last_missed_at = $2, + evidence = evidence || jsonb_build_object('last_absence', $7::jsonb) where ad_network_id = $1 and publisher_id = $6 - `, [relationship.ad_network_id, checkedAt, status, endedAt, misses, relationship.publisher_id]); + `, [relationship.ad_network_id, checkedAt, status, endedAt, misses, relationship.publisher_id, JSON.stringify({ checked_at: checkedAt, homepage: evidence?.homepage ?? null, ads_txt_sha256: evidence?.ads_txt_sha256 ?? null })]); } } @@ -153,16 +179,19 @@ export async function saveVerification(pool, publisher, result) { try { await client.query("begin"); const existing = await relationshipRows(client, publisher.id); - const failures = SUCCESSFUL_OUTCOMES.has(result.outcome) ? 0 : publisher.consecutive_failures + 1; + const conclusive = SUCCESSFUL_OUTCOMES.has(result.outcome) && !result.inconclusiveNetworkIds?.length; + const failures = conclusive ? 0 : (publisher.consecutive_failures ?? 0) + 1; const positive = result.matches?.confirmed ?? []; const review = result.matches?.review ?? []; const positiveIds = new Set([ ...positive.map((match) => String(match.networkId)), ...(result.unchangedNetworkIds ?? []).map(String), ]); + // Unknown evidence never adds misses; old confirmation expires below. + const protectedIds = new Set([...positiveIds, ...(result.inconclusiveNetworkIds ?? []).map(String)]); const missingActive = existing.filter((relationship) => new Set(["confirmed", "stale"]).has(relationship.status) - && !positiveIds.has(String(relationship.ad_network_id)) + && !protectedIds.has(String(relationship.ad_network_id)) ); if (result.outcome === "not_modified") { @@ -173,16 +202,27 @@ export async function saveVerification(pool, publisher, result) { where publisher_id = $1 and status in ('confirmed', 'stale') and ad_network_id = any($3::bigint[]) `, [publisher.id, result.checkedAt, result.unchangedNetworkIds ?? []]); - await ageMissingRelationships(client, existing, positiveIds, result.checkedAt); + await ageMissingRelationships(client, existing, protectedIds, result.checkedAt, result.evidence); } else { await upsertMatches(client, publisher.id, [...positive, ...review], result.checkedAt); if (SUCCESSFUL_OUTCOMES.has(result.outcome)) { - await ageMissingRelationships(client, existing, positiveIds, result.checkedAt); + await ageMissingRelationships(client, existing, protectedIds, result.checkedAt, result.evidence); + } + } + + for (const relationship of existing) { + // Every check without fresh page evidence can expire an old confirmation, + // including a valid absence. The status guard keeps an ended relationship ended. + if (!positiveIds.has(String(relationship.ad_network_id)) + && homepageConfirmationExpired(relationship, result.checkedAt)) { + await client.query(`update public.ad_network_publisher_relationships set status='stale' + where ad_network_id=$1 and publisher_id=$2 and status='confirmed'`, + [relationship.ad_network_id, relationship.publisher_id]); } } - const defaultNextCheck = nextCheckAt(result.outcome, failures, new Date(result.checkedAt)); - const relationshipNextCheck = missingActive.length && SUCCESSFUL_OUTCOMES.has(result.outcome) + const defaultNextCheck = nextCheckAt(result.inconclusiveNetworkIds?.length ? "temporary_error" : result.outcome, Math.max(0, failures - 1), new Date(result.checkedAt)); + const relationshipNextCheck = missingActive.length && conclusive ? new Date(new Date(result.checkedAt).getTime() + (missingActive.some((relationship) => relationship.consecutive_misses >= 1) ? 24 : 48) * 3_600_000) : null; @@ -201,8 +241,8 @@ export async function saveVerification(pool, publisher, result) { next_check_at = $11, lease_until = null where id = $1 - `, [publisher.id, result.checkedAt, publisherStatus(result.outcome), failures, - result.outcome, JSON.stringify(result.evidence), SUCCESSFUL_OUTCOMES.has(result.outcome), + `, [publisher.id, result.checkedAt, publisherStatus(result.inconclusiveNetworkIds?.length ? "temporary_error" : result.outcome), failures, + result.outcome, JSON.stringify(result.evidence), conclusive, result.sha256 ?? null, result.etag ?? null, result.lastModified ?? null, relationshipNextCheck ?? defaultNextCheck]); await client.query("commit"); diff --git a/scripts/ad-networks/verify-publisher.mjs b/scripts/ad-networks/verify-publisher.mjs index e95bc8c2..7163cb17 100644 --- a/scripts/ad-networks/verify-publisher.mjs +++ b/scripts/ad-networks/verify-publisher.mjs @@ -1,8 +1,8 @@ import { getDomain } from "tldts"; import { summarizeAdsTxt, homepageFacts } from "./enrichment-parsers.mjs"; import { PARSER_VERSION, parseAdsTxt } from "./ads-txt.mjs"; -import { fetchAdsTxt, fetchDeclaredSubdomainAdsTxt, fetchHomepage } from "./fetch-policy.mjs"; -import { MATCHER_VERSION, matchNetworks, needsHomepage } from "./matcher.mjs"; +import { fetchAdsTxt, fetchDeclaredSubdomainAdsTxt, fetchHomepage, homepageChallenge } from "./fetch-policy.mjs"; +import { MATCHER_VERSION, matchNetworks, needsHomepage, hasHomepageSignature } from "./matcher.mjs"; import { gateReviewMatches } from "./review-rules.mjs"; function publicEvidence(fetchResult, parsed, matches, checkedAt) { @@ -26,18 +26,61 @@ function publicEvidence(fetchResult, parsed, matches, checkedAt) { }; } +async function checkHomepage(domain, fetchPage) { + try { + const page = await fetchPage(domain); + if (page?.outcome === "inconclusive") return { ...page, html: undefined }; + if (!page?.html?.trim()) return { ...page, outcome: "inconclusive", reason: page?.reason ?? "homepage_unavailable" }; + if (page.outcome === "fetched") return page; // The fetcher already checked challenges. + if (homepageChallenge(page.html)) return { ...page, html: undefined, outcome: "inconclusive", reason: "challenge_page" }; + return { ...page, outcome: "fetched" }; + } catch (error) { + return { outcome: "inconclusive", reason: error.cause?.code ?? error.message }; + } +} + +function homepageResult(homepage, parsed, networks, publisher, matches, forceUnknown = false) { + const positive = new Set(matches?.confirmed.map(match => String(match.networkId)) ?? []); + const unknown = networks.filter(network => !positive.has(String(network.id)) + && (forceUnknown ? hasHomepageSignature(network) + : parsed && homepage?.outcome !== "fetched" && needsHomepage(parsed, [network], publisher.discovery_sources))); + return { + inconclusiveNetworkIds: unknown.map(network => network.id), + homepageEvidence: { + outcome: homepage?.outcome ?? "not_needed", http_status: homepage?.httpStatus ?? null, + final_url: homepage?.finalUrl ?? null, reason: homepage?.reason ?? null, + }, + inconclusiveSlugs: unknown.map(network => network.slug), + }; +} + export async function verifyPublisher(publisher, networks, { fetchAdsTxt: fetchAds = fetchAdsTxt, fetchDeclaredSubdomainAdsTxt: fetchSubdomain = fetchDeclaredSubdomainAdsTxt, fetchHomepage: fetchPage = fetchHomepage, } = {}) { const checkedAt = new Date().toISOString(); - const mayUseConditional = publisher.ads_txt_summary != null && publisher.last_check_evidence?.parser_version === PARSER_VERSION - && publisher.last_check_evidence?.matcher_version === MATCHER_VERSION; - let fetchResult = await fetchAds(publisher.normalized_domain, { - etag: mayUseConditional ? publisher.ads_txt_etag : null, - lastModified: mayUseConditional ? publisher.ads_txt_last_modified : null, - }); + // Cache the complete parse locally, never the truncated public preview. A 304 + // validates only this ads.txt representation; homepage evidence is fetched anew. + const cache = publisher.last_check_evidence?.ads_txt_cache; + const root = getDomain(publisher.normalized_domain, { allowPrivateDomains: true }); + // fetchAdsTxt always requests the root file. A declared subdomain's separate + // representation must never supply validators or a cached parse for that URL. + const rootCache = !cache?.checkedUrl || new URL(cache.checkedUrl).hostname === root; + const reusable = rootCache && cache?.parsed?.parserVersion === PARSER_VERSION + && Array.isArray(cache.parsed.sellers) && Array.isArray(cache.parsed.variables); + let fetchResult = await fetchAds(publisher.normalized_domain, reusable ? { + etag: cache.etag, lastModified: cache.lastModified, + } : {}); + let cachedParse = null; + if (fetchResult.outcome === "not_modified") { + if (reusable) { + cachedParse = cache.parsed; + fetchResult = { ...fetchResult, outcome: "parsed", sha256: cache.sha256, + checkedUrl: cache.checkedUrl, finalUrl: cache.finalUrl, + etag: fetchResult.etag ?? cache.etag, lastModified: fetchResult.lastModified ?? cache.lastModified }; + } else fetchResult = await fetchAds(publisher.normalized_domain, {}); + } if (fetchResult.outcome !== "parsed") { if (fetchResult.outcome === "no_ads_txt") { @@ -45,7 +88,7 @@ export async function verifyPublisher(publisher, networks, { const pageOnlyNetworks = networks.filter((network) => (network.program_signatures ?? []) .some((signature) => signature.kind === "html_substring" && signature.allow_without_ads_txt)); if (needsHomepage(emptyParsed, pageOnlyNetworks, publisher.discovery_sources)) { - const homepage = await fetchPage(publisher.normalized_domain).catch(() => null); + const homepage = await checkHomepage(publisher.normalized_domain, fetchPage); const matches = gateReviewMatches({ matches: matchNetworks({ parsed: emptyParsed, networks: pageOnlyNetworks, @@ -54,26 +97,26 @@ export async function verifyPublisher(publisher, networks, { discoverySources: publisher.discovery_sources, }), relationships: publisher.review_relationships, networks: pageOnlyNetworks, parsed: emptyParsed, fetchResult, homepage, checkedAt, discoverySources: publisher.discovery_sources }); - if (matches.confirmed.length || matches.review.length) { - return { - outcome: matches.confirmed.length ? "matched" : "no_ads_txt", - checkedAt, - matches, - homepageFacts: homepage ? homepageFacts(homepage.html, homepage.finalUrl) : null, - evidence: publicEvidence(fetchResult, emptyParsed, matches, checkedAt), - etag: fetchResult.etag, - lastModified: fetchResult.lastModified, - }; - } + const home = homepageResult(homepage, emptyParsed, networks, publisher, matches); + return { + inconclusiveNetworkIds: home.inconclusiveNetworkIds, + outcome: matches.confirmed.length ? "matched" : "no_ads_txt", + checkedAt, + matches, + homepageFacts: homepage?.outcome === "fetched" ? homepageFacts(homepage.html, homepage.finalUrl) : null, + evidence: { ...publicEvidence(fetchResult, emptyParsed, matches, checkedAt), homepage: home.homepageEvidence, inconclusive_networks: home.inconclusiveSlugs }, + etag: fetchResult.etag, + lastModified: fetchResult.lastModified, + }; } } const unchangedSlugs = fetchResult.outcome === "not_modified" ? publisher.last_check_evidence?.confirmed_networks ?? [] : []; const unchangedNetworkIds = networks - .filter((network) => unchangedSlugs.includes(network.slug)) + .filter((network) => !hasHomepageSignature(network) && unchangedSlugs.includes(network.slug)) .map((network) => network.id); - const evidence = publicEvidence(fetchResult, null, null, checkedAt); + const evidence = { ...publicEvidence(fetchResult, null, null, checkedAt), ...(reusable ? { ads_txt_cache: cache } : {}) }; if (fetchResult.outcome === "not_modified") { evidence.ads_txt_sha256 = publisher.ads_txt_hash ?? publisher.last_check_evidence?.ads_txt_sha256 ?? null; evidence.parser_version = publisher.last_check_evidence?.parser_version ?? null; @@ -84,7 +127,11 @@ export async function verifyPublisher(publisher, networks, { evidence.confirmed_networks = unchangedSlugs; evidence.review_networks = publisher.last_check_evidence?.review_networks ?? []; } + const home = homepageResult(null, null, networks, publisher, null, fetchResult.outcome === "not_modified"); + evidence.homepage = home.homepageEvidence; + evidence.inconclusive_networks = home.inconclusiveSlugs; return { + inconclusiveNetworkIds: home.inconclusiveNetworkIds, outcome: fetchResult.outcome, checkedAt, evidence, @@ -94,7 +141,7 @@ export async function verifyPublisher(publisher, networks, { }; } - let parsed = parseAdsTxt(fetchResult.text); + let parsed = cachedParse ?? parseAdsTxt(fetchResult.text); const registrableDomain = getDomain(publisher.normalized_domain, { allowPrivateDomains: true }); if ( registrableDomain @@ -113,6 +160,7 @@ export async function verifyPublisher(publisher, networks, { } fetchResult = subdomainResult; parsed = parseAdsTxt(fetchResult.text); + cachedParse = null; } if (!parsed.sellers.length && !parsed.variables.length && parsed.errors.length) { @@ -121,7 +169,7 @@ export async function verifyPublisher(publisher, networks, { outcome: "invalid_ads_txt", checkedAt, evidence: publicEvidence(fetchResult, parsed, null, checkedAt), - adsTxtSummary: summarizeAdsTxt(fetchResult.text), + adsTxtSummary: cachedParse ? null : summarizeAdsTxt(fetchResult.text), sha256: fetchResult.sha256, etag: fetchResult.etag, lastModified: fetchResult.lastModified, @@ -129,7 +177,7 @@ export async function verifyPublisher(publisher, networks, { } const homepage = needsHomepage(parsed, networks, publisher.discovery_sources) - ? await fetchPage(publisher.normalized_domain).catch(() => null) + ? await checkHomepage(publisher.normalized_domain, fetchPage) : null; const evidenceInput = { ...fetchResult, checkedAt }; const matches = gateReviewMatches({matches:matchNetworks({ @@ -140,14 +188,19 @@ export async function verifyPublisher(publisher, networks, { discoverySources: publisher.discovery_sources, }),relationships:publisher.review_relationships,networks,parsed,fetchResult,homepage,checkedAt, discoverySources:publisher.discovery_sources}); + const home = homepageResult(homepage, parsed, networks, publisher, matches); const outcome = matches.confirmed.length ? "matched" : "no_match"; return { outcome, checkedAt, + inconclusiveNetworkIds: home.inconclusiveNetworkIds, matches, - adsTxtSummary: summarizeAdsTxt(fetchResult.text), - homepageFacts: homepage ? homepageFacts(homepage.html, homepage.finalUrl) : null, - evidence: publicEvidence(fetchResult, parsed, matches, checkedAt), + adsTxtSummary: cachedParse ? null : summarizeAdsTxt(fetchResult.text), + homepageFacts: homepage?.outcome === "fetched" ? homepageFacts(homepage.html, homepage.finalUrl) : null, + evidence: { ...publicEvidence(fetchResult, parsed, matches, checkedAt), ads_txt_cache: { + parsed, sha256: fetchResult.sha256, etag: fetchResult.etag, lastModified: fetchResult.lastModified, + checkedUrl: fetchResult.checkedUrl, finalUrl: fetchResult.finalUrl, + }, homepage: home.homepageEvidence, inconclusive_networks: home.inconclusiveSlugs }, sha256: fetchResult.sha256, etag: fetchResult.etag, lastModified: fetchResult.lastModified, diff --git a/scripts/ad-networks/verify.mjs b/scripts/ad-networks/verify.mjs index 5639acaf..4278c130 100644 --- a/scripts/ad-networks/verify.mjs +++ b/scripts/ad-networks/verify.mjs @@ -12,6 +12,7 @@ import { saveLocalVerification, selectLocalPublishers, } from "./local-verifier-store.mjs"; +import { runVerificationBatch } from "./verification-batch.mjs"; import { verifyPublisher } from "./verify-publisher.mjs"; import { claimPublishers, @@ -117,14 +118,16 @@ function createSummary() { outcomes: {}, confirmedRelationships: 0, reviewRelationships: 0, + inconclusiveChecks: 0, + homepageFailures: {}, }; } async function processBatch(publishers, networks, summary) { - let cursor = 0; - const workers = Array.from({ length: Math.min(concurrency, publishers.length) }, async () => { - while (!stopping && cursor < publishers.length) { - const publisher = publishers[cursor++]; + await runVerificationBatch(publishers, { concurrency, deadline, + shouldStop: () => stopping, + release: apply ? release : async () => {}, + process: async publisher => { let result; try { result = await verifyPublisher(publisher, networks); @@ -143,10 +146,15 @@ async function processBatch(publishers, networks, summary) { summary.databaseFailures += 1; await Promise.resolve(release(publisher.id)).catch(() => {}); console.error(JSON.stringify({ domain: publisher.normalized_domain, status: "database_failed", error: error.message })); - continue; + return; } } summary.processed += 1; + if (result.inconclusiveNetworkIds?.length) summary.inconclusiveChecks++; + if (result.evidence.homepage?.outcome === "inconclusive") { + const reason = result.evidence.homepage.reason ?? "unknown"; + summary.homepageFailures[reason] = (summary.homepageFailures[reason] ?? 0) + 1; + } summary.outcomes[result.outcome] = (summary.outcomes[result.outcome] ?? 0) + 1; summary.confirmedRelationships += result.matches?.confirmed.length ?? 0; summary.reviewRelationships += result.matches?.review.length ?? 0; @@ -157,6 +165,8 @@ async function processBatch(publishers, networks, summary) { confirmed: result.matches?.confirmed.map((match) => match.slug) ?? [], review: result.matches?.review.map((match) => match.slug) ?? [], reason: result.evidence.reason ?? null, + homepage: result.evidence.homepage ?? null, + inconclusive: result.evidence.inconclusive_networks ?? [], })); } else if (summary.processed % 100 === 0) { const elapsed = (Date.now() - Date.parse(summary.startedAt)) / 1000; @@ -170,10 +180,8 @@ async function processBatch(publishers, networks, summary) { await (await import("node:fs/promises")).rename(temporary, file); console.log(JSON.stringify(progress)); } - } + }, }); - await Promise.all(workers); - if (apply && stopping) for (const publisher of publishers.slice(cursor)) await release(publisher.id); } async function saveSummary(summary) { @@ -186,6 +194,7 @@ async function saveSummary(summary) { } const summary = createSummary(); +const deadline = Date.now() + timeBudgetMinutes * 60_000; let stopping = false; for (const signal of ['SIGINT','SIGTERM']) process.on(signal, () => { stopping = true; summary.interrupted = true; }); try { @@ -204,7 +213,6 @@ try { await processBatch(publishers, networks, summary); await saveSummary(summary); } else { - const deadline = Date.now() + timeBudgetMinutes * 60_000; while (!stopping && summary.attempted < limit && Date.now() < deadline) { const batchSize = Math.min(concurrency * 32, limit - summary.attempted); const publishers = await claim({ limit: batchSize, leaseMinutes, domains: [] }); diff --git a/stack127_docs/monitoring.json b/stack127_docs/monitoring.json index 524d47cf..bf34f977 100644 --- a/stack127_docs/monitoring.json +++ b/stack127_docs/monitoring.json @@ -4,9 +4,9 @@ "about": "Entries for the Automation Monitor dashboard (https://monitor.ravitejaknts.com). One entry per scheduled job, workflow or service. key: / (machines: homelab, bloxodes-vps, stack127-vps, macbook-pro, macbook-air; job = cron run-job.sh name, systemd unit without .timer/.service, launchd label or T3 schedule title), github///, supabase/, or northflank/. name: short plain name. group: the section it is listed under. about: one sentence on what it does, written for the owner. impact (optional): what breaks if it stops. records (optional): stats_job_runs.job_name values it writes. ignore_records: job names that are sub-steps of other jobs.", "automations": [ {"key": "stack127-vps/launch-directory-metrics", "group": "Launch directories", "name": "Refresh launch directory ratings", "about": "On the 2nd of each month at 06:00 UTC, refreshes Ahrefs Domain Rating and Tranco popularity for published launch directories.", "impact": "Launch directory ratings and popularity become stale."}, - {"key": "stack127-vps/verification", "group": "Ad networks", "name": "Ad network site checks", "about": "Every hour at minute 40, checks up to 1,000 publisher sites (their ads.txt and pages), sends changed rows to production, and refreshes the site counts in titles and headings. It steps aside while a bigger job runs.", "impact": "Network pages stop gaining newly confirmed sites."}, + {"key": "stack127-vps/verification", "group": "Ad networks", "name": "Ad network site checks", "about": "Every hour at minute 40, checks up to 1,000 publisher sites (their ads.txt and pages), sends changed rows to production, and refreshes the site counts in titles and headings. Blocked or failed homepage checks never record departures, retry with backoff, and mark confirmation stale after 30 days without fresh evidence. It steps aside while a bigger job runs.", "impact": "Network pages stop gaining newly confirmed sites."}, {"key": "stack127-vps/snapshot", "group": "Ad networks", "name": "Daily ad network counts", "about": "Saves today's publisher counts for each ad network."}, - {"key": "stack127-vps/platform", "group": "Ad networks", "name": "Publisher platform and favicon", "about": "Fills in each publisher's website platform and icon, and refreshes facts older than 30 days."}, + {"key": "stack127-vps/platform", "group": "Ad networks", "name": "Publisher platform and favicon", "about": "Fills in each publisher's website platform and icon, and refreshes facts older than 30 days. Challenge pages remain unavailable."}, {"key": "stack127-vps/domain-age", "group": "Ad networks", "name": "Publisher domain age", "about": "Looks up when new publisher domains were registered."}, {"key": "stack127-vps/ads-txt-summary", "group": "Ad networks", "name": "ads.txt summaries", "about": "Summarizes each publisher's ads.txt file for the network pages."}, {"key": "stack127-vps/discovery", "group": "Ad networks", "name": "Find new publishers", "about": "Every Tuesday, reads the ad networks' official seller lists and adds new publisher domains to check."}, diff --git a/supabase/migrations/20261113120000_ad_network_homepage_miss_repairs.sql b/supabase/migrations/20261113120000_ad_network_homepage_miss_repairs.sql new file mode 100644 index 00000000..5116b5dd --- /dev/null +++ b/supabase/migrations/20261113120000_ad_network_homepage_miss_repairs.sql @@ -0,0 +1,23 @@ +begin; +set local lock_timeout = '10s'; +-- A target-side checkpoint survives collector syncs and commits atomically with +-- event/snapshot repair. Never put it in source-owned relationship evidence. +create table public.ad_network_homepage_miss_repairs ( + ad_network_id bigint not null, + publisher_id bigint not null, + corrected_at timestamptz not null, + version text not null, + repaired_at timestamptz not null default now(), + primary key (ad_network_id, publisher_id, corrected_at, version) +); +alter table public.ad_network_homepage_miss_repairs enable row level security; +revoke all on public.ad_network_homepage_miss_repairs from public, anon, authenticated; +grant select, insert on public.ad_network_homepage_miss_repairs to service_role; +do $$ begin + if exists(select 1 from pg_roles where rolname='stack127_app') then + grant select, insert on public.ad_network_homepage_miss_repairs to stack127_app; + create policy ad_network_homepage_miss_repairs_backend on public.ad_network_homepage_miss_repairs + for all to stack127_app using (true) with check (true); + end if; +end $$; +commit; diff --git a/tests/pages/ad-networks/fixtures/homepage-challenges.json b/tests/pages/ad-networks/fixtures/homepage-challenges.json new file mode 100644 index 00000000..861f890d --- /dev/null +++ b/tests/pages/ad-networks/fixtures/homepage-challenges.json @@ -0,0 +1,87 @@ +[ + { + "name": "CAPTCHA human title", + "challenge": true, + "html": "Please verify you are a human
" + }, + { + "name": "human heading", + "challenge": true, + "html": "Security

Verify you are human

" + }, + { + "name": "Cloudflare", + "challenge": true, + "html": "Just a moment...
" + }, + { + "name": "Sucuri", + "challenge": true, + "html": "You are being redirected...

Checking your browser

" + }, + { + "name": "Imperva Incapsula", + "challenge": true, + "html": "Request unsuccessful" + }, + { + "name": "Akamai", + "challenge": true, + "html": "Security verification" + }, + { + "name": "DataDome", + "challenge": true, + "html": "Publisher" + }, + { + "name": "PerimeterX", + "challenge": true, + "html": "Access to this page has been denied

Please verify you are a human

" + }, + { + "name": "hCaptcha interstitial", + "challenge": true, + "html": "Check

Please complete the CAPTCHA

" + }, + { + "name": "reCAPTCHA interstitial", + "challenge": true, + "html": "Please verify that you're a human
" + }, + { + "name": "wsidchk quoted example", + "challenge": false, + "html": "JavaScript tutorial
const name = 'wsidchk';
" + }, + { + "name": "comments and inert markup", + "challenge": false, + "html": "Home" + }, + { + "name": "article quoting a CAPTCHA", + "challenge": false, + "html": "How CAPTCHA works

Verify you are human is a common message.

" + }, + { + "name": "ordinary contact form", + "challenge": false, + "html": "Contact us
" + }, + { + "name": "Cloudflare passive JS detection", + "challenge": false, + "html": "Home" + }, + { + "name": "human heading without server-rendered CAPTCHA", + "challenge": true, + "html": "Security

Verify you are human

" + }, + { + "name": "hidden and quoted examples", + "challenge": false, + "html": "Guide

Verify you are human

" + } +] diff --git a/tests/pages/ad-networks/homepage-misses.test.mjs b/tests/pages/ad-networks/homepage-misses.test.mjs new file mode 100644 index 00000000..df421113 --- /dev/null +++ b/tests/pages/ad-networks/homepage-misses.test.mjs @@ -0,0 +1,399 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { mkdtempSync,rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { verifyPublisher } from '../../../scripts/ad-networks/verify-publisher.mjs'; +import { fetchHomepage,homepageChallenge } from '../../../scripts/ad-networks/fetch-policy.mjs'; +import { openLocalDatabase } from '../../../scripts/ad-networks/local-db.mjs'; +import { saveLocalVerification } from '../../../scripts/ad-networks/local-verifier-store.mjs'; +import { saveVerification,homepageConfirmationExpired } from '../../../scripts/ad-networks/verifier-store.mjs'; +import { correctionEligibility,correctionConfirmation,planCorrections,restoration,applySqliteCorrection } from '../../../scripts/ad-networks/correct-homepage-misses.mjs'; + +const network={id:32,slug:'google-adsense',manager_domains:[],ad_system_domains:['google.com'],program_signatures:[{kind:'adsense_account_match'}]}; +const manager={id:1,slug:'manager',manager_domains:['manager.com'],ad_system_domains:[],program_signatures:[]}; +const ads={outcome:'parsed',text:'google.com,pub-1234567890123456,DIRECT\n',sha256:'hash',httpStatus:200}; +const page={html:'',finalUrl:'https://publisher.example/'}; +const publisher={normalized_domain:'publisher.example',discovery_sources:[],last_check_evidence:{}}; +const signal='google.com,pub-1234567890123456,DIRECT + ca-pub-1234567890123456'; +async function verify(homepage,fetchAdsTxt=async()=>ads) { + return verifyPublisher(publisher,[network,manager],{fetchAdsTxt,fetchHomepage:async()=>{ + if(homepage instanceof Error)throw homepage; + return homepage; + }}); +} +function fixture(t) { + const dir=mkdtempSync(join(tmpdir(),'ad-homepage-')),db=openLocalDatabase(join(dir,'store.sqlite')); + t.after(()=>{db.close();rmSync(dir,{recursive:true,force:true});}); + const now='2026-10-01T00:00:00.000Z'; + db.prepare("insert into publishers(id,normalized_domain,display_domain,first_seen_at,last_discovered_at,created_at,updated_at) values(1,'publisher.example','publisher.example',?,?,?,?)").run(now,now,now,now); + db.prepare("insert into relationships(ad_network_id,publisher_id,status,relationship_type,verification_method,evidence,first_confirmed_at,last_confirmed_at,consecutive_misses,first_missed_at,last_missed_at,created_at,updated_at) values(32,1,'stale','active_network_use','ads_txt_and_adsense_account',?,?,?,2,?,?,?,?)") + .run(JSON.stringify({matched_signal:signal,checked_url:'https://publisher.example/ads.txt',checked_at:now,parser_version:'ads-txt-1.1.0'}),now,now,now,'2026-10-03T00:00:00.000Z',now,now); + return db; +} +for(const [name,homepage] of [['timeout',new Error('timeout')],['unavailable',null], + ['soft challenge',{html:'One moment, please...'}], + ['Cloudflare',{html:'Just a moment...
'}]]) { + test(name+' leaves a stale relationship completely unchanged',async t=>{ + const db=fixture(t),before=db.prepare('select * from relationships').get(),result=await verify(homepage); + assert.deepEqual(result.inconclusiveNetworkIds,[32]); + saveLocalVerification(db,{id:1,consecutive_failures:0},result); + assert.deepEqual(db.prepare('select * from relationships').get(),before); + const stored=db.prepare('select * from publishers').get(); + assert.equal(stored.consecutive_failures,1);assert.equal(stored.last_successful_check_at,null); + assert.equal(Date.parse(stored.next_check_at)-Date.parse(result.checkedAt),6*3600000); + }); +} +test('304 refetches ads.txt and obtains fresh homepage evidence',async()=>{ + let calls=0,pages=0; + const result=await verifyPublisher(publisher,[network],{ + fetchAdsTxt:async(_,conditional)=>{assert.ok(!conditional.etag);return ++calls===1?{outcome:'not_modified'}:ads;}, + fetchHomepage:async()=>{pages++;return page;}, + }); + assert.equal(calls,2);assert.equal(pages,1);assert.equal(result.matches.confirmed[0].networkId,32); +}); +test('a repeated 304 neither confirms nor misses a homepage relationship',async t=>{ + const db=fixture(t),before=db.prepare('select * from relationships').get(),result=await verify(null,async()=>({outcome:'not_modified'})); + assert.deepEqual(result.inconclusiveNetworkIds,[32]);assert.deepEqual(result.unchangedNetworkIds,[]); + saveLocalVerification(db,{id:1,consecutive_failures:0},result); + assert.deepEqual(db.prepare('select * from relationships').get(),before); +}); +for(const [name,home,body] of [['fetched homepage without tag',{html:'Real homepage'},ads], + ['ads.txt drops authorization',null,{...ads,text:'other.com,123,DIRECT'}]]) { + test(name+' still ends after three valid misses over seven days',async t=>{ + const db=fixture(t),result=await verify(home,async()=>body);result.checkedAt='2026-10-09T00:00:00.000Z'; + assert.deepEqual(result.inconclusiveNetworkIds,[]); + saveLocalVerification(db,{id:1,consecutive_failures:0},result); + const row=db.prepare('select * from relationships').get(); + assert.equal(row.status,'ended');assert.equal(row.consecutive_misses,3); + assert.equal(JSON.parse(row.evidence).last_absence.checked_at,result.checkedAt); + }); +} +test('independent manager evidence still confirms during a homepage failure',async()=>{ + const result=await verify(null,async()=>({...ads,text:ads.text+'MANAGERDOMAIN=manager.com'})); + assert.equal(result.matches.confirmed[0].networkId,1);assert.deepEqual(result.inconclusiveNetworkIds,[32]); +}); +test('fetcher records HTTP and challenge outcomes; normal Cloudflare analytics remains usable',async()=>{ + const deps={lookup:async()=>[{address:'93.184.216.34',family:4}],sharedHostSlot:async()=>()=>{}}; + for(const [status,html,reason] of [[403,'blocked','http_status'],[200,'One moment, please...','challenge_page'], + [200,'Home',undefined]]) { + const result=await fetchHomepage('example.com',{...deps,fetch:async()=>new Response(html,{status,headers:{'content-type':'text/html'}})}); + assert.equal(result.reason,reason);assert.equal(result.httpStatus,status);assert.equal(result.outcome,reason?'inconclusive':'fetched'); + } + assert.equal(homepageChallenge('How Cloudflare works

Access denied errors

'),false); +}); +test('PostgreSQL writer does not update an inconclusive relationship',async()=>{ + const queries=[],relationship={ad_network_id:'32',publisher_id:'1',status:'stale',consecutive_misses:2}; + const client={query:async(sql,args)=>{queries.push({sql,args});return{rows:sql.includes('for update')?[relationship]:[]};},release(){}}; + await saveVerification({connect:async()=>client},{id:1,consecutive_failures:0},await verify(null)); + assert.equal(queries.some(({sql})=>sql.includes('set status = $3')),false); + const write=queries.find(({sql})=>sql.includes('update public.ad_network_publishers set')); + assert.equal(write.args[3],1);assert.equal(write.args[6],false); +}); +const damaged={ad_network_id:32,publisher_id:1,normalized_domain:'publisher.example',status:'ended', + verification_method:'ads_txt_and_adsense_account',consecutive_misses:3,first_confirmed_at:'2026-09-26T00:00:00Z', + first_missed_at:'2026-10-20T00:00:00Z',last_missed_at:'2026-11-13T00:00:00Z',ended_at:'2026-11-13T00:00:00Z', + last_checked_at:'2026-11-13T00:00:00Z',last_check_outcome:'not_modified', + last_confirmed_at:'2026-09-26T00:00:00Z',relationship_type:'active_network_use', + evidence:{matched_signal:signal,ads_txt_sha256:'hash',checked_url:'https://publisher.example/ads.txt',checked_at:'2026-09-26',parser_version:'ads-txt-1.1.0'},last_check_evidence:{}}; +test('correction excludes real absences, unknown full fetches, old misses, changed signals and hashes',async()=>{ + assert.equal(correctionEligibility(damaged),'needs_live_confirmation'); + assert.equal(correctionEligibility({...damaged,verification_method:'ads_txt_managerdomain'}),'not_homepage_dependent'); + assert.equal(correctionEligibility({...damaged,last_check_outcome:'no_match'}),'no_proof_of_inconclusive_miss'); + assert.equal(correctionEligibility({...damaged,evidence:{...damaged.evidence,last_absence:{}}}),'recorded_absence'); + assert.equal(correctionEligibility({...damaged,first_missed_at:'2026-09-27'}),'before_fault_window'); + const positive=await verify(page); + assert.equal(correctionConfirmation(damaged,positive),'restore'); + assert.equal(correctionConfirmation(damaged,{...positive,sha256:'different'}),'ads_txt_changed_since_confirmation'); + assert.equal(correctionConfirmation({...damaged,evidence:{matched_signal:'other'}},positive),'live_signal_changed'); + const plan=await planCorrections([damaged,{...damaged,last_check_outcome:'no_match'}],[network],{verify:async()=>positive}); + assert.equal(plan.restores.length,1);assert.equal(plan.counts.no_proof_of_inconclusive_miss,1); +}); +test('SQLite correction preserves dates, queues priority checks, and is repeatable',async t=>{ + const db=fixture(t),row={...db.prepare('select * from relationships').get(),normalized_domain:'publisher.example', + last_check_outcome:'not_modified',last_checked_at:'2026-10-03T00:00:00.000Z'}; + const evidence=restoration(row,'2026-11-13T01:00:00Z'); + db.exec('begin immediate');assert.equal(applySqliteCorrection(db,{restores:[{...row,correction_evidence:evidence}]}).restored,1);db.exec('commit'); + const restored=db.prepare('select * from relationships').get(); + assert.equal(restored.status,'confirmed');assert.equal(restored.first_confirmed_at,row.first_confirmed_at); + assert.equal(restored.last_confirmed_at,row.last_confirmed_at);assert.equal(restored.consecutive_misses,0); + assert.equal(db.prepare('select verification_priority from publishers').get().verification_priority,100); + assert.equal(correctionEligibility(restored),'already_restored'); + const positive=await verify(page); + saveLocalVerification(db,{id:1,consecutive_failures:0},positive); + assert.equal(JSON.parse(db.prepare('select evidence from relationships').get().evidence).homepage_miss_correction.version,'homepage-misses-v1'); + const plan=await planCorrections([restored],[network],{verify:async()=>{throw Error('must not fetch');}}); + assert.equal(plan.restores.length,0); +}); + +test('development PostgreSQL repair removes only the invalid departure, replaces uncertain snapshots and repeats safely', + {skip:process.env.AD_NETWORKS_CORRECTION_DATABASE_TEST!=='1'},async()=>{ + const {targetPool}=await import('../../../scripts/ad-networks/runtime-db.mjs'); + const {applyPostgresCorrection,previewPostgresCorrection}=await import('../../../scripts/ad-networks/correct-homepage-misses.mjs'); + const pool=targetPool({environment:'development'}),client=await pool.connect(); + try { + await client.query('begin'); + const {readFileSync}=await import('node:fs'); + await client.query(readFileSync(new URL('../../../supabase/migrations/20261113120000_ad_network_homepage_miss_repairs.sql',import.meta.url),'utf8').replace(/^begin;|commit;$/gm,'')); + const networkId=(await client.query("select id from public.ad_networks where slug='google-adsense'")).rows[0].id; + const domain='homepage-correction-'+Date.now()+'.example'; + const pub=(await client.query("insert into public.ad_network_publishers(normalized_domain,display_domain) values($1,$1) returning id",[domain])).rows[0].id; + const row={...damaged,ad_network_id:networkId,publisher_id:pub}; + await client.query("insert into public.ad_network_publisher_relationships(ad_network_id,publisher_id,status,relationship_type,verification_method,evidence,first_confirmed_at,last_confirmed_at,consecutive_misses,first_missed_at,last_missed_at,ended_at) values($1,$2,'ended','active_network_use',$3,$4,$5,$5,3,$6,$7,$7)", + [networkId,pub,row.verification_method,JSON.stringify(row.evidence),row.first_confirmed_at,row.first_missed_at,row.last_missed_at]); + await client.query("insert into public.ad_network_daily_counts(ad_network_id,snapshot_on,listed_count,confirmed_count,stale_count,captured_at) values($1,'2026-11-12',1,0,1,'2026-11-12') on conflict(ad_network_id,snapshot_on) do update set captured_at=excluded.captured_at",[networkId]); + await client.query("insert into public.ad_network_check_observations(publisher_id,checked_at,confirmed_network_ids) values($1,$2,'{}')",[pub,row.last_missed_at]); + await client.query("insert into public.ad_network_relationship_events(ad_network_id,publisher_id,event,observed_at,relationship_type) values($1,$2,'joined','2026-11-13T00:10:00Z','active_network_use'),($1,$2,'baseline','2026-11-13T00:10:00Z','active_network_use')",[networkId,pub]); + const revisionBefore=(await client.query('select revision from public.stack127_ad_network_source_revision')).rows[0].revision; + const proof=restoration(row,new Date().toISOString()); + const repairPlan={restores:[{...row,correction_evidence:proof}],repaired:[]}; + const preview=await previewPostgresCorrection(client,repairPlan); + assert.equal(preview.wouldRemoveEvents,1);assert.equal(preview.wouldNeutralizeArrivals,1);assert.ok(preview.wouldInvalidateSnapshots>=1); + await client.query('savepoint interrupted_history'); + await applyPostgresCorrection(client,repairPlan); + await client.query('rollback to savepoint interrupted_history'); + assert.equal((await client.query('select count(*)::int n from public.ad_network_homepage_miss_repairs where publisher_id=$1',[pub])).rows[0].n,0); + assert.equal((await client.query("select count(*)::int n from public.ad_network_relationship_events where publisher_id=$1 and event='left'",[pub])).rows[0].n,1); + const first=await applyPostgresCorrection(client,repairPlan); + assert.notEqual((await client.query('select revision from public.stack127_ad_network_source_revision')).rows[0].revision,revisionBefore); + assert.ok((await client.query("select 1 from public.stack127_revalidation_events where entity_type='ad_networks' and slug='all'")).rowCount); + assert.equal(first.restored,1);assert.equal(first.removedEvents,1);assert.equal(first.neutralizedArrivals,1);assert.ok(first.invalidatedSnapshots>=1); + const after=(await client.query('select * from public.ad_network_publisher_relationships where ad_network_id=$1 and publisher_id=$2',[networkId,pub])).rows[0]; + assert.equal(after.status,'confirmed');assert.equal(after.first_confirmed_at.toISOString(),new Date(row.first_confirmed_at).toISOString()); + assert.equal((await client.query("select count(*)::int n from public.ad_network_relationship_events where ad_network_id=$1 and publisher_id=$2 and event='left'",[networkId,pub])).rows[0].n,0); + assert.equal((await client.query("select verification_priority from public.ad_network_publishers where id=$1",[pub])).rows[0].verification_priority,100); + const second=await applyPostgresCorrection(client,{restores:[],repaired:[after]}); + assert.equal(second.restored,0);assert.equal(second.removedEvents,0);assert.equal(second.invalidatedSnapshots,0); + assert.equal(second.snapshots,0);assert.equal(second.historyRepaired,0); + // A public check date is the successful verification, never a later blocked attempt. + await client.query("update public.ad_network_publishers set last_checked_at='2026-11-13',last_successful_check_at='2026-10-01' where id=$1",[pub]); + await client.query("update public.ad_network_publisher_relationships set status='stale' where publisher_id=$1",[pub]); + const {adNetworksLibrary}=await import('./loader-harness.mjs'); + const lib=adNetworksLibrary((sql,args)=>client.query(sql,args)); + const publicSite=await lib.loadAdNetworkCheckerSite(domain); + assert.equal(publicSite.site.last_checked_at,'2026-10-01T00:00:00.000Z'); + assert.equal(publicSite.relationships[0].status,'stale'); + assert.equal(publicSite.relationships[0].last_confirmed_at,new Date(row.last_confirmed_at).toISOString()); + const listing=await lib.loadAdNetworkSites({...lib.defaultAdNetworkSitesQuery(),q:domain,sort:'checked'}); + assert.equal(listing.rows[0].last_checked_at,'2026-10-01T00:00:00.000Z'); + } finally {await client.query('rollback');client.release();await pool.end();} +}); +test('a blocked homepage protects AdSense and corroborated tag networks together',async()=>{ + const mgid={id:33,slug:'mgid',manager_domains:[],ad_system_domains:['mgid.com'], + program_signatures:[{kind:'html_substring',value:'jsc.mgid.com',requires_direct_ad_system_domains:['mgid.com']}]}; + const result=await verifyPublisher(publisher,[network,mgid],{ + fetchAdsTxt:async()=>({...ads,text:ads.text+'mgid.com,123,DIRECT'}), + fetchHomepage:async()=>({outcome:'inconclusive',httpStatus:403,reason:'http_status'}), + }); + assert.deepEqual(result.inconclusiveNetworkIds,[32,33]); + assert.equal(result.evidence.homepage.http_status,403); +}); +test('homepage-only networks remain unknown when ads.txt is missing and the page is blocked',async()=>{ + const ethical={id:7,slug:'ethicalads',manager_domains:[],ad_system_domains:[], + program_signatures:[{kind:'html_substring',value:'data-ea-publisher=',allow_without_ads_txt:true}]}; + const result=await verifyPublisher(publisher,[ethical],{ + fetchAdsTxt:async()=>({outcome:'no_ads_txt',httpStatus:404}),fetchHomepage:async()=>null, + }); + assert.deepEqual(result.inconclusiveNetworkIds,[7]); +}); +test('inconclusive checks back off to seven days and preserve the original absence count',async t=>{ + const db=fixture(t),before=db.prepare('select * from relationships').get(); + for(const [failures,hours] of [[1,24],[2,72],[3,168],[4,168]]) { + const result=await verify(null); + saveLocalVerification(db,{id:1,consecutive_failures:failures},result); + const stored=db.prepare('select * from publishers').get(); + assert.equal(Date.parse(stored.next_check_at)-Date.parse(result.checkedAt),hours*3600000); + assert.deepEqual(db.prepare('select * from relationships').get(),before); + } +}); +test('bounded correction batches advance a domain cursor; stats-only makes no network requests',async()=>{ + const rows=['a.example','b.example'].map(normalized_domain=>({...damaged,normalized_domain})); + const positive=await verify(page); + const first=await planCorrections(rows,[network],{limit:1,verify:async()=>positive}); + assert.equal(first.checked,1);assert.equal(first.nextAfterDomain,'a.example');assert.equal(first.counts.live_check_limit,1); + const second=await planCorrections(rows,[network],{limit:1,afterDomain:first.nextAfterDomain,verify:async()=>positive}); + assert.equal(second.checked,1);assert.equal(second.nextAfterDomain,'b.example'); + const stats=await planCorrections(rows,[network],{statsOnly:true,verify:async()=>{throw Error('must not fetch');}}); + assert.equal(stats.checked,0); +}); + +test('active challenge fixtures exclude quoted examples, comments, inert HTML and contact forms',async()=>{ + const {readFileSync}=await import('node:fs'); + const fixtures=JSON.parse(readFileSync(new URL('./fixtures/homepage-challenges.json',import.meta.url),'utf8')); + for(const fixture of fixtures) assert.equal(homepageChallenge(fixture.html),fixture.challenge,fixture.name); +}); + +test('homepage confirmation expires at 30 days, preserves dates and misses, and a real positive recovers',async t=>{ + const db=fixture(t),confirmed='2026-10-01T00:00:00.000Z'; + db.prepare("update relationships set status='confirmed',consecutive_misses=0,first_missed_at=null,last_missed_at=null").run(); + db.prepare('update publishers set last_successful_check_at=?').run(confirmed); + for(const [checkedAt,expected] of [['2026-10-30T23:59:59.999Z','confirmed'],['2026-10-31T00:00:00.000Z','stale']]) { + const result=await verify(null); result.checkedAt=checkedAt; + saveLocalVerification(db,{id:1,consecutive_failures:0},result); + const row=db.prepare('select * from relationships').get(); + assert.equal(row.status,expected);assert.equal(row.consecutive_misses,0);assert.equal(row.ended_at,null); + assert.equal(row.last_confirmed_at,confirmed); + assert.equal(db.prepare('select last_successful_check_at from publishers').get().last_successful_check_at,confirmed); + } + const positive=await verify(page);positive.checkedAt='2026-11-01T00:00:00.000Z'; + saveLocalVerification(db,{id:1,consecutive_failures:0},positive); + const row=db.prepare('select * from relationships').get(); + assert.equal(row.status,'confirmed');assert.equal(row.last_confirmed_at,positive.checkedAt); +}); + +test('transport failures also expire homepage confirmation without creating a departure',async t=>{ + const db=fixture(t);db.exec("update relationships set status='confirmed'"); + const before=db.prepare('select * from relationships').get(); + saveLocalVerification(db,{id:1,consecutive_failures:0},{outcome:'temporary_error',checkedAt:'2026-11-02T00:00:00Z',evidence:{}}); + const after=db.prepare('select * from relationships').get(); + assert.equal(after.status,'stale');assert.equal(after.consecutive_misses,before.consecutive_misses); + assert.equal(after.last_confirmed_at,before.last_confirmed_at);assert.equal(after.ended_at,null); +}); + +test('PostgreSQL expires unknown homepage confirmation without moving successful dates',async()=>{ + const queries=[],relationship={ad_network_id:'32',publisher_id:'1',status:'confirmed',verification_method:'ads_txt_and_adsense_account',last_confirmed_at:'2026-10-01'}; + const client={query:async(sql,args)=>{queries.push({sql,args});return {rows:sql.includes('for update')?[relationship]:[]};},release(){}}; + const result=await verify(null); result.checkedAt='2026-11-01T00:00:00Z'; + await saveVerification({connect:async()=>client},{id:1},result); + assert.ok(queries.find(({sql})=>sql.includes("set status='stale'"))); + const write=queries.find(({sql})=>sql.includes('update public.ad_network_publishers set')); + assert.equal(write.args[6],false); +}); + +test('expiry uses the saved page evidence time when a not-modified check moved the confirmation date',()=>{ + const relationship={status:'confirmed',verification_method:'ads_txt_and_adsense_account', + last_confirmed_at:'2026-11-01T00:00:00Z',evidence:JSON.stringify({checked_at:'2026-10-01T00:00:00Z'})}; + assert.equal(homepageConfirmationExpired(relationship,'2026-11-13T00:00:00Z'),true); + assert.equal(homepageConfirmationExpired({...relationship,evidence:{checked_at:'2026-11-01T00:00:00Z'}},'2026-11-13T00:00:00Z'),false); + assert.equal(homepageConfirmationExpired({...relationship,evidence:'not json'},'2026-11-13T00:00:00Z'),false); + assert.equal(homepageConfirmationExpired({...relationship,verification_method:'ads_txt_manager_domain',evidence:{checked_at:'2026-01-01'}},'2026-11-13T00:00:00Z'),false); +}); + +for(const store of ['PostgreSQL','SQLite']) test(store+': the first valid absence also expires an old confirmation, without ending it',async t=>{ + const result={outcome:'no_match',checkedAt:'2026-11-13T00:00:00Z',evidence:{},matches:{confirmed:[],review:[]},inconclusiveNetworkIds:[]}; + if(store==='SQLite'){ + const db=fixture(t); + db.prepare("update relationships set status='confirmed',consecutive_misses=0,first_missed_at=null,last_missed_at=null,last_confirmed_at=?").run('2026-10-01T00:00:00.000Z'); + saveLocalVerification(db,{id:1,consecutive_failures:0},result); + const row=db.prepare('select * from relationships').get(); + assert.equal(row.status,'stale');assert.equal(row.ended_at,null);assert.equal(row.consecutive_misses,1); + return; + } + const queries=[],relationship={ad_network_id:'32',publisher_id:'1',status:'confirmed',verification_method:'ads_txt_and_adsense_account', + last_confirmed_at:'2026-10-01',consecutive_misses:0,first_missed_at:null,last_missed_at:null,evidence:{checked_at:'2026-10-01'}}; + const client={query:async(sql,args)=>{queries.push({sql,args});return {rows:sql.includes('for update')?[relationship]:[]};},release(){}}; + await saveVerification({connect:async()=>client},{id:1},result); + const stale=queries.find(({sql})=>sql.includes("set status='stale'")); + assert.ok(stale);assert.match(stale.sql,/status='confirmed'/); +}); + +for(const [name,networks,text,homeCount] of [ + ['homepage network',[network],ads.text,1],['manager-only publisher in whole catalog',[network,manager],'MANAGERDOMAIN=manager.com',0], +]) test('conditional ads.txt reuses the full parse for '+name,async t=>{ + const db=fixture(t); let pages=0; + const first=await verifyPublisher(publisher,networks,{fetchAdsTxt:async()=>({...ads,text,etag:'"v1"'}),fetchHomepage:async()=>{pages++;return page;}}); + saveLocalVerification(db,{id:1},first); + const stored=db.prepare('select * from publishers').get();stored.last_check_evidence=JSON.parse(stored.last_check_evidence);stored.discovery_sources=[]; + pages=0;let calls=0; + const second=await verifyPublisher(stored,networks,{fetchAdsTxt:async(_,conditional)=>{calls++;assert.equal(conditional.etag,'"v1"');return {outcome:'not_modified',httpStatus:304};}, + fetchHomepage:async()=>{pages++;return {html:'

Real home, tag removed

'};}}); + assert.equal(calls,1);assert.equal(pages,homeCount);assert.equal(second.sha256,'hash'); + assert.deepEqual(second.inconclusiveNetworkIds,[]); + assert.deepEqual(second.matches.confirmed.map(m=>m.networkId),homeCount?[]:[manager.id]); + assert.ok(second.evidence.ads_txt_cache.parsed.sellers); +}); + +test('cached ads.txt still needs a fresh homepage and cannot renew confirmation on a challenge',async()=>{ + const first=await verify(page,async()=>({...ads,etag:'v1'})); + let pages=0; + const second=await verifyPublisher({...publisher,last_check_evidence:first.evidence},[network],{ + fetchAdsTxt:async()=>({outcome:'not_modified'}),fetchHomepage:async()=>{pages++;return null;}, + }); + assert.equal(pages,1);assert.deepEqual(second.inconclusiveNetworkIds,[32]);assert.equal(second.matches.confirmed.length,0); +}); + +test('review-promoted corrections require both the original seller line and live page signature',async()=>{ + const mgid={id:33,slug:'mgid',ad_system_domains:['mgid.com'],program_signatures:[{kind:'html_substring',value:'jsc.mgid.com'}]}; + const row={...damaged,ad_network_id:33,verification_method:'ads_txt_and_page_signature', + evidence:{...damaged.evidence,matched_signal:'mgid.com,123,DIRECT',page_signature:'page:jsc.mgid.com'}}; + const check=async text=>verifyPublisher(publisher,[mgid],{fetchAdsTxt:async()=>({...ads,text}),fetchHomepage:async()=>({html:''})}); + const positive=await check('mgid.com,123,DIRECT');assert.equal(correctionConfirmation(row,positive),'restore'); + assert.equal(correctionConfirmation(row,await check('mgid.com,456,DIRECT')),'live_signal_changed'); + assert.equal(correctionConfirmation({...row,evidence:{...row.evidence,page_signature:'page:other-tag'}},positive),'live_signal_changed'); +}); + +test('history repairs honor independent bounds, cursors and durable completion checkpoints',async()=>{ + const restored=domain=>({...damaged,normalized_domain:domain,status:'confirmed',consecutive_misses:0,evidence:restoration(damaged,'2026-11-13T01:00:00Z')}); + const rows=['a.example','b.example','c.example'].map(restored); + const first=await planCorrections(rows,[],{statsOnly:true,limit:100,historyLimit:1}); + assert.equal(first.repaired.length,1);assert.equal(first.nextAfterDomain,'a.example');assert.equal(first.counts.history_limit,2); + const next=await planCorrections(rows,[],{statsOnly:true,limit:1,afterDomain:first.nextAfterDomain}); + assert.equal(next.repaired.length,1);assert.equal(next.repaired[0].normalized_domain,'b.example'); + const retry=await planCorrections([{...rows[0],history_repaired_at:'2026-11-13'},rows[1]],[],{statsOnly:true,limit:1}); + assert.equal(retry.repaired[0].normalized_domain,'b.example');assert.equal(retry.counts.history_complete,1); + // History still needs repair if an interrupted stage was followed by a real verifier miss. + const interrupted=await planCorrections([{...rows[0],status:'stale',consecutive_misses:1}],[],{statsOnly:true}); + assert.equal(interrupted.repaired.length,1);assert.equal(interrupted.restores.length,0); +}); + +test('wrong environment and SQLite aliases are rejected before writable opening',async t=>{ + const {assertCorrectionStore,runCorrection}=await import('../../../scripts/ad-networks/correct-homepage-misses.mjs'); + const {writeFileSync,symlinkSync}=await import('node:fs'); + const dir=mkdtempSync(join(tmpdir(),'ad-store-'));t.after(()=>rmSync(dir,{recursive:true,force:true})); + const source=join(dir,'source.sqlite'),other=join(dir,'other.sqlite'),alias=join(dir,'alias.sqlite'); + writeFileSync(source,'');writeFileSync(other,'');symlinkSync(other,alias); + const options={environment:'development',apply:true,store:'sqlite',sqlite:source}; + assert.throws(()=>assertCorrectionStore(options,{AD_NETWORKS_RUNTIME_ENVIRONMENT:'production',AD_NETWORK_DB_PATH:source}),/environment mismatch/); + assert.throws(()=>assertCorrectionStore(options,{}),/require AD_NETWORKS_RUNTIME_ENVIRONMENT/); + assert.throws(()=>assertCorrectionStore({...options,sqlite:alias},{AD_NETWORKS_RUNTIME_ENVIRONMENT:'development',AD_NETWORK_DB_PATH:source}),/source path/); + assert.doesNotThrow(()=>assertCorrectionStore(options,{AD_NETWORKS_RUNTIME_ENVIRONMENT:'development',AD_NETWORK_DB_PATH:source})); + await assert.rejects(runCorrection({...options,environment:'production'}),/--confirm/); +}); + +test('each domain is admitted before the deadline and unstarted leases are released',async()=>{ + const {runVerificationBatch}=await import('../../../scripts/ad-networks/verification-batch.mjs'); + let now=0;const started=[],released=[]; + await runVerificationBatch([1,2,3,4].map(id=>({id})),{concurrency:1,deadline:10,now:()=>now, + process:async row=>{started.push(row.id);now+=6;},release:async id=>released.push(id)}); + assert.deepEqual(started,[1,2]);assert.deepEqual(released,[3,4]); + await runVerificationBatch([{id:5}],{concurrency:1,deadline:10,now:()=>10, + process:async()=>assert.fail('must not start at deadline'),release:async id=>released.push(id)}); + assert.deepEqual(released,[3,4,5]); +}); + +test('interrupted SQLite source stage rolls back and can safely restart before sync',async t=>{ + const db=fixture(t),row={...db.prepare('select * from relationships').get(),normalized_domain:'publisher.example'}; + const planned={...row,correction_evidence:restoration(row,'2026-11-13T01:00:00Z')}; + db.exec('begin immediate');applySqliteCorrection(db,{restores:[planned]});db.exec('rollback'); + assert.deepEqual({...db.prepare('select * from relationships').get()},Object.fromEntries(Object.entries(row).filter(([key])=>key!=='normalized_domain'))); + db.exec('begin immediate');applySqliteCorrection(db,{restores:[planned]});db.exec('commit'); + const restored=db.prepare('select * from relationships').get(); + const resume=await planCorrections([{...restored,normalized_domain:row.normalized_domain}],[],{verify:async()=>assert.fail('restored source must not refetch')}); + assert.equal(resume.restores.length,0);assert.equal(resume.repaired.length,1); +}); + +test('a failed worker releases unstarted leases after all admitted workers settle',async()=>{ + const {runVerificationBatch}=await import('../../../scripts/ad-networks/verification-batch.mjs'); + const released=[],finished=[]; + await assert.rejects(runVerificationBatch([1,2,3].map(id=>({id})),{concurrency:2,deadline:100,now:()=>0, + process:async row=>{if(row.id===1)throw Error('save failed');await new Promise(resolve=>setImmediate(resolve));finished.push(row.id);}, + release:async id=>{assert.deepEqual(finished,[2]);released.push(id);}}),/save failed/); + assert.deepEqual(released,[3]); +}); + +test('public stale relationship reports unknown freshness and its last successful confirmation',async()=>{ + const {loadTypeScript}=await import('../../helpers/load-typescript.mjs'); + const view=loadTypeScript(new URL('../../../app/(site)/(tools)/ad-network-checker/ad-network-checker-view.ts',import.meta.url),{}); + assert.equal(view.statusLabel({status:'stale',last_confirmed_at:'2026-10-01T00:00:00Z'}),'Verification stale; last confirmed Oct 1, 2026'); +}); + +test('a declared subdomain cache never supplies validators for the root ads.txt file',async()=>{ + const {parseAdsTxt}=await import('../../../scripts/ad-networks/ads-txt.mjs'); + const rootAds={...ads,text:'SUBDOMAIN=blog.publisher.example\n'+ads.text,checkedUrl:'https://publisher.example/ads.txt'}; + const cache={parsed:parseAdsTxt('manager.com,999,DIRECT'),etag:'subdomain-v1',checkedUrl:'https://blog.publisher.example/ads.txt'}; + const result=await verifyPublisher({...publisher,normalized_domain:'blog.publisher.example',last_check_evidence:{ads_txt_cache:cache}},[network],{ + fetchAdsTxt:async(_,conditional)=>{assert.equal(conditional.etag,undefined);return rootAds;}, + fetchDeclaredSubdomainAdsTxt:async()=>({...ads,checkedUrl:'https://blog.publisher.example/ads.txt'}),fetchHomepage:async()=>page, + }); + assert.equal(result.matches.confirmed[0].networkId,32);assert.ok(result.adsTxtSummary); +});