Average DR of rated sites only: {shortName(name)} vs other big networks
+{name} sites by Domain Rating. The highlighted bar holds the median.
` log lines. Missed runs catch up by selecting due work at the next hour. The existing log rotation and automation manifest cover `logs/domain-rating.log`. Failure exits nonzero and the existing alert path handles it.
+
+After collection (including a failed partial run), `refresh.mjs` syncs all changed reviewed rows through the existing 500-row PostgreSQL batches, then calls the existing origin revalidation step once. There is no per-domain sync. SQLite candidates without a reviewed relationship are not sent to PostgreSQL. Sync hash checkpoints make retries harmless. The new migration excludes only DR and its checked date from the frontend source projection, so DR-only updates produce no cache event/source revision and no Cloudflare rebuild. Other enrichment or relationship changes in the same sync still queue their normal refresh. DR values are public on pages and APIs with linked Ahrefs credit. The daily snapshot at 00:10 UTC always inserts a dated row and queues a refresh, publishing new DR values daily even when the counts stay unchanged. Other public record changes can publish DR sooner.
+
+When rolling out this lock change, stop both Ahrefs jobs before updating their code so old and new lock paths cannot overlap. Keep the existing state file and use the same released helper for both jobs. Before enabling, the lead must release migration `20261114130000_ad_network_domain_rating.sql`, add the Infisical key, and update the released OVH checkout. Run a small bounded job through the wrapper and check its new start/end lines and progress JSON. Then add only the reviewed cron line, preserving verification's current pause. Confirm the dashboard lists `stack127-vps/domain-rating`. No server, cron or Infisical change was made during implementation.
+
+The local smoke check measured two managed development domains using a real locally available key in a temporary SQLite store. Repeating it processed zero domains. The SQL migration, DR-only cache behavior, groups, filters and summaries were tested inside a development transaction and rolled back. The installed wrapper and production permissions still need the lead's release check.
+
+Any DR answer must carry [Domain Rating by Ahrefs](https://ahrefs.com), say it is a third-party estimate, and show measured coverage. The [Ahrefs license](https://ahrefs.com/legal/domain-rating-license) applies. Private per-site results are for the owner. Future public plans expose network summaries only, never per-site DR values or DR dataset exports.
diff --git a/scripts/ad-networks/ovh/crontab.txt b/scripts/ad-networks/ovh/crontab.txt
index 33855e78..2859c955 100644
--- a/scripts/ad-networks/ovh/crontab.txt
+++ b/scripts/ad-networks/ovh/crontab.txt
@@ -9,3 +9,5 @@
0 4 * * 3 /srv/stack127/scripts/ad-networks/ovh/run-job.sh publisher-names --limit 5000 --time-budget-minutes 30
0 4 5 * * /srv/stack127/scripts/ad-networks/ovh/run-job.sh traffic --limit 5000 --time-budget-minutes 30
0 5 * * * /srv/stack127/scripts/ad-networks/ovh/run-job.sh backup
+# DR: bounded sequential work; leave 00:00-06:59 UTC for daily jobs and releases.
+5 7-23 * * * /srv/stack127/scripts/ad-networks/ovh/run-job.sh domain-rating --limit 500 --time-budget-minutes 30
diff --git a/scripts/ad-networks/refresh.mjs b/scripts/ad-networks/refresh.mjs
index 05da9b76..1848d7ae 100644
--- a/scripts/ad-networks/refresh.mjs
+++ b/scripts/ad-networks/refresh.mjs
@@ -5,12 +5,13 @@ import { homedir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { parseArgs } from 'node:util';
+import { ahrefsStateFile } from '../stack127/ahrefs-dr.mjs';
import { defaultLocalDatabasePath } from './local-db.mjs';
import { jobAlert, safeError } from './alerts.mjs';
const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
export const refreshJobs = ['sync', 'snapshot', 'verification', 'discovery', 'catalog', 'backup',
- 'platform', 'publisher-names', 'traffic', 'domain-age', 'ads-txt-summary', 'niche-export', 'rdap-import'];
+ 'platform', 'publisher-names', 'traffic', 'domain-age', 'domain-rating', 'ads-txt-summary', 'niche-export', 'rdap-import'];
export function refreshArgs(argv) {
const { values } = parseArgs({ args: argv, options: {
@@ -20,6 +21,7 @@ export function refreshArgs(argv) {
'state-directory': { type: 'string', default: join(homedir(), '.local/state/stack127/ad-networks') },
concurrency: { type: 'string', default: '4' }, limit: { type: 'string', default: '500' },
'time-budget-minutes': { type: 'string', default: '10' }, 'source-sqlite': { type: 'string' },
+ 'pace-ms': { type: 'string', default: '3000' },
'sync-only': { type: 'boolean', default: false }, full: { type: 'boolean', default: false },
} });
if (!refreshJobs.includes(values.job)) throw new Error(`Choose --job ${refreshJobs.join('|')}`);
@@ -28,6 +30,7 @@ export function refreshArgs(argv) {
for (const [key, max] of [['concurrency', 8], ['limit', 5000], ['time-budget-minutes', 45]]) {
if (!/^[1-9]\d*$/.test(values[key]) || Number(values[key]) > max) throw new Error(`${key} must be 1 to ${max}`);
}
+ if (!/^\d+$/.test(values['pace-ms']) || +values['pace-ms'] < 3000 || +values['pace-ms'] > 60000) throw new Error('pace-ms must be 3000 to 60000');
if (values.job === 'rdap-import' && !values['source-sqlite']) throw new Error('rdap-import needs --source-sqlite');
return { ...values, 'local-database': resolve(values['local-database']), 'state-directory': resolve(values['state-directory']) };
}
@@ -52,6 +55,7 @@ export function refreshCommand(options) {
'--run-directory', join(state, 'niche/input'), '--progress-file', join(state, 'niche-export.json')] };
return { script: options.job, args: [...local, '--concurrency', options.concurrency, '--limit', options.limit,
'--time-budget-minutes', options['time-budget-minutes'], '--progress-file', join(state, `${options.job}.json`),
+ ...(options.job === 'domain-rating' ? ['--pace-ms', options['pace-ms'], '--ahrefs-state-file', ahrefsStateFile(options.environment)] : []),
...(options.job === 'traffic' ? ['--cache-directory', join(state, 'crux')] : []),
...(options.job === 'publisher-names' ? ['--refresh'] : [])] };
}
diff --git a/scripts/ad-networks/sync.mjs b/scripts/ad-networks/sync.mjs
index 756d03fb..132eaa4b 100644
--- a/scripts/ad-networks/sync.mjs
+++ b/scripts/ad-networks/sync.mjs
@@ -14,7 +14,7 @@ const reviewed = new Set(['confirmed', 'stale', 'ended']);
const reviewedWhere = (prefix = '') => `(${prefix}status in ('confirmed','stale') or (${prefix}status='ended' and ${prefix}first_confirmed_at is not null))`;
const publisherFields = ['normalized_domain','display_domain','first_seen_at','last_discovered_at','last_checked_at',
'check_status','last_check_outcome','last_check_evidence','last_successful_check_at','discovery_sources'];
-export const enrichmentFields = ['niche','publisher_name','platform','domain_created_on','traffic_group','ads_txt_summary','favicon'];
+export const enrichmentFields = ['niche','publisher_name','platform','domain_created_on','domain_rating','domain_rating_checked_on','traffic_group','ads_txt_summary','favicon'];
// Recurring model work on homelab owns existing target niches. SQLite can seed
// a newly inserted publisher, but a full collector sync must never replace one.
export const publisherUpdateFields = fields => fields.filter(field => !['normalized_domain','niche'].includes(field));
diff --git a/scripts/launch-directories/README.md b/scripts/launch-directories/README.md
index 01bf1a3e..fb395180 100644
--- a/scripts/launch-directories/README.md
+++ b/scripts/launch-directories/README.md
@@ -51,7 +51,8 @@ Facts imports move row timestamps. Prepare a fresh correction file after importi
- [Ahrefs free DR endpoint](https://docs.ahrefs.com/en/api/reference/public/get-domain-rating-free): authenticated GET, explicit `version=dr3`, sequential requests at least three seconds apart. Floats round to the existing whole-number DR column. HTTP 429 backs off for 30 then 60 seconds, honors longer `Retry-After` values, and stops DR lookups after two retries or a requested wait over 60 seconds. HTTP 401/403 stops DR lookups immediately. Tranco can still finish. Exit 2 reports a partial run; successes can be saved without clearing failed metrics.
- [Tranco](https://tranco-list.eu/): resolve the latest standard list ID, then download its immutable top-million CSV. Validate all one million ordered rows, including Tranco's reserved wildcard placeholders, before treating an absent registrable domain as `below_1m`. Buckets match the reviewed facts importer. No extra ZIP dependency is needed.
-- Fetches hold no database transaction or connection. A single short write statement updates changed metrics, guarded by exact row versions. Existing triggers move public dates, queue parent and child refreshes, and change source fingerprints. The Node cache worker starts the usual Cloudflare rebuild; the job never purges or deploys itself.
+- Both Ahrefs jobs use `/srv/stack127-data/state/ahrefs.sqlite` for durable state and a separate `ahrefs.sqlite.lock` for the process-held request lock. Each request commits its pacing reservation before HTTP starts. A 429 commits its cooldown before body cleanup or callbacks. A hard kill releases the lock and retains committed state. A kill before the cooldown commit retains only the pacing reservation. Stop both jobs before rolling out this lock change, keep the state file, and update both to the same helper. Authentication stops apply only to the current requester.
+- Fetches hold no PostgreSQL transaction or connection. A single short write statement updates changed metrics, guarded by exact row versions. Existing triggers move public dates, queue parent and child refreshes, and change source fingerprints. The Node cache worker starts the usual Cloudflare rebuild; the job never purges or deploys itself.
- The [Ahrefs license](https://ahrefs.com/legal/domain-rating-license) requires the visible linked credit "Domain Rating by Ahrefs" beside displayed DR data. The pages use `ahrefs-credit.tsx` for it; keep it beside any new place that shows DR.
```sh
diff --git a/scripts/launch-directories/refresh-metrics.mjs b/scripts/launch-directories/refresh-metrics.mjs
index 82349f8c..cac48ed6 100644
--- a/scripts/launch-directories/refresh-metrics.mjs
+++ b/scripts/launch-directories/refresh-metrics.mjs
@@ -2,7 +2,8 @@
import { readFileSync } from "node:fs";
import { pathToFileURL } from "node:url";
import { parseEnv } from "node:util";
-import { setTimeout as sleep } from "node:timers/promises";
+import { ahrefsRequester, ahrefsStateFile } from "../stack127/ahrefs-dr.mjs";
+export { ahrefsRequester, parseDomainRating, retryDelay } from "../stack127/ahrefs-dr.mjs";
import pg from "pg";
import { getDomain } from "tldts";
import { factsArgs } from "./import-facts.mjs";
@@ -64,59 +65,9 @@ export async function latestTranco(wantedDomains, { fetchImplementation = fetch
return { list_id: list.list_id, ranks: parseTrancoCsv(await boundedText(response, 40 * 1024 * 1024), wantedDomains) };
}
-export function parseDomainRating(body) {
- const value = body?.domain_rating?.domain_rating;
- if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 100) throw new Error("Invalid Ahrefs Domain Rating response");
- // The existing schema and UI use whole-number DR. Ahrefs returns a float.
- return Math.round(value);
-}
-
-export function retryDelay(header, attempt, now = Date.now()) {
- const seconds = header && /^\d+(?:\.\d+)?$/.test(header.trim()) ? Number(header) * 1000 : null;
- const date = header && seconds === null ? Date.parse(header) - now : null;
- return Math.max(30000 * 2 ** attempt, seconds ?? (Number.isFinite(date) ? date : 0));
-}
-
-/** One requester per run. All domains and retries share its pacing clock. */
-export function ahrefsRequester({ apiKey, fetchImplementation = fetch, sleepImplementation = sleep,
- now = Date.now, paceMs = 3000, maxRetries = 2, maxBackoffMs = 60000 } = {}) {
- if (!apiKey?.trim()) throw new Error("AHREFS_API_KEY is required");
- let nextRequestAt = 0;
- let stopped = null;
- return async domain => {
- if (stopped) return { stopped };
- for (let attempt = 0; ; attempt++) {
- const wait = Math.max(0, nextRequestAt - now());
- if (wait) await sleepImplementation(wait);
- const url = new URL("https://api.ahrefs.com/v3/public/domain-rating-free");
- url.search = new URLSearchParams({ target: domain, version: "dr3", output: "json" });
- let response;
- try {
- response = await fetchImplementation(url, { headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json" }, signal: AbortSignal.timeout(30000), redirect: "error" });
- } catch { nextRequestAt = now() + paceMs; return { error: "Ahrefs request failed" }; }
- nextRequestAt = now() + paceMs;
- if (response.status === 429) {
- const delay = retryDelay(response.headers.get("retry-after"), attempt, now());
- await response.body?.cancel().catch(() => {});
- if (attempt >= maxRetries || delay > maxBackoffMs) { stopped = "rate_limited"; return { stopped }; }
- nextRequestAt = now() + Math.max(paceMs, delay);
- continue;
- }
- if (response.status === 401 || response.status === 403) {
- await response.body?.cancel().catch(() => {});
- stopped = "authentication_failed";
- return { stopped };
- }
- if (!response.ok) { await response.body?.cancel().catch(() => {}); return { error: `Ahrefs HTTP ${response.status}` }; }
- try { return { value: parseDomainRating(JSON.parse(await boundedText(response, 64000))) }; }
- catch { return { error: "Invalid Ahrefs response" }; }
- }
- };
-}
-
const metricKeys = ["domain_rating", "domain_rating_checked_on", "traffic_bucket", "traffic_checked_on"];
export async function collectMetrics(records, { apiKey, checkedOn = new Date().toISOString().slice(0, 10),
- loadTranco = latestTranco, requestRating = ahrefsRequester({ apiKey }) } = {}) {
+ loadTranco = latestTranco, stateFile, requestRating = ahrefsRequester({ apiKey, stateFile }) } = {}) {
const domains = new Map();
let tranco;
const failures = [];
@@ -194,7 +145,7 @@ async function main() {
where is_published order by slug limit $1`, [Number(args.limit ?? 2147483647)])).rows.map(row => row.record);
} finally { await pool.end(); }
// No database connection or transaction is held during external lookups.
- const { patches, ...summary } = await collectMetrics(records, { apiKey });
+ const { patches, ...summary } = await collectMetrics(records, { apiKey, stateFile: ahrefsStateFile(args.environment) });
let result = { changed: 0, conflicts: [] };
if (args.apply && patches.length) {
const writer = new pg.Pool(poolOptions);
diff --git a/scripts/stack127/ahrefs-dr.mjs b/scripts/stack127/ahrefs-dr.mjs
new file mode 100644
index 00000000..cf5c9873
--- /dev/null
+++ b/scripts/stack127/ahrefs-dr.mjs
@@ -0,0 +1,134 @@
+import { setTimeout as sleep } from "node:timers/promises";
+import { DatabaseSync } from "node:sqlite";
+import { mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { homedir } from "node:os";
+
+// Both OVH jobs run as the same user and must pass this same host-wide file.
+export const AHREFS_STATE_FILE = "/srv/stack127-data/state/ahrefs.sqlite";
+export function ahrefsStateFile(environment) {
+ return environment === "production" ? AHREFS_STATE_FILE : join(homedir(), ".local/state/stack127/ahrefs.sqlite");
+}
+
+/** Hold the request lock in a separate SQLite file so state writes can commit
+ * immediately. Process death releases this lock without undoing committed state.
+ * Poll SQLITE_BUSY asynchronously so abort signals and the run budget still work. */
+async function withSharedState(file, signal, work) {
+ mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
+ const lock = new DatabaseSync(`${file}.lock`);
+ const deadline = Date.now() + 120000;
+ try {
+ for (;;) {
+ signal?.throwIfAborted();
+ try { lock.exec("begin immediate"); break; }
+ catch (error) {
+ if (error.errcode !== 5 || Date.now() >= deadline) throw error;
+ await sleep(100, undefined, { signal });
+ }
+ }
+ const db = new DatabaseSync(file);
+ try {
+ db.exec(`pragma synchronous=FULL;
+ create table if not exists request_state (
+ id integer primary key check(id=1), next_request_at integer not null, cooldown_until integer not null)`);
+ const row = db.prepare("select * from request_state where id=1").get();
+ const state = { nextRequestAt: row?.next_request_at ?? 0, cooldownUntil: row?.cooldown_until ?? 0 };
+ const write = db.prepare(`insert into request_state values(1,?,?) on conflict(id) do update set
+ next_request_at=excluded.next_request_at,cooldown_until=excluded.cooldown_until`);
+ // No open state transaction: every run commits before control returns.
+ const persist = () => write.run(state.nextRequestAt, state.cooldownUntil);
+ return await work(state, persist);
+ } finally { db.close(); }
+ } finally { lock.close(); }
+}
+
+async function boundedText(response, maxBytes) {
+ if (Number(response.headers.get("content-length")) > maxBytes) throw new Error("Response is too large");
+ const chunks = [];
+ let bytes = 0;
+ for await (const chunk of response.body) {
+ bytes += chunk.length;
+ if (bytes > maxBytes) { await response.body.cancel().catch(() => {}); throw new Error("Response is too large"); }
+ chunks.push(chunk);
+ }
+ return Buffer.concat(chunks).toString("utf8");
+}
+
+export function parseDomainRating(body) {
+ const value = body?.domain_rating?.domain_rating;
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 100) throw new Error("Invalid Ahrefs Domain Rating response");
+ // The existing schema and UI use whole-number DR. Ahrefs returns a float.
+ return Math.round(value);
+}
+
+export function retryDelay(header, attempt, now = Date.now()) {
+ const seconds = header && /^\d+(?:\.\d+)?$/.test(header.trim()) ? Number(header) * 1000 : null;
+ const date = header && seconds === null ? Date.parse(header) - now : null;
+ return Math.max(30000 * 2 ** attempt, seconds ?? (Number.isFinite(date) ? date : 0));
+}
+
+/** All domains, retries and OVH jobs share one pacing and Retry-After clock. */
+export function ahrefsRequester({ apiKey, fetchImplementation = fetch, sleepImplementation = sleep,
+ now = Date.now, paceMs = 3000, maxRetries = 2, maxBackoffMs = 60000, signal, onRateLimit, stateFile } = {}) {
+ if (!apiKey?.trim()) throw new Error("AHREFS_API_KEY is required");
+ if (!Number.isFinite(paceMs) || paceMs < 3000) throw new Error("Ahrefs pacing must be at least 3000ms");
+ const localState = { nextRequestAt: 0, cooldownUntil: 0 };
+ const withState = work => stateFile ? withSharedState(stateFile, signal, work) : work(localState, () => {});
+ let stopped = null;
+ return async domain => {
+ if (stopped) return { stopped };
+ signal?.throwIfAborted();
+ for (let attempt = 0; ; attempt++) {
+ const result = await withState(async (state, persist) => {
+ // Long cooldowns stop promptly instead of sleeping through a scheduled run.
+ if (state.cooldownUntil - now() > maxBackoffMs) {
+ onRateLimit?.({ retryAt: state.cooldownUntil });
+ return { stopped: "rate_limited" };
+ }
+ const wait = Math.max(0, state.nextRequestAt - now(), state.cooldownUntil - now());
+ if (wait) await sleepImplementation(wait, undefined, { signal });
+ signal?.throwIfAborted();
+ const url = new URL("https://api.ahrefs.com/v3/public/domain-rating-free");
+ url.search = new URLSearchParams({ target: domain, version: "dr3", output: "json" });
+ // Reserve the next slot before HTTP starts, even if this process dies
+ // before receiving a response. Completion extends the same reservation.
+ state.nextRequestAt = now() + paceMs;
+ persist();
+ let response;
+ try {
+ response = await fetchImplementation(url, { headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json" }, signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(30000)]) : AbortSignal.timeout(30000), redirect: "error" });
+ } catch {
+ state.nextRequestAt = now() + paceMs;
+ persist();
+ signal?.throwIfAborted();
+ return { error: "Ahrefs request failed" };
+ }
+ // Keep the lock until the body has been consumed/cancelled too.
+ try {
+ if (response.status === 429) {
+ const delay = retryDelay(response.headers.get("retry-after"), attempt, now());
+ state.cooldownUntil = now() + Math.max(paceMs, delay);
+ // Commit before callbacks or body cleanup can throw, hang or die.
+ persist();
+ await response.body?.cancel().catch(() => {});
+ onRateLimit?.({ retryAt: state.cooldownUntil });
+ return attempt >= maxRetries || delay > maxBackoffMs ? { stopped: "rate_limited" } : { retry: true };
+ }
+ if (response.status === 401 || response.status === 403) {
+ await response.body?.cancel().catch(() => {});
+ return { stopped: "authentication_failed" };
+ }
+ if (!response.ok) { await response.body?.cancel().catch(() => {}); return { error: `Ahrefs HTTP ${response.status}` }; }
+ try { return { value: parseDomainRating(JSON.parse(await boundedText(response, 64000))) }; }
+ catch { signal?.throwIfAborted(); return { error: "Invalid Ahrefs response" }; }
+ } finally {
+ state.nextRequestAt = now() + paceMs;
+ persist();
+ }
+ });
+ if (result.retry) continue;
+ if (result.stopped) stopped = result.stopped;
+ return result;
+ }
+ };
+}
diff --git a/stack127_docs/mcp-and-api.md b/stack127_docs/mcp-and-api.md
index f18f08f5..34cd2555 100644
--- a/stack127_docs/mcp-and-api.md
+++ b/stack127_docs/mcp-and-api.md
@@ -20,14 +20,14 @@ For every row below, the REST route is **POST `/api/v1/ad-networks/`*
| `ad_networks_overview` | Start here: sites checked, distinct listed/managed totals, freshness, measurement, definitions, caveats, allowed values, tool guide. Input `{}`. |
| `list_ad_networks` | Counts, domain-age median and measured base, sourced joining/payment facts and checked dates. Filter published numeric minimum, traffic unit, or public numeric rule; sort by listed, managed, name or minimum traffic. |
| `get_ad_network` | One `network` slug, facts/sources, counts, domain-age buckets/median, registration years, niche/size/platform mix, co-listing and daily listed snapshots. |
-| `compare_ad_networks` | 2–6 distinct `networks`, one `dimension`: niche, registration_year, age_bucket, size_group or platform. Counts and shares use each network's filtered distinct-site base. |
+| `compare_ad_networks` | 2–6 distinct `networks`, one `dimension`: niche, registration_year, age_bucket, size_group, platform or domain_rating. Counts and shares use each network's filtered distinct-site base. |
| `count_sites` | Distinct-site counter; up to two `group_by` dimensions, including network. Global totals deduplicate sites listed at several networks. |
| `find_sites` | Public domains and listed memberships, types, confirmation dates, niche, platform, RDAP date and CrUX size. `limit` 1–200, `sort` domain/newest/oldest, opaque `cursor`. Keep filters/sort unchanged between pages. |
| `lookup_site` | Bare `domain`: current/past relationships, first/last confirmation and end dates, public ads.txt summary, classification, age, size and events. Unknown domain returns 404. |
| `network_movements` | Optional `networks`, inclusive UTC `from` and `to`; at most 366 days apart. Observed joins/leaves/newly-found only, with a strong fault warning. |
| `run_readonly_sql` | Disabled for this release. Future private SQL requires a dedicated database login and a separate switch. A single SELECT against the documented safe views. Never available to a public principal. |
-`count_sites`, `find_sites` and `compare_ad_networks` share filters: `networks` (OR), `relationship` (`listed`, default, or `managed`), `niche`, `registered_after`/`registered_before` (exclusive RDAP dates), `min_age`/`max_age` (inclusive completed years), `size_group`, `platform`, and `status` (`confirmed` or `stale`). Other filters combine with AND. Missing classifications remain null groups. `find_sites` returns all listed networks for each matched site, even if only one network matched the filter.
+`count_sites`, `find_sites` and `compare_ad_networks` share filters: `networks` (OR), `relationship` (`listed`, default, or `managed`), `niche`, `registered_after`/`registered_before` (exclusive RDAP dates), `min_age`/`max_age` (inclusive completed years), `size_group`, `platform`, `min_domain_rating`/`max_domain_rating` (inclusive integers 0–100; exclude unknowns), and `status` (`confirmed` or `stale`). Other filters combine with AND. Missing classifications remain null groups. `find_sites` returns all listed networks for each matched site, even if only one network matched the filter.
Listed means confirmed or stale. Managed means a listed `primary_or_exclusive_manager` relationship. This is the requested relationship-based measure; it differs from the public page's manager resolver, which also reconciles retained manager-domain evidence and ambiguous signals. Do not call those two measures interchangeable. The API reuses the public listed-status rule, domain-age bucket thresholds/median expression, source facts and observed-event definitions. Minimum traffic in sessions and pageviews is not interchangeable; use `traffic_unit`. `has_public_rule` specifically means a published numeric monthly traffic minimum, including an explicit zero; read full joining facts for alternatives.
@@ -43,6 +43,18 @@ Colors and font stacks come from `app/globals.css` and the ad network visualizat
Connected clients must refresh their tool list to see `get_brand_kit` and the updated server instructions.
+## Domain Rating (prepared; requires the DR migration and release)
+
+`find_sites` and `lookup_site` include `domain_rating` (rounded integer 0–100 or null) and `domain_rating_checked_on` (successful UTC YYYY-MM-DD or null). A failed lookup does not clear a prior value or advance its date. `stack127_data.ad_network_publishers` projects these two private fields; anonymous/authenticated roles still have no view access. `dataset_summary` adds `domain_rating_base` and `domain_rating_total`, measured and total distinct listed publishers respectively.
+
+`count_sites` and `compare_ad_networks` accept `min_domain_rating` and `max_domain_rating`; filters combine with the existing filters. `domain_rating` is a grouping/comparison dimension with exclusive buckets `0–9`, `10–19`, `20–29`, `30–49`, `50–69`, `70–100`, `unknown`. Zero is measured. Unknowns remain in the unfiltered base and unknown bucket; min/max filters exclude them. Grouped answers include `domain_rating_coverage` for all filtered distinct sites; network groups also include `network_domain_rating_coverage` with each network's measured count and full filtered base, even when group rows truncate.
+
+Every network in `get_ad_network` and `list_ad_networks` includes `median_domain_rating`, `average_domain_rating`, and `domain_rating_base`. Both summaries use measured listed sites only. The denominator is that row's `listed` count. Empty measured cohorts return null summaries and zero coverage. Network profiles also include the DR bucket distribution. Network memberships can overlap; do not add their measured counts to estimate dataset coverage.
+
+All successful answers, including optional private SQL, carry `data.domain_rating_metadata`: `attribution` = `Domain Rating by Ahrefs`, `url` = `https://ahrefs.com`, a ready-to-display linked `credit`, a `caveat` that DR is a third-party estimate, and `coverage` (`sites_with_rating`, `total_sites`, explicit dataset scope). This coverage describes all distinct listed sites, not just a page or SQL result; filtered groups and network rows provide their own bases. MCP text and structured content carry the same metadata. Clients must display [Domain Rating by Ahrefs](https://ahrefs.com) beside DR values/summaries, retain the third-party estimate caveat, and include coverage. DR estimates backlink strength; it is not traffic, revenue, quality or a recommendation.
+
+The [free endpoint](https://docs.ahrefs.com/en/api/reference/public/get-domain-rating-free) uses `dr3`. The [license](https://ahrefs.com/legal/domain-rating-license) allows integration with linked attribution, bars redistribution of original DR data or a competing product, and forbids circumventing limits. The authenticated MCP/REST integration stays private to the owner. The public website and bounded `/api/ad-networks/sites` and `/api/ad-networks/site` responses include per-site DR with linked attribution; their `attribution` metadata is present when a rating is returned, including zero. Public page summaries show rated/total coverage. Sitemaps and registry JSON-LD carry no DR, and there is no DR CSV export. No bulk DR export or resale is offered. Future public MCP/REST plans remain limited to derived summaries per network. See [the collector instructions](../scripts/ad-networks/ovh/README.md) for priority, pacing, source coverage and freshness.
+
## Authentication and future public plans
`authenticate(request) -> Principal` is the replacement point for future user API keys/OAuth and paid plans. Today both credentials map to `private-owner`, plan `private`. `PUBLIC_MODE` is a hardcoded false constant. There is no anonymous fallback, public switch, signup, billing endpoint, or public SQL access.
@@ -65,6 +77,7 @@ When this becomes a public product, it moves to its own subdomain and gets real
- **Sign-in:** OAuth for claude.ai and ChatGPT, so each user clicks Connect, signs in on stack127, and approves access. No secret URLs for public users. Per-user API keys for REST and header clients. Both replace `authenticate(request)`; the tools and their input and output stay the same.
- **Accounts and plans:** user accounts, plans (for example free and paid), billing, and per-plan limits on requests a day, rows per answer, and which tools are available. The existing rate-limit and usage-log design is per principal, so it extends to per user.
- **Read-only by design, with a second lock:** every tool is read-only today (read-only transactions, fixed parameterized queries, read-only views; the only write is our own rate-limit counter). Today the tools use the website's normal database login, which can write elsewhere, so the read-only transaction is the main protection. Before launch, an admin creates a dedicated database login with SELECT on the `stack127_data` views only (plus the counter table), and the public MCP and API use only that login, so even a code bug cannot write.
+- **DR in public plans:** expose only DR summaries per network with linked Ahrefs credit, third-party estimate caveat and measured coverage. Never expose per-site DR values, including in `find_sites`, `lookup_site`, downloads or dataset exports, under this planned API policy. The existing bounded website APIs are separate. Add plan-aware field projections before offering public MCP/REST access.
- **What stays private:** `run_readonly_sql` is never offered publicly. `publisher_name` and internal check fields are never returned. Whether site-by-site lists (`find_sites`, `lookup_site`) are paid-only is an owner decision to make before launch.
- **Before launch:** terms of use and an acceptable-use rule, public documentation for each tool, a status or uptime check, alerts on errors and abuse, and a review of data accuracy and its stated limits.
- **The owner's private access** moves to the same sign-in at launch, and the secret path is switched off.
diff --git a/stack127_docs/monitoring.json b/stack127_docs/monitoring.json
index bf34f977..2288c9da 100644
--- a/stack127_docs/monitoring.json
+++ b/stack127_docs/monitoring.json
@@ -3,10 +3,11 @@
"project": "stack127",
"about": "Entries for the Automation Monitor dashboard (https://monitor.ravitejaknts.com). One entry per scheduled job, workflow or service. key: / (machines: homelab, bloxodes-vps, stack127-vps, macbook-pro, macbook-air; job = cron run-job.sh name, systemd unit without .timer/.service, launchd label or T3 schedule title), github///, supabase/, or northflank/. name: short plain name. group: the section it is listed under. about: one sentence on what it does, written for the owner. impact (optional): what breaks if it stops. records (optional): stats_job_runs.job_name values it writes. ignore_records: job names that are sub-steps of other jobs.",
"automations": [
- {"key": "stack127-vps/launch-directory-metrics", "group": "Launch directories", "name": "Refresh launch directory ratings", "about": "On the 2nd of each month at 06:00 UTC, refreshes Ahrefs Domain Rating and Tranco popularity for published launch directories.", "impact": "Launch directory ratings and popularity become stale."},
+ {"key": "stack127-vps/launch-directory-metrics", "group": "Launch directories", "name": "Refresh launch directory ratings", "about": "On the 2nd of each month at 06:00 UTC, refreshes Ahrefs Domain Rating and Tranco popularity for published launch directories. Shares the process-held /srv/stack127-data/state/ahrefs.sqlite.lock request lock with ad-network DR collection. The separate ahrefs.sqlite commits pacing before HTTP and Retry-After before body cleanup or callbacks, retaining committed state after a crash.", "impact": "Launch directory ratings and popularity become stale."},
{"key": "stack127-vps/verification", "group": "Ad networks", "name": "Ad network site checks", "about": "Every hour at minute 40, checks up to 1,000 publisher sites (their ads.txt and pages), sends changed rows to production, and refreshes the site counts in titles and headings. Blocked or failed homepage checks never record departures, retry with backoff, and mark confirmation stale after 30 days without fresh evidence. It steps aside while a bigger job runs.", "impact": "Network pages stop gaining newly confirmed sites."},
{"key": "stack127-vps/snapshot", "group": "Ad networks", "name": "Daily ad network counts", "about": "Saves today's publisher counts for each ad network."},
{"key": "stack127-vps/platform", "group": "Ad networks", "name": "Publisher platform and favicon", "about": "Fills in each publisher's website platform and icon, and refreshes facts older than 30 days. Challenge pages remain unavailable."},
+ {"key": "stack127-vps/domain-rating", "group": "Ad networks", "name": "Publisher Domain Rating", "about": "At minute 05 of hours 07 through 23 UTC, checks up to 500 domains in 30 minutes; up to 8,500/day. OVH: 213,003 candidates, 123,361 listed, 39,823 managed; about 5 days for managed and 26 days for full first coverage. Both Ahrefs jobs share the process-held /srv/stack127-data/state/ahrefs.sqlite.lock request lock. The separate ahrefs.sqlite commits pacing before HTTP and cooldowns before body cleanup or callbacks, retaining committed state after a crash. DR-only sync keeps the frontend revision unchanged; the dated 00:10 daily snapshot queues the refresh that publishes DR daily. A sync overrun into :40 may skip hourly verification once; acceptable occasionally, investigate repeated skips.", "impact": "Public pages, public APIs and private MCP/REST stop gaining fresh Domain Rating estimates."},
{"key": "stack127-vps/domain-age", "group": "Ad networks", "name": "Publisher domain age", "about": "Looks up when new publisher domains were registered."},
{"key": "stack127-vps/ads-txt-summary", "group": "Ad networks", "name": "ads.txt summaries", "about": "Summarizes each publisher's ads.txt file for the network pages."},
{"key": "stack127-vps/discovery", "group": "Ad networks", "name": "Find new publishers", "about": "Every Tuesday, reads the ad networks' official seller lists and adds new publisher domains to check."},
diff --git a/supabase/migrations/20261114130000_ad_network_domain_rating.sql b/supabase/migrations/20261114130000_ad_network_domain_rating.sql
new file mode 100644
index 00000000..d512c430
--- /dev/null
+++ b/supabase/migrations/20261114130000_ad_network_domain_rating.sql
@@ -0,0 +1,120 @@
+begin;
+set local lock_timeout='10s';
+set local statement_timeout='120s';
+-- Add nullable columns without scanning existing publishers under ACCESS EXCLUSIVE.
+-- Reruns keep existing data, but reject an incompatible pre-existing schema.
+do $$
+declare column_name text; expected_type regtype; actual_type oid;
+begin
+ for column_name, expected_type in select * from (values
+ ('domain_rating', 'integer'::regtype), ('domain_rating_checked_on', 'date'::regtype)) as columns(name, type)
+ loop
+ select atttypid into actual_type from pg_attribute
+ where attrelid='public.ad_network_publishers'::regclass and attname=column_name and not attisdropped;
+ if found and actual_type <> expected_type::oid then
+ raise exception 'public.ad_network_publishers.% has type %, expected %', column_name, actual_type::regtype, expected_type;
+ end if;
+ end loop;
+end $$;
+alter table public.ad_network_publishers
+ add column if not exists domain_rating integer,
+ add column if not exists domain_rating_checked_on date;
+do $$ begin
+ if not exists(select 1 from pg_constraint where conrelid='public.ad_network_publishers'::regclass
+ and conname='ad_network_publishers_domain_rating_check') then
+ alter table public.ad_network_publishers add constraint ad_network_publishers_domain_rating_check
+ check(domain_rating between 0 and 100) not valid;
+ end if;
+end $$;
+comment on column public.ad_network_publishers.domain_rating is
+ 'Rounded Ahrefs dr3 estimate for pages and APIs. Display with linked Domain Rating by Ahrefs credit. Unknown is null.';
+
+create or replace view stack127_data.ad_network_publishers with (security_barrier=true) as
+ select id,normalized_domain,niche,platform,domain_created_on,traffic_group,last_successful_check_at,
+ case when ads_txt_summary is not null then jsonb_build_object(
+ 'lines',ads_txt_summary->'lines','direct',ads_txt_summary->'direct',
+ 'reseller',ads_txt_summary->'reseller','invalid',ads_txt_summary->'invalid',
+ 'top_sellers',ads_txt_summary->'top_sellers') end as ads_txt_summary,
+ p.domain_rating,p.domain_rating_checked_on
+ from public.ad_network_publishers p
+ where exists(select 1 from public.ad_network_publisher_relationships r
+ where r.publisher_id=p.id and r.status in ('confirmed','stale','ended'));
+revoke all on stack127_data.ad_network_publishers from public,anon,authenticated;
+
+create or replace view stack127_data.dataset_summary with (security_barrier=true) as
+ select count(*) filter(where last_successful_check_at is not null)::int as sites_checked,
+ min(last_successful_check_at) as oldest_site_check,max(last_successful_check_at) as newest_site_check,
+ greatest(max(last_successful_check_at),(select max(observed_at) from public.ad_network_relationship_events),
+ (select max(captured_at) from public.ad_network_daily_counts)) as as_of,
+ count(*) filter(where domain_rating is not null and exists(select 1 from public.ad_network_publisher_relationships r
+ where r.publisher_id=ad_network_publishers.id and r.status in ('confirmed','stale')))::int as domain_rating_base,
+ count(*) filter(where exists(select 1 from public.ad_network_publisher_relationships r
+ where r.publisher_id=ad_network_publishers.id and r.status in ('confirmed','stale')))::int as domain_rating_total
+ from public.ad_network_publishers;
+
+-- Pages and APIs display DR with linked attribution. DR-only writes do not
+-- move the frontend revision or enqueue a rebuild. The daily snapshot always
+-- inserts a dated row and queues the refresh that publishes new DR each day.
+-- Other enrichment and relationship changes still queue their normal refresh.
+create or replace function public.stack127_directory_record_value(p_table text, p_row jsonb)
+returns jsonb language plpgsql immutable security invoker set search_path = pg_catalog as $$
+begin
+ case p_table
+ when 'ad_network_publishers' then
+ return p_row - array['created_at','updated_at','next_check_at','lease_until','verification_priority','check_attempt_count','consecutive_failures','last_discovered_at','ads_txt_hash','ads_txt_etag','ads_txt_last_modified','domain_rating','domain_rating_checked_on'];
+ when 'ad_network_publisher_relationships' then
+ if p_row->>'status' not in ('confirmed','stale','ended') then return null; end if;
+ -- Verification mix and retained manager evidence are public report inputs.
+ return (p_row - array['created_at','updated_at','consecutive_misses','first_missed_at','last_missed_at','evidence'])
+ || jsonb_build_object('manager_evidence',jsonb_build_object(
+ 'manager_domain',p_row->'evidence'->'manager_domain',
+ 'matched_signal',p_row->'evidence'->'matched_signal'));
+ when 'ad_networks' then
+ return jsonb_build_object('id',p_row->'id','slug',p_row->'slug','name',p_row->'name',
+ 'official_url',p_row->'official_url','summary',p_row->'summary',
+ 'manager_domains',p_row->'manager_domains','ad_system_domains',p_row->'ad_system_domains');
+ when 'launch_directories' then
+ if p_row->>'is_published' <> 'true' then return null; end if;
+ return p_row - array['created_at','updated_at','research_notes'];
+ when 'terminal_apps' then
+ if p_row->>'published' <> 'true' then return null; end if;
+ return p_row - array['created_at','updated_at','self_host_app_id'];
+ when 'self_host_apps' then
+ if p_row->>'published' <> 'true' or p_row->>'self_hosting_status' <> 'self_hostable' then return null; end if;
+ return p_row - array['created_at','updated_at','data_checked_at','data_confidence','data_source_urls',
+ 'source_repo_id','github_repo_id','github_node_id','github_full_name','github_owner_type',
+ 'source_full_name','source_watchers','source_open_issues','source_languages','source_updated_at',
+ 'source_default_branch','source_repo_size_kb','source_disabled','source_is_fork','source_is_template',
+ 'github_watchers','github_open_issues','github_languages','github_updated_at','github_default_branch',
+ 'github_repo_size_kb','github_disabled','github_is_fork','github_is_template'];
+ when 'github_projects' then
+ return jsonb_build_object('id',p_row->'id','owner',p_row->'owner','repo',p_row->'repo',
+ 'stars',p_row->'stars','forks',p_row->'forks','primary_language',p_row->'primary_language','topics',p_row->'topics',
+ 'created_at',p_row->'created_at','pushed_at',p_row->'pushed_at','latest_release',p_row->'latest_release',
+ 'latest_release_at',p_row->'latest_release_at','last_commit_at',p_row->'last_commit_at',
+ 'checked_at',p_row->'checked_at','archived',p_row->'archived');
+ when 'self_host_app_install_options' then
+ return jsonb_build_object('app_id',p_row->'app_id','options',p_row->'options');
+ else
+ return p_row - array['created_at','updated_at','row_updated_at'];
+ end case;
+end;
+$$;
+revoke all on function public.stack127_directory_record_value(text,jsonb) from public, anon, authenticated;
+grant execute on function public.stack127_directory_record_value(text,jsonb) to service_role;
+
+commit;
+
+-- Release ACCESS EXCLUSIVE before scanning the existing rows. VALIDATE takes
+-- SHARE UPDATE EXCLUSIVE, allowing normal reads and writes during validation.
+-- https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-NOTES
+begin;
+set local lock_timeout='10s';
+set local statement_timeout='120s';
+do $$ begin
+ if exists(select 1 from pg_constraint where conrelid='public.ad_network_publishers'::regclass
+ and conname='ad_network_publishers_domain_rating_check' and not convalidated) then
+ alter table public.ad_network_publishers validate constraint ad_network_publishers_domain_rating_check;
+ end if;
+end $$;
+commit;
diff --git a/tests/data-api/core.test.mjs b/tests/data-api/core.test.mjs
index 035608a0..9af9db61 100644
--- a/tests/data-api/core.test.mjs
+++ b/tests/data-api/core.test.mjs
@@ -46,7 +46,7 @@ test('SQL parser rejects writes, batches, CTE mutations, locks, private fields b
'select * from (select slug from stack127_data.ad_networks) as n'])assert.equal(validateReadonlySql(sql),sql);
});
test('every core response carries definitions, caveats, filters, data time and citation',async()=>{
- const transaction=async run=>run({query:async sql=>sql.includes('dataset_summary')?[{as_of:'2026-11-14T00:00:00Z'}]:[{total:2}]});
+ const transaction=async run=>run({query:async sql=>sql.includes('dataset_summary')?[{as_of:'2026-11-14T00:00:00Z',domain_rating_base:1,domain_rating_total:2}]:[{total:2,domain_rating_base:1}]});
const answer=await executeTool('count_sites',{},transaction);
assert.equal(answer.data.total,2);assert.equal(answer.filters.relationship,'listed');assert.ok(answer.as_of);
assert.match(answer.definitions.not_ranked,/not small/);assert.ok(answer.definitions.managed);assert.ok(answer.definitions.shares);
@@ -86,7 +86,7 @@ test('SQL flag alone cannot run a query or expose a REST route',async()=>{
});
test('MCP text contains the full answer and applies the same byte limit after formatting',async()=>{
const {mcpAnswer}=await import('../../lib/data-api/mcp.ts');
- const answer=await executeTool('count_sites',{},async run=>run({query:async sql=>sql.includes('dataset_summary')?[{as_of:null}]:[{total:42}]}));
+ const answer=await executeTool('count_sites',{},async run=>run({query:async sql=>sql.includes('dataset_summary')?[{as_of:null,domain_rating_base:10,domain_rating_total:42}]:[{total:42,domain_rating_base:10}]}));
const result=mcpAnswer(answer);
assert.deepEqual(JSON.parse(result.content[0].text),result.structuredContent);
assert.equal(JSON.parse(result.content[0].text).data.total,42);
@@ -100,7 +100,7 @@ test('future SQL uses only its dedicated pool and read-only transaction, includi
const previous={app:globalThis.stack127DataApiPool,sql:globalThis.stack127DataSqlPool,flag:process.env.STACK127_DATA_SQL_ENABLED,url:process.env.STACK127_DATA_SQL_DATABASE_URL};
globalThis.stack127DataApiPool={connect:()=>assert.fail('arbitrary SQL used the app login')};
globalThis.stack127DataSqlPool={connect:async()=>({query:async(sql,values)=>{
- statements.push({sql,values});return {rows:sql.includes('dataset_summary')?[{as_of:null}]:[{value:1}]};
+ statements.push({sql,values});return {rows:sql.includes('dataset_summary')?[{as_of:null,domain_rating_base:10,domain_rating_total:42}]:[{value:1}]};
},release(){released++;}})};
Object.assign(process.env,{STACK127_DATA_SQL_ENABLED:'true',STACK127_DATA_SQL_DATABASE_URL:'postgresql://restricted:unused@localhost/research'});
try {
@@ -174,3 +174,29 @@ test('brand kit is listed in MCP and both front doors share its answer and authe
for(const key of keys){if(previousEnv[key]===undefined)delete process.env[key];else process.env[key]=previousEnv[key];}
}
});
+
+test('DR filters validate integer ranges, bind zero and combine with managed/network filters',()=>{
+ for(const f of [{min_domain_rating:-1},{max_domain_rating:101},{min_domain_rating:1.5},{min_domain_rating:50,max_domain_rating:20}])
+ assert.throws(()=>validateInput('count_sites',f),{status:400});
+ const f=filterSql(validateInput('count_sites',{networks:['mediavine'],relationship:'managed',min_domain_rating:0,max_domain_rating:69}));
+ assert.match(f.text,/p.domain_rating>=\$2/);assert.match(f.text,/p.domain_rating<=\$3/);
+ assert.deepEqual(f.values,[['mediavine'],0,69]);
+ assert.deepEqual(validateInput('count_sites',{group_by:['domain_rating']}).group_by,['domain_rating']);
+ assert.equal(validateInput('compare_ad_networks',{networks:['ezoic','mediavine'],dimension:'domain_rating'}).dimension,'domain_rating');
+});
+
+test('DR answers carry linked attribution, third-party caveat and measured coverage in MCP text too',async()=>{
+ const transaction=async run=>run({query:async sql=>sql.includes('dataset_summary')?
+ [{as_of:null,domain_rating_base:2,domain_rating_total:3}]:sql.includes('group by')?
+ [{group_1:'0–9',count:1},{group_1:'unknown',count:1}]:[{total:2,domain_rating_base:1}]});
+ const answer=await executeTool('count_sites',{group_by:['domain_rating']},transaction);
+ assert.deepEqual(answer.data.domain_rating_coverage,{sites_with_rating:1,total_sites:2,scope:'distinct sites matching all filters'});
+ assert.equal(answer.data.domain_rating_metadata.attribution,'Domain Rating by Ahrefs');
+ assert.equal(answer.data.domain_rating_metadata.url,'https://ahrefs.com');
+ assert.match(answer.data.domain_rating_metadata.caveat,/third-party estimate/);
+ assert.equal(answer.data.domain_rating_metadata.coverage.sites_with_rating,2);
+ assert.deepEqual(answer.data.rows.map(r=>r.domain_rating),['0–9','unknown']);
+ const {mcpAnswer}=await import('../../lib/data-api/mcp.ts');
+ const text=JSON.parse(mcpAnswer(answer).content[0].text);
+ assert.deepEqual(text.data.domain_rating_metadata,answer.data.domain_rating_metadata);
+});
diff --git a/tests/pages/ad-networks/ahrefs-crash.test.mjs b/tests/pages/ad-networks/ahrefs-crash.test.mjs
new file mode 100644
index 00000000..f0c1df39
--- /dev/null
+++ b/tests/pages/ad-networks/ahrefs-crash.test.mjs
@@ -0,0 +1,140 @@
+import assert from 'node:assert/strict';
+import { spawn } from 'node:child_process';
+import { once } from 'node:events';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { createServer } from 'node:http';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import test from 'node:test';
+import { ahrefsRequester } from '../../../scripts/stack127/ahrefs-dr.mjs';
+
+const startedAt = Date.parse('2026-11-14T12:00:00Z');
+const moduleUrl = new URL('../../../scripts/stack127/ahrefs-dr.mjs', import.meta.url).href;
+
+async function crashAt(t, point) {
+ const directory = mkdtempSync(join(tmpdir(), 's127-ahrefs-crash-'));
+ const stateFile = join(directory, 'ahrefs.sqlite');
+ let child, requests = 0, receivedRequest;
+ const inFlight = new Promise(resolve => { receivedRequest = resolve; });
+ const server = createServer((request, response) => {
+ requests++;
+ receivedRequest();
+ if (point !== 'during-request') {
+ response.writeHead(429, { 'Retry-After': '86400' });
+ response.end('limited');
+ }
+ });
+ t.after(async () => {
+ if (child && child.exitCode === null && child.signalCode === null) {
+ const exited = once(child, 'exit');
+ child.kill('SIGKILL');
+ await exited;
+ }
+ server.closeAllConnections();
+ await new Promise(resolve => server.close(resolve));
+ rmSync(directory, { recursive: true, force: true });
+ });
+ await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+ child = spawn(process.execPath, ['--input-type=module', '-e', `
+ import { ahrefsRequester } from ${JSON.stringify(moduleUrl)};
+ const [stateFile, point, url] = process.argv.slice(1);
+ function checkpoint(point) {
+ process.send({ point });
+ return new Promise(resolve => setTimeout(resolve, 60000));
+ }
+ const request = ahrefsRequester({ apiKey: 'test-only', stateFile, now: () => ${startedAt},
+ fetchImplementation: async () => {
+ if (point === 'before-request') await checkpoint(point);
+ const response = await fetch(url);
+ if (point === 'before-cleanup') return {
+ status: response.status, headers: response.headers,
+ body: { cancel: () => checkpoint(point) },
+ };
+ return response;
+ },
+ onRateLimit: () => {
+ process.send({ point: 'in-callback' });
+ // A synchronous callback must not be able to prevent the state commit.
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0);
+ },
+ });
+ await request('example.com');
+ throw new Error('request returned before the crash checkpoint');
+ `, stateFile, point, `http://127.0.0.1:${server.address().port}`], {
+ stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
+ });
+ let stderr = '';
+ child.stderr.on('data', data => { stderr += data; });
+ const exited = once(child, 'exit');
+ const reached = point === 'during-request' ? inFlight : once(child, 'message').then(([message]) => {
+ assert.equal(message.point, point);
+ });
+ await Promise.race([
+ reached,
+ exited.then(([code, signal]) => assert.fail(`child exited before ${point}: ${code}/${signal}\n${stderr}`)),
+ ]);
+ assert.equal(requests, point === 'before-request' ? 0 : 1);
+ child.kill('SIGKILL');
+ assert.deepEqual(await exited, [null, 'SIGKILL']);
+ return { stateFile, requests: () => requests };
+}
+
+for (const point of ['before-request', 'during-request']) {
+ test(`SIGKILL ${point} preserves the pacing reservation and releases the request lock`, { timeout: 10000 }, async t => {
+ const { stateFile, requests } = await crashAt(t, point);
+ let time = startedAt + 100, calls = 0;
+ const waits = [];
+ const request = ahrefsRequester({ apiKey: 'test-only', stateFile, now: () => time,
+ sleepImplementation: async ms => {
+ assert.equal(calls, 0, 'no HTTP before the reserved slot');
+ waits.push(ms);
+ time += ms;
+ },
+ fetchImplementation: async () => {
+ calls++;
+ assert.equal(time, startedAt + 3000);
+ return new Response(JSON.stringify({ domain_rating: { domain_rating: 15 } }));
+ },
+ });
+ assert.deepEqual(await request('next.example'), { value: 15 });
+ assert.deepEqual(waits, [2900]);
+ assert.equal(calls, 1);
+ assert.equal(requests(), point === 'before-request' ? 0 : 1);
+ });
+}
+
+for (const point of ['before-cleanup', 'in-callback']) {
+ test(`SIGKILL after 429 ${point} preserves Retry-After before cleanup or callbacks finish`, { timeout: 10000 }, async t => {
+ const { stateFile, requests } = await crashAt(t, point);
+ let calls = 0;
+ const options = { apiKey: 'test-only', stateFile,
+ sleepImplementation: async () => assert.fail('long cooldown must stop without waiting'),
+ fetchImplementation: async () => {
+ calls++;
+ return new Response(JSON.stringify({ domain_rating: { domain_rating: 20 } }));
+ },
+ };
+ // Read from a fresh connection after death: the cooldown must already be committed.
+ const db = new DatabaseSync(stateFile);
+ try {
+ assert.equal(db.prepare('select cooldown_until from request_state where id=1').get().cooldown_until, startedAt + 86400000);
+ } finally { db.close(); }
+ for (const elapsed of [100, 3600000, 86339999]) {
+ assert.deepEqual(await ahrefsRequester({ ...options, now: () => startedAt + elapsed })('next.example'), { stopped: 'rate_limited' });
+ assert.equal(calls, 0, 'no HTTP throughout the retained cooldown');
+ }
+ const controller = new AbortController();
+ await assert.rejects(ahrefsRequester({ ...options, now: () => startedAt + 86399999, signal: controller.signal,
+ sleepImplementation: async ms => {
+ assert.equal(ms, 1);
+ assert.equal(calls, 0, 'the final millisecond still blocks HTTP');
+ controller.abort();
+ },
+ })('next.example'), { name: 'AbortError' });
+ assert.equal(calls, 0);
+ assert.deepEqual(await ahrefsRequester({ ...options, now: () => startedAt + 86400001 })('next.example'), { value: 20 });
+ assert.equal(calls, 1);
+ assert.equal(requests(), 1);
+ });
+}
diff --git a/tests/pages/ad-networks/backend.test.mjs b/tests/pages/ad-networks/backend.test.mjs
index 90fc2335..7fc7401c 100644
--- a/tests/pages/ad-networks/backend.test.mjs
+++ b/tests/pages/ad-networks/backend.test.mjs
@@ -1,3 +1,4 @@
+import * as ahrefs from "../../../lib/ahrefs.ts";
import assert from 'node:assert/strict';
import test from 'node:test';
import { deriveManagedBy, buildAdsTxtPreview, aggregateAdNetworkManagers, aggregateAdNetworkTrafficShares } from '../../../lib/ad-network-site-facts.mjs';
@@ -43,7 +44,7 @@ test('sync defaults to a read-only development preview and requires explicit pro
});
test('sites inputs support both network and niche with bounded pagination and literal search',()=>{
assert.deepEqual(parseAdNetworkSitesParams(new URLSearchParams('network=mediavine&niche=food&q=Food%25&sort=oldest&page=2&pageSize=50')),
- {network:'mediavine',niche:'food',q:'food%',sort:'oldest',page:2,pageSize:50});
+ {network:'mediavine',niche:'food',q:'food%',sort:'oldest',page:2,pageSize:50,minDr:null});
for (const query of ['page=0','page=-1','page=1.5','page=10001','pageSize=101','pageSize=0','sort=drop','niche=made-up','network=x%27','q=a&q=b','x=1']) {
assert.throws(()=>parseAdNetworkSitesParams(new URLSearchParams(query)),AdNetworkInputError,query);
}
@@ -59,15 +60,15 @@ test('checker accepts URLs and international domains and rejects local or malfor
const { loadTypeScript } = await import('../../helpers/load-typescript.mjs');
const input = {parseAdNetworkSitesParams,parseAdNetworkSiteParams,AdNetworkInputError};
-function route(kind, {published=true,fail=false}={}) {
+function route(kind, {published=true,fail=false,rating=null}={}) {
let calls=0;
const lib={
- getAdNetworkSites: async query=>{calls++;if(fail)throw Error('database unavailable');return {rows:[],total:0,page:query.page,pageSize:query.pageSize,totalPages:1};},
- getAdNetworkCheckerSite: async domain=>{calls++;if(fail)throw Error('database unavailable');return domain==='known.com'?{site:{domain}}:null;},
+ getAdNetworkSites: async query=>{calls++;if(fail)throw Error('database unavailable');return {rows:[{domain:"known.com",domain_rating:rating}],total:1,page:query.page,pageSize:query.pageSize,totalPages:1,...ahrefs.ahrefsAttribution([rating])};},
+ getAdNetworkCheckerSite: async domain=>{calls++;if(fail)throw Error('database unavailable');return domain==='known.com'?{site:{domain,domain_rating:rating},similar_sites:[]}:null;},
};
const {GET}=loadTypeScript(new URL(`../../../app/api/ad-networks/${kind}/route.ts`,import.meta.url),{
'next/server':{NextResponse:{json:(body,options)=>new Response(JSON.stringify(body),options)}},
- '@/lib/ad-networks':lib,'@/lib/page-registry':{getPublishedPage:async()=>published?{}:null},'../input':input,
+ '@/lib/ahrefs':ahrefs,'@/lib/ad-networks':lib,'@/lib/page-registry':{getPublishedPage:async()=>published?{}:null},'../input':input,
});
return {GET,calls:()=>calls};
}
@@ -198,3 +199,14 @@ test('traffic shares retain confirmed managers after failed checks and count eac
assert.deepEqual(aggregateAdNetworkTrafficShares([...groups].reverse(),networks),expected);
assert.deepEqual(aggregateAdNetworkTrafficShares([],networks),[]);
});
+
+test('both public API responses credit Ahrefs for a present DR including zero, and omit credit for unknown DR',async()=>{
+ for(const kind of ['sites','site'])for(const rating of [0,42,null]) {
+ const api=route(kind,{rating});
+ const response=await api.GET(new Request(`https://stack127.com/api/ad-networks/${kind}?${kind==='site'?'domain=known.com':''}`));
+ assert.equal(response.status,200);
+ const body=await response.json();
+ if(rating===null)assert.equal(Object.hasOwn(body,'attribution'),false);
+ else assert.deepEqual(body.attribution,ahrefs.AHREFS_ATTRIBUTION);
+ }
+});
diff --git a/tests/pages/ad-networks/chart-data.test.mjs b/tests/pages/ad-networks/chart-data.test.mjs
index 66ee81af..ad849308 100644
--- a/tests/pages/ad-networks/chart-data.test.mjs
+++ b/tests/pages/ad-networks/chart-data.test.mjs
@@ -76,7 +76,7 @@ test('newest niche arrivals name the network actually joined while preserving le
const coverage={listed:125,classified:100,traffic:0,age:0,platform:0,publisher:0,checked:0,manager:0,niches:['food']};
const pageSizes=[];
const lib=adNetworksLibrary(async(sql,args)=>{
- if(sql.includes('limit $4 offset $5'))pageSizes.push(args[3]);
+ if(sql.includes('limit $5 offset $6'))pageSizes.push(args[4]);
if(sql.includes('arrived_network.name'))return {rows:[{...site,first_confirmed_at:'2026-10-15T12:30:00Z',arrived_network:{slug:'b',name:'Network B'}}]};
if(sql.includes('count(p.niche)'))return {rows:[coverage]};
if(sql.includes('as classified'))return {rows:[{listed:125,confirmed:100,classified:100}]};
diff --git a/tests/pages/ad-networks/checker-domains.test.mjs b/tests/pages/ad-networks/checker-domains.test.mjs
index 64b515cc..22dc87e7 100644
--- a/tests/pages/ad-networks/checker-domains.test.mjs
+++ b/tests/pages/ad-networks/checker-domains.test.mjs
@@ -1,3 +1,4 @@
+import * as ahrefs from "../../../lib/ahrefs.ts";
import assert from 'node:assert/strict';
import test from 'node:test';
import {adNetworksLibrary} from './loader-harness.mjs';
@@ -28,6 +29,7 @@ test('site API returns each relationship ad-system domains without substituting
});
const {GET}=loadTypeScript(new URL('../../../app/api/ad-networks/site/route.ts',import.meta.url),{
'next/server':{NextResponse:{json:(body,options)=>Response.json(body,options)}},
+ "@/lib/ahrefs":ahrefs,
'@/lib/ad-networks':{getAdNetworkCheckerSite:lib.loadAdNetworkCheckerSite},
'@/lib/page-registry':{getPublishedPage:async()=>({})},'../input':input,
});
diff --git a/tests/pages/ad-networks/domain-rating-database.test.mjs b/tests/pages/ad-networks/domain-rating-database.test.mjs
new file mode 100644
index 00000000..e33712df
--- /dev/null
+++ b/tests/pages/ad-networks/domain-rating-database.test.mjs
@@ -0,0 +1,122 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
+import { parseEnv } from 'node:util';
+import pg from 'pg';
+import { executeTool, domainRatingBucket } from '../../../lib/data-api/ad-networks/core.ts';
+
+let devEnv = {};
+try { devEnv = parseEnv(readFileSync(new URL('../../../.env.local', import.meta.url), 'utf8')); } catch { /* Optional development check. */ }
+const connection = process.env.DATABASE_POOLER_URL || process.env.DATABASE_URL;
+const devConnection = devEnv.DATABASE_POOLER_URL || devEnv.DATABASE_URL;
+const live = process.env.STACK127_DR_INTEGRATION === 'true' && connection === devConnection
+ && devEnv.SUPABASE_URL === 'https://yzasonowwlvritqicatj.supabase.co';
+
+test('development DR schema, cache projection, private views and all API queries; rollback leaves no changes', {skip:!live}, async () => {
+ const client = new pg.Client({connectionString:connection,connectionTimeoutMillis:15000,query_timeout:120000});
+ await client.connect();
+ try {
+ // Apply the migration separately: its validation must start after the DDL commits.
+ // Only test fixtures belong in this rolled-back transaction.
+ await client.query('begin');
+ const constraint=(await client.query("select convalidated from pg_constraint where conrelid='public.ad_network_publishers'::regclass and conname='ad_network_publishers_domain_rating_check'")).rows[0];
+ assert.equal(constraint?.convalidated,true);
+ await client.query('set local search_path=pg_temp,stack127_data,public');
+ const sync=(await client.query(`select domain_rating,domain_rating_checked_on::text from jsonb_populate_recordset(null::public.ad_network_publishers,
+ '[{\"domain_rating\":0,\"domain_rating_checked_on\":\"2026-11-14\"}]'::jsonb)`)).rows[0];
+ assert.deepEqual(sync,{domain_rating:0,domain_rating_checked_on:'2026-11-14'});
+ const before = (await client.query('select revision from public.stack127_ad_network_source_revision')).rows[0].revision;
+ await client.query(`update public.ad_network_publishers set domain_rating=42,domain_rating_checked_on=current_date
+ where id=(select min(id) from public.ad_network_publishers)`);
+ assert.equal((await client.query('select revision from public.stack127_ad_network_source_revision')).rows[0].revision,before,
+ 'DR-only updates must not queue a frontend refresh or change the source revision');
+ const projection = (await client.query(`select
+ public.stack127_directory_record_value('ad_network_publishers','{"id":1,"domain_rating":1,"domain_rating_checked_on":"2026-01-01","platform":"A"}') =
+ public.stack127_directory_record_value('ad_network_publishers','{"id":1,"domain_rating":99,"domain_rating_checked_on":"2026-11-14","platform":"A"}') as same,
+ public.stack127_directory_record_value('ad_network_publishers','{"id":1,"platform":"A"}') <>
+ public.stack127_directory_record_value('ad_network_publishers','{"id":1,"platform":"B"}') as other_changes`)).rows[0];
+ assert.equal(projection.same,true);assert.equal(projection.other_changes,true);
+ const reader={query:async(sql,values)=>(await client.query(sql,values)).rows};
+ const transaction=async work=>work(reader);
+ const real=await executeTool('find_sites',{limit:1},transaction);
+ assert.ok(Object.hasOwn(real.data.sites[0],'domain_rating'));
+ assert.ok(Object.hasOwn(real.data.sites[0],'domain_rating_checked_on'));
+ const anon=(await client.query("select has_table_privilege('anon','stack127_data.ad_network_publishers','SELECT') as allowed")).rows[0];
+ assert.equal(anon.allowed,false);
+ await client.query(`create temp table ad_network_publishers as select * from stack127_data.ad_network_publishers where false;
+ create temp table ad_networks as select * from stack127_data.ad_networks where false;
+ create temp table ad_network_publisher_relationships as select * from stack127_data.ad_network_publisher_relationships where false;
+ create temp table ad_network_relationship_events as select * from stack127_data.ad_network_relationship_events where false;
+ create temp table ad_network_daily_counts as select * from stack127_data.ad_network_daily_counts where false;
+ insert into ad_networks(id,slug,name) values(1,'test-a','Test A'),(2,'test-b','Test B'),(3,'test-empty','Empty');`);
+ const ratings=[0,9,10,19,20,29,30,49,50,69,70,100,null];
+ for(const [i,value] of ratings.entries()) {
+ await client.query(`insert into ad_network_publishers(id,normalized_domain,domain_rating,domain_rating_checked_on)
+ values($1,$2,$3,case when $3::int is not null then current_date end)`,[i+1,`site${i+1}.example`,value]);
+ await client.query(`insert into ad_network_publisher_relationships(ad_network_id,publisher_id,status,relationship_type)
+ values(1,$1,'confirmed','primary_or_exclusive_manager')`,[i+1]);
+ }
+ await client.query(`insert into ad_network_publisher_relationships(ad_network_id,publisher_id,status,relationship_type)
+ values(2,1,'stale','primary_or_exclusive_manager'),(2,13,'confirmed','active_network_use');`);
+ const buckets=(await reader.query(`select ${domainRatingBucket} as bucket,count(*)::int as count from ad_network_publishers p group by 1`));
+ assert.deepEqual(Object.fromEntries(buckets.map(r=>[r.bucket,r.count])),{'0–9':2,'10–19':2,'20–29':2,'30–49':2,'50–69':2,'70–100':2,unknown:1});
+ const grouped=await executeTool('count_sites',{group_by:['domain_rating']},transaction);
+ assert.equal(grouped.data.total,13);assert.equal(grouped.data.domain_rating_coverage.sites_with_rating,12);
+ assert.equal(grouped.data.rows.reduce((sum,r)=>sum+r.count,0),13);
+ const compare=await executeTool('compare_ad_networks',{networks:['test-a','test-b'],dimension:'domain_rating'},transaction);
+ assert.deepEqual(compare.data.network_domain_rating_coverage['test-b'],{sites_with_rating:1,total_sites:2});
+ assert.equal(compare.data.total,13);assert.equal(compare.data.network_bases['test-b'],2);
+ const filtered=await executeTool('count_sites',{min_domain_rating:30,max_domain_rating:69},transaction);
+ assert.equal(filtered.data.total,4);
+ const zero=await executeTool('find_sites',{min_domain_rating:0,max_domain_rating:0},transaction);
+ assert.equal(zero.data.sites.length,1);assert.equal(zero.data.sites[0].domain_rating,0);
+ const lookup=await executeTool('lookup_site',{domain:'site13.example'},transaction);
+ assert.equal(lookup.data.site.domain_rating,null);assert.equal(lookup.data.site.domain_rating_checked_on,null);
+ const networks=await executeTool('list_ad_networks',{},transaction);
+ const n=networks.data.networks.find(n=>n.slug==='test-a');
+ assert.equal(n.domain_rating_base,12);assert.equal(n.median_domain_rating,29.5);
+ assert.ok(Math.abs(n.average_domain_rating-ratings.filter(r=>r!==null).reduce((a,b)=>a+b,0)/12)<1e-10);
+ const empty=networks.data.networks.find(n=>n.slug==='test-empty');
+ assert.equal(empty.domain_rating_base,0);assert.equal(empty.median_domain_rating,null);assert.equal(empty.average_domain_rating,null);
+ const profile=await executeTool('get_ad_network',{network:'test-a'},transaction);
+ assert.equal(profile.data.network.domain_rating_base,12);
+ assert.equal(profile.data.distributions.domain_rating.total,13);
+ for(const answer of [real,grouped,compare,filtered,zero,lookup,networks,profile]) {
+ assert.equal(answer.data.domain_rating_metadata.attribution,'Domain Rating by Ahrefs');
+ assert.equal(answer.data.domain_rating_metadata.url,'https://ahrefs.com');
+ assert.ok(Number.isInteger(answer.data.domain_rating_metadata.coverage.sites_with_rating));
+ }
+ } finally { await client.query('rollback'); await client.end(); }
+});
+
+test('DR schema guards allow reruns, reject wrong types, and validate after releasing the exclusive lock', {skip:!live}, async()=>{
+ const client=new pg.Client({connectionString:connection,connectionTimeoutMillis:15000,query_timeout:120000});
+ await client.connect();
+ const migration=readFileSync(new URL('../../../supabase/migrations/20261114130000_ad_network_domain_rating.sql',import.meta.url),'utf8');
+ // Exercise the exact column/constraint SQL on disposable development tables.
+ // Views and source-revision functions are covered by the integration test above.
+ const ddl=migration.split('comment on column')[0].replaceAll('public.ad_network_publishers','pg_temp.dr_guard_test');
+ const validation=migration.slice(migration.indexOf('-- Release ACCESS EXCLUSIVE')).replaceAll('public.ad_network_publishers','pg_temp.dr_guard_test');
+ try {
+ await client.query('create temp table dr_guard_test(id integer); insert into dr_guard_test values(1)');
+ await client.query(ddl);
+ const constraint=()=>client.query("select convalidated from pg_constraint where conrelid='pg_temp.dr_guard_test'::regclass and conname='ad_network_publishers_domain_rating_check'");
+ assert.equal((await constraint()).rows[0].convalidated,false);
+ await client.query('commit');
+ await client.query(validation.replace(/commit;\s*$/,''));
+ assert.equal((await constraint()).rows[0].convalidated,true);
+ const locks=(await client.query("select mode from pg_locks where pid=pg_backend_pid() and relation='pg_temp.dr_guard_test'::regclass")).rows.map(row=>row.mode);
+ assert.ok(locks.includes('ShareUpdateExclusiveLock'));
+ assert.ok(!locks.includes('AccessExclusiveLock'));
+ await client.query('commit');
+ await client.query(ddl+'commit;'); await client.query(validation);
+ assert.equal((await constraint()).rows[0].convalidated,true);
+ assert.equal((await client.query('select count(*)::int as total from dr_guard_test')).rows[0].total,1);
+ await assert.rejects(client.query('insert into dr_guard_test(domain_rating) values(101)'),/ad_network_publishers_domain_rating_check/);
+ for(const columns of ['domain_rating text','domain_rating integer,domain_rating_checked_on text']) {
+ await client.query(`drop table dr_guard_test;create temp table dr_guard_test(${columns})`);
+ await assert.rejects(client.query(ddl),/has type text, expected (integer|date)/);
+ await client.query('rollback');
+ }
+ }finally{await client.query('rollback').catch(()=>{});await client.end();}
+});
diff --git a/tests/pages/ad-networks/domain-rating-report.test.mjs b/tests/pages/ad-networks/domain-rating-report.test.mjs
new file mode 100644
index 00000000..84227a13
--- /dev/null
+++ b/tests/pages/ad-networks/domain-rating-report.test.mjs
@@ -0,0 +1,135 @@
+import { AHREFS_ATTRIBUTION } from "../../../lib/ahrefs.ts";
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import pg from 'pg';
+import {
+ AD_NETWORK_DR_GATE, adNetworkDomainRatingReport, adNetworkSiteAvailability, hasAdNetworkDomainRatingCoverage,
+} from '../../../lib/ad-networks-report.ts';
+import {AdNetworkInputError, parseAdNetworkSitesParams} from '../../../app/api/ad-networks/input.ts';
+import {adNetworksLibrary} from './loader-harness.mjs';
+
+const coverage={listed:1000,classified:0,traffic:0,age:0,platform:0,publisher:0,checked:0,manager:0,niches:[],networks:[]};
+const summary=(slug,listed,rated,median,average)=>({slug,name:slug,listed,rated,median,average});
+
+test('the sites API accepts only the DR steps and the two DR sorts',()=>{
+ assert.equal(parseAdNetworkSitesParams(new URLSearchParams()).minDr,null);
+ assert.equal(parseAdNetworkSitesParams(new URLSearchParams('network=raptive&dr=30')).minDr,30);
+ for(const step of [10,20,30,50,70])assert.equal(parseAdNetworkSitesParams(new URLSearchParams(`dr=${step}`)).minDr,step);
+ assert.equal(parseAdNetworkSitesParams(new URLSearchParams('sort=highest-dr')).sort,'highest-dr');
+ assert.equal(parseAdNetworkSitesParams(new URLSearchParams('sort=lowest-dr')).sort,'lowest-dr');
+ // One cache key per view: no near-duplicates of a step, no repeats, no blanks.
+ for(const query of ['dr=31','dr=030','dr=30.0','dr=','dr=0','dr=100','dr=30&dr=30','sort=dr'])
+ assert.throws(()=>parseAdNetworkSitesParams(new URLSearchParams(query)),AdNetworkInputError,query);
+});
+
+test('DR column, sorts and filter appear only once sites in scope are rated',()=>{
+ for(const rated of [undefined,0]) {
+ const none=adNetworkSiteAvailability({...coverage,rated});
+ assert.ok(!none.availableColumns.includes('domain_rating'));
+ assert.ok(!none.availableSorts.includes('highest-dr'));
+ assert.ok(!none.availableFilters.includes('domain_rating'));
+ }
+ const some=adNetworkSiteAvailability({...coverage,rated:1});
+ assert.ok(some.availableColumns.includes('domain_rating'));
+ assert.deepEqual(some.availableSorts.filter(sort=>sort.endsWith('-dr')),['highest-dr','lowest-dr']);
+ assert.ok(some.availableFilters.includes('domain_rating'));
+});
+
+test('the DR block hides below the gate on rated sites and coverage',()=>{
+ assert.deepEqual(AD_NETWORK_DR_GATE,{minRated:20,minShare:0.25});
+ assert.equal(hasAdNetworkDomainRatingCoverage(19,19),false);
+ assert.equal(hasAdNetworkDomainRatingCoverage(20,81),false);
+ assert.equal(hasAdNetworkDomainRatingCoverage(20,80),true);
+ assert.equal(hasAdNetworkDomainRatingCoverage(0,0),false);
+ assert.equal(adNetworkDomainRatingReport('raptive',[summary('raptive',1000,249,30,30)],[{min:30,count:249}]),null);
+ assert.equal(adNetworkDomainRatingReport('missing',[summary('raptive',100,100,30,30)],[]),null);
+});
+
+test('the DR block bands sites, marks the rounded median band and compares the gated big networks',()=>{
+ const summaries=[
+ summary('raptive',100,90,29.5,34.2),summary('mediavine',200,150,28,30.4),
+ summary('ezoic',1000,100,20,19), // 10% rated: left out of the comparison
+ summary('journey-by-mediavine',300,300,14,15.6),summary('mgid',50,50,60,61),
+ ];
+ const report=adNetworkDomainRatingReport('raptive',summaries,[
+ {min:0,count:5},{min:20,count:30},{min:30,count:40},{min:70,count:10},{min:80,count:4},{min:100,count:1},
+ ]);
+ assert.deepEqual(report.bands.map(band=>band.label),['0–9','10–19','20–29','30–39','40–49','50–59','60–69','70+']);
+ assert.deepEqual(report.bands.map(band=>band.count),[5,0,30,40,0,0,0,15]);
+ // 29.5 shows as 30, so the highlighted band is 30–39.
+ assert.equal(report.median_band,'30-39');
+ assert.deepEqual(report.comparison.map(row=>[row.slug,row.current]),
+ [['raptive',true],['mediavine',false],['journey-by-mediavine',false]]);
+ // A network outside the big four still sees itself beside them.
+ const mgid=adNetworkDomainRatingReport('mgid',summaries,[{min:60,count:50}]);
+ assert.equal(mgid.comparison[0].slug,'mgid');assert.equal(mgid.comparison.length,4);
+ // Compared with nothing else, the strip hides.
+ assert.deepEqual(adNetworkDomainRatingReport('mgid',[summaries[4]],[{min:60,count:50}]).comparison,[]);
+});
+
+test('the sites loader filters by DR in both queries and sorts unrated sites last',async()=>{
+ const calls=[];
+ const {loadAdNetworkSites,defaultAdNetworkSitesQuery}=adNetworksLibrary(async(sql,args)=>{
+ calls.push({sql,args});
+ if(sql.includes('as classified'))return {rows:[{...coverage,rated:10}]};
+ if(sql.includes('select distinct n.slug')||sql.includes('select evidence,manager_relationships')||sql.includes('select slug,name,manager_domains'))return {rows:[]};
+ if(sql.includes('count(*)::int as total'))return {rows:[{total:3}]};
+ if(sql.includes('p.normalized_domain as domain'))return {rows:[]};
+ throw Error(`Unexpected SQL: ${sql}`);
+ });
+ const result=await loadAdNetworkSites({...defaultAdNetworkSitesQuery('raptive'),sort:'highest-dr',minDr:30});
+ assert.equal(result.effectiveSort,'highest-dr');
+ const count=calls.find(call=>call.sql.includes('count(*)::int as total'));
+ const table=calls.find(call=>call.sql.includes('p.normalized_domain as domain'));
+ for(const call of [count,table]) {
+ assert.match(call.sql,/\(\$4::int is null or p\.domain_rating>=\$4\)/);
+ assert.equal(call.args[3],30);
+ }
+ assert.ok(table.sql.includes('order by p.domain_rating desc nulls last,p.normalized_domain limit $5 offset $6'));
+ calls.length=0;
+ await loadAdNetworkSites({...defaultAdNetworkSitesQuery('raptive'),sort:'lowest-dr'});
+ assert.ok(calls.find(call=>call.sql.includes('p.normalized_domain as domain')).sql.includes('order by p.domain_rating asc nulls last'));
+ await assert.rejects(loadAdNetworkSites({...defaultAdNetworkSitesQuery('raptive'),minDr:31}),/Invalid sites query/);
+});
+
+// Read-only fixture query: the real ORDER BY runs in PostgreSQL; no tables are written.
+const connectionString=process.env.DATABASE_URL;
+test('both DR sorts put unrated sites last across pages',{skip:!connectionString},async()=>{
+ const orders={};
+ const library=adNetworksLibrary(async sql=>{
+ if(sql.includes('as classified'))return {rows:[{...coverage,rated:5}]};
+ if(sql.includes('count(*)::int as total'))return {rows:[{total:6}]};
+ const order=sql.match(/select p\.id from matches p order by (.*?) limit \$5 offset \$6/s);
+ if(order)orders.current=order[1];
+ return {rows:[]};
+ });
+ const rows=[[1,'a.com',40],[2,'b.com',null],[3,'c.com',70],[4,'d.com',40],[5,'e.com',null],[6,'f.com',5]]
+ .map(([id,normalized_domain,domain_rating])=>({id,normalized_domain,domain_rating}));
+ const client=new pg.Client({connectionString,connectionTimeoutMillis:15000,
+ options:'-c default_transaction_read_only=on -c statement_timeout=15000'});
+ try {
+ await client.connect();
+ for(const [sort,expected] of [['highest-dr',[3,1,4,6,2,5]],['lowest-dr',[6,1,4,3,2,5]]]) {
+ await library.loadAdNetworkSites({...library.defaultAdNetworkSitesQuery('raptive'),sort});
+ const sql=`with matches as materialized (
+ select * from jsonb_to_recordset($1::jsonb) p(id int,normalized_domain text,domain_rating int)
+ ) select p.id from matches p order by ${orders.current} limit $2 offset $3`;
+ const page=async(size,offset)=>(await client.query(sql,[JSON.stringify(rows),size,offset])).rows.map(row=>row.id);
+ assert.deepEqual([...(await page(2,0)),...(await page(2,2)),...(await page(2,4))],expected,sort);
+ }
+ } finally { await client.end(); }
+});
+
+test('the public sites loader carries linked attribution beside rated records and the response',async()=>{
+ for(const rating of [0,42,null]) {
+ const lib=adNetworksLibrary(async sql=>{
+ if(sql.includes('as classified'))return {rows:[{...coverage,rated:rating===null?0:1}]};
+ if(sql.includes('count(*)::int as total'))return {rows:[{total:1}]};
+ if(sql.includes('p.normalized_domain as domain'))return {rows:[{domain:'example.com',last_checked_at:null,evidence:{},manager_relationships:[],domain_rating:rating}]};
+ return {rows:[]};
+ });
+ const result=await lib.loadAdNetworkSites(lib.defaultAdNetworkSitesQuery());
+ if(rating===null) {assert.equal(Object.hasOwn(result,'attribution'),false);assert.equal(Object.hasOwn(result.rows[0],'attribution'),false);}
+ else {assert.deepEqual(result.attribution,AHREFS_ATTRIBUTION);assert.deepEqual(result.rows[0].attribution,AHREFS_ATTRIBUTION);}
+ }
+});
diff --git a/tests/pages/ad-networks/domain-rating.test.mjs b/tests/pages/ad-networks/domain-rating.test.mjs
new file mode 100644
index 00000000..588e9ebe
--- /dev/null
+++ b/tests/pages/ad-networks/domain-rating.test.mjs
@@ -0,0 +1,169 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { openLocalDatabase } from '../../../scripts/ad-networks/local-db.mjs';
+import { collectDomainRatings, domainRatingSites } from '../../../scripts/ad-networks/domain-rating.mjs';
+import { ahrefsRequester } from '../../../scripts/stack127/ahrefs-dr.mjs';
+
+function fixture(t) {
+ const dir = mkdtempSync(join(tmpdir(), 's127-dr-'));
+ const db = openLocalDatabase(join(dir, 'test.sqlite'));
+ t.after(() => { db.close(); rmSync(dir, { recursive: true, force: true }); });
+ const network = db.prepare('select id from ad_networks limit 1').get().id;
+ function site(id, status, manager = false, checked = null, value = null) {
+ db.prepare(`insert into publishers(id,normalized_domain,display_domain,first_seen_at,last_discovered_at,created_at,updated_at,domain_rating_checked_on,domain_rating)
+ values(?,?,?,datetime('now'),datetime('now'),datetime('now'),datetime('now'),?,?)`).run(id, `site${id}.example`, `site${id}.example`, checked, value);
+ if (status) db.prepare(`insert into relationships(ad_network_id,publisher_id,status,relationship_type,evidence,created_at,updated_at)
+ values(?,?,?,?, '{}',datetime('now'),datetime('now'))`).run(network, id, status, manager ? 'primary_or_exclusive_manager' : 'active_network_use');
+ }
+ return { db, site };
+}
+
+test('DR selection prioritizes listed managers, other listed, remaining never checked, then oldest due', t => {
+ const { db, site } = fixture(t);
+ site(1, 'ended', true); site(2, 'stale'); site(3, 'stale', true); site(4, 'confirmed', true);
+ site(5, 'confirmed', true, '2026-10-14', 10); site(6, null, false, '2026-09-01', 0);
+ site(7, 'confirmed', true, '2026-10-15', 100); // Exactly 30 days old is not due yet.
+ site(8, null); site(9, 'needs_review', true);
+ assert.deepEqual(domainRatingSites(db, { now: Date.parse('2026-11-14T12:00:00Z'), limit: 50 }).map(r => r.id), [3,4,2,1,8,9,6,5]);
+ assert.deepEqual(domainRatingSites(db, { now: Date.parse('2026-11-14T12:00:00Z'), limit: 2 }).map(r => r.id), [3,4]);
+ for (const value of [-1,101,1.5,'invalid']) assert.throws(() => db.prepare('update publishers set domain_rating=? where id=1').run(value));
+});
+
+test('failures keep prior DR/date, defer retry, and never block other domains; successes resume idempotently', async t => {
+ const { db, site } = fixture(t);
+ site(1, 'confirmed', true, '2020-01-01', 40); site(2, 'confirmed', true);
+ let calls = 0;
+ const result = await collectDomainRatings(db, { limit: 2 }, { requestRating: async domain => { calls++; return domain === 'site1.example' ? { error: 'unavailable' } : { value: 0 }; } });
+ assert.equal(result.status, 'failed'); assert.equal(result.measured, 1); assert.equal(result.failed, 1);
+ assert.deepEqual({...db.prepare('select domain_rating,domain_rating_checked_on from publishers where id=1').get()}, {domain_rating:40,domain_rating_checked_on:'2020-01-01'});
+ assert.equal(db.prepare('select domain_rating from publishers where id=2').get().domain_rating, 0);
+ assert.ok(db.prepare("select retry_at from enrichment_state where publisher_id=1 and kind='domain-rating'").get().retry_at);
+ await collectDomainRatings(db, { limit: 2 }, { requestRating: async () => assert.fail('already measured or retry deferred') });
+ assert.equal(calls, 2);
+});
+
+for (const status of [401,403,429]) test(`HTTP ${status} stops the run, keeps all values and does not start another domain`, async t => {
+ const { db, site } = fixture(t); site(1, 'confirmed', true); site(2, 'confirmed', true);
+ let time = 1, calls = 0;
+ const requestRating = ahrefsRequester({apiKey:'test-only', now:()=>time, sleepImplementation:async ms=>{time+=ms;},
+ fetchImplementation:async()=>{calls++;return new Response('', {status,headers:{'Retry-After':'40'}});}});
+ const result = await collectDomainRatings(db, {limit:2}, {requestRating});
+ assert.equal(result.status, 'failed'); assert.equal(result.measured, 0); assert.equal(calls, status===429?3:1);
+ assert.ok(db.prepare('select * from publishers').all().every(r=>r.domain_rating===null&&r.domain_rating_checked_on===null));
+});
+
+test('shared requester enforces the minimum pace, honors slower configuration and cancels a backoff', async () => {
+ assert.throws(()=>ahrefsRequester({apiKey:'test',paceMs:2999}), /3000/);
+ let time=1;
+ const times=[];
+ const request=ahrefsRequester({apiKey:'test',paceMs:5000,now:()=>time,sleepImplementation:async ms=>{time+=ms;},fetchImplementation:async()=>{
+ times.push(time);return new Response(JSON.stringify({domain_rating:{domain_rating:42.6}}));
+ }});
+ assert.deepEqual(await request('a.example'),{value:43});await request('b.example');assert.deepEqual(times,[1,5001]);
+ const controller=new AbortController(); let calls=0;
+ const cancelled=ahrefsRequester({apiKey:'test',signal:controller.signal,fetchImplementation:async()=>{
+ calls++;return new Response('',{status:429});
+ },sleepImplementation:async()=>{controller.abort();}});
+ await assert.rejects(cancelled('a.example'));assert.equal(calls,1);
+});
+
+test('run budget checkpoints partial work and prevents the next request', async t => {
+ const {db,site}=fixture(t);site(1,'confirmed');site(2,'confirmed');
+ let time=Date.now(),calls=0;
+ const result=await collectDomainRatings(db,{limit:2,'time-budget-minutes':1},{now:()=>time,requestRating:async()=>{calls++;time+=61000;return {value:20};}});
+ assert.equal(calls,1);assert.equal(result.status,'budget_complete');
+ assert.equal(domainRatingSites(db).length,1);
+});
+
+
+test('a long Retry-After survives the run and blocks all later domains/runs until it expires', async t => {
+ const {db,site}=fixture(t);site(1,'confirmed');site(2,'confirmed');
+ const previous=process.env.AHREFS_API_KEY;process.env.AHREFS_API_KEY='test-only';
+ t.after(()=>{if(previous===undefined)delete process.env.AHREFS_API_KEY;else process.env.AHREFS_API_KEY=previous;});
+ let time=Date.now(),calls=0;
+ const first=await collectDomainRatings(db,{limit:2},{now:()=>time,fetchImplementation:async()=>{
+ calls++;return new Response('',{status:429,headers:{'Retry-After':'86400'}});
+ }});
+ assert.equal(first.stopped,'rate_limited');assert.equal(calls,1);
+ assert.ok(db.prepare("select value from schema_meta where key='domain_rating_retry_at'").get().value);
+ const second=await collectDomainRatings(db,{limit:2},{now:()=>time+3600000,requestRating:async()=>assert.fail('Retry-After not reached')});
+ assert.equal(second.stopped,'rate_limited');assert.equal(second.processed,0);
+ time+=86400001;
+ const third=await collectDomainRatings(db,{limit:1},{now:()=>time,requestRating:async()=>({value:50})});
+ assert.equal(third.measured,1);
+});
+
+test('November 1 ad-network Retry-After blocks November 2 launch metrics on the shared host state', async t => {
+ const { db, site } = fixture(t); site(1, 'confirmed', true);
+ const dir = mkdtempSync(join(tmpdir(), 's127-ahrefs-shared-'));
+ t.after(() => rmSync(dir, {recursive:true,force:true}));
+ const stateFile = join(dir, 'ahrefs.sqlite');
+ const previous = process.env.AHREFS_API_KEY; process.env.AHREFS_API_KEY = 'test-only';
+ t.after(() => {if(previous===undefined)delete process.env.AHREFS_API_KEY;else process.env.AHREFS_API_KEY=previous;});
+ const firstAt = Date.parse('2026-11-01T23:05:00Z');
+ let calls = 0;
+ const adNetworks = await collectDomainRatings(db, {limit:1}, {stateFile, now:()=>firstAt,
+ fetchImplementation:async()=>{calls++; return new Response('',{status:429,headers:{'Retry-After':'86400'}});}});
+ assert.equal(adNetworks.stopped,'rate_limited'); assert.equal(calls,1);
+ const {collectMetrics} = await import('../../../scripts/launch-directories/refresh-metrics.mjs');
+ const records = [{slug:'example',website_url:'https://example.com',domain_rating:20,domain_rating_checked_on:'2026-10-02'}];
+ const launch = await collectMetrics(records, {checkedOn:'2026-11-02',
+ loadTranco:async()=>({list_id:'test',ranks:new Map()}),
+ requestRating:ahrefsRequester({apiKey:'test-only',stateFile,now:()=>Date.parse('2026-11-02T06:00:00Z'),
+ fetchImplementation:async()=>assert.fail('24-hour cooldown must block the other job'),
+ sleepImplementation:async()=>assert.fail('long cooldown must stop promptly')})});
+ assert.equal(launch.stopped,'rate_limited');
+ assert.ok(launch.patches.every(patch=>!Object.hasOwn(patch.changes,'domain_rating')));
+ assert.equal(launch.patches[0].changes.traffic_bucket,'below_1m','Tranco still runs during an Ahrefs cooldown');
+ const after = ahrefsRequester({apiKey:'test-only',stateFile,now:()=>firstAt+86400001,
+ fetchImplementation:async()=>new Response(JSON.stringify({domain_rating:{domain_rating:30}}))});
+ assert.deepEqual(await after('example.com'),{value:30});
+});
+
+test('independent requesters serialize in-flight HTTP and retain post-response pacing across restarts', async t => {
+ const dir=mkdtempSync(join(tmpdir(),'s127-ahrefs-lock-'));
+ t.after(()=>rmSync(dir,{recursive:true,force:true}));
+ const stateFile=join(dir,'ahrefs.sqlite');
+ let time=1000, release, entered;
+ const ready=new Promise(resolve=>{entered=resolve;});
+ const hold=new Promise(resolve=>{release=resolve;});
+ const first=ahrefsRequester({apiKey:'test',stateFile,now:()=>time,fetchImplementation:async()=>{
+ entered(); await hold; time=2000; return new Response(JSON.stringify({domain_rating:{domain_rating:10}}));
+ }});
+ const pending=first('first.example'); await ready;
+ let secondCalls=0;
+ const waits=[];
+ const options={apiKey:'test',stateFile,now:()=>time,sleepImplementation:async ms=>{waits.push(ms);time+=ms;},
+ fetchImplementation:async()=>{secondCalls++;return new Response(JSON.stringify({domain_rating:{domain_rating:20}}));}};
+ const second=ahrefsRequester(options)('second.example');
+ assert.equal(secondCalls,0); release();
+ assert.deepEqual(await pending,{value:10}); assert.deepEqual(await second,{value:20});
+ assert.deepEqual(waits,[3000]);
+ await ahrefsRequester(options)('third.example');
+ assert.deepEqual(waits,[3000,3000]);
+});
+
+test('a killed process releases the shared request lock for the next job', {timeout:10000}, async t => {
+ const {spawn}=await import('node:child_process');
+ const {once}=await import('node:events');
+ const dir=mkdtempSync(join(tmpdir(),'s127-ahrefs-crash-'));
+ t.after(()=>rmSync(dir,{recursive:true,force:true}));
+ const stateFile=join(dir,'ahrefs.sqlite');
+ const moduleUrl=new URL('../../../scripts/stack127/ahrefs-dr.mjs',import.meta.url).href;
+ const child=spawn(process.execPath,['--input-type=module','-e',`
+ import {ahrefsRequester} from ${JSON.stringify(moduleUrl)};
+ const request=ahrefsRequester({apiKey:'test-only',stateFile:process.argv[1],fetchImplementation:async()=>{
+ process.stdout.write('locked'); await new Promise(resolve=>setTimeout(resolve,60000));
+ }});
+ await request('example.com');
+ `,stateFile],{stdio:['ignore','pipe','pipe']});
+ t.after(()=>child.kill('SIGKILL'));
+ await Promise.race([once(child.stdout,'data'),once(child,'exit').then(()=>assert.fail('child exited before acquiring lock'))]);
+ const exit=once(child,'exit'); child.kill('SIGKILL'); await exit;
+ const request=ahrefsRequester({apiKey:'test-only',stateFile,
+ fetchImplementation:async()=>new Response(JSON.stringify({domain_rating:{domain_rating:15}}))});
+ assert.deepEqual(await request('next.example'),{value:15});
+});
diff --git a/tests/pages/ad-networks/incremental-sync.test.mjs b/tests/pages/ad-networks/incremental-sync.test.mjs
index d9e442fe..9666fcba 100644
--- a/tests/pages/ad-networks/incremental-sync.test.mjs
+++ b/tests/pages/ad-networks/incremental-sync.test.mjs
@@ -310,3 +310,16 @@ test('compact attribution preserves the real weekly chart, and cleanup bounds pr
progress=await clean();
assert.equal(Number(progress[0].publishers_scanned),0);assert.equal(Number(progress[1].relationships_scanned),0);
});
+
+
+test('DR sync preserves zero/date, sends only changed rows, and old snapshots omit DR', async t => {
+ const f=syncFixture(t);
+ await f.run();f.payloads.publishers.length=0;
+ f.db.prepare("update publishers set domain_rating=0,domain_rating_checked_on='2026-11-14' where id=1").run();
+ let result=await f.run();assert.equal(result.publishers_sent,1);
+ assert.equal(f.payloads.publishers[0].domain_rating,0);
+ assert.equal(f.payloads.publishers[0].domain_rating_checked_on,'2026-11-14');
+ result=await f.run();assert.equal(result.publishers_sent,0);
+ assert.ok(!Object.hasOwn(mapSyncPublisher({normalized_domain:'old.example'}),'domain_rating'));
+ assert.ok(!Object.hasOwn(mapSyncPublisher({normalized_domain:'old.example'}),'domain_rating_checked_on'));
+});
diff --git a/tests/pages/ad-networks/loader-harness.mjs b/tests/pages/ad-networks/loader-harness.mjs
index dc1f2927..aa93f716 100644
--- a/tests/pages/ad-networks/loader-harness.mjs
+++ b/tests/pages/ad-networks/loader-harness.mjs
@@ -1,3 +1,4 @@
+import * as ahrefs from "../../../lib/ahrefs.ts";
import fs from 'node:fs';
import vm from 'node:vm';
import ts from 'typescript';
@@ -11,6 +12,7 @@ import * as tldts from 'tldts';
export function adNetworksLibrary(query, overrides={}, globals={}) {
const exports={};
const imports={
+ './ahrefs':ahrefs,
'./db':{getPool:()=>({query})},
'next/cache':{unstable_cache:fn=>fn},
'./ad-network-niches':niches,'./ad-networks-report':report,'./ad-network-site-facts.mjs':facts,
diff --git a/tests/pages/ad-networks/publishers.test.mjs b/tests/pages/ad-networks/publishers.test.mjs
index 1e61a3e5..f9c99903 100644
--- a/tests/pages/ad-networks/publishers.test.mjs
+++ b/tests/pages/ad-networks/publishers.test.mjs
@@ -1,3 +1,4 @@
+import * as ahrefs from "../../../lib/ahrefs.ts";
import assert from "node:assert/strict";
import test from "node:test";
import { loadTypeScript } from "../../helpers/load-typescript.mjs";
@@ -11,6 +12,7 @@ import * as tldts from "tldts";
function library(rows, fail = false) {
const calls = [];
const exports = loadTypeScript(new URL("../../../lib/ad-networks.ts", import.meta.url), {
+ "./ahrefs":ahrefs,
"./db": { getPool: () => ({
query: async (sql, args) => {
calls.push({ sql, args });
diff --git a/tests/pages/ad-networks/refresh.test.mjs b/tests/pages/ad-networks/refresh.test.mjs
index 4e2f09b4..3044dfdc 100644
--- a/tests/pages/ad-networks/refresh.test.mjs
+++ b/tests/pages/ad-networks/refresh.test.mjs
@@ -190,3 +190,20 @@ test('sync contention preserves collected checkpoints and retries skip the colle
assert.equal((await refresh({ ...options, 'sync-only': true }, deps)).skipped, true);
assert.deepEqual(calls, ['sync']);
});
+
+
+test('DR job forwards a safe pace and publishes partial checkpoints once after a collector failure', async t => {
+ const state = await mkdtemp(join(tmpdir(), 's127-dr-refresh-'));
+ t.after(()=>rm(state,{recursive:true,force:true}));
+ const options=refreshArgs(['--job','domain-rating','--pace-ms','5000','--state-directory',state,'--apply']);
+ const command=refreshCommand(options);
+ assert.equal(command.script,'domain-rating');
+ const production=refreshCommand({...options,environment:'production'});
+ assert.equal(production.args[production.args.indexOf('--ahrefs-state-file')+1],'/srv/stack127-data/state/ahrefs.sqlite');
+ assert.notEqual(command.args[command.args.indexOf('--ahrefs-state-file')+1],'/srv/stack127-data/state/ahrefs.sqlite');
+ assert.deepEqual(command.args.slice(command.args.indexOf('--pace-ms'),command.args.indexOf('--pace-ms')+2),['--pace-ms','5000']);
+ assert.throws(()=>refreshArgs(['--job','domain-rating','--pace-ms','2999']),/pace-ms/);
+ const calls=[];
+ await assert.rejects(refresh(options,{run:async script=>{calls.push(script);if(script==='domain-rating')throw Error('authentication_failed');},alert:async()=>{}}),/authentication_failed/);
+ assert.deepEqual(calls,['domain-rating','sync','revalidate']);
+});
diff --git a/tests/pages/ad-networks/report.test.mjs b/tests/pages/ad-networks/report.test.mjs
index 9acb0c2b..a0e101d5 100644
--- a/tests/pages/ad-networks/report.test.mjs
+++ b/tests/pages/ad-networks/report.test.mjs
@@ -137,7 +137,7 @@ test('sites loader and parser default to 10, preserve explicit sizes, and bound
assert.equal(result.effectiveSort,'domain');assert.equal(result.page,8);assert.equal(result.totalPages,8);
assert.deepEqual(Array.from(result.availableSorts),['domain','checked']);
const table=calls.find(c=>c.sql.includes('p.normalized_domain as domain'));
- assert.match(table.sql,/order by p.normalized_domain limit \$4 offset \$5/);
+ assert.match(table.sql,/order by p.normalized_domain limit \$5 offset \$6/);
assert.deepEqual(Array.from(table.args).slice(-2),[10,70]);
const explicit=await loadAdNetworkSites({...query,page:99,pageSize:50});
assert.equal(explicit.pageSize,50);assert.equal(explicit.page,2);assert.equal(explicit.totalPages,2);
@@ -160,13 +160,13 @@ test('default and explicit traffic sort use age, domain and id ties in both pagi
}
const order="case p.traffic_group when 'top-1k' then 0 when 'top-10k' then 1 when 'top-100k' then 2 when 'top-1m' then 3 when 'not-ranked' then 4 else 5 end,p.domain_created_on asc nulls last,p.normalized_domain,p.id";
for(const {sql} of tables) {
- assert.ok(sql.includes(`order by ${order} limit $4 offset $5`));
+ assert.ok(sql.includes(`order by ${order} limit $5 offset $6`));
assert.ok(sql.trimEnd().endsWith(`order by ${order}`));
}
- assert.deepEqual(Array.from(tables[0].args),['mediavine',null,'',10,0]);
- assert.deepEqual(Array.from(tables[1].args),['mediavine',null,'',10,10]);
- assert.deepEqual(Array.from(tables[2].args),[null,'tech','',10,0]);
- assert.deepEqual(Array.from(tables[3].args),[null,'tech','',10,10]);
+ assert.deepEqual(Array.from(tables[0].args),['mediavine',null,'',null,10,0]);
+ assert.deepEqual(Array.from(tables[1].args),['mediavine',null,'',null,10,10]);
+ assert.deepEqual(Array.from(tables[2].args),[null,'tech','',null,10,0]);
+ assert.deepEqual(Array.from(tables[3].args),[null,'tech','',null,10,10]);
});
test('an all-network data event refreshes every network, niches and both APIs',()=>{
const targets=targetsForEvents([{entity_type:'ad_networks',slug:'all'}]);
diff --git a/tests/pages/ad-networks/sites-sort-database.test.mjs b/tests/pages/ad-networks/sites-sort-database.test.mjs
index c45a6ff7..5ffec8fa 100644
--- a/tests/pages/ad-networks/sites-sort-database.test.mjs
+++ b/tests/pages/ad-networks/sites-sort-database.test.mjs
@@ -14,7 +14,7 @@ test('traffic groups, oldest registrations, null dates and stable ties survive p
return {rows:[]};
});
await library.loadAdNetworkSites(library.defaultAdNetworkSitesQuery('mediavine'));
- const pagingOrder=tableSql.match(/select p.id from matches p order by (.*?) limit \$4 offset \$5/s)[1];
+ const pagingOrder=tableSql.match(/select p.id from matches p order by (.*?) limit \$5 offset \$6/s)[1];
const resultOrder=tableSql.slice(tableSql.lastIndexOf('order by '));
const site=(id,normalized_domain,traffic_group,domain_created_on)=>({id,normalized_domain,traffic_group,domain_created_on});
const rows=[