From fd0227d5f7078b64c10ab4e729a9902ba51ece83 Mon Sep 17 00:00:00 2001 From: RaviTejaKNTS Date: Sat, 14 Nov 2026 16:13:47 +0530 Subject: [PATCH] Add Ahrefs Domain Rating to the ad network directory A new OVH job, domain-rating, collects DR for ad network publisher sites from Ahrefs' free endpoint: managed sites first, then other listed sites, then the rest, then the oldest checks after 30 days. It runs hourly (500 domains, 30-minute budget), at least 3 seconds between requests. It shares one crash-safe pacing and cooldown state with the launch-directory metrics job, which uses the same key, so a Retry-After received by either stops both. DR-only writes do not move the frontend revision; the daily snapshot publishes new values. Pages: a Domain rating block on each network page (median, average, coverage, bands, comparison with the big networks, hidden below 20 rated sites or 25% coverage), a DR column, sorts and filter in the sites lists, a checker row, and average DR on the stats page. Every place that shows or returns DR, including the public JSON APIs and the private MCP and REST, carries the linked "Domain Rating by Ahrefs" credit, as the Ahrefs license requires. The migration adds nullable columns and a NOT VALID range check, then validates it in a second transaction to avoid a long table lock. Co-Authored-By: Claude Opus 5.5 --- .../[slug]/network-stats.module.css | 28 +++ .../ad-networks/[slug]/network-stats.tsx | 37 +++- .../(directories)/ad-networks/[slug]/page.tsx | 5 +- .../ad-networks/niches/[niche]/niche-view.tsx | 5 +- .../(directories)/ad-networks/stats/page.tsx | 58 +++--- .../launch-directories/[slug]/page.tsx | 2 +- .../launch-directories/ahrefs-credit.tsx | 11 -- .../compare/compare-view.tsx | 2 +- .../launch-directories.module.css | 3 - .../launch-directory-view.tsx | 2 +- .../launch-directories/launch-glance.tsx | 2 +- .../ad-network-checker-tool.tsx | 6 + app/api/ad-networks/input.ts | 15 +- app/api/ad-networks/site/route.ts | 3 +- app/ui/ad-networks/sites-table.module.css | 39 ++++ app/ui/ad-networks/sites-table.tsx | 46 +++-- app/ui/ahrefs-credit.module.css | 3 + app/ui/ahrefs-credit.tsx | 12 ++ app/ui/report.tsx | 2 +- content_docs/ad-networks/maintenance.md | 13 ++ .../ad-networks/production-runbook.md | 13 ++ lib/ad-networks-report.ts | 53 ++++++ lib/ad-networks-types.ts | 38 +++- lib/ad-networks.ts | 76 ++++++-- lib/ahrefs.ts | 11 ++ lib/data-api/ad-networks/core.ts | 36 ++-- lib/data-api/ad-networks/definitions.ts | 2 + lib/data-api/ad-networks/schemas.ts | 6 +- lib/data-api/ad-networks/sql.ts | 2 +- lib/data-api/mcp.ts | 10 +- package.json | 1 + scripts/ad-networks/domain-rating.mjs | 97 ++++++++++ scripts/ad-networks/local-db.mjs | 2 + scripts/ad-networks/ovh/README.md | 25 +++ scripts/ad-networks/ovh/crontab.txt | 2 + scripts/ad-networks/refresh.mjs | 6 +- scripts/ad-networks/sync.mjs | 2 +- scripts/launch-directories/README.md | 3 +- .../launch-directories/refresh-metrics.mjs | 57 +----- scripts/stack127/ahrefs-dr.mjs | 134 ++++++++++++++ stack127_docs/mcp-and-api.md | 17 +- stack127_docs/monitoring.json | 3 +- ...0261114130000_ad_network_domain_rating.sql | 120 +++++++++++++ tests/data-api/core.test.mjs | 32 +++- tests/pages/ad-networks/ahrefs-crash.test.mjs | 140 +++++++++++++++ tests/pages/ad-networks/backend.test.mjs | 22 ++- tests/pages/ad-networks/chart-data.test.mjs | 2 +- .../ad-networks/checker-domains.test.mjs | 2 + .../domain-rating-database.test.mjs | 122 +++++++++++++ .../ad-networks/domain-rating-report.test.mjs | 135 ++++++++++++++ .../pages/ad-networks/domain-rating.test.mjs | 169 ++++++++++++++++++ .../ad-networks/incremental-sync.test.mjs | 13 ++ tests/pages/ad-networks/loader-harness.mjs | 2 + tests/pages/ad-networks/publishers.test.mjs | 2 + tests/pages/ad-networks/refresh.test.mjs | 17 ++ tests/pages/ad-networks/report.test.mjs | 12 +- .../ad-networks/sites-sort-database.test.mjs | 2 +- 57 files changed, 1502 insertions(+), 180 deletions(-) create mode 100644 app/(site)/(directories)/ad-networks/[slug]/network-stats.module.css delete mode 100644 app/(site)/(directories)/launch-directories/ahrefs-credit.tsx create mode 100644 app/ui/ahrefs-credit.module.css create mode 100644 app/ui/ahrefs-credit.tsx create mode 100644 lib/ahrefs.ts create mode 100644 scripts/ad-networks/domain-rating.mjs create mode 100644 scripts/ad-networks/ovh/README.md create mode 100644 scripts/stack127/ahrefs-dr.mjs create mode 100644 supabase/migrations/20261114130000_ad_network_domain_rating.sql create mode 100644 tests/pages/ad-networks/ahrefs-crash.test.mjs create mode 100644 tests/pages/ad-networks/domain-rating-database.test.mjs create mode 100644 tests/pages/ad-networks/domain-rating-report.test.mjs create mode 100644 tests/pages/ad-networks/domain-rating.test.mjs diff --git a/app/(site)/(directories)/ad-networks/[slug]/network-stats.module.css b/app/(site)/(directories)/ad-networks/[slug]/network-stats.module.css new file mode 100644 index 00000000..f9213f97 --- /dev/null +++ b/app/(site)/(directories)/ad-networks/[slug]/network-stats.module.css @@ -0,0 +1,28 @@ +/* Domain rating block: the comparison under the histogram, then the caption and Ahrefs credit. */ +.drCompare { + border-top: 1px dashed var(--color-border); + display: grid; + gap: var(--space-4); + margin-top: var(--space-8); + padding-top: var(--space-7); +} + +/* Bars this short read better at reading width than across a wide canvas. */ +.drCompare > dl { + max-width: 640px; +} + +.drCompare > p, +.drNote { + color: var(--color-text-muted); + font: var(--text-small) / 1.4 var(--font-mono); + margin: 0; +} + +.drNote { + display: flex; + flex-wrap: wrap; + gap: var(--space-2) var(--space-6); + justify-content: space-between; + margin-top: var(--space-7); +} diff --git a/app/(site)/(directories)/ad-networks/[slug]/network-stats.tsx b/app/(site)/(directories)/ad-networks/[slug]/network-stats.tsx index 178dfd71..9450a4bf 100644 --- a/app/(site)/(directories)/ad-networks/[slug]/network-stats.tsx +++ b/app/(site)/(directories)/ad-networks/[slug]/network-stats.tsx @@ -12,7 +12,9 @@ import { JoinedLeftBars } from "@/app/ui/ad-networks/viz/joined-left"; import { SwitchFlow } from "@/app/ui/ad-networks/viz/switch-flow"; import { TypeScale } from "@/app/ui/ad-networks/viz/type-scale"; import { nicheName } from "@/app/ui/ad-networks/site-format"; +import { AhrefsCredit } from "@/app/ui/ahrefs-credit"; import { ordinal, percent } from "../ad-networks-shared"; +import styles from "./network-stats.module.css"; // Loader shares are 0 to 100. Never round a share that isn't 100% up to 100%. const pct = (share: number | undefined) => percent(share ?? 0); @@ -104,6 +106,7 @@ export function NetworkStats({ network, stats }: { network: AdNetworkListRow; st ...(otherPlatforms > 0 ? [{ label: "Other", value: otherPlatforms }] : []), ...(unknownPlatforms > 0 ? [{ label: "Unknown", value: unknownPlatforms, tone: "track" as const }] : []), ]; + const dr = stats.domain_rating; const topEndings = stats.domain_endings.slice(0, 5); const otherEndings = stats.domain_endings.slice(5).reduce((sum, row) => sum + (row.share ?? 0), 0); const endings = [ @@ -162,6 +165,38 @@ export function NetworkStats({ network, stats }: { network: AdNetworkListRow; st /> ) : null; + // Hidden until enough of this network's sites are rated (AD_NETWORK_DR_GATE in lib/ad-networks-report.ts). + // Ahrefs' license needs the linked credit wherever DR shows, so it sits in the block's last line. + const drBlock = dr ? ( + + + ({ label: band.label, value: band.count, tone: band.key === dr.median_band ? "ac" as const : "ac2" as const }))} + /> + {dr.comparison.length ? ( +
+

Average DR of rated sites only: {shortName(name)} vs other big networks

+ ({ + label: shortName(row.name), value: row.average, display: <>{Math.round(row.average)}
{formatNumber(row.rated)} of {formatNumber(row.listed)} rated, + highlight: row.current, href: row.current ? undefined : `/ad-networks/${row.slug}`, + }))} + tone="highlight" + /> +
+ ) : null} +

{name} sites by Domain Rating. The highlighted bar holds the median.

+
+ ) : null; const buildBlocks = [ measuredPlatforms.length ? ( @@ -214,7 +249,7 @@ export function NetworkStats({ network, stats }: { network: AdNetworkListRow; st }, { title: `What kind of sites use ${name}?`, has: kindBlocks.some(Boolean), content: }, { title: `Which sites joined or left ${name}?`, has: moveBlocks.some(Boolean), content: }, - { title: `How are ${name} sites built?`, has: Boolean(ageBlock) || buildBlocks.some(Boolean), content: <>{ageBlock} }, + { title: `How are ${name} sites built?`, has: Boolean(ageBlock || drBlock) || buildBlocks.some(Boolean), content: <>{ageBlock}{drBlock} }, { title: "What else is in their ads.txt?", has: adsTxtBlocks.some(Boolean), content: }, ]; diff --git a/app/(site)/(directories)/ad-networks/[slug]/page.tsx b/app/(site)/(directories)/ad-networks/[slug]/page.tsx index fac35d81..58a21bf7 100644 --- a/app/(site)/(directories)/ad-networks/[slug]/page.tsx +++ b/app/(site)/(directories)/ad-networks/[slug]/page.tsx @@ -71,8 +71,9 @@ export default async function AdNetworkPage({ params }: { params: Promise<{ slug {network.listed_count ? ( ) : null} diff --git a/app/(site)/(directories)/ad-networks/niches/[niche]/niche-view.tsx b/app/(site)/(directories)/ad-networks/niches/[niche]/niche-view.tsx index ea7f295c..e0bebf63 100644 --- a/app/(site)/(directories)/ad-networks/niches/[niche]/niche-view.tsx +++ b/app/(site)/(directories)/ad-networks/niches/[niche]/niche-view.tsx @@ -97,8 +97,9 @@ export function NicheView({ breadcrumbs, data, Intro, title, updatedAt }: { {niche.networks.length ? ( diff --git a/app/(site)/(directories)/ad-networks/stats/page.tsx b/app/(site)/(directories)/ad-networks/stats/page.tsx index a69e42c0..74968272 100644 --- a/app/(site)/(directories)/ad-networks/stats/page.tsx +++ b/app/(site)/(directories)/ad-networks/stats/page.tsx @@ -10,12 +10,13 @@ import { Report, ReportBars, ReportBlock, ReportGroup, ReportNumbers, ReportRow, import { ColumnChart } from "@/app/ui/ad-networks/viz/columns"; import { DivergingBars } from "@/app/ui/ad-networks/viz/diverging-bars"; import { DotPlot } from "@/app/ui/ad-networks/viz/dot-plot"; -import { formatCompact, logStepPosition } from "@/app/ui/ad-networks/viz/format"; +import { formatCompact, formatNumber, logStepPosition } from "@/app/ui/ad-networks/viz/format"; import { MultiLine, type LineSeries } from "@/app/ui/ad-networks/viz/multi-line"; import { ScatterPlot } from "@/app/ui/ad-networks/viz/scatter-plot"; import { SizeStacks } from "@/app/ui/ad-networks/viz/size-stacks"; import { StackedColumns } from "@/app/ui/ad-networks/viz/stacked-columns"; import { formatCount } from "@/app/ui/ad-networks/site-format"; +import { AhrefsCredit } from "@/app/ui/ahrefs-credit"; import { AppShell } from "@/app/ui/site/app-shell"; import { EntryHero } from "@/app/ui/site/entry-hero"; import { PageCanvas } from "@/app/ui/site/page-canvas"; @@ -139,6 +140,34 @@ export default async function AdNetworkStatsPage() { const ageRows = data.median_domain_age.slice(0, 9).map((row) => ({ label: row.name, shortLabel: shortName(row.name), value: Math.round(row.years) })); const wordpressRows = data.wordpress_share.slice(0, 8) .map((row) => ({ label: row.name, value: row.share, display: percent(row.share), href: `/ad-networks/${row.slug}` })); + // Only networks with enough rated sites (AD_NETWORK_DR_GATE), highest average first. + const drRows = (data.domain_rating_by_network ?? []).slice(0, 8) + .map((row) => ({ label: row.name, value: row.average, display: <>{Math.round(row.average)}
{formatNumber(row.rated)} of {formatNumber(row.listed)} rated, href: `/ad-networks/${row.slug}` })); + + // data-gate: the domain-age run fills median ages; the platform run fills WordPress shares; the + // domain-rating run fills average DR, with the Ahrefs credit its license needs. + const compareBlocks = [ + scatter.length > 1 ? ( + + + + ) : null, + ageRows.length ? ( + + + + ) : null, + wordpressRows.length ? ( + + + + ) : null, + drRows.length ? ( + Average backlink strength from 0 to 100, covering rated sites only. } key="dr" label="average domain rating"> + + + ) : null, + ].filter(Boolean); const ads = data.ads_txt_metrics; const fileSizeTop = Math.max(0, ...data.ads_txt.file_sizes.map((row) => row.count)); @@ -237,26 +266,15 @@ export default async function AdNetworkStatsPage() { ) : null} - {/* data-gate: the domain-age run fills median ages; the platform run fills WordPress shares. */} - {scatter.length > 1 || ageRows.length || wordpressRows.length ? ( + {compareBlocks.length ? ( - - {scatter.length > 1 ? ( - - - - ) : null} - {ageRows.length ? ( - - - - ) : null} - {wordpressRows.length ? ( - - - - ) : null} - + {/* Four blocks sit as two pairs; three keep the original row of three. */} + {compareBlocks.length === 4 ? ( + <> + {compareBlocks.slice(0, 2)} + {compareBlocks.slice(2)} + + ) : {compareBlocks}} ) : null} diff --git a/app/(site)/(directories)/launch-directories/[slug]/page.tsx b/app/(site)/(directories)/launch-directories/[slug]/page.tsx index e89c4f75..06977455 100644 --- a/app/(site)/(directories)/launch-directories/[slug]/page.tsx +++ b/app/(site)/(directories)/launch-directories/[slug]/page.tsx @@ -21,7 +21,7 @@ import { shortUrl, submitStatusText, trafficLabels, trafficSteps, waitShort, } from "../launch-format"; import { LaunchLogo } from "../launch-logo"; -import { AhrefsCredit } from "../ahrefs-credit"; +import { AhrefsCredit } from "@/app/ui/ahrefs-credit"; import styles from "./directory.module.css"; // Build every place during the release. A place without a published registry row still returns 404. diff --git a/app/(site)/(directories)/launch-directories/ahrefs-credit.tsx b/app/(site)/(directories)/launch-directories/ahrefs-credit.tsx deleted file mode 100644 index 1bd77c63..00000000 --- a/app/(site)/(directories)/launch-directories/ahrefs-credit.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import styles from "./launch-directories.module.css"; - -/** - * The credit Ahrefs' Domain Rating license requires next to every place DR is shown: - * the words "Domain Rating by Ahrefs", with "Ahrefs" linking to ahrefs.com. - */ -export function AhrefsCredit({ prefix }: { prefix?: string }) { - return - {prefix ? `${prefix} ` : null}Domain Rating by Ahrefs - ; -} diff --git a/app/(site)/(directories)/launch-directories/compare/compare-view.tsx b/app/(site)/(directories)/launch-directories/compare/compare-view.tsx index 13a4215f..45c9f1dc 100644 --- a/app/(site)/(directories)/launch-directories/compare/compare-view.tsx +++ b/app/(site)/(directories)/launch-directories/compare/compare-view.tsx @@ -9,7 +9,7 @@ import type { LaunchDirectory } from "@/lib/launch-directories"; import { LaunchCard, LaunchGrid, launchHref, type LaunchCategory } from "../launch-card"; import { costShort, formatDate, formatPrice, isSubmitReady, linkShort, shortUrl, submitStatusText, trafficLabels, trafficSteps, waitShort } from "../launch-format"; import { LaunchLogo } from "../launch-logo"; -import { AhrefsCredit } from "../ahrefs-credit"; +import { AhrefsCredit } from "@/app/ui/ahrefs-credit"; import styles from "./compare.module.css"; export type CompareRecord = Pick{site.managed_by.name} } : null, // data-gate: the traffic, niche, platform, domain-age, and publisher-names runs fill these facts. site.traffic_group ? { label: "traffic", value: } : null, + // data-gate: the domain-rating run. Ahrefs' license needs the linked credit beside the number. + typeof site.domain_rating === "number" ? { label: "domain rating", value: <> + {site.domain_rating}{site.domain_rating_checked_on ? <> · {formatDay(site.domain_rating_checked_on)} : null} +
+ } : null, niche && site.niche ? { label: "niche", value: {niche} } : null, site.platform ? { label: "platform", value: site.platform } : null, age ? { label: "domain registered", value: <>{age.year}{age.years !== null ? <> · {age.years} {age.years === 1 ? "yr" : "yrs"} : null} } : null, diff --git a/app/api/ad-networks/input.ts b/app/api/ad-networks/input.ts index fd1b8a94..11cde0e7 100644 --- a/app/api/ad-networks/input.ts +++ b/app/api/ad-networks/input.ts @@ -1,16 +1,18 @@ import { domainToASCII } from "node:url"; import { isIP } from "node:net"; import { getDomain } from "tldts"; -import { AD_NETWORK_SITES_PAGE_SIZE } from "../../../lib/ad-networks-report.ts"; +import { AD_NETWORK_DR_FILTER_STEPS, AD_NETWORK_SITES_PAGE_SIZE } from "../../../lib/ad-networks-report.ts"; import { getAdNetworkNiche } from "../../../lib/ad-network-niches.ts"; import type { AdNetworkNicheSlug } from "../../../lib/ad-network-niches.ts"; -import type { AdNetworkSitesQuery, AdNetworkSitesSort } from "../../../lib/ad-networks-types.ts"; +import type { AdNetworkDrFilter, AdNetworkSitesQuery, AdNetworkSitesSort } from "../../../lib/ad-networks-types.ts"; export class AdNetworkInputError extends Error {} -const sorts = new Set(["domain", "traffic", "oldest", "newest", "checked"]); +const sorts = new Set(["domain", "traffic", "oldest", "newest", "checked", "highest-dr", "lowest-dr"]); +// Only the select's exact steps, so each DR view has one cache key (`dr=30`, never `dr=030` or `dr=31`). +const drSteps = new Map(AD_NETWORK_DR_FILTER_STEPS.map((step) => [String(step), step])); const slug = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; export function parseAdNetworkSitesParams(params: URLSearchParams): AdNetworkSitesQuery { - const allowed = new Set(["network", "niche", "q", "sort", "page", "pageSize"]); + const allowed = new Set(["network", "niche", "q", "sort", "page", "pageSize", "dr"]); for (const key of params.keys()) { if (!allowed.has(key) || params.getAll(key).length > 1) throw new AdNetworkInputError(`Invalid parameter: ${key}`); } @@ -22,6 +24,8 @@ export function parseAdNetworkSitesParams(params: URLSearchParams): AdNetworkSit if (niche && !getAdNetworkNiche(niche)) throw new AdNetworkInputError("Invalid niche."); if (q.length > 100 || /[\x00-\x1f\x7f]/.test(q)) throw new AdNetworkInputError("Search must be at most 100 characters."); if (!sorts.has(sort)) throw new AdNetworkInputError("Invalid sort."); + const dr = params.get("dr"); + if (dr !== null && !drSteps.has(dr)) throw new AdNetworkInputError("Invalid dr."); function integer(key: string, fallback: number, max: number) { const raw = params.get(key); if (raw === null) return fallback; @@ -29,7 +33,8 @@ export function parseAdNetworkSitesParams(params: URLSearchParams): AdNetworkSit return Number(raw); } return { network, niche: niche as AdNetworkNicheSlug | null, q, sort: sort as AdNetworkSitesSort, - page: integer("page", 1, 10000), pageSize: integer("pageSize", AD_NETWORK_SITES_PAGE_SIZE, 100) }; + page: integer("page", 1, 10000), pageSize: integer("pageSize", AD_NETWORK_SITES_PAGE_SIZE, 100), + minDr: dr === null ? null : drSteps.get(dr) as AdNetworkDrFilter }; } export function parseAdNetworkSiteParams(params: URLSearchParams): string { if ([...params.keys()].some(key => key !== "domain") || params.getAll("domain").length !== 1) { diff --git a/app/api/ad-networks/site/route.ts b/app/api/ad-networks/site/route.ts index 9b8da8ef..64016011 100644 --- a/app/api/ad-networks/site/route.ts +++ b/app/api/ad-networks/site/route.ts @@ -1,3 +1,4 @@ +import { ahrefsAttribution } from "@/lib/ahrefs"; import { NextResponse } from "next/server"; import { getAdNetworkCheckerSite } from "@/lib/ad-networks"; import { getPublishedPage } from "@/lib/page-registry"; @@ -14,7 +15,7 @@ export async function GET(request: Request) { return NextResponse.json({ error: "Tool not found." }, { status: 404, headers: noStore }); } const site = await getAdNetworkCheckerSite(domain); - const result: AdNetworkSiteResponse = { domain, found: site !== null, site }; + const result: AdNetworkSiteResponse = { domain, found: site !== null, site, ...ahrefsAttribution([site?.site.domain_rating, ...(site?.similar_sites ?? []).map(row => row.domain_rating)]) }; return NextResponse.json(result, { headers: site ? { "Cache-Control": "public, max-age=0, must-revalidate, s-maxage=604800", "Cache-Tag": "stack127-public,ad-networks-catalog", "X-Robots-Tag": "noindex, follow", diff --git a/app/ui/ad-networks/sites-table.module.css b/app/ui/ad-networks/sites-table.module.css index 87fb59f3..7114b55a 100644 --- a/app/ui/ad-networks/sites-table.module.css +++ b/app/ui/ad-networks/sites-table.module.css @@ -45,6 +45,11 @@ white-space: nowrap; } +.table thead abbr { + text-decoration: none; + text-transform: none; +} + .table tbody th, .table tbody td { border-top: 1px solid var(--color-border); @@ -80,6 +85,28 @@ font: var(--text-small) var(--font-mono); } +/* DR: the number, then a small bar on the 0 to 100 scale. */ +.dr { + align-items: center; + display: inline-flex; + gap: var(--space-3); +} + +.drBar { + background: var(--color-surface-2); + border-radius: 1px; + display: inline-block; + height: 6px; + overflow: hidden; + width: 46px; +} + +.drBar i { + background: var(--color-accent-text); + display: block; + height: 100%; +} + .site { min-width: 0; } @@ -212,6 +239,18 @@ display: none; } + .table tbody td + td[data-column="dr"]::before { + content: " · DR "; + } + + .drBar { + display: none; + } + + .table tbody td[data-column="dr"] .none { + font: inherit; + } + .table tbody td + td[data-column="manager"]::before { content: " · managed by "; } diff --git a/app/ui/ad-networks/sites-table.tsx b/app/ui/ad-networks/sites-table.tsx index c608a064..7bfb1b88 100644 --- a/app/ui/ad-networks/sites-table.tsx +++ b/app/ui/ad-networks/sites-table.tsx @@ -4,6 +4,8 @@ import Link from "next/link"; import { useCallback, useEffect, useRef, useState, type MouseEvent, type ReactNode } from "react"; import { adNetworkNiches, type AdNetworkNicheSlug } from "@/lib/ad-network-niches"; import type { AdNetworkRef, AdNetworkSiteRow, AdNetworkSitesResponse, AdNetworkSitesSort } from "@/lib/ad-networks-types"; +import { AD_NETWORK_DR_FILTER_STEPS } from "@/lib/ad-networks-report"; +import { AhrefsCredit } from "@/app/ui/ahrefs-credit"; import { UiButton } from "@/app/ui/controls"; import { Pagination } from "@/app/ui/pagination"; import { StatePanel } from "@/app/ui/state-panel"; @@ -12,7 +14,7 @@ import { NOT_LISTED, checkerHref, domainAge, formatCount, formatDay, managerName import { TrafficMeter } from "./traffic-meter"; import styles from "./sites-table.module.css"; -export type SitesTableColumn = "niche" | "networks" | "traffic" | "age" | "platform" | "manager" | "checked"; +export type SitesTableColumn = "niche" | "networks" | "traffic" | "dr" | "age" | "platform" | "manager" | "checked"; export type SitesTableProps = { /** The page path. Filters, sort, and page live in its query, such as `?q=&niche=&page=2`. */ @@ -30,6 +32,8 @@ export type SitesTableProps = { nicheFilter?: boolean; /** Show a network filter with these networks. */ networkFilter?: readonly AdNetworkRef[]; + /** Show the minimum Domain Rating filter once sites in scope have ratings. */ + drFilter?: boolean; /** Words for the count line, such as `food sites`. */ noun?: string; /** @@ -40,7 +44,7 @@ export type SitesTableProps = { sorts?: readonly AdNetworkSitesSort[]; }; -type View = { q: string; niche: string; network: string; sort: AdNetworkSitesSort; page: number }; +type View = { q: string; niche: string; network: string; dr: string; sort: AdNetworkSitesSort; page: number }; const sortOptions: { value: AdNetworkSitesSort; label: string }[] = [ { value: "domain", label: "A to Z" }, @@ -48,13 +52,17 @@ const sortOptions: { value: AdNetworkSitesSort; label: string }[] = [ { value: "oldest", label: "Oldest domain" }, { value: "newest", label: "Newest domain" }, { value: "checked", label: "Last checked" }, + { value: "highest-dr", label: "Highest DR" }, + { value: "lowest-dr", label: "Lowest DR" }, ]; +const drOptions = [{ value: "", label: "Any" }, ...AD_NETWORK_DR_FILTER_STEPS.map((step) => ({ value: String(step), label: `${step}+` }))]; const allSorts = new Set(sortOptions.map((option) => option.value)); const mockupSorts: readonly AdNetworkSitesSort[] = ["traffic", "oldest", "newest"]; const apiColumn: Record = { niche: "niche", networks: "network", traffic: "traffic", + dr: "domain_rating", age: "domain_age", platform: "platform", manager: "manager", @@ -64,22 +72,25 @@ const columnLabels: Record = { niche: "niche", networks: "network", traffic: "traffic", + dr: "DR", age: "domain age", platform: "platform", manager: "ads managed by", checked: "checked", }; -function readView(defaultSort: AdNetworkSitesSort, props: Pick): View { +function readView(defaultSort: AdNetworkSitesSort, props: Pick): View { const search = new URLSearchParams(window.location.search); const sort = search.get("sort") ?? ""; const page = Number(search.get("page")); const niche = search.get("niche") ?? ""; const network = search.get("network") ?? ""; + const dr = search.get("dr") ?? ""; return { q: (search.get("q") ?? "").slice(0, 100), niche: props.nicheFilter && adNetworkNiches.some((item) => item.slug === niche) ? niche : "", network: props.networkFilter?.some((item) => item.slug === network) ? network : "", + dr: props.drFilter && drOptions.some((item) => item.value === dr) ? dr : "", sort: allSorts.has(sort) ? sort as AdNetworkSitesSort : defaultSort, page: Number.isInteger(page) && page > 1 && page <= 10000 ? page : 1, }; @@ -90,6 +101,7 @@ function viewHref(route: string, view: View, defaultSort: AdNetworkSitesSort, pa if (view.q.trim()) params.set("q", view.q.trim()); if (view.niche) params.set("niche", view.niche); if (view.network) params.set("network", view.network); + if (view.dr) params.set("dr", view.dr); if (view.sort !== defaultSort) params.set("sort", view.sort); if (page > 1) params.set("page", String(page)); const query = params.toString(); @@ -97,7 +109,7 @@ function viewHref(route: string, view: View, defaultSort: AdNetworkSitesSort, pa } const isDefault = (view: View, defaultSort: AdNetworkSitesSort) => - !view.q.trim() && !view.niche && !view.network && view.sort === defaultSort && view.page === 1; + !view.q.trim() && !view.niche && !view.network && !view.dr && view.sort === defaultSort && view.page === 1; function TrafficCell({ group }: { group: string | null }) { return group ? : {NOT_LISTED}; @@ -111,6 +123,12 @@ function AgeCell({ row }: { row: AdNetworkSiteRow }) { : <>{age.years} {age.years === 1 ? "yr" : "yrs"} · {age.year}; } +/** The site's DR with a small bar on the 0 to 100 scale. Unrated sites read "not yet". */ +function DrCell({ rating }: { rating: number | null | undefined }) { + if (rating === null || rating === undefined) return not yet; + return {rating}; +} + function Cell({ column, row }: { column: SitesTableColumn; row: AdNetworkSiteRow }) { switch (column) { case "niche": { @@ -127,6 +145,8 @@ function Cell({ column, row }: { column: SitesTableColumn; row: AdNetworkSiteRow ); case "traffic": return ; + case "dr": + return ; case "age": return ; case "platform": @@ -146,8 +166,8 @@ function Cell({ column, row }: { column: SitesTableColumn; row: AdNetworkSiteRow * row becomes a two-line block, so the table never scrolls sideways. */ export function SitesTable(props: SitesTableProps) { - const { columns, defaultSort, heading, headingId, initial, networkFilter, nicheFilter, noun = "sites", route, scope, sorts = mockupSorts } = props; - const [view, setView] = useState({ q: "", niche: "", network: "", sort: defaultSort, page: 1 }); + const { columns, defaultSort, drFilter, heading, headingId, initial, networkFilter, nicheFilter, noun = "sites", route, scope, sorts = mockupSorts } = props; + const [view, setView] = useState({ q: "", niche: "", network: "", dr: "", sort: defaultSort, page: 1 }); const [data, setData] = useState(initial); const [status, setStatus] = useState<"idle" | "loading" | "error">("idle"); const [attempt, setAttempt] = useState(0); @@ -176,7 +196,8 @@ export function SitesTable(props: SitesTableProps) { ); const showNicheFilter = Boolean(nicheFilter) && ((data.availableFilters?.includes("niche") && nicheOptions.length > 0) || Boolean(view.niche)); const showNetworkFilter = Boolean(networkFilter) && ((data.availableFilters?.includes("network") && networkOptions.length > 0) || Boolean(view.network)); - const filterRef = useRef({ nicheFilter, networkFilter }); + const showDrFilter = Boolean(drFilter) && (Boolean(data.availableFilters?.includes("domain_rating")) || Boolean(view.dr)); + const filterRef = useRef({ nicheFilter, networkFilter, drFilter }); // Read a shared link after hydration, and follow the back button. useEffect(() => { @@ -203,6 +224,7 @@ export function SitesTable(props: SitesTableProps) { if (network) params.set("network", network); if (niche) params.set("niche", niche); if (view.q.trim()) params.set("q", view.q.trim()); + if (view.dr) params.set("dr", view.dr); params.set("sort", view.sort); params.set("page", String(view.page)); params.set("pageSize", String(initial.pageSize)); @@ -249,14 +271,15 @@ export function SitesTable(props: SitesTableProps) { traffic: (row) => row.traffic_group, age: (row) => row.domain_created_on, platform: (row) => row.platform, + dr: (row) => row.domain_rating ?? null, }; const available = data.availableColumns; const shown = columns.filter((column) => available ? available.includes(apiColumn[column]) : !hasValue[column] || data.rows.some((row) => hasValue[column]!(row))); - const reset = () => update({ q: "", niche: "", network: "", sort: defaultSort }); - const filtered = Boolean(view.q.trim() || view.niche || view.network); + const reset = () => update({ q: "", niche: "", network: "", dr: "", sort: defaultSort }); + const filtered = Boolean(view.q.trim() || view.niche || view.network || view.dr); const first = data.total ? (data.page - 1) * data.pageSize + 1 : 0; const last = Math.min(data.total, data.page * data.pageSize); @@ -279,7 +302,7 @@ export function SitesTable(props: SitesTableProps) { site - {shown.map((column) => {columnLabels[column]})} + {shown.map((column) => {column === "dr" ? {columnLabels[column]} : columnLabels[column]})} @@ -297,6 +320,8 @@ export function SitesTable(props: SitesTableProps) {
{formatCount(first)}-{formatCount(last)} of {formatCount(data.total)} {noun} + {/* Ahrefs' license needs this credit wherever DR shows. */} + {shown.includes("dr") ? : null}
) : null} + {showDrFilter ? update({ dr })} options={drOptions} value={view.dr} /> : null} {shownSorts.length > 1 ? update({ sort: sort as AdNetworkSitesSort })} options={shownSorts} value={shownSort} /> : null} : null}
diff --git a/app/ui/ahrefs-credit.module.css b/app/ui/ahrefs-credit.module.css new file mode 100644 index 00000000..9085af85 --- /dev/null +++ b/app/ui/ahrefs-credit.module.css @@ -0,0 +1,3 @@ +.credit { color: var(--color-text-muted); font: 0.78rem / 1.4 var(--font-mono); } +.credit a { color: inherit; text-decoration: underline; text-underline-offset: 3px; } +.credit a:hover { color: var(--color-text); } diff --git a/app/ui/ahrefs-credit.tsx b/app/ui/ahrefs-credit.tsx new file mode 100644 index 00000000..0f12e7c5 --- /dev/null +++ b/app/ui/ahrefs-credit.tsx @@ -0,0 +1,12 @@ +import { AHREFS_ATTRIBUTION } from "@/lib/ahrefs"; +import styles from "./ahrefs-credit.module.css"; + +/** + * The credit Ahrefs' Domain Rating license requires next to every place DR is shown: + * the words "Domain Rating by Ahrefs", with "Ahrefs" linking to ahrefs.com. + */ +export function AhrefsCredit({ prefix }: { prefix?: string }) { + return + {prefix ? `${prefix} ` : null}Domain Rating by Ahrefs + ; +} diff --git a/app/ui/report.tsx b/app/ui/report.tsx index 3b9357c5..db01212d 100644 --- a/app/ui/report.tsx +++ b/app/ui/report.tsx @@ -103,7 +103,7 @@ export type ReportBar = { label: string; value: number; /** The number as shown at the end of the row, such as `31%` or `7,852`. */ - display: string; + display: ReactNode; href?: string; /** Marks the row the page is about, such as this network in a ranking. */ current?: boolean; diff --git a/content_docs/ad-networks/maintenance.md b/content_docs/ad-networks/maintenance.md index aeb7909a..761c7318 100644 --- a/content_docs/ad-networks/maintenance.md +++ b/content_docs/ad-networks/maintenance.md @@ -528,3 +528,16 @@ Only a fully valid, nonblank HTTP 200 ads.txt body counts an absence. Reviews re The September 2026 file has 1k, 5k, 10k, 50k, 100k, 500k, and 1m magnitude buckets. Both 1k and 5k map to `top-10k`; 50k maps to `top-100k`; 500k maps to `top-1m`. The best rank for the exact hostname, treating `www` as an alias, supplies the group. Other subdomains are never merged into their parent. `not-ranked` means absent from this complete top-million snapshot, not zero traffic. One malformed upstream ACE origin is skipped and counted; too many malformed origins or an unknown rank magnitude stops the import before writes. SQLite `enrichment_state(kind='traffic').data` records the month, Google creator, CC BY license link, source URL, cache credit, and the grouping adaptation. Sync retains it under `last_check_evidence.enrichment.traffic` in development. Keep this attribution with any public presentation of the derived traffic groups. Production publication and any visible attribution copy remain part of the coordinator's release review. No BigQuery credential is needed for this cached dataset path. + + +## Domain Rating enrichment (prepared November 14, 2026) + +`domain-rating` is a new SQLite enrichment job; see [its runner instructions](../../scripts/ad-networks/ovh/README.md). It is prepared in source only. The lead must release it and install its single cron entry; preserve the current verification pause and existing jobs. + +- Storage: `publishers.domain_rating` (integer 0–100 or null), `domain_rating_checked_on` (last successful UTC day), mirrored to `public.ad_network_publishers` by the existing incremental 500-row sync. Failed lookups keep both values; a separate `enrichment_state(kind='domain-rating')` checkpoint retries after 24 hours. Old snapshots without the columns leave target DR alone. Candidates are measured locally but become serving rows only after the existing reviewed-relationship admission. +- Priority: never-checked confirmed/stale manager sites, other listed sites, other never-checked sites, then oldest successful checks older than 30 days. One request at a time, at least three seconds apart. Honor 429/Retry-After and stop on authentication failure or repeated rate limits. Budget 30 minutes, limit 500, at :05 during hours 07–23 UTC; at most 8,500 domains/day. OVH has 213,003 candidates, including 123,361 listed and 39,823 managed sites. Managed sites take about 5 days and full first coverage about 26 days before throttling or slow responses. Both Ahrefs jobs share a process-held SQLite request lock in `/srv/stack127-data/state/ahrefs.sqlite.lock` and durable state in `/srv/stack127-data/state/ahrefs.sqlite`. Each request commits its pacing reservation before HTTP starts. A 429 commits its cooldown before body cleanup or callbacks. A hard kill releases the lock and retains committed state; a kill before the cooldown commit retains only the pacing reservation. The job user must be able to write that directory. Stop both jobs before rolling out the new lock path, preserving the existing state file. +- Env: add `AHREFS_API_KEY` to Infisical `stack127 /vps/ad-networks`. Cockpit's env sync writes `/srv/stack127-data/ad-networks.env`; there is no ad-network prepare-env step. Do not edit that file, the Mac mirrors, or the launch directory folder. This is a private job credential, never a build or public variable. +- Migration: `20261114130000_ad_network_domain_rating.sql` adds PostgreSQL fields, private views and coverage, and excludes DR-only updates from frontend source revisions. Public pages now show per-site DR and network summaries with linked Ahrefs attribution. Collection finishes before one ordinary sync; DR never syncs after every lookup. Other writes retain their existing refresh behavior; DR-only changes queue no frontend build. The 00:10 UTC daily snapshot always inserts a dated row and queues a refresh, publishing new DR values daily even when counts are unchanged. Sync can occasionally overrun into :40 and make hourly verification skip once. This is acceptable because the next hourly run resumes due checks; repeated skips need investigation. +- Access: private MCP and REST return per-site values/dates and network median/average/coverage, with [Domain Rating by Ahrefs](https://ahrefs.com), a third-party estimate caveat and explicit coverage. Public sites/checker APIs return per-site DR with attribution metadata, and page summaries show rated/total coverage. See [API rules](../../stack127_docs/mcp-and-api.md) and the [license](https://ahrefs.com/legal/domain-rating-license). + +The tiny local smoke check used a real key and two managed domains sampled from development in a temporary SQLite store. A second run processed zero. The schema migration was applied twice in development; both runs preserved all 106,685 publishers and the 25,159 development ratings dated 2026-11-12. API SQL fixtures run inside a rolled-back transaction. No migration, job, cron or credential was installed on production. diff --git a/content_docs/ad-networks/production-runbook.md b/content_docs/ad-networks/production-runbook.md index 0ae78629..aa3ccb14 100644 --- a/content_docs/ad-networks/production-runbook.md +++ b/content_docs/ad-networks/production-runbook.md @@ -481,3 +481,16 @@ order by slug; rollback; ``` + + +## Domain Rating: release and operation + +`domain-rating` is a new SQLite enrichment job; see [its runner instructions](../../scripts/ad-networks/ovh/README.md). It is prepared in source only. The lead must release it and install its single cron entry; preserve the current verification pause and existing jobs. + +- Storage: `publishers.domain_rating` (integer 0–100 or null), `domain_rating_checked_on` (last successful UTC day), mirrored to `public.ad_network_publishers` by the existing incremental 500-row sync. Failed lookups keep both values; a separate `enrichment_state(kind='domain-rating')` checkpoint retries after 24 hours. Old snapshots without the columns leave target DR alone. Candidates are measured locally but become serving rows only after the existing reviewed-relationship admission. +- Priority: never-checked confirmed/stale manager sites, other listed sites, other never-checked sites, then oldest successful checks older than 30 days. One request at a time, at least three seconds apart. Honor 429/Retry-After and stop on authentication failure or repeated rate limits. Budget 30 minutes, limit 500, at :05 during hours 07–23 UTC; at most 8,500 domains/day. OVH has 213,003 candidates, including 123,361 listed and 39,823 managed sites. Managed sites take about 5 days and full first coverage about 26 days before throttling or slow responses. Both Ahrefs jobs share a process-held SQLite request lock in `/srv/stack127-data/state/ahrefs.sqlite.lock` and durable state in `/srv/stack127-data/state/ahrefs.sqlite`. Each request commits its pacing reservation before HTTP starts. A 429 commits its cooldown before body cleanup or callbacks. A hard kill releases the lock and retains committed state; a kill before the cooldown commit retains only the pacing reservation. The job user must be able to write that directory. Stop both jobs before rolling out the new lock path, preserving the existing state file. +- Env: add `AHREFS_API_KEY` to Infisical `stack127 /vps/ad-networks`. Cockpit's env sync writes `/srv/stack127-data/ad-networks.env`; there is no ad-network prepare-env step. Do not edit that file, the Mac mirrors, or the launch directory folder. This is a private job credential, never a build or public variable. +- Migration: `20261114130000_ad_network_domain_rating.sql` adds PostgreSQL fields, private views and coverage, and excludes DR-only updates from frontend source revisions. Public pages now show per-site DR and network summaries with linked Ahrefs attribution. Collection finishes before one ordinary sync; DR never syncs after every lookup. Other writes retain their existing refresh behavior; DR-only changes queue no frontend build. The 00:10 UTC daily snapshot always inserts a dated row and queues a refresh, publishing new DR values daily even when counts are unchanged. Sync can occasionally overrun into :40 and make hourly verification skip once. This is acceptable because the next hourly run resumes due checks; repeated skips need investigation. +- Access: private MCP and REST return per-site values/dates and network median/average/coverage, with [Domain Rating by Ahrefs](https://ahrefs.com), a third-party estimate caveat and explicit coverage. Public sites/checker APIs return per-site DR with attribution metadata, and page summaries show rated/total coverage. See [API rules](../../stack127_docs/mcp-and-api.md) and the [license](https://ahrefs.com/legal/domain-rating-license). + +The tiny local smoke check used a real key and two managed domains sampled from development in a temporary SQLite store. A second run processed zero. The schema migration was applied twice in development; both runs preserved all 106,685 publishers and the 25,159 development ratings dated 2026-11-12. API SQL fixtures run inside a rolled-back transaction. No migration, job, cron or credential was installed on production. diff --git a/lib/ad-networks-report.ts b/lib/ad-networks-report.ts index 8513a835..42e26220 100644 --- a/lib/ad-networks-report.ts +++ b/lib/ad-networks-report.ts @@ -1,4 +1,5 @@ import type { + AdNetworkDomainRatingReport, AdNetworkDomainRatingSummary, AdNetworkDrFilter, AdNetworkCount, AdNetworkDailyCount, AdNetworkMovement, AdNetworkSitesAvailability, AdNetworkRef, AdNetworkSwitch, AdNetworkSwitchFlow, AdNetworkSwitchFlowSide, AdNetworkMovementPeriod, AdNetworkHistoryObservation, AdNetworkCheckerEvent, } from "./ad-networks-types.ts"; @@ -19,9 +20,57 @@ export function adNetworkNicheShares(rows: AdNetworkCount[]): AdNetworkCount[] { return sorted ? rows.map(row => ({ ...row, share: row.count / sorted * 100 })) : []; } +/** + * When a network's Domain Rating numbers are worth showing. A half-collected network would show a + * misleading median, so its DR block and comparison rows hide until at least `minRated` sites and + * `minShare` of its listed sites are rated. Change the bar here only. + */ +export const AD_NETWORK_DR_GATE = { minRated: 20, minShare: 0.25 } as const; + +export function hasAdNetworkDomainRatingCoverage(rated: number, listed: number) { + // data-gate: the domain-rating run has rated enough of this network's listed sites. + return listed > 0 && rated >= AD_NETWORK_DR_GATE.minRated && rated / listed >= AD_NETWORK_DR_GATE.minShare; +} + +/** The sites table's DR filter: sites rated at least this much. */ +export const AD_NETWORK_DR_FILTER_STEPS: readonly AdNetworkDrFilter[] = [10, 20, 30, 50, 70]; + +/** The big networks every network page compares its average DR against. */ +export const AD_NETWORK_DR_COMPARISON = ["mediavine", "raptive", "ezoic", "journey-by-mediavine"] as const; + +/** 10-point bands, with everything from 70 in the last one. */ +export const AD_NETWORK_DR_BANDS = [0, 10, 20, 30, 40, 50, 60, 70].map((min) => ({ + key: min === 70 ? "70+" : `${min}-${min + 9}`, + label: min === 70 ? "70+" : `${min}–${min + 9}`, + min, +})); +export const adNetworkDrBand = (rating: number) => AD_NETWORK_DR_BANDS[Math.min(7, Math.max(0, Math.floor(rating / 10)))]; + +/** + * One network's DR block from per-network summaries and its band counts. Null below the gate. + * The median is shown rounded, so the highlighted band is the one holding the rounded median. + */ +export function adNetworkDomainRatingReport(slug: string, summaries: readonly AdNetworkDomainRatingSummary[], + bandCounts: readonly { min: number; count: number }[]): AdNetworkDomainRatingReport | null { + const own = summaries.find((row) => row.slug === slug); + if (!own || !hasAdNetworkDomainRatingCoverage(own.rated, own.listed)) return null; + const bands = AD_NETWORK_DR_BANDS.map((band) => ({ ...band, + count: bandCounts.filter((row) => adNetworkDrBand(row.min).key === band.key).reduce((sum, row) => sum + row.count, 0) })); + const comparison = summaries + .filter((row) => row.slug === slug || ((AD_NETWORK_DR_COMPARISON as readonly string[]).includes(row.slug) + && hasAdNetworkDomainRatingCoverage(row.rated, row.listed))) + .map((row) => ({ ...row, current: row.slug === slug })) + .sort((a, b) => b.average - a.average || a.slug.localeCompare(b.slug)); + return { ...own, bands, median_band: adNetworkDrBand(Math.round(own.median)).key, + // A comparison of one network with itself says nothing. + comparison: comparison.length > 1 ? comparison : [] }; +} + export type AdNetworkSiteCoverage = { listed: number; classified: number; traffic: number; age: number; platform: number; publisher: number; checked: number; manager: number; + /** Sites with a Domain Rating. Optional so older fixtures read as none rated. */ + rated?: number; niches: string[]; networks: { slug: string; name: string }[]; }; @@ -40,6 +89,10 @@ export function adNetworkSiteAvailability(coverage: AdNetworkSiteCoverage): AdNe if (coverage.publisher > 0) availableColumns.push("publisher"); if (coverage.checked > 0) { availableColumns.push("checked"); availableSorts.push("checked"); } if (coverage.manager > 0) availableColumns.push("manager"); + // data-gate: domain-rating run. Unrated rows read "not yet" and sort last. + if ((coverage.rated ?? 0) > 0) { + availableColumns.push("domain_rating"); availableSorts.push("highest-dr", "lowest-dr"); availableFilters.push("domain_rating"); + } if (coverage.networks.length) { availableColumns.push("network"); availableFilters.push("network"); } return { availableColumns, availableSorts, availableFilters, filters: { niches: nicheReady ? adNetworkNiches.filter(n => coverage.niches.includes(n.slug)) : [], networks: coverage.networks } }; diff --git a/lib/ad-networks-types.ts b/lib/ad-networks-types.ts index 14440e7c..27909955 100644 --- a/lib/ad-networks-types.ts +++ b/lib/ad-networks-types.ts @@ -1,3 +1,4 @@ +import type { AhrefsAttribution } from "./ahrefs"; import type { AdNetworkNicheSlug } from "./ad-network-niches"; /** API dates are ISO strings. Null means the field has not been collected. */ @@ -8,25 +9,32 @@ export type AdNetworkListRow = AdNetworkRef & { id: string; official_url: string | null; summary: string | null; verified_count: number; listed_count: number; stale_count: number; ended_count: number; }; -export type AdNetworkSiteRow = { +export type AdNetworkSiteRow = AhrefsAttribution & { domain: string; publisher_name: string | null; niche: AdNetworkNicheSlug | null; traffic_group: string | null; domain_created_on: string | null; platform: string | null; /** MANAGERDOMAIN evidence, with a confirmed manager relationship as fallback. */ managed_by: AdNetworkRef | null; manager_domain: string | null; last_checked_at: string | null; favicon: string | null; networks: AdNetworkRef[]; + /** Ahrefs Domain Rating, 0 to 100, and the day it was read. Null until the domain-rating run rates + * the site. Show it only with the linked "Domain Rating by Ahrefs" credit. */ + domain_rating?: number | null; domain_rating_checked_on?: string | null; }; -export type AdNetworkSitesSort = "domain" | "traffic" | "oldest" | "newest" | "checked"; +export type AdNetworkSitesSort = "domain" | "traffic" | "oldest" | "newest" | "checked" | "highest-dr" | "lowest-dr"; +/** The DR filter's steps: sites rated at least this much. Values live in AD_NETWORK_DR_FILTER_STEPS. */ +export type AdNetworkDrFilter = 10 | 20 | 30 | 50 | 70; export type AdNetworkSitesQuery = { network: string | null; niche: AdNetworkNicheSlug | null; q: string; sort: AdNetworkSitesSort; page: number; pageSize: number; + /** Sites with a Domain Rating of at least this. Null shows every site, rated or not. */ + minDr: AdNetworkDrFilter | null; }; export type AdNetworkSitesAvailability = { - availableColumns: ("domain" | "niche" | "traffic" | "domain_age" | "platform" | "manager" | "checked" | "network" | "publisher")[]; + availableColumns: ("domain" | "niche" | "traffic" | "domain_age" | "platform" | "manager" | "checked" | "network" | "publisher" | "domain_rating")[]; availableSorts: AdNetworkSitesSort[]; - availableFilters: ("niche" | "network")[]; + availableFilters: ("niche" | "network" | "domain_rating")[]; filters: { niches: { slug: string; name: string; line: string }[]; networks: AdNetworkRef[] }; }; -export type AdNetworkSitesResponse = AdNetworkSitesAvailability & { +export type AdNetworkSitesResponse = AdNetworkSitesAvailability & AhrefsAttribution & { effectiveSort: AdNetworkSitesSort; rows: AdNetworkSiteRow[]; total: number; page: number; pageSize: number; totalPages: number; }; @@ -61,7 +69,21 @@ export type AdNetworkMovementPeriod = { joined: number; left: number; net: number; }; export type AdNetworkArrival = { site: AdNetworkSiteRow; network: AdNetworkRef; observed_at: string }; +/** One network's Domain Rating summary over its rated listed sites. */ +export type AdNetworkDomainRatingSummary = AdNetworkRef & AhrefsAttribution & { + listed: number; rated: number; median: number; average: number; +}; +/** A network's DR block. Null until enough of its sites are rated (see AD_NETWORK_DR_GATE). */ +export type AdNetworkDomainRatingReport = AdNetworkDomainRatingSummary & { + /** Sites per 10-point band, 0–9 to 70+, every band present. */ + bands: { key: string; label: string; min: number; count: number }[]; + /** The band holding the rounded median. */ + median_band: string; + /** This network beside the big networks that also pass the gate, highest average first. */ + comparison: (AdNetworkDomainRatingSummary & { current: boolean })[]; +}; export type AdNetworkStatsReport = { + domain_rating?: AdNetworkDomainRatingReport | null; domain_age_strip?: AdNetworkDomainAgeStrip | null; switch_flow?: AdNetworkSwitchFlow | null; movement_periods?: AdNetworkMovementPeriod[]; @@ -95,7 +117,9 @@ export type AdNetworkNichePageData = { niche: AdNetworkNicheRow; sites: AdNetworkSitesResponse; traffic: AdNetworkTrafficShare[]; switches: AdNetworkSwitch[]; newest: (AdNetworkRef & { sites: AdNetworkSiteRow[] })[]; }; -export type AdNetworkStatsPageData = { +export type AdNetworkStatsPageData = AhrefsAttribution & { + /** Networks with enough rated sites, highest average DR first. */ + domain_rating_by_network?: AdNetworkDomainRatingSummary[]; domain_age_strip?: AdNetworkDomainAgeStrip | null; domain_age_by_network?: (AdNetworkRef & { strip: AdNetworkDomainAgeStrip })[]; confirmed_share: number | null; multiple_networks_share: number | null; updated_at: string | null; @@ -159,7 +183,7 @@ export type AdNetworkCheckerSiteView = { domain_age_years: number | null; similar_available_columns: AdNetworkSitesAvailability["availableColumns"]; }; -export type AdNetworkSiteResponse = { domain: string; found: boolean; site: AdNetworkCheckerSiteView | null }; +export type AdNetworkSiteResponse = AhrefsAttribution & { domain: string; found: boolean; site: AdNetworkCheckerSiteView | null }; export type AdNetworkJoiningRule = AdNetworkRef & { amount: number; unit: "sessions" | "pageviews" | "visitors" | "visits" | "monthly active users"; label?: string }; export type AdNetworkDirectorySort = "most-sites" | "easiest-to-join" | "by-niche"; diff --git a/lib/ad-networks.ts b/lib/ad-networks.ts index 0d1bb609..4a7ff953 100644 --- a/lib/ad-networks.ts +++ b/lib/ad-networks.ts @@ -1,3 +1,4 @@ +import { ahrefsAttribution } from "./ahrefs"; import { getPublicSuffix } from "tldts"; import { adNetworkNiches, getAdNetworkNiche } from "./ad-network-niches"; import type { AdNetworkNicheSlug } from "./ad-network-niches"; @@ -7,13 +8,15 @@ import type { AdNetworkStatsPageData, AdNetworkShare, AdNetworkCount, AdNetworkDailyCount, AdNetworkMovement, AdNetworkSwitch, AdNetworkTrafficShare, AdNetworkCheckerSiteView, AdNetworkSiteRelationship, AdNetworkAdsTxtSummary, AdNetworkSitesAvailability, AdNetworkDirectoryData, AdNetworkJoiningRule, AdNetworkDomainAgeStrip, + AdNetworkDomainRatingSummary, } from "./ad-networks-types"; import { deriveManagedBy, buildAdsTxtPreview, aggregateAdNetworkManagers, aggregateAdNetworkTrafficShares } from "./ad-network-site-facts.mjs"; -import { AD_NETWORK_SITES_PAGE_SIZE, adNetworkSiteAvailability, adNetworkNicheShares, hasAdNetworkNicheCoverage, adNetworkMonthEnds, adNetworkYearChange, +import { AD_NETWORK_SITES_PAGE_SIZE, AD_NETWORK_DR_FILTER_STEPS, adNetworkSiteAvailability, adNetworkNicheShares, hasAdNetworkNicheCoverage, adNetworkMonthEnds, adNetworkYearChange, adNetworkMovementTotals, adNetworkManagerBreakdown, trafficOrder, domainAgeOrder, adNetworkSwitchFlow, adNetworkMovementPeriods, adNetworkPlatformMix, - adNetworkCheckerEvents, adNetworkObservedDomainAge, adNetworkUtcWindow } from "./ad-networks-report"; + adNetworkCheckerEvents, adNetworkObservedDomainAge, adNetworkUtcWindow, + adNetworkDomainRatingReport, hasAdNetworkDomainRatingCoverage } from "./ad-networks-report"; import type { AdNetworkSiteCoverage } from "./ad-networks-report"; import { adNetworkFacts } from "../app/(site)/(directories)/ad-networks/network-facts"; @@ -179,7 +182,7 @@ function cached(key: string, loader: (...args: T) => Pro } try { return await read; } finally { if (pending.get(argumentKey) === read) pending.delete(argumentKey); } - }, [key, "v15"], { revalidate: false, tags: [AD_NETWORKS_TAG] }); + }, [key, "v17"], { revalidate: false, tags: [AD_NETWORKS_TAG] }); } const iso = (value: Date | string | null): string | null => value === null ? null : new Date(value).toISOString(); const ref = (network: AdNetworkRef): AdNetworkRef => ({ slug: network.slug, name: network.name }); @@ -203,7 +206,8 @@ const managerEvidenceProjection = `jsonb_build_object('manager_domains',p.last_c where r.publisher_id=p.id and r.status='confirmed' and r.relationship_type='primary_or_exclusive_manager'), '[]'::jsonb) as manager_relationships`; const siteProjection = `p.normalized_domain as domain, p.publisher_name, p.niche, p.traffic_group, - p.domain_created_on::text, p.platform, p.last_successful_check_at as last_checked_at, p.favicon,${managerEvidenceProjection}, + p.domain_created_on::text, p.platform, p.last_successful_check_at as last_checked_at, p.favicon, + p.domain_rating, p.domain_rating_checked_on::text,${managerEvidenceProjection}, coalesce((select jsonb_agg(jsonb_build_object('slug',n.slug,'name',n.name) order by n.name) from public.ad_network_publisher_relationships r join public.ad_networks n on n.id=r.ad_network_id where r.publisher_id=p.id and r.status in ('confirmed','stale')),'[]'::jsonb) as networks`; @@ -240,20 +244,22 @@ export async function loadAdNetworkManagerStatistics() { const getAdNetworkManagerStatistics=cached("ad-network-manager-statistics",loadAdNetworkManagerStatistics); function toSite(row: SiteRecord, identities: Awaited>): AdNetworkSiteRow { const { evidence, manager_relationships, ...site } = row; - return { ...site, last_checked_at: iso(row.last_checked_at), ...deriveManagedBy(evidence, identities, manager_relationships) }; + return { ...site, ...ahrefsAttribution([site.domain_rating]), last_checked_at: iso(row.last_checked_at), ...deriveManagedBy(evidence, identities, manager_relationships) }; } const siteWhere = `exists(select 1 from public.ad_network_publisher_relationships r join public.ad_networks n on n.id=r.ad_network_id where r.publisher_id=p.id and r.status in ('confirmed','stale') and ($1::text is null or n.slug=$1)) and ($2::text is null or p.niche=$2) and ($3::text='' or position($3 in p.normalized_domain)>0 or position($3 in lower(coalesce(p.publisher_name,'')))>0)`; +/** The sites table adds its DR filter as $4. Unrated sites never pass a DR filter. */ +const siteListWhere = `${siteWhere} and ($4::int is null or p.domain_rating>=$4)`; /** Scope controls to the table's network/niche, independent of search and page. */ export async function loadAdNetworkSitesAvailability(network: string | null, niche: string | null): Promise { const [coverage, networks, managerGroups, identities] = await Promise.all([ queryAdNetworkReport>(`${scopedCoverageSites} select count(*)::int as listed,count(p.niche)::int as classified,count(p.traffic_group)::int as traffic, count(p.domain_created_on)::int as age,count(p.platform)::int as platform,count(p.publisher_name)::int as publisher, - count(p.last_checked_at)::int as checked, + count(p.last_checked_at)::int as checked,count(p.domain_rating)::int as rated, 0::int as manager, coalesce(array_agg(distinct p.niche) filter(where p.niche is not null),'{}') as niches from sites p`,[network,niche]), queryAdNetworkReport(`${scopedCoverageSites} select distinct n.slug,n.name from sites p @@ -275,8 +281,8 @@ export async function loadAdNetworkSitesAvailability(network: string | null, nic const coverageWhere = `exists(select 1 from public.ad_network_publisher_relationships r join public.ad_networks n on n.id=r.ad_network_id where r.publisher_id=p.id and r.status in ('confirmed','stale') and ($1::text is null or n.slug=$1)) and ($2::text is null or p.niche=$2)`; -const scopedCoverageSites = `with sites as materialized (select p.id,p.niche,p.traffic_group,p.domain_created_on,p.platform,p.publisher_name,p.last_successful_check_at as last_checked_at - from public.ad_network_publishers p where ${coverageWhere})`; +const scopedCoverageSites = `with sites as materialized (select p.id,p.niche,p.traffic_group,p.domain_created_on,p.platform,p.publisher_name,p.last_successful_check_at as last_checked_at, + p.domain_rating from public.ad_network_publishers p where ${coverageWhere})`; const scopedManagerSites = `with sites as materialized (select p.id, jsonb_build_object('manager_domains',p.report_manager_domains) as last_check_evidence from public.ad_network_publishers p @@ -289,17 +295,21 @@ const siteSort: Record = { oldest: "p.domain_created_on asc nulls last,p.normalized_domain", newest: "p.domain_created_on desc nulls last,p.normalized_domain", checked: "p.last_successful_check_at desc nulls last,p.normalized_domain", + // Unrated sites sort last both ways. + "highest-dr": "p.domain_rating desc nulls last,p.normalized_domain", + "lowest-dr": "p.domain_rating asc nulls last,p.normalized_domain", }; /** Uncached source loader for signed origin verification. */ export async function loadAdNetworkSites(query: AdNetworkSitesQuery): Promise { // Apply bounds here too: server page callers do not go through the API parser. if (!Object.hasOwn(siteSort, query.sort) || !Number.isInteger(query.pageSize) || query.pageSize < 1 || query.pageSize > 100 - || !Number.isInteger(query.page) || query.page < 1 || query.page > 10000 || query.q.length > 100) throw new Error("Invalid sites query"); + || !Number.isInteger(query.page) || query.page < 1 || query.page > 10000 || query.q.length > 100 + || (query.minDr !== null && !AD_NETWORK_DR_FILTER_STEPS.includes(query.minDr))) throw new Error("Invalid sites query"); const availability = await loadAdNetworkSitesAvailability(query.network,query.niche); // data-gate: unavailable traffic/age sorts fall back to domain until enrichment lands. const effectiveSort = availability.availableSorts.includes(query.sort) ? query.sort : "domain"; - const values = [query.network, query.niche, query.q.trim().toLowerCase()]; - const count = await queryAdNetworkReport<{total:number}>(`select count(*)::int as total from public.ad_network_publishers p where ${siteWhere}`, values); + const values = [query.network, query.niche, query.q.trim().toLowerCase(), query.minDr]; + const count = await queryAdNetworkReport<{total:number}>(`select count(*)::int as total from public.ad_network_publishers p where ${siteListWhere}`, values); const total = count.rows[0].total; const totalPages = Math.max(1, Math.ceil(total / query.pageSize)); const page = Math.min(query.page, totalPages); @@ -307,19 +317,19 @@ export async function loadAdNetworkSites(query: AdNetworkSitesQuery): Promise(`with matches as materialized ( - select p.id,p.normalized_domain,p.traffic_group,p.domain_created_on,p.last_successful_check_at - from public.ad_network_publishers p where ${siteWhere} + select p.id,p.normalized_domain,p.traffic_group,p.domain_created_on,p.last_successful_check_at,p.domain_rating + from public.ad_network_publishers p where ${siteListWhere} ), selected as materialized ( - select p.id from matches p order by ${siteSort[effectiveSort]} limit $4 offset $5 + select p.id from matches p order by ${siteSort[effectiveSort]} limit $5 offset $6 ) select ${siteProjection} from selected s join public.ad_network_publishers p on p.id=s.id order by ${siteSort[effectiveSort]}`, [...values, query.pageSize, (page - 1) * query.pageSize]), queryNetworkIdentities(), ]); - return { rows: result.rows.map(row => toSite(row, identities)), total, page, pageSize: query.pageSize, totalPages, effectiveSort, ...availability }; + return { ...ahrefsAttribution(result.rows.map(row => row.domain_rating)), rows: result.rows.map(row => toSite(row, identities)), total, page, pageSize: query.pageSize, totalPages, effectiveSort, ...availability }; } export const getAdNetworkSites=cached("ad-network-sites-traffic-age",loadAdNetworkSites); export function defaultAdNetworkSitesQuery(network: string | null = null, niche: AdNetworkNicheSlug | null = null, pageSize = AD_NETWORK_SITES_PAGE_SIZE): AdNetworkSitesQuery { - return { network, niche, q: "", sort: network || niche ? "traffic" : "domain", page: 1, pageSize }; + return { network, niche, q: "", sort: network || niche ? "traffic" : "domain", page: 1, pageSize, minDr: null }; } const scopedSites = `with sites as materialized (select p.id,p.normalized_domain,p.niche,p.publisher_name,p.traffic_group,p.platform,p.domain_created_on @@ -445,6 +455,28 @@ export async function loadAdNetworkDomainAgeByNetwork(niche: string | null=null) } export const getAdNetworkDomainAgeByNetwork=cached("ad-network-domain-age-by-network",loadAdNetworkDomainAgeByNetwork); +type DomainRatingRecord = AdNetworkDomainRatingSummary & { bands: { min: number; count: number }[] }; + +/** Every network's Domain Rating over its listed sites, with 10-point band counts (70 and up in one). + * Unrated sites count only in `listed`; median and average use rated sites only. */ +export async function loadAdNetworkDomainRatings(): Promise { + return (await queryAdNetworkReport(`with records as materialized ( + select n.id,n.slug,n.name,p.domain_rating from public.ad_network_publisher_relationships r + join public.ad_networks n on n.id=r.ad_network_id join public.ad_network_publishers p on p.id=r.publisher_id + where r.status in ('confirmed','stale') + ), bands as ( + select id,least(domain_rating/10,7)*10 as min,count(*)::int as count + from records where domain_rating is not null group by 1,2 + ) select r.slug,r.name,count(*)::int as listed,count(r.domain_rating)::int as rated, + coalesce(percentile_cont(0.5) within group(order by r.domain_rating),0)::float8 as median, + coalesce(avg(r.domain_rating),0)::float8 as average, + coalesce((select jsonb_agg(jsonb_build_object('min',b.min,'count',b.count) order by b.min) + from bands b where b.id=r.id),'[]'::jsonb) as bands + from records r group by r.id,r.slug,r.name order by r.slug`)).rows.map(row=>({...row,...ahrefsAttribution(row.rated ? [row.average] : [])})); +} +const getAdNetworkDomainRatings=cached("ad-network-domain-ratings",loadAdNetworkDomainRatings); +const domainRatingSummaries=(rows: DomainRatingRecord[]): AdNetworkDomainRatingSummary[] => rows.map(({bands:_,...row})=>row); + type Breakdowns = Pick; const loadBreakdowns = cached("ad-network-breakdowns", async (network: string): Promise => { const pool = {query:queryAdNetworkReport}; @@ -495,12 +527,14 @@ export async function getAdNetworkPageData(slug: string): Promiserow.slug===slug)?.bands??[]); // data-gate: a 90-day-old snapshot establishes the observation span for real gains/losses. const older = history.filter(row=>row.date<=adNetworkUtcWindow(90,observedDay).baseline).at(-1); - return { network,sites,stats:{ domain_age_strip,switch_flow:adNetworkSwitchFlow(ref(network),switches), + return { network,sites,stats:{ domain_rating,domain_age_strip,switch_flow:adNetworkSwitchFlow(ref(network),switches), movement_periods:adNetworkMovementPeriods(movements,observedDay),confirmed_share:network.listed_count ? share(network.verified_count,network.listed_count) : null, change_year:adNetworkYearChange(history,network.listed_count,observedDay),networks_tracked:ranking.length,...metadata,listed:network.listed_count,confirmed:network.verified_count, rank:network.listed_count ? ranking.findIndex(row=>row.slug===slug)+1 : null, @@ -728,6 +762,10 @@ export const getAdNetworkStatsPageData = cached("ad-network-market-report", asyn count(*) filter(where last_checked_at is null)::int as waiting from public.ad_network_publishers`), getAdNetworkDomainAgeStrip(),getAdNetworkDomainAgeByNetwork(), ]); + // data-gate: only networks with enough rated sites enter the DR comparison. + const domain_rating_by_network=domainRatingSummaries(await getAdNetworkDomainRatings()) + .filter(row=>hasAdNetworkDomainRatingCoverage(row.rated,row.listed)) + .sort((a,b)=>b.average-a.average||a.slug.localeCompare(b.slug)); // Cache each network's history separately so the aggregate never approaches the 2 MB limit. const history=await Promise.all(market_share.slice(0,5).map(async n=>({...ref(n),history:adNetworkMonthEnds(await loadHistory(n.slug))}))); const net_change:AdNetworkStatsPageData["net_change"]=[]; @@ -746,7 +784,7 @@ export const getAdNetworkStatsPageData = cached("ad-network-market-report", asyn const ads_txt_metrics=managerStats.measured ? {manager_share:share(managerStats.with_manager,managerStats.measured), broken_share:share(ads.rows[0].with_broken_lines,managerStats.measured),median_lines:ads.rows[0].median_lines} : null; const queueCount=queue.rows[0]; - return {domain_age_strip,domain_age_by_network,confirmed_share:niches.listed ? share(niches.confirmed,niches.listed) : null, + return {...ahrefsAttribution(domain_rating_by_network.map(row=>row.average)),domain_rating_by_network,domain_age_strip,domain_age_by_network,confirmed_share:niches.listed ? share(niches.confirmed,niches.listed) : null, multiple_networks_share:niches.listed ? share(multiple,niches.listed) : null,...metadata, joining_rules:joiningRules(market_share.filter(n=>n.count>0)).map(n=>({...n,sites:market_share.find(row=>row.slug===n.slug)!.count})), check_queue:queueCount.total ? {...queueCount,checked_share:share(queueCount.checked,queueCount.total)} : null,ads_txt_metrics, diff --git a/lib/ahrefs.ts b/lib/ahrefs.ts new file mode 100644 index 00000000..e4ea5401 --- /dev/null +++ b/lib/ahrefs.ts @@ -0,0 +1,11 @@ +/** Required beside every published Ahrefs Domain Rating value or summary. */ +export const AHREFS_ATTRIBUTION = { + text: "Domain Rating by Ahrefs", + url: "https://ahrefs.com", +} as const; + +export type AhrefsAttribution = { attribution?: typeof AHREFS_ATTRIBUTION }; + +export function ahrefsAttribution(ratings: readonly (number | null | undefined)[]): AhrefsAttribution { + return ratings.some(value => typeof value === "number") ? { attribution: AHREFS_ATTRIBUTION } : {}; +} diff --git a/lib/data-api/ad-networks/core.ts b/lib/data-api/ad-networks/core.ts index c590c304..80dd97c0 100644 --- a/lib/data-api/ad-networks/core.ts +++ b/lib/data-api/ad-networks/core.ts @@ -1,4 +1,5 @@ import { brandKit } from '../brand.ts'; +import { AHREFS_ATTRIBUTION } from "../../ahrefs.ts"; import { createHash } from 'node:crypto'; import { domainToASCII } from 'node:url'; import { adNetworkFacts } from '../../../app/(site)/(directories)/ad-networks/network-facts.ts'; @@ -13,7 +14,8 @@ import { validateReadonlySql } from './sql.ts'; const listed = "r.status in ('confirmed','stale')"; export const ageYears = 'extract(year from age(current_date,p.domain_created_on))'; const ageBucket = `case when p.domain_created_on is null then null when ${ageYears}<3 then '0–2 years' when ${ageYears}<6 then '3–5 years' when ${ageYears}<11 then '6–10 years' when ${ageYears}<16 then '11–15 years' else '16+ years' end`; -const dimensionSql = { network:'n.slug',niche:'p.niche',registration_year:'extract(year from p.domain_created_on)::int',age_bucket:ageBucket,size_group:'p.traffic_group',platform:'p.platform' }; +export const domainRatingBucket = `case when p.domain_rating is null then 'unknown' when p.domain_rating<10 then '0–9' when p.domain_rating<20 then '10–19' when p.domain_rating<30 then '20–29' when p.domain_rating<50 then '30–49' when p.domain_rating<70 then '50–69' else '70–100' end`; +const dimensionSql = { network:'n.slug',niche:'p.niche',registration_year:'extract(year from p.domain_created_on)::int',age_bucket:ageBucket,size_group:'p.traffic_group',platform:'p.platform',domain_rating:domainRatingBucket }; export function filterSql(f: Partial) { const values: unknown[] = []; const add = (v: unknown) => { values.push(v); return `$${values.length}`; }; @@ -28,6 +30,8 @@ export function filterSql(f: Partial) { if (f.registered_before) conditions.push(`p.domain_created_on<${add(f.registered_before)}::date`); if (f.min_age !== undefined) conditions.push(`${ageYears}>=${add(f.min_age)}`); if (f.max_age !== undefined) conditions.push(`${ageYears}<=${add(f.max_age)}`); + if (f.min_domain_rating !== undefined) conditions.push(`p.domain_rating>=${add(f.min_domain_rating)}`); + if (f.max_domain_rating !== undefined) conditions.push(`p.domain_rating<=${add(f.max_domain_rating)}`); return { text: conditions.join(' and '), values }; } const joins = 'from ad_network_publishers p join ad_network_publisher_relationships r on r.publisher_id=p.id join ad_networks n on n.id=r.ad_network_id'; @@ -44,6 +48,7 @@ export function validateInput(name: ToolName, input: unknown): Record; if ((f.registered_after && f.registered_before && f.registered_after>=f.registered_before) || (f.min_age !== undefined && f.max_age !== undefined && f.min_age>f.max_age) + || (f.min_domain_rating !== undefined && f.max_domain_rating !== undefined && f.min_domain_rating>f.max_domain_rating) || (f.networks && new Set(f.networks).size!==f.networks.length) || (f.group_by && new Set(f.group_by).size!==f.group_by.length) || (f.from && (f.from>f.to || Date.parse(f.to)-Date.parse(f.from)>366*86400000))) throw new ApiError(400,'invalid_input','Invalid range or duplicate dimension/network.'); @@ -53,23 +58,27 @@ export function validateInput(name: ToolName, input: unknown): Record, dimensions: (keyof typeof dimensionSql)[], cap: number) { const w=filterSql(f); - const base=(await db.query(`select count(distinct p.id)::int as total ${joins} where ${w.text}`,w.values))[0].total as number; - if (!dimensions.length) return { total:base,base:'distinct sites matching all filters; networks are OR',rows:[],truncated:false }; + const counts=(await db.query(`select count(distinct p.id)::int as total, + count(distinct p.id) filter(where p.domain_rating is not null)::int as domain_rating_base ${joins} where ${w.text}`,w.values))[0]; + const base=counts.total as number; + const domain_rating_coverage={sites_with_rating:counts.domain_rating_base,total_sites:base,scope:'distinct sites matching all filters'}; + if (!dimensions.length) return { total:base,domain_rating_coverage,base:'distinct sites matching all filters; networks are OR',rows:[],truncated:false }; const selections=dimensions.map((d,i)=>`${dimensionSql[d]} as group_${i+1}`); const rows=await db.query(`select ${selections.join(',')},count(distinct p.id)::int as count ${joins} where ${w.text} group by ${dimensions.map((_,i)=>i+1).join(',')} order by count desc,${dimensions.map((_,i)=>`${i+1} nulls last`).join(',')} limit ${cap+1}`,w.values); const truncated=rows.length>cap; - const networkBases=dimensions.includes('network') ? await db.query(`select n.slug,count(distinct p.id)::int as total ${joins} where ${w.text} group by n.slug`,w.values) : []; - return { total:base,base:'distinct filtered sites; network shares use their own filtered base; null groups include unknowns',dimensions, + const networkBases=dimensions.includes('network') ? await db.query(`select n.slug,count(distinct p.id)::int as total,count(distinct p.id) filter(where p.domain_rating is not null)::int as domain_rating_base ${joins} where ${w.text} group by n.slug`,w.values) : []; + return { total:base,domain_rating_coverage,base:'distinct filtered sites; network shares use their own filtered base; null groups include unknowns',dimensions, network_bases:dimensions.includes('network')?Object.fromEntries((f.networks??networkBases.map(n=>n.slug)).map(slug=>[slug,networkBases.find(n=>n.slug===slug)?.total??0])):undefined, + network_domain_rating_coverage:dimensions.includes('network')?Object.fromEntries((f.networks??networkBases.map(n=>n.slug)).map(slug=>[slug,{sites_with_rating:networkBases.find(n=>n.slug===slug)?.domain_rating_base??0,total_sites:networkBases.find(n=>n.slug===slug)?.total??0}])):undefined, rows:rows.slice(0,cap).map(row=>{ const networkIndex=dimensions.indexOf('network'); const denominator=networkIndex<0 ? base : networkBases.find(n=>n.slug===row[`group_${networkIndex+1}`])?.total ?? 0; return {...Object.fromEntries(dimensions.map((d,i)=>[d,row[`group_${i+1}`]])),count:row.count,base:denominator,share_percent:denominator ? row.count/denominator*100 : 0}; }),truncated }; } -type NetworkRow = {slug:string;name:string;official_url:string|null;summary:string|null;listed:number;managed:number;confirmed:number;stale:number;median_domain_age_years:number|null;domain_age_base:number}; +type NetworkRow = {slug:string;name:string;official_url:string|null;summary:string|null;listed:number;managed:number;confirmed:number;stale:number;median_domain_age_years:number|null;domain_age_base:number;median_domain_rating:number|null;average_domain_rating:number|null;domain_rating_base:number}; async function networks(db: Reader) { return (await db.query(`select n.slug,n.name,n.official_url,n.summary, count(r.publisher_id) filter(where ${listed})::int as listed, @@ -78,13 +87,16 @@ async function networks(db: Reader) { count(r.publisher_id) filter(where r.status='stale')::int as stale, percentile_cont(0.5) within group(order by extract(epoch from age(current_date,p.domain_created_on))/31557600) filter(where ${listed} and p.domain_created_on is not null)::float8 as median_domain_age_years, - count(r.publisher_id) filter(where ${listed} and p.domain_created_on is not null)::int as domain_age_base + count(r.publisher_id) filter(where ${listed} and p.domain_created_on is not null)::int as domain_age_base, + percentile_cont(0.5) within group(order by p.domain_rating) filter(where ${listed})::float8 as median_domain_rating, + avg(p.domain_rating) filter(where ${listed})::float8 as average_domain_rating, + count(r.publisher_id) filter(where ${listed} and p.domain_rating is not null)::int as domain_rating_base from ad_networks n left join ad_network_publisher_relationships r on r.ad_network_id=n.id left join ad_network_publishers p on p.id=r.publisher_id group by n.id,n.slug,n.name,n.official_url,n.summary order by n.slug limit 201`)).map(n=>({ ...n,facts:adNetworkFacts[n.slug]??null,url:`${sourceUrl}/${n.slug}`, })); } -const siteFields = 'p.normalized_domain as domain,p.niche,p.platform,p.domain_created_on::text,p.traffic_group as size_group,p.last_successful_check_at'; +const siteFields = 'p.normalized_domain as domain,p.niche,p.platform,p.domain_created_on::text,p.traffic_group as size_group,p.last_successful_check_at,p.domain_rating,p.domain_rating_checked_on::text'; const membership = `coalesce((select jsonb_agg(jsonb_build_object('network',n.slug,'status',r.status,'relationship_type',r.relationship_type, 'first_confirmed_at',r.first_confirmed_at,'last_confirmed_at',r.last_confirmed_at,'ended_at',r.ended_at) order by n.slug) from ad_network_publisher_relationships r join ad_networks n on n.id=r.ad_network_id @@ -142,7 +154,7 @@ export async function queryTool(db: Reader,name: ToolName,f: Record const stats=(await db.query('select * from dataset_summary'))[0]; const totals=(await db.query(`select count(distinct publisher_id)::int as listed,count(distinct publisher_id) filter(where relationship_type='primary_or_exclusive_manager')::int as managed from ad_network_publisher_relationships r where ${listed}`))[0]; return { ...stats,...totals,measurement:'We check the publisher’s ads.txt for seller signals and MANAGERDOMAIN. AdSense and page-signature networks also require homepage signals. Discovery lists alone do not confirm a relationship.', - available_values:{niche:adNetworkNiches.map(n=>({slug:n.slug,name:n.name})),size_group:['top-10k','top-100k','top-1m','not-ranked'], + available_values:{niche:adNetworkNiches.map(n=>({slug:n.slug,name:n.name})),domain_rating:['0–9','10–19','20–29','30–49','50–69','70–100','unknown'],size_group:['top-10k','top-100k','top-1m','not-ranked'], platform:(await db.query(`select distinct platform from ad_network_publishers where platform is not null order by platform limit 200`)).map(r=>r.platform), networks:(await db.query(`select slug,name from ad_networks order by slug limit 200`))}, tools:{ad_networks_overview:'Start here for definitions, coverage and values.',list_ad_networks:'Joining and payment facts and network totals.',get_ad_network:'Full network profile and distributions.',compare_ad_networks:'One dimension side by side.',count_sites:'Distinct counts with filters and up to two groups.',find_sites:'Bounded site list with cursor.',lookup_site:'One domain with current and past memberships.',network_movements:'Observed events with a strong quality warning.',...(sqlEnabled()?{run_readonly_sql:'Private SQL for documented safe views only.'}:{})}}; @@ -158,7 +170,7 @@ export async function queryTool(db: Reader,name: ToolName,f: Record case 'get_ad_network': { const n=(await networks(db)).find(n=>n.slug===f.network);if(!n)throw new ApiError(404,'not_found','Unknown network slug.'); const distributions:Record={}; - for(const dim of ['age_bucket','registration_year','niche','size_group','platform'] as const) distributions[dim]=await grouped(db,{networks:[f.network]},[dim],cap); + for(const dim of ['age_bucket','registration_year','niche','size_group','platform','domain_rating'] as const) distributions[dim]=await grouped(db,{networks:[f.network]},[dim],cap); const co=await db.query(`select n.slug,count(distinct r.publisher_id)::int as count from ad_network_publisher_relationships r join ad_networks n on n.id=r.ad_network_id where ${listed} and n.slug<>$1 and r.publisher_id in (select r.publisher_id from ad_network_publisher_relationships r join ad_networks n on n.id=r.ad_network_id where n.slug=$1 and ${listed}) @@ -213,7 +225,9 @@ export async function executeTool(name: ToolName,input: unknown,transaction?: ty const budget=boundResults(data,cap); const result={as_of:freshness.as_of,queried_at:new Date().toISOString(),definitions,caveats,filters:f, url:name==='get_ad_network'?`${sourceUrl}/${f.network}`:name==='lookup_site'?`https://stack127.com/ad-network-checker?domain=${encodeURIComponent(f.domain)}`:name==='network_movements'?'https://stack127.com/ad-networks/stats':sourceUrl, - data,limits:{max_rows:cap,...budget,truncated:JSON.stringify(data).includes('"truncated":true')}}; + data:{...data,domain_rating_metadata:{attribution:AHREFS_ATTRIBUTION.text,url:AHREFS_ATTRIBUTION.url, + credit:`[${AHREFS_ATTRIBUTION.text}](${AHREFS_ATTRIBUTION.url})`,caveat:definitions.domain_rating, + coverage:{sites_with_rating:freshness.domain_rating_base,total_sites:freshness.domain_rating_total,scope:'all distinct listed sites in the dataset; not the returned page or filtered cohort'}}},limits:{max_rows:cap,...budget,truncated:JSON.stringify(data).includes('"truncated":true')}}; if(Buffer.byteLength(JSON.stringify(result))>MAX_ANSWER_BYTES)throw new ApiError(413,'answer_too_large','Answer exceeds 1 MB. Narrow the query or lower the row limit.'); return result; }); diff --git a/lib/data-api/ad-networks/definitions.ts b/lib/data-api/ad-networks/definitions.ts index 1d4e4047..e642b470 100644 --- a/lib/data-api/ad-networks/definitions.ts +++ b/lib/data-api/ad-networks/definitions.ts @@ -1,6 +1,7 @@ export const definitions = { listed: "A site is listed when its relationship status is confirmed or stale. A site may be listed for several networks.", managed: "A managed site has a listed relationship with relationship_type = primary_or_exclusive_manager. Its own ads.txt names that network as manager; this does not prove exclusivity.", + domain_rating: 'Domain Rating by Ahrefs (https://ahrefs.com) is a third-party estimate of backlink strength, rounded to a whole number from dr3; it is not traffic, revenue, quality or a recommendation. Unknown is null. Summaries use measured listed sites only; check coverage before comparing networks.', domain_age: "RDAP domain_created_on, not the site's launch date. Age buckets use completed years as of the UTC query date; medians use the public page's age()/31557600 definition. Unknown dates stay unknown.", top_million: "Chrome UX Report traffic_group in (top-10k, top-100k, top-1m). These groups are exclusive. The imported CrUX month is not stored in the serving tables and is unknown here.", not_ranked: "Not ranked means absent from the imported CrUX list, not small or zero traffic.", @@ -14,6 +15,7 @@ export const caveats = [ 'ads.txt lists authorized sellers; it does not prove revenue, exclusivity, or an active contract. AdSense and page-signature networks also use homepage checks.', 'Stale means retained after an inconclusive check, not freshly confirmed. Last successful check is not a freshness guarantee for every field.', 'Unknown values remain null. Monthly traffic units and alternative joining rules are not interchangeable; read each network’s cited rules.', + 'Display the linked credit [Domain Rating by Ahrefs](https://ahrefs.com) beside every DR value or summary, with its coverage and third-party estimate caveat.', movementWarning, ]; export const sourceUrl = 'https://stack127.com/ad-networks'; diff --git a/lib/data-api/ad-networks/schemas.ts b/lib/data-api/ad-networks/schemas.ts index a35d6aab..2d36534b 100644 --- a/lib/data-api/ad-networks/schemas.ts +++ b/lib/data-api/ad-networks/schemas.ts @@ -1,13 +1,15 @@ import { z } from 'zod'; const slug = z.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(80); const date = z.string().regex(/^\d{4}-\d{2}-\d{2}$/).refine(s => !Number.isNaN(Date.parse(s)) && new Date(s).toISOString().slice(0,10) === s, 'Use a real YYYY-MM-DD date'); -export const dimensions = ['network','niche','registration_year','age_bucket','size_group','platform'] as const; +export const dimensions = ['network','niche','registration_year','age_bucket','size_group','platform','domain_rating'] as const; export const filtersShape = { networks: z.array(slug).min(1).max(44).optional().describe('Network slugs, matched with OR. Example: ["mediavine","ezoic"]. Omit for all.'), relationship: z.enum(['managed','listed']).default('listed').describe('managed = listed primary_or_exclusive_manager; listed = confirmed or stale.'), niche: z.string().max(80).optional().describe('One stack127 niche slug; see overview. Example: food. Unknown values match no sites.'), registered_after: date.optional().describe('Exclusive RDAP registration lower bound. Example: 2020-01-01.'), registered_before: date.optional().describe('Exclusive RDAP registration upper bound. Example: 2026-01-01.'), + min_domain_rating: z.number().int().min(0).max(100).optional().describe('Minimum Ahrefs Domain Rating, inclusive. Unknown ratings are excluded. Example: 30.'), + max_domain_rating: z.number().int().min(0).max(100).optional().describe('Maximum Ahrefs Domain Rating, inclusive. Unknown ratings are excluded. Example: 69.'), min_age: z.number().min(0).max(100).optional().describe('Minimum completed years since RDAP registration, inclusive. Example: 3.'), max_age: z.number().min(0).max(100).optional().describe('Maximum completed years since RDAP registration, inclusive. Example: 10.'), size_group: z.enum(['top-10k','top-100k','top-1m','not-ranked']).optional().describe('CrUX size group. not-ranked does not mean small. Example: top-1m.'), @@ -26,7 +28,7 @@ export const toolSchemas = { get_ad_network: z.object({ network: slug.describe('Network slug from overview. Example: mediavine.') }).strict(), compare_ad_networks: z.object({ ...filtersShape, networks: z.array(slug).min(2).max(6).describe('2–6 distinct network slugs. Example: ["mediavine","ezoic"].'), - dimension: z.enum(['niche','registration_year','age_bucket','size_group','platform']).describe('One comparison dimension. Example: niche.'), + dimension: z.enum(['niche','registration_year','age_bucket','size_group','platform','domain_rating']).describe('One comparison dimension. Example: niche.'), }).strict(), count_sites: z.object({ ...filtersShape, group_by: z.array(z.enum(dimensions)).max(2).default([]).describe('Up to two distinct dimensions. Example: ["network","niche"]. Empty returns only the distinct-site total.'), diff --git a/lib/data-api/ad-networks/sql.ts b/lib/data-api/ad-networks/sql.ts index fcc2c0e9..ac1ae949 100644 --- a/lib/data-api/ad-networks/sql.ts +++ b/lib/data-api/ad-networks/sql.ts @@ -2,7 +2,7 @@ import { parse } from 'pgsql-ast-parser'; import { ApiError } from '../config.ts'; export const sqlSchema = `Only these read-only views exist in stack127_data: ad_networks(id,slug,name,official_url,summary); -ad_network_publishers(id,normalized_domain,niche,platform,domain_created_on,traffic_group,last_successful_check_at,ads_txt_summary); +ad_network_publishers(id,normalized_domain,niche,platform,domain_created_on,traffic_group,last_successful_check_at,ads_txt_summary,domain_rating,domain_rating_checked_on); ad_network_publisher_relationships(ad_network_id,publisher_id,status,relationship_type,first_confirmed_at,last_confirmed_at,ended_at); ad_network_relationship_events(ad_network_id,publisher_id,event,observed_at,relationship_type); ad_network_daily_counts(ad_network_id,snapshot_on,listed_count,confirmed_count,stale_count,captured_at). diff --git a/lib/data-api/mcp.ts b/lib/data-api/mcp.ts index 024dae40..a1678e09 100644 --- a/lib/data-api/mcp.ts +++ b/lib/data-api/mcp.ts @@ -10,12 +10,12 @@ import { sqlSchema } from './ad-networks/sql.ts'; const descriptions: Record = { get_brand_kit:'For charts, images, tables and shared visuals: call once per visual task for stack127 colors, fonts, inline SVG logos and credit rules. Not needed for data questions. Input {}.', ad_networks_overview:'Start here before researching stack127 ad network data. Returns coverage totals, freshness, all definitions/caveats, allowed filters and a tool guide. Counts observed sites, not the whole market.', - list_ad_networks:'Use to choose networks by joining rules or compare listed/managed counts and payment terms. Returns sourced public facts with checkedAt, median RDAP domain age and measurement bases. Traffic units and alternative joining rules are not comparable without reading the facts.', - get_ad_network:'Use for a full profile of one network slug. Returns sourced joining/payment facts, confirmed/stale/listed/managed counts, age/year/niche/size/platform distributions, co-listed networks and bounded newest-first daily snapshots. Missing days are unknown; snapshots are listed counts.', + list_ad_networks:'Use to choose networks by joining rules or compare listed/managed counts and payment terms. Returns sourced public facts with checkedAt, median RDAP domain age, Ahrefs Domain Rating median/average/coverage and measurement bases. Traffic units and alternative joining rules are not comparable without reading the facts.', + get_ad_network:'Use for a full profile of one network slug. Returns sourced joining/payment facts, confirmed/stale/listed/managed counts, age/year/niche/size/platform/Domain Rating distributions, co-listed networks and bounded newest-first daily snapshots. Missing days are unknown; snapshots are listed counts.', compare_ad_networks:'Use for 2–6 networks side by side on one dimension. Returns distinct-site counts, per-network filtered bases, and percentage shares including unknown groups. A site may belong to several networks. Bounded rows; check truncated.', count_sites:'Use for flexible distinct-site counts with filters and up to two grouping dimensions. Networks are OR. Unfiltered totals deduplicate multi-network sites; network groups overlap. Returns explicit bases, shares and null/unknown groups. Bounded groups; check truncated.', - find_sites:'Use for a bounded list of matching public domains and all their listed network memberships, types and confirmation times. Also returns niche/platform/RDAP date/CrUX size and next_cursor. At most 200 sites; reuse cursor with identical filters/sort. No publisher names.', - lookup_site:'Use to inspect one bare domain. Returns current and past observed network relationships, confirmation/end times, public ads.txt summary, niche/platform/RDAP age/CrUX size and bounded newest-first events. No live crawl. A missing domain is unknown, not network-free.', + find_sites:'Use for a bounded list of matching public domains and all their listed network memberships, types and confirmation times. Also returns niche/platform/RDAP date/CrUX size/Ahrefs Domain Rating and checked date and next_cursor. At most 200 sites; reuse cursor with identical filters/sort. No publisher names.', + lookup_site:'Use to inspect one bare domain. Returns current and past observed network relationships, confirmation/end times, public ads.txt summary, niche/platform/RDAP age/CrUX size/Ahrefs Domain Rating and checked date, plus bounded newest-first events. No live crawl. A missing domain is unknown, not network-free.', network_movements:'Use to inspect observed joins, departures and newly found sites by network/day over at most 366 days. Strong warning: history starts around 2026-10-19 and checker faults repaired 2026-11-13/14 caused false departures. Not reliable churn/growth. Missing days are unknown. Baseline imports excluded.', run_readonly_sql:'Private only. Use when the other tools cannot express a read-only research query. Returns capped rows, metadata and truncation, using a dedicated restricted database login and 15s maximum statement timeout. Never a write tool. '+sqlSchema, }; @@ -35,7 +35,7 @@ export async function handleMcp(request: Request,secret?: string): Promise", using the data answer’s as_of date and its more specific page URL when available. Use the brand colors and dark theme by default.'}); + {instructions:'Start with ad_networks_overview. Cite the returned stack127 URL. Carry definitions, as_of, filters and caveats into research. Listed sites may overlap between networks. Movements are not reliable growth/churn. Display the linked credit [Domain Rating by Ahrefs](https://ahrefs.com) beside any DR values or summaries, state DR is a third-party estimate, and include coverage. All tools read data; SQL is private and optional. When making a chart, image or table for sharing, call get_brand_kit first. Title the visual by what it shows and add "Source: stack127.com/ad-networks · data as of ", using the data answer’s as_of date and its more specific page URL when available. Use the brand colors and dark theme by default.'}); for(const name of toolNames.filter(n=>n!=='run_readonly_sql'||sqlEnabled())) { server.registerTool(name,{ title:name.split('_').map(w=>w[0].toUpperCase()+w.slice(1)).join(' '),description:descriptions[name], diff --git a/package.json b/package.json index 02a72969..84af12ff 100644 --- a/package.json +++ b/package.json @@ -93,6 +93,7 @@ "stack127:content:export": "node scripts/stack127/content/export-page-content.mjs", "ad-networks:ads-txt-summary": "node --experimental-strip-types scripts/ad-networks/ads-txt-summary.mjs", "ad-networks:platform": "node --experimental-strip-types scripts/ad-networks/platform.mjs", + "ad-networks:domain-rating": "node --experimental-strip-types scripts/ad-networks/domain-rating.mjs", "ad-networks:domain-age": "node --experimental-strip-types scripts/ad-networks/domain-age.mjs", "ad-networks:publisher-names": "node --experimental-strip-types scripts/ad-networks/publisher-names.mjs", "ad-networks:traffic": "node --experimental-strip-types scripts/ad-networks/traffic.mjs", diff --git a/scripts/ad-networks/domain-rating.mjs b/scripts/ad-networks/domain-rating.mjs new file mode 100644 index 00000000..22470c0b --- /dev/null +++ b/scripts/ad-networks/domain-rating.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node +import { pathToFileURL } from 'node:url'; +import { ahrefsRequester, ahrefsStateFile } from '../stack127/ahrefs-dr.mjs'; +import { enrichmentArgs, enrichmentDatabase, saveState, progressFile } from './enrichment-store.mjs'; + +// Failed domains have a separate retry checkpoint. Never advance the successful +// measurement date on failure, and never let one bad domain starve the queue. +export function domainRatingSites(db, { now = Date.now(), limit = 500 } = {}) { + const cutoff = new Date(now - 30 * 86400000).toISOString().slice(0, 10); + return db.prepare(`select p.id,p.normalized_domain,p.domain_rating,p.domain_rating_checked_on + from publishers p left join enrichment_state s on s.publisher_id=p.id and s.kind='domain-rating' + where (p.domain_rating_checked_on is null or p.domain_rating_checked_on 5000) throw new Error('limit must be 1 to 5000'); + if (!Number.isFinite(budgetMs) || budgetMs <= 0 || budgetMs > 45 * 60000) throw new Error('time budget must be at most 45 minutes'); + if (!Number.isFinite(paceMs) || paceMs < 3000 || paceMs > 60000) throw new Error('pace-ms must be 3000 to 60000'); + const controller = new AbortController(); + let signal; + const stop = received => { signal = received; controller.abort(); }; + const interrupt = () => stop('SIGINT'), terminate = () => stop('SIGTERM'); + const deadline = now() + budgetMs; + const budget = setTimeout(() => controller.abort(), budgetMs); + const summary = { kind: 'domain-rating', status: 'running', startedAt: new Date(now()).toISOString(), processed: 0, measured: 0, failed: 0, stopped: null }; + const report = async () => { if (options['progress-file']) await progressFile(options['progress-file'], summary); }; + process.on('SIGINT', interrupt); process.on('SIGTERM', terminate); + try { + const retryAt = db.prepare("select value from schema_meta where key='domain_rating_retry_at'").get()?.value; + if (retryAt && Date.parse(retryAt) > now()) { + Object.assign(summary, { status: 'failed', stopped: 'rate_limited', retryAt, total: 0, finishedAt: new Date(now()).toISOString() }); + await report(); return summary; + } + const request = dependencies.requestRating ?? ahrefsRequester({ apiKey: process.env.AHREFS_API_KEY, paceMs, signal: controller.signal, now, + stateFile: dependencies.stateFile, fetchImplementation: dependencies.fetchImplementation, sleepImplementation: dependencies.sleepImplementation, + onRateLimit: ({ retryAt }) => { + const until = new Date(retryAt).toISOString(); + db.prepare("insert into schema_meta(key,value) values('domain_rating_retry_at',?) on conflict(key) do update set value=excluded.value").run(until); + summary.retryAt = until; + } }); + const rows = domainRatingSites(db, { now: now(), limit }); + summary.total = rows.length; + await report(); + for (const site of rows) { + if (controller.signal.aborted || now() >= deadline) break; + let result; + try { result = await request(site.normalized_domain); } + catch (error) { if (controller.signal.aborted) break; throw error; } + if (result.stopped) { summary.stopped = result.stopped; break; } + db.exec('begin immediate'); + try { + if (Number.isInteger(result.value) && result.value >= 0 && result.value <= 100) { + const checkedOn = new Date(now()).toISOString().slice(0, 10); + db.prepare('update publishers set domain_rating=?,domain_rating_checked_on=? where id=?').run(result.value, checkedOn, site.id); + saveState(db, site.id, 'domain-rating', 'measured', { version: 'dr3', attribution: 'Domain Rating by Ahrefs', url: 'https://ahrefs.com' }); + summary.measured++; + } else { + saveState(db, site.id, 'domain-rating', 'lookup_failed', {}, 86400); + summary.failed++; + } + db.exec('commit'); + } catch (error) { db.exec('rollback'); throw error; } + summary.processed++; + if (summary.processed % 50 === 0) { await report(); console.log(JSON.stringify(summary)); } + } + summary.status = signal ? 'stopped' : summary.stopped || summary.failed ? 'failed' : controller.signal.aborted || now() >= deadline ? 'budget_complete' : 'complete'; + summary.finishedAt = new Date(now()).toISOString(); + await report(); + if (signal) summary.signal = signal; + return summary; + } finally { + clearTimeout(budget); process.off('SIGINT', interrupt); process.off('SIGTERM', terminate); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + let db; + try { + const args = enrichmentArgs({ 'ahrefs-state-file': { type: 'string', default: ahrefsStateFile('development') }, 'pace-ms': { type: 'string', default: '3000' }, + limit: { type: 'string', default: '500' }, 'time-budget-minutes': { type: 'string', default: '30' } }); + db = enrichmentDatabase(args); + const summary = await collectDomainRatings(db, args, { stateFile: args['ahrefs-state-file'] }); + console.log(JSON.stringify(summary)); + if (summary.signal) process.exitCode = summary.signal === 'SIGINT' ? 130 : 143; + else if (summary.status === 'failed') process.exitCode = 1; + } catch { console.error('Domain Rating collector failed; check key, network, arguments and SQLite schema.'); process.exitCode = 1; } + finally { db?.close(); } +} diff --git a/scripts/ad-networks/local-db.mjs b/scripts/ad-networks/local-db.mjs index 847e30a4..2abc1076 100644 --- a/scripts/ad-networks/local-db.mjs +++ b/scripts/ad-networks/local-db.mjs @@ -216,6 +216,8 @@ export function openLocalDatabase(path = defaultLocalDatabasePath, options = {}) for (const name of ['niche','publisher_name','platform','domain_created_on','traffic_group','ads_txt_summary','favicon']) { if (!columns.has(name)) database.exec(`alter table publishers add column ${name} text`); } + if (!columns.has('domain_rating')) database.exec('alter table publishers add column domain_rating integer check(domain_rating is null or (domain_rating between 0 and 100 and typeof(domain_rating)=\'integer\'))'); + if (!columns.has('domain_rating_checked_on')) database.exec('alter table publishers add column domain_rating_checked_on text'); database.exec(`create table if not exists enrichment_state ( publisher_id integer not null references publishers(id), kind text not null, status text not null, checked_at text not null, retry_at text, data text not null default '{}', diff --git a/scripts/ad-networks/ovh/README.md b/scripts/ad-networks/ovh/README.md new file mode 100644 index 00000000..63a1f17b --- /dev/null +++ b/scripts/ad-networks/ovh/README.md @@ -0,0 +1,25 @@ +# Ad-network jobs on OVH + +See [the production runbook](../../../content_docs/ad-networks/production-runbook.md) for the existing jobs, locks, backups, sync and recovery. These files describe installation; changing this checkout does not install or enable anything. + +## Domain Rating + +The proposed cron entry is `5 7-23 * * * .../run-job.sh domain-rating --limit 500 --time-budget-minutes 30` (UTC). It leaves 00:00–06:59 for the existing daily jobs and 05:00–06:59 release window. Collection ends by :35. A slow or contended sync can overrun into :40 and make hourly verification skip once. This occasional skip is acceptable: the next hourly run resumes due checks, and missed enrichment runs select due work again. Repeated skips need investigation; the existing lock/retry wrapper remains authoritative. Preserve current pauses and unrelated jobs when the lead adds this line to the installed crontab after review. + +The job needs **`AHREFS_API_KEY` in Infisical project `stack127`, folder `/vps/ad-networks`, environment `prod`** (`stack127 /vps/ad-networks`). This is the same API key used by launch directory metrics, copied through Infisical, not a new public/build variable. Cockpit `workflow/infisical.md` maps that folder to `/srv/stack127-data/ad-networks.env`. Homelab's existing sync writes that file. There is no ad-network `prepare-env` script. The lead adds the named key through `infisical-env`; never edit the OVH file or read-only Mac mirrors. Do not put it in Dokploy, GitHub, or `NEXT_PUBLIC_*` variables. The launch directory folder stays unchanged. + +The collector uses the [free Ahrefs endpoint](https://docs.ahrefs.com/en/api/reference/public/get-domain-rating-free), `dr3`. It makes one request at a time, at least three seconds after the previous response. `--pace-ms` can slow it down (3000–60000), never speed it up. All retries share that clock. HTTP 429 honors numeric or dated `Retry-After`, with 30/60 second exponential backoff. A third 429, a delay longer than 60 seconds, or HTTP 401/403 stops the run and reports failure. It does not try other domains after the stop. Both this job and the monthly launch-directory metrics job use `/srv/stack127-data/state/ahrefs.sqlite`. A separate process-held SQLite lock in `ahrefs.sqlite.lock` serializes requests across both jobs and releases automatically on process death. Before HTTP starts, the state file commits a reservation for now plus the configured pace. HTTP completion extends that reservation. A 429 commits its Retry-After cooldown before body cleanup or callbacks. These committed values survive a hard kill, including cooldowns longer than an hour. A crash before the cooldown commit can retain only the pre-request pacing reservation. Authentication failures stop the current requester; they are not saved as shared stop states. The job user needs write access to that directory. The ad-network SQLite checkpoint also retains its own retry time. A run before that time reports the retained rate limit without HTTP requests. No attempt bypasses the limit. + +Priority: never-checked sites with a confirmed/stale manager relationship, other confirmed/stale sites, other never-checked sites, then the oldest successful dates strictly older than 30 days. Pending candidates stay local until the existing sync admits their reviewed relationships. Failed lookups retain both the prior value and successful date; a separate 24-hour retry checkpoint prevents bad domains from starving the queue. Successful zero is a measurement. Successful values are rounded to integers, 0–100; unknown stays null. + +Seventeen daily runs can measure at most 8,500 domains a day. OVH has 213,003 candidates: 123,361 listed and 39,823 managed. Managed sites take about 5 days; full first coverage takes about 26 days without throttling or slow responses. These groups overlap, so do not add them together. Steady state picks up dates after 30 days, so a full refresh cycle takes roughly 30 days plus the queue delay. Recheck capacity if the source exceeds 255,000 domains or upstream throttling persists. + +Each successful measurement and its checkpoint commit in one SQLite transaction. SIGTERM/SIGINT and the time budget cancel sleeps/HTTP; completed measurements remain. A restart retries unfinished work without losing or duplicating measurements. The existing wrapper uses a process-held `flock`, `nice`, time-bounded children, sync-lock retries, and the standard `== start domain-rating ...` / `== end domain-rating exit ` log lines. Missed runs catch up by selecting due work at the next hour. The existing log rotation and automation manifest cover `logs/domain-rating.log`. Failure exits nonzero and the existing alert path handles it. + +After collection (including a failed partial run), `refresh.mjs` syncs all changed reviewed rows through the existing 500-row PostgreSQL batches, then calls the existing origin revalidation step once. There is no per-domain sync. SQLite candidates without a reviewed relationship are not sent to PostgreSQL. Sync hash checkpoints make retries harmless. The new migration excludes only DR and its checked date from the frontend source projection, so DR-only updates produce no cache event/source revision and no Cloudflare rebuild. Other enrichment or relationship changes in the same sync still queue their normal refresh. DR values are public on pages and APIs with linked Ahrefs credit. The daily snapshot at 00:10 UTC always inserts a dated row and queues a refresh, publishing new DR values daily even when the counts stay unchanged. Other public record changes can publish DR sooner. + +When rolling out this lock change, stop both Ahrefs jobs before updating their code so old and new lock paths cannot overlap. Keep the existing state file and use the same released helper for both jobs. Before enabling, the lead must release migration `20261114130000_ad_network_domain_rating.sql`, add the Infisical key, and update the released OVH checkout. Run a small bounded job through the wrapper and check its new start/end lines and progress JSON. Then add only the reviewed cron line, preserving verification's current pause. Confirm the dashboard lists `stack127-vps/domain-rating`. No server, cron or Infisical change was made during implementation. + +The local smoke check measured two managed development domains using a real locally available key in a temporary SQLite store. Repeating it processed zero domains. The SQL migration, DR-only cache behavior, groups, filters and summaries were tested inside a development transaction and rolled back. The installed wrapper and production permissions still need the lead's release check. + +Any DR answer must carry [Domain Rating by Ahrefs](https://ahrefs.com), say it is a third-party estimate, and show measured coverage. The [Ahrefs license](https://ahrefs.com/legal/domain-rating-license) applies. Private per-site results are for the owner. Future public plans expose network summaries only, never per-site DR values or DR dataset exports. diff --git a/scripts/ad-networks/ovh/crontab.txt b/scripts/ad-networks/ovh/crontab.txt index 33855e78..2859c955 100644 --- a/scripts/ad-networks/ovh/crontab.txt +++ b/scripts/ad-networks/ovh/crontab.txt @@ -9,3 +9,5 @@ 0 4 * * 3 /srv/stack127/scripts/ad-networks/ovh/run-job.sh publisher-names --limit 5000 --time-budget-minutes 30 0 4 5 * * /srv/stack127/scripts/ad-networks/ovh/run-job.sh traffic --limit 5000 --time-budget-minutes 30 0 5 * * * /srv/stack127/scripts/ad-networks/ovh/run-job.sh backup +# DR: bounded sequential work; leave 00:00-06:59 UTC for daily jobs and releases. +5 7-23 * * * /srv/stack127/scripts/ad-networks/ovh/run-job.sh domain-rating --limit 500 --time-budget-minutes 30 diff --git a/scripts/ad-networks/refresh.mjs b/scripts/ad-networks/refresh.mjs index 05da9b76..1848d7ae 100644 --- a/scripts/ad-networks/refresh.mjs +++ b/scripts/ad-networks/refresh.mjs @@ -5,12 +5,13 @@ import { homedir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { parseArgs } from 'node:util'; +import { ahrefsStateFile } from '../stack127/ahrefs-dr.mjs'; import { defaultLocalDatabasePath } from './local-db.mjs'; import { jobAlert, safeError } from './alerts.mjs'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); export const refreshJobs = ['sync', 'snapshot', 'verification', 'discovery', 'catalog', 'backup', - 'platform', 'publisher-names', 'traffic', 'domain-age', 'ads-txt-summary', 'niche-export', 'rdap-import']; + 'platform', 'publisher-names', 'traffic', 'domain-age', 'domain-rating', 'ads-txt-summary', 'niche-export', 'rdap-import']; export function refreshArgs(argv) { const { values } = parseArgs({ args: argv, options: { @@ -20,6 +21,7 @@ export function refreshArgs(argv) { 'state-directory': { type: 'string', default: join(homedir(), '.local/state/stack127/ad-networks') }, concurrency: { type: 'string', default: '4' }, limit: { type: 'string', default: '500' }, 'time-budget-minutes': { type: 'string', default: '10' }, 'source-sqlite': { type: 'string' }, + 'pace-ms': { type: 'string', default: '3000' }, 'sync-only': { type: 'boolean', default: false }, full: { type: 'boolean', default: false }, } }); if (!refreshJobs.includes(values.job)) throw new Error(`Choose --job ${refreshJobs.join('|')}`); @@ -28,6 +30,7 @@ export function refreshArgs(argv) { for (const [key, max] of [['concurrency', 8], ['limit', 5000], ['time-budget-minutes', 45]]) { if (!/^[1-9]\d*$/.test(values[key]) || Number(values[key]) > max) throw new Error(`${key} must be 1 to ${max}`); } + if (!/^\d+$/.test(values['pace-ms']) || +values['pace-ms'] < 3000 || +values['pace-ms'] > 60000) throw new Error('pace-ms must be 3000 to 60000'); if (values.job === 'rdap-import' && !values['source-sqlite']) throw new Error('rdap-import needs --source-sqlite'); return { ...values, 'local-database': resolve(values['local-database']), 'state-directory': resolve(values['state-directory']) }; } @@ -52,6 +55,7 @@ export function refreshCommand(options) { '--run-directory', join(state, 'niche/input'), '--progress-file', join(state, 'niche-export.json')] }; return { script: options.job, args: [...local, '--concurrency', options.concurrency, '--limit', options.limit, '--time-budget-minutes', options['time-budget-minutes'], '--progress-file', join(state, `${options.job}.json`), + ...(options.job === 'domain-rating' ? ['--pace-ms', options['pace-ms'], '--ahrefs-state-file', ahrefsStateFile(options.environment)] : []), ...(options.job === 'traffic' ? ['--cache-directory', join(state, 'crux')] : []), ...(options.job === 'publisher-names' ? ['--refresh'] : [])] }; } diff --git a/scripts/ad-networks/sync.mjs b/scripts/ad-networks/sync.mjs index 756d03fb..132eaa4b 100644 --- a/scripts/ad-networks/sync.mjs +++ b/scripts/ad-networks/sync.mjs @@ -14,7 +14,7 @@ const reviewed = new Set(['confirmed', 'stale', 'ended']); const reviewedWhere = (prefix = '') => `(${prefix}status in ('confirmed','stale') or (${prefix}status='ended' and ${prefix}first_confirmed_at is not null))`; const publisherFields = ['normalized_domain','display_domain','first_seen_at','last_discovered_at','last_checked_at', 'check_status','last_check_outcome','last_check_evidence','last_successful_check_at','discovery_sources']; -export const enrichmentFields = ['niche','publisher_name','platform','domain_created_on','traffic_group','ads_txt_summary','favicon']; +export const enrichmentFields = ['niche','publisher_name','platform','domain_created_on','domain_rating','domain_rating_checked_on','traffic_group','ads_txt_summary','favicon']; // Recurring model work on homelab owns existing target niches. SQLite can seed // a newly inserted publisher, but a full collector sync must never replace one. export const publisherUpdateFields = fields => fields.filter(field => !['normalized_domain','niche'].includes(field)); diff --git a/scripts/launch-directories/README.md b/scripts/launch-directories/README.md index 01bf1a3e..fb395180 100644 --- a/scripts/launch-directories/README.md +++ b/scripts/launch-directories/README.md @@ -51,7 +51,8 @@ Facts imports move row timestamps. Prepare a fresh correction file after importi - [Ahrefs free DR endpoint](https://docs.ahrefs.com/en/api/reference/public/get-domain-rating-free): authenticated GET, explicit `version=dr3`, sequential requests at least three seconds apart. Floats round to the existing whole-number DR column. HTTP 429 backs off for 30 then 60 seconds, honors longer `Retry-After` values, and stops DR lookups after two retries or a requested wait over 60 seconds. HTTP 401/403 stops DR lookups immediately. Tranco can still finish. Exit 2 reports a partial run; successes can be saved without clearing failed metrics. - [Tranco](https://tranco-list.eu/): resolve the latest standard list ID, then download its immutable top-million CSV. Validate all one million ordered rows, including Tranco's reserved wildcard placeholders, before treating an absent registrable domain as `below_1m`. Buckets match the reviewed facts importer. No extra ZIP dependency is needed. -- Fetches hold no database transaction or connection. A single short write statement updates changed metrics, guarded by exact row versions. Existing triggers move public dates, queue parent and child refreshes, and change source fingerprints. The Node cache worker starts the usual Cloudflare rebuild; the job never purges or deploys itself. +- Both Ahrefs jobs use `/srv/stack127-data/state/ahrefs.sqlite` for durable state and a separate `ahrefs.sqlite.lock` for the process-held request lock. Each request commits its pacing reservation before HTTP starts. A 429 commits its cooldown before body cleanup or callbacks. A hard kill releases the lock and retains committed state. A kill before the cooldown commit retains only the pacing reservation. Stop both jobs before rolling out this lock change, keep the state file, and update both to the same helper. Authentication stops apply only to the current requester. +- Fetches hold no PostgreSQL transaction or connection. A single short write statement updates changed metrics, guarded by exact row versions. Existing triggers move public dates, queue parent and child refreshes, and change source fingerprints. The Node cache worker starts the usual Cloudflare rebuild; the job never purges or deploys itself. - The [Ahrefs license](https://ahrefs.com/legal/domain-rating-license) requires the visible linked credit "Domain Rating by Ahrefs" beside displayed DR data. The pages use `ahrefs-credit.tsx` for it; keep it beside any new place that shows DR. ```sh diff --git a/scripts/launch-directories/refresh-metrics.mjs b/scripts/launch-directories/refresh-metrics.mjs index 82349f8c..cac48ed6 100644 --- a/scripts/launch-directories/refresh-metrics.mjs +++ b/scripts/launch-directories/refresh-metrics.mjs @@ -2,7 +2,8 @@ import { readFileSync } from "node:fs"; import { pathToFileURL } from "node:url"; import { parseEnv } from "node:util"; -import { setTimeout as sleep } from "node:timers/promises"; +import { ahrefsRequester, ahrefsStateFile } from "../stack127/ahrefs-dr.mjs"; +export { ahrefsRequester, parseDomainRating, retryDelay } from "../stack127/ahrefs-dr.mjs"; import pg from "pg"; import { getDomain } from "tldts"; import { factsArgs } from "./import-facts.mjs"; @@ -64,59 +65,9 @@ export async function latestTranco(wantedDomains, { fetchImplementation = fetch return { list_id: list.list_id, ranks: parseTrancoCsv(await boundedText(response, 40 * 1024 * 1024), wantedDomains) }; } -export function parseDomainRating(body) { - const value = body?.domain_rating?.domain_rating; - if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 100) throw new Error("Invalid Ahrefs Domain Rating response"); - // The existing schema and UI use whole-number DR. Ahrefs returns a float. - return Math.round(value); -} - -export function retryDelay(header, attempt, now = Date.now()) { - const seconds = header && /^\d+(?:\.\d+)?$/.test(header.trim()) ? Number(header) * 1000 : null; - const date = header && seconds === null ? Date.parse(header) - now : null; - return Math.max(30000 * 2 ** attempt, seconds ?? (Number.isFinite(date) ? date : 0)); -} - -/** One requester per run. All domains and retries share its pacing clock. */ -export function ahrefsRequester({ apiKey, fetchImplementation = fetch, sleepImplementation = sleep, - now = Date.now, paceMs = 3000, maxRetries = 2, maxBackoffMs = 60000 } = {}) { - if (!apiKey?.trim()) throw new Error("AHREFS_API_KEY is required"); - let nextRequestAt = 0; - let stopped = null; - return async domain => { - if (stopped) return { stopped }; - for (let attempt = 0; ; attempt++) { - const wait = Math.max(0, nextRequestAt - now()); - if (wait) await sleepImplementation(wait); - const url = new URL("https://api.ahrefs.com/v3/public/domain-rating-free"); - url.search = new URLSearchParams({ target: domain, version: "dr3", output: "json" }); - let response; - try { - response = await fetchImplementation(url, { headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json" }, signal: AbortSignal.timeout(30000), redirect: "error" }); - } catch { nextRequestAt = now() + paceMs; return { error: "Ahrefs request failed" }; } - nextRequestAt = now() + paceMs; - if (response.status === 429) { - const delay = retryDelay(response.headers.get("retry-after"), attempt, now()); - await response.body?.cancel().catch(() => {}); - if (attempt >= maxRetries || delay > maxBackoffMs) { stopped = "rate_limited"; return { stopped }; } - nextRequestAt = now() + Math.max(paceMs, delay); - continue; - } - if (response.status === 401 || response.status === 403) { - await response.body?.cancel().catch(() => {}); - stopped = "authentication_failed"; - return { stopped }; - } - if (!response.ok) { await response.body?.cancel().catch(() => {}); return { error: `Ahrefs HTTP ${response.status}` }; } - try { return { value: parseDomainRating(JSON.parse(await boundedText(response, 64000))) }; } - catch { return { error: "Invalid Ahrefs response" }; } - } - }; -} - const metricKeys = ["domain_rating", "domain_rating_checked_on", "traffic_bucket", "traffic_checked_on"]; export async function collectMetrics(records, { apiKey, checkedOn = new Date().toISOString().slice(0, 10), - loadTranco = latestTranco, requestRating = ahrefsRequester({ apiKey }) } = {}) { + loadTranco = latestTranco, stateFile, requestRating = ahrefsRequester({ apiKey, stateFile }) } = {}) { const domains = new Map(); let tranco; const failures = []; @@ -194,7 +145,7 @@ async function main() { where is_published order by slug limit $1`, [Number(args.limit ?? 2147483647)])).rows.map(row => row.record); } finally { await pool.end(); } // No database connection or transaction is held during external lookups. - const { patches, ...summary } = await collectMetrics(records, { apiKey }); + const { patches, ...summary } = await collectMetrics(records, { apiKey, stateFile: ahrefsStateFile(args.environment) }); let result = { changed: 0, conflicts: [] }; if (args.apply && patches.length) { const writer = new pg.Pool(poolOptions); diff --git a/scripts/stack127/ahrefs-dr.mjs b/scripts/stack127/ahrefs-dr.mjs new file mode 100644 index 00000000..cf5c9873 --- /dev/null +++ b/scripts/stack127/ahrefs-dr.mjs @@ -0,0 +1,134 @@ +import { setTimeout as sleep } from "node:timers/promises"; +import { DatabaseSync } from "node:sqlite"; +import { mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { homedir } from "node:os"; + +// Both OVH jobs run as the same user and must pass this same host-wide file. +export const AHREFS_STATE_FILE = "/srv/stack127-data/state/ahrefs.sqlite"; +export function ahrefsStateFile(environment) { + return environment === "production" ? AHREFS_STATE_FILE : join(homedir(), ".local/state/stack127/ahrefs.sqlite"); +} + +/** Hold the request lock in a separate SQLite file so state writes can commit + * immediately. Process death releases this lock without undoing committed state. + * Poll SQLITE_BUSY asynchronously so abort signals and the run budget still work. */ +async function withSharedState(file, signal, work) { + mkdirSync(dirname(file), { recursive: true, mode: 0o700 }); + const lock = new DatabaseSync(`${file}.lock`); + const deadline = Date.now() + 120000; + try { + for (;;) { + signal?.throwIfAborted(); + try { lock.exec("begin immediate"); break; } + catch (error) { + if (error.errcode !== 5 || Date.now() >= deadline) throw error; + await sleep(100, undefined, { signal }); + } + } + const db = new DatabaseSync(file); + try { + db.exec(`pragma synchronous=FULL; + create table if not exists request_state ( + id integer primary key check(id=1), next_request_at integer not null, cooldown_until integer not null)`); + const row = db.prepare("select * from request_state where id=1").get(); + const state = { nextRequestAt: row?.next_request_at ?? 0, cooldownUntil: row?.cooldown_until ?? 0 }; + const write = db.prepare(`insert into request_state values(1,?,?) on conflict(id) do update set + next_request_at=excluded.next_request_at,cooldown_until=excluded.cooldown_until`); + // No open state transaction: every run commits before control returns. + const persist = () => write.run(state.nextRequestAt, state.cooldownUntil); + return await work(state, persist); + } finally { db.close(); } + } finally { lock.close(); } +} + +async function boundedText(response, maxBytes) { + if (Number(response.headers.get("content-length")) > maxBytes) throw new Error("Response is too large"); + const chunks = []; + let bytes = 0; + for await (const chunk of response.body) { + bytes += chunk.length; + if (bytes > maxBytes) { await response.body.cancel().catch(() => {}); throw new Error("Response is too large"); } + chunks.push(chunk); + } + return Buffer.concat(chunks).toString("utf8"); +} + +export function parseDomainRating(body) { + const value = body?.domain_rating?.domain_rating; + if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 100) throw new Error("Invalid Ahrefs Domain Rating response"); + // The existing schema and UI use whole-number DR. Ahrefs returns a float. + return Math.round(value); +} + +export function retryDelay(header, attempt, now = Date.now()) { + const seconds = header && /^\d+(?:\.\d+)?$/.test(header.trim()) ? Number(header) * 1000 : null; + const date = header && seconds === null ? Date.parse(header) - now : null; + return Math.max(30000 * 2 ** attempt, seconds ?? (Number.isFinite(date) ? date : 0)); +} + +/** All domains, retries and OVH jobs share one pacing and Retry-After clock. */ +export function ahrefsRequester({ apiKey, fetchImplementation = fetch, sleepImplementation = sleep, + now = Date.now, paceMs = 3000, maxRetries = 2, maxBackoffMs = 60000, signal, onRateLimit, stateFile } = {}) { + if (!apiKey?.trim()) throw new Error("AHREFS_API_KEY is required"); + if (!Number.isFinite(paceMs) || paceMs < 3000) throw new Error("Ahrefs pacing must be at least 3000ms"); + const localState = { nextRequestAt: 0, cooldownUntil: 0 }; + const withState = work => stateFile ? withSharedState(stateFile, signal, work) : work(localState, () => {}); + let stopped = null; + return async domain => { + if (stopped) return { stopped }; + signal?.throwIfAborted(); + for (let attempt = 0; ; attempt++) { + const result = await withState(async (state, persist) => { + // Long cooldowns stop promptly instead of sleeping through a scheduled run. + if (state.cooldownUntil - now() > maxBackoffMs) { + onRateLimit?.({ retryAt: state.cooldownUntil }); + return { stopped: "rate_limited" }; + } + const wait = Math.max(0, state.nextRequestAt - now(), state.cooldownUntil - now()); + if (wait) await sleepImplementation(wait, undefined, { signal }); + signal?.throwIfAborted(); + const url = new URL("https://api.ahrefs.com/v3/public/domain-rating-free"); + url.search = new URLSearchParams({ target: domain, version: "dr3", output: "json" }); + // Reserve the next slot before HTTP starts, even if this process dies + // before receiving a response. Completion extends the same reservation. + state.nextRequestAt = now() + paceMs; + persist(); + let response; + try { + response = await fetchImplementation(url, { headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json" }, signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(30000)]) : AbortSignal.timeout(30000), redirect: "error" }); + } catch { + state.nextRequestAt = now() + paceMs; + persist(); + signal?.throwIfAborted(); + return { error: "Ahrefs request failed" }; + } + // Keep the lock until the body has been consumed/cancelled too. + try { + if (response.status === 429) { + const delay = retryDelay(response.headers.get("retry-after"), attempt, now()); + state.cooldownUntil = now() + Math.max(paceMs, delay); + // Commit before callbacks or body cleanup can throw, hang or die. + persist(); + await response.body?.cancel().catch(() => {}); + onRateLimit?.({ retryAt: state.cooldownUntil }); + return attempt >= maxRetries || delay > maxBackoffMs ? { stopped: "rate_limited" } : { retry: true }; + } + if (response.status === 401 || response.status === 403) { + await response.body?.cancel().catch(() => {}); + return { stopped: "authentication_failed" }; + } + if (!response.ok) { await response.body?.cancel().catch(() => {}); return { error: `Ahrefs HTTP ${response.status}` }; } + try { return { value: parseDomainRating(JSON.parse(await boundedText(response, 64000))) }; } + catch { signal?.throwIfAborted(); return { error: "Invalid Ahrefs response" }; } + } finally { + state.nextRequestAt = now() + paceMs; + persist(); + } + }); + if (result.retry) continue; + if (result.stopped) stopped = result.stopped; + return result; + } + }; +} diff --git a/stack127_docs/mcp-and-api.md b/stack127_docs/mcp-and-api.md index f18f08f5..34cd2555 100644 --- a/stack127_docs/mcp-and-api.md +++ b/stack127_docs/mcp-and-api.md @@ -20,14 +20,14 @@ For every row below, the REST route is **POST `/api/v1/ad-networks/`* | `ad_networks_overview` | Start here: sites checked, distinct listed/managed totals, freshness, measurement, definitions, caveats, allowed values, tool guide. Input `{}`. | | `list_ad_networks` | Counts, domain-age median and measured base, sourced joining/payment facts and checked dates. Filter published numeric minimum, traffic unit, or public numeric rule; sort by listed, managed, name or minimum traffic. | | `get_ad_network` | One `network` slug, facts/sources, counts, domain-age buckets/median, registration years, niche/size/platform mix, co-listing and daily listed snapshots. | -| `compare_ad_networks` | 2–6 distinct `networks`, one `dimension`: niche, registration_year, age_bucket, size_group or platform. Counts and shares use each network's filtered distinct-site base. | +| `compare_ad_networks` | 2–6 distinct `networks`, one `dimension`: niche, registration_year, age_bucket, size_group, platform or domain_rating. Counts and shares use each network's filtered distinct-site base. | | `count_sites` | Distinct-site counter; up to two `group_by` dimensions, including network. Global totals deduplicate sites listed at several networks. | | `find_sites` | Public domains and listed memberships, types, confirmation dates, niche, platform, RDAP date and CrUX size. `limit` 1–200, `sort` domain/newest/oldest, opaque `cursor`. Keep filters/sort unchanged between pages. | | `lookup_site` | Bare `domain`: current/past relationships, first/last confirmation and end dates, public ads.txt summary, classification, age, size and events. Unknown domain returns 404. | | `network_movements` | Optional `networks`, inclusive UTC `from` and `to`; at most 366 days apart. Observed joins/leaves/newly-found only, with a strong fault warning. | | `run_readonly_sql` | Disabled for this release. Future private SQL requires a dedicated database login and a separate switch. A single SELECT against the documented safe views. Never available to a public principal. | -`count_sites`, `find_sites` and `compare_ad_networks` share filters: `networks` (OR), `relationship` (`listed`, default, or `managed`), `niche`, `registered_after`/`registered_before` (exclusive RDAP dates), `min_age`/`max_age` (inclusive completed years), `size_group`, `platform`, and `status` (`confirmed` or `stale`). Other filters combine with AND. Missing classifications remain null groups. `find_sites` returns all listed networks for each matched site, even if only one network matched the filter. +`count_sites`, `find_sites` and `compare_ad_networks` share filters: `networks` (OR), `relationship` (`listed`, default, or `managed`), `niche`, `registered_after`/`registered_before` (exclusive RDAP dates), `min_age`/`max_age` (inclusive completed years), `size_group`, `platform`, `min_domain_rating`/`max_domain_rating` (inclusive integers 0–100; exclude unknowns), and `status` (`confirmed` or `stale`). Other filters combine with AND. Missing classifications remain null groups. `find_sites` returns all listed networks for each matched site, even if only one network matched the filter. Listed means confirmed or stale. Managed means a listed `primary_or_exclusive_manager` relationship. This is the requested relationship-based measure; it differs from the public page's manager resolver, which also reconciles retained manager-domain evidence and ambiguous signals. Do not call those two measures interchangeable. The API reuses the public listed-status rule, domain-age bucket thresholds/median expression, source facts and observed-event definitions. Minimum traffic in sessions and pageviews is not interchangeable; use `traffic_unit`. `has_public_rule` specifically means a published numeric monthly traffic minimum, including an explicit zero; read full joining facts for alternatives. @@ -43,6 +43,18 @@ Colors and font stacks come from `app/globals.css` and the ad network visualizat Connected clients must refresh their tool list to see `get_brand_kit` and the updated server instructions. +## Domain Rating (prepared; requires the DR migration and release) + +`find_sites` and `lookup_site` include `domain_rating` (rounded integer 0–100 or null) and `domain_rating_checked_on` (successful UTC YYYY-MM-DD or null). A failed lookup does not clear a prior value or advance its date. `stack127_data.ad_network_publishers` projects these two private fields; anonymous/authenticated roles still have no view access. `dataset_summary` adds `domain_rating_base` and `domain_rating_total`, measured and total distinct listed publishers respectively. + +`count_sites` and `compare_ad_networks` accept `min_domain_rating` and `max_domain_rating`; filters combine with the existing filters. `domain_rating` is a grouping/comparison dimension with exclusive buckets `0–9`, `10–19`, `20–29`, `30–49`, `50–69`, `70–100`, `unknown`. Zero is measured. Unknowns remain in the unfiltered base and unknown bucket; min/max filters exclude them. Grouped answers include `domain_rating_coverage` for all filtered distinct sites; network groups also include `network_domain_rating_coverage` with each network's measured count and full filtered base, even when group rows truncate. + +Every network in `get_ad_network` and `list_ad_networks` includes `median_domain_rating`, `average_domain_rating`, and `domain_rating_base`. Both summaries use measured listed sites only. The denominator is that row's `listed` count. Empty measured cohorts return null summaries and zero coverage. Network profiles also include the DR bucket distribution. Network memberships can overlap; do not add their measured counts to estimate dataset coverage. + +All successful answers, including optional private SQL, carry `data.domain_rating_metadata`: `attribution` = `Domain Rating by Ahrefs`, `url` = `https://ahrefs.com`, a ready-to-display linked `credit`, a `caveat` that DR is a third-party estimate, and `coverage` (`sites_with_rating`, `total_sites`, explicit dataset scope). This coverage describes all distinct listed sites, not just a page or SQL result; filtered groups and network rows provide their own bases. MCP text and structured content carry the same metadata. Clients must display [Domain Rating by Ahrefs](https://ahrefs.com) beside DR values/summaries, retain the third-party estimate caveat, and include coverage. DR estimates backlink strength; it is not traffic, revenue, quality or a recommendation. + +The [free endpoint](https://docs.ahrefs.com/en/api/reference/public/get-domain-rating-free) uses `dr3`. The [license](https://ahrefs.com/legal/domain-rating-license) allows integration with linked attribution, bars redistribution of original DR data or a competing product, and forbids circumventing limits. The authenticated MCP/REST integration stays private to the owner. The public website and bounded `/api/ad-networks/sites` and `/api/ad-networks/site` responses include per-site DR with linked attribution; their `attribution` metadata is present when a rating is returned, including zero. Public page summaries show rated/total coverage. Sitemaps and registry JSON-LD carry no DR, and there is no DR CSV export. No bulk DR export or resale is offered. Future public MCP/REST plans remain limited to derived summaries per network. See [the collector instructions](../scripts/ad-networks/ovh/README.md) for priority, pacing, source coverage and freshness. + ## Authentication and future public plans `authenticate(request) -> Principal` is the replacement point for future user API keys/OAuth and paid plans. Today both credentials map to `private-owner`, plan `private`. `PUBLIC_MODE` is a hardcoded false constant. There is no anonymous fallback, public switch, signup, billing endpoint, or public SQL access. @@ -65,6 +77,7 @@ When this becomes a public product, it moves to its own subdomain and gets real - **Sign-in:** OAuth for claude.ai and ChatGPT, so each user clicks Connect, signs in on stack127, and approves access. No secret URLs for public users. Per-user API keys for REST and header clients. Both replace `authenticate(request)`; the tools and their input and output stay the same. - **Accounts and plans:** user accounts, plans (for example free and paid), billing, and per-plan limits on requests a day, rows per answer, and which tools are available. The existing rate-limit and usage-log design is per principal, so it extends to per user. - **Read-only by design, with a second lock:** every tool is read-only today (read-only transactions, fixed parameterized queries, read-only views; the only write is our own rate-limit counter). Today the tools use the website's normal database login, which can write elsewhere, so the read-only transaction is the main protection. Before launch, an admin creates a dedicated database login with SELECT on the `stack127_data` views only (plus the counter table), and the public MCP and API use only that login, so even a code bug cannot write. +- **DR in public plans:** expose only DR summaries per network with linked Ahrefs credit, third-party estimate caveat and measured coverage. Never expose per-site DR values, including in `find_sites`, `lookup_site`, downloads or dataset exports, under this planned API policy. The existing bounded website APIs are separate. Add plan-aware field projections before offering public MCP/REST access. - **What stays private:** `run_readonly_sql` is never offered publicly. `publisher_name` and internal check fields are never returned. Whether site-by-site lists (`find_sites`, `lookup_site`) are paid-only is an owner decision to make before launch. - **Before launch:** terms of use and an acceptable-use rule, public documentation for each tool, a status or uptime check, alerts on errors and abuse, and a review of data accuracy and its stated limits. - **The owner's private access** moves to the same sign-in at launch, and the secret path is switched off. diff --git a/stack127_docs/monitoring.json b/stack127_docs/monitoring.json index bf34f977..2288c9da 100644 --- a/stack127_docs/monitoring.json +++ b/stack127_docs/monitoring.json @@ -3,10 +3,11 @@ "project": "stack127", "about": "Entries for the Automation Monitor dashboard (https://monitor.ravitejaknts.com). One entry per scheduled job, workflow or service. key: / (machines: homelab, bloxodes-vps, stack127-vps, macbook-pro, macbook-air; job = cron run-job.sh name, systemd unit without .timer/.service, launchd label or T3 schedule title), github///, supabase/, or northflank/. name: short plain name. group: the section it is listed under. about: one sentence on what it does, written for the owner. impact (optional): what breaks if it stops. records (optional): stats_job_runs.job_name values it writes. ignore_records: job names that are sub-steps of other jobs.", "automations": [ - {"key": "stack127-vps/launch-directory-metrics", "group": "Launch directories", "name": "Refresh launch directory ratings", "about": "On the 2nd of each month at 06:00 UTC, refreshes Ahrefs Domain Rating and Tranco popularity for published launch directories.", "impact": "Launch directory ratings and popularity become stale."}, + {"key": "stack127-vps/launch-directory-metrics", "group": "Launch directories", "name": "Refresh launch directory ratings", "about": "On the 2nd of each month at 06:00 UTC, refreshes Ahrefs Domain Rating and Tranco popularity for published launch directories. Shares the process-held /srv/stack127-data/state/ahrefs.sqlite.lock request lock with ad-network DR collection. The separate ahrefs.sqlite commits pacing before HTTP and Retry-After before body cleanup or callbacks, retaining committed state after a crash.", "impact": "Launch directory ratings and popularity become stale."}, {"key": "stack127-vps/verification", "group": "Ad networks", "name": "Ad network site checks", "about": "Every hour at minute 40, checks up to 1,000 publisher sites (their ads.txt and pages), sends changed rows to production, and refreshes the site counts in titles and headings. Blocked or failed homepage checks never record departures, retry with backoff, and mark confirmation stale after 30 days without fresh evidence. It steps aside while a bigger job runs.", "impact": "Network pages stop gaining newly confirmed sites."}, {"key": "stack127-vps/snapshot", "group": "Ad networks", "name": "Daily ad network counts", "about": "Saves today's publisher counts for each ad network."}, {"key": "stack127-vps/platform", "group": "Ad networks", "name": "Publisher platform and favicon", "about": "Fills in each publisher's website platform and icon, and refreshes facts older than 30 days. Challenge pages remain unavailable."}, + {"key": "stack127-vps/domain-rating", "group": "Ad networks", "name": "Publisher Domain Rating", "about": "At minute 05 of hours 07 through 23 UTC, checks up to 500 domains in 30 minutes; up to 8,500/day. OVH: 213,003 candidates, 123,361 listed, 39,823 managed; about 5 days for managed and 26 days for full first coverage. Both Ahrefs jobs share the process-held /srv/stack127-data/state/ahrefs.sqlite.lock request lock. The separate ahrefs.sqlite commits pacing before HTTP and cooldowns before body cleanup or callbacks, retaining committed state after a crash. DR-only sync keeps the frontend revision unchanged; the dated 00:10 daily snapshot queues the refresh that publishes DR daily. A sync overrun into :40 may skip hourly verification once; acceptable occasionally, investigate repeated skips.", "impact": "Public pages, public APIs and private MCP/REST stop gaining fresh Domain Rating estimates."}, {"key": "stack127-vps/domain-age", "group": "Ad networks", "name": "Publisher domain age", "about": "Looks up when new publisher domains were registered."}, {"key": "stack127-vps/ads-txt-summary", "group": "Ad networks", "name": "ads.txt summaries", "about": "Summarizes each publisher's ads.txt file for the network pages."}, {"key": "stack127-vps/discovery", "group": "Ad networks", "name": "Find new publishers", "about": "Every Tuesday, reads the ad networks' official seller lists and adds new publisher domains to check."}, diff --git a/supabase/migrations/20261114130000_ad_network_domain_rating.sql b/supabase/migrations/20261114130000_ad_network_domain_rating.sql new file mode 100644 index 00000000..d512c430 --- /dev/null +++ b/supabase/migrations/20261114130000_ad_network_domain_rating.sql @@ -0,0 +1,120 @@ +begin; +set local lock_timeout='10s'; +set local statement_timeout='120s'; +-- Add nullable columns without scanning existing publishers under ACCESS EXCLUSIVE. +-- Reruns keep existing data, but reject an incompatible pre-existing schema. +do $$ +declare column_name text; expected_type regtype; actual_type oid; +begin + for column_name, expected_type in select * from (values + ('domain_rating', 'integer'::regtype), ('domain_rating_checked_on', 'date'::regtype)) as columns(name, type) + loop + select atttypid into actual_type from pg_attribute + where attrelid='public.ad_network_publishers'::regclass and attname=column_name and not attisdropped; + if found and actual_type <> expected_type::oid then + raise exception 'public.ad_network_publishers.% has type %, expected %', column_name, actual_type::regtype, expected_type; + end if; + end loop; +end $$; +alter table public.ad_network_publishers + add column if not exists domain_rating integer, + add column if not exists domain_rating_checked_on date; +do $$ begin + if not exists(select 1 from pg_constraint where conrelid='public.ad_network_publishers'::regclass + and conname='ad_network_publishers_domain_rating_check') then + alter table public.ad_network_publishers add constraint ad_network_publishers_domain_rating_check + check(domain_rating between 0 and 100) not valid; + end if; +end $$; +comment on column public.ad_network_publishers.domain_rating is + 'Rounded Ahrefs dr3 estimate for pages and APIs. Display with linked Domain Rating by Ahrefs credit. Unknown is null.'; + +create or replace view stack127_data.ad_network_publishers with (security_barrier=true) as + select id,normalized_domain,niche,platform,domain_created_on,traffic_group,last_successful_check_at, + case when ads_txt_summary is not null then jsonb_build_object( + 'lines',ads_txt_summary->'lines','direct',ads_txt_summary->'direct', + 'reseller',ads_txt_summary->'reseller','invalid',ads_txt_summary->'invalid', + 'top_sellers',ads_txt_summary->'top_sellers') end as ads_txt_summary, + p.domain_rating,p.domain_rating_checked_on + from public.ad_network_publishers p + where exists(select 1 from public.ad_network_publisher_relationships r + where r.publisher_id=p.id and r.status in ('confirmed','stale','ended')); +revoke all on stack127_data.ad_network_publishers from public,anon,authenticated; + +create or replace view stack127_data.dataset_summary with (security_barrier=true) as + select count(*) filter(where last_successful_check_at is not null)::int as sites_checked, + min(last_successful_check_at) as oldest_site_check,max(last_successful_check_at) as newest_site_check, + greatest(max(last_successful_check_at),(select max(observed_at) from public.ad_network_relationship_events), + (select max(captured_at) from public.ad_network_daily_counts)) as as_of, + count(*) filter(where domain_rating is not null and exists(select 1 from public.ad_network_publisher_relationships r + where r.publisher_id=ad_network_publishers.id and r.status in ('confirmed','stale')))::int as domain_rating_base, + count(*) filter(where exists(select 1 from public.ad_network_publisher_relationships r + where r.publisher_id=ad_network_publishers.id and r.status in ('confirmed','stale')))::int as domain_rating_total + from public.ad_network_publishers; + +-- Pages and APIs display DR with linked attribution. DR-only writes do not +-- move the frontend revision or enqueue a rebuild. The daily snapshot always +-- inserts a dated row and queues the refresh that publishes new DR each day. +-- Other enrichment and relationship changes still queue their normal refresh. +create or replace function public.stack127_directory_record_value(p_table text, p_row jsonb) +returns jsonb language plpgsql immutable security invoker set search_path = pg_catalog as $$ +begin + case p_table + when 'ad_network_publishers' then + return p_row - array['created_at','updated_at','next_check_at','lease_until','verification_priority','check_attempt_count','consecutive_failures','last_discovered_at','ads_txt_hash','ads_txt_etag','ads_txt_last_modified','domain_rating','domain_rating_checked_on']; + when 'ad_network_publisher_relationships' then + if p_row->>'status' not in ('confirmed','stale','ended') then return null; end if; + -- Verification mix and retained manager evidence are public report inputs. + return (p_row - array['created_at','updated_at','consecutive_misses','first_missed_at','last_missed_at','evidence']) + || jsonb_build_object('manager_evidence',jsonb_build_object( + 'manager_domain',p_row->'evidence'->'manager_domain', + 'matched_signal',p_row->'evidence'->'matched_signal')); + when 'ad_networks' then + return jsonb_build_object('id',p_row->'id','slug',p_row->'slug','name',p_row->'name', + 'official_url',p_row->'official_url','summary',p_row->'summary', + 'manager_domains',p_row->'manager_domains','ad_system_domains',p_row->'ad_system_domains'); + when 'launch_directories' then + if p_row->>'is_published' <> 'true' then return null; end if; + return p_row - array['created_at','updated_at','research_notes']; + when 'terminal_apps' then + if p_row->>'published' <> 'true' then return null; end if; + return p_row - array['created_at','updated_at','self_host_app_id']; + when 'self_host_apps' then + if p_row->>'published' <> 'true' or p_row->>'self_hosting_status' <> 'self_hostable' then return null; end if; + return p_row - array['created_at','updated_at','data_checked_at','data_confidence','data_source_urls', + 'source_repo_id','github_repo_id','github_node_id','github_full_name','github_owner_type', + 'source_full_name','source_watchers','source_open_issues','source_languages','source_updated_at', + 'source_default_branch','source_repo_size_kb','source_disabled','source_is_fork','source_is_template', + 'github_watchers','github_open_issues','github_languages','github_updated_at','github_default_branch', + 'github_repo_size_kb','github_disabled','github_is_fork','github_is_template']; + when 'github_projects' then + return jsonb_build_object('id',p_row->'id','owner',p_row->'owner','repo',p_row->'repo', + 'stars',p_row->'stars','forks',p_row->'forks','primary_language',p_row->'primary_language','topics',p_row->'topics', + 'created_at',p_row->'created_at','pushed_at',p_row->'pushed_at','latest_release',p_row->'latest_release', + 'latest_release_at',p_row->'latest_release_at','last_commit_at',p_row->'last_commit_at', + 'checked_at',p_row->'checked_at','archived',p_row->'archived'); + when 'self_host_app_install_options' then + return jsonb_build_object('app_id',p_row->'app_id','options',p_row->'options'); + else + return p_row - array['created_at','updated_at','row_updated_at']; + end case; +end; +$$; +revoke all on function public.stack127_directory_record_value(text,jsonb) from public, anon, authenticated; +grant execute on function public.stack127_directory_record_value(text,jsonb) to service_role; + +commit; + +-- Release ACCESS EXCLUSIVE before scanning the existing rows. VALIDATE takes +-- SHARE UPDATE EXCLUSIVE, allowing normal reads and writes during validation. +-- https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-NOTES +begin; +set local lock_timeout='10s'; +set local statement_timeout='120s'; +do $$ begin + if exists(select 1 from pg_constraint where conrelid='public.ad_network_publishers'::regclass + and conname='ad_network_publishers_domain_rating_check' and not convalidated) then + alter table public.ad_network_publishers validate constraint ad_network_publishers_domain_rating_check; + end if; +end $$; +commit; diff --git a/tests/data-api/core.test.mjs b/tests/data-api/core.test.mjs index 035608a0..9af9db61 100644 --- a/tests/data-api/core.test.mjs +++ b/tests/data-api/core.test.mjs @@ -46,7 +46,7 @@ test('SQL parser rejects writes, batches, CTE mutations, locks, private fields b 'select * from (select slug from stack127_data.ad_networks) as n'])assert.equal(validateReadonlySql(sql),sql); }); test('every core response carries definitions, caveats, filters, data time and citation',async()=>{ - const transaction=async run=>run({query:async sql=>sql.includes('dataset_summary')?[{as_of:'2026-11-14T00:00:00Z'}]:[{total:2}]}); + const transaction=async run=>run({query:async sql=>sql.includes('dataset_summary')?[{as_of:'2026-11-14T00:00:00Z',domain_rating_base:1,domain_rating_total:2}]:[{total:2,domain_rating_base:1}]}); const answer=await executeTool('count_sites',{},transaction); assert.equal(answer.data.total,2);assert.equal(answer.filters.relationship,'listed');assert.ok(answer.as_of); assert.match(answer.definitions.not_ranked,/not small/);assert.ok(answer.definitions.managed);assert.ok(answer.definitions.shares); @@ -86,7 +86,7 @@ test('SQL flag alone cannot run a query or expose a REST route',async()=>{ }); test('MCP text contains the full answer and applies the same byte limit after formatting',async()=>{ const {mcpAnswer}=await import('../../lib/data-api/mcp.ts'); - const answer=await executeTool('count_sites',{},async run=>run({query:async sql=>sql.includes('dataset_summary')?[{as_of:null}]:[{total:42}]})); + const answer=await executeTool('count_sites',{},async run=>run({query:async sql=>sql.includes('dataset_summary')?[{as_of:null,domain_rating_base:10,domain_rating_total:42}]:[{total:42,domain_rating_base:10}]})); const result=mcpAnswer(answer); assert.deepEqual(JSON.parse(result.content[0].text),result.structuredContent); assert.equal(JSON.parse(result.content[0].text).data.total,42); @@ -100,7 +100,7 @@ test('future SQL uses only its dedicated pool and read-only transaction, includi const previous={app:globalThis.stack127DataApiPool,sql:globalThis.stack127DataSqlPool,flag:process.env.STACK127_DATA_SQL_ENABLED,url:process.env.STACK127_DATA_SQL_DATABASE_URL}; globalThis.stack127DataApiPool={connect:()=>assert.fail('arbitrary SQL used the app login')}; globalThis.stack127DataSqlPool={connect:async()=>({query:async(sql,values)=>{ - statements.push({sql,values});return {rows:sql.includes('dataset_summary')?[{as_of:null}]:[{value:1}]}; + statements.push({sql,values});return {rows:sql.includes('dataset_summary')?[{as_of:null,domain_rating_base:10,domain_rating_total:42}]:[{value:1}]}; },release(){released++;}})}; Object.assign(process.env,{STACK127_DATA_SQL_ENABLED:'true',STACK127_DATA_SQL_DATABASE_URL:'postgresql://restricted:unused@localhost/research'}); try { @@ -174,3 +174,29 @@ test('brand kit is listed in MCP and both front doors share its answer and authe for(const key of keys){if(previousEnv[key]===undefined)delete process.env[key];else process.env[key]=previousEnv[key];} } }); + +test('DR filters validate integer ranges, bind zero and combine with managed/network filters',()=>{ + for(const f of [{min_domain_rating:-1},{max_domain_rating:101},{min_domain_rating:1.5},{min_domain_rating:50,max_domain_rating:20}]) + assert.throws(()=>validateInput('count_sites',f),{status:400}); + const f=filterSql(validateInput('count_sites',{networks:['mediavine'],relationship:'managed',min_domain_rating:0,max_domain_rating:69})); + assert.match(f.text,/p.domain_rating>=\$2/);assert.match(f.text,/p.domain_rating<=\$3/); + assert.deepEqual(f.values,[['mediavine'],0,69]); + assert.deepEqual(validateInput('count_sites',{group_by:['domain_rating']}).group_by,['domain_rating']); + assert.equal(validateInput('compare_ad_networks',{networks:['ezoic','mediavine'],dimension:'domain_rating'}).dimension,'domain_rating'); +}); + +test('DR answers carry linked attribution, third-party caveat and measured coverage in MCP text too',async()=>{ + const transaction=async run=>run({query:async sql=>sql.includes('dataset_summary')? + [{as_of:null,domain_rating_base:2,domain_rating_total:3}]:sql.includes('group by')? + [{group_1:'0–9',count:1},{group_1:'unknown',count:1}]:[{total:2,domain_rating_base:1}]}); + const answer=await executeTool('count_sites',{group_by:['domain_rating']},transaction); + assert.deepEqual(answer.data.domain_rating_coverage,{sites_with_rating:1,total_sites:2,scope:'distinct sites matching all filters'}); + assert.equal(answer.data.domain_rating_metadata.attribution,'Domain Rating by Ahrefs'); + assert.equal(answer.data.domain_rating_metadata.url,'https://ahrefs.com'); + assert.match(answer.data.domain_rating_metadata.caveat,/third-party estimate/); + assert.equal(answer.data.domain_rating_metadata.coverage.sites_with_rating,2); + assert.deepEqual(answer.data.rows.map(r=>r.domain_rating),['0–9','unknown']); + const {mcpAnswer}=await import('../../lib/data-api/mcp.ts'); + const text=JSON.parse(mcpAnswer(answer).content[0].text); + assert.deepEqual(text.data.domain_rating_metadata,answer.data.domain_rating_metadata); +}); diff --git a/tests/pages/ad-networks/ahrefs-crash.test.mjs b/tests/pages/ad-networks/ahrefs-crash.test.mjs new file mode 100644 index 00000000..f0c1df39 --- /dev/null +++ b/tests/pages/ad-networks/ahrefs-crash.test.mjs @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import test from 'node:test'; +import { ahrefsRequester } from '../../../scripts/stack127/ahrefs-dr.mjs'; + +const startedAt = Date.parse('2026-11-14T12:00:00Z'); +const moduleUrl = new URL('../../../scripts/stack127/ahrefs-dr.mjs', import.meta.url).href; + +async function crashAt(t, point) { + const directory = mkdtempSync(join(tmpdir(), 's127-ahrefs-crash-')); + const stateFile = join(directory, 'ahrefs.sqlite'); + let child, requests = 0, receivedRequest; + const inFlight = new Promise(resolve => { receivedRequest = resolve; }); + const server = createServer((request, response) => { + requests++; + receivedRequest(); + if (point !== 'during-request') { + response.writeHead(429, { 'Retry-After': '86400' }); + response.end('limited'); + } + }); + t.after(async () => { + if (child && child.exitCode === null && child.signalCode === null) { + const exited = once(child, 'exit'); + child.kill('SIGKILL'); + await exited; + } + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + rmSync(directory, { recursive: true, force: true }); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + child = spawn(process.execPath, ['--input-type=module', '-e', ` + import { ahrefsRequester } from ${JSON.stringify(moduleUrl)}; + const [stateFile, point, url] = process.argv.slice(1); + function checkpoint(point) { + process.send({ point }); + return new Promise(resolve => setTimeout(resolve, 60000)); + } + const request = ahrefsRequester({ apiKey: 'test-only', stateFile, now: () => ${startedAt}, + fetchImplementation: async () => { + if (point === 'before-request') await checkpoint(point); + const response = await fetch(url); + if (point === 'before-cleanup') return { + status: response.status, headers: response.headers, + body: { cancel: () => checkpoint(point) }, + }; + return response; + }, + onRateLimit: () => { + process.send({ point: 'in-callback' }); + // A synchronous callback must not be able to prevent the state commit. + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0); + }, + }); + await request('example.com'); + throw new Error('request returned before the crash checkpoint'); + `, stateFile, point, `http://127.0.0.1:${server.address().port}`], { + stdio: ['ignore', 'ignore', 'pipe', 'ipc'], + }); + let stderr = ''; + child.stderr.on('data', data => { stderr += data; }); + const exited = once(child, 'exit'); + const reached = point === 'during-request' ? inFlight : once(child, 'message').then(([message]) => { + assert.equal(message.point, point); + }); + await Promise.race([ + reached, + exited.then(([code, signal]) => assert.fail(`child exited before ${point}: ${code}/${signal}\n${stderr}`)), + ]); + assert.equal(requests, point === 'before-request' ? 0 : 1); + child.kill('SIGKILL'); + assert.deepEqual(await exited, [null, 'SIGKILL']); + return { stateFile, requests: () => requests }; +} + +for (const point of ['before-request', 'during-request']) { + test(`SIGKILL ${point} preserves the pacing reservation and releases the request lock`, { timeout: 10000 }, async t => { + const { stateFile, requests } = await crashAt(t, point); + let time = startedAt + 100, calls = 0; + const waits = []; + const request = ahrefsRequester({ apiKey: 'test-only', stateFile, now: () => time, + sleepImplementation: async ms => { + assert.equal(calls, 0, 'no HTTP before the reserved slot'); + waits.push(ms); + time += ms; + }, + fetchImplementation: async () => { + calls++; + assert.equal(time, startedAt + 3000); + return new Response(JSON.stringify({ domain_rating: { domain_rating: 15 } })); + }, + }); + assert.deepEqual(await request('next.example'), { value: 15 }); + assert.deepEqual(waits, [2900]); + assert.equal(calls, 1); + assert.equal(requests(), point === 'before-request' ? 0 : 1); + }); +} + +for (const point of ['before-cleanup', 'in-callback']) { + test(`SIGKILL after 429 ${point} preserves Retry-After before cleanup or callbacks finish`, { timeout: 10000 }, async t => { + const { stateFile, requests } = await crashAt(t, point); + let calls = 0; + const options = { apiKey: 'test-only', stateFile, + sleepImplementation: async () => assert.fail('long cooldown must stop without waiting'), + fetchImplementation: async () => { + calls++; + return new Response(JSON.stringify({ domain_rating: { domain_rating: 20 } })); + }, + }; + // Read from a fresh connection after death: the cooldown must already be committed. + const db = new DatabaseSync(stateFile); + try { + assert.equal(db.prepare('select cooldown_until from request_state where id=1').get().cooldown_until, startedAt + 86400000); + } finally { db.close(); } + for (const elapsed of [100, 3600000, 86339999]) { + assert.deepEqual(await ahrefsRequester({ ...options, now: () => startedAt + elapsed })('next.example'), { stopped: 'rate_limited' }); + assert.equal(calls, 0, 'no HTTP throughout the retained cooldown'); + } + const controller = new AbortController(); + await assert.rejects(ahrefsRequester({ ...options, now: () => startedAt + 86399999, signal: controller.signal, + sleepImplementation: async ms => { + assert.equal(ms, 1); + assert.equal(calls, 0, 'the final millisecond still blocks HTTP'); + controller.abort(); + }, + })('next.example'), { name: 'AbortError' }); + assert.equal(calls, 0); + assert.deepEqual(await ahrefsRequester({ ...options, now: () => startedAt + 86400001 })('next.example'), { value: 20 }); + assert.equal(calls, 1); + assert.equal(requests(), 1); + }); +} diff --git a/tests/pages/ad-networks/backend.test.mjs b/tests/pages/ad-networks/backend.test.mjs index 90fc2335..7fc7401c 100644 --- a/tests/pages/ad-networks/backend.test.mjs +++ b/tests/pages/ad-networks/backend.test.mjs @@ -1,3 +1,4 @@ +import * as ahrefs from "../../../lib/ahrefs.ts"; import assert from 'node:assert/strict'; import test from 'node:test'; import { deriveManagedBy, buildAdsTxtPreview, aggregateAdNetworkManagers, aggregateAdNetworkTrafficShares } from '../../../lib/ad-network-site-facts.mjs'; @@ -43,7 +44,7 @@ test('sync defaults to a read-only development preview and requires explicit pro }); test('sites inputs support both network and niche with bounded pagination and literal search',()=>{ assert.deepEqual(parseAdNetworkSitesParams(new URLSearchParams('network=mediavine&niche=food&q=Food%25&sort=oldest&page=2&pageSize=50')), - {network:'mediavine',niche:'food',q:'food%',sort:'oldest',page:2,pageSize:50}); + {network:'mediavine',niche:'food',q:'food%',sort:'oldest',page:2,pageSize:50,minDr:null}); for (const query of ['page=0','page=-1','page=1.5','page=10001','pageSize=101','pageSize=0','sort=drop','niche=made-up','network=x%27','q=a&q=b','x=1']) { assert.throws(()=>parseAdNetworkSitesParams(new URLSearchParams(query)),AdNetworkInputError,query); } @@ -59,15 +60,15 @@ test('checker accepts URLs and international domains and rejects local or malfor const { loadTypeScript } = await import('../../helpers/load-typescript.mjs'); const input = {parseAdNetworkSitesParams,parseAdNetworkSiteParams,AdNetworkInputError}; -function route(kind, {published=true,fail=false}={}) { +function route(kind, {published=true,fail=false,rating=null}={}) { let calls=0; const lib={ - getAdNetworkSites: async query=>{calls++;if(fail)throw Error('database unavailable');return {rows:[],total:0,page:query.page,pageSize:query.pageSize,totalPages:1};}, - getAdNetworkCheckerSite: async domain=>{calls++;if(fail)throw Error('database unavailable');return domain==='known.com'?{site:{domain}}:null;}, + getAdNetworkSites: async query=>{calls++;if(fail)throw Error('database unavailable');return {rows:[{domain:"known.com",domain_rating:rating}],total:1,page:query.page,pageSize:query.pageSize,totalPages:1,...ahrefs.ahrefsAttribution([rating])};}, + getAdNetworkCheckerSite: async domain=>{calls++;if(fail)throw Error('database unavailable');return domain==='known.com'?{site:{domain,domain_rating:rating},similar_sites:[]}:null;}, }; const {GET}=loadTypeScript(new URL(`../../../app/api/ad-networks/${kind}/route.ts`,import.meta.url),{ 'next/server':{NextResponse:{json:(body,options)=>new Response(JSON.stringify(body),options)}}, - '@/lib/ad-networks':lib,'@/lib/page-registry':{getPublishedPage:async()=>published?{}:null},'../input':input, + '@/lib/ahrefs':ahrefs,'@/lib/ad-networks':lib,'@/lib/page-registry':{getPublishedPage:async()=>published?{}:null},'../input':input, }); return {GET,calls:()=>calls}; } @@ -198,3 +199,14 @@ test('traffic shares retain confirmed managers after failed checks and count eac assert.deepEqual(aggregateAdNetworkTrafficShares([...groups].reverse(),networks),expected); assert.deepEqual(aggregateAdNetworkTrafficShares([],networks),[]); }); + +test('both public API responses credit Ahrefs for a present DR including zero, and omit credit for unknown DR',async()=>{ + for(const kind of ['sites','site'])for(const rating of [0,42,null]) { + const api=route(kind,{rating}); + const response=await api.GET(new Request(`https://stack127.com/api/ad-networks/${kind}?${kind==='site'?'domain=known.com':''}`)); + assert.equal(response.status,200); + const body=await response.json(); + if(rating===null)assert.equal(Object.hasOwn(body,'attribution'),false); + else assert.deepEqual(body.attribution,ahrefs.AHREFS_ATTRIBUTION); + } +}); diff --git a/tests/pages/ad-networks/chart-data.test.mjs b/tests/pages/ad-networks/chart-data.test.mjs index 66ee81af..ad849308 100644 --- a/tests/pages/ad-networks/chart-data.test.mjs +++ b/tests/pages/ad-networks/chart-data.test.mjs @@ -76,7 +76,7 @@ test('newest niche arrivals name the network actually joined while preserving le const coverage={listed:125,classified:100,traffic:0,age:0,platform:0,publisher:0,checked:0,manager:0,niches:['food']}; const pageSizes=[]; const lib=adNetworksLibrary(async(sql,args)=>{ - if(sql.includes('limit $4 offset $5'))pageSizes.push(args[3]); + if(sql.includes('limit $5 offset $6'))pageSizes.push(args[4]); if(sql.includes('arrived_network.name'))return {rows:[{...site,first_confirmed_at:'2026-10-15T12:30:00Z',arrived_network:{slug:'b',name:'Network B'}}]}; if(sql.includes('count(p.niche)'))return {rows:[coverage]}; if(sql.includes('as classified'))return {rows:[{listed:125,confirmed:100,classified:100}]}; diff --git a/tests/pages/ad-networks/checker-domains.test.mjs b/tests/pages/ad-networks/checker-domains.test.mjs index 64b515cc..22dc87e7 100644 --- a/tests/pages/ad-networks/checker-domains.test.mjs +++ b/tests/pages/ad-networks/checker-domains.test.mjs @@ -1,3 +1,4 @@ +import * as ahrefs from "../../../lib/ahrefs.ts"; import assert from 'node:assert/strict'; import test from 'node:test'; import {adNetworksLibrary} from './loader-harness.mjs'; @@ -28,6 +29,7 @@ test('site API returns each relationship ad-system domains without substituting }); const {GET}=loadTypeScript(new URL('../../../app/api/ad-networks/site/route.ts',import.meta.url),{ 'next/server':{NextResponse:{json:(body,options)=>Response.json(body,options)}}, + "@/lib/ahrefs":ahrefs, '@/lib/ad-networks':{getAdNetworkCheckerSite:lib.loadAdNetworkCheckerSite}, '@/lib/page-registry':{getPublishedPage:async()=>({})},'../input':input, }); diff --git a/tests/pages/ad-networks/domain-rating-database.test.mjs b/tests/pages/ad-networks/domain-rating-database.test.mjs new file mode 100644 index 00000000..e33712df --- /dev/null +++ b/tests/pages/ad-networks/domain-rating-database.test.mjs @@ -0,0 +1,122 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { parseEnv } from 'node:util'; +import pg from 'pg'; +import { executeTool, domainRatingBucket } from '../../../lib/data-api/ad-networks/core.ts'; + +let devEnv = {}; +try { devEnv = parseEnv(readFileSync(new URL('../../../.env.local', import.meta.url), 'utf8')); } catch { /* Optional development check. */ } +const connection = process.env.DATABASE_POOLER_URL || process.env.DATABASE_URL; +const devConnection = devEnv.DATABASE_POOLER_URL || devEnv.DATABASE_URL; +const live = process.env.STACK127_DR_INTEGRATION === 'true' && connection === devConnection + && devEnv.SUPABASE_URL === 'https://yzasonowwlvritqicatj.supabase.co'; + +test('development DR schema, cache projection, private views and all API queries; rollback leaves no changes', {skip:!live}, async () => { + const client = new pg.Client({connectionString:connection,connectionTimeoutMillis:15000,query_timeout:120000}); + await client.connect(); + try { + // Apply the migration separately: its validation must start after the DDL commits. + // Only test fixtures belong in this rolled-back transaction. + await client.query('begin'); + const constraint=(await client.query("select convalidated from pg_constraint where conrelid='public.ad_network_publishers'::regclass and conname='ad_network_publishers_domain_rating_check'")).rows[0]; + assert.equal(constraint?.convalidated,true); + await client.query('set local search_path=pg_temp,stack127_data,public'); + const sync=(await client.query(`select domain_rating,domain_rating_checked_on::text from jsonb_populate_recordset(null::public.ad_network_publishers, + '[{\"domain_rating\":0,\"domain_rating_checked_on\":\"2026-11-14\"}]'::jsonb)`)).rows[0]; + assert.deepEqual(sync,{domain_rating:0,domain_rating_checked_on:'2026-11-14'}); + const before = (await client.query('select revision from public.stack127_ad_network_source_revision')).rows[0].revision; + await client.query(`update public.ad_network_publishers set domain_rating=42,domain_rating_checked_on=current_date + where id=(select min(id) from public.ad_network_publishers)`); + assert.equal((await client.query('select revision from public.stack127_ad_network_source_revision')).rows[0].revision,before, + 'DR-only updates must not queue a frontend refresh or change the source revision'); + const projection = (await client.query(`select + public.stack127_directory_record_value('ad_network_publishers','{"id":1,"domain_rating":1,"domain_rating_checked_on":"2026-01-01","platform":"A"}') = + public.stack127_directory_record_value('ad_network_publishers','{"id":1,"domain_rating":99,"domain_rating_checked_on":"2026-11-14","platform":"A"}') as same, + public.stack127_directory_record_value('ad_network_publishers','{"id":1,"platform":"A"}') <> + public.stack127_directory_record_value('ad_network_publishers','{"id":1,"platform":"B"}') as other_changes`)).rows[0]; + assert.equal(projection.same,true);assert.equal(projection.other_changes,true); + const reader={query:async(sql,values)=>(await client.query(sql,values)).rows}; + const transaction=async work=>work(reader); + const real=await executeTool('find_sites',{limit:1},transaction); + assert.ok(Object.hasOwn(real.data.sites[0],'domain_rating')); + assert.ok(Object.hasOwn(real.data.sites[0],'domain_rating_checked_on')); + const anon=(await client.query("select has_table_privilege('anon','stack127_data.ad_network_publishers','SELECT') as allowed")).rows[0]; + assert.equal(anon.allowed,false); + await client.query(`create temp table ad_network_publishers as select * from stack127_data.ad_network_publishers where false; + create temp table ad_networks as select * from stack127_data.ad_networks where false; + create temp table ad_network_publisher_relationships as select * from stack127_data.ad_network_publisher_relationships where false; + create temp table ad_network_relationship_events as select * from stack127_data.ad_network_relationship_events where false; + create temp table ad_network_daily_counts as select * from stack127_data.ad_network_daily_counts where false; + insert into ad_networks(id,slug,name) values(1,'test-a','Test A'),(2,'test-b','Test B'),(3,'test-empty','Empty');`); + const ratings=[0,9,10,19,20,29,30,49,50,69,70,100,null]; + for(const [i,value] of ratings.entries()) { + await client.query(`insert into ad_network_publishers(id,normalized_domain,domain_rating,domain_rating_checked_on) + values($1,$2,$3,case when $3::int is not null then current_date end)`,[i+1,`site${i+1}.example`,value]); + await client.query(`insert into ad_network_publisher_relationships(ad_network_id,publisher_id,status,relationship_type) + values(1,$1,'confirmed','primary_or_exclusive_manager')`,[i+1]); + } + await client.query(`insert into ad_network_publisher_relationships(ad_network_id,publisher_id,status,relationship_type) + values(2,1,'stale','primary_or_exclusive_manager'),(2,13,'confirmed','active_network_use');`); + const buckets=(await reader.query(`select ${domainRatingBucket} as bucket,count(*)::int as count from ad_network_publishers p group by 1`)); + assert.deepEqual(Object.fromEntries(buckets.map(r=>[r.bucket,r.count])),{'0–9':2,'10–19':2,'20–29':2,'30–49':2,'50–69':2,'70–100':2,unknown:1}); + const grouped=await executeTool('count_sites',{group_by:['domain_rating']},transaction); + assert.equal(grouped.data.total,13);assert.equal(grouped.data.domain_rating_coverage.sites_with_rating,12); + assert.equal(grouped.data.rows.reduce((sum,r)=>sum+r.count,0),13); + const compare=await executeTool('compare_ad_networks',{networks:['test-a','test-b'],dimension:'domain_rating'},transaction); + assert.deepEqual(compare.data.network_domain_rating_coverage['test-b'],{sites_with_rating:1,total_sites:2}); + assert.equal(compare.data.total,13);assert.equal(compare.data.network_bases['test-b'],2); + const filtered=await executeTool('count_sites',{min_domain_rating:30,max_domain_rating:69},transaction); + assert.equal(filtered.data.total,4); + const zero=await executeTool('find_sites',{min_domain_rating:0,max_domain_rating:0},transaction); + assert.equal(zero.data.sites.length,1);assert.equal(zero.data.sites[0].domain_rating,0); + const lookup=await executeTool('lookup_site',{domain:'site13.example'},transaction); + assert.equal(lookup.data.site.domain_rating,null);assert.equal(lookup.data.site.domain_rating_checked_on,null); + const networks=await executeTool('list_ad_networks',{},transaction); + const n=networks.data.networks.find(n=>n.slug==='test-a'); + assert.equal(n.domain_rating_base,12);assert.equal(n.median_domain_rating,29.5); + assert.ok(Math.abs(n.average_domain_rating-ratings.filter(r=>r!==null).reduce((a,b)=>a+b,0)/12)<1e-10); + const empty=networks.data.networks.find(n=>n.slug==='test-empty'); + assert.equal(empty.domain_rating_base,0);assert.equal(empty.median_domain_rating,null);assert.equal(empty.average_domain_rating,null); + const profile=await executeTool('get_ad_network',{network:'test-a'},transaction); + assert.equal(profile.data.network.domain_rating_base,12); + assert.equal(profile.data.distributions.domain_rating.total,13); + for(const answer of [real,grouped,compare,filtered,zero,lookup,networks,profile]) { + assert.equal(answer.data.domain_rating_metadata.attribution,'Domain Rating by Ahrefs'); + assert.equal(answer.data.domain_rating_metadata.url,'https://ahrefs.com'); + assert.ok(Number.isInteger(answer.data.domain_rating_metadata.coverage.sites_with_rating)); + } + } finally { await client.query('rollback'); await client.end(); } +}); + +test('DR schema guards allow reruns, reject wrong types, and validate after releasing the exclusive lock', {skip:!live}, async()=>{ + const client=new pg.Client({connectionString:connection,connectionTimeoutMillis:15000,query_timeout:120000}); + await client.connect(); + const migration=readFileSync(new URL('../../../supabase/migrations/20261114130000_ad_network_domain_rating.sql',import.meta.url),'utf8'); + // Exercise the exact column/constraint SQL on disposable development tables. + // Views and source-revision functions are covered by the integration test above. + const ddl=migration.split('comment on column')[0].replaceAll('public.ad_network_publishers','pg_temp.dr_guard_test'); + const validation=migration.slice(migration.indexOf('-- Release ACCESS EXCLUSIVE')).replaceAll('public.ad_network_publishers','pg_temp.dr_guard_test'); + try { + await client.query('create temp table dr_guard_test(id integer); insert into dr_guard_test values(1)'); + await client.query(ddl); + const constraint=()=>client.query("select convalidated from pg_constraint where conrelid='pg_temp.dr_guard_test'::regclass and conname='ad_network_publishers_domain_rating_check'"); + assert.equal((await constraint()).rows[0].convalidated,false); + await client.query('commit'); + await client.query(validation.replace(/commit;\s*$/,'')); + assert.equal((await constraint()).rows[0].convalidated,true); + const locks=(await client.query("select mode from pg_locks where pid=pg_backend_pid() and relation='pg_temp.dr_guard_test'::regclass")).rows.map(row=>row.mode); + assert.ok(locks.includes('ShareUpdateExclusiveLock')); + assert.ok(!locks.includes('AccessExclusiveLock')); + await client.query('commit'); + await client.query(ddl+'commit;'); await client.query(validation); + assert.equal((await constraint()).rows[0].convalidated,true); + assert.equal((await client.query('select count(*)::int as total from dr_guard_test')).rows[0].total,1); + await assert.rejects(client.query('insert into dr_guard_test(domain_rating) values(101)'),/ad_network_publishers_domain_rating_check/); + for(const columns of ['domain_rating text','domain_rating integer,domain_rating_checked_on text']) { + await client.query(`drop table dr_guard_test;create temp table dr_guard_test(${columns})`); + await assert.rejects(client.query(ddl),/has type text, expected (integer|date)/); + await client.query('rollback'); + } + }finally{await client.query('rollback').catch(()=>{});await client.end();} +}); diff --git a/tests/pages/ad-networks/domain-rating-report.test.mjs b/tests/pages/ad-networks/domain-rating-report.test.mjs new file mode 100644 index 00000000..84227a13 --- /dev/null +++ b/tests/pages/ad-networks/domain-rating-report.test.mjs @@ -0,0 +1,135 @@ +import { AHREFS_ATTRIBUTION } from "../../../lib/ahrefs.ts"; +import assert from 'node:assert/strict'; +import test from 'node:test'; +import pg from 'pg'; +import { + AD_NETWORK_DR_GATE, adNetworkDomainRatingReport, adNetworkSiteAvailability, hasAdNetworkDomainRatingCoverage, +} from '../../../lib/ad-networks-report.ts'; +import {AdNetworkInputError, parseAdNetworkSitesParams} from '../../../app/api/ad-networks/input.ts'; +import {adNetworksLibrary} from './loader-harness.mjs'; + +const coverage={listed:1000,classified:0,traffic:0,age:0,platform:0,publisher:0,checked:0,manager:0,niches:[],networks:[]}; +const summary=(slug,listed,rated,median,average)=>({slug,name:slug,listed,rated,median,average}); + +test('the sites API accepts only the DR steps and the two DR sorts',()=>{ + assert.equal(parseAdNetworkSitesParams(new URLSearchParams()).minDr,null); + assert.equal(parseAdNetworkSitesParams(new URLSearchParams('network=raptive&dr=30')).minDr,30); + for(const step of [10,20,30,50,70])assert.equal(parseAdNetworkSitesParams(new URLSearchParams(`dr=${step}`)).minDr,step); + assert.equal(parseAdNetworkSitesParams(new URLSearchParams('sort=highest-dr')).sort,'highest-dr'); + assert.equal(parseAdNetworkSitesParams(new URLSearchParams('sort=lowest-dr')).sort,'lowest-dr'); + // One cache key per view: no near-duplicates of a step, no repeats, no blanks. + for(const query of ['dr=31','dr=030','dr=30.0','dr=','dr=0','dr=100','dr=30&dr=30','sort=dr']) + assert.throws(()=>parseAdNetworkSitesParams(new URLSearchParams(query)),AdNetworkInputError,query); +}); + +test('DR column, sorts and filter appear only once sites in scope are rated',()=>{ + for(const rated of [undefined,0]) { + const none=adNetworkSiteAvailability({...coverage,rated}); + assert.ok(!none.availableColumns.includes('domain_rating')); + assert.ok(!none.availableSorts.includes('highest-dr')); + assert.ok(!none.availableFilters.includes('domain_rating')); + } + const some=adNetworkSiteAvailability({...coverage,rated:1}); + assert.ok(some.availableColumns.includes('domain_rating')); + assert.deepEqual(some.availableSorts.filter(sort=>sort.endsWith('-dr')),['highest-dr','lowest-dr']); + assert.ok(some.availableFilters.includes('domain_rating')); +}); + +test('the DR block hides below the gate on rated sites and coverage',()=>{ + assert.deepEqual(AD_NETWORK_DR_GATE,{minRated:20,minShare:0.25}); + assert.equal(hasAdNetworkDomainRatingCoverage(19,19),false); + assert.equal(hasAdNetworkDomainRatingCoverage(20,81),false); + assert.equal(hasAdNetworkDomainRatingCoverage(20,80),true); + assert.equal(hasAdNetworkDomainRatingCoverage(0,0),false); + assert.equal(adNetworkDomainRatingReport('raptive',[summary('raptive',1000,249,30,30)],[{min:30,count:249}]),null); + assert.equal(adNetworkDomainRatingReport('missing',[summary('raptive',100,100,30,30)],[]),null); +}); + +test('the DR block bands sites, marks the rounded median band and compares the gated big networks',()=>{ + const summaries=[ + summary('raptive',100,90,29.5,34.2),summary('mediavine',200,150,28,30.4), + summary('ezoic',1000,100,20,19), // 10% rated: left out of the comparison + summary('journey-by-mediavine',300,300,14,15.6),summary('mgid',50,50,60,61), + ]; + const report=adNetworkDomainRatingReport('raptive',summaries,[ + {min:0,count:5},{min:20,count:30},{min:30,count:40},{min:70,count:10},{min:80,count:4},{min:100,count:1}, + ]); + assert.deepEqual(report.bands.map(band=>band.label),['0–9','10–19','20–29','30–39','40–49','50–59','60–69','70+']); + assert.deepEqual(report.bands.map(band=>band.count),[5,0,30,40,0,0,0,15]); + // 29.5 shows as 30, so the highlighted band is 30–39. + assert.equal(report.median_band,'30-39'); + assert.deepEqual(report.comparison.map(row=>[row.slug,row.current]), + [['raptive',true],['mediavine',false],['journey-by-mediavine',false]]); + // A network outside the big four still sees itself beside them. + const mgid=adNetworkDomainRatingReport('mgid',summaries,[{min:60,count:50}]); + assert.equal(mgid.comparison[0].slug,'mgid');assert.equal(mgid.comparison.length,4); + // Compared with nothing else, the strip hides. + assert.deepEqual(adNetworkDomainRatingReport('mgid',[summaries[4]],[{min:60,count:50}]).comparison,[]); +}); + +test('the sites loader filters by DR in both queries and sorts unrated sites last',async()=>{ + const calls=[]; + const {loadAdNetworkSites,defaultAdNetworkSitesQuery}=adNetworksLibrary(async(sql,args)=>{ + calls.push({sql,args}); + if(sql.includes('as classified'))return {rows:[{...coverage,rated:10}]}; + if(sql.includes('select distinct n.slug')||sql.includes('select evidence,manager_relationships')||sql.includes('select slug,name,manager_domains'))return {rows:[]}; + if(sql.includes('count(*)::int as total'))return {rows:[{total:3}]}; + if(sql.includes('p.normalized_domain as domain'))return {rows:[]}; + throw Error(`Unexpected SQL: ${sql}`); + }); + const result=await loadAdNetworkSites({...defaultAdNetworkSitesQuery('raptive'),sort:'highest-dr',minDr:30}); + assert.equal(result.effectiveSort,'highest-dr'); + const count=calls.find(call=>call.sql.includes('count(*)::int as total')); + const table=calls.find(call=>call.sql.includes('p.normalized_domain as domain')); + for(const call of [count,table]) { + assert.match(call.sql,/\(\$4::int is null or p\.domain_rating>=\$4\)/); + assert.equal(call.args[3],30); + } + assert.ok(table.sql.includes('order by p.domain_rating desc nulls last,p.normalized_domain limit $5 offset $6')); + calls.length=0; + await loadAdNetworkSites({...defaultAdNetworkSitesQuery('raptive'),sort:'lowest-dr'}); + assert.ok(calls.find(call=>call.sql.includes('p.normalized_domain as domain')).sql.includes('order by p.domain_rating asc nulls last')); + await assert.rejects(loadAdNetworkSites({...defaultAdNetworkSitesQuery('raptive'),minDr:31}),/Invalid sites query/); +}); + +// Read-only fixture query: the real ORDER BY runs in PostgreSQL; no tables are written. +const connectionString=process.env.DATABASE_URL; +test('both DR sorts put unrated sites last across pages',{skip:!connectionString},async()=>{ + const orders={}; + const library=adNetworksLibrary(async sql=>{ + if(sql.includes('as classified'))return {rows:[{...coverage,rated:5}]}; + if(sql.includes('count(*)::int as total'))return {rows:[{total:6}]}; + const order=sql.match(/select p\.id from matches p order by (.*?) limit \$5 offset \$6/s); + if(order)orders.current=order[1]; + return {rows:[]}; + }); + const rows=[[1,'a.com',40],[2,'b.com',null],[3,'c.com',70],[4,'d.com',40],[5,'e.com',null],[6,'f.com',5]] + .map(([id,normalized_domain,domain_rating])=>({id,normalized_domain,domain_rating})); + const client=new pg.Client({connectionString,connectionTimeoutMillis:15000, + options:'-c default_transaction_read_only=on -c statement_timeout=15000'}); + try { + await client.connect(); + for(const [sort,expected] of [['highest-dr',[3,1,4,6,2,5]],['lowest-dr',[6,1,4,3,2,5]]]) { + await library.loadAdNetworkSites({...library.defaultAdNetworkSitesQuery('raptive'),sort}); + const sql=`with matches as materialized ( + select * from jsonb_to_recordset($1::jsonb) p(id int,normalized_domain text,domain_rating int) + ) select p.id from matches p order by ${orders.current} limit $2 offset $3`; + const page=async(size,offset)=>(await client.query(sql,[JSON.stringify(rows),size,offset])).rows.map(row=>row.id); + assert.deepEqual([...(await page(2,0)),...(await page(2,2)),...(await page(2,4))],expected,sort); + } + } finally { await client.end(); } +}); + +test('the public sites loader carries linked attribution beside rated records and the response',async()=>{ + for(const rating of [0,42,null]) { + const lib=adNetworksLibrary(async sql=>{ + if(sql.includes('as classified'))return {rows:[{...coverage,rated:rating===null?0:1}]}; + if(sql.includes('count(*)::int as total'))return {rows:[{total:1}]}; + if(sql.includes('p.normalized_domain as domain'))return {rows:[{domain:'example.com',last_checked_at:null,evidence:{},manager_relationships:[],domain_rating:rating}]}; + return {rows:[]}; + }); + const result=await lib.loadAdNetworkSites(lib.defaultAdNetworkSitesQuery()); + if(rating===null) {assert.equal(Object.hasOwn(result,'attribution'),false);assert.equal(Object.hasOwn(result.rows[0],'attribution'),false);} + else {assert.deepEqual(result.attribution,AHREFS_ATTRIBUTION);assert.deepEqual(result.rows[0].attribution,AHREFS_ATTRIBUTION);} + } +}); diff --git a/tests/pages/ad-networks/domain-rating.test.mjs b/tests/pages/ad-networks/domain-rating.test.mjs new file mode 100644 index 00000000..588e9ebe --- /dev/null +++ b/tests/pages/ad-networks/domain-rating.test.mjs @@ -0,0 +1,169 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { openLocalDatabase } from '../../../scripts/ad-networks/local-db.mjs'; +import { collectDomainRatings, domainRatingSites } from '../../../scripts/ad-networks/domain-rating.mjs'; +import { ahrefsRequester } from '../../../scripts/stack127/ahrefs-dr.mjs'; + +function fixture(t) { + const dir = mkdtempSync(join(tmpdir(), 's127-dr-')); + const db = openLocalDatabase(join(dir, 'test.sqlite')); + t.after(() => { db.close(); rmSync(dir, { recursive: true, force: true }); }); + const network = db.prepare('select id from ad_networks limit 1').get().id; + function site(id, status, manager = false, checked = null, value = null) { + db.prepare(`insert into publishers(id,normalized_domain,display_domain,first_seen_at,last_discovered_at,created_at,updated_at,domain_rating_checked_on,domain_rating) + values(?,?,?,datetime('now'),datetime('now'),datetime('now'),datetime('now'),?,?)`).run(id, `site${id}.example`, `site${id}.example`, checked, value); + if (status) db.prepare(`insert into relationships(ad_network_id,publisher_id,status,relationship_type,evidence,created_at,updated_at) + values(?,?,?,?, '{}',datetime('now'),datetime('now'))`).run(network, id, status, manager ? 'primary_or_exclusive_manager' : 'active_network_use'); + } + return { db, site }; +} + +test('DR selection prioritizes listed managers, other listed, remaining never checked, then oldest due', t => { + const { db, site } = fixture(t); + site(1, 'ended', true); site(2, 'stale'); site(3, 'stale', true); site(4, 'confirmed', true); + site(5, 'confirmed', true, '2026-10-14', 10); site(6, null, false, '2026-09-01', 0); + site(7, 'confirmed', true, '2026-10-15', 100); // Exactly 30 days old is not due yet. + site(8, null); site(9, 'needs_review', true); + assert.deepEqual(domainRatingSites(db, { now: Date.parse('2026-11-14T12:00:00Z'), limit: 50 }).map(r => r.id), [3,4,2,1,8,9,6,5]); + assert.deepEqual(domainRatingSites(db, { now: Date.parse('2026-11-14T12:00:00Z'), limit: 2 }).map(r => r.id), [3,4]); + for (const value of [-1,101,1.5,'invalid']) assert.throws(() => db.prepare('update publishers set domain_rating=? where id=1').run(value)); +}); + +test('failures keep prior DR/date, defer retry, and never block other domains; successes resume idempotently', async t => { + const { db, site } = fixture(t); + site(1, 'confirmed', true, '2020-01-01', 40); site(2, 'confirmed', true); + let calls = 0; + const result = await collectDomainRatings(db, { limit: 2 }, { requestRating: async domain => { calls++; return domain === 'site1.example' ? { error: 'unavailable' } : { value: 0 }; } }); + assert.equal(result.status, 'failed'); assert.equal(result.measured, 1); assert.equal(result.failed, 1); + assert.deepEqual({...db.prepare('select domain_rating,domain_rating_checked_on from publishers where id=1').get()}, {domain_rating:40,domain_rating_checked_on:'2020-01-01'}); + assert.equal(db.prepare('select domain_rating from publishers where id=2').get().domain_rating, 0); + assert.ok(db.prepare("select retry_at from enrichment_state where publisher_id=1 and kind='domain-rating'").get().retry_at); + await collectDomainRatings(db, { limit: 2 }, { requestRating: async () => assert.fail('already measured or retry deferred') }); + assert.equal(calls, 2); +}); + +for (const status of [401,403,429]) test(`HTTP ${status} stops the run, keeps all values and does not start another domain`, async t => { + const { db, site } = fixture(t); site(1, 'confirmed', true); site(2, 'confirmed', true); + let time = 1, calls = 0; + const requestRating = ahrefsRequester({apiKey:'test-only', now:()=>time, sleepImplementation:async ms=>{time+=ms;}, + fetchImplementation:async()=>{calls++;return new Response('', {status,headers:{'Retry-After':'40'}});}}); + const result = await collectDomainRatings(db, {limit:2}, {requestRating}); + assert.equal(result.status, 'failed'); assert.equal(result.measured, 0); assert.equal(calls, status===429?3:1); + assert.ok(db.prepare('select * from publishers').all().every(r=>r.domain_rating===null&&r.domain_rating_checked_on===null)); +}); + +test('shared requester enforces the minimum pace, honors slower configuration and cancels a backoff', async () => { + assert.throws(()=>ahrefsRequester({apiKey:'test',paceMs:2999}), /3000/); + let time=1; + const times=[]; + const request=ahrefsRequester({apiKey:'test',paceMs:5000,now:()=>time,sleepImplementation:async ms=>{time+=ms;},fetchImplementation:async()=>{ + times.push(time);return new Response(JSON.stringify({domain_rating:{domain_rating:42.6}})); + }}); + assert.deepEqual(await request('a.example'),{value:43});await request('b.example');assert.deepEqual(times,[1,5001]); + const controller=new AbortController(); let calls=0; + const cancelled=ahrefsRequester({apiKey:'test',signal:controller.signal,fetchImplementation:async()=>{ + calls++;return new Response('',{status:429}); + },sleepImplementation:async()=>{controller.abort();}}); + await assert.rejects(cancelled('a.example'));assert.equal(calls,1); +}); + +test('run budget checkpoints partial work and prevents the next request', async t => { + const {db,site}=fixture(t);site(1,'confirmed');site(2,'confirmed'); + let time=Date.now(),calls=0; + const result=await collectDomainRatings(db,{limit:2,'time-budget-minutes':1},{now:()=>time,requestRating:async()=>{calls++;time+=61000;return {value:20};}}); + assert.equal(calls,1);assert.equal(result.status,'budget_complete'); + assert.equal(domainRatingSites(db).length,1); +}); + + +test('a long Retry-After survives the run and blocks all later domains/runs until it expires', async t => { + const {db,site}=fixture(t);site(1,'confirmed');site(2,'confirmed'); + const previous=process.env.AHREFS_API_KEY;process.env.AHREFS_API_KEY='test-only'; + t.after(()=>{if(previous===undefined)delete process.env.AHREFS_API_KEY;else process.env.AHREFS_API_KEY=previous;}); + let time=Date.now(),calls=0; + const first=await collectDomainRatings(db,{limit:2},{now:()=>time,fetchImplementation:async()=>{ + calls++;return new Response('',{status:429,headers:{'Retry-After':'86400'}}); + }}); + assert.equal(first.stopped,'rate_limited');assert.equal(calls,1); + assert.ok(db.prepare("select value from schema_meta where key='domain_rating_retry_at'").get().value); + const second=await collectDomainRatings(db,{limit:2},{now:()=>time+3600000,requestRating:async()=>assert.fail('Retry-After not reached')}); + assert.equal(second.stopped,'rate_limited');assert.equal(second.processed,0); + time+=86400001; + const third=await collectDomainRatings(db,{limit:1},{now:()=>time,requestRating:async()=>({value:50})}); + assert.equal(third.measured,1); +}); + +test('November 1 ad-network Retry-After blocks November 2 launch metrics on the shared host state', async t => { + const { db, site } = fixture(t); site(1, 'confirmed', true); + const dir = mkdtempSync(join(tmpdir(), 's127-ahrefs-shared-')); + t.after(() => rmSync(dir, {recursive:true,force:true})); + const stateFile = join(dir, 'ahrefs.sqlite'); + const previous = process.env.AHREFS_API_KEY; process.env.AHREFS_API_KEY = 'test-only'; + t.after(() => {if(previous===undefined)delete process.env.AHREFS_API_KEY;else process.env.AHREFS_API_KEY=previous;}); + const firstAt = Date.parse('2026-11-01T23:05:00Z'); + let calls = 0; + const adNetworks = await collectDomainRatings(db, {limit:1}, {stateFile, now:()=>firstAt, + fetchImplementation:async()=>{calls++; return new Response('',{status:429,headers:{'Retry-After':'86400'}});}}); + assert.equal(adNetworks.stopped,'rate_limited'); assert.equal(calls,1); + const {collectMetrics} = await import('../../../scripts/launch-directories/refresh-metrics.mjs'); + const records = [{slug:'example',website_url:'https://example.com',domain_rating:20,domain_rating_checked_on:'2026-10-02'}]; + const launch = await collectMetrics(records, {checkedOn:'2026-11-02', + loadTranco:async()=>({list_id:'test',ranks:new Map()}), + requestRating:ahrefsRequester({apiKey:'test-only',stateFile,now:()=>Date.parse('2026-11-02T06:00:00Z'), + fetchImplementation:async()=>assert.fail('24-hour cooldown must block the other job'), + sleepImplementation:async()=>assert.fail('long cooldown must stop promptly')})}); + assert.equal(launch.stopped,'rate_limited'); + assert.ok(launch.patches.every(patch=>!Object.hasOwn(patch.changes,'domain_rating'))); + assert.equal(launch.patches[0].changes.traffic_bucket,'below_1m','Tranco still runs during an Ahrefs cooldown'); + const after = ahrefsRequester({apiKey:'test-only',stateFile,now:()=>firstAt+86400001, + fetchImplementation:async()=>new Response(JSON.stringify({domain_rating:{domain_rating:30}}))}); + assert.deepEqual(await after('example.com'),{value:30}); +}); + +test('independent requesters serialize in-flight HTTP and retain post-response pacing across restarts', async t => { + const dir=mkdtempSync(join(tmpdir(),'s127-ahrefs-lock-')); + t.after(()=>rmSync(dir,{recursive:true,force:true})); + const stateFile=join(dir,'ahrefs.sqlite'); + let time=1000, release, entered; + const ready=new Promise(resolve=>{entered=resolve;}); + const hold=new Promise(resolve=>{release=resolve;}); + const first=ahrefsRequester({apiKey:'test',stateFile,now:()=>time,fetchImplementation:async()=>{ + entered(); await hold; time=2000; return new Response(JSON.stringify({domain_rating:{domain_rating:10}})); + }}); + const pending=first('first.example'); await ready; + let secondCalls=0; + const waits=[]; + const options={apiKey:'test',stateFile,now:()=>time,sleepImplementation:async ms=>{waits.push(ms);time+=ms;}, + fetchImplementation:async()=>{secondCalls++;return new Response(JSON.stringify({domain_rating:{domain_rating:20}}));}}; + const second=ahrefsRequester(options)('second.example'); + assert.equal(secondCalls,0); release(); + assert.deepEqual(await pending,{value:10}); assert.deepEqual(await second,{value:20}); + assert.deepEqual(waits,[3000]); + await ahrefsRequester(options)('third.example'); + assert.deepEqual(waits,[3000,3000]); +}); + +test('a killed process releases the shared request lock for the next job', {timeout:10000}, async t => { + const {spawn}=await import('node:child_process'); + const {once}=await import('node:events'); + const dir=mkdtempSync(join(tmpdir(),'s127-ahrefs-crash-')); + t.after(()=>rmSync(dir,{recursive:true,force:true})); + const stateFile=join(dir,'ahrefs.sqlite'); + const moduleUrl=new URL('../../../scripts/stack127/ahrefs-dr.mjs',import.meta.url).href; + const child=spawn(process.execPath,['--input-type=module','-e',` + import {ahrefsRequester} from ${JSON.stringify(moduleUrl)}; + const request=ahrefsRequester({apiKey:'test-only',stateFile:process.argv[1],fetchImplementation:async()=>{ + process.stdout.write('locked'); await new Promise(resolve=>setTimeout(resolve,60000)); + }}); + await request('example.com'); + `,stateFile],{stdio:['ignore','pipe','pipe']}); + t.after(()=>child.kill('SIGKILL')); + await Promise.race([once(child.stdout,'data'),once(child,'exit').then(()=>assert.fail('child exited before acquiring lock'))]); + const exit=once(child,'exit'); child.kill('SIGKILL'); await exit; + const request=ahrefsRequester({apiKey:'test-only',stateFile, + fetchImplementation:async()=>new Response(JSON.stringify({domain_rating:{domain_rating:15}}))}); + assert.deepEqual(await request('next.example'),{value:15}); +}); diff --git a/tests/pages/ad-networks/incremental-sync.test.mjs b/tests/pages/ad-networks/incremental-sync.test.mjs index d9e442fe..9666fcba 100644 --- a/tests/pages/ad-networks/incremental-sync.test.mjs +++ b/tests/pages/ad-networks/incremental-sync.test.mjs @@ -310,3 +310,16 @@ test('compact attribution preserves the real weekly chart, and cleanup bounds pr progress=await clean(); assert.equal(Number(progress[0].publishers_scanned),0);assert.equal(Number(progress[1].relationships_scanned),0); }); + + +test('DR sync preserves zero/date, sends only changed rows, and old snapshots omit DR', async t => { + const f=syncFixture(t); + await f.run();f.payloads.publishers.length=0; + f.db.prepare("update publishers set domain_rating=0,domain_rating_checked_on='2026-11-14' where id=1").run(); + let result=await f.run();assert.equal(result.publishers_sent,1); + assert.equal(f.payloads.publishers[0].domain_rating,0); + assert.equal(f.payloads.publishers[0].domain_rating_checked_on,'2026-11-14'); + result=await f.run();assert.equal(result.publishers_sent,0); + assert.ok(!Object.hasOwn(mapSyncPublisher({normalized_domain:'old.example'}),'domain_rating')); + assert.ok(!Object.hasOwn(mapSyncPublisher({normalized_domain:'old.example'}),'domain_rating_checked_on')); +}); diff --git a/tests/pages/ad-networks/loader-harness.mjs b/tests/pages/ad-networks/loader-harness.mjs index dc1f2927..aa93f716 100644 --- a/tests/pages/ad-networks/loader-harness.mjs +++ b/tests/pages/ad-networks/loader-harness.mjs @@ -1,3 +1,4 @@ +import * as ahrefs from "../../../lib/ahrefs.ts"; import fs from 'node:fs'; import vm from 'node:vm'; import ts from 'typescript'; @@ -11,6 +12,7 @@ import * as tldts from 'tldts'; export function adNetworksLibrary(query, overrides={}, globals={}) { const exports={}; const imports={ + './ahrefs':ahrefs, './db':{getPool:()=>({query})}, 'next/cache':{unstable_cache:fn=>fn}, './ad-network-niches':niches,'./ad-networks-report':report,'./ad-network-site-facts.mjs':facts, diff --git a/tests/pages/ad-networks/publishers.test.mjs b/tests/pages/ad-networks/publishers.test.mjs index 1e61a3e5..f9c99903 100644 --- a/tests/pages/ad-networks/publishers.test.mjs +++ b/tests/pages/ad-networks/publishers.test.mjs @@ -1,3 +1,4 @@ +import * as ahrefs from "../../../lib/ahrefs.ts"; import assert from "node:assert/strict"; import test from "node:test"; import { loadTypeScript } from "../../helpers/load-typescript.mjs"; @@ -11,6 +12,7 @@ import * as tldts from "tldts"; function library(rows, fail = false) { const calls = []; const exports = loadTypeScript(new URL("../../../lib/ad-networks.ts", import.meta.url), { + "./ahrefs":ahrefs, "./db": { getPool: () => ({ query: async (sql, args) => { calls.push({ sql, args }); diff --git a/tests/pages/ad-networks/refresh.test.mjs b/tests/pages/ad-networks/refresh.test.mjs index 4e2f09b4..3044dfdc 100644 --- a/tests/pages/ad-networks/refresh.test.mjs +++ b/tests/pages/ad-networks/refresh.test.mjs @@ -190,3 +190,20 @@ test('sync contention preserves collected checkpoints and retries skip the colle assert.equal((await refresh({ ...options, 'sync-only': true }, deps)).skipped, true); assert.deepEqual(calls, ['sync']); }); + + +test('DR job forwards a safe pace and publishes partial checkpoints once after a collector failure', async t => { + const state = await mkdtemp(join(tmpdir(), 's127-dr-refresh-')); + t.after(()=>rm(state,{recursive:true,force:true})); + const options=refreshArgs(['--job','domain-rating','--pace-ms','5000','--state-directory',state,'--apply']); + const command=refreshCommand(options); + assert.equal(command.script,'domain-rating'); + const production=refreshCommand({...options,environment:'production'}); + assert.equal(production.args[production.args.indexOf('--ahrefs-state-file')+1],'/srv/stack127-data/state/ahrefs.sqlite'); + assert.notEqual(command.args[command.args.indexOf('--ahrefs-state-file')+1],'/srv/stack127-data/state/ahrefs.sqlite'); + assert.deepEqual(command.args.slice(command.args.indexOf('--pace-ms'),command.args.indexOf('--pace-ms')+2),['--pace-ms','5000']); + assert.throws(()=>refreshArgs(['--job','domain-rating','--pace-ms','2999']),/pace-ms/); + const calls=[]; + await assert.rejects(refresh(options,{run:async script=>{calls.push(script);if(script==='domain-rating')throw Error('authentication_failed');},alert:async()=>{}}),/authentication_failed/); + assert.deepEqual(calls,['domain-rating','sync','revalidate']); +}); diff --git a/tests/pages/ad-networks/report.test.mjs b/tests/pages/ad-networks/report.test.mjs index 9acb0c2b..a0e101d5 100644 --- a/tests/pages/ad-networks/report.test.mjs +++ b/tests/pages/ad-networks/report.test.mjs @@ -137,7 +137,7 @@ test('sites loader and parser default to 10, preserve explicit sizes, and bound assert.equal(result.effectiveSort,'domain');assert.equal(result.page,8);assert.equal(result.totalPages,8); assert.deepEqual(Array.from(result.availableSorts),['domain','checked']); const table=calls.find(c=>c.sql.includes('p.normalized_domain as domain')); - assert.match(table.sql,/order by p.normalized_domain limit \$4 offset \$5/); + assert.match(table.sql,/order by p.normalized_domain limit \$5 offset \$6/); assert.deepEqual(Array.from(table.args).slice(-2),[10,70]); const explicit=await loadAdNetworkSites({...query,page:99,pageSize:50}); assert.equal(explicit.pageSize,50);assert.equal(explicit.page,2);assert.equal(explicit.totalPages,2); @@ -160,13 +160,13 @@ test('default and explicit traffic sort use age, domain and id ties in both pagi } const order="case p.traffic_group when 'top-1k' then 0 when 'top-10k' then 1 when 'top-100k' then 2 when 'top-1m' then 3 when 'not-ranked' then 4 else 5 end,p.domain_created_on asc nulls last,p.normalized_domain,p.id"; for(const {sql} of tables) { - assert.ok(sql.includes(`order by ${order} limit $4 offset $5`)); + assert.ok(sql.includes(`order by ${order} limit $5 offset $6`)); assert.ok(sql.trimEnd().endsWith(`order by ${order}`)); } - assert.deepEqual(Array.from(tables[0].args),['mediavine',null,'',10,0]); - assert.deepEqual(Array.from(tables[1].args),['mediavine',null,'',10,10]); - assert.deepEqual(Array.from(tables[2].args),[null,'tech','',10,0]); - assert.deepEqual(Array.from(tables[3].args),[null,'tech','',10,10]); + assert.deepEqual(Array.from(tables[0].args),['mediavine',null,'',null,10,0]); + assert.deepEqual(Array.from(tables[1].args),['mediavine',null,'',null,10,10]); + assert.deepEqual(Array.from(tables[2].args),[null,'tech','',null,10,0]); + assert.deepEqual(Array.from(tables[3].args),[null,'tech','',null,10,10]); }); test('an all-network data event refreshes every network, niches and both APIs',()=>{ const targets=targetsForEvents([{entity_type:'ad_networks',slug:'all'}]); diff --git a/tests/pages/ad-networks/sites-sort-database.test.mjs b/tests/pages/ad-networks/sites-sort-database.test.mjs index c45a6ff7..5ffec8fa 100644 --- a/tests/pages/ad-networks/sites-sort-database.test.mjs +++ b/tests/pages/ad-networks/sites-sort-database.test.mjs @@ -14,7 +14,7 @@ test('traffic groups, oldest registrations, null dates and stable ties survive p return {rows:[]}; }); await library.loadAdNetworkSites(library.defaultAdNetworkSitesQuery('mediavine')); - const pagingOrder=tableSql.match(/select p.id from matches p order by (.*?) limit \$4 offset \$5/s)[1]; + const pagingOrder=tableSql.match(/select p.id from matches p order by (.*?) limit \$5 offset \$6/s)[1]; const resultOrder=tableSql.slice(tableSql.lastIndexOf('order by ')); const site=(id,normalized_domain,traffic_group,domain_created_on)=>({id,normalized_domain,traffic_group,domain_created_on}); const rows=[