From d7ebadf278577fa62b2b78c2e7b94f5409547894 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 8 Nov 2026 17:21:57 +0000 Subject: [PATCH] crawler: site starts and frontier pool are tunable; pool defaults to 3 (0.49.2) Two ceilings hit scaling toward 1M pages/h (2026-11-08): - Site starts ran max(2, concurrency/4) = 8 at once on dev2. A start is mostly politeness waits (robots, sitemaps, home page, probes, 1.5 s apart), so too few sites were active: 144k page jobs delayed, 0 ready. CRAWL_SITE_CONCURRENCY. - Each process opened up to 8 frontier connections; 11 processes held 70 idle, the pitcrawl role's cap, and the 12th could not connect, so its site starts hung. CRAWL_DATABASE_POOL, default 3. Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- services/crawler/README.md | 2 ++ services/crawler/src/config.ts | 8 ++++++++ services/crawler/src/queues.ts | 4 ++-- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 4442789..68bcd9c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pitworld", - "version": "0.49.1", + "version": "0.49.2", "mcpName": "world.pit/pit", "description": "pit.world MCP server: private web search and Moshpit name lookups for AI agents (stdio, no API key). Also a web app, API, CLI and TUI.", "license": "MIT", diff --git a/services/crawler/README.md b/services/crawler/README.md index 977e4b7..3e1a146 100644 --- a/services/crawler/README.md +++ b/services/crawler/README.md @@ -126,6 +126,8 @@ curl 127.0.0.1:8807/api/v1/corpus/stats | `CRAWL_RECRAWL_DAYS` | 30 | a site is read again this often | | `CRAWL_DISCOVER` | 0 | 1: crawl linked sites too, up to `CRAWL_MAX_SITES` (100,000,000; at the limit links to new sites are dropped and /status says so) | | `CRAWL_PUBLISH_PAGES` | 1 | 0: pages go to the local index only, not nichedb /c/sites (dev2 since 2026-11-08: 10-15 GB a day in nichedb filled the disk). MCP records still publish | +| `CRAWL_SITE_CONCURRENCY` | concurrency/4, min 2 | site starts at once (robots, sitemaps, home page, probes: mostly politeness waits). dev2 runs 64 | +| `CRAWL_DATABASE_POOL` | 3 | Postgres connections per process for the site frontier; the pitcrawl role is capped, so keep processes x pool under it | | `CRAWL_SITE_QUEUE_HIGH` | 50000 | past this many waiting site jobs, new sites stay parked in Postgres and are fed in as the queue drains | | `CRAWL_DATABASE_URL` | (none) | Postgres for the site frontier: table `pitcrawl.sites` (known, parked, recrawl schedule). pit.world's own Supabase on dev2. Unset, the frontier is in memory (tests only). `pit-crawler migrate-sites [--drop]` moves an older Redis frontier in | | `CRAWL_MAX_PAGE_BACKLOG` | 50000 | site starts wait while more page jobs than this are queued | diff --git a/services/crawler/src/config.ts b/services/crawler/src/config.ts index 6f18bab..2fb0d2f 100644 --- a/services/crawler/src/config.ts +++ b/services/crawler/src/config.ts @@ -68,6 +68,10 @@ export type Config = { databaseUrl: string /** Past this many waiting site jobs, newly scheduled sites are parked compactly and fed in as the queue drains. */ siteQueueHigh: number + /** Site starts at once in a site-role process. Mostly waiting on politeness gaps, not CPU. */ + siteConcurrency: number + /** Postgres connections per process for the site frontier (the pitcrawl role is capped). */ + databasePool: number /** Records waiting for nichedb beyond this are dropped oldest first. */ outboxMax: number /** Page reads pause while more records than this wait to be published: crawl at the pace the index absorbs. */ @@ -127,6 +131,10 @@ export function configFromEnv(env: Record = process. maxSites: num(env.CRAWL_MAX_SITES, 100_000_000, 1), databaseUrl: env.CRAWL_DATABASE_URL ?? '', siteQueueHigh: num(env.CRAWL_SITE_QUEUE_HIGH, 50_000, 1), + siteConcurrency: num(env.CRAWL_SITE_CONCURRENCY, Math.max(2, Math.ceil(num(env.CRAWL_CONCURRENCY, 16, 1) / 4)), 1), + // 8 a process held 70 idle connections across 11 processes and the 12th could not + // connect, so its site starts hung (2026-11-08). Each process needs only a few. + databasePool: num(env.CRAWL_DATABASE_POOL, 3, 1), seenTtlMs: num(env.CRAWL_SEEN_TTL_HOURS, 72, 1) * 3_600_000, maxPageBacklog: num(env.CRAWL_MAX_PAGE_BACKLOG, 50_000, 1), outboxMax: num(env.CRAWL_OUTBOX_MAX, 200_000, 1), diff --git a/services/crawler/src/queues.ts b/services/crawler/src/queues.ts index 05c1d5a..e99dc52 100644 --- a/services/crawler/src/queues.ts +++ b/services/crawler/src/queues.ts @@ -142,7 +142,7 @@ export function createRuntime(config: Config, log: (line: string) => void = cons const pool = config.obscuraUrls.length ? new ObscuraPool(config.obscuraUrls, config.obscuraToken) : null const index = openIndex(config) if (!config.databaseUrl) log('CRAWL_DATABASE_URL unset: the site frontier is in memory and forgets every site on restart') - const sites: SiteStore = config.databaseUrl ? new PgSiteStore(config.databaseUrl) : new MemorySiteStore() + const sites: SiteStore = config.databaseUrl ? new PgSiteStore(config.databaseUrl, config.databasePool) : new MemorySiteStore() // Only a publishing process writes the archive; a fetch-only worker never opens it. const archive = config.archiveDir && config.roles.includes('publish') ? new PageArchive(config.archiveDir) : null const crawler = createCrawler({ @@ -241,7 +241,7 @@ export async function startWorkers(config: Config, log: (line: string) => void = if (!out.delayMs) log(`site ${job.data.site}: ${out.outcome}, ${out.admitted} pages queued`) return out }), - { ...common, connection: workerConnection(), concurrency: Math.max(2, Math.ceil(config.concurrency / 4)) }, + { ...common, connection: workerConnection(), concurrency: config.siteConcurrency }, ) workers.push(siteWorker) // Backpressure without losing places: a site job delayed for the page backlog came