From 2eefc575c2c03ed5d3e6293db65747e0df1fe8f2 Mon Sep 17 00:00:00 2001 From: RaviTejaKNTS Date: Sat, 14 Nov 2026 12:26:39 +0530 Subject: [PATCH 1/2] Keep the OVH job checkout current and stop discovery failing on one source - Add an hourly OVH updater that fast-forwards /srv/stack127 to main in a safe UTC window (minutes 15-28). It stages dependencies first, holds the job locks, rolls back in the same run on failure, and journals each phase so a hard kill recovers on the next run. Job wrappers and the cache worker refuse to run on an interrupted update. - Discovery warns on one isolated source failure and fails when every source fails, a source fails twice in a row, or local storage fails. Pause the Outbrain seller list, which has returned HTTP 406 since Nov 3. - Remove the two manual ad-network preview workflows; previews run on OVH. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ad-network-discovery.yml | 26 - .github/workflows/ad-network-verification.yml | 42 -- content_docs/ad-networks/maintenance.md | 23 +- content_docs/ad-networks/plan.md | 2 +- .../ad-networks/production-runbook.md | 17 +- .../launch-directories/production-runbook.md | 6 +- scripts/ad-networks/discover.mjs | 97 ++-- scripts/ad-networks/discovery-download.mjs | 68 +++ scripts/ad-networks/discovery-run.mjs | 29 + scripts/ad-networks/fetch-policy.mjs | 20 +- scripts/ad-networks/lib.mjs | 5 +- scripts/ad-networks/local-discovery-store.mjs | 2 + scripts/ad-networks/ovh/run-job.sh | 15 +- scripts/ad-networks/publisher-names.mjs | 4 +- scripts/ad-networks/sources.mjs | 11 +- scripts/ad-networks/storage-errors.mjs | 18 + scripts/launch-directories/ovh/run-job.sh | 15 +- scripts/stack127/cache/ovh/run-worker.sh | 9 +- scripts/stack127/checkout/README.md | 238 +++++++++ scripts/stack127/checkout/crontab.txt | 3 + scripts/stack127/checkout/update.sh | 330 ++++++++++++ stack127_docs/monitoring.json | 3 +- .../ad-networks/discovery-failures.test.mjs | 191 +++++++ tests/pages/ad-networks/discovery.test.mjs | 8 +- tests/stack127/checkout-update.test.mjs | 494 ++++++++++++++++++ 25 files changed, 1504 insertions(+), 172 deletions(-) delete mode 100644 .github/workflows/ad-network-discovery.yml delete mode 100644 .github/workflows/ad-network-verification.yml create mode 100644 scripts/ad-networks/discovery-download.mjs create mode 100644 scripts/ad-networks/discovery-run.mjs create mode 100644 scripts/ad-networks/storage-errors.mjs create mode 100644 scripts/stack127/checkout/README.md create mode 100644 scripts/stack127/checkout/crontab.txt create mode 100644 scripts/stack127/checkout/update.sh create mode 100644 tests/pages/ad-networks/discovery-failures.test.mjs create mode 100644 tests/stack127/checkout-update.test.mjs diff --git a/.github/workflows/ad-network-discovery.yml b/.github/workflows/ad-network-discovery.yml deleted file mode 100644 index 6b651fde5..000000000 --- a/.github/workflows/ad-network-discovery.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Preview ad network publisher discovery - -on: - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ad-network-discovery - cancel-in-progress: false - -jobs: - discover: - runs-on: ubuntu-latest - timeout-minutes: 90 - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: npm - - run: npm ci - - name: Preview candidates without database writes - run: npm run ad-networks:discover -- --store postgres --kind sellers-json diff --git a/.github/workflows/ad-network-verification.yml b/.github/workflows/ad-network-verification.yml deleted file mode 100644 index 00ce2a056..000000000 --- a/.github/workflows/ad-network-verification.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Preview ad network publisher verification - -on: - workflow_dispatch: - inputs: - limit: - description: Maximum domains to check without saving results - required: false - default: "100" - -permissions: - contents: read - -concurrency: - group: ad-network-verification - cancel-in-progress: false - -jobs: - verify: - runs-on: ubuntu-latest - timeout-minutes: 110 - env: - DATABASE_URL: ${{ secrets.DATABASE_POOLER_URL }} - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: npm - - run: npm ci - - name: Preview verification without database writes - env: - VERIFICATION_LIMIT: ${{ inputs.limit || '100' }} - run: npm run ad-networks:verify -- --store postgres --limit "$VERIFICATION_LIMIT" --concurrency 16 --time-budget-minutes 95 --lease-minutes 110 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: ad-network-verification-${{ github.run_id }} - path: tmp/ad-networks/verification-summary.json - if-no-files-found: warn - retention-days: 30 diff --git a/content_docs/ad-networks/maintenance.md b/content_docs/ad-networks/maintenance.md index 761c73188..2ebbe0314 100644 --- a/content_docs/ad-networks/maintenance.md +++ b/content_docs/ad-networks/maintenance.md @@ -2,7 +2,7 @@ ## Recurring production jobs (since 2026-10-19) -The jobs run as plain cron on the OVH server and write straight to the production database. Niche sorting runs on homelab with its logged-in Claude CLI and the reviewed Haiku alias; no model API key is needed. All ad-network pages stay unpublished and return 404. The [production runbook](production-runbook.md) covers paths, checks, updates, pausing and recovery. +The jobs run as plain cron on the OVH server and write straight to the production database. Niche sorting runs on homelab with its logged-in Claude CLI and the reviewed Haiku alias; no model API key is needed. All ad-network pages stay unpublished and return 404. The [production runbook](production-runbook.md) covers paths, checks, updates, pausing and recovery. The [hourly checkout updater](../../scripts/stack127/checkout/README.md) keeps OVH job code current. Manual updates and the first install run start only at UTC minutes 15–28, protecting scheduled jobs from losing their turn to updater locks. Set its hold before a maintenance window, wait for an active update, and remove your hold only after the window is complete. Keep the runbook's exact cron prefixes and cache timer steps. | Job | Schedule, UTC | Work | | --- | --- | --- | @@ -19,7 +19,7 @@ The jobs run as plain cron on the OVH server and write straight to the productio The crontab and its wrapper are [`scripts/ad-networks/ovh/`](../../scripts/ad-networks/ovh/). One session-level PostgreSQL advisory lock lets only one heavy job run at a time. A scheduled job that finds it taken retries every minute for up to 6 hours, and verification steps aside while one waits. `flock` stops the same job from starting twice. Site-based enrichment stops admitting work after 30 minutes or 5,000 sites. Source-feed jobs need a complete feed and have an outer timeout. Existing host limits remain authoritative. -`node --experimental-strip-types scripts/ad-networks/refresh.mjs --job --environment development` prints the full collector, sync, and revalidation plan without changing anything. `--apply` runs it. Production additionally requires `--environment production --confirm stack127-production`. Every scheduled run ends with a consistent reviewed sync and an authenticated on-demand origin revalidation. Completed checkpoints are still synced when a collector fails, and the run still counts as failed. A failed sync or revalidation never reports success. Snapshots and backups also use this final pipeline. Normal operation has no separate human sync step. +`node --experimental-strip-types scripts/ad-networks/refresh.mjs --job --environment development` prints the full collector, sync, and revalidation plan without changing anything. `--apply` runs it. Production additionally requires `--environment production --confirm stack127-production`. Every scheduled run ends with a consistent reviewed sync and an authenticated on-demand origin revalidation. Completed checkpoints are still synced when a collector fails. Discovery logs one isolated source failure as a warning and succeeds if another source completed; all sources failing or the same source failing on consecutive runs fails the job. Source status persists in SQLite `source_state` (or the existing JSON state file for PostgreSQL discovery). A successful HTTP 304 clears failure status while preserving feed metadata. Database write failures always fail. Other collector failures still fail the run. A failed sync or revalidation never reports success. Snapshots and backups also use this final pipeline. Normal operation has no separate human sync step. Sync's database triggers already queue `ad_networks:all` and affected page changes. The runner's extra revalidation immediately expires the origin data/page caches. It deliberately does not delete queued events. The existing Supabase cache worker verifies/purges the legacy host, or dispatches the Cloudflare static frontend refresh, before acknowledging events. In Cloudflare mode the public files update after that workflow completes. No collector publishes routes. @@ -47,13 +47,22 @@ Job failures reuse the existing `GITHUB_ALERT_TOKEN` and the repository's GitHub Keep the one-time homelab RDAP pass and gather supervisor running. When both finish, follow "Merge the homelab RDAP pass" in the runbook: `merge-rdap.mjs`, run through `refresh --job rdap-import`, merges only newer RDAP checkpoints and registration dates by domain into OVH's store, then syncs and revalidates. It never replaces the working SQLite file, niches, site checks, sources, or relationships. -## GitHub preview workflows +### Outbrain seller-feed access -The repository discovery and verification workflows are manual-only previews. Neither has a schedule or passes `--apply`. Discovery needs no database credential in preview mode. Verification reads candidates from PostgreSQL and writes only its summary artifact, without saving database results. Its default preview batch is 100 domains. +On November 14, 2026, plain GETs to `https://www.outbrain.com/sellers.json` returned HTTP 406 with an empty body from both OVH and the Mac. From OVH, `application/json`, wildcard, broader JSON/plain-text and full HTML Accept values all failed with the unchanged `stack127bot/1.0 (+https://stack127.com/ad-networks)` identity. Compression and language negotiation also failed. The response included `X-Served-By: cache-lcy-…`, `Cache-Control: private, max-age=600`, and `Content-Length: 0`. -The verifier rejects `--store postgres --apply`. Apply verification to the SQLite working store, then copy reviewed results through `sync`; this keeps one verification write path. PostgreSQL previews still load protected review signals. +This is an unresolved upstream refusal; the response does not identify its exact rule. It is not proven to be an Accept-only problem, unlike the homepage cases in PR #126. Discovery (including curl downloads) and publisher-name requests now consistently accept JSON, plain text and wildcard content while keeping the bot identity. This does not establish that Outbrain is fixed. The source `sellers-json:teads-outbrain` is paused in `sources.mjs` (since November 14, 2026), so the weekly job stops failing on it. Its last good run on October 27 read 974 domains and added no new ones. To try it again, run `npm run ad-networks:discover -- --kind sellers-json` with the source re-enabled in a reviewed change; do not impersonate a browser. Existing SQLite `last_status='failed'` also counts as the preceding failed run under the new policy. -These workflows do not change the independent homelab workers described below. Keep production pages unpublished and draft tables private. Before enabling a production sync or schedule, review the queue import, permissions, identity rules, sample results, and limits. Publication is a separate registry decision. +## Manual previews on OVH + +The manual GitHub discovery and verification workflows have been removed. Use the OVH checkout for previews, without `--apply`. First set the [checkout hold](../../scripts/stack127/checkout/README.md#maintenance-hold-and-manual-commands) and wait for an active updater. Remove your hold after the preview. Discovery needs no database credential. Verification reads the protected candidates and writes a local summary, without saving database results. + +```sh +ssh ovh 'cd /srv/stack127 && npm run ad-networks:discover -- --store postgres --kind sellers-json' +ssh ovh 'cd /srv/stack127 && set -a && . /srv/stack127-data/ad-networks.env && set +a && npm run ad-networks:verify -- --store postgres --limit 100 --concurrency 4 --time-budget-minutes 10' +``` + +The verifier rejects `--store postgres --apply`. Apply verification to the SQLite working store, then copy reviewed results through `sync`; this keeps one verification write path. PostgreSQL previews still load protected review signals. Publication is a separate registry decision. ## Phase 2 one-time import @@ -143,7 +152,7 @@ The default is a dry run. `--apply` is required for database writes. `--source` HTTP ETag, Last-Modified, candidate counts, and source failures are stored in SQLite. A 304 response skips an unchanged source. Source count drops greater than 50% cause a failure for review; disappearance from a source never deletes a candidate. The old PostgreSQL path remains available with `--store postgres` for a later controlled sync. -Check the workflow log for failed sources, unusually low domain counts, and changes in redirects or robots rules. When a source URL changes, update the manifest and this guide, run its dry run, then apply that source in development. Keep `source_key` stable across URL changes so existing provenance remains idempotent. Before a new source is scheduled, record its exact URL, extraction method, permitted use, robots result, and review date. +Check the OVH discovery log for failed sources, unusually low domain counts, and changes in redirects or robots rules. When a source URL changes, update the manifest and this guide, run its dry run, then apply that source in development. Keep `source_key` stable across URL changes so existing provenance remains idempotent. Before a new source is scheduled, record its exact URL, extraction method, permitted use, robots result, and review date. ## Verify publisher relationships diff --git a/content_docs/ad-networks/plan.md b/content_docs/ad-networks/plan.md index 754d44670..eb15699ef 100644 --- a/content_docs/ad-networks/plan.md +++ b/content_docs/ad-networks/plan.md @@ -143,7 +143,7 @@ This phase establishes the page and database contract. It does not assign a netw Database history is reconciled through the cleanup release. Keep the canonical versions in both managed projects. The old development ledger gaps and SQL experiments are historical context, not pending migrations. -The GitHub discovery and verification workflows remain manual-only, without `--apply`. They preview candidates and checks without saving database changes. Keep their production schedules disabled until a separate activation review. The independent homelab SQLite workers remain separate. +The manual GitHub discovery and verification workflows have been removed. Run previews without `--apply` in the OVH checkout, using the hold and commands in [the maintenance guide](maintenance.md#manual-previews-on-ovh). Recurring production jobs run on OVH. ### 2. Gather the initial domain list diff --git a/content_docs/ad-networks/production-runbook.md b/content_docs/ad-networks/production-runbook.md index aa3ccb14e..3801228f2 100644 --- a/content_docs/ad-networks/production-runbook.md +++ b/content_docs/ad-networks/production-runbook.md @@ -18,6 +18,8 @@ On OVH (`ssh ovh`, user `ubuntu`, UTC): The env file also sets `GITHUB_ALERT_TOKEN`, `AD_NETWORK_DB_PATH`, and `AD_NETWORKS_STATE_DIRECTORY`. Never print it. Homelab owns env values; Mac copies must not be edited. +The shared checkout updates at minute 25 each hour through the [installed updater](../../scripts/stack127/checkout/README.md). Cron and manual updates start only at UTC minutes 15–28, so the ten-minute timeout plus 30-second kill period releases job locks before :40. It skips maintenance holds and active jobs, uses fast-forward only, and installs dependencies only when needed. Never pull or install packages underneath a running job. + The wrapper is [`run-job.sh`](../../scripts/ad-networks/ovh/run-job.sh). The full schedule is [`crontab.txt`](../../scripts/ad-networks/ovh/crontab.txt). Homelab's `stack127-ad-network-niche.timer` runs daily at 06:00 UTC (11:30 IST), with up to 3,000 sites per run. ## Counts in titles and headings @@ -67,13 +69,13 @@ Ordinary vacuum makes dead space reusable; it does not generally shrink files on The coordinator performs these steps after review and release authorization. Keep cron removed and pause homelab's niche timer while doing the first sync. 1. Confirm the latest database backup and SQLite backup are usable. Wait for any old ad-network processes to stop. Keep the existing OVH working store. -2. Update OVH to the reviewed release: +2. Let the [hourly checkout updater](../../scripts/stack127/checkout/README.md) reach the reviewed release (install it with that guide first if needed). Run it by hand only at UTC minutes 15–28, before opening a maintenance window. The first install run has the same time limit: ```bash - ssh ovh 'cd /srv/stack127 && git pull --ff-only && npm ci --omit=dev --ignore-scripts && git rev-parse HEAD' + ssh ovh '/bin/bash /srv/stack127-data/checkout/run-job.sh stack127-checkout-update && git -C /srv/stack127 rev-parse HEAD' ``` - Check that SHA against the released revision. Check the env target privately: `AD_NETWORKS_RUNTIME_ENVIRONMENT=production`, loopback port 5432, and login `stack127_app`. Source it with `set -a; . /srv/stack127-data/ad-networks.env; set +a`. Do not use a parser that leaves surrounding quotes in values. + Exit 76 means outside UTC minutes 15–28, held or busy; wait for the next safe window and retry without bypassing locks. Check that SHA against the released revision. Set the checkout hold and wait for the updater using the linked guide before the manual steps below; remove your hold only after all release/maintenance work is complete. Check the env target privately: `AD_NETWORKS_RUNTIME_ENVIRONMENT=production`, loopback port 5432, and login `stack127_app`. Source it with `set -a; . /srv/stack127-data/ad-networks.env; set +a`. Do not use a parser that leaves surrounding quotes in values. 3. If the separate evidence cleanup is approved, run its SQL as the database administrator. Each pass selects at most 5,000 keys per table before projecting JSON. Temporary cursors advance in one database session, including through unchanged batches. Stop when both scanned counts are zero, even if earlier trimmed counts were zero. For example, on OVH from `/srv/stack127`: ```bash @@ -191,6 +193,11 @@ From the operator's local checkout, pause frontend dispatch and save the existin ```bash # Local shell. The implementation agent must not run these production mutations. export ADN_WINDOW=$(date -u +%Y%m%dT%H%M%SZ) +# Hold code first; refuse an existing hold. Wait for any update already in flight. +ssh ovh 'set -eu +install -d -m 755 /srv/stack127-data/state +(set -C; : > /srv/stack127-data/state/checkout-hold) +flock -w 660 /srv/stack127-data/checkout/updater.lock true' ssh ovh python3 - "$ADN_WINDOW" <<'PYTHON' import pathlib import subprocess @@ -214,7 +221,7 @@ ssh ovh 'sudo -n systemctl stop stack127-cache-worker.timer' ssh ovh 'while systemctl is-active --quiet stack127-cache-worker.service; do sleep 2; done' ``` -Coordinate other writers and releases for the same window. The authorized operator pauses homelab niche imports with `ssh homelab 'systemctl --user stop stack127-ad-network-niche.timer'` and waits until `ssh homelab 'systemctl --user is-active stack127-ad-network-niche.service'` reports inactive. Do not kill an active importer. These are release instructions, not permission for the implementation agent to operate on homelab. Wait for existing OVH collectors, syncs and retry wrappers to finish; inspect with: +Keep this hold through the complete window. Update code before setting it, using the updater only at UTC minutes 15–28; never bypass the start window while collectors are paused. It survives restarts, but does not pause data jobs or dispatch. If waiting for the updater fails, leave the hold and investigate before proceeding. Coordinate other writers and releases for the same window. The authorized operator pauses homelab niche imports with `ssh homelab 'systemctl --user stop stack127-ad-network-niche.timer'` and waits until `ssh homelab 'systemctl --user is-active stack127-ad-network-niche.service'` reports inactive. Do not kill an active importer. These are release instructions, not permission for the implementation agent to operate on homelab. Wait for existing OVH collectors, syncs and retry wrappers to finish; inspect with: ```bash ssh ovh 'ps -eo pid,etime,args | grep -E "[s]cripts/ad-networks/(ovh/run-job.sh|refresh.mjs|verify.mjs|sync.mjs|niche-import.mjs)"' @@ -389,6 +396,8 @@ PYTHON ssh ovh 'crontab -l' ``` +After the complete window and public acceptance pass, remove only the checkout hold created in step 1 with `ssh ovh 'rm /srv/stack127-data/state/checkout-hold'`. If no new window was opened, do not remove an existing hold belonging to another operator. The next minute-25 update catches up. + After the next hourly `:40` run finishes, inspect `ssh ovh 'tail -n 100 /srv/stack127-data/logs/verification.log'`. Require a `start verification` timestamp after `$ADN_RESUMED_AT` and its following `end verification exit 0`. An old exit line or an active crontab alone does not pass. If that next run fails, keep the release incomplete and investigate before claiming acceptance. Check the durable `ad_networks/all` queue event drains after a successful refresh and the next hourly verification exits successfully. Do not remove the backups until acceptance. The whole false-loss spike **will not disappear**: blocked checks, changed signals and historical misses without proof remain unresolved. Missing old snapshots mean unknown history, not zero losses. diff --git a/content_docs/launch-directories/production-runbook.md b/content_docs/launch-directories/production-runbook.md index 79a08bce6..0b6437051 100644 --- a/content_docs/launch-directories/production-runbook.md +++ b/content_docs/launch-directories/production-runbook.md @@ -30,7 +30,7 @@ PR CI can read production before the new columns exist. The fact projection uses Use the run ID printed for this exact commit, and require the workflow's deployed-image check to pass too. Stop if main advances or the release fails. Confirm both migrations `20261021120000` and `20261021120100` were applied. No local build. -2. **Check production schema access and update the OVH job checkout.** This query is read-only. The existing table-wide app grants cover added columns, and `stack127_backend_access` covers their rows. No new table is added, so rerunning `app-role.sql` is unnecessary. +2. **Check production schema access and the automatic OVH checkout update.** This query is read-only. The existing table-wide app grants cover added columns, and `stack127_backend_access` covers their rows. No new table is added, so rerunning `app-role.sql` is unnecessary. ```sh node --input-type=module <<'NODE' @@ -51,10 +51,10 @@ PR CI can read production before the new columns exist. The fact projection uses console.log('App role, RLS, new columns, triggers and 155 source records pass'); } finally {await pool.end();} NODE - ssh ovh 'cd /srv/stack127 && git pull --ff-only origin main && git rev-parse HEAD && node --input-type=module -e '\''await Promise.all([import("pg"),import("tldts")]); console.log("Dependencies present")'\''' + ssh ovh '/bin/bash /srv/stack127-data/checkout/run-job.sh stack127-checkout-update && git -C /srv/stack127 rev-parse HEAD' ``` - OVH's checkout must report the same released main SHA. Both dependencies are already production dependencies; no package install is added by this feature. Stop if its checkout is dirty or cannot fast-forward. The Node cache worker now reads the released cache map. Its systemd timer remains as configured. + Install the [shared checkout updater](../../scripts/stack127/checkout/README.md#install-after-review-and-merge) first if needed. It runs hourly at minute 25 and installs dependencies only when the lockfile changes or dependencies are missing. Manual runs, including the first install run, must start at UTC minutes 15–28. This leaves the ten-minute timeout and 30-second kill period clear of verification at :40 and the monthly 06:00 launch-directory start. Exit 76 means outside that window or a hold or job is active; wait for the next safe window and retry. Require the same released main SHA. Before the manual OVH work below, set the hold and wait for an active update using that guide. Remove your hold only after the work and public acceptance are complete. A dirty or diverged tree fails without resetting local edits. The cache worker keeps its existing timer and lock. 3. **Publish the 155 child pages and compare through the directory publisher.** Use the reviewed child manifest already in `tmp/page-publishing/launch-directories/manifest.json`. The identical `manifest-final.json` is a second frozen copy. New plans need unused filenames. diff --git a/scripts/ad-networks/discover.mjs b/scripts/ad-networks/discover.mjs index f19b43e2e..9cca5e0c9 100644 --- a/scripts/ad-networks/discover.mjs +++ b/scripts/ad-networks/discover.mjs @@ -1,10 +1,7 @@ #!/usr/bin/env node -import { createReadStream } from "node:fs"; -import { readFile, mkdir, mkdtemp, rename, rm, writeFile } from "node:fs/promises"; -import { spawn } from "node:child_process"; -import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import { readFile, mkdir, rename, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; import { parseArgs } from "node:util"; import pg from "pg"; import { @@ -14,6 +11,9 @@ import { publisherLinkDomains, sellerDomainsStream, } from "./lib.mjs"; +import { downloadToFile } from "./discovery-download.mjs"; +import { runDiscoverySources } from "./discovery-run.mjs"; +import { SELLERS_ACCEPT } from "./fetch-policy.mjs"; import { createLocalDiscoveryStore } from "./local-discovery-store.mjs"; import { defaultLocalDatabasePath, openLocalDatabase } from "./local-db.mjs"; import { selectSources, sources } from "./sources.mjs"; @@ -123,52 +123,11 @@ async function existingSourceDomains(pool, sourceKey) { return domains; } -async function downloadToFile(source, previous, { maxTimeSeconds, accept }) { - const directory = await mkdtemp(join(tmpdir(), "stack127-source-")); - const file = join(directory, "source-data"); - const headerFile = join(directory, "headers.txt"); - const args = [ - "--ipv4", "--http1.1", "--location", "--silent", "--show-error", "--fail", - "--retry", "4", "--retry-all-errors", "--connect-timeout", "20", - "--max-time", String(maxTimeSeconds), "--user-agent", "stack127bot/1.0 (+https://stack127.com/ad-networks)", - "--header", `Accept: ${accept}`, - "--dump-header", headerFile, "--output", file, "--write-out", "%{http_code} %{url_effective}", - ]; - if (apply && previous?.url === source.url) { - if (previous.etag && !/[\r\n]/.test(previous.etag)) args.push("--header", `If-None-Match: ${previous.etag}`); - if (previous.lastModified && !/[\r\n]/.test(previous.lastModified)) args.push("--header", `If-Modified-Since: ${previous.lastModified}`); - } - args.push(source.url); - try { - const result = await new Promise((resolveDownload, rejectDownload) => { - const child = spawn("curl", args, { stdio: ["ignore", "pipe", "pipe"] }); - let output = ""; - let errors = ""; - child.stdout.on("data", (chunk) => { output += chunk.toString(); }); - child.stderr.on("data", (chunk) => { errors = `${errors}${chunk.toString()}`.slice(-1000); }); - child.on("error", rejectDownload); - child.on("close", (code) => code === 0 ? resolveDownload(output.trim()) : rejectDownload(new Error(`${source.key} download failed: ${errors.trim() || `curl exit ${code}`}`))); - }); - const headerText = await readFile(headerFile, "utf8"); - const headerValue = (name) => [...headerText.matchAll(new RegExp(`^${name}:\\s*(.+)$`, "gim"))].at(-1)?.[1]?.trim() ?? null; - const [status, finalUrl] = result.split(" ", 2); - return { - status: Number(status), - url: finalUrl, - body: status === "304" ? null : createReadStream(file), - headers: { get: headerValue }, - cleanup: () => rm(directory, { recursive: true, force: true }), - }; - } catch (error) { - await rm(directory, { recursive: true, force: true }); - throw error; - } -} async function sourceResponse(source, previous) { if (!source.skipRobotsCheck) await checkRobots(source.url); if (source.kind === "majestic-csv") { - const response = await downloadToFile(source, previous, { maxTimeSeconds: 600, accept: "text/csv" }); + const response = await downloadToFile(source, previous, { maxTimeSeconds: 600, accept: "text/csv", apply }); if (response.url !== source.url) { try { await checkRobots(response.url); } catch (error) { await response.cleanup(); throw error; } @@ -177,8 +136,9 @@ async function sourceResponse(source, previous) { } if (source.transport === "curl-file") { const response = await downloadToFile(source, previous, { + apply, maxTimeSeconds: Math.ceil((source.timeoutMs ?? 180_000) / 1000), - accept: source.kind === "sellers-json" ? "application/json" : "text/html", + accept: source.kind === "sellers-json" ? SELLERS_ACCEPT : "text/html", }); if (response.url !== source.url) { try { await checkRobots(response.url); } @@ -187,7 +147,7 @@ async function sourceResponse(source, previous) { return response; } const accept = source.kind === "sellers-json" - ? "application/json" + ? SELLERS_ACCEPT : source.kind === "publisher-list-html" ? "text/html" : "text/csv"; @@ -216,15 +176,19 @@ async function processSource(source, storage, state) { const seenAt = new Date().toISOString(); const stats = { source: source.key, status: apply ? "applied" : "dry_run", candidates: 0, newDomains: 0, newSightings: 0 }; - const previouslyAttributed = source.kind === "majestic-csv" && storage - ? await storage.existingSourceDomains(source.key) - : null; + let previouslyAttributed = null; + if (source.kind === "majestic-csv" && storage) { + try { previouslyAttributed = await storage.existingSourceDomains(source.key); } + catch (error) { error.discoveryStorage = true; throw error; } + } const batch = []; const flush = async () => { if (!batch.length) return; if (storage) { const unique = [...new Map(batch.map((candidate) => [candidate.domain, candidate])).values()]; - const result = await storage.upsertBatch(source, unique, seenAt); + let result; + try { result = await storage.upsertBatch(source, unique, seenAt); } + catch (error) { error.discoveryStorage = true; throw error; } stats.newDomains += result.inserted; stats.newSightings += result.newSightings; } @@ -280,8 +244,6 @@ async function processSource(source, storage, state) { candidates: stats.candidates, completedAt: seenAt, }; - if (storage?.saveSourceState) storage.saveSourceState(source.key, state[source.key]); - else await saveState(state); } return stats; } @@ -302,20 +264,23 @@ const storage = localStore ?? (pool ? { existingSourceDomains: (sourceKey) => existingSourceDomains(pool, sourceKey), } : null); -let failures = 0; try { const state = apply ? (localStore ? localStore.readState() : await readState()) : {}; - for (const source of selected) { - try { - console.log(JSON.stringify(await processSource(source, storage, state))); - } catch (error) { - failures += 1; - localStore?.saveSourceFailure(source, error); - console.error(JSON.stringify({ source: source.key, status: "failed", error: error.message })); - } - } + process.exitCode = await runDiscoverySources({ + sources: selected, state, + processSource: source => processSource(source, storage, state), + saveSuccess: async (source, entry) => { + if (!apply) return; + if (localStore) localStore.saveSourceState(source.key, entry); + else await saveState(state); + }, + saveFailure: async (source, error) => { + if (!apply) return; + if (localStore) localStore.saveSourceFailure(source, error); + else await saveState(state); + }, + }); } finally { if (pool) await pool.end(); localDatabase?.close(); } -if (failures) process.exitCode = 1; diff --git a/scripts/ad-networks/discovery-download.mjs b/scripts/ad-networks/discovery-download.mjs new file mode 100644 index 000000000..092edfc4b --- /dev/null +++ b/scripts/ad-networks/discovery-download.mjs @@ -0,0 +1,68 @@ +import { createReadStream } from 'node:fs'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { spawn } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { storageError } from './storage-errors.mjs'; + +export async function downloadToFile(source, previous, { maxTimeSeconds, accept, apply = false }, dependencies = {}) { + const makeTemp = dependencies.mkdtemp ?? mkdtemp; + const read = dependencies.readFile ?? readFile; + const start = dependencies.spawn ?? spawn; + let directory; + try { directory = await makeTemp(join(tmpdir(), "stack127-source-")); } + catch (error) { throw storageError(error); } + const file = join(directory, "source-data"); + const headerFile = join(directory, "headers.txt"); + const args = [ + "--ipv4", "--http1.1", "--location", "--silent", "--show-error", "--fail", + "--retry", "4", "--retry-all-errors", "--connect-timeout", "20", + "--max-time", String(maxTimeSeconds), "--user-agent", "stack127bot/1.0 (+https://stack127.com/ad-networks)", + "--header", `Accept: ${accept}`, + "--dump-header", headerFile, "--output", file, "--write-out", "%{http_code} %{url_effective}", + ]; + if (apply && previous?.url === source.url) { + if (previous.etag && !/[\r\n]/.test(previous.etag)) args.push("--header", `If-None-Match: ${previous.etag}`); + if (previous.lastModified && !/[\r\n]/.test(previous.lastModified)) args.push("--header", `If-Modified-Since: ${previous.lastModified}`); + } + args.push(source.url); + try { + const result = await new Promise((resolveDownload, rejectDownload) => { + const child = start("curl", args, { stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; + let errors = ""; + child.stdout.on("data", (chunk) => { output += chunk.toString(); }); + child.stderr.on("data", (chunk) => { errors = `${errors}${chunk.toString()}`.slice(-1000); }); + child.on("error", rejectDownload); + child.on("close", (code) => { + if (code === 0) return resolveDownload(output.trim()); + const error = new Error(`${source.key} download failed: ${errors.trim() || `curl exit ${code}`}`); + // curl 23: local output/header write failed; 26: local input read failed. + rejectDownload([23, 26].includes(code) ? storageError(error) : error); + }); + }); + let headerText; + try { headerText = await read(headerFile, "utf8"); } + catch (error) { throw storageError(error); } + const headerValue = (name) => [...headerText.matchAll(new RegExp(`^${name}:\\s*(.+)$`, "gim"))].at(-1)?.[1]?.trim() ?? null; + const [status, finalUrl] = result.split(" ", 2); + const body = status === "304" ? null : createReadStream(file); + body?.on('error', error => { error.discoveryStorage = true; }); + const cleanup = async () => { + try { await rm(directory, { recursive: true, force: true }); } + catch (error) { throw storageError(error); } + }; + return { + status: Number(status), + url: finalUrl, + body, + headers: { get: headerValue }, + cleanup, + }; + } catch (error) { + try { await rm(directory, { recursive: true, force: true }); } + catch (cleanupError) { throw storageError(cleanupError); } + throw error; + } +} + diff --git a/scripts/ad-networks/discovery-run.mjs b/scripts/ad-networks/discovery-run.mjs new file mode 100644 index 000000000..2e280d416 --- /dev/null +++ b/scripts/ad-networks/discovery-run.mjs @@ -0,0 +1,29 @@ +import { isStorageError } from './storage-errors.mjs'; + +// One isolated upstream failure is a warning. A persistent failure must stay red. +// Storage failures are never downgraded to upstream warnings. +export async function runDiscoverySources({ sources, state, processSource, saveSuccess, saveFailure, log = console.log, warn = console.error }) { + let failures = 0, repeated = false; + for (const source of sources) { + const previous = state[source.key]; + let result; + try { + result = await processSource(source); + } catch (error) { + if (isStorageError(error)) throw error; + failures++; + const consecutive = previous?.lastStatus === 'failed'; + repeated ||= consecutive; + state[source.key] = { ...previous, url: source.url, lastStatus: 'failed', lastError: error.message }; + await saveFailure(source, error, state[source.key]); + warn(JSON.stringify({ source: source.key, status: 'failed', error: error.message, consecutiveFailure: consecutive })); + warn(`${consecutive ? 'ERROR' : 'WARNING'}: discovery source ${source.key} failed${consecutive ? ' on consecutive runs' : ' once; retry next run'}: ${error.message}`); + continue; + } + state[source.key] = { ...state[source.key], lastStatus: 'complete', lastError: null }; + await saveSuccess(source, state[source.key]); + log(JSON.stringify(result)); + } + if (failures === sources.length) warn('ERROR: every discovery source failed; no source completed'); + return failures === sources.length || repeated ? 1 : 0; +} diff --git a/scripts/ad-networks/fetch-policy.mjs b/scripts/ad-networks/fetch-policy.mjs index 5d25f4986..5efce9fc1 100644 --- a/scripts/ad-networks/fetch-policy.mjs +++ b/scripts/ad-networks/fetch-policy.mjs @@ -4,8 +4,11 @@ import dns from "node:dns/promises"; import { isIP } from "node:net"; import { getDomain } from "tldts"; import { sharedHostSlot } from './shared-fetch-slots.mjs'; +import { isStorageError, storageError } from './storage-errors.mjs'; export const USER_AGENT = "stack127bot/1.0 (+https://stack127.com/ad-networks)"; +// Seller feeds can be served as JSON, plain text, or generic binary content. +export const SELLERS_ACCEPT = "application/json, text/plain;q=0.9, */*;q=0.8"; const TRANSIENT_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]); const MAX_REDIRECTS = 5; const MAX_EXTERNAL_REDIRECTS = 1; @@ -91,12 +94,16 @@ async function requestOnce(url, options, dependencies) { const release = () => { if (released) return; released = true; try {sharedUnlock?.();} - catch(error){console.error(`Shared fetch lease release failed for ${host}: ${error.message}; its bounded lease will expire.`);} + catch(error){ + if (options.failOnStorageError) throw storageError(error); + console.error(`Shared fetch lease release failed for ${host}: ${error.message}; its bounded lease will expire.`); + } finally {setTimeout(() => { unlock(); if (hostSlots.get(host) === slot) hostSlots.delete(host); }, 1000);} }; await previous; try { - sharedUnlock=await dependencies.sharedHostSlot(host,{leaseMs:2*(options.timeoutMs??15000)+60000}); + try { sharedUnlock=await dependencies.sharedHostSlot(host,{leaseMs:2*(options.timeoutMs??15000)+60000}); } + catch (error) { throw storageError(error); } const response = await requestPublic(url, options, dependencies); if (!response.body) { release(); return response; } const reader = response.body.getReader(); @@ -104,8 +111,11 @@ async function requestOnce(url, options, dependencies) { async pull(controller) { try { const { done, value } = await reader.read(); - if (done) { controller.close(); release(); } else controller.enqueue(value); - } catch (error) { controller.error(error); release(); } + if (done) { release(); controller.close(); } else controller.enqueue(value); + } catch (error) { + try { release(); } catch (releaseError) { error = releaseError; } + controller.error(error); + } }, async cancel(reason) { try { await reader.cancel(reason); } finally { release(); } }, }); @@ -152,7 +162,7 @@ export async function safeFetch(value, options = {}, dependencies = {}) { try { response = await requestOnce(url, options, runtime); } catch (error) { - if (attempt + 1 >= (options.attempts ?? 3)) throw error; + if (isStorageError(error) || attempt + 1 >= (options.attempts ?? 3)) throw error; await sleep(Math.min(1000 * (2 ** attempt), 5000)); continue; } diff --git a/scripts/ad-networks/lib.mjs b/scripts/ad-networks/lib.mjs index b6a286461..97d8e4489 100644 --- a/scripts/ad-networks/lib.mjs +++ b/scripts/ad-networks/lib.mjs @@ -4,6 +4,7 @@ import { domainToASCII } from "node:url"; import { createInterface } from "node:readline"; import { Readable } from "node:stream"; import { assertPublicUrl, readLimitedText, safeFetch } from "./fetch-policy.mjs"; +import { isStorageError } from './storage-errors.mjs'; const userAgent = "stack127bot/1.0 (+https://stack127.com/ad-networks)"; dns.setDefaultResultOrder("ipv4first"); @@ -258,7 +259,7 @@ export async function fetchWithRetry(url, options = {}) { signal: AbortSignal.timeout(options.timeoutMs ?? 45_000), }); } catch (error) { - if (attempt === 2) throw new Error(`${url}: ${error.cause?.code ?? error.message}`, { cause: error }); + if (isStorageError(error) || attempt === 2) throw new Error(`${url}: ${error.cause?.code ?? error.message}`, { cause: error }); await sleep(1000 * (2 ** attempt)); continue; } @@ -275,7 +276,7 @@ export async function fetchWithRetry(url, options = {}) { export async function checkRobots(url, dependencies = {}) { const target = await assertPublicUrl(url, dependencies.lookup); const robotsUrl = `${target.origin}/robots.txt`; - const { response } = await safeFetch(robotsUrl, { timeoutMs: 20_000, headers: { Accept: "text/plain" } }, dependencies); + const { response } = await safeFetch(robotsUrl, { timeoutMs: 20_000, failOnStorageError: true, headers: { Accept: "text/plain" } }, dependencies); if (response.status === 404) { await response.body?.cancel(); return; } if (!response.ok) { await response.body?.cancel(); diff --git a/scripts/ad-networks/local-discovery-store.mjs b/scripts/ad-networks/local-discovery-store.mjs index 84942ce5c..e32b51ef1 100644 --- a/scripts/ad-networks/local-discovery-store.mjs +++ b/scripts/ad-networks/local-discovery-store.mjs @@ -96,6 +96,8 @@ export function createLocalDiscoveryStore(database) { lastModified: row.last_modified, candidates: row.candidate_count, completedAt: row.completed_at, + lastStatus: row.last_status, + lastError: row.last_error, }; } return state; diff --git a/scripts/ad-networks/ovh/run-job.sh b/scripts/ad-networks/ovh/run-job.sh index 3052c973e..780742ab6 100755 --- a/scripts/ad-networks/ovh/run-job.sh +++ b/scripts/ad-networks/ovh/run-job.sh @@ -4,6 +4,14 @@ # jobs retry for up to 6 hours, and verification steps aside while one waits. set -u job=$1; shift +# Fail visibly even after a hard-killed updater has released its OS locks. +if [ -e /srv/stack127-data/checkout/update-in-progress ]; then + exec >>"/srv/stack127-data/logs/$job.log" 2>&1 + echo "== $(date -u +%FT%TZ) start $job $*" + echo 'ERROR: checkout update in progress or interrupted' + echo "== $(date -u +%FT%TZ) end $job exit 1" + exit 1 +fi cd /srv/stack127 || exit 1 set -a; . /srv/stack127-data/ad-networks.env; set +a pending=/tmp/adn-pending-$job-$$ @@ -27,7 +35,12 @@ tries=0 sync_only= while :; do rc=0 - nice -n 10 flock -n -E 76 "/tmp/adn-$job.lock" node --no-warnings --experimental-strip-types scripts/ad-networks/refresh.mjs \ + nice -n 10 flock -n -E 76 "/tmp/adn-$job.lock" sh -c ' + if [ -e /srv/stack127-data/checkout/update-in-progress ]; then + echo "ERROR: checkout update in progress or interrupted"; exit 1 + fi + exec "$@" + ' sh node --no-warnings --experimental-strip-types scripts/ad-networks/refresh.mjs \ --job "$job" --environment production --confirm stack127-production --apply \ --local-database "$AD_NETWORK_DB_PATH" --state-directory "$AD_NETWORKS_STATE_DIRECTORY" $sync_only "$@" || rc=$? [ "$rc" = 76 ] && { echo "already running"; break; } diff --git a/scripts/ad-networks/publisher-names.mjs b/scripts/ad-networks/publisher-names.mjs index 7a32c5dc9..14830d5d2 100644 --- a/scripts/ad-networks/publisher-names.mjs +++ b/scripts/ad-networks/publisher-names.mjs @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'; import { sources } from './sources.mjs'; import { checkRobots, normalizeDomain, sellerDomainsStream } from './lib.mjs'; -import { safeFetch } from './fetch-policy.mjs'; +import { safeFetch, SELLERS_ACCEPT } from './fetch-policy.mjs'; import { businessName } from './enrichment-parsers.mjs'; import { enrichmentArgs,enrichmentDatabase,confirmedSites,progressFile,saveState } from './enrichment-store.mjs'; const startedAt=new Date().toISOString(); @@ -18,7 +18,7 @@ try { if(previous?.value===cohort && !args.refresh){results.push({source:source.key,status:'cached'});continue;} let rows=[]; try { - const request=await safeFetch(source.url,{timeoutMs:180000,beforeRequest:async url=>{if(!source.skipRobotsCheck)await checkRobots(url.href);}}); + const request=await safeFetch(source.url,{headers:{Accept:SELLERS_ACCEPT},timeoutMs:180000,beforeRequest:async url=>{if(!source.skipRobotsCheck)await checkRobots(url.href);}}); if(!request.response.ok){await request.response.body?.cancel();throw new Error(`HTTP ${request.response.status}`);} // Stream even the large Google feed. Retain business names only for confirmed domains. for await(const row of sellerDomainsStream(request.response.body,seller=>{ diff --git a/scripts/ad-networks/sources.mjs b/scripts/ad-networks/sources.mjs index c7eeda0d5..cce507aad 100644 --- a/scripts/ad-networks/sources.mjs +++ b/scripts/ad-networks/sources.mjs @@ -89,8 +89,10 @@ export const sources = [ robotsReviewedAt: "2026-09-28", verificationPriority: 80, normalRequestsPerRun: 2, - recurring: true, - pausedReason: null, + // Paused 2026-11-14: HTTP 406 to every honest request since 2026-11-03. Its + // last good run (2026-10-27) read 974 domains and added no new ones. + recurring: false, + pausedReason: "www.outbrain.com/sellers.json returns HTTP 406 since 2026-11-03", }, { key: "official-publishers:ethicalads", @@ -126,7 +128,8 @@ export function selectSources(keys = []) { const selected = sources.filter((source) => keys.includes(source.key) || keys.includes(source.networkSlug)); const unknown = keys.filter((key) => !sources.some((source) => source.key === key || source.networkSlug === key)); if (unknown.length) throw new Error(`Unknown source: ${unknown.join(", ")}`); - const held = selected.filter((source) => !source.recurring); + // A network name selects its active sources; a paused source named directly fails. + const held = selected.filter((source) => !source.recurring && (keys.includes(source.key) || selected.every((item) => !item.recurring))); if (held.length) throw new Error(`Source is paused: ${held.map((source) => `${source.key} (${source.pausedReason})`).join(", ")}`); - return selected; + return selected.filter((source) => source.recurring); } diff --git a/scripts/ad-networks/storage-errors.mjs b/scripts/ad-networks/storage-errors.mjs new file mode 100644 index 000000000..5ce359605 --- /dev/null +++ b/scripts/ad-networks/storage-errors.mjs @@ -0,0 +1,18 @@ +// Keep local infrastructure failures fatal through network/parser wrappers. +const filesystemCodes = new Set([ + 'ENOSPC', 'EDQUOT', 'EIO', 'EROFS', 'EACCES', 'EPERM', 'EMFILE', 'ENFILE', + 'ENOENT', 'ENOTDIR', 'EISDIR', 'EFBIG', 'EBADF', 'ENOMEM', +]); + +export function isStorageError(error, seen = new Set()) { + if (!error || typeof error !== 'object' || seen.has(error)) return false; + seen.add(error); + return Boolean(error.discoveryStorage || filesystemCodes.has(error.code) + || /^(?:ERR_)?SQLITE_/.test(error.code ?? '') + || isStorageError(error.cause, seen) + || (Array.isArray(error.errors) && error.errors.some(cause => isStorageError(cause, seen)))); +} + +export function storageError(error) { + return Object.assign(new Error(error.message, { cause: error }), { discoveryStorage: true }); +} diff --git a/scripts/launch-directories/ovh/run-job.sh b/scripts/launch-directories/ovh/run-job.sh index e5483c28d..3ffa27555 100755 --- a/scripts/launch-directories/ovh/run-job.sh +++ b/scripts/launch-directories/ovh/run-job.sh @@ -6,6 +6,14 @@ export PATH job=${1:-} [ "$job" = launch-directory-metrics ] || { echo "Usage: run-job.sh launch-directory-metrics [options]" >&2; exit 1; } shift +# Fail visibly even after a hard-killed updater has released its OS locks. +if [ -e /srv/stack127-data/checkout/update-in-progress ]; then + exec >>"/srv/stack127-data/logs/$job.log" 2>&1 + echo "== $(date -u +%FT%TZ) start $job $*" + echo 'ERROR: checkout update in progress or interrupted' + echo "== $(date -u +%FT%TZ) end $job exit 1" + exit 1 +fi cd /srv/stack127 job_env=/srv/stack127-data/launch-directory-metrics.env [ -f "$job_env" ] && [ "$(stat -c '%a' "$job_env")" = 600 ] && [ "$(stat -c '%u' "$job_env")" = "$(id -u)" ] || { @@ -15,7 +23,12 @@ exec >>"/srv/stack127-data/logs/$job.log" 2>&1 echo "== $(date -u +%FT%TZ) start $job $*" rc=0 nice -n 10 flock -n -E 76 /srv/stack127-data/launch-directory-metrics.lock \ - node --experimental-strip-types scripts/launch-directories/refresh-metrics.mjs \ + sh -c ' + if [ -e /srv/stack127-data/checkout/update-in-progress ]; then + echo "ERROR: checkout update in progress or interrupted"; exit 1 + fi + exec "$@" + ' sh node --experimental-strip-types scripts/launch-directories/refresh-metrics.mjs \ --env-file "$job_env" --environment production --confirm stack127-production --apply "$@" || rc=$? echo "== $(date -u +%FT%TZ) end $job exit $rc" exit "$rc" diff --git a/scripts/stack127/cache/ovh/run-worker.sh b/scripts/stack127/cache/ovh/run-worker.sh index 3e20a49cf..154903a29 100755 --- a/scripts/stack127/cache/ovh/run-worker.sh +++ b/scripts/stack127/cache/ovh/run-worker.sh @@ -15,5 +15,12 @@ fi cd /srv/stack127 # The lock is outside /tmp so systemd PrivateTmp and manual invocations share it. # Node reads dotenv directly; values are never evaluated as shell commands. +# Check the update journal only after taking the lock: a busy lock (also held by a +# healthy updater) is a quiet skip, and a journal left behind after that is a real failure. exec flock -n -E 0 /srv/stack127-data/cache-worker.lock \ - node --experimental-strip-types --env-file="$worker_env" scripts/stack127/cache/worker.mjs + sh -c ' + if [ -e /srv/stack127-data/checkout/update-in-progress ]; then + echo "ERROR: checkout update in progress or interrupted" >&2; exit 1 + fi + exec "$@" + ' sh node --experimental-strip-types --env-file="$worker_env" scripts/stack127/cache/worker.mjs diff --git a/scripts/stack127/checkout/README.md b/scripts/stack127/checkout/README.md new file mode 100644 index 000000000..753bf5690 --- /dev/null +++ b/scripts/stack127/checkout/README.md @@ -0,0 +1,238 @@ +# Keep the shared OVH job checkout current + +An hourly cron job fast-forwards `/srv/stack127` to `origin/main`. It runs as +`ubuntu` at minute 25 UTC, away from verification at minute 40. A missed or +skipped update is retried next hour, including after a restart. Both cron and +manual runs start only at UTC minutes **15–28**. The ten-minute timeout plus +30-second kill period then finishes before verification at :40, with at least +30 seconds to spare. Other shared-checkout cron starts are at :00 and 00:10. +This prevents the updater's non-blocking locks from costing a monthly or weekly +job its only scheduled turn. Recheck this window against OVH's crontab whenever +a shared-checkout schedule changes. Cron must be enabled. This updates job code; it does not deploy the website or publish data. +The separate self-host checkout and installed supervisor are unchanged. + +The installed updater lives outside Git at +`/srv/stack127-data/checkout/run-job.sh`. It takes its own OS lock, checks the +maintenance hold, live `/tmp/adn-pending-*` PIDs (including six-hour retry +wrappers), every known ad-network job lock and existing `/tmp/adn-*.lock`, the +launch-directory lock, the cache-worker lock, and readable `/proc/*/cwd` links +pointing into `/srv/stack127`. It rechecks the hold, PIDs and cwd after locking. +All locks stay held through Git and dependency installation. Keep the known job +list in `update.sh` current when adding a job; a new job may not have created its +lock yet. It never deletes job lock files or pending markers. + +The cache worker already takes `/srv/stack127-data/cache-worker.lock` before +starting Node. This path works with its systemd `PrivateTmp`. A worker that loses +the lock exits normally. If it sees the update journal, it fails visibly as +described below. Either way it retries on its next 30-second timer. No service +change or sudo is needed. There remains a small race +where systemd opens the checkout-owned shell wrapper while Git replaces it; +the lock prevents Node and its dependencies from running during the update. +An invocation failing during that shell-start race recovers on the next timer. +Already running wrappers and commands are caught by the cwd scan. Other users' +unreadable cwd links cannot be inspected without sudo. Manual operators must +use the hold; the scan is an extra guard, not a substitute. + +The updater refuses ordinary dirty files, untracked files, unpublished commits, +a branch other than `main`, or a non-fast-forward. It never stashes, rebases, +force-fetches, or resets an ordinary dirty tree. An atomic journal records the +phase, base and target before Git starts. Fetch records the current HEAD as both +base and target. It fetches only `origin/main`, without tags, pruning, +`FETCH_HEAD` writes or auto-maintenance. After an interrupted fetch, recovery +removes only `refs/remotes/origin/main.lock`; it never resets the index or +working tree. HEAD must still equal the recorded base. Any edits then fail the +ordinary clean-tree check and remain untouched. + +After fetch and ancestry checks, the updater prepares the target packages in +`/srv/stack127-data/checkout/staging/new`, using `package.json` and +`package-lock.json` read directly from the fetched commit. It runs +`npm ci --omit=dev --ignore-scripts --no-audit --no-fund` there when the lockfile +hash differs from the last successful install, or `node_modules` is absent. +The first run with no hash installs once to establish that baseline. A failed +staging install leaves live code, dependencies and the old success marker +unchanged. A code-only change does not reinstall packages. + +Staging and `/srv/stack127` must be on the **same filesystem**. The updater +checks this before touching live code. It rejects absolute dependency symlinks +and links escaping the staged `node_modules`, including local linked packages. +Relative `.bin` symlinks survive the rename. Install as `ubuntu`, the same user +as the cron jobs and cache worker; staged files must belong to that user. +`--ignore-scripts` prevents install hooks from embedding the staging path. +The staging tree is private (the updater uses umask 077); jobs running as +`ubuntu` can still read and execute it after the rename. Allow disk space for +both the old and new dependency trees during an update. + +An `apply` journal records the base and target before the live fast-forward. +The updater then renames live `node_modules` to `staging/old-node_modules` and +renames the staged tree into place. Renames cannot fall back to copying across +filesystems. It writes the new dependency marker **last**, keeps the old tree +and marker available until it removes the journal, then deletes staging. + +Any ordinary error or TERM/INT signal during the live update runs recovery +**before releasing the job locks**. Recovery rolls back to the recorded base, +restores the old dependency folder and marker, and removes the journal only +after the tree is clean. It first resets the index to target without checking +out files, then resets to base, so files created by an interrupted Git checkout +are removed too. A hard kill or power loss can interrupt that recovery; the +next safe-window updater run repeats it before fetching. Every rename boundary +is recoverable. A failed cleanup after the journal has gone leaves the new code +and new dependencies consistent; the next run clears leftover staging. + +Recovery may remove only `index.lock`, `HEAD.lock`, `ORIG_HEAD.lock` and +`refs/heads/main.lock` for an apply/merge journal. It validates HEAD against the +recorded base and target. Older merge-only journals are still accepted: these +complete the recorded target after staging a fresh dependency install, because +the old supervisor may already have damaged the live dependencies. Startup +recovery refuses while an ubuntu Git process is alive. Same-run recovery has +already waited for its own synchronous Git child to exit. Locks outside the +journaled phase are preserved and fail for operator inspection, as does any +Git lock without a journal. Unrelated untracked files remain and block recovery; +the journal stays for review if recovery itself cannot finish. + +The ad-network, launch-directory and cache-worker wrappers check +`/srv/stack127-data/checkout/update-in-progress` before starting, and check again +after acquiring their job lock. If it exists, they print +`ERROR: checkout update in progress or interrupted` and exit **1**, so the run +fails visibly instead of showing as skipped. Cron wrappers keep the dashboard's +start/end markers. The cache worker uses its existing service and journal; +there is no unit change. A hard-killed update therefore blocks jobs until the +updater recovers. These guards reach OVH with the merged checkout on its next +successful update; they do not protect an older checkout before that update. + +As with the self-host journal, **apply/merge recovery** may overwrite tracked +files and untracked files in the way of tracked target paths. **Never edit this +checkout without first setting the hold and waiting for the updater to finish.** +After a crash, inspect the journal before making manual edits; do not leave +operator edits for automatic recovery to overwrite. An invalid journal or +unexpected HEAD fails without reset. Never delete the journal to bypass a guard. + +Drift between the installed updater and the new repo copy fails visibly after +updating; the new code and dependencies are already consistent. Fetching +continues on later runs so a reviewed repair can still arrive. Reinstall the +reviewed copy by atomic rename below. It never rewrites itself. + +The entire update has a ten-minute timeout, then 30 seconds to terminate its +process group. Git cannot prompt and has connection/low-speed timeouts. npm has +a 1 GB V8 heap cap, and the run uses low CPU priority. Output keeps its first +48 KiB and last 16 KiB. Logs use the dashboard's exact start/end markers in +`/srv/stack127-data/logs/stack127-checkout-update.log`; existing log rotation +covers this path. Exit 76 means outside the safe start window or held/busy, and +shows as skipped. Other nonzero exits are failures. An overlapping updater is silent to preserve the active +run's marker pair. A SIGKILL can leave a start without an end; OS locks release +when the remaining children stop, and cron retries. + +## Install after review and merge + +Run from the lead's local checkout containing the reviewed, merged source. Do +not install during a data-maintenance window, an active release, or while main +is advancing. Preserve every existing pause prefix in crontab. These commands +make no change to cron's other entries. Copying the supervisor from here also +bootstraps a stale OVH checkout without a manual Git pull. Run the install and +its **first updater invocation at UTC minutes 15–28**. Check the server's clock +with `ssh ovh 'date -u'`. If the install crosses the window boundary, the updater +exits 76; retry it in the next safe window before adding cron. + +```sh +( +set -eu +ssh ovh 'set -eu +minute=$(date -u +%M) +case "$minute" in 1[5-9]|2[0-8]) ;; *) echo "Install and first run require UTC minutes 15–28" >&2; exit 76 ;; esac +test ! -e /srv/stack127-data/state/checkout-hold +[ "$(id -un)" = ubuntu ] +systemctl is-enabled cron +command -v git npm node python3 timeout flock sha256sum pgrep +install -d -m 700 /srv/stack127-data/checkout /srv/stack127-data/checkout/staging +install -d -m 755 /srv/stack127-data/logs +[ "$(stat -c %d /srv/stack127)" = "$(stat -c %d /srv/stack127-data/checkout/staging)" ] || { echo "Staging and checkout must share a filesystem" >&2; exit 1; }' +scp scripts/stack127/checkout/update.sh ovh:/srv/stack127-data/checkout/run-job.sh.new +scp scripts/stack127/checkout/crontab.txt ovh:/srv/stack127-data/checkout/crontab.reviewed +ssh ovh 'set -eu +bash -n /srv/stack127-data/checkout/run-job.sh.new +chmod 700 /srv/stack127-data/checkout/run-job.sh.new +mv /srv/stack127-data/checkout/run-job.sh.new /srv/stack127-data/checkout/run-job.sh +/bin/bash /srv/stack127-data/checkout/run-job.sh stack127-checkout-update +git -C /srv/stack127 rev-parse HEAD +tail -n 30 /srv/stack127-data/logs/stack127-checkout-update.log' +) +``` + +Watch this first run to its end. The wrappers on OVH before this release do +not check the update journal, so only a hard kill of this one run (SIGKILL, +reboot) could let a job start on half-updated files. If the first run ends +without an `end stack127-checkout-update` line or leaves +`/srv/stack127-data/checkout/update-in-progress`, rerun it at once inside the +window: recovery rolls the checkout back and finishes the update. After the +first success, every wrapper carries the journal guard. + +An outside-window, held or busy run exits 76. Wait for UTC minutes 15–28 and +for the existing job or hold to clear, then retry the first run: + +```sh +ssh ovh '/bin/bash /srv/stack127-data/checkout/run-job.sh stack127-checkout-update && git -C /srv/stack127 rev-parse HEAD' +``` + +Never remove locks to force it through. Require exit 0 and compare its SHA with +the reviewed released main. Stop if main advanced unexpectedly. Then append cron once: + +```sh +ssh ovh 'set -eu +crontab -l > /srv/stack127-data/checkout/crontab.before +if ! grep -q "^# BEGIN stack127 checkout update" /srv/stack127-data/checkout/crontab.before; then + if grep -q "/srv/stack127-data/checkout/run-job.sh" /srv/stack127-data/checkout/crontab.before; then + echo "Existing updater entry without managed block; inspect before installing" >&2 + exit 1 + fi + cat /srv/stack127-data/checkout/crontab.before /srv/stack127-data/checkout/crontab.reviewed > /srv/stack127-data/checkout/crontab.new + crontab /srv/stack127-data/checkout/crontab.new +fi +crontab -l +cmp /srv/stack127-data/checkout/run-job.sh /srv/stack127/scripts/stack127/checkout/update.sh' +``` + +Require one `25 * * * *` updater line, all prior entries and pauses, and a fresh +`end stack127-checkout-update exit 0` after the next cron run. The dashboard +probe derives the name from `run-job.sh stack127-checkout-update`, then reads +start/end markers from that job's log under its configured worker directory. +The name maps to `stack127-vps/stack127-checkout-update` in `monitoring.json`. +After homelab's main checkout receives this manifest, confirm the **Server +upkeep → Update OVH job code** row, hourly minute-25 schedule, result and log in +`https://monitor.ravitejaknts.com`. No dashboard code change is needed. + +Reinstall future updater changes with the same syntax-check, chmod and atomic +rename. Use the hold below first if an updater is active; do not replace a +running supervisor. Do not install a different version from another branch. + +## Maintenance hold and manual commands + +For a manual update, use the command above only at UTC minutes 15–28, before +opening the maintenance hold. Outside that window the updater exits 76 without +taking job locks or changing code. After maintenance, remove your hold and let +the next minute-25 cron run catch up, or run manually in the next safe window. +Do not bypass the time guard for installation, recovery or a paused collector. + +The hold survives a restart. It pauses only the updater, not collectors or cache +dispatch. Keep the exact cron-prefix and timer procedure in the [ad-network +runbook](../../../content_docs/ad-networks/production-runbook.md#1-open-a-maintenance-window). +Before touching code or starting manual work in `/srv/stack127`, create the hold +and wait for an already-running updater. Refuse an existing hold rather than +claiming another operator's window: + +```sh +ssh ovh 'set -eu +install -d -m 755 /srv/stack127-data/state +(set -C; : > /srv/stack127-data/state/checkout-hold) +flock -w 660 /srv/stack127-data/checkout/updater.lock true' +``` + +If waiting fails, leave the hold and investigate. Verify the expected revision +before running commands. Keep the hold through the complete maintenance window, +including corrections, sync, history repair and acceptance. When that work is +complete, remove only the hold you created: + +```sh +ssh ovh 'rm /srv/stack127-data/state/checkout-hold' +``` + +The next hourly run catches up. Never clear someone else's hold, delete a +journal to bypass recovery, or pull/install packages underneath active jobs. diff --git a/scripts/stack127/checkout/crontab.txt b/scripts/stack127/checkout/crontab.txt new file mode 100644 index 000000000..f9f825830 --- /dev/null +++ b/scripts/stack127/checkout/crontab.txt @@ -0,0 +1,3 @@ +# BEGIN stack127 checkout update (ubuntu, UTC; install outside Git first) +25 * * * * /bin/bash /srv/stack127-data/checkout/run-job.sh stack127-checkout-update >> /srv/stack127-data/logs/stack127-checkout-update.log 2>&1 +# END stack127 checkout update diff --git a/scripts/stack127/checkout/update.sh b/scripts/stack127/checkout/update.sh new file mode 100644 index 000000000..8448cdfe4 --- /dev/null +++ b/scripts/stack127/checkout/update.sh @@ -0,0 +1,330 @@ +#!/bin/bash +# Install outside Git as /srv/stack127-data/checkout/run-job.sh +set -euo pipefail + +skip() { echo "SKIPPED: $*; retry next hour"; return 76; } + +# A 10-minute timeout plus 30 seconds to kill children must finish before :40. +# Even a start at :28:59 leaves 30 seconds of margin. Override in scratch tests. +utc_minute() { date -u +%M; } +check_window() { + local minute + minute=$(utc_minute) + [[ "$minute" =~ ^[0-5][0-9]$ ]] || { echo 'ERROR: cannot read UTC minute'; return 1; } + (( 10#$minute >= 15 && 10#$minute <= 28 )) || { + skip "outside safe start window (UTC minutes 15–28; now :$minute)"; return 76; + } +} + +check_idle() { + [[ ! -e "$hold" ]] || { skip "checkout hold exists: $hold"; return 76; } + local marker pid cwd entry + for marker in "$lock_tmp"/adn-pending-*; do + [[ -f "$marker" ]] || continue + IFS= read -r pid < "$marker" || continue + [[ "$pid" =~ ^[1-9][0-9]*$ ]] || continue + if kill -0 "$pid" 2>/dev/null; then skip "ad-network retry wrapper $pid is alive"; return 76; fi + done + # Reading /proc does not require sudo for ubuntu-owned jobs. Ignore processes + # that disappear mid-scan and other users' unreadable cwd links. + [[ -d "$proc_root/self" ]] || { echo 'ERROR: /proc is unavailable'; return 1; } + for entry in "$proc_root"/[0-9]*/cwd; do + cwd=$(readlink "$entry" 2>/dev/null) || continue + case "$cwd" in "$checkout"|"$checkout"/*) + skip "process ${entry%/cwd} has its cwd in the checkout"; return 76 ;; + esac + done +} + +lock_jobs() { + local lock fd=100 name + local locks=("$data/launch-directory-metrics.lock" "$data/cache-worker.lock") + # Include future existing ad-network locks as well as every current job, even + # on a new install before its first run has created the lock file. + for name in verification snapshot platform domain-age ads-txt-summary discovery publisher-names traffic backup; do + locks+=("$lock_tmp/adn-$name.lock") + done + for lock in "$lock_tmp"/adn-*.lock; do [[ ! -e "$lock" ]] || locks+=("$lock"); done + while IFS= read -r lock; do + eval "exec $fd>\"\$lock\"" + flock -n -E 76 "$fd" || { skip "job lock busy: $lock"; return 76; } + fd=$((fd + 1)) + done < <(printf '%s\n' "${locks[@]}" | sort -u) + # Locks stay open across Git AND npm. New cron jobs cannot start data steps; + # the worker's existing persistent lock works even with systemd PrivateTmp. +} + +# os.rename fails across filesystems instead of silently copying a live tree. +rename_tree() { + [[ ! -e "$2" && ! -L "$2" ]] || { echo "ERROR: rename destination exists: $2"; return 1; } + python3 -c 'import os,sys; os.rename(sys.argv[1], sys.argv[2])' "$1" "$2" +} + +prepare_dependencies() { + local target=$1 force=${2:-} hash installed='' + rm -rf -- "$root/staging" + mkdir -p "$root/staging/new" + git show "$target:package.json" > "$root/staging/new/package.json" + git show "$target:package-lock.json" > "$root/staging/new/package-lock.json" + hash=$(sha256sum "$root/staging/new/package-lock.json"); hash=${hash%% *} + if [[ -f "$root/dependencies.sha256" ]]; then + read -r installed < "$root/dependencies.sha256" + cp "$root/dependencies.sha256" "$root/staging/base-dependencies.sha256" + fi + [[ -d "$checkout/node_modules" ]] || : > "$root/staging/no-base-node-modules" + if [[ "$hash" != "$installed" || ! -d "$checkout/node_modules" || "$force" = force ]]; then + # Nothing in the live checkout (including its success marker) changes here. + (cd "$root/staging/new" && npm ci --omit=dev --ignore-scripts --no-audit --no-fund) + # npm's .bin links are relative. Reject local/link dependencies and any link + # escaping node_modules: those cannot safely move with the installed tree. + python3 - "$root/staging/new/node_modules" "$checkout" <<'PYTHON' +import os, sys +from pathlib import Path +modules = Path(sys.argv[1]) +if not modules.is_dir() or modules.is_symlink(): + sys.exit("ERROR: staging did not create a real node_modules directory") +if modules.stat().st_dev != os.stat(sys.argv[2]).st_dev: + sys.exit("ERROR: staging and checkout must share a filesystem for atomic renames") +for directory, dirs, files in os.walk(modules): + for name in dirs + files: + entry = Path(directory) / name + if entry.lstat().st_uid != os.getuid(): + sys.exit("ERROR: staged dependencies must belong to the job user") + if entry.is_symlink(): + target = os.readlink(entry) + if os.path.isabs(target) or not entry.resolve().is_relative_to(modules.resolve()): + sys.exit("ERROR: staged dependency link cannot be relocated: " + str(entry)) +PYTHON + printf '%s\n' "$hash" > "$root/staging/target-dependencies.sha256" + fi +} + +swap_dependencies() { + [[ -f "$root/staging/target-dependencies.sha256" ]] || return 0 + if [[ -e "$checkout/node_modules" || -L "$checkout/node_modules" ]]; then + rename_tree "$checkout/node_modules" "$root/staging/old-node_modules" + fi + rename_tree "$root/staging/new/node_modules" "$checkout/node_modules" + # Commit the dependency marker only after the new directory is in place. + cp "$root/staging/target-dependencies.sha256" "$root/dependencies.sha256.tmp" + mv "$root/dependencies.sha256.tmp" "$root/dependencies.sha256" +} + +restore_dependencies() { + [[ -f "$root/staging/target-dependencies.sha256" ]] || return 0 + # Move the new tree back before restoring the old one. Repeating recovery + # after another hard kill is safe at every rename boundary. + if [[ -d "$root/staging/old-node_modules" || -L "$root/staging/old-node_modules" ]]; then + if [[ -e "$checkout/node_modules" || -L "$checkout/node_modules" ]]; then + rename_tree "$checkout/node_modules" "$root/staging/new/node_modules" + fi + rename_tree "$root/staging/old-node_modules" "$checkout/node_modules" + elif [[ -f "$root/staging/no-base-node-modules" && ! -d "$root/staging/new/node_modules" ]]; then + rename_tree "$checkout/node_modules" "$root/staging/new/node_modules" + fi + if [[ -f "$root/staging/base-dependencies.sha256" ]]; then + cp "$root/staging/base-dependencies.sha256" "$root/dependencies.sha256.tmp" + mv "$root/dependencies.sha256.tmp" "$root/dependencies.sha256" + else + rm -f "$root/dependencies.sha256" + fi +} + +recover_update() { + local journal="$root/update-in-progress" phase base target current git_dir lock relative git_rc=0 + local owned_locks=() + [[ -f "$journal" ]] || return 0 + # Children inherit the OS locks. Refuse recovery while manual Git runs. + if [[ "${1:-}" != owned ]]; then + pgrep -u "$(id -u)" -x git >/dev/null || git_rc=$? + [[ "$git_rc" = 1 ]] || { echo 'ERROR: Git still running or process list unavailable; refusing recovery'; return 1; } + fi + read -r phase base target < "$journal" + # Accept journals from the original merge-only supervisor too. + if [[ "$phase" =~ ^[0-9a-f]{40,64}$ ]]; then target=$base; base=$phase; phase=merge; fi + [[ "$base" =~ ^[0-9a-f]{40,64}$ && "$target" =~ ^[0-9a-f]{40,64}$ ]] && + git cat-file -e "$base^{commit}" && git cat-file -e "$target^{commit}" && + git merge-base --is-ancestor "$base" "$target" || { echo 'ERROR: invalid update journal'; return 1; } + current=$(git rev-parse HEAD) + git_dir=$(git rev-parse --absolute-git-dir) + case "$phase" in + fetch) + [[ "$base" = "$target" && "$current" = "$base" ]] || { echo 'ERROR: HEAD changed outside interrupted fetch; refusing recovery'; return 1; } + owned_locks=(refs/remotes/origin/main.lock) + ;; + apply|merge) + [[ "$current" = "$base" || "$current" = "$target" ]] || { echo 'ERROR: HEAD changed outside interrupted update; refusing recovery'; return 1; } + owned_locks=(index.lock HEAD.lock ORIG_HEAD.lock refs/heads/main.lock) + ;; + *) echo 'ERROR: invalid update journal phase'; return 1 ;; + esac + # Check every lock before deleting any. Never remove unrelated Git locks. + while IFS= read -r -d '' lock; do + relative=${lock#"$git_dir/"} + case " ${owned_locks[*]} " in + *" $relative "*) ;; + *) echo "ERROR: Git lock outside journaled $phase phase: $relative; inspect manually"; return 1 ;; + esac + done < <(find "$git_dir" -type f -name '*.lock' -print0) + for relative in "${owned_locks[@]}"; do + lock="$git_dir/$relative" + if [[ -f "$lock" ]]; then + echo "RECOVERY: removing stale Git lock $relative" + rm -- "$lock" + fi + done + case "$phase" in + apply) + [[ -d "$root/staging/new" ]] || { echo 'ERROR: update staging missing; journal retained for review'; return 1; } + echo "RECOVERY: rolling back interrupted update to $base" + # An interrupted checkout may have created target paths before moving + # HEAD. Put those paths in the index first so resetting base removes them. + git reset --mixed "$target" + git reset --hard "$base" + restore_dependencies + ;; + merge) + # Old supervisors could leave the target with incomplete dependencies. + # Rebuild in staging before completing their legacy merge journal. + echo "RECOVERY: completing interrupted fast-forward to $target" + prepare_dependencies "$target" force + git reset --hard "$target" + swap_dependencies + ;; + fetch) echo 'RECOVERY: interrupted fetch; working tree preserved' ;; + esac + if [[ "$phase" != fetch ]]; then + [[ -z "$(git status --porcelain --untracked-files=normal)" ]] || { echo 'ERROR: recovery left local files; journal retained for review'; return 1; } + fi + rm "$journal" +} + +update_checkout() { + cd "$checkout" + [[ "$(git symbolic-ref -q HEAD)" = refs/heads/main ]] || { echo 'ERROR: checkout must be on main'; return 1; } + local journal="$root/update-in-progress" base target git_dir + recover_update + [[ -z "$(git status --porcelain --untracked-files=normal)" ]] || { echo 'ERROR: checkout is dirty; refusing to change local files'; return 1; } + git_dir=$(git rev-parse --absolute-git-dir) + # Unjournaled Git locks could belong to an operator. Never remove them. + [[ -z "$(find "$git_dir" -type f -name '*.lock' -print -quit)" ]] || { echo 'ERROR: unjournaled Git lock; inspect manually'; return 1; } + base=$(git rev-parse HEAD) + printf 'fetch %s %s\n' "$base" "$base" > "$journal.tmp" + transaction_active=true + mv "$journal.tmp" "$journal" + git -c http.lowSpeedLimit=1024 -c http.lowSpeedTime=30 fetch --no-tags --no-prune --no-write-fetch-head --no-auto-maintenance origin refs/heads/main:refs/remotes/origin/main + rm "$journal" + git merge-base --is-ancestor HEAD origin/main || { echo 'ERROR: checkout has local commits or cannot fast-forward; nothing merged'; return 1; } + target=$(git rev-parse origin/main) + prepare_dependencies "$target" + printf 'apply %s %s\n' "$base" "$target" > "$journal.tmp" + mv "$journal.tmp" "$journal" + if [[ "$base" != "$target" ]]; then git merge --ff-only "$target"; fi + [[ "$(git rev-parse HEAD)" = "$target" && -z "$(git status --porcelain --untracked-files=normal)" ]] || { echo 'ERROR: checkout did not reach main'; return 1; } + swap_dependencies + # Removing the journal commits the code + dependency pair. Keep backups until + # this point; even failure writing the marker rolls back in this same run. + rm "$journal" + rm -rf -- "$root/staging" + echo "revision $target" +} + +finish_update() { + local rc=$1 recovery_rc + trap - EXIT + trap '' TERM INT + if [[ "$transaction_active" = true && -f "$root/update-in-progress" ]]; then + echo "ERROR: update failed (exit $rc); recovering before releasing job locks" + set +e + # Use a fresh errexit context: testing a shell function with `if` or `||` + # disables errexit inside it and could otherwise hide a failed rollback. + # Our synchronous Git child has exited; unrelated Git processes cannot + # block same-run rollback. Startup recovery still checks for live Git. + (set -e; recover_update owned) + recovery_rc=$? + if [[ "$recovery_rc" != 0 ]]; then + echo 'ERROR: recovery failed; journal retained and job guards will refuse work' + rc=1 + fi + fi + exit "$rc" +} + +execute() ( + # The exit trap runs before this subshell releases any inherited job locks. + # TERM/INT and ordinary command failures recover here; SIGKILL keeps a journal. + transaction_active=false + trap 'finish_update $?' EXIT + trap 'exit 143' TERM + trap 'exit 130' INT + cd / # The updater must not count its own cwd as an active checkout user. + check_window + check_idle + lock_jobs + check_idle + check_window + update_checkout + # Fetching still works on drift, so the next reviewed main can bring a repair. + cmp -s "$self" "$checkout/scripts/stack127/checkout/update.sh" || { + echo 'ERROR: installed checkout updater differs from main; perform reviewed atomic reinstall in scripts/stack127/checkout/README.md'; return 1; + } +) + +bounded_output() { + # Do not use Node while npm ci can replace its dependencies. Keep head + tail, + # drain the pipe, and leave the producer exit code to pipefail. + python3 -c ' +import sys +head_left = 49152 +tail = b"" +dropped = 0 +while True: + chunk = sys.stdin.buffer.read(8192) + if not chunk: break + part, chunk = chunk[:head_left], chunk[head_left:] + sys.stdout.buffer.write(part) + sys.stdout.buffer.flush() + head_left -= len(part) + tail += chunk + if len(tail) > 16384: + dropped += len(tail) - 16384 + tail = tail[-16384:] +if dropped: sys.stdout.buffer.write(("\n[output capped: %s bytes omitted]\n" % dropped).encode()) +sys.stdout.buffer.write(tail) +' +} + +main() { + export HOME=/home/ubuntu PATH=/usr/local/bin:/usr/bin:/bin + export GIT_TERMINAL_PROMPT=0 GIT_SSH_COMMAND='ssh -o BatchMode=yes -o ConnectTimeout=15' + export NODE_OPTIONS=--max-old-space-size=1024 + umask 077 + checkout=/srv/stack127 + data=/srv/stack127-data + root="$data/checkout" + hold="$data/state/checkout-hold" + lock_tmp=/tmp + proc_root=/proc + self=$(realpath "$0") + cd / # Keep the supervisor and timeout out of the checkout cwd scan too. + [[ "${1:-}" != --execute ]] || { execute; return; } + local job=stack127-checkout-update rc=0 + mkdir -p "$root" "$data/logs" + exec >>"$data/logs/$job.log" 2>&1 + exec 9>"$root/updater.lock" + # Overlap stays silent so it cannot interleave another run's markers. + flock -n -E 76 9 || return 76 + echo "== $(date -u +%FT%TZ) start $job" + trap 'rc=$?; trap - EXIT; echo "== $(date -u +%FT%TZ) end stack127-checkout-update exit $rc"; exit "$rc"' EXIT + trap 'exit 143' TERM + trap 'exit 130' INT + [[ $# = 1 && "$1" = "$job" ]] || { echo "Usage: run-job.sh $job"; return 2; } + # Whole process group is bounded, including Git, npm and inherited job locks. + # Low CPU priority and a V8 heap cap keep this small job below data collectors. + ulimit -c 0 + timeout --kill-after=30s 10m nice -n 15 /bin/bash "$self" --execute 2>&1 | bounded_output || rc=$? + return "$rc" +} + +# Source functions in scratch-repository tests without invoking production paths. +if [[ "${BASH_SOURCE[0]}" = "$0" ]]; then main "$@"; fi diff --git a/stack127_docs/monitoring.json b/stack127_docs/monitoring.json index 2288c9da8..43b4841d1 100644 --- a/stack127_docs/monitoring.json +++ b/stack127_docs/monitoring.json @@ -3,6 +3,7 @@ "project": "stack127", "about": "Entries for the Automation Monitor dashboard (https://monitor.ravitejaknts.com). One entry per scheduled job, workflow or service. key: / (machines: homelab, bloxodes-vps, stack127-vps, macbook-pro, macbook-air; job = cron run-job.sh name, systemd unit without .timer/.service, launchd label or T3 schedule title), github///, supabase/, or northflank/. name: short plain name. group: the section it is listed under. about: one sentence on what it does, written for the owner. impact (optional): what breaks if it stops. records (optional): stats_job_runs.job_name values it writes. ignore_records: job names that are sub-steps of other jobs.", "automations": [ + {"key": "stack127-vps/stack127-checkout-update", "group": "Server upkeep", "name": "Update OVH job code", "about": "Every hour at minute 25 UTC, safely updates the shared OVH checkout and changed dependencies; waits for maintenance holds and active jobs to clear.", "impact": "Ad-network jobs, launch-directory metrics and the cache worker keep running old code."}, {"key": "stack127-vps/launch-directory-metrics", "group": "Launch directories", "name": "Refresh launch directory ratings", "about": "On the 2nd of each month at 06:00 UTC, refreshes Ahrefs Domain Rating and Tranco popularity for published launch directories. Shares the process-held /srv/stack127-data/state/ahrefs.sqlite.lock request lock with ad-network DR collection. The separate ahrefs.sqlite commits pacing before HTTP and Retry-After before body cleanup or callbacks, retaining committed state after a crash.", "impact": "Launch directory ratings and popularity become stale."}, {"key": "stack127-vps/verification", "group": "Ad networks", "name": "Ad network site checks", "about": "Every hour at minute 40, checks up to 1,000 publisher sites (their ads.txt and pages), sends changed rows to production, and refreshes the site counts in titles and headings. Blocked or failed homepage checks never record departures, retry with backoff, and mark confirmation stale after 30 days without fresh evidence. It steps aside while a bigger job runs.", "impact": "Network pages stop gaining newly confirmed sites."}, {"key": "stack127-vps/snapshot", "group": "Ad networks", "name": "Daily ad network counts", "about": "Saves today's publisher counts for each ad network."}, @@ -16,8 +17,6 @@ {"key": "stack127-vps/backup", "group": "Ad networks", "name": "Ad networks data backup (OVH)", "about": "Makes a consistent daily copy of the ad networks database on the Stack127 VPS and keeps 7 copies.", "impact": "Recent ad network data could be lost."}, {"key": "homelab/stack127-ad-network-backup", "group": "Ad networks", "name": "Ad networks data backup (homelab)", "about": "Retired after the move to OVH; the homelab backup timer is disabled and does not make backups."}, {"key": "homelab/stack127-ad-network-discovery", "group": "Ad networks", "name": "Find new publishers (old homelab job)", "about": "The old homelab version of publisher discovery. Switched off after the move to OVH."}, - {"key": "github/stack127labs/stack127/ad-network-discovery.yml", "group": "Ad networks", "name": "Preview publisher discovery", "about": "A manual dry run that previews new publisher domains without saving anything."}, - {"key": "github/stack127labs/stack127/ad-network-verification.yml", "group": "Ad networks", "name": "Preview publisher checks", "about": "A manual dry run that previews ad network checks without saving anything."}, {"key": "homelab/stack127-ad-network-niche", "group": "Ad networks", "name": "Sort sites into niches", "about": "Every day at 06:00 UTC (11:30 IST), an AI model sorts up to 3,000 unsorted publisher sites into niches and saves accepted labels to production.", "impact": "New sites stay out of the niche pages."}, {"key": "stack127-vps/self-host-awesome-list", "group": "Self-host apps", "name": "Refresh public self-host list", "about": "Monday at 14:02 UTC, exports production apps, checks files and count drops, and pushes changed generated files to the public awesome-self-hosted-apps repository; catch-up retries missed runs.", "impact": "The public list and its downloadable data become stale."}, {"key": "stack127-vps/self-host-catch-up", "group": "Self-host apps", "name": "Catch up missed self-host jobs", "about": "At minute 30 each hour except 05 and 06 UTC, updates the checkout, checks the installed supervisor, and retries self-host jobs missing their latest scheduled success. Overlapping launches leave the active run log intact; checkout-lock waits skip for retry.", "impact": "Missed self-host jobs wait for their next nominal schedule, and supervisor drift goes unnoticed."}, diff --git a/tests/pages/ad-networks/discovery-failures.test.mjs b/tests/pages/ad-networks/discovery-failures.test.mjs new file mode 100644 index 000000000..708926db2 --- /dev/null +++ b/tests/pages/ad-networks/discovery-failures.test.mjs @@ -0,0 +1,191 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { DatabaseSync } from 'node:sqlite'; +import { createLocalDiscoveryStore } from '../../../scripts/ad-networks/local-discovery-store.mjs'; +import { runDiscoverySources } from '../../../scripts/ad-networks/discovery-run.mjs'; +import { isStorageError } from '../../../scripts/ad-networks/storage-errors.mjs'; +import { downloadToFile } from '../../../scripts/ad-networks/discovery-download.mjs'; +import { fetchWithRetry, checkRobots, sellerDomainsStream } from '../../../scripts/ad-networks/lib.mjs'; +import { sharedHostSlot, closeSharedFetchSlots } from '../../../scripts/ad-networks/shared-fetch-slots.mjs'; +import { EventEmitter } from 'node:events'; +import { PassThrough, Readable } from 'node:stream'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const sources = ['one', 'two'].map(key => ({ key, url: `https://${key}.example/sellers.json` })); +function fixture(t) { + const db = new DatabaseSync(':memory:'); + t.after(() => db.close()); + db.exec(`create table source_state(source_key text primary key, source_url text, etag text, last_modified text, + candidate_count integer, completed_at text, last_status text, last_error text); + create table publishers(id integer primary key, normalized_domain text unique, display_domain text, + first_seen_at text,last_discovered_at text,next_check_at text,verification_priority integer,created_at text,updated_at text); + create table publisher_sources(publisher_id integer,source_key text,source_url text,domain_as_supplied text, + license_attribution text,first_seen_at text,last_seen_at text,primary key(publisher_id,source_key));`); + const store = createLocalDiscoveryStore(db); + const warnings = []; + const run = (failed, { unchanged = false, storageError = false } = {}) => { + const state = store.readState(); // Reload as a separate cron invocation would. + return runDiscoverySources({ sources, state, + async processSource(source) { + if (failed.includes(source.key)) throw Object.assign(new Error('HTTP 406'), { discoveryStorage: storageError }); + if (!unchanged) state[source.key] = { url: source.url, candidates: 12, completedAt: '2026-11-14T00:00:00Z', etag: 'one' }; + return { source: source.key, status: unchanged ? 'unchanged' : 'applied' }; + }, + saveSuccess: (source, entry) => store.saveSourceState(source.key, entry), + saveFailure: (source, error) => store.saveSourceFailure(source, error), + log() {}, warn: line => warnings.push(line), + }); + }; + return { run, store, warnings }; +} + +test('one upstream failure warns; a second failure of that source on the next run fails', async t => { + const f = fixture(t); + assert.equal(await f.run(['one']), 0); + assert.match(f.warnings.join('\n'), /WARNING: discovery source one failed once/); + assert.equal(await f.run(['one']), 1); + assert.match(f.warnings.join('\n'), /ERROR: discovery source one failed on consecutive runs/); +}); + +test('every source failing fails even on the first run', async t => { + assert.equal(await fixture(t).run(['one', 'two']), 1); +}); + +test('successful 304 clears failure state without losing feed metadata', async t => { + const f = fixture(t); + assert.equal(await f.run([]), 0); + assert.equal(await f.run(['one']), 0); + assert.equal(await f.run([], { unchanged: true }), 0); + assert.equal(f.store.readState().one.etag, 'one'); + assert.equal(f.store.readState().one.candidates, 12); + assert.equal(f.store.readState().one.lastStatus, 'complete'); + assert.equal(await f.run(['one']), 0); +}); + +test('failure history belongs to each source, not the overall job', async t => { + const f = fixture(t); + assert.equal(await f.run(['one']), 0); + assert.equal(await f.run(['two']), 0); + assert.equal(await f.run(['one']), 0); +}); + +test('database writes and failure-state persistence errors always stop the run', async t => { + await assert.rejects(fixture(t).run(['one'], { storageError: true }), /HTTP 406/); + await assert.rejects(runDiscoverySources({ sources, state: {}, + processSource() { throw new Error('upstream'); }, + saveFailure() { throw new Error('disk full'); }, saveSuccess() {}, + }), /disk full/); +}); + +// A healthy second source must never hide a failure of local infrastructure. +async function runWithFailure(fail) { + let healthySourceRan = false, savedFailure = false; + const promise = runDiscoverySources({ sources, state: {}, + async processSource(source) { + if (source.key === 'one') return fail(); + healthySourceRan = true; + return { status: 'complete' }; + }, + saveSuccess() {}, saveFailure() { savedFailure = true; }, log() {}, warn() {}, + }); + await assert.rejects(promise, error => { + assert.equal(healthySourceRan, false); + assert.equal(savedFailure, false); + return isStorageError(error); + }); +} + +for (const code of ['ENOSPC', 'EDQUOT', 'EIO', 'EROFS', 'EACCES', 'EMFILE', 'ERR_SQLITE_ERROR']) { + test(`local ${code} stays fatal through error wrappers`, async () => { + await runWithFailure(() => { + throw new Error('storage wrapper', { cause: Object.assign(new Error('disk failure'), { code }) }); + }); + }); +} + +test('fetch retry preserves ENOSPC and stops without network retries', async t => { + let attempts = 0; + t.mock.method(globalThis, 'fetch', async () => { + attempts++; + throw new Error('fetch storage failure', { cause: Object.assign(new Error('disk full'), { code: 'ENOSPC' }) }); + }); + await runWithFailure(() => fetchWithRetry(sources[0].url)); + assert.equal(attempts, 1); +}); + +const downloadOptions = { maxTimeSeconds: 1, accept: 'application/json' }; +function curlResult(code) { + return () => { + const child = new EventEmitter(); + child.stdout = new PassThrough(); child.stderr = new PassThrough(); + process.nextTick(() => { + child.stdout.end('200 https://one.example/sellers.json'); + child.emit('close', code); + }); + return child; + }; +} + +test('temp creation, downloaded-header reads, and curl output writes stay fatal', async () => { + const noSpace = () => { throw Object.assign(new Error('disk full'), { code: 'ENOSPC' }); }; + for (const dependencies of [ + { mkdtemp: noSpace }, + { spawn: curlResult(0), readFile: noSpace }, + { spawn: curlResult(23) }, + ]) { + await runWithFailure(() => downloadToFile(sources[0], null, downloadOptions, dependencies)); + } +}); + +test('downloaded body read failures stay fatal through the seller parser', async () => { + await runWithFailure(async () => { + const response = await downloadToFile(sources[0], null, downloadOptions, { + spawn: curlResult(0), readFile: async () => '', + }); + try { for await (const candidate of sellerDomainsStream(response.body)) void candidate; } + finally { await response.cleanup(); } + }); + await runWithFailure(async () => { + const body = Readable.from((async function* () { + throw Object.assign(new Error('disk full'), { code: 'ENOSPC' }); + })()); + for await (const candidate of sellerDomainsStream(body)) void candidate; + }); +}); + +test('HTTP and curl connection errors remain isolated source failures', async () => { + for (const fail of [ + () => downloadToFile(sources[0], null, downloadOptions, { spawn: curlResult(22) }), + () => downloadToFile(sources[0], null, downloadOptions, { spawn: curlResult(7) }), + () => { throw new Error('fetch failed', { cause: Object.assign(new Error('connection reset'), { code: 'ECONNRESET' }) }); }, + ]) { + let healthySourceRan = false; + assert.equal(await runDiscoverySources({ sources, state: {}, + processSource(source) { if (source.key === 'one') return fail(); healthySourceRan = true; return {}; }, + saveSuccess() {}, saveFailure() {}, log() {}, warn() {}, + }), 0); + assert.equal(healthySourceRan, true); + } +}); + +test('robots throttle database open, acquire and release failures stay fatal', async t => { + const directory = mkdtempSync(join(tmpdir(), 'discovery-throttle-')); + t.after(() => { closeSharedFetchSlots(); rmSync(directory, { recursive: true, force: true }); }); + const dependencies = { lookup: async () => [{ address: '8.8.8.8', family: 4 }] }; + const throttleFailure = () => { throw new Error('throttle transaction failure'); }; + for (const sharedSlot of [ + host => sharedHostSlot(host, { databasePath: directory }), // Real SQLite open failure. + throttleFailure, + async () => throttleFailure, + ]) { + // Cover both a no-body response and release after streaming a robots body. + for (const status of [404, 200]) { + await runWithFailure(() => checkRobots(sources[0].url, { + ...dependencies, sharedHostSlot: sharedSlot, + fetch: async () => status === 404 ? new Response(null, { status }) : new Response('User-agent: *\nAllow: /'), + })); + } + } +}); diff --git a/tests/pages/ad-networks/discovery.test.mjs b/tests/pages/ad-networks/discovery.test.mjs index 1ee50c6db..9b3df57f4 100644 --- a/tests/pages/ad-networks/discovery.test.mjs +++ b/tests/pages/ad-networks/discovery.test.mjs @@ -92,15 +92,13 @@ test("robots rules honor explicit seller-file exceptions and broad bans", () => test("scheduled source selection excludes feeds with unresolved robots access", () => { assert.equal(sources.filter((source) => source.kind === "sellers-json").length, 39); - assert.equal(selectSources().length, 30); + assert.equal(selectSources().length, 29); assert.throws(() => selectSources(["adapex"]), /paused/); assert.throws(() => selectSources(["amazon-publisher-services"]), /paused/); assert.equal(selectSources(["concert"])[0].key, "sellers-json:concert"); assert.equal(selectSources(["google-adsense"])[0].verificationPriority, 95); assert.equal(selectSources(["taboola"])[0].transport, "curl-file"); - assert.deepEqual(selectSources(["teads"]).map((source) => source.key).sort(), [ - "sellers-json:teads", - "sellers-json:teads-outbrain", - ]); + assert.deepEqual(selectSources(["teads"]).map((source) => source.key), ["sellers-json:teads"]); + assert.throws(() => selectSources(["sellers-json:teads-outbrain"]), /paused/); assert.equal(selectSources(["ethicalads"])[0].kind, "publisher-list-html"); }); diff --git a/tests/stack127/checkout-update.test.mjs b/tests/stack127/checkout-update.test.mjs new file mode 100644 index 000000000..39560c145 --- /dev/null +++ b/tests/stack127/checkout-update.test.mjs @@ -0,0 +1,494 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdtempSync, mkdirSync, readFileSync, readlinkSync, realpathSync, rmSync, statSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; + +const source = readFileSync(new URL('../../scripts/stack127/checkout/update.sh', import.meta.url), 'utf8'); +const git = (cwd, ...args) => execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const quote = s => `'${s.replaceAll("'", "'\\''")}'`; +function fixture(t) { + const dir = mkdtempSync(path.join(tmpdir(), 'checkout-update-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const origin = path.join(dir, 'origin'), checkout = path.join(dir, 'checkout'); + const data = path.join(dir, 'data'), root = path.join(data, 'checkout'), bin = path.join(dir, 'bin'); + const lockTmp = path.join(dir, 'tmp'), proc = path.join(dir, 'proc'); + for (const p of [origin, root, bin, lockTmp, path.join(proc, 'self'), path.join(data, 'state')]) mkdirSync(p, { recursive: true }); + // Use the same OS flock on macOS via Python; never fake lock success. + writeFileSync(path.join(bin, 'flock'), `#!/usr/bin/env python3 +import fcntl, sys +try: fcntl.flock(int(sys.argv[-1]), fcntl.LOCK_EX | fcntl.LOCK_NB) +except BlockingIOError: sys.exit(76) +`, { mode: 0o755 }); + // Only Git running inside this fixture counts. Unrelated Git on the test + // machine (editors, other checkouts) must not trip the recovery guard. + writeFileSync(path.join(bin, 'pgrep'), `#!/bin/sh +real=$(PATH=/usr/bin:/bin command -v pgrep) || exit 3 +for pid in $("$real" "$@"); do + if [ -d /proc/$pid ]; then cwd=$(readlink /proc/$pid/cwd 2>/dev/null) + else cwd=$(lsof -a -p "$pid" -d cwd -Fn 2>/dev/null | sed -n 's/^n//p'); fi + case "$cwd" in ${quote(realpathSync(dir))}*) exit 0 ;; esac +done +exit 1 +`, { mode: 0o755 }); + writeFileSync(path.join(bin, 'sha256sum'), '#!/usr/bin/env python3\nimport hashlib,sys\nprint(hashlib.sha256(open(sys.argv[1],"rb").read()).hexdigest())\n', { mode: 0o755 }); + const calls = path.join(dir, 'npm-calls'); + writeFileSync(path.join(bin, 'npm'), `#!/bin/sh +printf '%s\\n' "$*" >> ${quote(calls)} +mkdir -p node_modules/.bin node_modules/example +printf new > node_modules/version +printf '#!/bin/sh\\necho relocated\\n' > node_modules/example/cli.sh +chmod +x node_modules/example/cli.sh +ln -s ../example/cli.sh node_modules/.bin/example +# npm must never run in the live checkout, nor invalidate its success marker. +test "$PWD" != ${quote(checkout)} || exit 91 +if [ -f ${quote(path.join(dir, 'npm-fail'))} ]; then exit 9; fi +`, { mode: 0o755 }); + const updaterPath = 'scripts/stack127/checkout/update.sh'; + mkdirSync(path.join(origin, path.dirname(updaterPath)), { recursive: true }); + writeFileSync(path.join(origin, updaterPath), source); + writeFileSync(path.join(origin, 'tracked.txt'), 'base\n'); + writeFileSync(path.join(origin, 'package.json'), '{"name":"fixture","version":"1.0.0"}\n'); + writeFileSync(path.join(origin, 'package-lock.json'), '{"lockfileVersion":3}\n'); + git(origin, 'init', '-b', 'main'); + for (const p of [origin]) { git(p, 'config', 'user.name', 'Fixture'); git(p, 'config', 'user.email', 'fixture@example.invalid'); } + git(origin, 'add', '.'); git(origin, 'commit', '-m', 'base'); + git(dir, 'clone', origin, checkout); + mkdirSync(path.join(checkout, 'node_modules')); + writeFileSync(path.join(checkout, 'node_modules/version'), 'base'); + // Match production's ignored dependency tree. + writeFileSync(path.join(checkout, '.git/info/exclude'), 'node_modules/\n'); + const installed = path.join(root, 'update.sh'); writeFileSync(installed, source); + const hold = path.join(data, 'state/checkout-hold'); + const setup = `source ${quote(installed)}; checkout=${quote(checkout)}; data=${quote(data)}; root=${quote(root)}; hold=${quote(hold)}; lock_tmp=${quote(lockTmp)}; proc_root=${quote(proc)}; self=${quote(installed)}; export PATH=${quote(bin)}:"$PATH"; utc_minute() { echo 25; }; `; + function run(command = 'execute') { + return execFileSync('/bin/bash', ['-c', setup + command], { cwd: dir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 15000 }); + } + const commit = message => { git(origin, 'add', '.'); git(origin, 'commit', '-m', message); return git(origin, 'rev-parse', 'HEAD'); }; + const seed = () => writeFileSync(path.join(root, 'dependencies.sha256'), createHash('sha256').update(readFileSync(path.join(checkout, 'package-lock.json'))).digest('hex') + '\n'); + seed(); + const base = git(checkout, 'rev-parse', 'HEAD'); + return { dir, origin, checkout, data, root, hold, proc, lockTmp, installed, run, setup, commit, base, calls, seed }; +} +const skipped = action => assert.throws(action, e => e.status === 76 && /SKIPPED:/.test(e.stdout)); + +test('hold prevents fetch and mutation', t => { + const f = fixture(t); + writeFileSync(f.hold, 'maintenance'); + git(f.checkout, 'remote', 'set-url', 'origin', path.join(f.dir, 'missing')); + skipped(() => f.run()); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), f.base); + assert.equal(existsSync(f.calls), false); +}); + +test('live retry PID and checkout cwd skip; stale retry PID does not', t => { + const f = fixture(t), marker = path.join(f.lockTmp, 'adn-pending-discovery-1'); + writeFileSync(marker, String(process.pid) + '\n'); + skipped(() => f.run()); + writeFileSync(marker, '2147483647\n'); + f.run(); + const processDir = path.join(f.proc, String(process.pid)); mkdirSync(processDir); + symlinkSync(f.checkout, path.join(processDir, 'cwd')); + skipped(() => f.run()); +}); + +for (const job of ['adn-discovery.lock', 'adn-future-job.lock', 'launch-directory-metrics.lock', 'cache-worker.lock']) { + test(`active ${job} prevents an update using a real OS lock`, async t => { + const f = fixture(t); + const lock = path.join(job.startsWith('adn-') ? f.lockTmp : f.data, job); + const child = spawn('python3', ['-c', 'import fcntl,sys,time; f=open(sys.argv[1],"w"); fcntl.flock(f,fcntl.LOCK_EX); print("ready",flush=True); time.sleep(30)', lock], { stdio: ['ignore', 'pipe', 'pipe'] }); + const exit = once(child, 'exit'); + t.after(() => child.kill('SIGKILL')); + await once(child.stdout, 'data'); + skipped(() => f.run()); + child.kill('SIGKILL'); await exit; + f.run(); + }); +} + +test('dirty tree and unpublished commits refuse without resetting edits', t => { + const f = fixture(t); + writeFileSync(path.join(f.checkout, 'tracked.txt'), 'operator edit'); + assert.throws(() => f.run(), e => e.status === 1 && /checkout is dirty/.test(e.stdout)); + assert.equal(readFileSync(path.join(f.checkout, 'tracked.txt'), 'utf8'), 'operator edit'); + git(f.checkout, 'config', 'user.name', 'Fixture'); git(f.checkout, 'config', 'user.email', 'fixture@example.invalid'); + git(f.checkout, 'add', '.'); git(f.checkout, 'commit', '-m', 'local change'); + const head = git(f.checkout, 'rev-parse', 'HEAD'); + assert.throws(() => f.run(), e => e.status === 1 && /local commits or cannot fast-forward/.test(e.stdout)); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), head); + assert.equal(existsSync(path.join(f.root, 'update-in-progress')), false); +}); + +test('code-only fast-forward avoids npm; changed lockfile installs once with production flags', t => { + const f = fixture(t); + writeFileSync(path.join(f.origin, 'tracked.txt'), 'updated'); const target = f.commit('code only'); + f.run(); assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), target); + assert.equal(existsSync(f.calls), false); + writeFileSync(path.join(f.origin, 'package-lock.json'), '{"lockfileVersion":3,"packages":{}}\n'); f.commit('dependency change'); + f.run(); f.run(); + assert.equal(readFileSync(f.calls, 'utf8'), 'ci --omit=dev --ignore-scripts --no-audit --no-fund\n'); +}); + +test('failed staging install leaves HEAD, working dependencies and marker unchanged', t => { + const f = fixture(t); + const oldModules = statSync(path.join(f.checkout, 'node_modules')).ino; + const oldMarker = readFileSync(path.join(f.root, 'dependencies.sha256'), 'utf8'); + writeFileSync(path.join(f.origin, 'package-lock.json'), '{"lockfileVersion":3,"packages":{}}\n'); f.commit('lock change'); + writeFileSync(path.join(f.dir, 'npm-fail'), ''); + assert.throws(() => f.run(), e => e.status === 9); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), f.base); + assert.equal(statSync(path.join(f.checkout, 'node_modules')).ino, oldModules); + assert.equal(readFileSync(path.join(f.checkout, 'node_modules/version'), 'utf8'), 'base'); + assert.equal(readFileSync(path.join(f.root, 'dependencies.sha256'), 'utf8'), oldMarker); + assert.equal(existsSync(path.join(f.root, 'update-in-progress')), false); + rmSync(path.join(f.dir, 'npm-fail')); f.run(); + assert.equal(readFileSync(f.calls, 'utf8').trim().split('\n').length, 2); +}); + +for (const failure of ['merge', 'first rename', 'second rename', 'marker']) { + test(`failed ${failure} restores base and dependencies in the same run`, t => { + const f = fixture(t); + writeFileSync(path.join(f.origin, 'tracked.txt'), 'target'); + writeFileSync(path.join(f.origin, 'new.txt'), 'created by interrupted merge'); + writeFileSync(path.join(f.origin, 'package-lock.json'), '{"lockfileVersion":3,"packages":{}}\n'); + const target = f.commit('dependency change'); + const marker = readFileSync(path.join(f.root, 'dependencies.sha256'), 'utf8'); + const modules = statSync(path.join(f.checkout, 'node_modules')).ino; + let inject; + if (failure === 'merge') { + // Simulate Git failing after checking out target paths but before moving HEAD. + inject = `git() { if [[ "$1" = merge ]]; then command git checkout ${quote(target)} -- tracked.txt new.txt; return 23; fi; command git "$@"; }; `; + } else if (failure === 'marker') { + inject = `mv() { if [[ "$2" = ${quote(path.join(f.root, 'dependencies.sha256'))} && ! -f ${quote(path.join(f.dir, 'failed-once'))} ]]; then touch ${quote(path.join(f.dir, 'failed-once'))}; return 23; fi; command mv "$@"; }; `; + } else { + const destination = failure === 'first rename' ? path.join(f.root, 'staging/old-node_modules') : path.join(f.checkout, 'node_modules'); + inject = `original_rename=$(declare -f rename_tree); eval "\${original_rename/rename_tree/original_rename_tree}"; rename_tree() { if [[ "$2" = ${quote(destination)} && ! -f ${quote(path.join(f.dir, 'failed-once'))} ]]; then touch ${quote(path.join(f.dir, 'failed-once'))}; return 23; fi; original_rename_tree "$@"; }; `; + } + assert.throws(() => f.run(inject + 'execute'), e => e.status === 23 && /RECOVERY: rolling back/.test(e.stdout)); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), f.base); + assert.equal(git(f.checkout, 'status', '--porcelain'), ''); + assert.equal(readFileSync(path.join(f.checkout, 'tracked.txt'), 'utf8'), 'base\n'); + assert.equal(existsSync(path.join(f.checkout, 'new.txt')), false); + assert.equal(statSync(path.join(f.checkout, 'node_modules')).ino, modules); + assert.equal(readFileSync(path.join(f.checkout, 'node_modules/version'), 'utf8'), 'base'); + assert.equal(readFileSync(path.join(f.root, 'dependencies.sha256'), 'utf8'), marker); + assert.equal(existsSync(path.join(f.root, 'update-in-progress')), false); + f.run(); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), target); + }); +} + +test('successful update swaps relocatable dependencies and writes the marker last', t => { + const f = fixture(t); + const marker = readFileSync(path.join(f.root, 'dependencies.sha256'), 'utf8'); + const modules = statSync(path.join(f.checkout, 'node_modules')).ino; + writeFileSync(path.join(f.origin, 'package-lock.json'), '{"lockfileVersion":3,"packages":{}}\n'); + const target = f.commit('dependency change'); + // At marker publication, HEAD and new dependencies must match and the old + // dependency tree must still be available for rollback under the journal. + const hook = `mv() { if [[ "$2" = ${quote(path.join(f.root, 'dependencies.sha256'))} ]]; then + [[ "$(git rev-parse HEAD)" = ${quote(target)} ]] + [[ "$(cat node_modules/version)" = new ]] + [[ "$(cat ${quote(path.join(f.root, 'staging/old-node_modules/version'))})" = base ]] + [[ "$(cat ${quote(path.join(f.root, 'dependencies.sha256'))})" = ${quote(marker.trim())} ]] + [[ -f ${quote(path.join(f.root, 'update-in-progress'))} ]] + fi; command mv "$@"; }; execute`; + f.run(hook); + assert.notEqual(statSync(path.join(f.checkout, 'node_modules')).ino, modules); + assert.equal(readlinkSync(path.join(f.checkout, 'node_modules/.bin/example')), '../example/cli.sh'); + assert.equal(execFileSync(path.join(f.checkout, 'node_modules/.bin/example'), { encoding: 'utf8' }), 'relocated\n'); + assert.equal(statSync(path.join(f.checkout, 'node_modules')).uid, process.getuid()); + assert.equal(readFileSync(path.join(f.root, 'dependencies.sha256'), 'utf8'), createHash('sha256').update(readFileSync(path.join(f.checkout, 'package-lock.json'))).digest('hex') + '\n'); + assert.equal(existsSync(path.join(f.root, 'update-in-progress')), false); + assert.equal(existsSync(path.join(f.root, 'staging/old-node_modules')), false); +}); + +test('journal recovery repairs interrupted Git files and locks; unjournaled locks are preserved', t => { + const f = fixture(t); + writeFileSync(path.join(f.origin, 'tracked.txt'), 'target'); writeFileSync(path.join(f.origin, 'new.txt'), 'new'); + const target = f.commit('target'); git(f.checkout, 'fetch', 'origin', 'main'); + const lock = path.join(f.checkout, '.git/index.lock'); writeFileSync(lock, ''); + assert.throws(() => f.run(), e => /unjournaled Git lock/.test(e.stdout)); + assert.equal(existsSync(lock), true); + writeFileSync(path.join(f.root, 'update-in-progress'), `${f.base} ${target}\n`); + writeFileSync(path.join(f.checkout, 'tracked.txt'), 'half-written'); + writeFileSync(path.join(f.checkout, 'new.txt'), 'new'); + assert.match(f.run(), /RECOVERY: completing interrupted fast-forward/); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), target); + assert.equal(git(f.checkout, 'status', '--porcelain'), ''); + assert.equal(existsSync(lock), false); + assert.equal(existsSync(path.join(f.root, 'update-in-progress')), false); +}); + +test('updater fetches repairs even on install drift and reports the need to reinstall', t => { + const f = fixture(t); + writeFileSync(path.join(f.origin, 'scripts/stack127/checkout/update.sh'), source + '\n# reviewed change\n'); const target = f.commit('updater change'); + assert.throws(() => f.run(), e => /installed checkout updater differs/.test(e.stdout)); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), target); +}); + +test('SIGKILL during Git leaves a journal and recovers on the next run', async t => { + const f = fixture(t); + writeFileSync(path.join(f.origin, '.gitattributes'), 'z-new.txt filter=interrupt\n'); + writeFileSync(path.join(f.origin, 'a-new.txt'), 'created before interrupt\n'); + writeFileSync(path.join(f.origin, 'tracked.txt'), 'updated\n'); + writeFileSync(path.join(f.origin, 'z-new.txt'), 'filtered\n'); + const target = f.commit('interrupted merge'); + const ready = path.join(f.dir, 'ready'); + git(f.checkout, 'config', 'filter.interrupt.smudge', `printf half-written > ${quote(path.join(f.checkout, 'tracked.txt'))}; touch ${quote(ready)}; sleep 30; cat`); + const child = spawn('/bin/bash', ['-c', f.setup + 'execute'], { cwd: f.dir, detached: true, stdio: 'ignore' }); + const exit = once(child, 'exit'); + t.after(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} }); + const deadline = Date.now() + 10000; + while (!existsSync(ready)) { + assert.ok(Date.now() < deadline, 'Git never reached the interrupt point'); + await new Promise(resolve => setTimeout(resolve, 20)); + } + assert.equal(readFileSync(path.join(f.root, 'update-in-progress'), 'utf8'), `apply ${f.base} ${target}\n`); + // Even a manual recovery call cannot reset a live Git operation. + assert.throws(() => f.run('update_checkout'), e => /Git still running/.test(e.stdout)); + process.kill(-child.pid, 'SIGKILL'); await exit; + git(f.checkout, 'config', '--unset', 'filter.interrupt.smudge'); + assert.match(f.run(), /RECOVERY: rolling back interrupted update/); + assert.equal(readFileSync(path.join(f.checkout, 'tracked.txt'), 'utf8'), 'updated\n'); + assert.equal(readFileSync(path.join(f.checkout, 'a-new.txt'), 'utf8'), 'created before interrupt\n'); + assert.equal(git(f.checkout, 'status', '--porcelain'), ''); + assert.equal(existsSync(path.join(f.checkout, '.git/index.lock')), false); + f.run(); // No stale job lock or repeated recovery. +}); + +test('installed supervisor logs skipped, successful and failed runs with the dashboard identity', t => { + const f = fixture(t); + const bin = path.join(f.dir, 'bin'); + // The local timeout adapter preserves descriptors and bounds the subprocess; + // production uses GNU timeout, absent on a base Mac installation. + writeFileSync(path.join(bin, 'timeout'), '#!/usr/bin/env python3\nimport subprocess,sys\nsys.exit(subprocess.run(sys.argv[3:], timeout=12, close_fds=False).returncode)\n', { mode: 0o755 }); + const scoped = source.replace('utc_minute() { date -u +%M; }', 'utc_minute() { echo 25; }').replaceAll('/srv/stack127-data', f.data).replaceAll('/srv/stack127', f.checkout) + .replace('proc_root=/proc', `proc_root=${quote(f.proc)}`) + .replace('lock_tmp=/tmp', `lock_tmp=${quote(f.lockTmp)}`) + .replace('PATH=/usr/local/bin:/usr/bin:/bin', `PATH=${quote(bin)}:"$PATH"`) + .replace('HOME=/home/ubuntu', `HOME=${quote(f.dir)}`); + writeFileSync(f.installed, scoped); + writeFileSync(path.join(f.origin, 'scripts/stack127/checkout/update.sh'), scoped); + f.commit('safe installed fixture'); + const run = () => execFileSync('/bin/bash', [f.installed, 'stack127-checkout-update'], { cwd: f.dir, stdio: 'pipe', timeout: 15000 }); + const logPath = path.join(f.data, 'logs/stack127-checkout-update.log'); + writeFileSync(f.hold, ''); assert.throws(run, e => e.status === 76); + rmSync(f.hold); + try { run(); } catch (error) { throw new Error(`${error.message}\n${readFileSync(logPath, 'utf8')}`); } + writeFileSync(path.join(f.checkout, 'tracked.txt'), 'human edit'); + assert.throws(run, e => e.status === 1); + const log = readFileSync(logPath, 'utf8'); + assert.equal((log.match(/== .* start stack127-checkout-update\n/g) || []).length, 3); + for (const code of [76, 0, 1]) assert.match(log, new RegExp(`== .* end stack127-checkout-update exit ${code}\\n`)); + assert.match(log, /checkout is dirty/); +}); + +test('failed staging with missing live dependencies does not publish a success marker', t => { + const f = fixture(t); + rmSync(path.join(f.checkout, 'node_modules'), { recursive: true }); + rmSync(path.join(f.root, 'dependencies.sha256')); + writeFileSync(path.join(f.dir, 'npm-fail'), ''); + assert.throws(() => f.run(), e => e.status === 9); + assert.equal(existsSync(path.join(f.root, 'dependencies.sha256')), false); + assert.equal(existsSync(path.join(f.checkout, 'node_modules')), false); + rmSync(path.join(f.dir, 'npm-fail')); + f.run(); f.run(); + assert.equal(readFileSync(f.calls, 'utf8').trim().split('\n').length, 2); +}); + +test('UTC start window covers only minutes 15–28, including manual runs', t => { + const f = fixture(t); + for (let minute = 0; minute < 60; minute++) { + const command = `utc_minute() { echo ${String(minute).padStart(2, '0')}; }; check_window`; + if (minute >= 15 && minute <= 28) f.run(command); + else skipped(() => f.run(command)); + } + git(f.checkout, 'remote', 'set-url', 'origin', path.join(f.dir, 'missing')); + for (const minute of [0, 10, 14, 29, 39, 40, 59]) { + skipped(() => f.run(`utc_minute() { echo ${String(minute).padStart(2, '0')}; }; execute`)); + } + assert.equal(existsSync(path.join(f.root, 'update-in-progress')), false); + assert.equal(existsSync(f.calls), false); + // Never take job locks if it is already too late to start. + assert.equal(existsSync(path.join(f.lockTmp, 'adn-discovery.lock')), false); +}); + +test('fetch recovery preserves edits and refuses locks outside its phase', t => { + const f = fixture(t); + const journal = path.join(f.root, 'update-in-progress'); + const ownLock = path.join(f.checkout, '.git/refs/remotes/origin/main.lock'); + const otherLock = path.join(f.checkout, '.git/index.lock'); + writeFileSync(journal, `fetch ${f.base} ${f.base}\n`); + writeFileSync(ownLock, ''); writeFileSync(otherLock, ''); + writeFileSync(path.join(f.checkout, 'tracked.txt'), 'operator edit'); + assert.throws(() => f.run(), e => /Git lock outside journaled fetch phase/.test(e.stdout)); + assert.equal(existsSync(ownLock), true); + assert.equal(existsSync(otherLock), true); + rmSync(otherLock); + assert.throws(() => f.run(), e => /working tree preserved/.test(e.stdout) && /checkout is dirty/.test(e.stdout)); + assert.equal(readFileSync(path.join(f.checkout, 'tracked.txt'), 'utf8'), 'operator edit'); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), f.base); + assert.equal(existsSync(ownLock), false); +}); + +test('SIGKILL during a real fetch ref transaction recovers its lock on the next run', async t => { + const f = fixture(t); + writeFileSync(path.join(f.origin, 'tracked.txt'), 'fetched update\n'); + const target = f.commit('fetch target'); + const ready = path.join(f.dir, 'fetch-ready'); + const hook = path.join(f.checkout, '.git/hooks/reference-transaction'); + writeFileSync(hook, `#!/bin/sh +if [ "$1" = prepared ]; then + if grep -q ' refs/remotes/origin/main$'; then + touch ${quote(ready)} + sleep 30 + fi +fi +`, { mode: 0o755 }); + const child = spawn('/bin/bash', ['-c', f.setup + 'execute'], { cwd: f.dir, detached: true, stdio: 'ignore' }); + const exit = once(child, 'exit'); + t.after(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} }); + const deadline = Date.now() + 10000; + while (!existsSync(ready)) { + assert.ok(Date.now() < deadline, 'fetch never reached its ref transaction'); + await new Promise(resolve => setTimeout(resolve, 20)); + } + const lock = path.join(f.checkout, '.git/refs/remotes/origin/main.lock'); + assert.equal(existsSync(lock), true); + assert.equal(readFileSync(path.join(f.root, 'update-in-progress'), 'utf8'), `fetch ${f.base} ${f.base}\n`); + process.kill(-child.pid, 'SIGKILL'); await exit; + rmSync(hook); + assert.equal(existsSync(lock), true, 'SIGKILL must leave the real fetch lock'); + assert.match(f.run(), /RECOVERY: interrupted fetch; working tree preserved/); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), target); + assert.equal(existsSync(lock), false); + assert.equal(existsSync(path.join(f.root, 'update-in-progress')), false); + f.run(); +}); + +for (const script of ['scripts/ad-networks/ovh/run-job.sh', 'scripts/launch-directories/ovh/run-job.sh', 'scripts/stack127/cache/ovh/run-worker.sh']) { + for (const timing of ['before wrapper starts', 'after initial check before job lock']) { + test(`${script} fails visibly with an interrupted-update journal ${timing}`, t => { + const f = fixture(t), bin = path.join(f.dir, 'bin'); + const journal = path.join(f.root, 'update-in-progress'); + mkdirSync(path.join(f.data, 'logs')); + const job = script.includes('ad-networks') ? 'discovery' : 'launch-directory-metrics'; + const worker = script.includes('cache/ovh'); + // Keep the wrapper's logic, including its POSIX shell and post-lock check. + // Only production paths, PATH and GNU stat are adapted for this scratch tree. + const wrapper = readFileSync(new URL(`../../${script}`, import.meta.url), 'utf8') + .replaceAll('/srv/stack127-data', f.data).replaceAll('/srv/stack127', f.checkout) + .replaceAll('/tmp/adn-', `${f.lockTmp}/adn-`) + .replace('PATH=/usr/local/bin:/usr/bin:/bin', `PATH=${quote(bin)}:"$PATH"`); + const installed = path.join(f.dir, 'wrapper.sh'); writeFileSync(installed, wrapper); + writeFileSync(path.join(bin, 'stat'), `#!/usr/bin/env python3 +import os,sys +s=os.stat(sys.argv[-1]); print(oct(s.st_mode & 0o777)[2:] if sys.argv[2] == '%a' else s.st_uid) +`, { mode: 0o755 }); + writeFileSync(path.join(bin, 'node'), `#!/bin/sh\ntouch ${quote(path.join(f.dir, 'node-started'))}\n`, { mode: 0o755 }); + writeFileSync(path.join(bin, 'flock'), `#!/bin/sh +# Inject a crash journal at the lock boundary; the inner shell must recheck it. +touch ${quote(journal)} +shift 4 +exec "$@" +`, { mode: 0o755 }); + for (const name of ['ad-networks.env', 'launch-directory-metrics.env', 'cache-worker.env']) { + writeFileSync(path.join(f.data, name), 'AD_NETWORK_DB_PATH=fixture\nAD_NETWORKS_STATE_DIRECTORY=fixture\n', { mode: 0o600 }); + } + if (timing === 'before wrapper starts') writeFileSync(journal, 'interrupted\n'); + assert.throws(() => execFileSync('/bin/sh', [installed, ...(worker ? [] : [job])], { + encoding: 'utf8', env: { ...process.env, PATH: `${bin}:${process.env.PATH}` }, stdio: 'pipe', timeout: 5000, + }), e => { + assert.equal(e.status, 1); + const output = worker ? e.stderr : readFileSync(path.join(f.data, `logs/${job}.log`), 'utf8'); + assert.match(output, /ERROR: checkout update in progress or interrupted/); + if (!worker) assert.match(output, new RegExp(`end ${job} exit 1`)); + return true; + }); + assert.equal(existsSync(path.join(f.dir, 'node-started')), false); + }); + } +} + +for (const boundary of ['old moved aside', 'new installed', 'marker written']) { + test(`SIGKILL with changed dependencies at ${boundary} recovers the old pair`, async t => { + const f = fixture(t); + const oldMarker = readFileSync(path.join(f.root, 'dependencies.sha256'), 'utf8'); + const oldModules = statSync(path.join(f.checkout, 'node_modules')).ino; + writeFileSync(path.join(f.origin, 'package-lock.json'), '{"lockfileVersion":3,"packages":{}}\n'); + f.commit('dependency update'); + const ready = path.join(f.dir, 'swap-ready'); + const pause = `touch ${quote(ready)}; sleep 30`; + const hook = boundary === 'marker written' + ? `mv() { command mv "$@"; if [[ "$2" = ${quote(path.join(f.root, 'dependencies.sha256'))} ]]; then ${pause}; fi; }; ` + : `original_rename=$(declare -f rename_tree); eval "\${original_rename/rename_tree/original_rename_tree}"; rename_tree() { + original_rename_tree "$@" + if [[ "$2" = ${quote(boundary === 'old moved aside' ? path.join(f.root, 'staging/old-node_modules') : path.join(f.checkout, 'node_modules'))} ]]; then ${pause}; fi + }; `; + const child = spawn('/bin/bash', ['-c', f.setup + hook + 'execute'], { cwd: f.dir, detached: true, stdio: 'ignore' }); + const exit = once(child, 'exit'); + t.after(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} }); + const deadline = Date.now() + 10000; + while (!existsSync(ready)) { + assert.ok(Date.now() < deadline, 'updater never reached the swap boundary'); + await new Promise(resolve => setTimeout(resolve, 20)); + } + process.kill(-child.pid, 'SIGKILL'); await exit; + assert.equal(existsSync(path.join(f.root, 'update-in-progress')), true); + // Recovery must restore the base pair before a new staging attempt, even + // if that next attempt fails, rather than merely finishing a new update. + writeFileSync(path.join(f.dir, 'npm-fail'), ''); + assert.throws(() => f.run(), e => e.status === 9 && /RECOVERY: rolling back/.test(e.stdout)); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), f.base); + assert.equal(statSync(path.join(f.checkout, 'node_modules')).ino, oldModules); + assert.equal(readFileSync(path.join(f.root, 'dependencies.sha256'), 'utf8'), oldMarker); + assert.equal(git(f.checkout, 'status', '--porcelain'), ''); + assert.equal(existsSync(path.join(f.root, 'update-in-progress')), false); + rmSync(path.join(f.dir, 'npm-fail')); f.run(); + }); +} + +test('SIGTERM after merging rolls back in the same run', t => { + const f = fixture(t); + writeFileSync(path.join(f.origin, 'tracked.txt'), 'new'); f.commit('new code'); + // Signal the execute subshell after a real successful merge. The parent shell + // must not be the signal target (Bash retains its parent's $$ in subshells). + const hook = `git() { command git "$@"; if [[ "$1" = merge ]]; then /bin/sh -c 'kill -TERM "$PPID"'; fi; }; execute`; + assert.throws(() => f.run(hook), e => e.status === 143 && /RECOVERY: rolling back/.test(e.stdout)); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), f.base); + assert.equal(readFileSync(path.join(f.checkout, 'node_modules/version'), 'utf8'), 'base'); + assert.equal(existsSync(path.join(f.root, 'update-in-progress')), false); +}); + +test('a real Git smudge failure removes partial target files and restores base immediately', t => { + const f = fixture(t); + writeFileSync(path.join(f.origin, '.gitattributes'), 'z-new.txt filter=fail\n'); + writeFileSync(path.join(f.origin, 'a-new.txt'), 'written before failure\n'); + writeFileSync(path.join(f.origin, 'z-new.txt'), 'fails smudge\n'); + f.commit('failing checkout filter'); + git(f.checkout, 'config', 'filter.fail.smudge', 'exit 1'); + git(f.checkout, 'config', 'filter.fail.required', 'true'); + assert.throws(() => f.run(), e => e.status !== 0 && /RECOVERY: rolling back/.test(e.stdout) && /smudge/.test(e.stderr)); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), f.base); + assert.equal(existsSync(path.join(f.checkout, 'a-new.txt')), false); + assert.equal(git(f.checkout, 'status', '--porcelain'), ''); + assert.equal(readFileSync(path.join(f.checkout, 'node_modules/version'), 'utf8'), 'base'); + assert.equal(existsSync(path.join(f.root, 'update-in-progress')), false); +}); + +test('a staged absolute dependency link is rejected without touching the live pair', t => { + const f = fixture(t); + writeFileSync(path.join(f.origin, 'package-lock.json'), '{"lockfileVersion":3,"packages":{}}\n'); + f.commit('dependency change'); + const marker = readFileSync(path.join(f.root, 'dependencies.sha256'), 'utf8'); + assert.throws(() => f.run('npm() { command npm "$@"; ln -s /tmp node_modules/absolute; }; execute'), + e => e.status === 1 && /dependency link cannot be relocated/.test(e.stderr)); + assert.equal(git(f.checkout, 'rev-parse', 'HEAD'), f.base); + assert.equal(readFileSync(path.join(f.checkout, 'node_modules/version'), 'utf8'), 'base'); + assert.equal(readFileSync(path.join(f.root, 'dependencies.sha256'), 'utf8'), marker); + assert.equal(existsSync(path.join(f.root, 'update-in-progress')), false); +}); From 149b3d77b34181c3155714f4f37e2016f40e8167 Mon Sep 17 00:00:00 2001 From: RaviTejaKNTS Date: Sat, 14 Nov 2026 17:04:52 +0530 Subject: [PATCH 2/2] Keep the ad-network wrapper's update guard on one line Main's new wrapper test records the collector command line by line; a multi-line inline guard split each call into several records. Co-Authored-By: Claude Opus 5.5 --- scripts/ad-networks/ovh/run-job.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/ad-networks/ovh/run-job.sh b/scripts/ad-networks/ovh/run-job.sh index 780742ab6..e12555cdd 100755 --- a/scripts/ad-networks/ovh/run-job.sh +++ b/scripts/ad-networks/ovh/run-job.sh @@ -35,12 +35,7 @@ tries=0 sync_only= while :; do rc=0 - nice -n 10 flock -n -E 76 "/tmp/adn-$job.lock" sh -c ' - if [ -e /srv/stack127-data/checkout/update-in-progress ]; then - echo "ERROR: checkout update in progress or interrupted"; exit 1 - fi - exec "$@" - ' sh node --no-warnings --experimental-strip-types scripts/ad-networks/refresh.mjs \ + nice -n 10 flock -n -E 76 "/tmp/adn-$job.lock" sh -c '[ ! -e /srv/stack127-data/checkout/update-in-progress ] || { echo "ERROR: checkout update in progress or interrupted"; exit 1; }; exec "$@"' sh node --no-warnings --experimental-strip-types scripts/ad-networks/refresh.mjs \ --job "$job" --environment production --confirm stack127-production --apply \ --local-database "$AD_NETWORK_DB_PATH" --state-directory "$AD_NETWORKS_STATE_DIRECTORY" $sync_only "$@" || rc=$? [ "$rc" = 76 ] && { echo "already running"; break; }