From a4db78fa28de17af23b01b898df5b4be1e544c4d Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Fri, 9 Oct 2026 13:11:57 -0700 Subject: [PATCH] fix(crawler): stop the crawl looping on null repository nodes GitHub returns a null node for a repository it cannot return. Reading its name threw inside the page loop, the catch only logged it, and the cursor never moved, so the crawler re-requested the same page every few seconds. Run 37978104944 did that for an hour until it was cancelled, and run 37864259112 did it for 25 minutes before ending green with nothing crawled. Null nodes are now skipped and counted, a GraphQL response without repositories fails the run, and finding no readable repositories fails it too. GraphQL 5xx answers are retried like rate limits, every request times out after 30 seconds, and the crawl job times out after 45 minutes. --- .github/workflows/scheduled.yml | 1 + crawler/src/main.js | 46 ++++++++++++++++++++------------- crawler/test/main.test.js | 17 ++++++++++++ 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index fa02a85f..65820333 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -19,6 +19,7 @@ jobs: pubcode-crawler: name: Crawl Git Repos for bcgovpubcode.yml runs-on: ubuntu-26.04 + timeout-minutes: 45 permissions: contents: read environment: prod diff --git a/crawler/src/main.js b/crawler/src/main.js index b858da61..0ae04a48 100644 --- a/crawler/src/main.js +++ b/crawler/src/main.js @@ -12,6 +12,8 @@ const API_URL = process.env.API_URL; const REPO_NAMES = process.env.REPO_NAMES; //comma separated list of repo names within bcgov org const BASE_DELAY = process.env.BASE_DELAY ? parseInt(process.env.BASE_DELAY, 10) : 60000; // 1 minute base delay const MAX_DELAY = process.env.MAX_DELAY ? parseInt(process.env.MAX_DELAY, 10) : 600000; // 10 minutes maximum delay +// Fail a stuck request instead of waiting forever; the API route also cuts requests off at 30 seconds. +axios.defaults.timeout = 30000; const JITTER_FACTOR = Math.random() * 0.3; // Random jitter between 0-30% /** * Fetches the bcgovpubcode yaml for the specified repo and branch @@ -83,7 +85,10 @@ export function isRecentlyUpdated(lastUpdated, now = new Date()) { export function reposFromEdges(edges) { const repos = []; for (const edge of edges) { - if (edge.node?.defaultBranchRef?.name && !edge.node.isArchived) { + if (!edge.node) { + continue; // GitHub returns a null node, with an error, for a repository it cannot return + } + if (edge.node.defaultBranchRef?.name && !edge.node.isArchived) { repos.push({ name: edge.node.name, defaultBranch: edge.node.defaultBranchRef.name, @@ -218,7 +223,8 @@ async function getGraphQlResponseOnQuery(query) { }); return response.data; } catch (error) { - if (error.response?.status === 403 && error.response?.data?.message?.includes("rate limit")) { + const status = error.response?.status; + if ((status === 403 && error.response?.data?.message?.includes("rate limit")) || status >= 500) { retries++; if (retries >= maxRetries) throw error; @@ -235,7 +241,7 @@ async function getGraphQlResponseOnQuery(query) { // Retry 3: ~240000ms (4 min) + up to 72000ms jitter = ~312s // Retry 4: ~480000ms (8 min) + up to 144000ms jitter = ~624s // Retry 5: ~600000ms (10 min, capped, jitter ignored) = 10 min - console.log(`Rate limit hit. Retrying in ${Math.round(delay/1000)} seconds... (Attempt ${retries}/${maxRetries})`); + console.log(`GitHub GraphQL answered ${status}. Retrying in ${Math.round(delay/1000)} seconds... (Attempt ${retries}/${maxRetries})`); await new Promise(resolve => setTimeout(resolve, delay)); } else { throw error; @@ -291,23 +297,27 @@ const performCrawling = async () => { } } }`; - try { - const responseData = await getGraphQlResponseOnQuery(query); - if (responseData.data?.organization?.repositories?.edges?.length > 0) { - const edges = responseData.data.organization.repositories.edges; - repoWithDetailsArray.push(...reposFromEdges(edges)); - cursor = edges[edges.length - 1].cursor; // the last cursor is used for the next page - if (responseData.data.organization.repositories.edges?.length < 100) { - moreRecords = false; - } - } else { - moreRecords = false; - } - console.info("iteration completed, cursor at ", cursor); - } catch (e) { - console.error(e); + const responseData = await getGraphQlResponseOnQuery(query); + const edges = responseData.data?.organization?.repositories?.edges; + if (!edges) { + throw new Error(`GitHub GraphQL returned no repositories: ${JSON.stringify(responseData.errors)}`); } + const unreadable = edges.filter((edge) => !edge.node).length; + if (unreadable > 0) { + console.warn(`Skipping ${unreadable} repositories GitHub returned as null: ${responseData.errors?.[0]?.message}`); + } + repoWithDetailsArray.push(...reposFromEdges(edges)); + if (edges.length > 0) { + cursor = edges[edges.length - 1].cursor; // the last cursor is used for the next page + } + if (edges.length < 100) { + moreRecords = false; + } + console.info("iteration completed, cursor at ", cursor); } while (moreRecords); + if (repoWithDetailsArray.length === 0) { + throw new Error("No readable repositories found under bcgov; check that GIT_TOKEN can read them"); + } const yamlAsJsons = await getAllPubCodeYamlsAsJSON(true); await bulkLoadPubCodes(yamlAsJsons); }; diff --git a/crawler/test/main.test.js b/crawler/test/main.test.js index 6b118936..254c1f50 100644 --- a/crawler/test/main.test.js +++ b/crawler/test/main.test.js @@ -90,6 +90,23 @@ test("reposFromEdges skips archived repos and repos without a default branch", ( ]); }); +test("reposFromEdges skips the null nodes GitHub returns for unreadable repositories", () => { + const readable = { + node: { + name: "readable", + isArchived: false, + defaultBranchRef: { name: "main" }, + pushedAt: "2026-10-01T00:00:00Z", + repositoryTopics: { nodes: [] }, + }, + cursor: "c1", + }; + + const repos = reposFromEdges([{ node: null, cursor: "c0" }, readable, { node: null, cursor: "c2" }]); + + assert.deepEqual(repos.map((repo) => repo.name), ["readable"]); +}); + test("isRecentlyUpdated accepts pushes within the last day only", () => { const now = new Date("2026-10-08T12:00:00Z");