diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml index f8af5528..dbe9741b 100644 --- a/.github/workflows/pr-open.yml +++ b/.github/workflows/pr-open.yml @@ -76,10 +76,26 @@ jobs: with: target: ${{ github.event.number }} + utility-tests: + name: Utility Tests + runs-on: ubuntu-26.04 + permissions: + contents: read + defaults: + run: + working-directory: utilities/remove-deleted-pubcode + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: "24.x" + - run: npm ci + - run: npm test + results: name: Results if: always() - needs: [builds, deploys, tests] + needs: [builds, deploys, tests, utility-tests] runs-on: ubuntu-26.04 timeout-minutes: 1 steps: diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index afc3771a..4dfdb7a1 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -58,7 +58,7 @@ jobs: # Get API key API_KEY=$(oc get secrets/pubcode --template={{.data.API_KEY}} | base64 -d) - API_URL=https://$(oc get route/pubcode-api --template={{.spec.host}}) + API_URL=https://$(oc get ingress/pubcode-api --template='{{(index .spec.rules 0).host}}') API_KEY="${API_KEY}" API_URL="${API_URL}" node src/main.js validate-ministry-list: @@ -147,5 +147,5 @@ jobs: # Get API key API_KEY=$(oc get secrets/pubcode --template={{.data.API_KEY}} | base64 -d) - API_URL=https://$(oc get route/pubcode-api --template={{.spec.host}}) + API_URL=https://$(oc get ingress/pubcode-api --template='{{(index .spec.rules 0).host}}') API_KEY="${API_KEY}" API_URL="${API_URL}" node index.js diff --git a/utilities/remove-deleted-pubcode/index.js b/utilities/remove-deleted-pubcode/index.js index 25a5cece..28fa6b03 100644 --- a/utilities/remove-deleted-pubcode/index.js +++ b/utilities/remove-deleted-pubcode/index.js @@ -1,81 +1,76 @@ import axios from "axios"; +import { pathToFileURL } from "node:url"; -const API_KEY = process.env.API_KEY; -const API_URL = process.env.API_URL; -// Set the URL for the initial HTTP GET request -const pubcodeURL = process.env.PUBCODE_URL || "https://pubcode-api.apps.silver.devops.gov.bc.ca/api/pub-code"; -// Set the URL for the secondary API call - -async function doProcess() { - try { - // Perform the initial HTTP GET request - const pubCodeResponse = await axios.get(pubcodeURL); - const items = pubCodeResponse.data; - - // Initialize an array to store the final results - const results = []; - - // Iterate through items and make secondary API calls - for (const item of items) { - try { - // get result from GitHub API - const repoName = item.repo_name; - const defaultBranch = item.default_branch; - try{ - await getYamlFromRepo(repoName, defaultBranch); - }catch (e){ - if (e.response?.status === 404) { - results.push(repoName); - } - } - } catch (error) { - console.error(`Failed to fetch data for ${item.repo_name}:`, error); - process.exit(1); +/** + * Returns the repo names whose bcgovpubcode.yml/.yaml is gone (404) from their default branch. + * Any other error keeps the entry, so a GitHub outage never hides products. + */ +export async function findRemovedRepos(items, getYaml) { + const removed = []; + for (const item of items) { + try { + await getYaml(item.repo_name, item.default_branch); + } catch (e) { + if (e.response?.status === 404) { + removed.push(item.repo_name); } } - await markSoftDeleted(results); - - } catch (error) { - console.error('Failed to fetch data:', error); } + return removed; } + async function getYamlFromRepo(repoName, branchName) { - let yamlResponse; try { - yamlResponse = await axios.get(`https://raw.githubusercontent.com/bcgov/${repoName}/${branchName}/bcgovpubcode.yml`); + return await axios.get(`https://raw.githubusercontent.com/bcgov/${repoName}/${branchName}/bcgovpubcode.yml`); } catch (e) { - if (e.response?.status === 404) { - yamlResponse = await axios.get(`https://raw.githubusercontent.com/bcgov/${repoName}/${branchName}/bcgovpubcode.yaml`); + if (e.response?.status !== 404) { + throw e; } + return await axios.get(`https://raw.githubusercontent.com/bcgov/${repoName}/${branchName}/bcgovpubcode.yaml`); } - return yamlResponse; } -async function markSoftDeleted(repoNames) { - if (repoNames.length > 0) { - console.info(`Found ${repoNames.length} yaml files to mark as soft delete.`); - console.info(repoNames); - //send to backend api bulk load endpoint - for (const repoName of repoNames) { - try { - await axios.delete(`${API_URL}/api/pub-code/${repoName}`, { - headers: { - "X-API-KEY": API_KEY - } - }); - } catch (e) { - console.error(e.response?.status); - console.error(e.response?.config?.url); - } - } - } else { +/** + * Soft deletes each repo through deleteRepo; throws after trying all of them if any failed. + */ +export async function markSoftDeleted(repoNames, deleteRepo) { + if (repoNames.length === 0) { console.info(`No yaml files to mark as soft delete.`); + return; + } + console.info(`Found ${repoNames.length} yaml files to mark as soft delete.`); + console.info(repoNames); + const failed = []; + for (const repoName of repoNames) { + try { + await deleteRepo(repoName); + } catch (e) { + console.error(`Failed to soft delete ${repoName}: ${e.response?.status ?? e.message}`); + failed.push(repoName); + } + } + if (failed.length > 0) { + throw new Error(`Failed to soft delete ${failed.length} of ${repoNames.length} repos: ${failed.join(", ")}`); } } -try{ - await doProcess(); -}catch (e) { - console.error(e); - process.exit(1); + +async function main() { + const { API_KEY, API_URL } = process.env; + if (!API_KEY || !API_URL) { + throw new Error("API_KEY and API_URL are required"); + } + const { data: items } = await axios.get(`${API_URL}/api/pub-code`); + const removed = await findRemovedRepos(items, getYamlFromRepo); + await markSoftDeleted(removed, (repoName) => + axios.delete(`${API_URL}/api/pub-code/${repoName}`, { headers: { "X-API-KEY": API_KEY } }) + ); } +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + await main(); + } catch (e) { + console.error(e.message); + process.exit(1); + } +} diff --git a/utilities/remove-deleted-pubcode/index.test.js b/utilities/remove-deleted-pubcode/index.test.js new file mode 100644 index 00000000..417fd26f --- /dev/null +++ b/utilities/remove-deleted-pubcode/index.test.js @@ -0,0 +1,44 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { findRemovedRepos, markSoftDeleted } from "./index.js"; + +const httpError = (status) => Object.assign(new Error(`HTTP ${status}`), { response: { status } }); + +test("findRemovedRepos returns only repos whose pubcode file is gone", async () => { + const items = [ + { repo_name: "kept", default_branch: "main" }, + { repo_name: "removed", default_branch: "master" }, + { repo_name: "github-down", default_branch: "main" }, + ]; + const calls = []; + const getYaml = async (repo, branch) => { + calls.push(`${repo}@${branch}`); + if (repo === "removed") throw httpError(404); + if (repo === "github-down") throw httpError(503); + return { data: "version: 1" }; + }; + assert.deepEqual(await findRemovedRepos(items, getYaml), ["removed"]); + assert.deepEqual(calls, ["kept@main", "removed@master", "github-down@main"]); +}); + +test("markSoftDeleted deletes every removed repo", async () => { + const deleted = []; + await markSoftDeleted(["a", "b"], async (repo) => deleted.push(repo)); + assert.deepEqual(deleted, ["a", "b"]); +}); + +test("markSoftDeleted does nothing when no repos were removed", async () => { + await markSoftDeleted([], async () => assert.fail("should not delete")); +}); + +test("markSoftDeleted tries every repo, then fails if any delete failed", async () => { + const attempted = []; + await assert.rejects( + markSoftDeleted(["a", "b", "c"], async (repo) => { + attempted.push(repo); + if (repo === "b") throw httpError(401); + }), + /Failed to soft delete 1 of 3 repos: b/ + ); + assert.deepEqual(attempted, ["a", "b", "c"]); +}); diff --git a/utilities/remove-deleted-pubcode/package.json b/utilities/remove-deleted-pubcode/package.json index f5fc57e0..5d91dfd8 100644 --- a/utilities/remove-deleted-pubcode/package.json +++ b/utilities/remove-deleted-pubcode/package.json @@ -5,7 +5,7 @@ "main": "index.js", "type": "module", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "node --test" }, "author": "Om Mishra", "license": "Apache-2.0",