From befd52ee90a5c1eb52fd5000e08e846acdc5f340 Mon Sep 17 00:00:00 2001 From: Derek Roberts Date: Thu, 8 Oct 2026 18:55:46 -0700 Subject: [PATCH] ci: gate PRs on per-workflow results checks and run unit tests Renovate PRs could auto-merge without running any tests for the crawler, the API, the schema script or the soft-delete utility. - analysis.yml: add API tests (against a MongoDB service using the production image), unit tests for crawler, schema/script and utilities/remove-deleted-pubcode, and an `Analysis Results` job that fails if any job failed or was cancelled. - pr-open.yml: rename `Results` to `PR Results`; the utility tests move to analysis.yml. - pr-validate.yml: rename the job to `Validate Results`. - crawler and schema/script: export the logic as functions and only run the script when executed directly, so it can be tested. Refs #597 --- .github/workflows/analysis.yml | 73 ++++++++++++ .github/workflows/pr-open.yml | 36 +++--- .github/workflows/pr-validate.yml | 2 +- AGENTS.md | 4 +- api/package.json | 3 +- api/test/pub-code-api.test.js | 145 +++++++++++++++++++++++ crawler/package.json | 3 +- crawler/src/main.js | 187 +++++++++++++++++------------- crawler/test/main.test.js | 120 +++++++++++++++++++ schema/script/index.js | 103 ++++++++++------ schema/script/package.json | 3 + schema/script/test/index.test.js | 61 ++++++++++ 12 files changed, 603 insertions(+), 137 deletions(-) create mode 100644 api/test/pub-code-api.test.js create mode 100644 crawler/test/main.test.js create mode 100644 schema/script/test/index.test.js diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml index 8b28b1b2..11d9b90c 100644 --- a/.github/workflows/analysis.yml +++ b/.github/workflows/analysis.yml @@ -42,3 +42,76 @@ jobs: uses: github/codeql-action/upload-sarif@v4 with: sarif_file: "trivy-results.sarif" + + api-tests: + name: API Tests + runs-on: ubuntu-26.04 + timeout-minutes: 10 + permissions: + contents: read + services: + mongo: + image: ghcr.io/bcgov/nr-containers/mongo:7.0.43 + options: >- + --health-cmd "mongosh --quiet --eval 'db.adminCommand({ ping: 1 })'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 27017:27017 + defaults: + run: + working-directory: api + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: "24.x" + - run: npm ci + - run: npm test + env: + MONGO_URI: mongodb://localhost:27017/pubcode-test + + unit-tests: + name: Unit Tests (${{ matrix.dir }}) + runs-on: ubuntu-26.04 + timeout-minutes: 10 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + dir: [crawler, schema/script, utilities/remove-deleted-pubcode] + defaults: + run: + working-directory: ${{ matrix.dir }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: "24.x" + - run: npm ci + - run: npm test + + # This job is the merge gate for this workflow. Every job in this + # workflow must be listed in 'needs', or its failure won't block the PR. + results: + name: Analysis Results + needs: [trivy, api-tests, unit-tests] + if: always() + runs-on: ubuntu-26.04 + timeout-minutes: 1 + steps: + - name: Log Job Results Context + run: | + echo "=== Upstream Job Statuses ===" + echo '${{ toJson(needs) }}' + + - name: Evaluate Overall Status + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: | + echo "At least one required job failed or was cancelled." + exit 1 + + - name: Success Message + run: echo "All required jobs passed or were intentionally skipped." diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml index dbe9741b..d596532c 100644 --- a/.github/workflows/pr-open.yml +++ b/.github/workflows/pr-open.yml @@ -76,29 +76,25 @@ jobs: with: target: ${{ github.event.number }} - utility-tests: - name: Utility Tests - runs-on: ubuntu-26.04 - permissions: - contents: read - defaults: - run: - working-directory: utilities/remove-deleted-pubcode - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-node@v7 - with: - node-version: "24.x" - - run: npm ci - - run: npm test - + # This job is the merge gate for this workflow. Every job in this + # workflow must be listed in 'needs', or its failure won't block the PR. results: - name: Results + name: PR Results + needs: [builds, deploys, tests] if: always() - needs: [builds, deploys, tests, utility-tests] runs-on: ubuntu-26.04 timeout-minutes: 1 steps: - - name: Fail if any dependent job failed or was cancelled + - name: Log Job Results Context + run: | + echo "=== Upstream Job Statuses ===" + echo '${{ toJson(needs) }}' + + - name: Evaluate Overall Status if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 + run: | + echo "At least one required job failed or was cancelled." + exit 1 + + - name: Success Message + run: echo "All required jobs passed or were intentionally skipped." diff --git a/.github/workflows/pr-validate.yml b/.github/workflows/pr-validate.yml index b4704b3c..326cffac 100644 --- a/.github/workflows/pr-validate.yml +++ b/.github/workflows/pr-validate.yml @@ -12,7 +12,7 @@ permissions: {} jobs: validate: - name: Validate PR + name: Validate Results if: (! github.event.pull_request.draft) runs-on: ubuntu-26.04 permissions: diff --git a/AGENTS.md b/AGENTS.md index f8edc5ce..cdfb4a82 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,11 +5,13 @@ Repository facts for automated coding assistants. Teams may edit or remove this ## Layout - `api/` (Express, Mongoose), `frontend/` (Vite, React), `crawler/` (GitHub crawl of `bcgovpubcode.yml`), `database/` (MongoDB image), `schema/` (JSON Schema) - Helm chart: `charts/pubcode` (`charts/pubcode/values.yaml`) -- Workflows: `.github/workflows/` (PR: `pr-open.yml`, `pr-close.yml`, `pr-validate.yml`; merge: `merge.yml`; crawler and schema jobs: `scheduled.yml`; reusable: `.deploy.yml`, `.tests.yml`) +- Workflows: `.github/workflows/` (PR: `pr-open.yml`, `analysis.yml`, `pr-close.yml`, `pr-validate.yml`; merge: `merge.yml`; crawler and schema jobs: `scheduled.yml`; reusable: `.deploy.yml`, `.tests.yml`) ## Build, test, deploy - Local stack: `docker compose up` (MongoDB, API on port 3005, Caddy frontend on port 3002). Caddy serves `frontend/dist`, so run `cd frontend && npm ci && npm run build` first. API: `cd api && npm ci && npm run dev`. Crawler: `cd crawler && npm ci && npm run dev` - Frontend (`cd frontend`, Node 24): `npm run dev` (Vite port 3001 in `frontend/vite.config.js`), `npm run build`, `npm run lint`. E2E: `npx cypress run --config baseUrl=http://localhost:3001` with `npm run dev` up. `frontend/cypress.config.js` sets `baseUrl` to `http://localhost:5173`. CI (`.github/workflows/.tests.yml`) overrides `baseUrl` to the deployed route +- Tests (`node --test`): `npm test` in `crawler/`, `schema/script/`, `utilities/remove-deleted-pubcode/` and `api/`. The API tests need MongoDB, e.g. `MONGO_URI=mongodb://localhost:27017/pubcode-test npm test`. CI runs them in `analysis.yml` +- Merge gates: `PR Results` (`pr-open.yml`) and `Analysis Results` (`analysis.yml`) fail if any job in their workflow failed or was cancelled, so add new jobs to their `needs`. `Validate Results` is the only job in `pr-validate.yml` - Images `api`, `database`, and `frontend` are built in GitHub Actions by `bcgov/actions/builder` (`pr-open.yml`). OpenShift Helm deploys run from GitHub Actions (`pr-open.yml` for pull requests; `merge.yml` deploys test, then prod), not a workstation ## Shared actions diff --git a/api/package.json b/api/package.json index 8f5f86b9..91737a94 100644 --- a/api/package.json +++ b/api/package.json @@ -26,7 +26,8 @@ }, "scripts": { "start": "node src/server.js", - "dev": "nodemon src/server.js" + "dev": "nodemon src/server.js", + "test": "node --test" }, "author": "Public Code API", "license": "Apache-2.0", diff --git a/api/test/pub-code-api.test.js b/api/test/pub-code-api.test.js new file mode 100644 index 00000000..b7da5318 --- /dev/null +++ b/api/test/pub-code-api.test.js @@ -0,0 +1,145 @@ +"use strict"; +// Integration tests: the API against a real MongoDB given by MONGO_URI. +const { test, before, after, beforeEach } = require("node:test"); +const assert = require("node:assert/strict"); + +const MONGO_URI = process.env.MONGO_URI; +if (!MONGO_URI) { + throw new Error("MONGO_URI is required, e.g. mongodb://localhost:27017/pubcode-test"); +} +process.env.API_KEY = "test-api-key"; +process.env.LOG_LEVEL = "error"; +// The email service refuses to load without configuration; no email is sent while EMAIL_RECIPIENTS is unset. +process.env.CHES_TOKEN_URL = "http://localhost/token"; +process.env.CHES_CLIENT_ID = "test"; +process.env.CHES_CLIENT_SECRET = "test"; +process.env.CHES_API_URL = "http://localhost/ches"; +delete process.env.EMAIL_RECIPIENTS; + +const mongoose = require("mongoose"); +const app = require("../src/app"); +const pubcodeEntity = require("../src/entities/pub-code-entity"); +const cacheService = require("../src/services/cache-service"); + +let server; +let baseUrl; + +const pubcode = (repoName, overrides = {}) => ({ + repo_name: repoName, + version: 1, + product_information: { product_name: `Product ${repoName}`, ministry: ["Forests"] }, + data_management_roles: { product_owner: "Owner" }, + product_technology_information: { backend_language: ["JavaScript"] }, + github_info: { default_branch: "main", stars: 1 }, + ...overrides, +}); + +function request(method, path, { body, apiKey } = {}) { + const headers = { "Content-Type": "application/json" }; + if (apiKey) { + headers["X-API-KEY"] = apiKey; + } + return fetch(`${baseUrl}${path}`, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + }); +} + +// bulk-load answers before it has written to the database, so wait for the write. +async function waitFor(condition, timeoutMs = 5000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await condition()) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error("Timed out waiting for the database"); +} + +before(async () => { + await mongoose.connect(MONGO_URI, { serverSelectionTimeoutMS: 10000 }); + await new Promise((resolve) => { + server = app.listen(0, resolve); + }); + baseUrl = `http://127.0.0.1:${server.address().port}`; +}); + +after(async () => { + await new Promise((resolve) => server.close(resolve)); + await mongoose.connection.dropDatabase(); + await mongoose.disconnect(); +}); + +beforeEach(async () => { + await pubcodeEntity.deleteMany({}); +}); + +test("bulk-load and delete reject requests without the API key", async () => { + assert.equal((await request("POST", "/api/pub-code/bulk-load", { body: [pubcode("a")] })).status, 401); + assert.equal((await request("POST", "/api/pub-code/bulk-load", { body: [pubcode("a")], apiKey: "wrong" })).status, 401); + assert.equal((await request("DELETE", "/api/pub-code/a")).status, 401); + assert.equal(await pubcodeEntity.countDocuments(), 0); +}); + +test("bulk-load rejects a body that is not a non-empty array", async () => { + for (const body of [[], { repo_name: "a" }]) { + const response = await request("POST", "/api/pub-code/bulk-load", { body, apiKey: "test-api-key" }); + assert.equal(response.status, 400); + } +}); + +test("bulk-load stores valid entries, skips invalid ones, and the list flattens them", async () => { + const invalid = pubcode("invalid"); + delete invalid.product_information; + + const response = await request("POST", "/api/pub-code/bulk-load", { + body: [pubcode("repo-a"), invalid], + apiKey: "test-api-key", + }); + assert.equal(response.status, 200); + await waitFor(async () => (await pubcodeEntity.countDocuments({ repo_name: "repo-a" })) === 1); + assert.equal(await pubcodeEntity.countDocuments({ repo_name: "invalid" }), 0); + + await cacheService.loadAllPubCodes(); + const list = await (await request("GET", "/api/pub-code")).json(); + assert.deepEqual(list, [ + { + repo_name: "repo-a", + product_name: "Product repo-a", + ministry: ["Forests"], + backend_language: ["JavaScript"], + product_owner: "Owner", + default_branch: "main", + stars: 1, + }, + ]); +}); + +test("bulk-load replaces an existing entry instead of duplicating it", async () => { + await pubcodeEntity.create(pubcode("repo-a")); + + await request("POST", "/api/pub-code/bulk-load", { + body: [pubcode("repo-a", { product_information: { product_name: "Renamed" } })], + apiKey: "test-api-key", + }); + + await waitFor(async () => (await pubcodeEntity.findOne({ repo_name: "repo-a" }))?.product_information?.product_name === "Renamed"); + assert.equal(await pubcodeEntity.countDocuments({ repo_name: "repo-a" }), 1); +}); + +test("delete soft-deletes an entry, hides it from the list, and a later bulk-load restores it", async () => { + await pubcodeEntity.create([pubcode("repo-a"), pubcode("repo-b")]); + + assert.equal((await request("DELETE", "/api/pub-code/missing", { apiKey: "test-api-key" })).status, 404); + assert.equal((await request("DELETE", "/api/pub-code/repo-a", { apiKey: "test-api-key" })).status, 200); + assert.equal((await pubcodeEntity.findOne({ repo_name: "repo-a" })).is_deleted, true); + + await cacheService.loadAllPubCodes(); + const visible = (await (await request("GET", "/api/pub-code")).json()).map((entry) => entry.repo_name); + assert.deepEqual(visible, ["repo-b"]); + + await request("POST", "/api/pub-code/bulk-load", { body: [pubcode("repo-a")], apiKey: "test-api-key" }); + await waitFor(async () => (await pubcodeEntity.findOne({ repo_name: "repo-a" }))?.is_deleted === false); +}); diff --git a/crawler/package.json b/crawler/package.json index 5fd6ea21..a6efd0b1 100644 --- a/crawler/package.json +++ b/crawler/package.json @@ -4,7 +4,8 @@ "version": "0.0.0", "type": "module", "scripts": { - "dev": "nodemon --watch src ./src/main.js" + "dev": "nodemon --watch src ./src/main.js", + "test": "node --test" }, "dependencies": { "async-retry": "^1.3.3", diff --git a/crawler/src/main.js b/crawler/src/main.js index 83dc6b39..b858da61 100644 --- a/crawler/src/main.js +++ b/crawler/src/main.js @@ -1,6 +1,7 @@ import axios from "axios"; import * as dotenv from "dotenv"; import * as jsYaml from "js-yaml"; +import { pathToFileURL } from "node:url"; dotenv.config(); const token = process.env.GIT_TOKEN; @@ -18,7 +19,7 @@ const JITTER_FACTOR = Math.random() * 0.3; // Random jitter between 0-30% * @param branchName * @returns {Promise>} */ -async function getYamlFromRepo(repoName, branchName) { +export async function getYamlFromRepo(repoName, branchName) { let yamlResponse; try { yamlResponse = await axios.get( @@ -41,7 +42,7 @@ async function getYamlFromRepo(repoName, branchName) { * @returns {JSON object of yaml with additional github attributes} */ -function processYamlFromHttpResponse(yamlResponse, repoWithDetails) { +export function processYamlFromHttpResponse(yamlResponse, repoWithDetails) { const yaml = yamlResponse.data; const yamlJson = jsYaml.load(yaml); yamlJson.repo_name = repoWithDetails.name; @@ -64,6 +65,45 @@ function processYamlFromHttpResponse(yamlResponse, repoWithDetails) { const DAY_IN_MILLIS = 24 * 60 * 60 * 1000; +/** + * Whether a repo was pushed to within the last day + * @param lastUpdated ISO timestamp of the last push + * @param now current time + * @returns {boolean} + */ +export function isRecentlyUpdated(lastUpdated, now = new Date()) { + return now.getTime() - new Date(lastUpdated).getTime() <= DAY_IN_MILLIS; +} + +/** + * Map GraphQL repository edges to repo details, skipping archived repos and repos without a default branch + * @param edges + * @returns {*[]} + */ +export function reposFromEdges(edges) { + const repos = []; + for (const edge of edges) { + if (edge.node?.defaultBranchRef?.name && !edge.node.isArchived) { + repos.push({ + name: edge.node.name, + defaultBranch: edge.node.defaultBranchRef.name, + stars: edge.node.stargazers?.totalCount, + lastUpdated: edge.node.pushedAt, + license: edge.node.licenseInfo?.name, + watchers: edge.node.watchers?.totalCount, + topics: edge.node.repositoryTopics.nodes.map( + (node) => node.topic.name + ), + }); + } else { + console.warn( + `skipping ${edge.node.name} as it does not have default branch or is archived., Default branch: '${edge.node.defaultBranchRef?.name}', isArchived: '${edge.node.isArchived}'` + ); + } + } + return repos; +} + /** * Fetches all the bcgovpubcode yaml files from the specified repos and converts them to JSON * @param compareLastUpdateDate @@ -74,9 +114,7 @@ async function getAllPubCodeYamlsAsJSON(compareLastUpdateDate) { for (const repoWithDetails of repoWithDetailsArray) { //if date comparison is enabled for this workflow and last_updated is not within last 1 day skip if (compareLastUpdateDate) { - const currentDate = new Date(); - const lastUpdatedDate = new Date(repoWithDetails.lastUpdated); - if (currentDate.getTime() - lastUpdatedDate.getTime() > DAY_IN_MILLIS) { + if (!isRecentlyUpdated(repoWithDetails.lastUpdated)) { console.debug( `Skipping ${repoWithDetails.name} repo as last updated date is more than 1 day.` ); @@ -256,26 +294,9 @@ const performCrawling = async () => { try { const responseData = await getGraphQlResponseOnQuery(query); if (responseData.data?.organization?.repositories?.edges?.length > 0) { - for (const edge of responseData.data.organization.repositories.edges) { - if (edge.node?.defaultBranchRef?.name && !edge.node.isArchived) { - repoWithDetailsArray.push({ - name: edge.node.name, - defaultBranch: edge.node.defaultBranchRef.name, - stars: edge.node.stargazers?.totalCount, - lastUpdated: edge.node.pushedAt, - license: edge.node.licenseInfo?.name, - watchers: edge.node.watchers?.totalCount, - topics: edge.node.repositoryTopics.nodes.map( - (node) => node.topic.name - ), - }); - } else { - console.warn( - `skipping ${edge.node.name} as it does not have default branch or is archived., Default branch: '${edge.node.defaultBranchRef?.name}', isArchived: '${edge.node.isArchived}'` - ); - } - cursor = edge.cursor; // keep overriding, the last cursor will be used - } + const edges = responseData.data.organization.repositories.edges; + repoWithDetailsArray.push(...reposFromEdges(edges)); + cursor = edges[edges.length - 1].cursor; // the last cursor is used for the next page if (responseData.data.organization.repositories.edges?.length < 100) { moreRecords = false; } @@ -291,62 +312,68 @@ const performCrawling = async () => { await bulkLoadPubCodes(yamlAsJsons); }; -if (!token || !API_KEY || !API_URL) { - console.error("Please provide GIT_TOKEN, API_KEY and API_URL in .env file"); - process.exit(1); -} else { - console.info("Starting crawling... and API_URL is ", API_URL); -} -if (REPO_NAMES?.length > 0) { - const repoNames = REPO_NAMES.split(","); - for (const repoName of repoNames) { - try { - const query = `query { - repository(owner: "bcgov", name: "${repoName}") { - name, - description, - defaultBranchRef{ - name - }, - repositoryTopics(first:20) { - nodes { - topic { - name +async function main() { + if (!token || !API_KEY || !API_URL) { + console.error("Please provide GIT_TOKEN, API_KEY and API_URL in .env file"); + process.exit(1); + } else { + console.info("Starting crawling... and API_URL is ", API_URL); + } + if (REPO_NAMES?.length > 0) { + const repoNames = REPO_NAMES.split(","); + for (const repoName of repoNames) { + try { + const query = `query { + repository(owner: "bcgov", name: "${repoName}") { + name, + description, + defaultBranchRef{ + name + }, + repositoryTopics(first:20) { + nodes { + topic { + name + } } - } - }, - updatedAt, - pushedAt, - stargazers { - totalCount - }, - watchers { - totalCount - }, - licenseInfo { - name - }, - } - }`; - const responseData = await getGraphQlResponseOnQuery(query); - const repo = responseData.data?.repository; - repoWithDetailsArray.push({ - name: repo?.name, - defaultBranch: repo?.defaultBranchRef?.name, - stars: repo?.stargazers?.totalCount, - lastUpdated: repo?.pushedAt, - license: repo?.licenseInfo?.name, - watchers: repo?.watchers?.totalCount, - topics: repo?.repositoryTopics?.nodes.map((node) => node.topic.name), - }); - } catch (e) { - console.error( - `Error while fetching yaml file for ${repoName} repo. Error: ${e.message}` - ); + }, + updatedAt, + pushedAt, + stargazers { + totalCount + }, + watchers { + totalCount + }, + licenseInfo { + name + }, + } + }`; + const responseData = await getGraphQlResponseOnQuery(query); + const repo = responseData.data?.repository; + repoWithDetailsArray.push({ + name: repo?.name, + defaultBranch: repo?.defaultBranchRef?.name, + stars: repo?.stargazers?.totalCount, + lastUpdated: repo?.pushedAt, + license: repo?.licenseInfo?.name, + watchers: repo?.watchers?.totalCount, + topics: repo?.repositoryTopics?.nodes.map((node) => node.topic.name), + }); + } catch (e) { + console.error( + `Error while fetching yaml file for ${repoName} repo. Error: ${e.message}` + ); + } } + const yamlAsJsons = await getAllPubCodeYamlsAsJSON(false); + await bulkLoadPubCodes(yamlAsJsons); + } else { + await performCrawling(); } - const yamlAsJsons = await getAllPubCodeYamlsAsJSON(false); - await bulkLoadPubCodes(yamlAsJsons); -} else { - await performCrawling(); +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + await main(); } diff --git a/crawler/test/main.test.js b/crawler/test/main.test.js new file mode 100644 index 00000000..6b118936 --- /dev/null +++ b/crawler/test/main.test.js @@ -0,0 +1,120 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import axios from "axios"; +import { + getYamlFromRepo, + isRecentlyUpdated, + processYamlFromHttpResponse, + reposFromEdges, +} from "../src/main.js"; + +const repoWithDetails = { + name: "nr-example", + defaultBranch: "main", + stars: 3, + lastUpdated: "2026-10-07T21:18:30Z", + license: "Apache License 2.0", + watchers: 5, + topics: ["nrs"], +}; + +test("processYamlFromHttpResponse adds repo details and GitHub info", () => { + const yaml = [ + "product_information:", + " product_name: Example", + " ministry:", + " - Forests", + "version: 1", + ].join("\n"); + + const result = processYamlFromHttpResponse({ data: yaml }, repoWithDetails); + + assert.equal(result.repo_name, "nr-example"); + assert.deepEqual(result.product_information, { + product_name: "Example", + ministry: ["Forests"], + }); + assert.equal(result.version, 1); + assert.deepEqual(result.github_info, { + last_updated: "2026-10-07", + license: "Apache License 2.0", + watchers: 5, + stars: 3, + default_branch: "main", + topics: ["nrs"], + }); +}); + +test("processYamlFromHttpResponse maps the legacy bcgov_pubcode_version field to version", () => { + const result = processYamlFromHttpResponse( + { data: "bcgov_pubcode_version: 2\n" }, + repoWithDetails + ); + + assert.equal(result.version, 2); + assert.equal("bcgov_pubcode_version" in result, false); +}); + +test("reposFromEdges skips archived repos and repos without a default branch", () => { + const node = (name, overrides = {}) => ({ + node: { + name, + isArchived: false, + defaultBranchRef: { name: "main" }, + stargazers: { totalCount: 1 }, + watchers: { totalCount: 2 }, + licenseInfo: { name: "MIT License" }, + pushedAt: "2026-10-01T00:00:00Z", + repositoryTopics: { nodes: [{ topic: { name: "a" } }, { topic: { name: "b" } }] }, + ...overrides, + }, + cursor: `cursor-${name}`, + }); + + const repos = reposFromEdges([ + node("active"), + node("archived", { isArchived: true }), + node("empty", { defaultBranchRef: null }), + ]); + + assert.deepEqual(repos, [ + { + name: "active", + defaultBranch: "main", + stars: 1, + lastUpdated: "2026-10-01T00:00:00Z", + license: "MIT License", + watchers: 2, + topics: ["a", "b"], + }, + ]); +}); + +test("isRecentlyUpdated accepts pushes within the last day only", () => { + const now = new Date("2026-10-08T12:00:00Z"); + + assert.equal(isRecentlyUpdated("2026-10-08T00:00:00Z", now), true); + assert.equal(isRecentlyUpdated("2026-10-07T12:00:00Z", now), true); + assert.equal(isRecentlyUpdated("2026-10-07T11:59:59Z", now), false); +}); + +test("getYamlFromRepo falls back to bcgovpubcode.yaml when the .yml file is missing", async (t) => { + const requested = []; + t.mock.method(axios, "get", async (url) => { + requested.push(url); + if (url.endsWith(".yml")) { + const error = new Error("Not Found"); + error.response = { status: 404 }; + throw error; + } + return { data: "version: 1\n" }; + }); + + const response = await getYamlFromRepo("nr-example", "main"); + + assert.equal(response.data, "version: 1\n"); + assert.deepEqual(requested, [ + "https://raw.githubusercontent.com/bcgov/nr-example/main/bcgovpubcode.yml", + "https://raw.githubusercontent.com/bcgov/nr-example/main/bcgovpubcode.yaml", + ]); +}); diff --git a/schema/script/index.js b/schema/script/index.js index 1e9ac27d..cadba42a 100644 --- a/schema/script/index.js +++ b/schema/script/index.js @@ -3,43 +3,75 @@ import axios from "axios"; import * as core from "@actions/core"; import * as cheerio from "cheerio"; import pkg from 'lodash'; +import { pathToFileURL } from "node:url"; const { isEqual } = pkg; const fs_promises = fs.promises; -const jsonSchemaResponse = await axios({ - method: "get", - url: "https://raw.githubusercontent.com/bcgov/pubcode/main/schema/bcgovpubcode.json", - headers: { - "Content-Type": "application/json", - "Cache-Control": "no-cache" - } -}); -const jsonSchema = jsonSchemaResponse.data; -const ministryNames = jsonSchema?.definitions?.product_information?.properties?.ministry?.items?.enum; +const MINISTRIES_URL = "https://www2.gov.bc.ca/gov/content/governments/organizational-structure/ministries-organizations/ministries"; -//https://www2.gov.bc.ca/gov/content/governments/organizational-structure/ministries-organizations/ministries -if (ministryNames) { - let htmlResponse = await axios({ - method: "get", - url: "https://www2.gov.bc.ca/gov/content/governments/organizational-structure/ministries-organizations/ministries" - }); - const dom = cheerio.load(htmlResponse.data); - const items = dom(`#body li `); - let index = 0; +/** + * Ministry names listed in the schema's ministry enum + * @param jsonSchema + * @returns {string[]|undefined} + */ +export function ministryNamesFromSchema(jsonSchema) { + return jsonSchema?.definitions?.product_information?.properties?.ministry?.items?.enum; +} + +/** + * Ministry names listed on the BC Gov ministries page + * @param html page HTML + * @returns {string[]} + */ +export function ministryNamesFromHtml(html) { + const dom = cheerio.load(html); const ministryNamesFromWeb = []; - items.each(function(idx, el) { - const name = dom(el).text().replace(/\u00a0/g, " "); - ministryNamesFromWeb.push(name); + dom(`#body li `).each(function(idx, el) { + ministryNamesFromWeb.push(dom(el).text().replace(/\u00a0/g, " ")); }); - if (!isEqual(ministryNamesFromWeb, ministryNames)) { - console.error("Ministry name mismatch between web and schema", ministryNamesFromWeb, ministryNames); - jsonSchema.definitions.product_information.properties.ministry.items["enum"] = ministryNamesFromWeb; - core.setOutput("schemaChanged", "true"); - await fs_promises.writeFile("../bcgovpubcode.json", JSON.stringify(jsonSchema, null, 2)); - } else { - core.setOutput("schemaChanged", "false"); + return ministryNamesFromWeb; +} + +/** + * Replace the schema's ministry enum when it differs from the web list + * @param jsonSchema schema, updated in place + * @param ministryNamesFromWeb + * @returns {boolean} whether the schema changed + */ +export function updateMinistryNames(jsonSchema, ministryNamesFromWeb) { + const ministryNames = ministryNamesFromSchema(jsonSchema); + if (isEqual(ministryNamesFromWeb, ministryNames)) { + return false; } -// BC Data Catalogue option, in future if there is a direct API endpoint to get list of active ministry names, that would be much better. + console.error("Ministry name mismatch between web and schema", ministryNamesFromWeb, ministryNames); + jsonSchema.definitions.product_information.properties.ministry.items["enum"] = ministryNamesFromWeb; + return true; +} + +async function main() { + const jsonSchemaResponse = await axios({ + method: "get", + url: "https://raw.githubusercontent.com/bcgov/pubcode/main/schema/bcgovpubcode.json", + headers: { + "Content-Type": "application/json", + "Cache-Control": "no-cache" + } + }); + const jsonSchema = jsonSchemaResponse.data; + + //https://www2.gov.bc.ca/gov/content/governments/organizational-structure/ministries-organizations/ministries + if (ministryNamesFromSchema(jsonSchema)) { + const htmlResponse = await axios({ + method: "get", + url: MINISTRIES_URL + }); + if (updateMinistryNames(jsonSchema, ministryNamesFromHtml(htmlResponse.data))) { + core.setOutput("schemaChanged", "true"); + await fs_promises.writeFile("../bcgovpubcode.json", JSON.stringify(jsonSchema, null, 2)); + } else { + core.setOutput("schemaChanged", "false"); + } + // BC Data Catalogue option, in future if there is a direct API endpoint to get list of active ministry names, that would be much better. /*const guids = [ "96860f84-dc45-4eb0-b0cb-9dedd3b58fe9", "3239fc90-cc88-49a8-94d9-e2d391bf0a75", @@ -78,7 +110,12 @@ if (ministryNames) { } });*/ -} else { - console.error("Ministry names not found in pubcode JSON schema"); - process.exit(1); + } else { + console.error("Ministry names not found in pubcode JSON schema"); + process.exit(1); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + await main(); } diff --git a/schema/script/package.json b/schema/script/package.json index 15893a7c..141fbf1f 100644 --- a/schema/script/package.json +++ b/schema/script/package.json @@ -3,6 +3,9 @@ "private": true, "type": "module", "version": "1.0.0", + "scripts": { + "test": "node --test" + }, "dependencies": { "axios": "^1.6.4", "cheerio": "^1.0.0-rc.12", diff --git a/schema/script/test/index.test.js b/schema/script/test/index.test.js new file mode 100644 index 00000000..62f06608 --- /dev/null +++ b/schema/script/test/index.test.js @@ -0,0 +1,61 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { + ministryNamesFromHtml, + ministryNamesFromSchema, + updateMinistryNames, +} from "../index.js"; + +const schemaWith = (names) => ({ + definitions: { + product_information: { + properties: { ministry: { items: { enum: names } } }, + }, + }, +}); + +test("the published schema lists ministry names", async () => { + const schema = JSON.parse( + await readFile(new URL("../../bcgovpubcode.json", import.meta.url), "utf8") + ); + + const names = ministryNamesFromSchema(schema); + + assert.ok(Array.isArray(names) && names.length > 0); + assert.ok(names.every((name) => typeof name === "string" && name.trim() === name)); +}); + +test("ministryNamesFromHtml reads list items under #body and normalizes non-breaking spaces", () => { + const html = ` + +
+
    +
  • Agriculture and Food
    • +
  • Forests\u00a0and Range
    • +
+
`; + + assert.deepEqual(ministryNamesFromHtml(html), [ + "Agriculture and Food", + "Forests and Range", + ]); +}); + +test("updateMinistryNames leaves a matching schema unchanged", () => { + const schema = schemaWith(["Agriculture and Food", "Forests"]); + + assert.equal(updateMinistryNames(schema, ["Agriculture and Food", "Forests"]), false); + assert.deepEqual(ministryNamesFromSchema(schema), ["Agriculture and Food", "Forests"]); +}); + +test("updateMinistryNames replaces the enum when the web list differs", (t) => { + t.mock.method(console, "error", () => {}); + const schema = schemaWith(["Agriculture and Food", "Forests"]); + + assert.equal(updateMinistryNames(schema, ["Agriculture and Food", "Water, Land and Resource Stewardship"]), true); + assert.deepEqual(ministryNamesFromSchema(schema), [ + "Agriculture and Food", + "Water, Land and Resource Stewardship", + ]); +});