diff --git a/.github/scripts/check_pin_freshness.py b/.github/scripts/check_pin_freshness.py new file mode 100644 index 0000000..f901bdc --- /dev/null +++ b/.github/scripts/check_pin_freshness.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Audit GitHub Actions pin freshness against the upstream registries. + +`check_action_pins.py` validates pin **shape** — does the @ref match the +policy bucket. This script validates **freshness** — does the @ref still +resolve to something upstream, and does the trailing `# vN.M.P` comment +on a SHA pin still match the tag's current SHA? + +Filed as #136 after PR #121 surfaced `astral-sh/setup-uv@v5` going +silently dead — the tag stopped resolving to anything in March 2026, +producing 0-jobs / 0-seconds CI failures. The shape gate doesn't catch +that class; this freshness gate does. + +Behaviour: + +- Walks every workflow + composite-action file via the same + `parse_workflow` machinery as `check_action_pins.py`. +- For each tag pin (`@v8`, `@v8.0.0`): GET + `https://api.github.com/repos//git/refs/tags/`. A 404 means + the tag no longer exists upstream — emit `::warning::` (or `::error::` + under strict mode). +- For each SHA pin (`@<40-hex>` + trailing `# vN.M.P` comment): GET + `/repos//git/refs/tags/` to fetch the tag's + current SHA. If the tag exists and resolves to a different SHA than + the pin, the upstream re-tagged — warn (potential supply-chain shift). + If the tag's SHA is a tag object (annotated tag), dereference one + level via `git/tags/` to get the commit SHA before comparing. +- API failures (network, 4xx other than 404, 5xx) downgrade to + `::warning::` — the gate's job is to surface drift, not be a + transient-network tripwire. + +Default: warn-not-fail (`exit 0` even on findings, with annotations). +With `PIN_FRESHNESS_STRICT=1`, findings escalate to errors (`exit 1`), +matching the `ASPIRATIONAL_STRICT=1` toggle pattern from #153. + +Exit codes: + 0 — every pin resolves cleanly OR strict mode is off and findings + are surfaced as warnings only + 1 — strict mode is on and one or more pins failed freshness checks + 2 — script-level error (workflows dir missing, parse failure, no + `GITHUB_TOKEN` set so we can't query the API) + +Usage (from repo root, in CI with token): + + GITHUB_TOKEN=... python .github/scripts/check_pin_freshness.py +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import sys +import urllib.error +import urllib.request +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from types import ModuleType + +# Reuse `parse_workflow`, `_collect_yaml_files`, `_VERSION_COMMENT_RE`, +# `_SHA_RE`, etc. from check_action_pins.py rather than duplicate them. +# Importlib-based load mirrors the test pattern used elsewhere in the +# repo so this script stays standalone (no setup.py wiring needed). +_SCRIPT_DIR = Path(__file__).parent + + +def _load_pin_module() -> ModuleType: + spec = importlib.util.spec_from_file_location( + "check_action_pins", _SCRIPT_DIR / "check_action_pins.py" + ) + if spec is None or spec.loader is None: + msg = "could not load check_action_pins.py" + raise RuntimeError(msg) + module = importlib.util.module_from_spec(spec) + # Register in sys.modules BEFORE exec_module — `@dataclass` walks + # `sys.modules[cls.__module__]` while processing the class, and the + # ActionRef dataclass would AttributeError without this line. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +_pins = _load_pin_module() +_API_BASE = "https://api.github.com" + + +def _fetch_json(url: str, token: str) -> dict[str, object] | None: + """GET a GitHub API URL, return parsed JSON or None on any failure. + + Failures (404, 5xx, network, JSON-parse) all collapse to None — the + caller decides how to surface them. Keeps this gate from being a + transient-CI tripwire. + """ + req = urllib.request.Request( # noqa: S310 — fixed api.github.com host + url, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with urllib.request.urlopen(req, timeout=10) as response: # noqa: S310 + payload = json.loads(response.read().decode("utf-8")) + except urllib.error.URLError, TimeoutError, json.JSONDecodeError: + return None + return payload if isinstance(payload, dict) else None + + +def _resolve_tag_sha(action: str, tag: str, token: str) -> str | None: + """Return the commit SHA the tag points at, or None on missing/error. + + Annotated tags resolve via two GETs: first `/git/refs/tags/` to + get the tag-object SHA, then `/git/tags/` to dereference to the + commit. Lightweight tags resolve in one GET (the ref's `object.sha` + is the commit directly). + """ + ref = _fetch_json(f"{_API_BASE}/repos/{action}/git/refs/tags/{tag}", token) + if ref is None: + return None + obj = ref.get("object") + if not isinstance(obj, dict): + return None + obj_type = obj.get("type") + obj_sha = obj.get("sha") + if not isinstance(obj_sha, str): + return None + if obj_type == "commit": + return obj_sha + if obj_type == "tag": + # Annotated tag — dereference to the commit it points at. + annotated = _fetch_json(f"{_API_BASE}/repos/{action}/git/tags/{obj_sha}", token) + if annotated is None: + return None + inner = annotated.get("object") + if isinstance(inner, dict): + inner_sha = inner.get("sha") + if isinstance(inner_sha, str): + return inner_sha + return None + + +def _check_tag_pin(ref: object, token: str) -> str | None: + """Tag pin: ensure the upstream tag still exists. Returns warning text or None.""" + tag = ref.pin # type: ignore[attr-defined] + sha = _resolve_tag_sha(ref.action, tag, token) # type: ignore[attr-defined] + if sha is None: + return ( + f"{ref.action}@{tag} — upstream tag no longer resolves " # type: ignore[attr-defined] + "(404 or API failure). If 404, the tag was deleted/renamed; " + "bump to a current tag or SHA pin." + ) + return None + + +def _check_sha_pin(ref: object, token: str) -> str | None: + """SHA pin: trailing tag comment must still resolve to the same SHA.""" + if not ref.comment: # type: ignore[attr-defined] + return None # shape audit owns the missing-comment case + match = _pins._VERSION_COMMENT_RE.search(ref.comment) # type: ignore[attr-defined] + if not match: + return None + documented_tag = match.group(0) + upstream_sha = _resolve_tag_sha(ref.action, documented_tag, token) # type: ignore[attr-defined] + if upstream_sha is None: + return ( + f"{ref.action}@{ref.pin[:8]}… (commented `{documented_tag}`) " # type: ignore[attr-defined] + "— upstream tag no longer resolves; comment may be stale." + ) + if upstream_sha.lower() != ref.pin.lower(): # type: ignore[attr-defined] + return ( + f"{ref.action}@{ref.pin[:8]}… (commented `{documented_tag}`) " # type: ignore[attr-defined] + f"— upstream tag has been re-tagged to " + f"{upstream_sha[:8]}…; pin no longer matches the documented tag." + ) + return None + + +def main() -> int: + token = os.environ.get("GITHUB_TOKEN", "") + if not token: + print( + "::error::GITHUB_TOKEN required for pin-freshness audit " + "(API rate limit + private-repo access)." + ) + return 2 + + yml_files = _pins._collect_yaml_files() + if not yml_files: + print("::error::no workflow / composite-action files found") + return 2 + + refs = [] + for path in yml_files: + refs.extend(_pins.parse_workflow(path)) + + strict = os.environ.get("PIN_FRESHNESS_STRICT", "") == "1" + findings: list[tuple[object, str]] = [] + for ref in refs: + if not ref.pin: + continue # shape audit catches missing-@ + if _pins._SHA_RE.match(ref.pin): + problem = _check_sha_pin(ref, token) + else: + problem = _check_tag_pin(ref, token) + if problem is not None: + findings.append((ref, problem)) + + severity = "error" if strict else "warning" + for ref, problem in findings: + print(f"::{severity} file={ref.file},line={ref.line}::{problem}") # type: ignore[attr-defined] + + summary = ( + f"Pin-freshness audit: {len(refs)} pins checked across " + f"{len(yml_files)} files; {len(findings)} finding(s)" + ) + # Surface the finding count as a workflow output so the calling + # workflow can decide whether to open a tracking issue. Skipped when + # GITHUB_OUTPUT isn't set (local runs / tests). + output_path = os.environ.get("GITHUB_OUTPUT", "") + if output_path: + with Path(output_path).open("a", encoding="utf-8") as fh: + fh.write(f"findings_count={len(findings)}\n") + if findings: + suffix = " (strict — failing)" if strict else " (warn-only)" + print(summary + suffix + ".") + return 1 if strict else 0 + print(summary + ".") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/check_required_contexts.py b/.github/scripts/check_required_contexts.py index 0ff657a..a59036c 100644 --- a/.github/scripts/check_required_contexts.py +++ b/.github/scripts/check_required_contexts.py @@ -55,6 +55,14 @@ "workflow_run-triggered after release.yml + workflow_dispatch only;" " opens its own roll-up PR (which goes through ci.yml as normal)." ), + "pin-freshness-audit.yml": ( + "Weekly cron + workflow_dispatch; warn-only by default with auto-" + " filed tracking issue. Never appears on PR check sets." + ), + "changelog-prestage.yml": ( + "workflow_dispatch only; opens its own pre-stage PR before a" + " release PR is opened. Never appears on PR check sets." + ), } diff --git a/.github/workflows/changelog-prestage.yml b/.github/workflows/changelog-prestage.yml new file mode 100644 index 0000000..aa90d41 --- /dev/null +++ b/.github/workflows/changelog-prestage.yml @@ -0,0 +1,169 @@ +name: Pre-stage CHANGELOG before release + +# Triggered manually (`workflow_dispatch`) before opening a release PR. +# Inserts the `## [] - ` heading + footer link into develop's +# CHANGELOG ahead of time, and merges main into develop on the prestage +# branch so develop's CHANGELOG already has the structural shape main +# carries. The release PR opened afterwards is conflict-free by +# construction. +# +# Why: release PRs commonly hit a same-line CHANGELOG conflict on the +# release-branch push because git sees both develop and main editing +# the region right after `## [Unreleased]`. The pre-#168 pattern was for +# the operator to manually `git merge origin/main` on the release branch +# every time; this workflow packages the same merge plus the heading +# insertion as a reviewable PR. +# +# Distinct from `changelog-rollup.yml`: rollup runs *after* a release +# tag is cut, bumps the version, and is auto-triggered by release.yml +# success. Prestage runs *before* the tag, doesn't bump version (the +# post-release rollup does), and is operator-triggered. Both share +# `.github/scripts/rollup_changelog.py` — prestage uses the `--no-bump` +# flag. + +on: + workflow_dispatch: + inputs: + tag: + description: "Tag about to be released (e.g. v1.11.0)" + required: true + type: string + prior_tag: + description: "Prior release tag for the compare link (auto-resolves if empty)" + required: false + type: string + default: "" + date: + description: "Release date YYYY-MM-DD (defaults to today UTC)" + required: false + type: string + default: "" + +permissions: + contents: write + pull-requests: write + +jobs: + prestage: + name: Open prestage PR + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: develop + # Full history so `git describe --abbrev=0 --tags` can resolve + # the prior tag, and the `git merge origin/main` step has a + # complete merge-base. + fetch-depth: 0 + # Prefer RELEASE_BOT_TOKEN so the prestage branch's push fires + # `pull_request` workflows on the auto-PR. Falls back to + # GITHUB_TOKEN when the secret isn't set — the auto-PR still + # opens, but its CI doesn't run until a user pushes on top. + token: ${{ secrets.RELEASE_BOT_TOKEN || secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.14" + + - name: Resolve inputs + id: resolve + run: | + set -euo pipefail + TAG="${{ inputs.tag }}" + if [ -z "${TAG}" ]; then + echo "::error::tag input is required" + exit 1 + fi + # Auto-resolve prior tag if not provided (mirrors changelog-rollup.yml). + PRIOR="${{ inputs.prior_tag }}" + if [ -z "${PRIOR}" ]; then + if PRIOR=$(git describe --abbrev=0 --tags --match 'v*.*.*' "${TAG}^" 2>/dev/null); then + echo "prior tag (resolved): ${PRIOR}" + else + # Try the most recent tag on main if `^` doesn't resolve + # (the tag isn't cut yet — that's the whole point of prestage). + if PRIOR=$(git describe --abbrev=0 --tags --match 'v*.*.*' origin/main 2>/dev/null); then + echo "prior tag (resolved from main): ${PRIOR}" + else + PRIOR="" + echo "no prior tag (first release)" + fi + fi + fi + DATE="${{ inputs.date }}" + if [ -z "${DATE}" ]; then + DATE=$(date -u +%Y-%m-%d) + fi + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" + echo "prior=${PRIOR}" >> "$GITHUB_OUTPUT" + echo "date=${DATE}" >> "$GITHUB_OUTPUT" + echo "branch=chore/changelog-prestage-${TAG}" >> "$GITHUB_OUTPUT" + + - name: Create prestage branch + merge main + env: + BRANCH: ${{ steps.resolve.outputs.branch }} + run: | + set -euo pipefail + # Idempotent: if the branch already exists from a previous replay, + # bail rather than force-push. + if git ls-remote --exit-code --heads origin "${BRANCH}" >/dev/null 2>&1; then + echo "::warning::branch ${BRANCH} already exists; skipping push to avoid clobbering an in-flight prestage PR" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -b "${BRANCH}" + # Bring main's CHANGELOG state into develop. Prefer auto-merge; + # if there's a real conflict (rare in steady-state — develop and + # main usually share their post-rollup history), fail fast with + # a clear message so the operator can resolve manually. + if ! git merge --no-edit origin/main; then + echo "::error::main → develop merge hit a conflict the prestage can't auto-resolve. Resolve manually on a local branch and re-run, or open the release PR by hand and merge main into the release branch (the pre-#168 pattern)." + git merge --abort + exit 1 + fi + + - name: Run rollup script in pre-stage mode + run: | + python .github/scripts/rollup_changelog.py \ + --tag "${{ steps.resolve.outputs.tag }}" \ + --prior-tag "${{ steps.resolve.outputs.prior }}" \ + --date "${{ steps.resolve.outputs.date }}" \ + --no-bump + + - name: Open prestage PR + env: + # Same fallback as the checkout step (#174) so the auto-PR + # fires `pull_request` workflows on creation when the secret + # is provisioned; falls back to GITHUB_TOKEN otherwise. + GH_TOKEN: ${{ secrets.RELEASE_BOT_TOKEN || secrets.GITHUB_TOKEN }} + BRANCH: ${{ steps.resolve.outputs.branch }} + TAG: ${{ steps.resolve.outputs.tag }} + DATE: ${{ steps.resolve.outputs.date }} + run: | + set -euo pipefail + git add CHANGELOG.md + git commit -m "chore: pre-stage CHANGELOG for ${TAG} - ${DATE}" + git push origin "${BRANCH}" + gh pr create \ + --base develop \ + --head "${BRANCH}" \ + --title "chore: pre-stage CHANGELOG for ${TAG} - ${DATE}" \ + --body "$(cat <...${TAG}\` footer link. + - **Does not** bump \`pyproject.toml\` / \`uv.lock\` — that's the post-release rollup's job. + + ## Operator next step + + Merge this PR into develop, then open the release PR develop → main with title \`release: ${TAG}\`. The release PR will be conflict-free because develop and main now agree on the CHANGELOG's structural shape. + + See [docs/DEVELOPMENT.md#creating-a-release](docs/DEVELOPMENT.md#creating-a-release) for the full cycle. + EOF + )" diff --git a/.github/workflows/pin-freshness-audit.yml b/.github/workflows/pin-freshness-audit.yml new file mode 100644 index 0000000..92bbf6f --- /dev/null +++ b/.github/workflows/pin-freshness-audit.yml @@ -0,0 +1,77 @@ +name: Pin freshness audit + +# Validates that every action pin (workflow + composite) still resolves +# upstream — closes the silently-deprecated-tag class. Complements the +# shape-only `Action pinning audit` job in ci.yml (which checks pin +# *shape* on every PR; this checks pin *freshness* on a schedule). +# +# Schedule: weekly + workflow_dispatch. Not on every PR — rate-limit +# aware (5000 req/hr per token, this audit costs ~70 req/run) and not +# blocking. Findings are surfaced as `::warning::` annotations and (when +# any are found) auto-file an issue tagged `harness,security`. +# +# Script + 15 unit tests live in +# `.github/scripts/check_pin_freshness.py` + `tests/test_check_pin_freshness.py`. + +on: + schedule: + # Monday 06:00 UTC — alongside artifact-cleanup. Avoids weekend + # noise; weekly cadence is enough for a non-blocking gate. + - cron: "0 6 * * 1" + workflow_dispatch: + inputs: + strict: + description: "Run in strict mode (findings → errors, exit 1)" + type: boolean + default: false + +permissions: + contents: read + issues: write + +jobs: + audit: + name: Pin freshness audit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.14" + + - name: Run pin-freshness audit + id: audit + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PIN_FRESHNESS_STRICT: ${{ inputs.strict && '1' || '' }} + run: python .github/scripts/check_pin_freshness.py + + - name: File issue on findings + # Only fire when default-mode (warn) found something — strict mode + # already failed the workflow and operator attention is automatic. + if: > + always() && + steps.audit.outputs.findings_count != '0' && + steps.audit.outputs.findings_count != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + FINDINGS: ${{ steps.audit.outputs.findings_count }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + gh issue create \ + --title "chore: pin-freshness audit found ${FINDINGS} stale pin(s)" \ + --label "harness,security" \ + --body "$(cat </-`. Open one issue per branch so the project board stays usable. ## Commit messages @@ -43,6 +43,18 @@ The subject is **lowercase** after the colon. Title Case prose (`Add the thing`) - **Screenshots** (UI changes only) 5. Wait for green CI + a code-owner review before merging. +### Solo-owner merge policy + +This repo runs with a single code owner (`* @constk` in `CODEOWNERS`). GitHub forbids a PR author from approving their own PR, so the standard "1 code-owner review" gate cannot be satisfied without an admin override. While in this state, the **intended workflow is**: + +```sh +gh pr merge --admin --squash --delete-branch +``` + +…for `feat:` / `fix:` / `chore:` PRs, and `--admin --merge` (preserves history) for `release:` PRs. The `enforce_admins: false` line in `.github/branch-protection/{develop,main}.json` is the documented escape hatch — admin merge here is the policy, not a deviation from it. + +When a second collaborator joins, drop the `--admin` flag and adopt standard PR review. Update this section + `CODEOWNERS` in the same PR. + ## Local pre-push gate ```sh diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index c844ffc..db7bb90 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -101,6 +101,9 @@ Subject is **lowercase after the colon** (Title Case is rejected unless it's an | `artifact-cleanup.yml` | weekly | No | | `eval-nightly.yml` | `workflow_dispatch` only by default | No | | `codeql.yml` | `workflow_dispatch` only (placeholder) | No | +| `pin-freshness-audit.yml` | weekly + `workflow_dispatch` | No — async second layer of action-pinning policy | +| `changelog-rollup.yml` | after `release.yml` succeeds + `workflow_dispatch` | No — opens a `chore: roll up CHANGELOG …` PR against develop | +| `changelog-prestage.yml` | `workflow_dispatch` only | No — operator-triggered before opening the release PR (closes the same-line CHANGELOG conflict class) | ### Action-pinning policy @@ -112,12 +115,29 @@ Audited by the `Action pinning audit` CI job (`.github/scripts/check_action_pins When bumping a third-party action, update the SHA *and* the trailing comment in the same PR. Dependabot's `github-actions` ecosystem opens those PRs automatically. +A second layer runs out-of-band: `.github/workflows/pin-freshness-audit.yml` (weekly Monday 06:00 UTC + `workflow_dispatch`) re-resolves every pin against `api.github.com`. It catches the silently-deprecated-tag class — a tag pin that no longer resolves, or a SHA pin whose documented `# vN.M.P` tag has been re-tagged upstream. Default warn-only with an auto-filed `harness,security` issue; `PIN_FRESHNESS_STRICT=1` (workflow_dispatch input) escalates to a hard failure. Where the on-PR `Action pinning audit` checks pin **shape**, this checks pin **freshness** — separate failure classes. + ### Version-bump policy Audited by the `Version bump check` CI job (`.github/scripts/check_version_bump.py`). Every PR bumps `[project] version` in `pyproject.toml` AND the matching `[[package]]` block in `uv.lock`. The bump direction follows commitizen's `bump_map` in `pyproject.toml` — `feat:` is MINOR, everything else is PATCH. `release:` PRs are exempt because the dev version IS the release version. The `uv.lock` self-version is hand-edited (one line); avoid `uv lock` mid-PR because it would re-resolve transitive deps and pull in unintended upgrades. The `Version bump check` gate enforces both halves. +### Creating a release + +The release flow chains four workflows and one script: + +1. **Pre-stage CHANGELOG** (`changelog-prestage.yml`, manual dispatch) — pass the new tag (e.g. `v0.3.0`); the workflow opens a `chore: pre-stage CHANGELOG …` PR against develop that merges `origin/main` into the branch and inserts the new `## [] - ` heading + footer compare-link. Merge that PR. +2. **Open the release PR** — `release: vX.Y.Z` from `develop` → `main`. Conflict-free now that develop has main's CHANGELOG shape. Admin-merge with `gh pr merge --admin --merge` once green. +3. **Tag the merge commit** — `git tag vX.Y.Z && git push origin vX.Y.Z`. Triggers `release.yml`. +4. **`release.yml`** builds the image, pushes to GHCR, generates the CycloneDX SBOM, publishes the GitHub Release. +5. **`changelog-rollup.yml`** auto-fires on the successful release and opens a `chore: roll up CHANGELOG …` PR against develop that bumps `pyproject.toml` + `uv.lock` PATCH (so develop's `[Unreleased]` section can accumulate again). + +The shared script is `.github/scripts/rollup_changelog.py`: + +- `rollup_changelog.py --tag vX.Y.Z --prior-tag vA.B.C --date YYYY-MM-DD` — full rollup (CHANGELOG edits + version bump). Used by `changelog-rollup.yml` post-release. +- `rollup_changelog.py … --no-bump` — CHANGELOG edits only. Used by `changelog-prestage.yml` pre-release. + ### Testing policy Audited by the `Tests required` CI job (`.github/scripts/check_tests_present.py`). `feat:` and `fix:` PRs that touch `src/` MUST also touch `tests/`. Other prefixes get a `::warning::` if `src/` is touched without tests but don't block. The 75 % coverage gate alone doesn't catch behaviour-change-without-test (a single new line on already-covered code can pass), which is why this is a separate axis. diff --git a/pyproject.toml b/pyproject.toml index 1d80f8d..c1f5158 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "harness-python-react" -version = "0.2.6" +version = "0.2.9" description = "Production-quality LLM-driven coding harness — Python (FastAPI) backend, Vite + React + TypeScript frontend." readme = "README.md" requires-python = ">=3.14" diff --git a/tests/test_check_pin_freshness.py b/tests/test_check_pin_freshness.py new file mode 100644 index 0000000..4f62ee2 --- /dev/null +++ b/tests/test_check_pin_freshness.py @@ -0,0 +1,243 @@ +"""Tests for `.github/scripts/check_pin_freshness.py` (#136). + +Mocks `_fetch_json` (the only network seam) and exercises: + +- Tag pin resolution (lightweight + annotated tag). +- SHA pin re-tag detection (upstream tag now points at a different SHA). +- 404 / network failure handling (warn, never fail). +- Strict mode (`PIN_FRESHNESS_STRICT=1`) escalates warnings to failures. +- `GITHUB_OUTPUT` integration writes `findings_count` for the workflow. +- `GITHUB_TOKEN` missing → exit 2 (script-level error). +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any +from unittest.mock import patch + +if TYPE_CHECKING: + import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +SCRIPT_PATH = REPO_ROOT / ".github" / "scripts" / "check_pin_freshness.py" + + +def _load_script() -> Any: + spec = importlib.util.spec_from_file_location("check_pin_freshness", SCRIPT_PATH) + if spec is None or spec.loader is None: + msg = f"Could not load script at {SCRIPT_PATH}" + raise RuntimeError(msg) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +cpf = _load_script() + + +def _make_ref(action: str, pin: str, comment: str | None = None) -> Any: + """Build a minimal ActionRef-shaped object for the freshness checks.""" + return cpf._pins.ActionRef( + file=Path("fake.yml"), line=1, action=action, pin=pin, comment=comment + ) + + +# ---------- _resolve_tag_sha ---------- + + +def test_resolve_lightweight_tag() -> None: + """Lightweight tag → object.type == 'commit', sha is the commit SHA.""" + ref_payload = {"object": {"type": "commit", "sha": "abc" * 13 + "abcd"}} + with patch.object(cpf, "_fetch_json", return_value=ref_payload): + assert cpf._resolve_tag_sha("foo/bar", "v1.0.0", "fake") == "abc" * 13 + "abcd" + + +def test_resolve_annotated_tag() -> None: + """Annotated tag → two GETs; second dereferences the tag object to commit.""" + ref_payload = {"object": {"type": "tag", "sha": "tagobj_sha"}} + tag_payload = {"object": {"sha": "commit_sha"}} + with patch.object(cpf, "_fetch_json", side_effect=[ref_payload, tag_payload]): + assert cpf._resolve_tag_sha("foo/bar", "v1.0.0", "fake") == "commit_sha" + + +def test_resolve_returns_none_on_404() -> None: + """`_fetch_json` returning None propagates as None — no crash.""" + with patch.object(cpf, "_fetch_json", return_value=None): + assert cpf._resolve_tag_sha("foo/bar", "v9.9.9", "fake") is None + + +def test_resolve_returns_none_on_malformed_payload() -> None: + """Missing object / non-string sha → None (defensive).""" + with patch.object(cpf, "_fetch_json", return_value={"unrelated": "shape"}): + assert cpf._resolve_tag_sha("foo/bar", "v1.0.0", "fake") is None + + +# ---------- _check_tag_pin ---------- + + +def test_tag_pin_passes_when_resolved() -> None: + ref = _make_ref("actions/checkout", "v4") + with patch.object(cpf, "_resolve_tag_sha", return_value="some_sha"): + assert cpf._check_tag_pin(ref, "fake") is None + + +def test_tag_pin_warns_when_unresolved() -> None: + ref = _make_ref("astral-sh/setup-uv", "v5") + with patch.object(cpf, "_resolve_tag_sha", return_value=None): + message = cpf._check_tag_pin(ref, "fake") + assert message is not None + assert "v5" in message + assert "no longer resolves" in message + + +# ---------- _check_sha_pin ---------- + + +def test_sha_pin_passes_when_tag_still_resolves_to_pin() -> None: + sha = "a" * 40 + ref = _make_ref("aquasecurity/trivy-action", sha, comment="# v0.36.0") + with patch.object(cpf, "_resolve_tag_sha", return_value=sha): + assert cpf._check_sha_pin(ref, "fake") is None + + +def test_sha_pin_warns_on_retag() -> None: + """Upstream re-tag: same tag now resolves to a different SHA.""" + pinned = "a" * 40 + upstream = "b" * 40 + ref = _make_ref("aquasecurity/trivy-action", pinned, comment="# v0.36.0") + with patch.object(cpf, "_resolve_tag_sha", return_value=upstream): + message = cpf._check_sha_pin(ref, "fake") + assert message is not None + assert "re-tagged" in message + assert "v0.36.0" in message + + +def test_sha_pin_warns_when_documented_tag_404() -> None: + sha = "a" * 40 + ref = _make_ref("aquasecurity/trivy-action", sha, comment="# v0.36.0") + with patch.object(cpf, "_resolve_tag_sha", return_value=None): + message = cpf._check_sha_pin(ref, "fake") + assert message is not None + assert "no longer resolves" in message + + +def test_sha_pin_silent_without_comment() -> None: + """Missing comment is the shape audit's job, not freshness.""" + sha = "a" * 40 + ref = _make_ref("aquasecurity/trivy-action", sha, comment=None) + assert cpf._check_sha_pin(ref, "fake") is None + + +# ---------- main() ---------- + + +def _setup_workflow_dir(tmp_path: Path) -> Path: + workflows = tmp_path / "workflows" + workflows.mkdir() + (workflows / "ci.yml").write_text( + "jobs:\n j:\n steps:\n" + " - uses: actions/checkout@v4\n" + " - uses: aquasecurity/trivy-action@" + ("a" * 40) + " # v0.36.0\n", + encoding="utf-8", + ) + # `tmp_path / "actions"` deliberately not created — exercises the + # optional-dir branch in `_collect_yaml_files`. + return workflows + + +def test_main_exits_2_without_token( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + assert cpf.main() == 2 + assert "GITHUB_TOKEN required" in capsys.readouterr().out + + +def test_main_warns_not_fails_in_default_mode( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Stale tag → ::warning::, exit 0 (gate not a tripwire).""" + workflows = _setup_workflow_dir(tmp_path) + monkeypatch.setenv("GITHUB_TOKEN", "fake") + monkeypatch.delenv("PIN_FRESHNESS_STRICT", raising=False) + monkeypatch.delenv("GITHUB_OUTPUT", raising=False) + with ( + patch.object(cpf._pins, "WORKFLOWS_DIR", workflows), + patch.object(cpf._pins, "ACTIONS_DIR", tmp_path / "no-such"), + # checkout@v4 resolves; trivy SHA's documented tag has been re-tagged. + patch.object( + cpf, + "_resolve_tag_sha", + side_effect=["abc" * 13 + "abcd", "b" * 40], + ), + ): + assert cpf.main() == 0 + out = capsys.readouterr().out + assert "::warning" in out + assert "re-tagged" in out + + +def test_main_strict_mode_fails_on_finding( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + workflows = _setup_workflow_dir(tmp_path) + monkeypatch.setenv("GITHUB_TOKEN", "fake") + monkeypatch.setenv("PIN_FRESHNESS_STRICT", "1") + monkeypatch.delenv("GITHUB_OUTPUT", raising=False) + with ( + patch.object(cpf._pins, "WORKFLOWS_DIR", workflows), + patch.object(cpf._pins, "ACTIONS_DIR", tmp_path / "no-such"), + patch.object( + cpf, + "_resolve_tag_sha", + side_effect=["abc" * 13 + "abcd", None], + ), + ): + assert cpf.main() == 1 + assert "::error" in capsys.readouterr().out + + +def test_main_writes_findings_count_to_github_output( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """`$GITHUB_OUTPUT` integration so the calling workflow can branch on it.""" + workflows = _setup_workflow_dir(tmp_path) + output_file = tmp_path / "github_output" + output_file.write_text("", encoding="utf-8") + monkeypatch.setenv("GITHUB_TOKEN", "fake") + monkeypatch.setenv("GITHUB_OUTPUT", str(output_file)) + monkeypatch.delenv("PIN_FRESHNESS_STRICT", raising=False) + with ( + patch.object(cpf._pins, "WORKFLOWS_DIR", workflows), + patch.object(cpf._pins, "ACTIONS_DIR", tmp_path / "no-such"), + patch.object( + cpf, + "_resolve_tag_sha", + side_effect=["abc" * 13 + "abcd", "b" * 40], + ), + ): + assert cpf.main() == 0 + assert "findings_count=1" in output_file.read_text(encoding="utf-8") + + +def test_main_passes_clean_when_all_resolve( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + workflows = _setup_workflow_dir(tmp_path) + monkeypatch.setenv("GITHUB_TOKEN", "fake") + monkeypatch.delenv("PIN_FRESHNESS_STRICT", raising=False) + monkeypatch.delenv("GITHUB_OUTPUT", raising=False) + pin_sha = "a" * 40 + with ( + patch.object(cpf._pins, "WORKFLOWS_DIR", workflows), + patch.object(cpf._pins, "ACTIONS_DIR", tmp_path / "no-such"), + patch.object(cpf, "_resolve_tag_sha", side_effect=["x" * 40, pin_sha]), + ): + assert cpf.main() == 0 + out = capsys.readouterr().out + assert "0 finding(s)" in out + assert "::warning" not in out diff --git a/uv.lock b/uv.lock index c20d4d6..8fe0b4f 100644 --- a/uv.lock +++ b/uv.lock @@ -328,7 +328,7 @@ wheels = [ [[package]] name = "harness-python-react" -version = "0.2.6" +version = "0.2.9" source = { virtual = "." } dependencies = [ { name = "fastapi" },