From 165a67b94f682c4c26f114c5b601f42a70d0ef54 Mon Sep 17 00:00:00 2001 From: Astrid Gealer Date: Sun, 25 Oct 2026 00:13:50 +0000 Subject: [PATCH] ci(security): cache gosec results and binary across runs The gosec step in the security-scan job took 304s of the job's ~7 minutes on a recent green main run. gosec keeps nothing between runs, so it re-parses and type-checks every package and dependency from source, and CI never saved the binary that scripts/run-gosec.sh caches, so every run also did `go install gosec`. Frontend-only PRs paid all of it for a verdict that could not have changed. Result cache: a new step hashes every input to gosec's verdict (all .go files with their paths, every go.mod/go.sum, go.work/go.work.sum, scripts/run-gosec.sh and ci.yml) and keys it with the pinned gosec version, the effective Go version, runner OS and arch. On an exact hit the restored, already-merged SARIF goes straight to the existing upload step, so the Security tab stays populated. On a miss the scan runs exactly as before. Only a clean scan (exit 0, SARIF merged) is saved, so a finding or a scan error is never replayed from the cache. Binary cache: ~/.cache/gosec/- is restored and saved with split actions/cache steps keyed on the same two versions, so a failing scanner elsewhere in the job does not stop the save. run-gosec.sh gains --print-versions so the workflow reads the gosec pin, Go version and binary cache dir from the script instead of duplicating them; test-run-gosec.sh covers it. The cache steps are an optimisation only: if computing the key fails, they are skipped and gosec runs uncached. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 108 +++++++++++++++++++++++++++++++++++++- scripts/run-gosec.sh | 18 +++++++ scripts/test-run-gosec.sh | 18 +++++++ 3 files changed, 142 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d60f27cad..7b4654166 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -820,13 +820,89 @@ jobs: if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success' run: bash scripts/test-run-gosec.sh + # gosec result cache. A full scan re-parses and type-checks every package + # and dependency from source (about 300s), even when no Go input changed, + # as on a frontend-only PR. gosec's verdict depends only on the inputs + # hashed below, so a run whose inputs exactly match an earlier clean scan + # would reach the same verdict. That run reuses the clean scan's merged + # SARIF instead. + - name: Compute gosec cache keys + id: gosec_key + # An optimisation only: if this step breaks, the restores and saves + # below are skipped and gosec scans exactly as it would uncached. + continue-on-error: true + if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success' + run: | + set -euo pipefail + # Pinned gosec, effective Go version and binary cache dir, read from + # scripts/run-gosec.sh so the pin lives in one place. + versions=$(scripts/run-gosec.sh --print-versions) + printf '%s\n' "$versions" | tee -a "$GITHUB_OUTPUT" + gosec_version=$(printf '%s\n' "$versions" | sed -n 's/^gosec_version=//p') + go_version=$(printf '%s\n' "$versions" | sed -n 's/^go_version=//p') + # Every input to gosec's verdict: all Go sources (both scanned modules + # and anything else on disk, testdata included), every go.mod/go.sum + # (go.sum pins dependency content), go.work/go.work.sum, the wrapper + # (pin, rule set, exclude dirs) and this workflow (the SARIF merge + # below). sha256sum lines carry the path as well as the content, so a + # moved or renamed file changes the key; hashFiles() would not. + # Including too much only costs a miss. + inputs_hash=$( + { + find . -path ./.git -prune -o -type f \( -name '*.go' -o -name go.mod \ + -o -name go.sum -o -name go.work -o -name go.work.sum \) -print0 + printf '%s\0' scripts/run-gosec.sh .github/workflows/ci.yml + } | LC_ALL=C sort -z | xargs -0 sha256sum | sha256sum | cut -d' ' -f1 + ) + echo "results_key=gosec-sarif-${RUNNER_OS}-${RUNNER_ARCH}-${gosec_version}-${go_version}-${inputs_hash}" | tee -a "$GITHUB_OUTPUT" + echo "bin_key=gosec-bin-${RUNNER_OS}-${RUNNER_ARCH}-${gosec_version}-${go_version}" | tee -a "$GITHUB_OUTPUT" + + - name: Restore gosec results + id: gosec_results + # Exact key only (no restore-keys): a result from different inputs is + # never reused. A miss or a cache service error leaves no file, and the + # scan below runs as usual. + if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success' && steps.gosec_key.outcome == 'success' + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ runner.temp }}/gosec-results.sarif + key: ${{ steps.gosec_key.outputs.results_key }} + + # Split restore/save rather than the combined actions/cache, whose save + # is a post step gated on `post-if: success()`: any failing scanner in + # this job would stop the binary being saved. Not needed on a results + # hit, which runs no gosec. + - name: Restore gosec binary + id: gosec_bin + if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success' && steps.gosec_key.outcome == 'success' && steps.gosec_results.outputs.cache-hit != 'true' + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.gosec_key.outputs.bin_dir }} + key: ${{ steps.gosec_key.outputs.bin_key }} + - name: Run gosec Security Scanner id: gosec # always(): don't let an earlier scanner's failure (e.g. npm audit) # skip Go SAST coverage. The job still fails if gosec itself fails. # Still requires checkout and Go setup to have succeeded. if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success' + env: + RESULTS_CACHE_HIT: ${{ steps.gosec_results.outputs.cache-hit }} run: | + # On an exact results-cache hit the restored SARIF is the merged + # output of a clean scan of identical inputs (only clean scans are + # saved, see "Save gosec results"), so hand it to the upload step + # as-is. If it is unreadable, fall through to a real scan. + merged="$RUNNER_TEMP/gosec-results.sarif" + if [ "$RESULTS_CACHE_HIT" = "true" ]; then + if jq -e '.runs | length > 0' "$merged" >/dev/null 2>&1; then + echo "Go inputs match a clean gosec scan: reusing its SARIF ($(jq '.runs | length' "$merged") runs)" + exit 0 + fi + echo "::warning::restored gosec SARIF is missing or unreadable; scanning instead" + rm -f "$merged" + fi + # scripts/run-gosec.sh owns the pinned gosec version, toolchain, # module list and rule set, shared with the pre-commit hook and # `make security-scan-go` (issue #340). It installs gosec with the @@ -836,11 +912,17 @@ jobs: # # gosec exits non-zero both for real findings and for a processing # error. The script still scans every module after a failure, and - # `|| status=1` exempts it from errexit so the merge below always + # `|| rc=$?` exempts it from errexit so the merge below always # runs (issue #1717); `status` records whether the job should still # fail at the end. status=0 - scripts/run-gosec.sh --sarif-dir "$RUNNER_TEMP" || status=1 + rc=0 + scripts/run-gosec.sh --sarif-dir "$RUNNER_TEMP" || rc=$? + [ "$rc" -eq 0 ] || status=1 + # run-gosec.sh exits 2 only on a setup failure (a failed install + # included). Any other exit means the binary is in its cache dir, + # ready for "Save gosec binary". + [ "$rc" -eq 2 ] || echo "binary_ready=true" >> "$GITHUB_OUTPUT" shopt -s nullglob for out in "$RUNNER_TEMP"/gosec-*.sarif; do tag=$(basename "$out" .sarif) @@ -867,6 +949,9 @@ jobs: jq -s '{version: "2.1.0", "$schema": "https://json.schemastore.org/sarif-2.1.0.json", runs: [.[].runs[]]}' \ "${sarif_files[@]}" > "$merged" echo "Merged $(jq '.runs | length' "$merged") SARIF runs" + # Only a clean scan (exit 0, SARIF merged) may be cached, so a + # finding or a scan error is never replayed from the cache. + [ "$status" -ne 0 ] || echo "clean_scan=true" >> "$GITHUB_OUTPUT" exit "$status" - name: Upload gosec results to GitHub Security @@ -879,6 +964,25 @@ jobs: with: sarif_file: ${{ runner.temp }}/gosec-results.sarif + - name: Save gosec results + # Only after a clean scan of these exact inputs (never a hit, a + # finding or a scan error). A pull_request run saves to its own + # merge-ref scope, which main and other PRs cannot restore from. + if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success' && steps.gosec_key.outcome == 'success' && steps.gosec.outputs.clean_scan == 'true' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ runner.temp }}/gosec-results.sarif + key: ${{ steps.gosec_key.outputs.results_key }} + + - name: Save gosec binary + # Saved even when gosec reports findings: the binary depends only on + # the gosec and Go versions in the key, not on the scan. + if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success' && steps.gosec_key.outcome == 'success' && steps.gosec_bin.outputs.cache-hit != 'true' && steps.gosec.outputs.binary_ready == 'true' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ${{ steps.gosec_key.outputs.bin_dir }} + key: ${{ steps.gosec_key.outputs.bin_key }} + - name: Run Trivy vulnerability scanner (filesystem) id: trivy_fs # always(): don't let an earlier scanner's failure skip Trivy fs diff --git a/scripts/run-gosec.sh b/scripts/run-gosec.sh index 120867266..9bd4c5b6c 100755 --- a/scripts/run-gosec.sh +++ b/scripts/run-gosec.sh @@ -11,6 +11,7 @@ # Usage: # run-gosec.sh [--sarif-dir DIR] [--module MOD [PKG...]]... # run-gosec.sh --list-modules +# run-gosec.sh --print-versions # # With no --module every module is scanned whole (./...). PKG paths are # relative to the module root (for example ./internal/api); default ./... @@ -18,6 +19,11 @@ # where is "root" for the root module and the module path with slashes # turned into dashes otherwise (tests/e2e -> tests-e2e). # +# --print-versions prints gosec_version=, go_version= and bin_dir= lines (the +# pinned gosec, the effective Go version and the binary cache directory) and +# exits without installing or scanning. CI keys its gosec caches on them, so +# they are never duplicated in the workflow. +# # Exit codes: 0 clean, 1 findings or a scan error, 2 setup/usage failure. # # The gosec binary is cached per (gosec version, effective Go version), because @@ -37,6 +43,13 @@ if [[ "${1:-}" == "--list-modules" ]]; then exit 0 fi +print_versions=0 +if [[ "${1:-}" == "--print-versions" ]]; then + [[ $# -eq 1 ]] || { echo "run-gosec: --print-versions takes no other arguments" >&2; exit 2; } + print_versions=1 + shift +fi + REPO_ROOT="$(git rev-parse --show-toplevel)" sarif_dir="" @@ -81,6 +94,11 @@ cache_root="${XDG_CACHE_HOME:-${HOME}/.cache}/gosec" gosec_dir="${cache_root}/${GOSEC_VERSION}-${go_version}" gosec_bin="${gosec_dir}/gosec" +if [[ $print_versions -eq 1 ]]; then + printf 'gosec_version=%s\ngo_version=%s\nbin_dir=%s\n' "$GOSEC_VERSION" "$go_version" "$gosec_dir" + exit 0 +fi + if [[ ! -x "$gosec_bin" ]]; then echo "run-gosec: installing gosec@${GOSEC_VERSION} (${go_version}) -> ${gosec_dir}" >&2 mkdir -p "$cache_root" diff --git a/scripts/test-run-gosec.sh b/scripts/test-run-gosec.sh index 587091573..f96766f74 100755 --- a/scripts/test-run-gosec.sh +++ b/scripts/test-run-gosec.sh @@ -131,6 +131,24 @@ if run "unknown argument" 2 bash "$SCRIPT_DIR/run-gosec.sh" -exclude=G104; then pass "extra gosec flags such as -exclude are rejected" fi +# CI keys its gosec result and binary caches on these lines, so they must name +# the pin, the Go version and the directory the binary is really cached in. +if run "print versions" 0 bash "$SCRIPT_DIR/run-gosec.sh" --print-versions; then + want="$(printf 'gosec_version=%s\ngo_version=%s\nbin_dir=%s' \ + "$gosec_version" "$(go env GOVERSION)" "$fake_dir")" + if [[ "$(cat "$work/out.txt")" == "$want" ]]; then + pass "--print-versions names the pin, Go version and binary cache dir" + else + failed "--print-versions names the pin, Go version and binary cache dir" + printf ' want:\n%s\n got:\n%s\n' "$want" "$(cat "$work/out.txt")" + fi + expect_calls "--print-versions runs no scan" +fi + +if run "print versions with extra arguments" 2 bash "$SCRIPT_DIR/run-gosec.sh" --print-versions --sarif-dir x; then + pass "--print-versions takes no other arguments" +fi + # --- pre-commit hook ----------------------------------------------------------- # Invoked through run (below), which shellcheck cannot see.