From 518c32a93b235ce92fe6cee4e2fd5a4c6adb1a71 Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Tue, 20 Oct 2026 17:28:53 +0200 Subject: [PATCH 1/3] ci: add a pinned gitleaks scan to the security job Run the gitleaks CLI (not gitleaks-action, which needs an org licence) over the checkout in the security-scan job. The v8.30.1 release tarball is pinned by version and sha256, and .gitleaks.toml extends the default rules with three file-and-literal-scoped allowlists for existing test fixtures. Any finding fails the job. Closes #101 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 21 +++++++++++++++++++++ .gitleaks.toml | 25 +++++++++++++++++++++++++ CHANGELOG.md | 1 + 3 files changed, 47 insertions(+) create mode 100644 .gitleaks.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f89d317e..2b3df9fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -465,6 +465,27 @@ jobs: go-version-file: pkg/go.mod cache-dependency-path: '**/go.sum' + - name: Secret scan (gitleaks) + # The gitleaks CLI, not gitleaks-action (which needs an org licence). + # The release tarball is pinned by version and sha256 (from the + # release's gitleaks__checksums.txt), so a moved tag or a + # tampered asset fails the step. Scans the checked-out tree with the + # repo's .gitleaks.toml; any finding exits 1. + if: always() && steps.checkout.outcome == 'success' + env: + GITLEAKS_VERSION: '8.30.1' + GITLEAKS_SHA256: '551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb' + run: | + set -euo pipefail + archive="${RUNNER_TEMP}/gitleaks.tar.gz" + curl -fsSL -o "${archive}" \ + "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + echo "${GITLEAKS_SHA256} ${archive}" | sha256sum -c - + mkdir "${RUNNER_TEMP}/gitleaks" + tar -xzf "${archive}" -C "${RUNNER_TEMP}/gitleaks" gitleaks + "${RUNNER_TEMP}/gitleaks/gitleaks" dir . \ + --config .gitleaks.toml --redact --no-banner --exit-code 1 + - name: Run govulncheck CVE scanner (all modules) # Independent scanners still run after another scanner fails. if: always() && steps.checkout.outcome == 'success' && steps.setup_go.outcome == 'success' diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..49d0aa49 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,25 @@ +# gitleaks configuration for the CI "Secret scan (gitleaks)" step. +# Extends the default ruleset; every allowlist below is scoped to one file AND +# one literal, so a real secret added to the same file still fails the scan. +title = "cloud-commitments-go gitleaks config" + +[extend] +useDefault = true + +[[allowlists]] +description = "git-secrets 1.3.0 release commit SHA pinned in pre-commit.yml" +condition = "AND" +paths = ['''\.github/workflows/pre-commit\.yml$'''] +regexes = ['''ad82d68ee924906a0401dfd48de5057731a9bc84'''] + +[[allowlists]] +description = "synthetic bearer token fixture in purchaseguard tests" +condition = "AND" +paths = ['''providers/aws/internal/purchaseguard/guard_test\.go$'''] +regexes = ['''tok-0123456789abcdef'''] + +[[allowlists]] +description = "synthetic hex token fixture in the Azure idempotency marker tests" +condition = "AND" +paths = ['''providers/azure/services/internal/reservations/idempotencymarker_test\.go$'''] +regexes = ['''4f1c2b0e9d8a7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a2f1e0d9c8b7a6f5e4d3c2b'''] diff --git a/CHANGELOG.md b/CHANGELOG.md index e9fcf8b1..620b2109 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -141,6 +141,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). - **`provider`:** `ProviderConfig` is renamed to `Config` (#39). `ProviderConfig` stays as a deprecated type alias, so existing consumers keep compiling; migrate to `provider.Config`. The provider modules move over once their pinned `pkg` version includes the rename. - **`provider`:** `ProviderConfig` is renamed to `Config` (#39). `ProviderConfig` stays as a type alias, so existing consumers keep compiling; migrate to `provider.Config`. The provider modules move over once their pinned `pkg` version includes the rename. - **`httpclient`:** dials to the RFC 8215 local-use NAT64 prefix `64:ff9b:1::/48` are refused when any RFC 6052 layout (/48, /56, /64, /96) embeds a metadata IPv4 address; Teredo `2001::/32` stays allowed (#238). +- **CI:** the security-scan job runs a pinned gitleaks CLI (v8.30.1, sha256-verified) over the checkout with `.gitleaks.toml` and fails on any finding (#101). - **`common.HoursPerMonth`:** exported 730-hour average month (`HoursPerYear / 12`) next to `HoursPerYear`; the remaining bare 730 and 8760 literals in the aws and gcp providers use the shared constants (#109). - **`gcp`:** region-lookup permission errors are now classified by type only (googleapi 403 or gRPC `PermissionDenied`) through the shared `IsPermissionError`. gRPC `PermissionDenied` is now skipped with a warning like a 403, and an untyped error whose text merely contains "403" and "permission" now propagates instead of being skipped (#6). - **BREAKING `gcp`: stored contexts removed (#6).** `computeengine.NewClient` From bace1c194290fd1083ad80012c73410f75e6bbd6 Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Tue, 20 Oct 2026 18:15:58 +0200 Subject: [PATCH 2/3] fix(ci): scope gitleaks allowlists by literal only and self-test them In gitleaks 8.30.1 an allowlist with paths and condition = "AND" skips the whole file, so a real token planted next to a fixture went undetected. Match only the exact fixture literals, and add scripts/test-gitleaks-config.sh (run in CI after the scan) which plants a PAT-shaped string beside each fixture and requires the scan to fail. Refs #101 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 ++- .gitleaks.toml | 12 +++------ scripts/test-gitleaks-config.sh | 47 +++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 scripts/test-gitleaks-config.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b3df9fc..3b004e08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -470,7 +470,8 @@ jobs: # The release tarball is pinned by version and sha256 (from the # release's gitleaks__checksums.txt), so a moved tag or a # tampered asset fails the step. Scans the checked-out tree with the - # repo's .gitleaks.toml; any finding exits 1. + # repo's .gitleaks.toml; any finding exits 1. The config self-test then + # proves a secret planted beside an allowlisted fixture is still caught. if: always() && steps.checkout.outcome == 'success' env: GITLEAKS_VERSION: '8.30.1' @@ -485,6 +486,7 @@ jobs: tar -xzf "${archive}" -C "${RUNNER_TEMP}/gitleaks" gitleaks "${RUNNER_TEMP}/gitleaks/gitleaks" dir . \ --config .gitleaks.toml --redact --no-banner --exit-code 1 + GITLEAKS="${RUNNER_TEMP}/gitleaks/gitleaks" bash scripts/test-gitleaks-config.sh - name: Run govulncheck CVE scanner (all modules) # Independent scanners still run after another scanner fails. diff --git a/.gitleaks.toml b/.gitleaks.toml index 49d0aa49..ecfdb549 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -1,6 +1,8 @@ # gitleaks configuration for the CI "Secret scan (gitleaks)" step. -# Extends the default ruleset; every allowlist below is scoped to one file AND -# one literal, so a real secret added to the same file still fails the scan. +# Extends the default ruleset; every allowlist below matches one exact literal. +# Do NOT add `paths` with condition = "AND" here: in gitleaks 8.30.1 that skips +# the whole file, so a real secret planted next to a fixture goes undetected. +# scripts/test-gitleaks-config.sh guards this. title = "cloud-commitments-go gitleaks config" [extend] @@ -8,18 +10,12 @@ useDefault = true [[allowlists]] description = "git-secrets 1.3.0 release commit SHA pinned in pre-commit.yml" -condition = "AND" -paths = ['''\.github/workflows/pre-commit\.yml$'''] regexes = ['''ad82d68ee924906a0401dfd48de5057731a9bc84'''] [[allowlists]] description = "synthetic bearer token fixture in purchaseguard tests" -condition = "AND" -paths = ['''providers/aws/internal/purchaseguard/guard_test\.go$'''] regexes = ['''tok-0123456789abcdef'''] [[allowlists]] description = "synthetic hex token fixture in the Azure idempotency marker tests" -condition = "AND" -paths = ['''providers/azure/services/internal/reservations/idempotencymarker_test\.go$'''] regexes = ['''4f1c2b0e9d8a7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a2f1e0d9c8b7a6f5e4d3c2b'''] diff --git a/scripts/test-gitleaks-config.sh b/scripts/test-gitleaks-config.sh new file mode 100644 index 00000000..ae523e05 --- /dev/null +++ b/scripts/test-gitleaks-config.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Self-test for .gitleaks.toml: each allowlisted fixture literal must scan +# clean, and a real-shaped secret planted in the same file must still fail +# (gitleaks 8.30.1 skips a whole file when an allowlist combines `paths` with +# condition = "AND"). Usage: GITLEAKS=/path/to/gitleaks bash scripts/test-gitleaks-config.sh +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GITLEAKS="${GITLEAKS:-gitleaks}" +command -v "$GITLEAKS" >/dev/null || { echo "gitleaks not found (set GITLEAKS)" >&2; exit 1; } + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +# Assembled at runtime so this file itself scans clean. +PAT="ghp""_aB3dE5gH7jK9mN1pQ3sT5vW7yZ9bC1dE3fG5" + +scan() { "$GITLEAKS" dir "$1" --config "$REPO_ROOT/.gitleaks.toml" --no-banner --redact --exit-code 1 >/dev/null 2>&1; } + +pass=0 fail=0 +check() { # label expected_rc dir + local rc=0 + scan "$3" || rc=$? + if [ "$rc" -eq "$2" ]; then pass=$((pass + 1)); else fail=$((fail + 1)); echo "FAIL: $1 (expected rc $2, got $rc)" >&2; fi +} + +# Every allowlist entry: its path, and its literal (read back from the config). +i=0 +while IFS='|' read -r path literal; do + i=$((i + 1)) + d="$tmp/c$i" + mkdir -p "$d/$(dirname "$path")" + printf 'const token = "%s"\n' "$literal" >"$d/$path" + check "fixture literal in $path is allowed" 0 "$d" + printf 'const real = "%s"\n' "$PAT" >>"$d/$path" + check "real-shaped PAT beside the fixture in $path is caught" 1 "$d" +done <<'ENTRIES' +.github/workflows/pre-commit.yml|ad82d68ee924906a0401dfd48de5057731a9bc84 +providers/aws/internal/purchaseguard/guard_test.go|tok-0123456789abcdef +providers/azure/services/internal/reservations/idempotencymarker_test.go|4f1c2b0e9d8a7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a2f1e0d9c8b7a6f5e4d3c2b +ENTRIES + +# The real tree must be clean under the repo config. +check "repository tree is clean" 0 "$REPO_ROOT" + +echo "$pass PASS, $fail FAIL" +[ "$fail" -eq 0 ] From 7a05fb31bacf66863ccbb65edd7762c5010b363a Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Tue, 20 Oct 2026 20:44:13 +0200 Subject: [PATCH 3/3] fix(ci): anchor gitleaks allowlist regexes and harden the self-test Anchor each allowlist regex with ^...$ so only the exact fixture secret is suppressed, not a longer token containing it. The self-test now reads the literals from the config, fails if the config uses paths, condition, stopwords, commits, targetRules or regexTarget (and proves an injected key is detected), and checks that a fixture plus a suffix is caught. Refs #101 Co-Authored-By: Claude Opus 5.5 --- .gitleaks.toml | 16 ++++---- scripts/test-gitleaks-config.sh | 69 +++++++++++++++++++++++---------- 2 files changed, 57 insertions(+), 28 deletions(-) diff --git a/.gitleaks.toml b/.gitleaks.toml index ecfdb549..583f614a 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -1,8 +1,10 @@ # gitleaks configuration for the CI "Secret scan (gitleaks)" step. -# Extends the default ruleset; every allowlist below matches one exact literal. -# Do NOT add `paths` with condition = "AND" here: in gitleaks 8.30.1 that skips -# the whole file, so a real secret planted next to a fixture goes undetected. -# scripts/test-gitleaks-config.sh guards this. +# Extends the default ruleset; every allowlist below is a regex anchored with +# ^...$ so it matches only the exact fixture secret, never a longer token that +# merely contains it. Do NOT add `paths`, `condition`, `stopwords`, `commits`, +# `targetRules` or `regexTarget`: in gitleaks 8.30.1 `paths` with +# condition = "AND" skips the whole file, so a real secret planted next to a +# fixture goes undetected. scripts/test-gitleaks-config.sh enforces this. title = "cloud-commitments-go gitleaks config" [extend] @@ -10,12 +12,12 @@ useDefault = true [[allowlists]] description = "git-secrets 1.3.0 release commit SHA pinned in pre-commit.yml" -regexes = ['''ad82d68ee924906a0401dfd48de5057731a9bc84'''] +regexes = ['''^ad82d68ee924906a0401dfd48de5057731a9bc84$'''] [[allowlists]] description = "synthetic bearer token fixture in purchaseguard tests" -regexes = ['''tok-0123456789abcdef'''] +regexes = ['''^tok-0123456789abcdef$'''] [[allowlists]] description = "synthetic hex token fixture in the Azure idempotency marker tests" -regexes = ['''4f1c2b0e9d8a7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a2f1e0d9c8b7a6f5e4d3c2b'''] +regexes = ['''^4f1c2b0e9d8a7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a2f1e0d9c8b7a6f5e4d3c2b$'''] diff --git a/scripts/test-gitleaks-config.sh b/scripts/test-gitleaks-config.sh index ae523e05..f46cf865 100644 --- a/scripts/test-gitleaks-config.sh +++ b/scripts/test-gitleaks-config.sh @@ -1,11 +1,19 @@ #!/bin/bash -# Self-test for .gitleaks.toml: each allowlisted fixture literal must scan -# clean, and a real-shaped secret planted in the same file must still fail -# (gitleaks 8.30.1 skips a whole file when an allowlist combines `paths` with -# condition = "AND"). Usage: GITLEAKS=/path/to/gitleaks bash scripts/test-gitleaks-config.sh +# Self-test for .gitleaks.toml. Allowlisted fixture literals are derived from +# the config itself (each `regexes` entry, ^ and $ stripped) and located in the +# tracked tree. Checks that: +# - the config uses no `paths`, `condition`, `stopwords`, `commits`, +# `targetRules` or `regexTarget` key (gitleaks 8.30.1 skips a whole file +# when `paths` is combined with condition = "AND"), and that injecting one +# is detected; +# - each literal scans clean on its own; +# - the literal with a prefix or suffix, or beside a PAT-shaped string, is caught; +# - the real tree is clean. +# Usage: GITLEAKS=/path/to/gitleaks bash scripts/test-gitleaks-config.sh set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CONFIG="$REPO_ROOT/.gitleaks.toml" GITLEAKS="${GITLEAKS:-gitleaks}" command -v "$GITLEAKS" >/dev/null || { echo "gitleaks not found (set GITLEAKS)" >&2; exit 1; } @@ -15,33 +23,52 @@ trap 'rm -rf "$tmp"' EXIT # Assembled at runtime so this file itself scans clean. PAT="ghp""_aB3dE5gH7jK9mN1pQ3sT5vW7yZ9bC1dE3fG5" -scan() { "$GITLEAKS" dir "$1" --config "$REPO_ROOT/.gitleaks.toml" --no-banner --redact --exit-code 1 >/dev/null 2>&1; } - pass=0 fail=0 -check() { # label expected_rc dir +check() { # label expected_rc actual_rc + if [ "$3" -eq "$2" ]; then pass=$((pass + 1)); else fail=$((fail + 1)); echo "FAIL: $1 (expected rc $2, got $3)" >&2; fi +} + +scan() { # dir; prints nothing, returns gitleaks rc local rc=0 - scan "$3" || rc=$? - if [ "$rc" -eq "$2" ]; then pass=$((pass + 1)); else fail=$((fail + 1)); echo "FAIL: $1 (expected rc $2, got $rc)" >&2; fi + "$GITLEAKS" dir "$1" --config "$CONFIG" --no-banner --redact --exit-code 1 >/dev/null 2>&1 || rc=$? + echo "$rc" +} + +# 0 when the config is free of the forbidden keys, 1 otherwise. +forbidden_keys() { + if grep -qE '^[[:space:]]*(paths|condition|stopwords|commits|targetRules|regexTarget)[[:space:]]*=' "$1"; then return 1; fi } +rc=0; forbidden_keys "$CONFIG" || rc=$? +check "config has no paths/condition/stopwords/commits/targetRules/regexTarget" 0 "$rc" +for key in 'paths = ['"'''"'unrelated\.txt'"'''"']' 'condition = "AND"' 'stopwords = ["x"]' 'commits = ["abc"]' 'targetRules = ["generic-api-key"]' 'regexTarget = "line"'; do + { cat "$CONFIG"; printf '\n[[allowlists]]\n%s\n' "$key"; } >"$tmp/injected.toml" + rc=0; forbidden_keys "$tmp/injected.toml" || rc=$? + check "injected '$key' is detected" 1 "$rc" +done + +# Literals from the config: regexes = ['''^literal$''']. +literals=$(sed -nE "s/^regexes = \['''\^(.*)\\$'''\]\$/\1/p" "$CONFIG") +[ -n "$literals" ] || { echo "no anchored regexes found in $CONFIG" >&2; exit 1; } +unanchored=$(grep -E "^regexes" "$CONFIG" | grep -vE "\['''\^.*\\$'''\]\$" || true) +[ -z "$unanchored" ] || { echo "FAIL: regexes not anchored with ^...\$:" >&2; echo "$unanchored" >&2; fail=$((fail + 1)); } -# Every allowlist entry: its path, and its literal (read back from the config). i=0 -while IFS='|' read -r path literal; do +while IFS= read -r literal; do i=$((i + 1)) + path=$(git -C "$REPO_ROOT" grep -lF -e "$literal" -- . ':!.gitleaks.toml' ':!scripts/test-gitleaks-config.sh' | head -1) + [ -n "$path" ] || { echo "FAIL: allowlisted literal #$i not found in the tree" >&2; fail=$((fail + 1)); continue; } d="$tmp/c$i" mkdir -p "$d/$(dirname "$path")" printf 'const token = "%s"\n' "$literal" >"$d/$path" - check "fixture literal in $path is allowed" 0 "$d" + check "fixture literal in $path is allowed" 0 "$(scan "$d")" + printf 'const token = "%sx9"\n' "$literal" >"$d/$path" + check "fixture literal plus a suffix in $path is caught" 1 "$(scan "$d")" + printf 'const token = "%s"\n' "$literal" >"$d/$path" printf 'const real = "%s"\n' "$PAT" >>"$d/$path" - check "real-shaped PAT beside the fixture in $path is caught" 1 "$d" -done <<'ENTRIES' -.github/workflows/pre-commit.yml|ad82d68ee924906a0401dfd48de5057731a9bc84 -providers/aws/internal/purchaseguard/guard_test.go|tok-0123456789abcdef -providers/azure/services/internal/reservations/idempotencymarker_test.go|4f1c2b0e9d8a7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a2f1e0d9c8b7a6f5e4d3c2b -ENTRIES - -# The real tree must be clean under the repo config. -check "repository tree is clean" 0 "$REPO_ROOT" + check "PAT-shaped string beside the fixture in $path is caught" 1 "$(scan "$d")" +done <<<"$literals" + +check "repository tree is clean" 0 "$(scan "$REPO_ROOT")" echo "$pass PASS, $fail FAIL" [ "$fail" -eq 0 ]