diff --git a/.github/scripts/check-commit-emails.sh b/.github/scripts/check-commit-emails.sh new file mode 100644 index 0000000000000..77a0883d6588b --- /dev/null +++ b/.github/scripts/check-commit-emails.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +event=${1:?event required} +base=${2:?base SHA required} +head=${3:?head SHA required} +[[ $event == pull_request || $event == push ]] +[[ $base =~ ^[0-9a-f]{40}$ && $head =~ ^[0-9a-f]{40}$ ]] +# Missing/deleted history must fail, never silently reduce the checked range. +git cat-file -e "$base^{commit}" +git cat-file -e "$head^{commit}" + +allowed() { + local noreply='^[A-Za-z0-9+_.-]+(\[bot\])?@users[.]noreply[.]github[.]com$' + [[ $1 == dimitrikennedy@gmail.com || $1 == noreply@github.com || + $1 =~ $noreply ]] +} + +options=() +if [[ $event == push ]]; then + options+=(--first-parent) +fi +commits=$(git rev-list "${options[@]}" "$base..$head") +failed=0 +for commit in $commits; do + author=$(git show -s --format=%ae "$commit") + committer=$(git show -s --format=%ce "$commit") + parents=$(git show -s --format=%P "$commit") + # Upstream merge commits can have upstream identities. Side-branch commits + # are excluded on pushes; ordinary first-parent commits are never exempt. + if [[ $event == push && $parents == *' '* ]] && ! allowed "$author"; then + continue + fi + if ! allowed "$author" || ! allowed "$committer"; then + # Do not reproduce a potentially private address in public CI logs. + echo "Commit $commit has an author or committer email outside the allowlist." >&2 + failed=1 + fi +done +exit "$failed" diff --git a/.github/scripts/check-commit-emails.test.sh b/.github/scripts/check-commit-emails.test.sh new file mode 100644 index 0000000000000..e899653a1f327 --- /dev/null +++ b/.github/scripts/check-commit-emails.test.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(git rev-parse --show-toplevel) +script="$root/.github/scripts/check-commit-emails.sh" +mkdir -p "$root/.generated" +fixture=$(mktemp -d "$root/.generated/commit-emails.XXXXXX") +trap 'rm -rf "$fixture"' EXIT +cd "$fixture" +git init -q +tree=$(git mktree result 2>&1 || actual=$? + if [[ $actual != "$expected" ]]; then + echo "Expected status $expected, got $actual: $*" >&2 + cat result >&2 + exit 1 + fi + if grep -q "$bad" result; then + echo 'Disallowed email leaked into output' >&2 + exit 1 + fi +} + +base=$(commit "$bad" "$bad") +valid=$(commit "$good" "$good" -p "$base") +expect 0 pull_request "$base" "$valid" +expect 0 push "$base" "$valid" +expect 0 pull_request "$base" "$base" +for email in '123+example@users.noreply.github.com' 'example@users.noreply.github.com' '123+example[bot]@users.noreply.github.com' noreply@github.com; do + head=$(commit "$email" "$email" -p "$base") + expect 0 pull_request "$base" "$head" +done +for email in "$bad" 'example@users.noreply.github.com.invalid' '@users.noreply.github.com'; do + head=$(commit "$email" "$good" -p "$base") + expect 1 pull_request "$base" "$head" + expect 1 push "$base" "$head" + head=$(commit "$good" "$email" -p "$base") + expect 1 pull_request "$base" "$head" +done +invalid=$(commit "$good" "$bad" -p "$base") +tip=$(commit "$good" "$good" -p "$invalid") +expect 1 pull_request "$base" "$tip" +expect 1 push "$base" "$tip" +upstream=$(commit "$bad" "$bad" -p "$base") +merge=$(commit "$bad" "$bad" -p "$valid" -p "$upstream") +expect 0 push "$base" "$merge" +expect 1 pull_request "$base" "$merge" +merge=$(commit "$good" "$bad" -p "$valid" -p "$upstream") +expect 1 push "$base" "$merge" +merge=$(commit "$good" "$good" -p "$valid" -p "$upstream") +expect 0 push "$base" "$merge" +# PR base reachability excludes upstream history already merged into the base. +topic=$(commit "$good" "$good" -p "$upstream") +expect 0 pull_request "$merge" "$topic" +expect 128 push 0000000000000000000000000000000000000000 "$valid" +expect 1 pull_request invalid "$valid" +echo 'Commit email range tests passed.' diff --git a/.github/scripts/commit-hygiene.test.sh b/.github/scripts/commit-hygiene.test.sh new file mode 100644 index 0000000000000..45e6a32b24d4c --- /dev/null +++ b/.github/scripts/commit-hygiene.test.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(git rev-parse --show-toplevel) +gitleaks=${1:?absolute path to verified gitleaks executable required} +mkdir -p "$root/.generated" +fixture=$(mktemp -d "$root/.generated/commit-hygiene.XXXXXX") +trap 'rm -rf "$fixture"' EXIT +export GIT_AUTHOR_NAME=Example GIT_COMMITTER_NAME=Example +export GIT_AUTHOR_EMAIL=example@users.noreply.github.com +export GIT_COMMITTER_EMAIL=$GIT_AUTHOR_EMAIL +# An inert stand-in for the private denylist secret. It must never be printed. +fixture_name=acme-fixture-client +LEAK_DENYLIST=$(cat <<'EOF' +[[rules]] +id = "fixture-private-name" +description = "Inert denylist fixture" +regex = '''(?i)\bacme-fixture-client\b''' +keywords = ["acme-fixture-client"] +EOF +) +export LEAK_DENYLIST +git init -q "$fixture/source" +cd "$fixture/source" +tree=$(git mktree .gitleaks.toml <<'EOF' +[[rules]] +id = "merge-fixture" +description = "Inert regression marker" +regex = '''fixture-only-merge-marker''' +EOF +blob=$(printf 'fixture-only-merge-marker\n' | git hash-object -w --stdin) +merge_tree=$(printf '100644 blob %s\tresolution.txt\n' "$blob" | git mktree) +merge=$(git commit-tree "$merge_tree" -p "$left" -p "$right" -m merge) +git update-ref refs/heads/fixture "$merge" +# The old invocation misses the addition that exists only in the merge tree. +"$gitleaks" git --redact --log-opts="$base..$merge" >"$fixture/old-scan.log" 2>&1 +status=0 +bash "$root/.github/scripts/scan-commit-secrets.sh" "$base" "$merge" "$gitleaks" >"$fixture/merge-scan.log" 2>&1 || status=$? +[[ $status == 1 ]] +grep -q 'leaks found:' "$fixture/merge-scan.log" +bash "$root/.github/scripts/scan-commit-secrets.sh" "$base" "$left" "$gitleaks" >"$fixture/clean-scan.log" 2>&1 + +# The private-name denylist fails closed without its secret. +status=0 +LEAK_DENYLIST= bash "$root/.github/scripts/scan-commit-secrets.sh" "$base" "$left" "$gitleaks" >"$fixture/no-secret.log" 2>&1 || status=$? +[[ $status == 1 ]] +grep -q 'LEAK_DENYLIST repository secret is not set' "$fixture/no-secret.log" +status=0 +LEAK_DENYLIST='not toml [[' bash "$root/.github/scripts/scan-commit-secrets.sh" "$base" "$left" "$gitleaks" >"$fixture/bad-secret.log" 2>&1 || status=$? +[[ $status == 1 ]] +grep -q 'Private-name denylist scan failed' "$fixture/bad-secret.log" + +# A denylisted name fails the check wherever it appears, and only the rule ID is printed. +expect_denied() { + local label=$1 commit=$2 where=$3 status=0 + bash "$root/.github/scripts/scan-commit-secrets.sh" "$base" "$commit" "$gitleaks" >"$fixture/$label.log" 2>&1 || status=$? + [[ $status == 1 ]] || { echo "denylist missed: $label" >&2; exit 1; } + grep -q "rule fixture-private-name matched .* in $where" "$fixture/$label.log" + if grep -qi "$fixture_name" "$fixture/$label.log"; then + echo "denylist output leaked the name: $label" >&2 + exit 1 + fi +} +name_blob=$(printf 'const owner = "%s";\n' "$fixture_name" | git hash-object -w --stdin) +plain_blob=$(printf 'x\n' | git hash-object -w --stdin) +expect_denied content "$(git commit-tree "$(printf '100644 blob %s\tsrc.ts\n' "$name_blob" | git mktree)" -p "$base" -m content)" patches +expect_denied message "$(git commit-tree "$tree" -p "$base" -m "work for $fixture_name")" identifiers +expect_denied path "$(git commit-tree "$(printf '100644 blob %s\t%s.md\n' "$plain_blob" "$fixture_name" | git mktree)" -p "$base" -m path)" identifiers +expect_denied author "$(GIT_AUTHOR_NAME="$fixture_name" git commit-tree "$tree" -p "$base" -m author)" identifiers +HEAD_REF="feature/$fixture_name" expect_denied ref "$left" identifiers +bash "$root/.github/scripts/scan-commit-secrets.sh" "$base" "$left" "$gitleaks" >"$fixture/clean-denylist.log" 2>&1 +# Redaction commits that delete or rename away a denylisted path still pass. +named=$(git commit-tree "$(printf '100644 blob %s\t%s.md\n' "$plain_blob" "$fixture_name" | git mktree)" -p "$base" -m named) +redacted=$(git commit-tree "$(printf '100644 blob %s\tneutral.md\n' "$plain_blob" | git mktree)" -p "$named" -m redact) +bash "$root/.github/scripts/scan-commit-secrets.sh" "$named" "$redacted" "$gitleaks" >"$fixture/redaction.log" 2>&1 + +# Reproduce a nested gitlink without .gitmodules, as found in vendored history. +git update-index --add --cacheinfo "160000,$base,.repos/example/submodules/dependency" +git update-index --add --cacheinfo "100644,$blob,.github/fixture.txt" +checkout_tree=$(git write-tree) +checkout_commit=$(git commit-tree "$checkout_tree" -p "$base" -m checkout) +git update-ref refs/heads/checkout "$checkout_commit" +git checkout -q -f checkout +status=0 +git submodule foreach --recursive ':' >"$fixture/full-checkout.log" 2>&1 || status=$? +[[ $status != 0 ]] +grep -q 'No url found for submodule path' "$fixture/full-checkout.log" + +git init -q "$fixture/fetched" +cd "$fixture/fetched" +git remote add origin "$fixture/source" +git -c credential.helper= fetch -q --no-tags --no-recurse-submodules origin "$base" "$checkout_commit" +git checkout -q --detach "$checkout_commit" +[[ -f .github/fixture.txt ]] +[[ $(git rev-parse --is-shallow-repository) == false ]] +git cat-file -e "$base^{commit}" +if git config --local --get-regexp 'credential|extraheader|sshCommand'; then + echo 'Checkout unexpectedly stored authentication configuration' >&2 + exit 1 +fi +bash "$root/.github/scripts/check-commit-emails.sh" pull_request "$base" "$checkout_commit" +bash "$root/.github/scripts/scan-commit-secrets.sh" "$base" "$checkout_commit" "$gitleaks" >"$fixture/checkout-scan.log" 2>&1 +echo 'Merge secret scan, private-name denylist and anonymous checkout regressions passed.' diff --git a/.github/scripts/scan-commit-secrets.sh b/.github/scripts/scan-commit-secrets.sh new file mode 100755 index 0000000000000..c346747b55a97 --- /dev/null +++ b/.github/scripts/scan-commit-secrets.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +base=${1:?base SHA required} +head=${2:?head SHA required} +gitleaks=${3:?gitleaks executable required} +[[ $base =~ ^[0-9a-f]{40}$ && $head =~ ^[0-9a-f]{40}$ ]] +git cat-file -e "$base^{commit}" +git cat-file -e "$head^{commit}" + +status=0 +# Compare merges with each parent so conflict-resolution additions are scanned. +"$gitleaks" git --redact --log-opts="-m $base..$head" || status=1 + +# Private-name denylist. The rules live only in the LEAK_DENYLIST repository secret +# (gitleaks TOML), so this public repository never contains the names it guards. +# Missing or unusable rules fail closed. Only rule IDs and where they matched are +# printed; gitleaks output is discarded because it can echo file paths or messages. +denylist=${LEAK_DENYLIST-} +if [[ -z ${denylist//[[:space:]]/} ]]; then + echo "::error title=Private-name denylist missing::The LEAK_DENYLIST repository secret is not set or is empty (forked PRs never receive it). Failing closed; a maintainer must add the secret." >&2 + exit 1 +fi +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +(umask 077 && printf '%s\n' "$denylist" >"$work/denylist.toml") + +denylist_scan() { + local kind=$1 + shift + local report="$work/$kind.json" code=0 + "$gitleaks" "$@" --no-banner --redact --exit-code 0 --config "$work/denylist.toml" \ + --report-format json --report-path "$report" >/dev/null 2>&1 || code=$? + if [[ $code != 0 || ! -s $report ]] || ! jq -e 'type == "array"' "$report" >/dev/null 2>&1; then + echo "::error title=Private-name denylist scan failed::gitleaks could not scan $kind with the LEAK_DENYLIST rules (exit $code); output suppressed. Check that the secret is valid gitleaks TOML." >&2 + exit 1 + fi + local hits + hits=$(jq -r --arg kind "$kind" \ + 'group_by(.RuleID)[] | "\(.[0].RuleID | gsub("[^A-Za-z0-9._-]"; "?"))\t\($kind)\t\(length)"' "$report") + if [[ -n $hits ]]; then + while IFS=$'\t' read -r rule where count; do + echo "::error title=Private-name denylist::rule $rule matched $count time(s) in $where" >&2 + done <<<"$hits" + status=1 + fi +} + +# Added lines in every new commit, including merge resolutions. +denylist_scan patches git --log-opts="-m $base..$head" + +# Identifiers that never appear in patches: commit messages, author and committer +# identities, paths a commit adds or renames to (removing a name is allowed) and the PR head ref. +mkdir "$work/identifiers" +while read -r sha; do + { + git show -s --format='%an%n%ae%n%cn%n%ce%n%B' "$sha" + git diff-tree -r -m --root --no-commit-id --name-only --diff-filter=d -M "$sha" + } >"$work/identifiers/$sha.txt" +done < <(git rev-list "$base..$head") +printf '%s\n' "${HEAD_REF-}" >"$work/identifiers/head-ref.txt" +denylist_scan identifiers dir "$work/identifiers" + +exit "$status" diff --git a/.github/workflows/commit-hygiene.yml b/.github/workflows/commit-hygiene.yml new file mode 100644 index 0000000000000..913f05294181c --- /dev/null +++ b/.github/workflows/commit-hygiene.yml @@ -0,0 +1,64 @@ +name: Commit hygiene + +on: + pull_request: + branches: [hd/nightly] + push: + branches: [hd/nightly] + +permissions: + contents: read + +jobs: + scan: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.after }} + steps: + # This public repository needs no checkout credentials. Avoid checkout's + # recursive auth cleanup, which fails on vendored gitlinks without URLs. + - name: Fetch full history without credentials + env: + GIT_CONFIG_NOSYSTEM: '1' + GIT_CONFIG_GLOBAL: /dev/null + GIT_TERMINAL_PROMPT: '0' + shell: bash + run: | + set -euo pipefail + [[ "$BASE_SHA" =~ ^[0-9a-f]{40}$ && "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] + git init . + git remote add origin "${{ github.server_url }}/${{ github.repository }}.git" + git -c credential.helper= fetch --no-tags --no-recurse-submodules origin "$BASE_SHA" "$HEAD_SHA" + git checkout --detach "$HEAD_SHA" + - name: Validate range and commit emails + env: + EVENT_NAME: ${{ github.event_name }} + run: bash .github/scripts/check-commit-emails.sh "$EVENT_NAME" "$BASE_SHA" "$HEAD_SHA" + - name: Download and verify gitleaks + id: gitleaks + if: ${{ !cancelled() }} + shell: bash + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/gitleaks" + cd "$RUNNER_TEMP/gitleaks" + curl --fail --silent --show-error --location --proto '=https' --tlsv1.2 \ + https://github.com/gitleaks/gitleaks/releases/download/v8.30.1/gitleaks_8.30.1_linux_x64.tar.gz \ + --output gitleaks.tar.gz + echo '551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb gitleaks.tar.gz' | sha256sum --check --strict + tar -xzf gitleaks.tar.gz gitleaks + - name: Test commit hygiene regressions + if: ${{ !cancelled() && steps.gitleaks.outcome == 'success' }} + run: | + bash .github/scripts/check-commit-emails.test.sh + bash .github/scripts/commit-hygiene.test.sh "$RUNNER_TEMP/gitleaks/gitleaks" + - name: Scan every new commit for secrets and private names + if: ${{ !cancelled() && steps.gitleaks.outcome == 'success' }} + shell: bash + env: + # gitleaks TOML rules; never committed. Unset or empty fails closed. + LEAK_DENYLIST: ${{ secrets.LEAK_DENYLIST }} + HEAD_REF: ${{ github.head_ref }} + run: bash .github/scripts/scan-commit-secrets.sh "$BASE_SHA" "$HEAD_SHA" "$RUNNER_TEMP/gitleaks/gitleaks" diff --git a/docs/operations/commit-hygiene.md b/docs/operations/commit-hygiene.md new file mode 100644 index 0000000000000..791800411b82b --- /dev/null +++ b/docs/operations/commit-hygiene.md @@ -0,0 +1,48 @@ +# Commit hygiene + +The `Commit hygiene` workflow checks PRs targeting `hd/nightly` and pushes to that +branch. It scans `base..head` with gitleaks v8.30.1, including merged side branches +and merge patches (`-m`) so conflict-resolution additions are checked. +The release archive must match the SHA-256 pinned in the workflow before it runs. +Findings are redacted; diagnostics are not enabled. + +The same scan step also applies a private-name denylist. Its gitleaks TOML rules +come only from the `LEAK_DENYLIST` repository secret, so this public repository +never contains the names it guards. It checks the added lines of every new commit +(including merge resolutions), commit messages, author and committer identities, +paths a commit adds or renames to, and the PR head ref. Removing a name passes. +A missing, empty or invalid secret fails the check closed; forked PRs never +receive the secret and therefore fail too. Output +lists only rule IDs and where they matched (`patches` or `identifiers`), so use +neutral rule IDs. Gitleaks output for this pass is discarded. + +For PRs, both the author and committer emails of every commit not already reachable +from the PR base SHA must match the allowlist in +[check-commit-emails.sh](../../.github/scripts/check-commit-emails.sh). It accepts +GitHub noreply addresses and the one maintainer address already used in the +fork's history. It does not read mailmap aliases or print rejected addresses. This checks email metadata, not authorship authenticity. + +For pushes, the email check considers only first-parent commits in `before..after`. +Merge commits whose author is outside the allowlist are exempt so upstream merges +can retain upstream identities. This deliberately trusts such merge commits and +their side-branch history; it does not prove they came from a particular upstream. +Ordinary first-parent commits always have both emails checked, as do merge commits +with allowlisted authors. PRs importing upstream commits not yet in their base +must first have that history integrated into `hd/nightly` or use the push policy. + +Missing range endpoints (including an all-zero endpoint for branch creation or +deletion) fail closed. The workflow checks out full history and does not substitute +the latest branch tip for an unavailable event SHA. The public fork is fetched +anonymously, without storing credentials. This avoids recursive credential cleanup +in `actions/checkout`, which fails on vendored gitlinks that lack `.gitmodules` +entries. The fetch has no depth limit and checkout uses the event's head SHA. + +Run the focused email-policy fixtures with +`bash .github/scripts/check-commit-emails.test.sh`. They create an isolated Git +repository under the worktree's ignored `.generated` directory and remove it when +finished. Run `bash .github/scripts/commit-hygiene.test.sh ` +to verify anonymous checkout with malformed gitlinks, detection of an inert +marker introduced only in a merge commit, and the denylist's fail-closed and +rule-ID-only behavior with an inert fixture rule. Both fixture suites also run in CI. +Workflow changes also need YAML/actionlint validation and +`vp run check:changed origin/hd/nightly` when the repository toolchain is available.