From ee0e27a5c70905ebf074d81c1526ae9b4eebe5ca Mon Sep 17 00:00:00 2001 From: Adrien Langou Date: Fri, 4 Sep 2026 15:36:25 +0200 Subject: [PATCH 1/5] ci(trivy): add artifact and PR configuration scans Signed-off-by: Adrien Langou --- .github/workflows/trivy-changes.yml | 144 +++++++++ .github/workflows/trivy-scan.yml | 280 +++++++++++++++++ .gitignore | 3 + .trivyignore.yaml | 62 ++++ CI.md | 94 +++++- architecture/build.md | 86 ++++++ flake.nix | 2 + .../agents/gator/skills/gator-gate/SKILL.md | 1 + tasks/scripts/trivy-scan.sh | 286 ++++++++++++++++++ 9 files changed, 955 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/trivy-changes.yml create mode 100644 .github/workflows/trivy-scan.yml create mode 100644 .trivyignore.yaml create mode 100755 tasks/scripts/trivy-scan.sh diff --git a/.github/workflows/trivy-changes.yml b/.github/workflows/trivy-changes.yml new file mode 100644 index 0000000000..cf66ca0e81 --- /dev/null +++ b/.github/workflows/trivy-changes.yml @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Trivy Changes + +on: + pull_request: + merge_group: + types: [checks_requested] + workflow_dispatch: + inputs: + base_sha: + description: Base commit SHA to compare + required: true + type: string + head_sha: + description: Candidate commit SHA to compare + required: true + type: string + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + changes: + name: Detect deployment configuration changes + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + should_run: ${{ steps.default.outputs.should_run || steps.changed.outputs.any_changed }} + steps: + - id: default + if: github.event_name != 'pull_request' + run: echo "should_run=true" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: github.event_name == 'pull_request' + with: + persist-credentials: false + + - id: changed + if: github.event_name == 'pull_request' + uses: tj-actions/changed-files@aa08304bd477b800d468db44fe10f6c61f7f7b11 # v42.1.0 + with: + files: | + deploy/docker/** + deploy/helm/** + .trivyignore.yaml + flake.nix + flake.lock + tasks/scripts/trivy-scan.sh + .github/workflows/trivy-changes.yml + + scan: + name: Scan changed deployment configuration + needs: changes + if: needs.changes.outputs.should_run == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + BASE_REF: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || inputs.base_sha }} + HEAD_REF: ${{ inputs.head_sha || github.sha }} + defaults: + run: + shell: nix develop --command bash -euo pipefail {0} + steps: + - name: Check out candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.HEAD_REF }} + persist-credentials: false + + - name: Check out baseline + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.BASE_REF }} + path: .trivy-base + persist-credentials: false + + - name: Set up Nix + uses: ./.github/actions/setup-nix + + - name: Scan baseline + env: + TRIVY_SOURCE_ROOT: ${{ github.workspace }}/.trivy-base + TRIVY_IGNORE_FILE: ${{ github.workspace }}/.trivyignore.yaml + TRIVY_REPORT_DIR: ${{ runner.temp }}/trivy-base + run: | + mkdir -p "$TRIVY_REPORT_DIR" + "$GITHUB_WORKSPACE/tasks/scripts/trivy-scan.sh" config + + - name: Scan candidate + env: + TRIVY_REPORT_DIR: ${{ runner.temp }}/trivy-head + run: | + mkdir -p "$TRIVY_REPORT_DIR" + tasks/scripts/trivy-scan.sh config + + - name: Reject new high or critical findings + run: | + tasks/scripts/trivy-scan.sh gate-config-diff \ + "$RUNNER_TEMP/trivy-base" "$RUNNER_TEMP/trivy-head" + + - name: Upload reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: trivy-changes-${{ github.run_id }} + path: | + ${{ runner.temp }}/trivy-base + ${{ runner.temp }}/trivy-head + if-no-files-found: ignore + retention-days: 14 + + result: + name: OpenShell / Trivy Changes + needs: [changes, scan] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check scan result + env: + CHANGES_RESULT: ${{ needs.changes.result }} + SHOULD_RUN: ${{ needs.changes.outputs.should_run }} + SCAN_RESULT: ${{ needs.scan.result }} + run: | + set -euo pipefail + if [ "$CHANGES_RESULT" != "success" ]; then + echo "::error::Change detection concluded $CHANGES_RESULT." + exit 1 + fi + if [ "$SHOULD_RUN" = "true" ] && [ "$SCAN_RESULT" != "success" ]; then + echo "::error::Trivy scan concluded $SCAN_RESULT." + exit 1 + fi + if [ "$SHOULD_RUN" != "true" ]; then + echo "No Helm or Dockerfile changes to scan." + fi diff --git a/.github/workflows/trivy-scan.yml b/.github/workflows/trivy-scan.yml new file mode 100644 index 0000000000..fa37db9922 --- /dev/null +++ b/.github/workflows/trivy-scan.yml @@ -0,0 +1,280 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Trivy Scan + +# Scans the artifacts a release publishes: the final container images and the +# deployment configuration. Callers pass OCI references, so this workflow is +# self-contained and knows nothing about how a release is assembled. +# +# Findings are informational while we learn what this reports in practice: they +# produce a warning, not a failure. A scanner that cannot run still fails, so a +# broken scan cannot look clean. Set fail-on-findings to flip the gate on. + +on: + workflow_call: + inputs: + images: + description: Newline-separated image references to scan + required: false + type: string + default: "" + chart-ref: + description: | + Packaged Helm chart to scan, for example + oci://ghcr.io/nvidia/openshell/helm-chart:0.0.116 + required: false + type: string + default: "" + severity: + description: Severities that fail the workflow + required: false + type: string + default: HIGH,CRITICAL + ignore-unfixed: + description: Ignore image vulnerabilities with no upstream fix + required: false + type: boolean + default: true + fail-on-findings: + description: Fail the run on findings instead of warning + required: false + type: boolean + default: false + upload-sarif: + description: Upload results to GitHub Code Scanning + required: false + type: boolean + default: true + secrets: + CACHIX_AUTH_TOKEN: + description: Token used to write Nix build outputs to Cachix + required: false + + workflow_dispatch: + inputs: + images: + description: Newline-separated image references to scan + required: false + type: string + default: "" + chart-ref: + description: Packaged Helm chart OCI reference to scan + required: false + type: string + default: "" + severity: + description: Severities that fail the workflow + required: false + type: string + default: HIGH,CRITICAL + ignore-unfixed: + description: Ignore image vulnerabilities with no upstream fix + required: false + type: boolean + default: true + fail-on-findings: + description: Fail the run on findings instead of warning + required: false + type: boolean + default: false + upload-sarif: + description: Upload results to GitHub Code Scanning + required: false + type: boolean + default: true + +permissions: + contents: read + +defaults: + run: + shell: nix develop --command bash -euo pipefail {0} + +env: + TRIVY_SEVERITY: ${{ inputs.severity }} + TRIVY_REPORT_DIR: reports/trivy + +jobs: + image: + name: Image vulnerabilities (informational) + if: inputs.images != '' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: read + security-events: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # Trivy reads ~/.docker/config.json, which `nix develop` leaves alone. + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Set up Nix + uses: ./.github/actions/setup-nix + with: + cachix-auth-token: ${{ github.event_name != 'pull_request' && secrets.CACHIX_AUTH_TOKEN || '' }} + + - name: Scan images + id: scan + env: + IMAGES: ${{ inputs.images }} + TRIVY_IGNORE_UNFIXED: ${{ inputs.ignore-unfixed }} + run: | + refs=() + while IFS= read -r ref; do + [ -n "$ref" ] || continue + refs+=("$ref") + done <<<"$IMAGES" + tasks/scripts/trivy-scan.sh images "${refs[@]}" + + - name: Upload SARIF to Code Scanning + if: ${{ !cancelled() && steps.scan.conclusion == 'success' && inputs.upload-sarif }} + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 + with: + sarif_file: reports/trivy + category: trivy-image + + - name: Upload reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: trivy-image-${{ github.run_id }} + path: reports/trivy + if-no-files-found: ignore + retention-days: 14 + + # Separate from the scan so a tripped gate still publishes its reports. + - name: Report findings + if: ${{ !cancelled() && steps.scan.conclusion == 'success' }} + env: + FAIL_ON_FINDINGS: ${{ inputs.fail-on-findings }} + run: | + set +e + tasks/scripts/trivy-scan.sh gate + status=$? + set -e + case "$status" in + 0) echo "No findings at ${TRIVY_SEVERITY}." ;; + 10) + if [ "$FAIL_ON_FINDINGS" = "true" ]; then + echo "::error::Image findings at ${TRIVY_SEVERITY}." + exit 1 + fi + echo "::warning::Image findings at ${TRIVY_SEVERITY}; this check is informational." + ;; + *) echo "::error::Trivy could not evaluate the reports (exit $status)."; exit "$status" ;; + esac + + config: + name: Configuration misconfigurations (informational) + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: read + security-events: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # `helm pull` needs registry credentials only when a packaged chart is + # requested, but logging in unconditionally keeps the step list flat. + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Set up Nix + uses: ./.github/actions/setup-nix + with: + cachix-auth-token: ${{ github.event_name != 'pull_request' && secrets.CACHIX_AUTH_TOKEN || '' }} + + - name: Scan configuration + id: scan + env: + CHART_REF: ${{ inputs.chart-ref }} + run: | + args=() + if [ -n "$CHART_REF" ]; then + args+=(--chart-ref "$CHART_REF") + fi + tasks/scripts/trivy-scan.sh config "${args[@]}" + + - name: Upload SARIF to Code Scanning + if: ${{ !cancelled() && steps.scan.conclusion == 'success' && inputs.upload-sarif }} + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 + with: + sarif_file: reports/trivy + category: trivy-config + + - name: Upload reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: trivy-config-${{ github.run_id }} + path: reports/trivy + if-no-files-found: ignore + retention-days: 14 + + - name: Report findings + if: ${{ !cancelled() && steps.scan.conclusion == 'success' }} + env: + FAIL_ON_FINDINGS: ${{ inputs.fail-on-findings }} + run: | + set +e + tasks/scripts/trivy-scan.sh gate + status=$? + set -e + case "$status" in + 0) echo "No findings at ${TRIVY_SEVERITY}." ;; + 10) + if [ "$FAIL_ON_FINDINGS" = "true" ]; then + echo "::error::Configuration findings at ${TRIVY_SEVERITY}." + exit 1 + fi + echo "::warning::Configuration findings at ${TRIVY_SEVERITY}; this check is informational." + ;; + *) echo "::error::Trivy could not evaluate the reports (exit $status)."; exit "$status" ;; + esac + + # Republishes whether the scans ran, not what they found, so a broken scanner + # cannot pass as healthy while findings stay informational. + result: + name: OpenShell / Trivy (informational) + needs: [image, config] + if: always() + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # The image job is skipped when no references are passed, which + # check-job-results treats as a failure, so drop skipped jobs first. + - name: Drop skipped jobs + id: required + env: + JOB_RESULTS: ${{ toJSON(needs) }} + run: | + { + echo 'results<>"$GITHUB_OUTPUT" + + - uses: ./.github/actions/check-job-results + with: + results: ${{ steps.required.outputs.results }} diff --git a/.gitignore b/.gitignore index 3ef1a37697..a7867fe2fe 100644 --- a/.gitignore +++ b/.gitignore @@ -67,6 +67,9 @@ pip-delete-this-directory.txt coverage.out coverage/ htmlcov/ + +# Trivy scan reports (tasks/scripts/trivy-scan-*.sh) +/reports/ .tox/ .nox/ .coverage diff --git a/.trivyignore.yaml b/.trivyignore.yaml new file mode 100644 index 0000000000..893ac16502 --- /dev/null +++ b/.trivyignore.yaml @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Trivy exceptions. Passed explicitly with --ignorefile by tasks/scripts/trivy-scan.sh, +# because Trivy auto-loads a plain `.trivyignore` but not the YAML variant. +# +# An entry belongs here only when the finding is wrong: the condition it +# reports is not true of this repository, or it is an artifact of how the scan +# renders the chart. Nothing else qualifies. Findings that describe hardening +# we have not done, or a risk we have accepted, stay in the report where they +# can be seen and argued about, even when that means the gate fails. +# +# Always scope an entry with `paths`, naming individual files. An `id` on its +# own disables the check everywhere, which would also hide a genuine occurrence +# elsewhere. +# +# Use `**/.yaml`. Paths match the location Trivy reports, which is +# relative to the scanned target, and the same template is reported two ways: +# `helm/openshell/templates/x.yaml` when scanning deploy/, and +# `helm-chart-.tgz:templates/x.yaml` when scanning the published chart. +# Only a leading `**/` matches both. `*templates/x.yaml` silently stops applying +# to the repository scan, and `**/templates/x.yaml` to the packaged one. +# +# `paths` is also as narrow as this file can get: for misconfigurations Trivy +# offers no per-occurrence scoping, so a second, legitimate finding of the same +# check in a listed file would be hidden too. Inline `#trivy:ignore:` comments +# would fix that and do work for Dockerfiles, but Trivy 0.74 does not apply them +# to Helm templates. Revisit when it does. + +misconfigurations: + # helm template renders without a namespace, so every workload appears to be + # in "default". The namespace comes from `helm install -n` and no template + # hardcodes one. + - id: KSV-0110 + paths: + - "**/statefulset.yaml" + - "**/deployment.yaml" + statement: >- + An artifact of rendering the chart outside a cluster. The namespace is + supplied at install time. + + # This ConfigMap stores the *name* of a key inside an external Secret + # (proxy_auth_secret_key = "proxy-auth"), not the credential. Scoped to the + # one file, because elsewhere this check is what would catch a real + # credential committed into a ConfigMap. + - id: KSV-01010 + paths: + - "**/gateway-config.yaml" + statement: >- + The ConfigMap holds the name of a key in an external Secret, not a + credential. + + # ghcr.io/nvidia/openshell is where this project publishes its own images. + # Trivy's default trusted-registry list cannot be extended in the version we + # run, so the check cannot be taught about our registry. Scoped to the two + # workload templates so third-party images referenced elsewhere still report. + - id: KSV-0125 + paths: + - "**/statefulset.yaml" + - "**/deployment.yaml" + statement: >- + Images come from ghcr.io/nvidia/openshell, this project's own registry. diff --git a/CI.md b/CI.md index c29abdba0c..480f5626e1 100644 --- a/CI.md +++ b/CI.md @@ -24,7 +24,9 @@ Three opt-in labels enable the long-running E2E suites: When multiple labels are present, `Branch E2E Checks` builds each generic multi-architecture artifact set once and fans out enabled suites in parallel. Runtime-specific reusable workflows define the Docker, Podman, VM, and Kubernetes lanes. Composite actions own the replaceable Podman, KVM, kind, and mise setup. Each lane depends only on the artifact categories it consumes: VM does not wait for container-driver artifacts or supervisor images, and GPU does not wait for the gateway image. Docker, Podman, GPU, Rust, Python, MCP, and VM E2E reuse matching prebuilt gateway and CLI binaries instead of compiling debug binaries in test jobs. Standalone-driver lanes additionally reuse driver-free gateway and compute-driver artifacts. Kubernetes managed-driver lanes consume published gateway and supervisor images, while the standalone-driver lane composes its gateway image from prebuilt binaries. The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while Kubernetes HA and credential-driver behavior are under active iteration: failures are visible in the workflow run but do not publish a required CI gate status. -The GitHub ruleset should require the `OpenShell / ...` statuses published by `Required CI Gates`, not the push-triggered workflow jobs directly. +The GitHub ruleset should require the `OpenShell / ...` statuses published by +`Required CI Gates` plus the direct `OpenShell / Trivy Changes` result, not the +push-triggered workflow jobs themselves. ## Informational security reports @@ -79,6 +81,81 @@ nix develop --command actionlint -shellcheck= -pyflakes= nix develop --command zizmor --offline --persona=regular --min-severity=high --no-exit-codes . ``` +## Artifact scanning + +`Trivy Scan` differs from the reports above in what it looks at rather than in +how it reports: it scans what a release publishes instead of what a change +contains — the final container images, the Helm charts, the final image +Dockerfiles, and the raw Kubernetes manifests. Nix provides Trivy and Helm, and +the jobs run on GitHub-hosted runners like the other scanners. + +Findings are informational for now, while we learn what the scanner reports in +practice. They raise a warning and the run stays green; a scanner that cannot +run still fails, so a broken scan cannot look clean. The `fail-on-findings` +input flips that to a hard failure once the findings have been worked through. + +The workflow is reusable and takes OCI references as input, so it knows nothing +about how a release is assembled. `HIGH` and `CRITICAL` are what get reported as +findings; everything below is listed without comment. Image scanning +additionally ignores vulnerabilities with no upstream fix, because a base-image +CVE without a patch would otherwise be permanent noise. That option does not +apply to misconfigurations. + +It is not listed in any release workflow's `needs:`, so no publication depends +on it. Wiring it into `release-dev.yml` and `release-tag.yml` is a separate +change, and one that only makes sense once findings fail. + +The configuration scan targets `deploy/` in one pass, which covers both charts, +the published Dockerfiles and the raw manifests. The macOS Dockerfiles export a +binary from `FROM scratch` and the CI image is toolchain rather than a release +artifact, so both are skipped. + +Chart coverage additionally depends on value combinations. The chart defaults +render 10 of the chart's 19 templates, while some conditional resources only +render with overrides stored under `deploy/helm/openshell/ci/values-*.yaml`. +The scan exercises each of these CI fixtures to cover resources such as the +high-availability Deployment, Gateway API objects, OpenShift Route, and broader +workspace-mode ClusterRole. These fixtures are test inputs, not a set of +separately supported product profiles. + +Exceptions live in `.trivyignore.yaml`, one justification per entry. Trivy +auto-loads a plain `.trivyignore` but not the YAML variant, so the scripts pass +`--ignorefile` explicitly. An entry qualifies only when the finding is wrong: +the condition it reports is not true of this repository, or it is an artifact of +how the scan renders the chart. Hardening we have not done and risks we have +accepted stay in the report, where they can be seen and argued about, even when +that means the gate fails. + +Four checks report today: `KSV-0014` (`readOnlyRootFilesystem` unset on the +gateway container), `KSV-0041` and `KSV-0056` (RBAC grants the managed workspace +mode needs and that RBAC cannot express more narrowly), and `DS-0002` (the +supervisor image runs as root by design). Resolving or consciously accepting +each of those is what has to happen before `fail-on-findings` is worth turning +on. + +Scans write full-severity reports and never fail on findings, so a report is +always available to upload; a separate `gate` step re-reads them and applies the +threshold. Run them locally with: + +```shell +nix develop --command tasks/scripts/trivy-scan.sh config +nix develop --command tasks/scripts/trivy-scan.sh images ghcr.io/nvidia/openshell/gateway:dev +nix develop --command tasks/scripts/trivy-scan.sh gate +``` + +### Pull-request change gate + +`Trivy Changes` runs directly on pull requests and merge groups. It detects +changes to Helm charts, release Dockerfiles, and the Trivy tooling, then scans +both the base revision and the candidate with the same scanner logic. The check +fails only when the candidate introduces a new `HIGH` or `CRITICAL` +misconfiguration, so existing findings do not block unrelated work. Reports +from both revisions are retained as workflow artifacts. + +This check analyzes Helm and Dockerfile configuration. It does not build +container images, so package and operating-system CVEs remain the responsibility +of the release-artifact image scan. + ## Commit signing copy-pr-bot decides whether to mirror a PR automatically based on whether the author is trusted. For org members and collaborators, "trusted" means **all commits in the PR are cryptographically signed**. Unsigned commits, even from an org member, force the bot to wait for a maintainer's `/ok to test `. @@ -158,14 +235,18 @@ GitHub merge queue is required for `main`. Repository administrators must enable - `OpenShell / E2E` - `OpenShell / GPU E2E` - `OpenShell / Helm Lint` +- `OpenShell / Trivy Changes` -Do not require the underlying workflow job names directly. `Required CI Gates` publishes stable commit statuses for both PR-head mirror commits and merge-group commits. +`Required CI Gates` publishes the stable statuses for mirror-based workflows. +`Trivy Changes` runs directly on pull requests and merge groups and publishes +its own stable result status. Merge-group runs use the `merge_group` event. The event is distinct from `pull_request` and `push`, and GitHub will not report required checks for queued PRs unless the workflows include it. In this repository: - `Branch Checks` runs the standard non-E2E gates on the merge-group SHA. - `Branch E2E Checks` runs core E2E and GPU E2E for merge groups. Kubernetes HA E2E remains optional and label-driven on PRs. - `Helm Lint` runs for merge groups without the PR diff optimization, because the merge-group branch is the final integration state. +- `Trivy Changes` compares the merge-group configuration with its base and rejects new High or Critical findings. - `Required CI Gates` posts the same `OpenShell / ...` statuses to the merge-group SHA and does not require a `pull-request/` mirror for merge-group events. Maintainers should add ready PRs to the queue rather than pressing a direct merge button. GitHub removes a PR from the queue if the merge-group checks fail or time out. @@ -204,6 +285,8 @@ The bot's full administrator documentation is internal to NVIDIA. The only comma | `.github/workflows/dependency-review.yml` | Reports dependency changes when GitHub Dependency Graph is available; otherwise publishes a neutral warning. | | `.github/workflows/codeql.yml` | Runs nightly informational CodeQL analysis on `main` for Rust and the Go, Python, and TypeScript SDKs and retains SARIF artifacts. | | `.github/workflows/codex-security.yml` | Scans the cumulative diff from the previous stable release to each pre-release candidate and publishes train-scoped SARIF on `main`. | +| `.github/workflows/trivy-changes.yml` | Blocks pull requests and merge groups that introduce new High or Critical Helm or Dockerfile misconfigurations. | +| `.github/workflows/trivy-scan.yml` | Reusable scan of published container images and deployment configuration. Findings are informational by default and can be configured to fail the workflow. | ## Release workflows @@ -223,8 +306,13 @@ Require these statuses in the branch ruleset for PR and merge-queue CI: - `OpenShell / E2E` - `OpenShell / GPU E2E` - `OpenShell / Helm Lint` +- `OpenShell / Trivy Changes` -Do not require the underlying workflow jobs directly. PR workflow jobs only appear after copy-pr-bot mirrors trusted code, and merge-group workflow jobs run on temporary queue branches. The stable `OpenShell / ...` contexts prove the expected workflow completed for the commit that GitHub is about to merge. +For mirror-based workflows, require the statuses published by +`Required CI Gates`, not their underlying jobs. `OpenShell / Trivy Changes` is +the stable result job of the direct pull-request workflow. Together these +contexts prove the expected checks completed for the commit GitHub is about to +merge. Do not add the informational Actionlint, Zizmor, Dependency Review, or CodeQL jobs to the required status list while they remain in observation mode. diff --git a/architecture/build.md b/architecture/build.md index d45cc25666..0d6b5402c7 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -439,6 +439,92 @@ tags and gating stable promotion on qualification results are part of [RFC 0014](../rfc/0014-release-stability/release-qualification.md) and are not implemented yet. +## Artifact Scanning + +Trivy runs in two places: a reusable release-artifact workflow and a +pull-request change gate. + +### Release Artifacts + +`.github/workflows/trivy-scan.yml` is reusable and takes OCI references as +input, so it has no knowledge of how a release is assembled and no dependency on +release job ordering. Nix supplies both Trivy and Helm, and the jobs stay on +GitHub-hosted runners like the other scanners. + +Findings are informational during the observation phase: they warn and the run +stays green, while a scanner that cannot run still fails. The `fail-on-findings` +input turns them into failures, which is a prerequisite for wiring the workflow +into a release `needs:` rather than something to do at the same time. + +Two scopes, with deliberately different reporting semantics: + +- **Images.** `HIGH` and `CRITICAL` are reported, and vulnerabilities with no + upstream fix are ignored. Without that exclusion a base-image CVE with no + available patch would be permanent noise, and a gate nobody can act on once + findings start failing. Published tags are multi-arch indexes and Trivy + defaults to the runner's own platform, so each architecture is scanned + separately. +- **Configuration.** The same severity threshold, but the unfixed exclusion does not + apply to misconfigurations. One pass over `deploy/` covers both charts, the + published Dockerfiles and the raw manifests. Coverage then depends on value + combinations: the chart defaults render 10 of the chart's 19 templates, so + CI value fixtures exercise conditional resources such as the high-availability + Deployment, Gateway API objects, OpenShift Route, and broader workspace-mode + ClusterRole. Each fixture is scanned on its own, and the packaged chart is + scanned from its published OCI reference to cover the artifact consumers + actually install. + +Trivy has no OCI artifact target, and `trivy image` rejects the Helm config media +type, so a packaged chart has to be fetched with `helm pull` before it can be +scanned. Trivy reports locations relative to the scanned target, so +`tasks/scripts/trivy-scan.sh` rewrites SARIF URIs to repository-relative paths; +without that, Code Scanning resolves alerts against files that do not exist. That +rewrite and the profile loop are the only repository-specific logic: severity +filtering, the pass/fail decision and the summary table all come from +`trivy convert --exit-code`, so nothing reimplements counting. + +`.trivyignore.yaml` holds exceptions, and the bar for adding one is that the +finding is wrong: the condition it reports is not true of this repository, or it +is an artifact of how the scan renders the chart. Hardening that has not been +done and risks that have been accepted stay in the report instead, so the +scanner keeps describing the real posture rather than a curated one. Trivy +auto-loads a plain `.trivyignore` but not the YAML variant, so the scripts pass +`--ignorefile` explicitly. + +That bar means four checks report today: `KSV-0014`, `KSV-0041`, `KSV-0056` and +`DS-0002`. Reports are written before findings are evaluated, so a warning or a +failure still publishes SARIF and artifacts. Introducing the tooling and settling +its findings are separate changes, in that order. + +### Pull-Request Change Gate + +`.github/workflows/trivy-changes.yml` gates changes rather than releases. It +runs on `pull_request` and `merge_group`; `workflow_dispatch` takes explicit +base and head SHAs for diagnostics. A detection job decides whether the change +touches `deploy/docker/**`, `deploy/helm/**`, or the scanner inputs themselves +(`.trivyignore.yaml`, `flake.nix`, `flake.lock`, `tasks/scripts/trivy-scan.sh`, +and the workflow file). + +When it does, the scan job checks out both the baseline and the candidate and +runs the candidate's `trivy-scan.sh config` over each tree with the candidate's +`.trivyignore.yaml`, so a scanner or ignore-policy change is judged by its own +rules on both sides. `gate-config-diff` then compares semantic finding +identities — rule ID, target, namespace, message, and cause +provider/service/resource — and fails only on identities absent from the +baseline. The four findings above therefore keep reporting without blocking +every pull request, while a newly introduced `HIGH` or `CRITICAL` +misconfiguration fails the check. Both report sets are uploaded as workflow +artifacts. + +The `result` job publishes a stable `OpenShell / Trivy Changes` status that +succeeds when no relevant files changed, so the check can be required +unconditionally. + +This gate scans Helm and Dockerfile configuration only. It builds no image, so +it cannot detect OS or package CVEs in the image a change would produce. +Final-image vulnerability scanning stays with the release-artifact workflow +above. + See `CI.md` for the contributor workflow, labels, and maintainer merge-queue workflow. ## Docs Site diff --git a/flake.nix b/flake.nix index e361fd1597..1302713450 100644 --- a/flake.nix +++ b/flake.nix @@ -57,7 +57,9 @@ pkg-config # Coverage. lcov + kubernetes-helm syft + trivy uv zizmor zstd diff --git a/scripts/agents/gator/skills/gator-gate/SKILL.md b/scripts/agents/gator/skills/gator-gate/SKILL.md index d66d5029ea..7768a38d1e 100644 --- a/scripts/agents/gator/skills/gator-gate/SKILL.md +++ b/scripts/agents/gator/skills/gator-gate/SKILL.md @@ -961,6 +961,7 @@ Required gates include at least: - `OpenShell / Branch Checks` - `OpenShell / Helm Lint` +- `OpenShell / Trivy Changes` - `OpenShell / E2E` when `test:e2e` is applied - `OpenShell / GPU E2E` when `test:e2e-gpu` is applied diff --git a/tasks/scripts/trivy-scan.sh b/tasks/scripts/trivy-scan.sh new file mode 100755 index 0000000000..aaf34a5b58 --- /dev/null +++ b/tasks/scripts/trivy-scan.sh @@ -0,0 +1,286 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# Scan release artifacts with Trivy. +# +# trivy-scan.sh config [--chart-ref ] +# trivy-scan.sh images [...] +# trivy-scan.sh gate +# trivy-scan.sh gate-config-diff +# +# `config` and `images` write full-severity reports and never fail on findings, +# so a report is always available to upload. `gate` then re-reads those reports +# and fails if any finding reaches TRIVY_SEVERITY. +# +# Environment: +# TRIVY_SEVERITY severities that fail `gate` (default HIGH,CRITICAL) +# TRIVY_IGNORE_UNFIXED skip image vulnerabilities with no fix (default true) +# TRIVY_PLATFORMS image platforms (default "linux/amd64 linux/arm64") +# TRIVY_REPORT_DIR output directory (default reports/trivy) +# TRIVY_SOURCE_ROOT source tree to scan (default repository root) +# TRIVY_IGNORE_FILE ignore file to apply (default repository copy) + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SOURCE_ROOT="${TRIVY_SOURCE_ROOT:-${REPO_ROOT}}" +IGNORE_FILE="${TRIVY_IGNORE_FILE:-${REPO_ROOT}/.trivyignore.yaml}" +cd "${SOURCE_ROOT}" + +SEVERITY="${TRIVY_SEVERITY:-HIGH,CRITICAL}" +REPORT_DIR="${TRIVY_REPORT_DIR:-reports/trivy}" +IGNORE_UNFIXED="${TRIVY_IGNORE_UNFIXED:-true}" +PLATFORMS="${TRIVY_PLATFORMS:-linux/amd64 linux/arm64}" + +# Rendering the chart outside a cluster cannot satisfy the Agent Sandbox API +# discovery check, and that template calls `fail`. +PREFLIGHT_OFF=(--helm-set agentSandbox.preflight.enabled=false) + +# These Dockerfiles produce no runnable image: the macOS ones export a binary +# from `FROM scratch`, and the CI image is toolchain, not a release artifact. +SKIP_DOCKERFILES=( + --skip-files 'deploy/docker/Dockerfile.ci' + --skip-files 'deploy/docker/Dockerfile.*-macos' +) + +# Run one scan. Reports keep every severity; `gate` applies the threshold. +# `prefix` is prepended to SARIF locations, which Trivy reports relative to the +# scanned target while Code Scanning resolves them from the repository root. +scan() { + local subcommand=$1 slug=$2 prefix=$3 + shift 3 + + echo "==> ${slug}" + trivy "${subcommand}" --skip-version-check --quiet \ + --ignorefile "${IGNORE_FILE}" \ + --format json --output "${REPORT_DIR}/${slug}.json" "$@" + trivy convert --quiet \ + --format sarif --output "${REPORT_DIR}/${slug}.sarif" \ + "${REPORT_DIR}/${slug}.json" + + if [ -n "${prefix}" ]; then + jq --arg p "${prefix}" ' + (.. | objects | select(has("artifactLocation")) | .artifactLocation.uri) + |= $p + (. | sub("^[^:]*\\.tgz:"; "")) + ' "${REPORT_DIR}/${slug}.sarif" >"${REPORT_DIR}/${slug}.sarif.tmp" + mv "${REPORT_DIR}/${slug}.sarif.tmp" "${REPORT_DIR}/${slug}.sarif" + fi +} + +# Scanning deploy/ in one pass covers both charts, the published Dockerfiles and +# the raw manifests, and keeps every reported path relative to the same root. +scan_config() { + scan config config-defaults deploy/ "${PREFLIGHT_OFF[@]}" \ + "${SKIP_DOCKERFILES[@]}" deploy + + # The chart defaults render 10 of its 19 templates. The high-availability + # Deployment, the Gateway API objects, the OpenShift Route and the wider + # workspace-mode ClusterRole only render under CI value fixtures. + local values fixture + for values in deploy/helm/openshell/ci/values-*.yaml; do + fixture="$(basename "${values}" .yaml | sed 's/^values-//')" + scan config "config-fixture-${fixture}" deploy/ "${PREFLIGHT_OFF[@]}" \ + "${SKIP_DOCKERFILES[@]}" --helm-values "${values}" deploy + done +} + +# Trivy has no OCI artifact target and rejects the Helm config media type, so a +# published chart has to be pulled before it can be scanned. It reads the +# archive directly, and skips secret scanning on packaged charts. +scan_packaged_chart() { + local ref=$1 + if [[ "${ref}" != *:* || "${ref##*/}" != *:* ]]; then + echo "Error: --chart-ref needs a version tag, e.g. oci://host/chart:1.2.3" >&2 + exit 2 + fi + + local dir + dir="$(mktemp -d)" + trap 'rm -rf "${dir}"' RETURN + + helm pull "${ref%:*}" --version "${ref##*:}" --destination "${dir}" + scan config config-packaged-chart deploy/helm/openshell/ \ + "${PREFLIGHT_OFF[@]}" "$(find "${dir}" -name '*.tgz' -print -quit)" +} + +scan_images() { + local extra=() + [ "${IGNORE_UNFIXED}" = "true" ] && extra+=(--ignore-unfixed) + + local image platform slug + for image in "$@"; do + # Published tags are multi-arch indexes and Trivy defaults to the runner's + # own platform, so each architecture needs its own scan. + for platform in ${PLATFORMS}; do + slug="image-$(printf '%s' "${image#*/}-${platform}" | tr -cs 'A-Za-z0-9._-' '-')" + scan image "${slug}" "" --platform "${platform}" --scanners vuln \ + "${extra[@]}" "${image}" + done + done +} + +# Re-read the reports and apply the threshold. The table doubles as the run +# summary, so nothing here reimplements counting. +gate() { + local report result findings=0 + + # `find` rather than `compgen -G`: compgen belongs to bash's programmable + # completion, which the non-interactive bash in the Nix dev shell does not + # ship, so it fails with "command not found" there. + if [ -z "$(find "${REPORT_DIR}" -maxdepth 1 -name '*.json' -print -quit)" ]; then + echo "Error: no reports in ${REPORT_DIR}; run 'config' or 'images' first" >&2 + exit 2 + fi + + for report in "${REPORT_DIR}"/*.json; do + set +e + trivy convert --quiet --exit-code 10 --severity "${SEVERITY}" \ + --format table "${report}" + result=$? + set -e + + case "${result}" in + 0) ;; + 10) findings=1 ;; + *) + echo "Error: Trivy could not evaluate ${report} (exit ${result})" >&2 + return "${result}" + ;; + esac + done + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### Trivy (gate: \`${SEVERITY}\`)" + echo '```' + for report in "${REPORT_DIR}"/*.json; do + trivy convert --quiet --severity "${SEVERITY}" --format table "${report}" + done + echo '```' + } >>"${GITHUB_STEP_SUMMARY}" + fi + + [ "${findings}" -eq 0 ] || return 10 +} + +collect_config_findings() { + local report_dir=$1 + + if [ -z "$(find "${report_dir}" -maxdepth 1 -name '*.json' -print -quit)" ]; then + echo "Error: no reports in ${report_dir}" >&2 + return 2 + fi + + jq -s --arg severities "${SEVERITY}" ' + [ + .[] + | .Results[]? as $result + | $result.Misconfigurations[]? + | .Severity as $severity + | select(($severities | split(",") | index($severity)) != null) + | { + key: ([ + .ID, + $result.Target, + (.Namespace // ""), + (.Message // ""), + (.CauseMetadata.Provider // ""), + (.CauseMetadata.Service // ""), + (.CauseMetadata.Resource // "") + ] | @json), + severity: .Severity, + id: .ID, + target: $result.Target, + title: .Title + } + ] + | unique_by(.key) + ' "${report_dir}"/*.json +} + +# Compare semantic finding identities instead of line numbers, so unrelated +# edits that move a finding do not make existing debt look newly introduced. +gate_config_diff() ( + set -euo pipefail + + local baseline_dir=$1 candidate_dir=$2 + local inventory_dir baseline candidate new_findings finding_count + inventory_dir="$(mktemp -d)" + trap 'rm -rf "${inventory_dir}"' EXIT + baseline="${inventory_dir}/baseline.json" + candidate="${inventory_dir}/candidate.json" + new_findings="${inventory_dir}/new.json" + + collect_config_findings "${baseline_dir}" >"${baseline}" + collect_config_findings "${candidate_dir}" >"${candidate}" + jq --slurpfile baseline "${baseline}" ' + ($baseline[0] | map(.key)) as $known + | [.[] | select(.key as $key | ($known | index($key)) == null)] + ' "${candidate}" >"${new_findings}" + + finding_count="$(jq 'length' "${new_findings}")" + if [ "${finding_count}" -eq 0 ]; then + echo "No new configuration findings at ${SEVERITY}." + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + echo "No new Trivy configuration findings at \`${SEVERITY}\`." \ + >>"${GITHUB_STEP_SUMMARY}" + fi + exit 0 + fi + + echo "::error::Trivy reported ${finding_count} new configuration finding(s) at ${SEVERITY}." + jq -r '.[] | "::error::[\(.severity)] \(.id) in deploy/\(.target): \(.title)"' \ + "${new_findings}" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + { + echo "### New Trivy configuration findings" + echo + jq -r '.[] | "- **\(.severity)** `\(.id)` in `deploy/\(.target)`: \(.title)"' \ + "${new_findings}" + } >>"${GITHUB_STEP_SUMMARY}" + fi + exit 10 +) + +command -v trivy >/dev/null || { echo "Error: trivy not on PATH; run inside 'nix develop'" >&2; exit 2; } + +case "${1:-}" in + config) + shift + mkdir -p "${REPORT_DIR}" + scan_config + if [ "${1:-}" = "--chart-ref" ]; then + [ -n "${2:-}" ] || { echo "Error: --chart-ref needs a value" >&2; exit 2; } + scan_packaged_chart "$2" + fi + ;; + images) + shift + [ $# -gt 0 ] || { echo "Error: images needs at least one reference" >&2; exit 2; } + mkdir -p "${REPORT_DIR}" + scan_images "$@" + ;; + gate) + gate + ;; + gate-config-diff) + shift + [ $# -eq 2 ] || { + echo "Error: gate-config-diff needs baseline and candidate report directories" >&2 + exit 2 + } + gate_config_diff "$1" "$2" + ;; + *) + cat >&2 <<'USAGE' +Usage: + trivy-scan.sh config [--chart-ref ] + trivy-scan.sh images [...] + trivy-scan.sh gate + trivy-scan.sh gate-config-diff +USAGE + exit 2 + ;; +esac From 0a96d9ced870dbf30be01de2ba359b07b751efc6 Mon Sep 17 00:00:00 2001 From: Adrien Langou Date: Mon, 7 Sep 2026 17:33:21 +0200 Subject: [PATCH 2/5] fix(ci): harden Trivy gate detection and finding diff Signed-off-by: Adrien Langou --- .github/workflows/trivy-changes.yml | 8 ++- CI.md | 5 +- architecture/build.md | 27 +++++--- tasks/scripts/trivy-scan.sh | 98 ++++++++++++++++++++--------- 4 files changed, 100 insertions(+), 38 deletions(-) diff --git a/.github/workflows/trivy-changes.yml b/.github/workflows/trivy-changes.yml index cf66ca0e81..28339805f5 100644 --- a/.github/workflows/trivy-changes.yml +++ b/.github/workflows/trivy-changes.yml @@ -33,7 +33,7 @@ jobs: contents: read pull-requests: read outputs: - should_run: ${{ steps.default.outputs.should_run || steps.changed.outputs.any_changed }} + should_run: ${{ steps.default.outputs.should_run || steps.changed.outputs.any_modified }} steps: - id: default if: github.event_name != 'pull_request' @@ -48,9 +48,15 @@ jobs: if: github.event_name == 'pull_request' uses: tj-actions/changed-files@aa08304bd477b800d468db44fe10f6c61f7f7b11 # v42.1.0 with: + # `any_modified` covers deletions, which `any_changed` omits, and a + # failed diff has to fail the job: both otherwise report no relevant + # change, and removing the scanner or a value fixture would skip the + # scan behind a green status. + fail_on_initial_diff_error: true files: | deploy/docker/** deploy/helm/** + deploy/kube/** .trivyignore.yaml flake.nix flake.lock diff --git a/CI.md b/CI.md index 480f5626e1..55abeb3d3d 100644 --- a/CI.md +++ b/CI.md @@ -146,8 +146,9 @@ nix develop --command tasks/scripts/trivy-scan.sh gate ### Pull-request change gate `Trivy Changes` runs directly on pull requests and merge groups. It detects -changes to Helm charts, release Dockerfiles, and the Trivy tooling, then scans -both the base revision and the candidate with the same scanner logic. The check +changes to Helm charts, release Dockerfiles, the raw Kubernetes manifests, and +the Trivy tooling — deletions included — then scans both the base revision and +the candidate with the same scanner logic. The check fails only when the candidate introduces a new `HIGH` or `CRITICAL` misconfiguration, so existing findings do not block unrelated work. Reports from both revisions are retained as workflow artifacts. diff --git a/architecture/build.md b/architecture/build.md index 0d6b5402c7..b6b153d219 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -478,7 +478,11 @@ Trivy has no OCI artifact target, and `trivy image` rejects the Helm config medi type, so a packaged chart has to be fetched with `helm pull` before it can be scanned. Trivy reports locations relative to the scanned target, so `tasks/scripts/trivy-scan.sh` rewrites SARIF URIs to repository-relative paths; -without that, Code Scanning resolves alerts against files that do not exist. That +without that, Code Scanning resolves alerts against files that do not exist. The +prefix comes from whichever chart declares the published name rather than from a +fixed directory, because a chart's published name is not its directory name and +the two charts share template filenames: a hardcoded prefix would report +`openshell-workspace` alerts against the gateway chart's `role.yaml`. That rewrite and the profile loop are the only repository-specific logic: severity filtering, the pass/fail decision and the summary table all come from `trivy convert --exit-code`, so nothing reimplements counting. @@ -501,18 +505,27 @@ its findings are separate changes, in that order. `.github/workflows/trivy-changes.yml` gates changes rather than releases. It runs on `pull_request` and `merge_group`; `workflow_dispatch` takes explicit base and head SHAs for diagnostics. A detection job decides whether the change -touches `deploy/docker/**`, `deploy/helm/**`, or the scanner inputs themselves -(`.trivyignore.yaml`, `flake.nix`, `flake.lock`, `tasks/scripts/trivy-scan.sh`, -and the workflow file). +touches `deploy/docker/**`, `deploy/helm/**`, `deploy/kube/**`, or the scanner +inputs themselves (`.trivyignore.yaml`, `flake.nix`, `flake.lock`, +`tasks/scripts/trivy-scan.sh`, and the workflow file); the watched paths track +what the scan covers, so a change to the raw manifests cannot land unscanned. +Detection counts deletions and treats a failed diff as a +failure, so removing the scanner, a value fixture, or the ignore file cannot skip +the scan behind a passing status. When it does, the scan job checks out both the baseline and the candidate and runs the candidate's `trivy-scan.sh config` over each tree with the candidate's `.trivyignore.yaml`, so a scanner or ignore-policy change is judged by its own rules on both sides. `gate-config-diff` then compares semantic finding identities — rule ID, target, namespace, message, and cause -provider/service/resource — and fails only on identities absent from the -baseline. The four findings above therefore keep reporting without blocking -every pull request, while a newly introduced `HIGH` or `CRITICAL` +provider/service/resource — together with how many times each occurs. Line +numbers stay out of the identity so that edits which merely move a finding do not +look new, and the count stops a second offending block from hiding behind an +identity the baseline already reports: `KSV-0041` covers two rules of the +workspace-mode ClusterRole today, so a third fails. Counts are taken per report +and reduced with `max`, never summed, so a new value fixture rendering the same +templates adds no debt. The four findings above therefore keep reporting without +blocking every pull request, while a newly introduced `HIGH` or `CRITICAL` misconfiguration fails the check. Both report sets are uploaded as workflow artifacts. diff --git a/tasks/scripts/trivy-scan.sh b/tasks/scripts/trivy-scan.sh index aaf34a5b58..116db01bb9 100755 --- a/tasks/scripts/trivy-scan.sh +++ b/tasks/scripts/trivy-scan.sh @@ -96,12 +96,31 @@ scan_packaged_chart() { exit 2 fi - local dir + # A published chart name is not its directory name — the gateway chart is + # `helm-chart` under deploy/helm/openshell — and both published charts share + # template filenames. Resolving the SARIF prefix and the report slug from the + # chart that declares the published name keeps `openshell-workspace` alerts off + # the gateway chart's `role.yaml` instead of silently reattributing them. + local repo chart_name chart_dir="" candidate dir + repo="${ref%:*}" + chart_name="${repo##*/}" + for candidate in deploy/helm/*/; do + [ -f "${candidate}Chart.yaml" ] || continue + [ "$(sed -n 's/^name:[[:space:]]*//p' "${candidate}Chart.yaml" | head -1)" \ + = "${chart_name}" ] || continue + chart_dir="${candidate}" + break + done + if [ -z "${chart_dir}" ]; then + echo "Error: no chart under deploy/helm declares name '${chart_name}'" >&2 + exit 2 + fi + dir="$(mktemp -d)" trap 'rm -rf "${dir}"' RETURN - helm pull "${ref%:*}" --version "${ref##*:}" --destination "${dir}" - scan config config-packaged-chart deploy/helm/openshell/ \ + helm pull "${repo}" --version "${ref##*:}" --destination "${dir}" + scan config "config-packaged-${chart_name}" "${chart_dir}" \ "${PREFLIGHT_OFF[@]}" "$(find "${dir}" -name '*.tgz' -print -quit)" } @@ -173,35 +192,49 @@ collect_config_findings() { return 2 fi + # One identity can cover several offending blocks: the key deliberately omits + # line numbers, so two rules in the same ClusterRole granting `secrets` are + # indistinguishable. Occurrences are therefore counted per report and reduced + # with `max`, never summed, because every value fixture scans the same tree and + # repeats its findings across reports while a template repeats them within one. jq -s --arg severities "${SEVERITY}" ' [ .[] - | .Results[]? as $result - | $result.Misconfigurations[]? - | .Severity as $severity - | select(($severities | split(",") | index($severity)) != null) - | { - key: ([ - .ID, - $result.Target, - (.Namespace // ""), - (.Message // ""), - (.CauseMetadata.Provider // ""), - (.CauseMetadata.Service // ""), - (.CauseMetadata.Resource // "") - ] | @json), - severity: .Severity, - id: .ID, - target: $result.Target, - title: .Title - } + | [ + .Results[]? as $result + | $result.Misconfigurations[]? + | .Severity as $severity + | select(($severities | split(",") | index($severity)) != null) + | { + key: ([ + .ID, + $result.Target, + (.Namespace // ""), + (.Message // ""), + (.CauseMetadata.Provider // ""), + (.CauseMetadata.Service // ""), + (.CauseMetadata.Resource // "") + ] | @json), + severity: .Severity, + id: .ID, + target: $result.Target, + title: .Title + } + ] + | group_by(.key) + | map(.[0] + { count: length }) + | .[] ] - | unique_by(.key) + | group_by(.key) + | map(max_by(.count)) ' "${report_dir}"/*.json } # Compare semantic finding identities instead of line numbers, so unrelated # edits that move a finding do not make existing debt look newly introduced. +# Identities carry an occurrence count rather than mere presence, so adding a +# second offending block under an identity the baseline already reports still +# fails. gate_config_diff() ( set -euo pipefail @@ -216,11 +249,16 @@ gate_config_diff() ( collect_config_findings "${baseline_dir}" >"${baseline}" collect_config_findings "${candidate_dir}" >"${candidate}" jq --slurpfile baseline "${baseline}" ' - ($baseline[0] | map(.key)) as $known - | [.[] | select(.key as $key | ($known | index($key)) == null)] + ($baseline[0] | map({ (.key): .count }) | add // {}) as $known + | [ + .[] + | (($known[.key]) // 0) as $before + | select(.count > $before) + | . + { baseline_count: $before, new_count: (.count - $before) } + ] ' "${candidate}" >"${new_findings}" - finding_count="$(jq 'length' "${new_findings}")" + finding_count="$(jq '[.[].new_count] | add // 0' "${new_findings}")" if [ "${finding_count}" -eq 0 ]; then echo "No new configuration findings at ${SEVERITY}." if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then @@ -231,13 +269,17 @@ gate_config_diff() ( fi echo "::error::Trivy reported ${finding_count} new configuration finding(s) at ${SEVERITY}." - jq -r '.[] | "::error::[\(.severity)] \(.id) in deploy/\(.target): \(.title)"' \ + jq -r '.[] + | "::error::[\(.severity)] \(.id) in deploy/\(.target): \(.title)" + + " (\(.new_count) new, \(.baseline_count) in baseline)"' \ "${new_findings}" if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then { echo "### New Trivy configuration findings" echo - jq -r '.[] | "- **\(.severity)** `\(.id)` in `deploy/\(.target)`: \(.title)"' \ + jq -r '.[] + | "- **\(.severity)** `\(.id)` in `deploy/\(.target)`: \(.title)" + + " (\(.new_count) new, \(.baseline_count) in baseline)"' \ "${new_findings}" } >>"${GITHUB_STEP_SUMMARY}" fi From 35d31552176429019ee0a53e6cc704e40c60b777 Mon Sep 17 00:00:00 2001 From: Adrien Langou Date: Mon, 7 Sep 2026 17:34:07 +0200 Subject: [PATCH 3/5] feat(ci): scan released artifacts in release pipelines Signed-off-by: Adrien Langou --- .github/workflows/release-dev.yml | 20 ++++++++++++++++++++ .github/workflows/release-tag.yml | 22 ++++++++++++++++++++++ .github/workflows/trivy-scan.yml | 17 +++++++++-------- CI.md | 11 ++++++++--- architecture/build.md | 16 +++++++++++----- tasks/scripts/trivy-scan.sh | 13 +++++++++---- 6 files changed, 79 insertions(+), 20 deletions(-) diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 18b5f33a2e..426a0fc07e 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -628,6 +628,26 @@ jobs: release-kind: dev pin-sha: ${{ github.sha }} + # Scans what this run published, so it can only follow publication and cannot + # gate it. Findings stay informational; a scanner that cannot run still fails. + # The SHA-pinned chart is used over the floating 0.0.0-dev tag so the scan is + # tied to the images above rather than to whatever dev points at. + scan-released-artifacts: + needs: release-helm + permissions: + contents: read + packages: read + security-events: write + uses: ./.github/workflows/trivy-scan.yml + with: + images: | + ghcr.io/nvidia/openshell/gateway:${{ github.sha }} + ghcr.io/nvidia/openshell/supervisor:${{ github.sha }} + charts: | + oci://ghcr.io/nvidia/openshell/helm-chart:0.0.0-dev.${{ github.sha }} + oci://ghcr.io/nvidia/openshell/openshell-workspace:0.0.0-dev.${{ github.sha }} + secrets: inherit + trigger-wheel-publish: name: Trigger Wheel Publish needs: [compute-versions, release-dev] diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index b3c45c97fb..112c228da5 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -747,6 +747,28 @@ jobs: app-version: ${{ needs.compute-versions.outputs.semver }} release-kind: public + # Scans what this run published, so it can only follow publication and cannot + # gate it. Findings stay informational; a scanner that cannot run still fails. + # SARIF upload is off here because the run's ref is a tag: Code Scanning keys + # alerts by ref, and Release Dev already publishes them against main. The + # scope is still granted because the called jobs declare it unconditionally. + scan-released-artifacts: + needs: [compute-versions, release-helm] + permissions: + contents: read + packages: read + security-events: write + uses: ./.github/workflows/trivy-scan.yml + with: + upload-sarif: false + images: | + ghcr.io/nvidia/openshell/gateway:${{ needs.compute-versions.outputs.semver }} + ghcr.io/nvidia/openshell/supervisor:${{ needs.compute-versions.outputs.semver }} + charts: | + oci://ghcr.io/nvidia/openshell/helm-chart:${{ needs.compute-versions.outputs.semver }} + oci://ghcr.io/nvidia/openshell/openshell-workspace:${{ needs.compute-versions.outputs.semver }} + secrets: inherit + trigger-wheel-publish: name: Trigger Wheel Publish needs: [compute-versions, release] diff --git a/.github/workflows/trivy-scan.yml b/.github/workflows/trivy-scan.yml index fa37db9922..9346716a6b 100644 --- a/.github/workflows/trivy-scan.yml +++ b/.github/workflows/trivy-scan.yml @@ -19,9 +19,9 @@ on: required: false type: string default: "" - chart-ref: + charts: description: | - Packaged Helm chart to scan, for example + Newline-separated packaged Helm charts to scan, for example oci://ghcr.io/nvidia/openshell/helm-chart:0.0.116 required: false type: string @@ -58,8 +58,8 @@ on: required: false type: string default: "" - chart-ref: - description: Packaged Helm chart OCI reference to scan + charts: + description: Newline-separated packaged Helm chart OCI references to scan required: false type: string default: "" @@ -202,12 +202,13 @@ jobs: - name: Scan configuration id: scan env: - CHART_REF: ${{ inputs.chart-ref }} + CHARTS: ${{ inputs.charts }} run: | args=() - if [ -n "$CHART_REF" ]; then - args+=(--chart-ref "$CHART_REF") - fi + while IFS= read -r ref; do + [ -n "$ref" ] || continue + args+=(--chart-ref "$ref") + done <<<"$CHARTS" tasks/scripts/trivy-scan.sh config "${args[@]}" - name: Upload SARIF to Code Scanning diff --git a/CI.md b/CI.md index 55abeb3d3d..27b64abc82 100644 --- a/CI.md +++ b/CI.md @@ -101,9 +101,14 @@ additionally ignores vulnerabilities with no upstream fix, because a base-image CVE without a patch would otherwise be permanent noise. That option does not apply to misconfigurations. -It is not listed in any release workflow's `needs:`, so no publication depends -on it. Wiring it into `release-dev.yml` and `release-tag.yml` is a separate -change, and one that only makes sense once findings fail. +`release-dev.yml` and `release-tag.yml` both call it once publication has +finished, passing the images and the two charts that run published. Because it +scans published artifacts, it can only follow publication and never gates it — +no release waits on the result. Release Dev uploads SARIF against `main`; the +tag release keeps reports as artifacts only, since Code Scanning keys alerts by +ref and a tag ref would duplicate what `main` already shows. Findings remain +informational there too: with four checks reporting today, `fail-on-findings` +would break every release, so flipping it stays a separate change. The configuration scan targets `deploy/` in one pass, which covers both charts, the published Dockerfiles and the raw manifests. The macOS Dockerfiles export a diff --git a/architecture/build.md b/architecture/build.md index b6b153d219..7e7457b197 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -447,9 +447,15 @@ pull-request change gate. ### Release Artifacts `.github/workflows/trivy-scan.yml` is reusable and takes OCI references as -input, so it has no knowledge of how a release is assembled and no dependency on -release job ordering. Nix supplies both Trivy and Helm, and the jobs stay on -GitHub-hosted runners like the other scanners. +input, so it has no knowledge of how a release is assembled. Nix supplies both +Trivy and Helm, and the jobs stay on GitHub-hosted runners like the other +scanners. + +Both release workflows call it after their Helm publication step, passing the +images and every chart that run published. Scanning published artifacts means it +can only follow publication, so it reports on a release rather than gating one: +no publication job depends on the result. Findings stay informational while four +checks report, because `fail-on-findings` would otherwise break every release. Findings are informational during the observation phase: they warn and the run stays green, while a scanner that cannot run still fails. The `fail-on-findings` @@ -470,8 +476,8 @@ Two scopes, with deliberately different reporting semantics: combinations: the chart defaults render 10 of the chart's 19 templates, so CI value fixtures exercise conditional resources such as the high-availability Deployment, Gateway API objects, OpenShift Route, and broader workspace-mode - ClusterRole. Each fixture is scanned on its own, and the packaged chart is - scanned from its published OCI reference to cover the artifact consumers + ClusterRole. Each fixture is scanned on its own, and each packaged chart is + scanned from its published OCI reference to cover the artifacts consumers actually install. Trivy has no OCI artifact target, and `trivy image` rejects the Helm config media diff --git a/tasks/scripts/trivy-scan.sh b/tasks/scripts/trivy-scan.sh index 116db01bb9..35edc41889 100755 --- a/tasks/scripts/trivy-scan.sh +++ b/tasks/scripts/trivy-scan.sh @@ -7,7 +7,7 @@ set -euo pipefail # Scan release artifacts with Trivy. # -# trivy-scan.sh config [--chart-ref ] +# trivy-scan.sh config [--chart-ref ]... # trivy-scan.sh images [...] # trivy-scan.sh gate # trivy-scan.sh gate-config-diff @@ -293,10 +293,15 @@ case "${1:-}" in shift mkdir -p "${REPORT_DIR}" scan_config - if [ "${1:-}" = "--chart-ref" ]; then + # A release publishes every chart under deploy/helm, so `--chart-ref` + # repeats. Unparsed arguments are rejected rather than ignored: a misspelled + # flag would otherwise leave the packaged charts unscanned and still exit 0. + while [ "${1:-}" = "--chart-ref" ]; do [ -n "${2:-}" ] || { echo "Error: --chart-ref needs a value" >&2; exit 2; } scan_packaged_chart "$2" - fi + shift 2 + done + [ $# -eq 0 ] || { echo "Error: unexpected argument '$1' after config" >&2; exit 2; } ;; images) shift @@ -318,7 +323,7 @@ case "${1:-}" in *) cat >&2 <<'USAGE' Usage: - trivy-scan.sh config [--chart-ref ] + trivy-scan.sh config [--chart-ref ]... trivy-scan.sh images [...] trivy-scan.sh gate trivy-scan.sh gate-config-diff From ed218dd040380ff797907f4490ca00d0e461d942 Mon Sep 17 00:00:00 2001 From: Adrien Langou Date: Tue, 8 Sep 2026 16:23:57 +0200 Subject: [PATCH 4/5] fix(ci): harden and simplify Trivy scans Signed-off-by: Adrien Langou --- .github/workflows/release-dev.yml | 20 --- .github/workflows/release-tag.yml | 22 --- .github/workflows/trivy-changes.yml | 21 ++- .github/workflows/trivy-scan.yml | 185 +++++----------------- .trivyignore.yaml | 44 +----- CI.md | 95 ++++-------- architecture/build.md | 137 +++++----------- flake.nix | 1 + tasks/scripts/trivy-scan-test.sh | 101 ++++++++++++ tasks/scripts/trivy-scan.sh | 233 ++++++++++++++++------------ 10 files changed, 367 insertions(+), 492 deletions(-) create mode 100755 tasks/scripts/trivy-scan-test.sh diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 426a0fc07e..18b5f33a2e 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -628,26 +628,6 @@ jobs: release-kind: dev pin-sha: ${{ github.sha }} - # Scans what this run published, so it can only follow publication and cannot - # gate it. Findings stay informational; a scanner that cannot run still fails. - # The SHA-pinned chart is used over the floating 0.0.0-dev tag so the scan is - # tied to the images above rather than to whatever dev points at. - scan-released-artifacts: - needs: release-helm - permissions: - contents: read - packages: read - security-events: write - uses: ./.github/workflows/trivy-scan.yml - with: - images: | - ghcr.io/nvidia/openshell/gateway:${{ github.sha }} - ghcr.io/nvidia/openshell/supervisor:${{ github.sha }} - charts: | - oci://ghcr.io/nvidia/openshell/helm-chart:0.0.0-dev.${{ github.sha }} - oci://ghcr.io/nvidia/openshell/openshell-workspace:0.0.0-dev.${{ github.sha }} - secrets: inherit - trigger-wheel-publish: name: Trigger Wheel Publish needs: [compute-versions, release-dev] diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 112c228da5..b3c45c97fb 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -747,28 +747,6 @@ jobs: app-version: ${{ needs.compute-versions.outputs.semver }} release-kind: public - # Scans what this run published, so it can only follow publication and cannot - # gate it. Findings stay informational; a scanner that cannot run still fails. - # SARIF upload is off here because the run's ref is a tag: Code Scanning keys - # alerts by ref, and Release Dev already publishes them against main. The - # scope is still granted because the called jobs declare it unconditionally. - scan-released-artifacts: - needs: [compute-versions, release-helm] - permissions: - contents: read - packages: read - security-events: write - uses: ./.github/workflows/trivy-scan.yml - with: - upload-sarif: false - images: | - ghcr.io/nvidia/openshell/gateway:${{ needs.compute-versions.outputs.semver }} - ghcr.io/nvidia/openshell/supervisor:${{ needs.compute-versions.outputs.semver }} - charts: | - oci://ghcr.io/nvidia/openshell/helm-chart:${{ needs.compute-versions.outputs.semver }} - oci://ghcr.io/nvidia/openshell/openshell-workspace:${{ needs.compute-versions.outputs.semver }} - secrets: inherit - trigger-wheel-publish: name: Trigger Wheel Publish needs: [compute-versions, release] diff --git a/.github/workflows/trivy-changes.yml b/.github/workflows/trivy-changes.yml index 28339805f5..642ca99757 100644 --- a/.github/workflows/trivy-changes.yml +++ b/.github/workflows/trivy-changes.yml @@ -61,6 +61,7 @@ jobs: flake.nix flake.lock tasks/scripts/trivy-scan.sh + tasks/scripts/trivy-scan-test.sh .github/workflows/trivy-changes.yml scan: @@ -92,10 +93,27 @@ jobs: - name: Set up Nix uses: ./.github/actions/setup-nix + - name: Test report comparison + run: tasks/scripts/trivy-scan-test.sh + + - name: Validate candidate ignore policy + run: tasks/scripts/trivy-scan.sh validate-ignore + + # Ignore-policy changes take effect only after merge. Applying the + # baseline policy to both scans prevents a candidate from exempting a new + # finding in the same change that introduces it. + - name: Prepare baseline ignore policy + run: | + if [ -f .trivy-base/.trivyignore.yaml ]; then + cp .trivy-base/.trivyignore.yaml "$RUNNER_TEMP/trivy-baseline-ignore.yaml" + else + printf 'misconfigurations: []\n' >"$RUNNER_TEMP/trivy-baseline-ignore.yaml" + fi + - name: Scan baseline env: TRIVY_SOURCE_ROOT: ${{ github.workspace }}/.trivy-base - TRIVY_IGNORE_FILE: ${{ github.workspace }}/.trivyignore.yaml + TRIVY_IGNORE_FILE: ${{ runner.temp }}/trivy-baseline-ignore.yaml TRIVY_REPORT_DIR: ${{ runner.temp }}/trivy-base run: | mkdir -p "$TRIVY_REPORT_DIR" @@ -103,6 +121,7 @@ jobs: - name: Scan candidate env: + TRIVY_IGNORE_FILE: ${{ runner.temp }}/trivy-baseline-ignore.yaml TRIVY_REPORT_DIR: ${{ runner.temp }}/trivy-head run: | mkdir -p "$TRIVY_REPORT_DIR" diff --git a/.github/workflows/trivy-scan.yml b/.github/workflows/trivy-scan.yml index 9346716a6b..5722d079b7 100644 --- a/.github/workflows/trivy-scan.yml +++ b/.github/workflows/trivy-scan.yml @@ -3,84 +3,64 @@ name: Trivy Scan -# Scans the artifacts a release publishes: the final container images and the -# deployment configuration. Callers pass OCI references, so this workflow is -# self-contained and knows nothing about how a release is assembled. -# -# Findings are informational while we learn what this reports in practice: they -# produce a warning, not a failure. A scanner that cannot run still fails, so a -# broken scan cannot look clean. Set fail-on-findings to flip the gate on. +# Manual or reusable scan of deployment configuration and supplied OCI +# artifacts. Findings are informational by default; scanner errors remain fatal. on: workflow_call: inputs: images: description: Newline-separated image references to scan - required: false type: string default: "" charts: - description: | - Newline-separated packaged Helm charts to scan, for example - oci://ghcr.io/nvidia/openshell/helm-chart:0.0.116 - required: false + description: Newline-separated packaged Helm chart OCI references type: string default: "" severity: description: Severities that fail the workflow - required: false type: string default: HIGH,CRITICAL ignore-unfixed: description: Ignore image vulnerabilities with no upstream fix - required: false type: boolean default: true fail-on-findings: description: Fail the run on findings instead of warning - required: false type: boolean default: false upload-sarif: description: Upload results to GitHub Code Scanning - required: false type: boolean default: true secrets: CACHIX_AUTH_TOKEN: description: Token used to write Nix build outputs to Cachix - required: false workflow_dispatch: inputs: images: description: Newline-separated image references to scan - required: false type: string default: "" charts: description: Newline-separated packaged Helm chart OCI references to scan - required: false type: string default: "" severity: description: Severities that fail the workflow - required: false type: string default: HIGH,CRITICAL ignore-unfixed: description: Ignore image vulnerabilities with no upstream fix - required: false type: boolean default: true fail-on-findings: description: Fail the run on findings instead of warning - required: false type: boolean default: false upload-sarif: description: Upload results to GitHub Code Scanning - required: false type: boolean default: true @@ -92,91 +72,14 @@ defaults: shell: nix develop --command bash -euo pipefail {0} env: - TRIVY_SEVERITY: ${{ inputs.severity }} + TRIVY_SEVERITY: ${{ inputs.severity || 'HIGH,CRITICAL' }} TRIVY_REPORT_DIR: reports/trivy jobs: - image: - name: Image vulnerabilities (informational) - if: inputs.images != '' - runs-on: ubuntu-latest - timeout-minutes: 30 - permissions: - contents: read - packages: read - security-events: write - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # Trivy reads ~/.docker/config.json, which `nix develop` leaves alone. - - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ github.token }} - - - name: Set up Nix - uses: ./.github/actions/setup-nix - with: - cachix-auth-token: ${{ github.event_name != 'pull_request' && secrets.CACHIX_AUTH_TOKEN || '' }} - - - name: Scan images - id: scan - env: - IMAGES: ${{ inputs.images }} - TRIVY_IGNORE_UNFIXED: ${{ inputs.ignore-unfixed }} - run: | - refs=() - while IFS= read -r ref; do - [ -n "$ref" ] || continue - refs+=("$ref") - done <<<"$IMAGES" - tasks/scripts/trivy-scan.sh images "${refs[@]}" - - - name: Upload SARIF to Code Scanning - if: ${{ !cancelled() && steps.scan.conclusion == 'success' && inputs.upload-sarif }} - uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 - with: - sarif_file: reports/trivy - category: trivy-image - - - name: Upload reports - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: trivy-image-${{ github.run_id }} - path: reports/trivy - if-no-files-found: ignore - retention-days: 14 - - # Separate from the scan so a tripped gate still publishes its reports. - - name: Report findings - if: ${{ !cancelled() && steps.scan.conclusion == 'success' }} - env: - FAIL_ON_FINDINGS: ${{ inputs.fail-on-findings }} - run: | - set +e - tasks/scripts/trivy-scan.sh gate - status=$? - set -e - case "$status" in - 0) echo "No findings at ${TRIVY_SEVERITY}." ;; - 10) - if [ "$FAIL_ON_FINDINGS" = "true" ]; then - echo "::error::Image findings at ${TRIVY_SEVERITY}." - exit 1 - fi - echo "::warning::Image findings at ${TRIVY_SEVERITY}; this check is informational." - ;; - *) echo "::error::Trivy could not evaluate the reports (exit $status)."; exit "$status" ;; - esac - - config: - name: Configuration misconfigurations (informational) + scan: + name: OpenShell / Trivy (informational) runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 permissions: contents: read packages: read @@ -186,8 +89,6 @@ jobs: with: persist-credentials: false - # `helm pull` needs registry credentials only when a packaged chart is - # requested, but logging in unconditionally keeps the step list flat. - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io @@ -200,7 +101,7 @@ jobs: cachix-auth-token: ${{ github.event_name != 'pull_request' && secrets.CACHIX_AUTH_TOKEN || '' }} - name: Scan configuration - id: scan + id: config env: CHARTS: ${{ inputs.charts }} run: | @@ -211,24 +112,50 @@ jobs: done <<<"$CHARTS" tasks/scripts/trivy-scan.sh config "${args[@]}" + - name: Scan images + id: images + if: ${{ !cancelled() }} + env: + IMAGES: ${{ inputs.images }} + TRIVY_IGNORE_UNFIXED: ${{ inputs.ignore-unfixed }} + run: | + refs=() + while IFS= read -r ref; do + [ -n "$ref" ] || continue + refs+=("$ref") + done <<<"$IMAGES" + if [ "${#refs[@]}" -gt 0 ]; then + tasks/scripts/trivy-scan.sh images "${refs[@]}" + fi + - name: Upload SARIF to Code Scanning - if: ${{ !cancelled() && steps.scan.conclusion == 'success' && inputs.upload-sarif }} + if: >- + ${{ + !cancelled() + && steps.config.conclusion == 'success' + && steps.images.conclusion == 'success' + && inputs.upload-sarif + }} uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 with: sarif_file: reports/trivy - category: trivy-config - name: Upload reports if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: trivy-config-${{ github.run_id }} + name: trivy-${{ github.run_id }} path: reports/trivy if-no-files-found: ignore retention-days: 14 - name: Report findings - if: ${{ !cancelled() && steps.scan.conclusion == 'success' }} + if: >- + ${{ + !cancelled() + && steps.config.conclusion == 'success' + && steps.images.conclusion == 'success' + }} env: FAIL_ON_FINDINGS: ${{ inputs.fail-on-findings }} run: | @@ -240,42 +167,10 @@ jobs: 0) echo "No findings at ${TRIVY_SEVERITY}." ;; 10) if [ "$FAIL_ON_FINDINGS" = "true" ]; then - echo "::error::Configuration findings at ${TRIVY_SEVERITY}." + echo "::error::Trivy findings at ${TRIVY_SEVERITY}." exit 1 fi - echo "::warning::Configuration findings at ${TRIVY_SEVERITY}; this check is informational." + echo "::warning::Trivy findings at ${TRIVY_SEVERITY}; this check is informational." ;; *) echo "::error::Trivy could not evaluate the reports (exit $status)."; exit "$status" ;; esac - - # Republishes whether the scans ran, not what they found, so a broken scanner - # cannot pass as healthy while findings stay informational. - result: - name: OpenShell / Trivy (informational) - needs: [image, config] - if: always() - runs-on: ubuntu-latest - defaults: - run: - shell: bash - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - # The image job is skipped when no references are passed, which - # check-job-results treats as a failure, so drop skipped jobs first. - - name: Drop skipped jobs - id: required - env: - JOB_RESULTS: ${{ toJSON(needs) }} - run: | - { - echo 'results<>"$GITHUB_OUTPUT" - - - uses: ./.github/actions/check-job-results - with: - results: ${{ steps.required.outputs.results }} diff --git a/.trivyignore.yaml b/.trivyignore.yaml index 893ac16502..5e6afc010e 100644 --- a/.trivyignore.yaml +++ b/.trivyignore.yaml @@ -1,36 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Trivy exceptions. Passed explicitly with --ignorefile by tasks/scripts/trivy-scan.sh, -# because Trivy auto-loads a plain `.trivyignore` but not the YAML variant. -# -# An entry belongs here only when the finding is wrong: the condition it -# reports is not true of this repository, or it is an artifact of how the scan -# renders the chart. Nothing else qualifies. Findings that describe hardening -# we have not done, or a risk we have accepted, stay in the report where they -# can be seen and argued about, even when that means the gate fails. -# -# Always scope an entry with `paths`, naming individual files. An `id` on its -# own disables the check everywhere, which would also hide a genuine occurrence -# elsewhere. -# -# Use `**/.yaml`. Paths match the location Trivy reports, which is -# relative to the scanned target, and the same template is reported two ways: -# `helm/openshell/templates/x.yaml` when scanning deploy/, and -# `helm-chart-.tgz:templates/x.yaml` when scanning the published chart. -# Only a leading `**/` matches both. `*templates/x.yaml` silently stops applying -# to the repository scan, and `**/templates/x.yaml` to the packaged one. -# -# `paths` is also as narrow as this file can get: for misconfigurations Trivy -# offers no per-occurrence scoping, so a second, legitimate finding of the same -# check in a listed file would be hidden too. Inline `#trivy:ignore:` comments -# would fix that and do work for Dockerfiles, but Trivy 0.74 does not apply them -# to Helm templates. Revisit when it does. +# False positives only; unimplemented hardening and accepted risks stay visible. +# The script passes this YAML file explicitly and requires each exception to use +# `**/`, which matches source and packaged-chart paths. +# Trivy cannot scope Helm exceptions to one occurrence, so keep IDs and paths +# narrow. misconfigurations: - # helm template renders without a namespace, so every workload appears to be - # in "default". The namespace comes from `helm install -n` and no template - # hardcodes one. + # The namespace comes from `helm install -n`, not the rendered workload. - id: KSV-0110 paths: - "**/statefulset.yaml" @@ -39,10 +17,7 @@ misconfigurations: An artifact of rendering the chart outside a cluster. The namespace is supplied at install time. - # This ConfigMap stores the *name* of a key inside an external Secret - # (proxy_auth_secret_key = "proxy-auth"), not the credential. Scoped to the - # one file, because elsewhere this check is what would catch a real - # credential committed into a ConfigMap. + # The ConfigMap stores an external Secret key name, not a credential. - id: KSV-01010 paths: - "**/gateway-config.yaml" @@ -50,10 +25,7 @@ misconfigurations: The ConfigMap holds the name of a key in an external Secret, not a credential. - # ghcr.io/nvidia/openshell is where this project publishes its own images. - # Trivy's default trusted-registry list cannot be extended in the version we - # run, so the check cannot be taught about our registry. Scoped to the two - # workload templates so third-party images referenced elsewhere still report. + # Trivy cannot add this project's GHCR namespace to its trusted registries. - id: KSV-0125 paths: - "**/statefulset.yaml" diff --git a/CI.md b/CI.md index 27b64abc82..b0596f72be 100644 --- a/CI.md +++ b/CI.md @@ -83,64 +83,23 @@ nix develop --command zizmor --offline --persona=regular --min-severity=high --n ## Artifact scanning -`Trivy Scan` differs from the reports above in what it looks at rather than in -how it reports: it scans what a release publishes instead of what a change -contains — the final container images, the Helm charts, the final image -Dockerfiles, and the raw Kubernetes manifests. Nix provides Trivy and Helm, and -the jobs run on GitHub-hosted runners like the other scanners. - -Findings are informational for now, while we learn what the scanner reports in -practice. They raise a warning and the run stays green; a scanner that cannot -run still fails, so a broken scan cannot look clean. The `fail-on-findings` -input flips that to a hard failure once the findings have been worked through. - -The workflow is reusable and takes OCI references as input, so it knows nothing -about how a release is assembled. `HIGH` and `CRITICAL` are what get reported as -findings; everything below is listed without comment. Image scanning -additionally ignores vulnerabilities with no upstream fix, because a base-image -CVE without a patch would otherwise be permanent noise. That option does not -apply to misconfigurations. - -`release-dev.yml` and `release-tag.yml` both call it once publication has -finished, passing the images and the two charts that run published. Because it -scans published artifacts, it can only follow publication and never gates it — -no release waits on the result. Release Dev uploads SARIF against `main`; the -tag release keeps reports as artifacts only, since Code Scanning keys alerts by -ref and a tag ref would duplicate what `main` already shows. Findings remain -informational there too: with four checks reporting today, `fail-on-findings` -would break every release, so flipping it stays a separate change. - -The configuration scan targets `deploy/` in one pass, which covers both charts, -the published Dockerfiles and the raw manifests. The macOS Dockerfiles export a -binary from `FROM scratch` and the CI image is toolchain rather than a release -artifact, so both are skipped. - -Chart coverage additionally depends on value combinations. The chart defaults -render 10 of the chart's 19 templates, while some conditional resources only -render with overrides stored under `deploy/helm/openshell/ci/values-*.yaml`. -The scan exercises each of these CI fixtures to cover resources such as the -high-availability Deployment, Gateway API objects, OpenShift Route, and broader -workspace-mode ClusterRole. These fixtures are test inputs, not a set of -separately supported product profiles. - -Exceptions live in `.trivyignore.yaml`, one justification per entry. Trivy -auto-loads a plain `.trivyignore` but not the YAML variant, so the scripts pass -`--ignorefile` explicitly. An entry qualifies only when the finding is wrong: -the condition it reports is not true of this repository, or it is an artifact of -how the scan renders the chart. Hardening we have not done and risks we have -accepted stay in the report, where they can be seen and argued about, even when -that means the gate fails. - -Four checks report today: `KSV-0014` (`readOnlyRootFilesystem` unset on the -gateway container), `KSV-0041` and `KSV-0056` (RBAC grants the managed workspace -mode needs and that RBAC cannot express more narrowly), and `DS-0002` (the -supervisor image runs as root by design). Resolving or consciously accepting -each of those is what has to happen before `fail-on-findings` is worth turning -on. - -Scans write full-severity reports and never fail on findings, so a report is -always available to upload; a separate `gate` step re-reads them and applies the -threshold. Run them locally with: +`Trivy Scan` is a self-contained `workflow_dispatch`/`workflow_call` step. It +always scans deployment configuration and optionally scans supplied OCI image +and chart references. It is not wired into a release workflow; a future analysis +orchestrator can call it directly. + +The single job scans `deploy/`, every Helm CI values fixture, requested packaged +charts, and both Linux architectures of each requested image. Findings are +informational by default, but scanner failures still fail the job and +`fail-on-findings` enables enforcement. Reports retain every severity; the gate +uses `HIGH,CRITICAL` by default. Each SARIF report has a unique automation ID so +the report directory can be uploaded in one operation. + +`.trivyignore.yaml` is reserved for false positives. Every entry must use at +least one `**/` path; `yq` and `jq` validate this structure +before scanning. + +Run the scanner locally with: ```shell nix develop --command tasks/scripts/trivy-scan.sh config @@ -150,17 +109,15 @@ nix develop --command tasks/scripts/trivy-scan.sh gate ### Pull-request change gate -`Trivy Changes` runs directly on pull requests and merge groups. It detects -changes to Helm charts, release Dockerfiles, the raw Kubernetes manifests, and -the Trivy tooling — deletions included — then scans both the base revision and -the candidate with the same scanner logic. The check -fails only when the candidate introduces a new `HIGH` or `CRITICAL` -misconfiguration, so existing findings do not block unrelated work. Reports -from both revisions are retained as workflow artifacts. +`Trivy Changes` scans the base and candidate when a pull request or merge group +changes deployment configuration or scanner inputs. It fails only for new +`HIGH` or `CRITICAL` misconfigurations and retains both report sets. -This check analyzes Helm and Dockerfile configuration. It does not build -container images, so package and operating-system CVEs remain the responsibility -of the release-artifact image scan. +The candidate ignore file is validated, but the baseline policy applies to both +scans so a change cannot exempt its own finding. Existing profiles are compared +independently; a new profile reuses the maximum known occurrence count. Invalid +Trivy reports fail closed. Image package and operating-system CVEs remain the +responsibility of the standalone scan. ## Commit signing @@ -292,7 +249,7 @@ The bot's full administrator documentation is internal to NVIDIA. The only comma | `.github/workflows/codeql.yml` | Runs nightly informational CodeQL analysis on `main` for Rust and the Go, Python, and TypeScript SDKs and retains SARIF artifacts. | | `.github/workflows/codex-security.yml` | Scans the cumulative diff from the previous stable release to each pre-release candidate and publishes train-scoped SARIF on `main`. | | `.github/workflows/trivy-changes.yml` | Blocks pull requests and merge groups that introduce new High or Critical Helm or Dockerfile misconfigurations. | -| `.github/workflows/trivy-scan.yml` | Reusable scan of published container images and deployment configuration. Findings are informational by default and can be configured to fail the workflow. | +| `.github/workflows/trivy-scan.yml` | Manual or reusable scan of supplied OCI image/chart references and deployment configuration. Findings are informational by default and can be configured to fail the workflow. | ## Release workflows diff --git a/architecture/build.md b/architecture/build.md index 7e7457b197..a7c31562d7 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -441,108 +441,47 @@ implemented yet. ## Artifact Scanning -Trivy runs in two places: a reusable release-artifact workflow and a -pull-request change gate. - -### Release Artifacts - -`.github/workflows/trivy-scan.yml` is reusable and takes OCI references as -input, so it has no knowledge of how a release is assembled. Nix supplies both -Trivy and Helm, and the jobs stay on GitHub-hosted runners like the other -scanners. - -Both release workflows call it after their Helm publication step, passing the -images and every chart that run published. Scanning published artifacts means it -can only follow publication, so it reports on a release rather than gating one: -no publication job depends on the result. Findings stay informational while four -checks report, because `fail-on-findings` would otherwise break every release. - -Findings are informational during the observation phase: they warn and the run -stays green, while a scanner that cannot run still fails. The `fail-on-findings` -input turns them into failures, which is a prerequisite for wiring the workflow -into a release `needs:` rather than something to do at the same time. - -Two scopes, with deliberately different reporting semantics: - -- **Images.** `HIGH` and `CRITICAL` are reported, and vulnerabilities with no - upstream fix are ignored. Without that exclusion a base-image CVE with no - available patch would be permanent noise, and a gate nobody can act on once - findings start failing. Published tags are multi-arch indexes and Trivy - defaults to the runner's own platform, so each architecture is scanned - separately. -- **Configuration.** The same severity threshold, but the unfixed exclusion does not - apply to misconfigurations. One pass over `deploy/` covers both charts, the - published Dockerfiles and the raw manifests. Coverage then depends on value - combinations: the chart defaults render 10 of the chart's 19 templates, so - CI value fixtures exercise conditional resources such as the high-availability - Deployment, Gateway API objects, OpenShift Route, and broader workspace-mode - ClusterRole. Each fixture is scanned on its own, and each packaged chart is - scanned from its published OCI reference to cover the artifacts consumers - actually install. - -Trivy has no OCI artifact target, and `trivy image` rejects the Helm config media -type, so a packaged chart has to be fetched with `helm pull` before it can be -scanned. Trivy reports locations relative to the scanned target, so -`tasks/scripts/trivy-scan.sh` rewrites SARIF URIs to repository-relative paths; -without that, Code Scanning resolves alerts against files that do not exist. The -prefix comes from whichever chart declares the published name rather than from a -fixed directory, because a chart's published name is not its directory name and -the two charts share template filenames: a hardcoded prefix would report -`openshell-workspace` alerts against the gateway chart's `role.yaml`. That -rewrite and the profile loop are the only repository-specific logic: severity -filtering, the pass/fail decision and the summary table all come from -`trivy convert --exit-code`, so nothing reimplements counting. - -`.trivyignore.yaml` holds exceptions, and the bar for adding one is that the -finding is wrong: the condition it reports is not true of this repository, or it -is an artifact of how the scan renders the chart. Hardening that has not been -done and risks that have been accepted stay in the report instead, so the -scanner keeps describing the real posture rather than a curated one. Trivy -auto-loads a plain `.trivyignore` but not the YAML variant, so the scripts pass -`--ignorefile` explicitly. - -That bar means four checks report today: `KSV-0014`, `KSV-0041`, `KSV-0056` and -`DS-0002`. Reports are written before findings are evaluated, so a warning or a -failure still publishes SARIF and artifacts. Introducing the tooling and settling -its findings are separate changes, in that order. +Two entry points share `tasks/scripts/trivy-scan.sh`: a standalone analysis +workflow and a pull-request change gate. Nix supplies Trivy, Helm and `yq`. + +### Standalone Scan + +`.github/workflows/trivy-scan.yml` runs only from `workflow_dispatch` or +`workflow_call` and takes OCI references as inputs, so it knows nothing about +how an artifact was assembled or published. Neither release workflow calls it +and no publication job depends on it; the intended consumer is a later +analysis-orchestration workflow. + +A single job, `OpenShell / Trivy (informational)`, always scans the deployment +configuration in its own checkout and adds OCI images and packaged charts when +the caller supplies references. Trivy defaults to the runner's platform, so each +platform of a multi-arch tag is scanned separately, and packaged charts need +`helm pull` because Trivy has no OCI artifact target. Findings only warn, while +a scanner that cannot run fails the job; `fail-on-findings` makes them fatal. + +The whole report directory uploads to Code Scanning in one operation, so each +report carries a unique `automationDetails.id`, and SARIF URIs — which Trivy +emits relative to the scanned target — are rewritten to repository-relative +paths. Exceptions live in `.trivyignore.yaml`, passed with `--ignorefile`, and +must use `**/` paths, the only shape both scan targets +report; `validate-ignore` checks that structurally with `yq` and `jq`. ### Pull-Request Change Gate -`.github/workflows/trivy-changes.yml` gates changes rather than releases. It -runs on `pull_request` and `merge_group`; `workflow_dispatch` takes explicit -base and head SHAs for diagnostics. A detection job decides whether the change -touches `deploy/docker/**`, `deploy/helm/**`, `deploy/kube/**`, or the scanner -inputs themselves (`.trivyignore.yaml`, `flake.nix`, `flake.lock`, -`tasks/scripts/trivy-scan.sh`, and the workflow file); the watched paths track -what the scan covers, so a change to the raw manifests cannot land unscanned. -Detection counts deletions and treats a failed diff as a -failure, so removing the scanner, a value fixture, or the ignore file cannot skip -the scan behind a passing status. - -When it does, the scan job checks out both the baseline and the candidate and -runs the candidate's `trivy-scan.sh config` over each tree with the candidate's -`.trivyignore.yaml`, so a scanner or ignore-policy change is judged by its own -rules on both sides. `gate-config-diff` then compares semantic finding -identities — rule ID, target, namespace, message, and cause -provider/service/resource — together with how many times each occurs. Line -numbers stay out of the identity so that edits which merely move a finding do not -look new, and the count stops a second offending block from hiding behind an -identity the baseline already reports: `KSV-0041` covers two rules of the -workspace-mode ClusterRole today, so a third fails. Counts are taken per report -and reduced with `max`, never summed, so a new value fixture rendering the same -templates adds no debt. The four findings above therefore keep reporting without -blocking every pull request, while a newly introduced `HIGH` or `CRITICAL` -misconfiguration fails the check. Both report sets are uploaded as workflow -artifacts. - -The `result` job publishes a stable `OpenShell / Trivy Changes` status that -succeeds when no relevant files changed, so the check can be required -unconditionally. - -This gate scans Helm and Dockerfile configuration only. It builds no image, so -it cannot detect OS or package CVEs in the image a change would produce. -Final-image vulnerability scanning stays with the release-artifact workflow -above. +`.github/workflows/trivy-changes.yml` runs on `pull_request` and `merge_group`; +`workflow_dispatch` takes explicit base and head SHAs. It scans the baseline and +candidate trees with the candidate's scanner and fails only on newly introduced +`HIGH` or `CRITICAL` misconfigurations, behind a stable +`OpenShell / Trivy Changes` status that succeeds when nothing relevant changed, +so the check can be required unconditionally. It builds no image, so image CVEs +need the standalone scan. Three invariants: + +- A structurally invalid Trivy report is an error, not an empty finding set. +- Findings compare per profile against the same baseline profile, by semantic + identity and count rather than line number; a profile absent from the baseline + falls back to that identity's maximum across all profiles. +- The candidate's ignore file is validated, but the baseline's policy applies to + both scans, so an exemption takes effect only after merge. See `CI.md` for the contributor workflow, labels, and maintainer merge-queue workflow. diff --git a/flake.nix b/flake.nix index 1302713450..b6cb70e387 100644 --- a/flake.nix +++ b/flake.nix @@ -61,6 +61,7 @@ syft trivy uv + yq-go zizmor zstd ]; diff --git a/tasks/scripts/trivy-scan-test.sh b/tasks/scripts/trivy-scan-test.sh new file mode 100755 index 0000000000..6cbd9765e5 --- /dev/null +++ b/tasks/scripts/trivy-scan-test.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SCANNER="${REPO_ROOT}/tasks/scripts/trivy-scan.sh" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +make_case() { + BASE="${TMP_DIR}/$1/base" + HEAD="${TMP_DIR}/$1/head" + mkdir -p "${BASE}" "${HEAD}" +} + +write_report() { + local path=$1 count=$2 + jq -n --argjson count "${count}" ' + { + SchemaVersion: 2, + ArtifactName: "deploy", + ArtifactType: "filesystem", + Results: (if $count == 0 then [] else [{ + Target: "helm/openshell/templates/clusterrole.yaml", + Misconfigurations: [ + range(0; $count) | { + ID: "KSV-0041", + Title: "Manage secrets", + Message: "Role permits management of secrets", + Namespace: "builtin.kubernetes.KSV041", + Severity: "HIGH", + CauseMetadata: {Provider: "Kubernetes", Service: "RBAC", Resource: "ClusterRole.openshell"} + } + ] + }] end) + } + ' >"${path}" +} + +expect_status() { + local expected=$1 description=$2 + shift 2 + + set +e + "$@" >/dev/null 2>&1 + local actual=$? + set -e + if [ "${actual}" -ne "${expected}" ]; then + echo "FAIL: ${description}: expected exit ${expected}, got ${actual}" >&2 + "$@" || true + exit 1 + fi +} + +make_case profile-expansion +write_report "${BASE}/config-defaults.json" 0 +write_report "${BASE}/config-fixture-workspace.json" 2 +write_report "${HEAD}/config-defaults.json" 1 +write_report "${HEAD}/config-fixture-workspace.json" 2 +expect_status 10 "finding newly exposed in an existing profile" \ + "${SCANNER}" gate-config-diff "${BASE}" "${HEAD}" + +make_case new-profile +write_report "${BASE}/config-defaults.json" 0 +write_report "${BASE}/config-fixture-workspace.json" 2 +write_report "${HEAD}/config-defaults.json" 0 +write_report "${HEAD}/config-fixture-workspace.json" 2 +write_report "${HEAD}/config-fixture-new.json" 2 +expect_status 0 "new profile repeating known findings" \ + "${SCANNER}" gate-config-diff "${BASE}" "${HEAD}" + +make_case malformed +write_report "${BASE}/config-defaults.json" 1 +printf '{}\n' >"${HEAD}/config-defaults.json" +expect_status 5 "structurally invalid candidate report" \ + "${SCANNER}" gate-config-diff "${BASE}" "${HEAD}" + +cat >"${TMP_DIR}/valid-ignore.yaml" <<'EOF' +misconfigurations: + - id: KSV-0041 + paths: + - "**/clusterrole.yaml" +EOF +expect_status 0 "concretely scoped ignore path" \ + env TRIVY_IGNORE_FILE="${TMP_DIR}/valid-ignore.yaml" \ + "${SCANNER}" validate-ignore + +cat >"${TMP_DIR}/broad-ignore.yaml" <<'EOF' +misconfigurations: +- id: KSV-0041 + paths: + - "**/*" +EOF +expect_status 2 "broad ignore path" \ + env TRIVY_IGNORE_FILE="${TMP_DIR}/broad-ignore.yaml" \ + "${SCANNER}" validate-ignore + +echo "Trivy scan tests passed." diff --git a/tasks/scripts/trivy-scan.sh b/tasks/scripts/trivy-scan.sh index 35edc41889..cdec0d7443 100755 --- a/tasks/scripts/trivy-scan.sh +++ b/tasks/scripts/trivy-scan.sh @@ -5,24 +5,8 @@ set -euo pipefail -# Scan release artifacts with Trivy. -# -# trivy-scan.sh config [--chart-ref ]... -# trivy-scan.sh images [...] -# trivy-scan.sh gate -# trivy-scan.sh gate-config-diff -# # `config` and `images` write full-severity reports and never fail on findings, -# so a report is always available to upload. `gate` then re-reads those reports -# and fails if any finding reaches TRIVY_SEVERITY. -# -# Environment: -# TRIVY_SEVERITY severities that fail `gate` (default HIGH,CRITICAL) -# TRIVY_IGNORE_UNFIXED skip image vulnerabilities with no fix (default true) -# TRIVY_PLATFORMS image platforms (default "linux/amd64 linux/arm64") -# TRIVY_REPORT_DIR output directory (default reports/trivy) -# TRIVY_SOURCE_ROOT source tree to scan (default repository root) -# TRIVY_IGNORE_FILE ignore file to apply (default repository copy) +# then `gate` applies TRIVY_SEVERITY (default HIGH,CRITICAL). REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" SOURCE_ROOT="${TRIVY_SOURCE_ROOT:-${REPO_ROOT}}" @@ -34,20 +18,50 @@ REPORT_DIR="${TRIVY_REPORT_DIR:-reports/trivy}" IGNORE_UNFIXED="${TRIVY_IGNORE_UNFIXED:-true}" PLATFORMS="${TRIVY_PLATFORMS:-linux/amd64 linux/arm64}" -# Rendering the chart outside a cluster cannot satisfy the Agent Sandbox API -# discovery check, and that template calls `fail`. +# Disable cluster discovery while rendering charts offline. PREFLIGHT_OFF=(--helm-set agentSandbox.preflight.enabled=false) -# These Dockerfiles produce no runnable image: the macOS ones export a binary -# from `FROM scratch`, and the CI image is toolchain, not a release artifact. +# These Dockerfiles do not produce release runtime images. SKIP_DOCKERFILES=( --skip-files 'deploy/docker/Dockerfile.ci' --skip-files 'deploy/docker/Dockerfile.*-macos' ) -# Run one scan. Reports keep every severity; `gate` applies the threshold. -# `prefix` is prepended to SARIF locations, which Trivy reports relative to the -# scanned target while Code Scanning resolves them from the repository root. +# Reject ignore entries broader than one concrete basename. +validate_ignore_file() { + [ -f "${IGNORE_FILE}" ] || { + echo "Error: Trivy ignore file not found: ${IGNORE_FILE}" >&2 + return 2 + } + + command -v yq >/dev/null || { + echo "Error: yq not on PATH; run inside 'nix develop'" >&2 + return 2 + } + if ! yq --output-format json '.' "${IGNORE_FILE}" | + jq -e ' + (.misconfigurations // []) as $entries + | (($entries | type) == "array") + and all($entries[]; + . as $entry + | (($entry.id | type) == "string") + and (($entry.paths | type) == "array") + and (($entry.paths | length) > 0) + and all($entry.paths[]; + . as $path + | (($path | type) == "string") + and ($path | startswith("**/")) + and (($path | ltrimstr("**/") | length) > 0) + and (($path | ltrimstr("**/") | test("[/*?\\[\\]]")) | not) + ) + ) + ' >/dev/null; then + echo "Error: every Trivy ignore must use at least one '**/' path" >&2 + return 2 + fi +} + +# Run one scan and normalize its SARIF metadata and repository path. scan() { local subcommand=$1 slug=$2 prefix=$3 shift 3 @@ -60,24 +74,23 @@ scan() { --format sarif --output "${REPORT_DIR}/${slug}.sarif" \ "${REPORT_DIR}/${slug}.json" - if [ -n "${prefix}" ]; then - jq --arg p "${prefix}" ' - (.. | objects | select(has("artifactLocation")) | .artifactLocation.uri) - |= $p + (. | sub("^[^:]*\\.tgz:"; "")) + jq --arg p "${prefix}" --arg automation_id "trivy/${slug}/" ' + .runs[] |= (.automationDetails.id = $automation_id) + | if $p == "" then + . + else + (.. | objects | select(has("artifactLocation")) | .artifactLocation.uri) + |= $p + (. | sub("^[^:]*\\.tgz:"; "")) + end ' "${REPORT_DIR}/${slug}.sarif" >"${REPORT_DIR}/${slug}.sarif.tmp" - mv "${REPORT_DIR}/${slug}.sarif.tmp" "${REPORT_DIR}/${slug}.sarif" - fi + mv "${REPORT_DIR}/${slug}.sarif.tmp" "${REPORT_DIR}/${slug}.sarif" } -# Scanning deploy/ in one pass covers both charts, the published Dockerfiles and -# the raw manifests, and keeps every reported path relative to the same root. +# Scan deploy/ defaults and conditional Helm fixtures. scan_config() { scan config config-defaults deploy/ "${PREFLIGHT_OFF[@]}" \ "${SKIP_DOCKERFILES[@]}" deploy - # The chart defaults render 10 of its 19 templates. The high-availability - # Deployment, the Gateway API objects, the OpenShift Route and the wider - # workspace-mode ClusterRole only render under CI value fixtures. local values fixture for values in deploy/helm/openshell/ci/values-*.yaml; do fixture="$(basename "${values}" .yaml | sed 's/^values-//')" @@ -86,9 +99,7 @@ scan_config() { done } -# Trivy has no OCI artifact target and rejects the Helm config media type, so a -# published chart has to be pulled before it can be scanned. It reads the -# archive directly, and skips secret scanning on packaged charts. +# Trivy needs a local chart archive rather than an OCI reference. scan_packaged_chart() { local ref=$1 if [[ "${ref}" != *:* || "${ref##*/}" != *:* ]]; then @@ -96,11 +107,6 @@ scan_packaged_chart() { exit 2 fi - # A published chart name is not its directory name — the gateway chart is - # `helm-chart` under deploy/helm/openshell — and both published charts share - # template filenames. Resolving the SARIF prefix and the report slug from the - # chart that declares the published name keeps `openshell-workspace` alerts off - # the gateway chart's `role.yaml` instead of silently reattributing them. local repo chart_name chart_dir="" candidate dir repo="${ref%:*}" chart_name="${repo##*/}" @@ -130,8 +136,6 @@ scan_images() { local image platform slug for image in "$@"; do - # Published tags are multi-arch indexes and Trivy defaults to the runner's - # own platform, so each architecture needs its own scan. for platform in ${PLATFORMS}; do slug="image-$(printf '%s' "${image#*/}-${platform}" | tr -cs 'A-Za-z0-9._-' '-')" scan image "${slug}" "" --platform "${platform}" --scanners vuln \ @@ -140,14 +144,9 @@ scan_images() { done } -# Re-read the reports and apply the threshold. The table doubles as the run -# summary, so nothing here reimplements counting. gate() { local report result findings=0 - # `find` rather than `compgen -G`: compgen belongs to bash's programmable - # completion, which the non-interactive bash in the Nix dev shell does not - # ship, so it fails with "command not found" there. if [ -z "$(find "${REPORT_DIR}" -maxdepth 1 -name '*.json' -print -quit)" ]; then echo "Error: no reports in ${REPORT_DIR}; run 'config' or 'images' first" >&2 exit 2 @@ -192,49 +191,58 @@ collect_config_findings() { return 2 fi - # One identity can cover several offending blocks: the key deliberately omits - # line numbers, so two rules in the same ClusterRole granting `secrets` are - # indistinguishable. Occurrences are therefore counted per report and reduced - # with `max`, never summed, because every value fixture scans the same tree and - # repeats its findings across reports while a template repeats them within one. - jq -s --arg severities "${SEVERITY}" ' - [ - .[] - | [ - .Results[]? as $result - | $result.Misconfigurations[]? - | .Severity as $severity - | select(($severities | split(",") | index($severity)) != null) - | { - key: ([ - .ID, - $result.Target, - (.Namespace // ""), - (.Message // ""), - (.CauseMetadata.Provider // ""), - (.CauseMetadata.Service // ""), - (.CauseMetadata.Resource // "") - ] | @json), - severity: .Severity, - id: .ID, - target: $result.Target, - title: .Title - } - ] - | group_by(.key) - | map(.[0] + { count: length }) - | .[] - ] - | group_by(.key) - | map(max_by(.count)) - ' "${report_dir}"/*.json + # Preserve report/profile identity while counting repeated findings. + local report profile + { + for report in "${report_dir}"/*.json; do + profile="$(basename "${report}" .json)" + jq --arg profile "${profile}" --arg severities "${SEVERITY}" ' + if .SchemaVersion != 2 + or ((.ArtifactName | type) != "string") + or ((.ArtifactType | type) != "string") + or (.Results != null and ((.Results | type) != "array")) + then + error("invalid Trivy JSON report: " + $profile) + else + { + profile: $profile, + findings: ([ + .Results[]? as $result + | $result.Misconfigurations[]? + | .Severity as $severity + | select(($severities | split(",") | index($severity)) != null) + | ([ + .ID, + $result.Target, + (.Namespace // ""), + (.Message // ""), + (.CauseMetadata.Provider // ""), + (.CauseMetadata.Service // ""), + (.CauseMetadata.Resource // "") + ] | @json) as $semantic_key + | { + key: ([$profile, $semantic_key] | @json), + semantic_key: $semantic_key, + profile: $profile, + severity: .Severity, + id: .ID, + target: $result.Target, + title: .Title + } + ] + | group_by(.key) + | map(.[0] + { count: length })) + } + end + ' "${report}" + done + } | jq -s '{ + profiles: map(.profile), + findings: (map(.findings) | add // []) + }' } -# Compare semantic finding identities instead of line numbers, so unrelated -# edits that move a finding do not make existing debt look newly introduced. -# Identities carry an occurrence count rather than mere presence, so adding a -# second offending block under an identity the baseline already reports still -# fails. +# Compare semantic identities and occurrence counts, excluding line numbers. gate_config_diff() ( set -euo pipefail @@ -249,10 +257,22 @@ gate_config_diff() ( collect_config_findings "${baseline_dir}" >"${baseline}" collect_config_findings "${candidate_dir}" >"${candidate}" jq --slurpfile baseline "${baseline}" ' - ($baseline[0] | map({ (.key): .count }) | add // {}) as $known + ($baseline[0].findings | map({ (.key): .count }) | add // {}) as $by_profile + | ($baseline[0].profiles) as $known_profiles + | ($baseline[0].findings + | group_by(.semantic_key) + | map({ + key: .[0].semantic_key, + value: (map(.count) | max) + }) + | from_entries) as $across_profiles | [ - .[] - | (($known[.key]) // 0) as $before + .findings[] + | . as $finding + | (if ($known_profiles | index($finding.profile)) != null + then (($by_profile[$finding.key]) // 0) + else (($across_profiles[$finding.semantic_key]) // 0) + end) as $before | select(.count > $before) | . + { baseline_count: $before, new_count: (.count - $before) } ] @@ -270,7 +290,7 @@ gate_config_diff() ( echo "::error::Trivy reported ${finding_count} new configuration finding(s) at ${SEVERITY}." jq -r '.[] - | "::error::[\(.severity)] \(.id) in deploy/\(.target): \(.title)" + | "::error::[\(.severity)] \(.id) in \(.profile) (deploy/\(.target)): \(.title)" + " (\(.new_count) new, \(.baseline_count) in baseline)"' \ "${new_findings}" if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then @@ -278,7 +298,8 @@ gate_config_diff() ( echo "### New Trivy configuration findings" echo jq -r '.[] - | "- **\(.severity)** `\(.id)` in `deploy/\(.target)`: \(.title)" + | "- **\(.severity)** `\(.id)` in `\(.profile)`" + + " (`deploy/\(.target)`): \(.title)" + " (\(.new_count) new, \(.baseline_count) in baseline)"' \ "${new_findings}" } >>"${GITHUB_STEP_SUMMARY}" @@ -286,16 +307,21 @@ gate_config_diff() ( exit 10 ) -command -v trivy >/dev/null || { echo "Error: trivy not on PATH; run inside 'nix develop'" >&2; exit 2; } +require_trivy() { + command -v trivy >/dev/null || { + echo "Error: trivy not on PATH; run inside 'nix develop'" >&2 + exit 2 + } +} case "${1:-}" in config) shift + require_trivy + validate_ignore_file mkdir -p "${REPORT_DIR}" scan_config - # A release publishes every chart under deploy/helm, so `--chart-ref` - # repeats. Unparsed arguments are rejected rather than ignored: a misspelled - # flag would otherwise leave the packaged charts unscanned and still exit 0. + # Reject unparsed arguments so requested charts cannot be silently skipped. while [ "${1:-}" = "--chart-ref" ]; do [ -n "${2:-}" ] || { echo "Error: --chart-ref needs a value" >&2; exit 2; } scan_packaged_chart "$2" @@ -306,10 +332,13 @@ case "${1:-}" in images) shift [ $# -gt 0 ] || { echo "Error: images needs at least one reference" >&2; exit 2; } + require_trivy + validate_ignore_file mkdir -p "${REPORT_DIR}" scan_images "$@" ;; gate) + require_trivy gate ;; gate-config-diff) @@ -320,6 +349,9 @@ case "${1:-}" in } gate_config_diff "$1" "$2" ;; + validate-ignore) + validate_ignore_file + ;; *) cat >&2 <<'USAGE' Usage: @@ -327,6 +359,7 @@ Usage: trivy-scan.sh images [...] trivy-scan.sh gate trivy-scan.sh gate-config-diff + trivy-scan.sh validate-ignore USAGE exit 2 ;; From bd7b1cdc707c0103f7de1d271f542eba52ccdb9b Mon Sep 17 00:00:00 2001 From: Adrien Langou Date: Wed, 9 Sep 2026 15:29:07 +0200 Subject: [PATCH 5/5] fix(ci): consolidate Trivy reports and prevent collisions Signed-off-by: Adrien Langou --- .github/workflows/trivy-changes.yml | 2 + .github/workflows/trivy-scan.yml | 53 ++++++++- CI.md | 40 +++++-- architecture/build.md | 47 +++----- tasks/scripts/trivy-config-report.jq | 48 ++++++++ tasks/scripts/trivy-scan-test.sh | 156 +++++++++++++++++++++++- tasks/scripts/trivy-scan.sh | 171 ++++++++++++++++++++------- 7 files changed, 422 insertions(+), 95 deletions(-) create mode 100644 tasks/scripts/trivy-config-report.jq diff --git a/.github/workflows/trivy-changes.yml b/.github/workflows/trivy-changes.yml index 642ca99757..612ce891cc 100644 --- a/.github/workflows/trivy-changes.yml +++ b/.github/workflows/trivy-changes.yml @@ -62,6 +62,8 @@ jobs: flake.lock tasks/scripts/trivy-scan.sh tasks/scripts/trivy-scan-test.sh + tasks/scripts/trivy-config-report.jq + .github/workflows/trivy-scan.yml .github/workflows/trivy-changes.yml scan: diff --git a/.github/workflows/trivy-scan.yml b/.github/workflows/trivy-scan.yml index 5722d079b7..ccfe80bd69 100644 --- a/.github/workflows/trivy-scan.yml +++ b/.github/workflows/trivy-scan.yml @@ -80,10 +80,12 @@ jobs: name: OpenShell / Trivy (informational) runs-on: ubuntu-latest timeout-minutes: 60 + outputs: + sarif-batches: ${{ steps.sarif.outputs.batches }} + artifact-id: ${{ steps.reports.outputs.artifact-id }} permissions: contents: read packages: read - security-events: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -100,6 +102,9 @@ jobs: with: cachix-auth-token: ${{ github.event_name != 'pull_request' && secrets.CACHIX_AUTH_TOKEN || '' }} + - name: Test scan reporting + run: tasks/scripts/trivy-scan-test.sh + - name: Scan configuration id: config env: @@ -128,7 +133,8 @@ jobs: tasks/scripts/trivy-scan.sh images "${refs[@]}" fi - - name: Upload SARIF to Code Scanning + - name: Prepare consolidated SARIF uploads + id: sarif if: >- ${{ !cancelled() @@ -136,11 +142,10 @@ jobs: && steps.images.conclusion == 'success' && inputs.upload-sarif }} - uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 - with: - sarif_file: reports/trivy + run: tasks/scripts/trivy-scan.sh prepare-sarif - name: Upload reports + id: reports if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -174,3 +179,41 @@ jobs: ;; *) echo "::error::Trivy could not evaluate the reports (exit $status)."; exit "$status" ;; esac + + upload-sarif: + name: Publish Trivy SARIF (${{ matrix.batch }}) + needs: scan + # Findings may fail the scan job after reports are prepared. Publish those + # complete results too, but never publish an incomplete/failed scan. + if: >- + ${{ + !cancelled() + && inputs.upload-sarif + && needs.scan.outputs.sarif-batches != '' + && needs.scan.outputs.artifact-id != '' + }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + batch: ${{ fromJSON(needs.scan.outputs.sarif-batches) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + artifact-ids: ${{ needs.scan.outputs.artifact-id }} + merge-multiple: true + path: reports/trivy + + # One configuration run, plus one run per image/platform or packaged + # chart. Each batch holds at most GitHub's limit of 20 SARIF runs. + - uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7 + with: + sarif_file: reports/trivy/code-scanning/uploads/${{ matrix.batch }} diff --git a/CI.md b/CI.md index b0596f72be..a84bd90638 100644 --- a/CI.md +++ b/CI.md @@ -88,12 +88,28 @@ always scans deployment configuration and optionally scans supplied OCI image and chart references. It is not wired into a release workflow; a future analysis orchestrator can call it directly. -The single job scans `deploy/`, every Helm CI values fixture, requested packaged -charts, and both Linux architectures of each requested image. Findings are -informational by default, but scanner failures still fail the job and -`fail-on-findings` enables enforcement. Reports retain every severity; the gate -uses `HIGH,CRITICAL` by default. Each SARIF report has a unique automation ID so -the report directory can be uploaded in one operation. +The scan job checks static deployment files (including Dockerfiles) once, both +local charts with default values, the OpenShell `HELM_PROFILES` selected in +`tasks/scripts/trivy-scan.sh`, requested packaged charts, and both Linux +architectures of each requested image. The profile list includes development +and E2E overlays for regression coverage, but excludes `values-spire-stack.yaml`, +which belongs to the external SPIRE chart. A new OpenShell values fixture needs +an explicit entry in this list to receive Trivy coverage. + +Findings are informational by default, but scanner failures still fail the job +and `fail-on-findings` enables enforcement. Reports retain every severity; the +gate uses `HIGH,CRITICAL` by default. + +All detailed JSON reports are retained in one artifact. The workflow summary and +Code Scanning configuration report deduplicate the same rule, target, resource, +and message across profiles, listing the affected profiles in each finding. +Images and packaged charts retain separate analyses, keyed by full reference +and platform so different registries or versions cannot overwrite each other. +SARIF is generated only for publication. The upload job publishes +only this consolidated configuration SARIF and the artifact analyses, in batches +of at most 20 runs to respect GitHub's per-file limit. It also publishes complete +reports when `fail-on-findings` makes the scan job fail; incomplete scans are +retained as artifacts but are not published to Code Scanning. `.trivyignore.yaml` is reserved for false positives. Every entry must use at least one `**/` path; `yq` and `jq` validate this structure @@ -105,8 +121,12 @@ Run the scanner locally with: nix develop --command tasks/scripts/trivy-scan.sh config nix develop --command tasks/scripts/trivy-scan.sh images ghcr.io/nvidia/openshell/gateway:dev nix develop --command tasks/scripts/trivy-scan.sh gate +nix develop --command tasks/scripts/trivy-scan.sh prepare-sarif ``` +Use a fresh `TRIVY_REPORT_DIR` for each scan session. `prepare-sarif` creates +`code-scanning/uploads//`; only these directories are intended for upload. + ### Pull-request change gate `Trivy Changes` scans the base and candidate when a pull request or merge group @@ -115,9 +135,11 @@ changes deployment configuration or scanner inputs. It fails only for new The candidate ignore file is validated, but the baseline policy applies to both scans so a change cannot exempt its own finding. Existing profiles are compared -independently; a new profile reuses the maximum known occurrence count. Invalid -Trivy reports fail closed. Image package and operating-system CVEs remain the -responsibility of the standalone scan. +independently using the detailed reports, before any presentation deduplication; +a new profile reuses the maximum known occurrence count. A selected fixture +missing from the candidate fails the scan; a new fixture absent from the base +has no baseline profile. Invalid Trivy reports fail closed. Image package and +operating-system CVEs remain the responsibility of the standalone scan. ## Commit signing diff --git a/architecture/build.md b/architecture/build.md index a7c31562d7..3cc5102f4d 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -444,46 +444,29 @@ implemented yet. Two entry points share `tasks/scripts/trivy-scan.sh`: a standalone analysis workflow and a pull-request change gate. Nix supplies Trivy, Helm and `yq`. -### Standalone Scan - -`.github/workflows/trivy-scan.yml` runs only from `workflow_dispatch` or -`workflow_call` and takes OCI references as inputs, so it knows nothing about -how an artifact was assembled or published. Neither release workflow calls it -and no publication job depends on it; the intended consumer is a later -analysis-orchestration workflow. - -A single job, `OpenShell / Trivy (informational)`, always scans the deployment -configuration in its own checkout and adds OCI images and packaged charts when -the caller supplies references. Trivy defaults to the runner's platform, so each -platform of a multi-arch tag is scanned separately, and packaged charts need -`helm pull` because Trivy has no OCI artifact target. Findings only warn, while -a scanner that cannot run fails the job; `fail-on-findings` makes them fatal. - -The whole report directory uploads to Code Scanning in one operation, so each -report carries a unique `automationDetails.id`, and SARIF URIs — which Trivy -emits relative to the scanned target — are rewritten to repository-relative -paths. Exceptions live in `.trivyignore.yaml`, passed with `--ignorefile`, and -must use `**/` paths, the only shape both scan targets -report; `validate-ignore` checks that structurally with `yq` and `jq`. - -### Pull-Request Change Gate - -`.github/workflows/trivy-changes.yml` runs on `pull_request` and `merge_group`; -`workflow_dispatch` takes explicit base and head SHAs. It scans the baseline and -candidate trees with the candidate's scanner and fails only on newly introduced -`HIGH` or `CRITICAL` misconfigurations, behind a stable -`OpenShell / Trivy Changes` status that succeeds when nothing relevant changed, -so the check can be required unconditionally. It builds no image, so image CVEs -need the standalone scan. Three invariants: +The standalone workflow scans deployment configuration and supplied OCI +references independently of release publication. Detailed JSON reports feed the +differential gate; the summary and published SARIF consolidate configuration +findings across profiles while preserving resource identity and affected profiles. +Images and packaged charts retain separate identities based on their full +references. Publication batches respect GitHub's limit of 20 SARIF runs. + +The PR/merge-group gate scans base and candidate with the same scanner and rejects +new `HIGH` or `CRITICAL` configuration findings. Its stable +`OpenShell / Trivy Changes` status succeeds when nothing relevant changed. +Image CVEs need the standalone scan. The reporting and gate invariants are: - A structurally invalid Trivy report is an error, not an empty finding set. +- Scanner failures prevent publication of incomplete analyses; findings alone + do not prevent publishing complete reports. - Findings compare per profile against the same baseline profile, by semantic identity and count rather than line number; a profile absent from the baseline falls back to that identity's maximum across all profiles. - The candidate's ignore file is validated, but the baseline's policy applies to both scans, so an exemption takes effect only after merge. -See `CI.md` for the contributor workflow, labels, and maintainer merge-queue workflow. +See [CI.md](../CI.md#artifact-scanning) for profiles, report paths, severity +settings, exceptions, and the contributor and maintainer workflows. ## Docs Site diff --git a/tasks/scripts/trivy-config-report.jq b/tasks/scripts/trivy-config-report.jq new file mode 100644 index 0000000000..c5c5dc613b --- /dev/null +++ b/tasks/scripts/trivy-config-report.jq @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Inputs are raw, repository-relative Trivy config reports. Consolidation is +# presentation only: the gate continues comparing unmodified reports per profile. +if length == 0 or any(.[]; + .SchemaVersion != 2 + or (.ArtifactName | type) != "string" + or (.ArtifactType | type) != "string" + or (.TrivyProfile | type) != "string" + or (.Results != null and (.Results | type) != "array") +) then error("invalid Trivy configuration reports") else . end +| .[0] as $template +| [ + .[] as $report + | $report.Results[]? + | (.Misconfigurations[]?.TrivyProfile) = $report.TrivyProfile + ] as $results +| $template +| .ArtifactName = "deployment configuration (all profiles)" +| del(.TrivyProfile) +| .Results = ( + $results + | group_by([.Target, .Class, .Type]) + | map( + . as $targets + | .[0] + | .Misconfigurations = ( + [$targets[] | .Misconfigurations[]?] + | group_by([ + .ID, .Namespace, .Severity, .Message, + .CauseMetadata.Provider, .CauseMetadata.Service, + .CauseMetadata.Resource + ]) + | map( + . as $findings + | .[0] + | .Message += ("\nProfiles: " + ($findings | map(.TrivyProfile) | unique | join(", "))) + | del(.TrivyProfile) + ) + ) + | .MisconfSummary = { + Successes: 0, + Failures: (.Misconfigurations | length), + Exceptions: 0 + } + ) + ) diff --git a/tasks/scripts/trivy-scan-test.sh b/tasks/scripts/trivy-scan-test.sh index 6cbd9765e5..8ce49e544b 100755 --- a/tasks/scripts/trivy-scan-test.sh +++ b/tasks/scripts/trivy-scan-test.sh @@ -9,6 +9,9 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" SCANNER="${REPO_ROOT}/tasks/scripts/trivy-scan.sh" TMP_DIR="$(mktemp -d)" trap 'rm -rf "${TMP_DIR}"' EXIT +# Synthetic findings must not write to the real Actions summary or outputs. +export GITHUB_OUTPUT="${TMP_DIR}/github-output" +export GITHUB_STEP_SUMMARY="${TMP_DIR}/github-summary" make_case() { BASE="${TMP_DIR}/$1/base" @@ -17,20 +20,27 @@ make_case() { } write_report() { - local path=$1 count=$2 - jq -n --argjson count "${count}" ' + local path=$1 count=$2 profile + profile="$(basename "${path}" .json)" + jq -n --argjson count "${count}" --arg profile "${profile}" ' { SchemaVersion: 2, ArtifactName: "deploy", ArtifactType: "filesystem", + TrivyProfile: $profile, Results: (if $count == 0 then [] else [{ - Target: "helm/openshell/templates/clusterrole.yaml", + Target: "deploy/helm/openshell/templates/clusterrole.yaml", + Class: "config", + Type: "kubernetes", + MisconfSummary: {Successes: 0, Failures: $count, Exceptions: 0}, Misconfigurations: [ range(0; $count) | { ID: "KSV-0041", Title: "Manage secrets", Message: "Role permits management of secrets", Namespace: "builtin.kubernetes.KSV041", + Type: "Kubernetes Security Check", + Status: "FAIL", Severity: "HIGH", CauseMetadata: {Provider: "Kubernetes", Service: "RBAC", Resource: "ClusterRole.openshell"} } @@ -45,12 +55,12 @@ expect_status() { shift 2 set +e - "$@" >/dev/null 2>&1 + "$@" >"${TMP_DIR}/last-command.log" 2>&1 local actual=$? set -e if [ "${actual}" -ne "${expected}" ]; then echo "FAIL: ${description}: expected exit ${expected}, got ${actual}" >&2 - "$@" || true + cat "${TMP_DIR}/last-command.log" >&2 exit 1 fi } @@ -72,12 +82,24 @@ write_report "${HEAD}/config-fixture-new.json" 2 expect_status 0 "new profile repeating known findings" \ "${SCANNER}" gate-config-diff "${BASE}" "${HEAD}" +make_case occurrence-increase +write_report "${BASE}/config-defaults.json" 1 +write_report "${HEAD}/config-defaults.json" 2 +expect_status 10 "additional occurrence of an existing finding" \ + "${SCANNER}" gate-config-diff "${BASE}" "${HEAD}" +expect_status 0 "findings below the requested threshold" \ + env TRIVY_SEVERITY=CRITICAL "${SCANNER}" gate-config-diff "${BASE}" "${HEAD}" + make_case malformed write_report "${BASE}/config-defaults.json" 1 printf '{}\n' >"${HEAD}/config-defaults.json" expect_status 5 "structurally invalid candidate report" \ "${SCANNER}" gate-config-diff "${BASE}" "${HEAD}" +make_case no-reports +expect_status 2 "missing reports cannot pass the differential gate" \ + "${SCANNER}" gate-config-diff "${BASE}" "${HEAD}" + cat >"${TMP_DIR}/valid-ignore.yaml" <<'EOF' misconfigurations: - id: KSV-0041 @@ -98,4 +120,128 @@ expect_status 2 "broad ignore path" \ env TRIVY_IGNORE_FILE="${TMP_DIR}/broad-ignore.yaml" \ "${SCANNER}" validate-ignore +# Consolidate more than 20 Helm profiles without losing resource identity or +# the per-profile input counts needed by the gate. Exercise real Trivy SARIF +# conversion with synthetic JSON; this requires no network or image downloads. +make_case sarif +write_report "${HEAD}/config-static.json" 0 +write_report "${HEAD}/config-defaults.json" 1 +for profile in {1..21}; do + write_report "${HEAD}/config-fixture-${profile}.json" 1 +done +write_report "${TMP_DIR}/resource-source.json" 2 +jq ' + .TrivyProfile = "config-fixture-resources" + | .Results[0].Misconfigurations[0].CauseMetadata.StartLine = 40 + | .Results[0].Misconfigurations[1].CauseMetadata.Resource = "ClusterRole.other" +' "${TMP_DIR}/resource-source.json" >"${HEAD}/config-fixture-resources.json" + +# Image reports must remain separate, even if they contain identical findings. +for report in {1..25}; do + write_report "${HEAD}/image-${report}.json" 0 +done +expect_status 0 "prepare one config analysis and bounded image upload batches" \ + env TRIVY_SEVERITY=CRITICAL TRIVY_REPORT_DIR="${HEAD}" GITHUB_OUTPUT="${TMP_DIR}/outputs" \ + "${SCANNER}" prepare-sarif +jq -e ' + [.Results[].Misconfigurations[]] as $findings + | ($findings | length) == 2 + and all($findings[]; .Message | contains("Profiles: ")) + and any($findings[]; + .CauseMetadata.Resource == "ClusterRole.openshell" + and (.Message | contains("config-defaults")) + and (.Message | contains("config-fixture-resources"))) +' "${HEAD}/consolidated/config.json" >/dev/null +jq -e '.Results[0].Misconfigurations | length == 2' \ + "${HEAD}/config-fixture-resources.json" >/dev/null +jq -e ' + (.runs | length) == 1 + and .runs[0].automationDetails.id == "trivy/config/" + and (.runs[0].results | length) == 2 + and (.runs[0].originalUriBaseIds == null) + and (.runs[0] as $run | all($run.results[]; + .ruleId == $run.tool.driver.rules[.ruleIndex].id + and all(.locations[].physicalLocation.artifactLocation; + .uri == "deploy/helm/openshell/templates/clusterrole.yaml" and .uriBaseId == null))) +' "${HEAD}/code-scanning/uploads/0/config.sarif" >/dev/null +jq -se '[.[].runs[]] | length == 20' "${HEAD}/code-scanning/uploads/0/"*.sarif >/dev/null +jq -se '[.[].runs[]] | length == 6' "${HEAD}/code-scanning/uploads/1/"*.sarif >/dev/null +[[ "$(<"${TMP_DIR}/outputs")" == 'batches=["0","1"]' ]] + +# Report findings and the workflow summary use the same consolidated view. +expect_status 10 "consolidated findings remain blocking" \ + env TRIVY_REPORT_DIR="${HEAD}" GITHUB_STEP_SUMMARY="${TMP_DIR}/summary" \ + "${SCANNER}" gate +test -s "${TMP_DIR}/summary" + +# Empty configuration is a valid clearing analysis, not a skipped upload. +make_case empty-sarif +write_report "${HEAD}/config-static.json" 0 +write_report "${HEAD}/config-defaults.json" 0 +expect_status 0 "empty configuration SARIF" \ + env TRIVY_REPORT_DIR="${HEAD}" "${SCANNER}" prepare-sarif +jq -e '.runs[0].results | length == 0' \ + "${HEAD}/code-scanning/uploads/0/config.sarif" >/dev/null + +make_case malformed-sarif +write_report "${HEAD}/config-static.json" 0 +printf '{}\n' >"${HEAD}/config-defaults.json" +expect_status 5 "reject malformed input during consolidation" \ + env TRIVY_REPORT_DIR="${HEAD}" "${SCANNER}" prepare-sarif + +# Record scan arguments without downloading Trivy's checks database or charts. +# shellcheck disable=SC2329 # Invoked by the scanner subprocess via export -f. +trivy() { + [ "${TRIVY_TEST_FAIL:-false}" = false ] || return 9 + local output="" previous="" arg + printf '%s\n' "$@" | jq -Rs 'split("\n")[:-1]' >>"${TRIVY_TEST_CALLS}" + for arg in "$@"; do + [ "${previous}" = --output ] && output="${arg}" + previous="${arg}" + done + jq -n --arg target "${!#}" \ + '{SchemaVersion: 2, ArtifactName: $target, ArtifactType: "filesystem", Results: []}' >"${output}" +} +# shellcheck disable=SC2329 # Invoked by the scanner subprocess via export -f. +helm() { touch "${!#}/chart.tgz"; } +export -f trivy helm +export TRIVY_TEST_CALLS="${TMP_DIR}/scan-calls" +expect_status 0 "scan selected profiles and retain distinct packaged chart versions/registries" \ + env TRIVY_REPORT_DIR="${TMP_DIR}/scan-reports" "${SCANNER}" config \ + --chart-ref oci://registry-a.example/charts/helm-chart:1.0.0 \ + --chart-ref oci://registry-a.example/charts/helm-chart:1.1.0 \ + --chart-ref oci://registry-b.example/charts/helm-chart:1.0.0 +jq -se ' + length == 21 + and ([.[] | select(.[-1] == "deploy")] | length) == 1 + and ([.[] | select(.[-1] == "deploy/helm")] | length) == 1 + and ([.[] | select(.[-1] == "deploy/helm/openshell")] | length) == 16 + and all(.[]; (join(" ") | contains("values-spire-stack.yaml")) | not) + and all(.[]; index("UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL") != null) +' "${TRIVY_TEST_CALLS}" >/dev/null +jq -se 'length == 3' "${TMP_DIR}/scan-reports/"config-packaged-*.json >/dev/null +test -z "$(find "${TMP_DIR}/scan-reports" -name '*.sarif' -print -quit)" + +expect_status 0 "retain image registry, path and platform identity" \ + env TRIVY_REPORT_DIR="${TMP_DIR}/images" "${SCANNER}" images \ + registry-a.example/ns/image:dev registry-b.example/ns/image:dev \ + registry-a.example/ns-image:dev +jq -se 'length == 6 and (map(.ArtifactName) | unique | length) == 3' \ + "${TMP_DIR}/images/"*.json >/dev/null + +expect_status 9 "scanner failures propagate" \ + env TRIVY_TEST_FAIL=true TRIVY_REPORT_DIR="${TMP_DIR}/scan-error" "${SCANNER}" config + +# Check missing fixture handling without changing the working checkout. +mkdir -p "${TMP_DIR}/candidate/tasks/scripts" "${TMP_DIR}/baseline" +cp "${SCANNER}" "${TMP_DIR}/candidate/tasks/scripts/trivy-scan.sh" +cp "${REPO_ROOT}/.trivyignore.yaml" "${TMP_DIR}/candidate/" +expect_status 2 "a selected candidate fixture cannot silently disappear" \ + env TRIVY_REPORT_DIR="${TMP_DIR}/missing-candidate" \ + "${TMP_DIR}/candidate/tasks/scripts/trivy-scan.sh" config +expect_status 0 "new candidate profiles may be absent from the baseline" \ + env TRIVY_SOURCE_ROOT="${TMP_DIR}/baseline" TRIVY_REPORT_DIR="${TMP_DIR}/missing-baseline" \ + "${TMP_DIR}/candidate/tasks/scripts/trivy-scan.sh" config +unset -f trivy helm + echo "Trivy scan tests passed." diff --git a/tasks/scripts/trivy-scan.sh b/tasks/scripts/trivy-scan.sh index cdec0d7443..3dd58ebdce 100755 --- a/tasks/scripts/trivy-scan.sh +++ b/tasks/scripts/trivy-scan.sh @@ -27,6 +27,15 @@ SKIP_DOCKERFILES=( --skip-files 'deploy/docker/Dockerfile.*-macos' ) +# Explicit OpenShell variants, including dev/E2E regression coverage. +# spire-stack belongs to the external SPIRE chart and is intentionally absent. +HELM_PROFILES=( + cert-manager credential-driver-kubernetes-secrets credential-driver-vault + gateway gateway-tls high-availability openshift-route-cert-manager spire + tls-disabled workspace-managed workspace-operator + corporate-proxy-e2e keycloak sidecar sidecar-kata skaffold +) + # Reject ignore entries broader than one concrete basename. validate_ignore_file() { [ -f "${IGNORE_FILE}" ] || { @@ -61,7 +70,23 @@ validate_ignore_file() { fi } -# Run one scan and normalize its SARIF metadata and repository path. +# Convert a report whose targets already use repository-relative paths. +convert_sarif() { + local report=$1 output=$2 category=$3 + trivy convert --quiet \ + --severity UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL --exit-code 0 \ + --format sarif --output "${output}" "${report}" + # `convert` can set ROOTPATH to the input JSON file. Our targets are relative + # to the checkout, not to the report file or the original scan directory. + jq --arg automation_id "trivy/${category}/" ' + .runs[] |= (.automationDetails.id = $automation_id) + | del(.runs[].originalUriBaseIds) + | (.. | objects | select(has("artifactLocation")) | .artifactLocation) + |= del(.uriBaseId) + ' "${output}" >"${output}.tmp" + mv "${output}.tmp" "${output}" +} + scan() { local subcommand=$1 slug=$2 prefix=$3 shift 3 @@ -69,36 +94,45 @@ scan() { echo "==> ${slug}" trivy "${subcommand}" --skip-version-check --quiet \ --ignorefile "${IGNORE_FILE}" \ + --severity UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL --exit-code 0 \ --format json --output "${REPORT_DIR}/${slug}.json" "$@" - trivy convert --quiet \ - --format sarif --output "${REPORT_DIR}/${slug}.sarif" \ - "${REPORT_DIR}/${slug}.json" - - jq --arg p "${prefix}" --arg automation_id "trivy/${slug}/" ' - .runs[] |= (.automationDetails.id = $automation_id) - | if $p == "" then - . - else - (.. | objects | select(has("artifactLocation")) | .artifactLocation.uri) - |= $p + (. | sub("^[^:]*\\.tgz:"; "")) - end - ' "${REPORT_DIR}/${slug}.sarif" >"${REPORT_DIR}/${slug}.sarif.tmp" - mv "${REPORT_DIR}/${slug}.sarif.tmp" "${REPORT_DIR}/${slug}.sarif" + if [ -n "${prefix}" ]; then + jq --arg prefix "${prefix}" --arg profile "${slug}" ' + .TrivyProfile = $profile + | (.Results[]?.Target) |= $prefix + sub("^[^:]*\\.tgz:"; "") + ' "${REPORT_DIR}/${slug}.json" >"${REPORT_DIR}/${slug}.json.tmp" + mv "${REPORT_DIR}/${slug}.json.tmp" "${REPORT_DIR}/${slug}.json" + fi } -# Scan deploy/ defaults and conditional Helm fixtures. +# Scan static deployment files once, then charts with their applicable values. scan_config() { - scan config config-defaults deploy/ "${PREFLIGHT_OFF[@]}" \ - "${SKIP_DOCKERFILES[@]}" deploy + scan config config-static deploy/ "${SKIP_DOCKERFILES[@]}" \ + --skip-dirs deploy/helm deploy + scan config config-defaults deploy/helm/ "${PREFLIGHT_OFF[@]}" deploy/helm local values fixture - for values in deploy/helm/openshell/ci/values-*.yaml; do - fixture="$(basename "${values}" .yaml | sed 's/^values-//')" - scan config "config-fixture-${fixture}" deploy/ "${PREFLIGHT_OFF[@]}" \ - "${SKIP_DOCKERFILES[@]}" --helm-values "${values}" deploy + for fixture in "${HELM_PROFILES[@]}"; do + values="deploy/helm/openshell/ci/values-${fixture}.yaml" + if [ ! -f "${values}" ]; then + # A newly added profile has no baseline report yet. A missing candidate + # fixture is an error: intentional removal must also update HELM_PROFILES. + [ "${SOURCE_ROOT}" != "${REPO_ROOT}" ] && continue + echo "Error: Helm profile not found: ${values}" >&2 + return 2 + fi + scan config "config-fixture-${fixture}" deploy/helm/openshell/ \ + "${PREFLIGHT_OFF[@]}" --helm-values "${values}" deploy/helm/openshell done } +# A readable label plus the full reference hash avoids registry/tag collisions. +artifact_slug() { + local digest + digest="$(printf '%s' "$1" | sha256sum)" + printf '%s-%s' "$(printf '%s' "${1##*/}" | tr -cs 'A-Za-z0-9._-' '-' | cut -c1-80)" "${digest%% *}" +} + # Trivy needs a local chart archive rather than an OCI reference. scan_packaged_chart() { local ref=$1 @@ -126,7 +160,7 @@ scan_packaged_chart() { trap 'rm -rf "${dir}"' RETURN helm pull "${repo}" --version "${ref##*:}" --destination "${dir}" - scan config "config-packaged-${chart_name}" "${chart_dir}" \ + scan config "config-packaged-$(artifact_slug "${ref}")" "${chart_dir}" \ "${PREFLIGHT_OFF[@]}" "$(find "${dir}" -name '*.tgz' -print -quit)" } @@ -137,7 +171,7 @@ scan_images() { local image platform slug for image in "$@"; do for platform in ${PLATFORMS}; do - slug="image-$(printf '%s' "${image#*/}-${platform}" | tr -cs 'A-Za-z0-9._-' '-')" + slug="image-$(artifact_slug "${image}")-${platform//\//-}" scan image "${slug}" "" --platform "${platform}" --scanners vuln \ "${extra[@]}" "${image}" done @@ -152,7 +186,17 @@ gate() { exit 2 fi - for report in "${REPORT_DIR}"/*.json; do + local reports=() + if [ -f "${REPORT_DIR}/config-defaults.json" ]; then + consolidate_config + reports+=("${REPORT_DIR}/consolidated/config.json") + fi + for report in "${REPORT_DIR}"/image-*.json "${REPORT_DIR}"/config-packaged-*.json; do + [ -f "${report}" ] && reports+=("${report}") + done + [ "${#reports[@]}" -gt 0 ] || { echo "Error: no complete scan reports" >&2; return 2; } + + for report in "${reports[@]}"; do set +e trivy convert --quiet --exit-code 10 --severity "${SEVERITY}" \ --format table "${report}" @@ -173,7 +217,7 @@ gate() { { echo "### Trivy (gate: \`${SEVERITY}\`)" echo '```' - for report in "${REPORT_DIR}"/*.json; do + for report in "${reports[@]}"; do trivy convert --quiet --severity "${SEVERITY}" --format table "${report}" done echo '```' @@ -183,21 +227,56 @@ gate() { [ "${findings}" -eq 0 ] || return 10 } -collect_config_findings() { - local report_dir=$1 +consolidate_config() { + local report + local reports=("${REPORT_DIR}/config-static.json" "${REPORT_DIR}/config-defaults.json") + for report in "${REPORT_DIR}"/config-fixture-*.json; do + [ -f "${report}" ] && reports+=("${report}") + done + mkdir -p "${REPORT_DIR}/consolidated" + jq -s -f "${REPO_ROOT}/tasks/scripts/trivy-config-report.jq" \ + "${reports[@]}" >"${REPORT_DIR}/consolidated/config.json" +} - if [ -z "$(find "${report_dir}" -maxdepth 1 -name '*.json' -print -quit)" ]; then - echo "Error: no reports in ${report_dir}" >&2 +# Keep detailed reports for the differential gate, but publish one deduplicated +# configuration analysis. Images and packaged charts retain separate identities. +prepare_sarif() { + local staging report slug batch=0 count=0 + staging="${REPORT_DIR}/code-scanning" + if [ -e "${staging}" ]; then + echo "Error: ${staging} already exists; use a fresh report directory" >&2 return 2 fi + mkdir -p "${staging}/uploads/0" + consolidate_config + convert_sarif "${REPORT_DIR}/consolidated/config.json" "${staging}/uploads/0/config.sarif" config + count=1 + for report in "${REPORT_DIR}"/image-*.json "${REPORT_DIR}"/config-packaged-*.json; do + [ -f "${report}" ] || continue + # upload-sarif combines a directory into one file; GitHub accepts at most + # 20 runs per file. Every report produced here contains exactly one run. + if [ "${count}" -eq 20 ]; then + batch=$((batch + 1)) + count=0 + mkdir -p "${staging}/uploads/${batch}" + fi + slug="$(basename "${report}" .json)" + convert_sarif "${report}" "${staging}/uploads/${batch}/${slug}.sarif" "${slug}" + count=$((count + 1)) + done + if [ -n "${GITHUB_OUTPUT:-}" ]; then + printf 'batches=%s\n' "$(jq -cn --argjson last "${batch}" '[range(0; $last + 1) | tostring]')" \ + >>"${GITHUB_OUTPUT}" + fi +} +collect_config_findings() { # Preserve report/profile identity while counting repeated findings. - local report profile - { - for report in "${report_dir}"/*.json; do - profile="$(basename "${report}" .json)" - jq --arg profile "${profile}" --arg severities "${SEVERITY}" ' - if .SchemaVersion != 2 + # With no reports, the unmatched glob makes jq fail on the missing input. + jq -n --arg severities "${SEVERITY}" ' + [inputs + | (input_filename | split("/")[-1] | rtrimstr(".json")) as $profile + | if .SchemaVersion != 2 or ((.ArtifactName | type) != "string") or ((.ArtifactType | type) != "string") or (.Results != null and ((.Results | type) != "array")) @@ -234,12 +313,11 @@ collect_config_findings() { | map(.[0] + { count: length })) } end - ' "${report}" - done - } | jq -s '{ - profiles: map(.profile), - findings: (map(.findings) | add // []) - }' + ] | { + profiles: map(.profile), + findings: (map(.findings) | add // []) + } + ' "$1"/*.json } # Compare semantic identities and occurrence counts, excluding line numbers. @@ -290,7 +368,7 @@ gate_config_diff() ( echo "::error::Trivy reported ${finding_count} new configuration finding(s) at ${SEVERITY}." jq -r '.[] - | "::error::[\(.severity)] \(.id) in \(.profile) (deploy/\(.target)): \(.title)" + | "::error::[\(.severity)] \(.id) in \(.profile) (\(.target)): \(.title)" + " (\(.new_count) new, \(.baseline_count) in baseline)"' \ "${new_findings}" if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then @@ -299,7 +377,7 @@ gate_config_diff() ( echo jq -r '.[] | "- **\(.severity)** `\(.id)` in `\(.profile)`" - + " (`deploy/\(.target)`): \(.title)" + + " (`\(.target)`): \(.title)" + " (\(.new_count) new, \(.baseline_count) in baseline)"' \ "${new_findings}" } >>"${GITHUB_STEP_SUMMARY}" @@ -341,6 +419,10 @@ case "${1:-}" in require_trivy gate ;; + prepare-sarif) + require_trivy + prepare_sarif + ;; gate-config-diff) shift [ $# -eq 2 ] || { @@ -358,6 +440,7 @@ Usage: trivy-scan.sh config [--chart-ref ]... trivy-scan.sh images [...] trivy-scan.sh gate + trivy-scan.sh prepare-sarif trivy-scan.sh gate-config-diff trivy-scan.sh validate-ignore USAGE