From ee451b24a58e987fb804793ee24d5e84eaba3ebd Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Wed, 9 Sep 2026 13:35:30 +0530 Subject: [PATCH 01/17] DLPX-98872 linux-pkg: generate per-package CycloneDX SBOM via Syft deb scan and publish S3 sidecar PR URL: https://www.github.com/delphix/linux-pkg/pull/418 --- .github/scripts/verify-sbom-scan-flag.sh | 24 ++ .github/workflows/main.yml | 5 + buildpkg.sh | 3 + ...6-09-08-sbom-per-package-sidecar-design.md | 279 ++++++++++++++++++ lib/common.sh | 61 ++++ packages/bcc/config.sh | 1 + packages/challenge-response/config.sh | 1 + packages/cloud-init/config.sh | 1 + packages/connstat/config.sh | 1 + packages/containerized-masking/config.sh | 1 + packages/crash-python/config.sh | 1 + packages/crypt-blowfish/config.sh | 2 + packages/cyclonedx-cli/config.sh | 1 + packages/delphix-go/config.sh | 1 + packages/delphix-kernel/config.sh | 1 + packages/delphix-platform/config.sh | 1 + packages/delphix-rust/config.sh | 1 + packages/delphix-sso-app/config.sh | 1 + packages/docker-python-image/config.sh | 4 +- packages/drgn/config.sh | 1 + packages/dwarves/config.sh | 1 + packages/fluentd-gems/config.sh | 1 + packages/gdb-python/config.sh | 1 + packages/grub2/config.sh | 1 + packages/host-jdks/config.sh | 1 + packages/libkdumpfile/config.sh | 1 + packages/linux-kernel-aws/config.sh | 3 + packages/linux-kernel-azure/config.sh | 3 + packages/linux-kernel-gcp/config.sh | 3 + packages/linux-kernel-generic/config.sh | 3 + packages/linux-kernel-oracle/config.sh | 3 + packages/makedumpfile/config.sh | 1 + packages/masking/config.sh | 1 + packages/misc-debs/config.sh | 1 + packages/nfs-utils/config.sh | 1 + packages/performance-diagnostics/config.sh | 1 + packages/ptools/config.sh | 1 + packages/python-rtslib-fb/config.sh | 1 + packages/savedump/config.sh | 1 + packages/sdb/config.sh | 1 + packages/syft/config.sh | 1 + packages/targetcli-fb/config.sh | 1 + packages/virtualization/config.sh | 1 + packages/windows-connector/config.sh | 1 + packages/zfs/config.sh | 1 + query-packages.sh | 3 +- 46 files changed, 427 insertions(+), 2 deletions(-) create mode 100755 .github/scripts/verify-sbom-scan-flag.sh create mode 100644 docs/specs/2026-09-08-sbom-per-package-sidecar-design.md diff --git a/.github/scripts/verify-sbom-scan-flag.sh b/.github/scripts/verify-sbom-scan-flag.sh new file mode 100755 index 00000000..94691ddd --- /dev/null +++ b/.github/scripts/verify-sbom-scan-flag.sh @@ -0,0 +1,24 @@ +#!/bin/bash -ex + +set -o pipefail + +cd "$(git rev-parse --show-toplevel)" + +# +# Every package must explicitly set SBOM_DEEP_SCAN to "true" or "false" in +# its config.sh -- see docs on generate_sbom() in lib/common.sh. There is +# no default: a package that hasn't been classified yet must fail CI +# rather than silently ship without a CycloneDX sidecar or without an +# explicit decision that it doesn't need one. +# +unclassified=$(./query-packages.sh list -o name,sbom-deep-scan all | + awk -F'\t' '$2 == "none" { print $1 }') + +if [[ -n "$unclassified" ]]; then + echo "The following packages have not set SBOM_DEEP_SCAN (\"true\" or" \ + "\"false\") in their config.sh:" + echo "$unclassified" + exit 1 +fi + +echo "All packages have classified SBOM_DEEP_SCAN" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e527d7c7..f4f9a559 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,3 +29,8 @@ jobs: steps: - uses: actions/checkout@v1 - run: ./.github/scripts/verify-query-packages.sh + verify-sbom-scan-flag: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - run: ./.github/scripts/verify-sbom-scan-flag.sh \ No newline at end of file diff --git a/buildpkg.sh b/buildpkg.sh index 891afb2a..57d184d1 100755 --- a/buildpkg.sh +++ b/buildpkg.sh @@ -137,6 +137,9 @@ stage build logmust cd "$WORKDIR" stage store_build_info +logmust cd "$WORKDIR" +stage generate_sbom + logmust cd "$WORKDIR" stage post_build_checks diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md new file mode 100644 index 00000000..7e97bccc --- /dev/null +++ b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md @@ -0,0 +1,279 @@ +# SBOM Per-Package Sidecar Generation — Design + +- **Date:** 2026-09-08 +- **Jira:** [DLPX-98872](https://perforce.atlassian.net/browse/DLPX-98872) +- **Epic:** [CP-13455](https://perforce.atlassian.net/browse/CP-13455) — CycloneDX SBOM for Delphix engine product images +- **Implements:** Phase 2 (CP-13465) of `appliance-build`'s + `docs/specs/2026-06-23-sbom-generation-implementation-plan.md` +- **Companion to (in the `appliance-build` repo):** `docs/specs/2026-06-15-sbom-generation-design.md` + (CP-13456, the overall design) and `docs/specs/2026-08-13-syft-cyclonedx-cli-provisioning-design.md` + (CP-13600, how `syft`/`cyclonedx-cli` get onto a build host) +- **Implemented in:** this change (DLPX-98872) — see *Implementation* below. + +## Builds on: Phase 1 (already implemented) + +Phase 1 — the `appliance-build` per-image base scan (CP-13464) — and its `syft`/ +`cyclonedx-cli` provisioning prerequisite (CP-13600) are implemented, each with an open PR: + +| Repo | PR | What it does | +|---|---|---| +| `syft` | [#1](https://github.com/delphix/syft/pull/1) | Packaging repo for `delphix-syft` (fetches the pinned upstream release binary) | +| `cyclonedx-cli` | [#1](https://github.com/delphix/cyclonedx-cli/pull/1) | Packaging repo for `delphix-cyclonedx-cli` (same pattern) | +| `linux-pkg` | [#414](https://github.com/delphix/linux-pkg/pull/414) | Adds `packages/syft/`, `packages/cyclonedx-cli/`, listed in `package-lists/build/main.pkgs` (this branch is rebased on top of it) | +| `appliance-build` | [#892](https://github.com/delphix/appliance-build/pull/892) | Installs both `.deb`s onto the build host in `build-ancillary-repository.sh`; new `95-generate-sbom.binary` hook runs `syft scan dir:binary --select-catalogers dpkg` + `cyclonedx-cli validate` to emit `-.cdx.json` | +| `devops-gate` | [#4717](https://github.com/delphix/devops-gate/pull/4717) | `appliance_build_stage0.groovy` best-effort-fetches and archives the per-image `.cdx.json` | + +Net effect: `syft` and `cyclonedx-cli` are already real, buildable `linux-pkg` packages, +already on every build host's `PATH` (build-host-only tooling — never shipped in the image), +and every image already gets a flat, dpkg-only base SBOM. **Phase 2 has no new tooling +dependency to add** — it only needs to *invoke* `syft`, which is already available. + +## Problem + +The Phase 1 base scan lists every installed `.deb` as a flat `pkg:deb` component. That is +correct and sufficient for 3rd-party Debian packages, but wrong for Delphix's own +first-party packages: `masking`, `virtualization`, `delphix-sso-app`, `containerized-masking`, +`windows-connector`, `zfs`, `ptools`, and `delphix-rust` all bundle third-party components +across ecosystems (jars, npm, wheels, Rust crates) that `dpkg` cannot see — dpkg only knows +the files *it* placed, not what's vendored inside a jar or statically linked into a Rust +binary. Those components are invisible to a vulnerability scanner today. + +Per the original design's key decision, this composition must be captured **upstream, in +`linux-pkg`**, at build time, where the package's own build artifacts are present — not +reconstructed later from a stripped, compiled `.deb`. + +## Design and implementation + +All of the following is implemented in this change, entirely within `linux-pkg` — no other +repo needs to change for Phase 2 (see *S3 upload* below for why). + +### 1. `SBOM_DEEP_SCAN` — per-package opt-in flag + +Mirrors the existing `MEND_SCAN_APPLICABLE` convention: a plain variable set in a package's +`config.sh`. Unlike `MEND_SCAN_APPLICABLE` (opt-in only, unset elsewhere), this flag is a real +tri-state — "must deep-scan" / "explicitly doesn't need it" / "nobody has classified this +yet" — enforced by the CI lint in §3, so **every** package's `config.sh` now has an explicit +`SBOM_DEEP_SCAN="true"` or `SBOM_DEEP_SCAN="false"` (see §5). + +```bash +# packages//config.sh +SBOM_DEEP_SCAN="true" +``` + +### 2. `query-packages.sh` — surface the field + +`query-packages.sh` has a closed, hardcoded field enum — no generic "query any config.sh +variable" mechanism exists. Two edits: + +- `ALL_OUTPUT_FIELDS`: appended `sbom-deep-scan`. +- `print_package()`'s `case "$field"` block, alongside the `mend-scan)` arm: + ```bash + sbom-deep-scan) outarray+=("${SBOM_DEEP_SCAN:-none}") ;; + ``` + +`-o`'s comma-delimited parsing and validation against `ALL_OUTPUT_FIELDS` then works for the +new field automatically. `./query-packages.sh list -o name,sbom-deep-scan all` is the +mechanism both the CI lint (§3) and `appliance-build`'s future Phase 3 consumer use to find +flagged packages. + +### 3. CI lint — every package must be classified + +No existing precedent to extend — `MEND_SCAN_APPLICABLE` is opt-in with zero enforcement +anywhere in `linux-pkg`. New script `.github/scripts/verify-sbom-scan-flag.sh`, wired as a +`verify-sbom-scan-flag` job in `.github/workflows/main.yml` alongside the existing +`verify-query-packages*` jobs: + +```bash +unclassified=$(./query-packages.sh list -o name,sbom-deep-scan all | + awk -F'\t' '$2 == "none" { print $1 }') +if [[ -n "$unclassified" ]]; then + echo "The following packages have not set SBOM_DEEP_SCAN (\"true\" or" \ + "\"false\") in their config.sh:" + echo "$unclassified" + exit 1 +fi +``` + +The one-time classification of all 40 packages (§5) lands in the same change, so this lint +never lands red. + +### 4. `generate_sbom()` — sidecar generation + +Modeled on `store_build_info()` — a default stage function in `lib/common.sh` that packages +don't need to override, gated on the new flag: + +```bash +# lib/common.sh +function generate_sbom() { + if [[ "$SBOM_DEEP_SCAN" != "true" ]]; then + return 0 + fi + + local debs=("$WORKDIR/artifacts/"*.deb) + if [[ ! -e "${debs[0]}" ]]; then + die "SBOM_DEEP_SCAN is set but no .deb was found in" \ + "'$WORKDIR/artifacts'" + fi + + local sbom_file="$WORKDIR/artifacts/$PACKAGE.cdx.json" + local sbom_scratch_dir + sbom_scratch_dir="$(logmust mktemp -d)" + + # A package can emit more than one .deb from a single build (e.g. + # "zfs" splits into zfs-dkms, zfsutils-linux, etc.) -- scan each one + # into its own document, then merge into a single sidecar so there's + # exactly one .cdx.json per package, matching how + # appliance-build associates a sidecar to a package via COMPONENTS. + local deb sbom_parts=() + for deb in "${debs[@]}"; do + local part + part="$sbom_scratch_dir/$(basename "$deb").cdx.json" + logmust syft scan "$deb" \ + --source-name "$PACKAGE" \ + --source-version "$PACKAGE_VERSION" \ + -o "cyclonedx-json@1.6=$part" + sbom_parts+=("$part") + done + + if [[ ${#sbom_parts[@]} -eq 1 ]]; then + logmust cp "${sbom_parts[0]}" "$sbom_file" + else + logmust cyclonedx-cli merge \ + --input-files "${sbom_parts[@]}" \ + --output-format json \ + --output-file "$sbom_file" + fi + logmust rm -rf "$sbom_scratch_dir" + + logmust cyclonedx-cli validate \ + --input-file "$sbom_file" \ + --input-format json \ + --input-version v1_6 \ + --fail-on-errors +} +``` + +Wired into `buildpkg.sh` as a new stage between the two that already bracket this point: + +```bash +logmust cd "$WORKDIR" +stage store_build_info +logmust cd "$WORKDIR" +stage generate_sbom # <-- new +logmust cd "$WORKDIR" +stage post_build_checks +``` + +`stage` silently skips undefined hooks, so this is inert for every package until +`generate_sbom` is defined — defining it once in `lib/common.sh`, gated internally on the +flag, means no package needs its own override for the baseline case (only per-ecosystem +overrides, out of scope for Phase 2, would override the function in a package's `config.sh`, +the same way packages already override `build()`). + +**Resolved — multiple `.deb`s per package:** handled above via per-deb scan + `cyclonedx-cli +merge` into one sidecar. Verify against a real `zfs` build before merging (see *Follow-ups*). + +**Open — Syft's `.deb`-scan support:** not yet verified against the real `syft`/ +`cyclonedx-cli` binaries in this environment (neither is installed here). Confirm the pinned +`delphix-syft` version (`SYFT_VERSION` in the `syft` packaging repo) supports scanning a +standalone `.deb` file as a source (Phase 1 only exercises Syft against a directory/rootfs, +not a single `.deb`). If unsupported, fall back to extracting the `.deb`'s `data.tar.*` into +a temp dir first and scanning that directory instead. + +### 5. Package classification + +Per the original design's "does this package bundle third-party composition" criterion (not +strictly "is it 1st-party" — see the `zfs` case), `SBOM_DEEP_SCAN="true"` is set for: + +| Package | Why | +|---|---| +| `masking` | Java/Gradle app bundling jars + npm frontend | +| `virtualization` | Java/Gradle app bundling jars + npm frontend | +| `delphix-sso-app` | Java/Gradle app | +| `containerized-masking` | Java/Gradle app | +| `windows-connector` | Java/Gradle app | +| `zfs` | OpenZFS fork bundling Delphix's Rust object agent (crates invisible to dpkg) | +| `ptools` | Rust | +| `delphix-rust` | Rust | + +The remaining 32 packages (kernel packages, `misc-debs`, `syft`/`cyclonedx-cli` themselves, +etc.) get `SBOM_DEEP_SCAN="false"` — plain 3rd-party forks or single-ecosystem tools already +fully represented by the Phase 1 flat `pkg:deb` component. `delphix-go` is a judgment call: +the top-level design's Tooling section separately calls out a possible Go override +(`cyclonedx-gomod`); defaulted to `"false"` here (baseline Syft-on-deb has a Go +binary-build-info cataloger that may already cover it) — revisit under Phase 4's evaluation +if gaps are found. + +### S3 upload — no new plumbing needed + +Confirmed by reading `devops-gate/jenkins/jobs/pipelines/linux_pkg_build_package.groovy`'s +`Publish` stage: it runs `aws s3 sync --delete --only-show-errors . ${env.S3_PACKAGE_PATH}` +from inside `dir("linux-pkg/workdir/artifacts")` — i.e., it uploads **the entire artifacts +directory verbatim**, whatever stages before it dropped there (`store_build_info`'s +`GIT_HASH`/`BUILD_INFO` files, the built `.deb`(s), etc.). `generate_sbom()` writing +`.cdx.json` into that same directory is automatically picked up by this existing +sync — **no `devops-gate` change required for Phase 2**, unlike Phase 1 which needed a new +fetch step in `appliance_build_stage0.groovy` because that pipeline wasn't already pulling +per-package artifact directories at all. + +## Architecture diagram + +``` ++---------------------------------------------------------------------------+ +| linux-pkg package build (buildpkg.sh) | +| | +| stage build --> $WORKDIR/artifacts/*.deb | +| stage store_build_info --> GIT_HASH, BUILD_INFO, ... | +| stage generate_sbom --> [only if SBOM_DEEP_SCAN="true"] | +| syft scan -o cyclonedx-json@1.6 | +| (merge via cyclonedx-cli if >1 .deb) | +| --> $WORKDIR/artifacts/.cdx.json | +| cyclonedx-cli validate --fail-on-errors | +| stage post_build_checks | ++------------------------------------+--------------------------------------+ + | + | devops-gate Publish stage: + | aws s3 sync (whole artifacts/ dir, + | unmodified from Phase 1) + v ++---------------------------------------------------------------------------+ +| S3: combined-packages/packages// | +| *.deb | +| .cdx.json <-- NEW: sidecar, sits beside the deb(s) | +| GIT_HASH, BUILD_INFO, ... (unchanged) | ++---------------------------------------------------------------------------+ + | + | (Phase 3, not in scope here: + | appliance-build fetches + merges + | this sidecar into the Phase 1 base) + v + [out of scope for Phase 2] +``` + +## Out of scope for Phase 2 + +- Per-ecosystem overrides (`cargo-cyclonedx` for Rust, `cyclonedx-gradle-plugin` for + Java) — Phase 4's evaluation decides if the Syft-on-deb baseline here is good enough + first. +- `appliance-build` fetching/merging these sidecars into the per-image document — that's + Phase 3 (CP-13466), which explicitly depends on this phase completing. +- DCT/Hyperscale sidecars — deferred in the top-level design, not filed as a story yet. +- The `devops-gate` publishing switch (CSV → CycloneDX) — unrelated to sidecar generation. + +## Implementation status + +- [x] `SBOM_DEEP_SCAN` classification added to all 40 packages' `config.sh` (§5). +- [x] `sbom-deep-scan` field added to `query-packages.sh` (§2). +- [x] `generate_sbom()` added to `lib/common.sh` and wired as `stage generate_sbom` in + `buildpkg.sh` (§4), including the multi-`.deb` merge case. +- [x] CI lint added (§3), landed in the same change as the classification pass. +- [x] Verified locally: `verify-sbom-scan-flag.sh` and the existing + `verify-query-packages.sh` both pass; `shellcheck`/`shfmt` clean on every touched file. + +## Follow-ups before merging + +1. Validate against a real `syft`/`cyclonedx-cli` build host: confirm `syft scan ` + works as expected (the open question in §4), and run an actual `zfs` build (multi-`.deb` + package) and `masking` build (single-`.deb`, Java+npm) end-to-end — confirm + `$WORKDIR/artifacts/.cdx.json` is produced, schema-valid, and lands in S3. +2. File as sub-tasks of DLPX-98872. \ No newline at end of file diff --git a/lib/common.sh b/lib/common.sh index 1faab9c7..d4ed860c 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1461,6 +1461,67 @@ function store_build_info() { fi } +# +# Generate a per-package CycloneDX SBOM sidecar by running Syft against +# this package's built .deb(s). Only packages that bundle third-party +# composition (jars, npm, wheels, Rust crates, ...) opt in via +# SBOM_DEEP_SCAN="true" in their config.sh -- everything else is left as +# a flat pkg:deb component by appliance-build's base chroot scan, so a +# deb scan here would add nothing. The resulting .cdx.json is +# dropped in $WORKDIR/artifacts/ alongside the .deb(s), where it's picked +# up by the same S3 sync as every other build artifact -- no separate +# upload path needed. +# +function generate_sbom() { + if [[ "$SBOM_DEEP_SCAN" != "true" ]]; then + return 0 + fi + + local debs=("$WORKDIR/artifacts/"*.deb) + if [[ ! -e "${debs[0]}" ]]; then + die "SBOM_DEEP_SCAN is set but no .deb was found in" \ + "'$WORKDIR/artifacts'" + fi + + local sbom_file="$WORKDIR/artifacts/$PACKAGE.cdx.json" + local sbom_scratch_dir + sbom_scratch_dir="$(logmust mktemp -d)" + + # + # A package can emit more than one .deb from a single build (e.g. + # "zfs" splits into zfs-dkms, zfsutils-linux, etc.) -- scan each one + # into its own document, then merge them into a single sidecar so + # there's exactly one .cdx.json per package, matching how + # appliance-build associates a sidecar to a package via COMPONENTS. + # + local deb sbom_parts=() + for deb in "${debs[@]}"; do + local part + part="$sbom_scratch_dir/$(basename "$deb").cdx.json" + logmust syft scan "$deb" \ + --source-name "$PACKAGE" \ + --source-version "$PACKAGE_VERSION" \ + -o "cyclonedx-json@1.6=$part" + sbom_parts+=("$part") + done + + if [[ ${#sbom_parts[@]} -eq 1 ]]; then + logmust cp "${sbom_parts[0]}" "$sbom_file" + else + logmust cyclonedx-cli merge \ + --input-files "${sbom_parts[@]}" \ + --output-format json \ + --output-file "$sbom_file" + fi + logmust rm -rf "$sbom_scratch_dir" + + logmust cyclonedx-cli validate \ + --input-file "$sbom_file" \ + --input-format json \ + --input-version v1_6 \ + --fail-on-errors +} + function set_secret_build_args() { _SECRET_BUILD_ARGS=() diff --git a/packages/bcc/config.sh b/packages/bcc/config.sh index 047b6922..1094b594 100644 --- a/packages/bcc/config.sh +++ b/packages/bcc/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/bcc.git" +SBOM_DEEP_SCAN="false" UPSTREAM_GIT_URL=https://github.com/iovisor/bcc.git UPSTREAM_GIT_BRANCH=master diff --git a/packages/challenge-response/config.sh b/packages/challenge-response/config.sh index 997f96f8..6c29a67d 100644 --- a/packages/challenge-response/config.sh +++ b/packages/challenge-response/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/challenge-response.git" +SBOM_DEEP_SCAN="false" function prepare() { install_build_deps_from_control_file diff --git a/packages/cloud-init/config.sh b/packages/cloud-init/config.sh index dd720656..aab2ad00 100644 --- a/packages/cloud-init/config.sh +++ b/packages/cloud-init/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/cloud-init.git" +SBOM_DEEP_SCAN="false" UPSTREAM_SOURCE_PACKAGE=cloud-init diff --git a/packages/connstat/config.sh b/packages/connstat/config.sh index 4e50e1b4..08ea16ed 100644 --- a/packages/connstat/config.sh +++ b/packages/connstat/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/connstat.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="@linux-kernel dwarves" function prepare() { diff --git a/packages/containerized-masking/config.sh b/packages/containerized-masking/config.sh index 36b06aca..a53a234e 100644 --- a/packages/containerized-masking/config.sh +++ b/packages/containerized-masking/config.sh @@ -40,6 +40,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" # PACKAGE_NEEDS_DOCKER="true" MEND_SCAN_APPLICABLE="true" +SBOM_DEEP_SCAN="true" MEND_SCAN_IMAGES="'delphix-masking-proxy', 'delphix-masking-database', 'delphix-masking-app'" SKIP_COPYRIGHTS_CHECK=true diff --git a/packages/crash-python/config.sh b/packages/crash-python/config.sh index 495597fc..ac839c98 100644 --- a/packages/crash-python/config.sh +++ b/packages/crash-python/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/crash-python.git" +SBOM_DEEP_SCAN="false" function prepare() { logmust install_build_deps_from_control_file diff --git a/packages/crypt-blowfish/config.sh b/packages/crypt-blowfish/config.sh index 04244a4b..b1f26638 100644 --- a/packages/crypt-blowfish/config.sh +++ b/packages/crypt-blowfish/config.sh @@ -17,6 +17,8 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/crypt-blowfish.git" +SBOM_DEEP_SCAN="false" + SKIP_COPYRIGHTS_CHECK=true function build() { diff --git a/packages/cyclonedx-cli/config.sh b/packages/cyclonedx-cli/config.sh index 3ee39875..ed56ae59 100755 --- a/packages/cyclonedx-cli/config.sh +++ b/packages/cyclonedx-cli/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/cyclonedx-cli.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-go/config.sh b/packages/delphix-go/config.sh index f9926295..9dfc17d1 100755 --- a/packages/delphix-go/config.sh +++ b/packages/delphix-go/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-go.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-kernel/config.sh b/packages/delphix-kernel/config.sh index 125836ff..c556cc25 100644 --- a/packages/delphix-kernel/config.sh +++ b/packages/delphix-kernel/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-kernel.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="@linux-kernel" function prepare() { diff --git a/packages/delphix-platform/config.sh b/packages/delphix-platform/config.sh index 42e4ff4c..c83aa809 100644 --- a/packages/delphix-platform/config.sh +++ b/packages/delphix-platform/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-platform.git" +SBOM_DEEP_SCAN="false" function prepare() { logmust cd "$WORKDIR/repo" diff --git a/packages/delphix-rust/config.sh b/packages/delphix-rust/config.sh index 4f1f06c0..c1613a2b 100755 --- a/packages/delphix-rust/config.sh +++ b/packages/delphix-rust/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-rust.git" +SBOM_DEEP_SCAN="true" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 6d1a9c05..480f79d0 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -19,6 +19,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/saml-app.git" MEND_SCAN_APPLICABLE="true" MEND_SCAN_USING_SUDO="true" +SBOM_DEEP_SCAN="true" function prepare() { logmust install_pkgs openjdk-17-jdk-headless: diff --git a/packages/docker-python-image/config.sh b/packages/docker-python-image/config.sh index b80e6172..47f1f501 100644 --- a/packages/docker-python-image/config.sh +++ b/packages/docker-python-image/config.sh @@ -17,12 +17,14 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/docker-python-image.git" +SBOM_DEEP_SCAN="false" + # # debian/rules' override_dh_install runs 'docker pull' to fetch the python # image it repackages, so the build needs a docker daemon. The build container # gets the host's daemon through its socket rather than running one of its own. # -PACKAGE_NEEDS_DOCKER="true" +PACKAGE_NEEDS_DOCKER="false" function prepare() { logmust install_build_deps_from_control_file diff --git a/packages/drgn/config.sh b/packages/drgn/config.sh index 0387a3a7..3f29a512 100644 --- a/packages/drgn/config.sh +++ b/packages/drgn/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/drgn.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="libkdumpfile" UPSTREAM_GIT_URL="https://github.com/osandov/drgn.git" diff --git a/packages/dwarves/config.sh b/packages/dwarves/config.sh index 46965c54..86cfcc66 100644 --- a/packages/dwarves/config.sh +++ b/packages/dwarves/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dwarves.git" +SBOM_DEEP_SCAN="false" UPSTREAM_GIT_URL="https://github.com/acmel/dwarves.git" UPSTREAM_GIT_BRANCH="master" diff --git a/packages/fluentd-gems/config.sh b/packages/fluentd-gems/config.sh index b511f1bb..2110330b 100644 --- a/packages/fluentd-gems/config.sh +++ b/packages/fluentd-gems/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/fluentd-gems.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/gdb-python/config.sh b/packages/gdb-python/config.sh index 014215f0..c36d22dd 100644 --- a/packages/gdb-python/config.sh +++ b/packages/gdb-python/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/gdb-python.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="libkdumpfile" function prepare() { diff --git a/packages/grub2/config.sh b/packages/grub2/config.sh index 82452184..e280c484 100644 --- a/packages/grub2/config.sh +++ b/packages/grub2/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none +SBOM_DEEP_SCAN="false" SKIP_COPYRIGHTS_CHECK=true URI="s3://release-de-images/internal-artifacts/2025.3.0.1/1.0.53/input-artifacts/combined-packages/packages/grub2" diff --git a/packages/host-jdks/config.sh b/packages/host-jdks/config.sh index 5aec9a30..d0b93c80 100755 --- a/packages/host-jdks/config.sh +++ b/packages/host-jdks/config.sh @@ -23,6 +23,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/host-jdks.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/libkdumpfile/config.sh b/packages/libkdumpfile/config.sh index d1dd6f52..91976030 100644 --- a/packages/libkdumpfile/config.sh +++ b/packages/libkdumpfile/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/libkdumpfile.git" +SBOM_DEEP_SCAN="false" UPSTREAM_GIT_URL="https://codeberg.org/ptesarik/libkdumpfile.git" UPSTREAM_GIT_BRANCH="tip" diff --git a/packages/linux-kernel-aws/config.sh b/packages/linux-kernel-aws/config.sh index b2596d2f..1f00b06c 100644 --- a/packages/linux-kernel-aws/config.sh +++ b/packages/linux-kernel-aws/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/linux-kernel-azure/config.sh b/packages/linux-kernel-azure/config.sh index 713b9b26..376cbadf 100644 --- a/packages/linux-kernel-azure/config.sh +++ b/packages/linux-kernel-azure/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/linux-kernel-gcp/config.sh b/packages/linux-kernel-gcp/config.sh index 713b9b26..376cbadf 100644 --- a/packages/linux-kernel-gcp/config.sh +++ b/packages/linux-kernel-gcp/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/linux-kernel-generic/config.sh b/packages/linux-kernel-generic/config.sh index 713b9b26..376cbadf 100644 --- a/packages/linux-kernel-generic/config.sh +++ b/packages/linux-kernel-generic/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/linux-kernel-oracle/config.sh b/packages/linux-kernel-oracle/config.sh index 713b9b26..376cbadf 100644 --- a/packages/linux-kernel-oracle/config.sh +++ b/packages/linux-kernel-oracle/config.sh @@ -31,3 +31,6 @@ default) die "invalid linux-kernel package source '$linux_package_source'" ;; esac + +# shellcheck disable=SC2034 +SBOM_DEEP_SCAN="false" diff --git a/packages/makedumpfile/config.sh b/packages/makedumpfile/config.sh index 8eb407a5..3ad8172d 100644 --- a/packages/makedumpfile/config.sh +++ b/packages/makedumpfile/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/makedumpfile.git" +SBOM_DEEP_SCAN="false" UPSTREAM_SOURCE_PACKAGE="makedumpfile" diff --git a/packages/masking/config.sh b/packages/masking/config.sh index 92163bdb..2229e02d 100644 --- a/packages/masking/config.sh +++ b/packages/masking/config.sh @@ -20,6 +20,7 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" MEND_SCAN_APPLICABLE="true" +SBOM_DEEP_SCAN="true" function prepare() { logmust read_list "$WORKDIR/repo/packaging/build-dependencies" diff --git a/packages/misc-debs/config.sh b/packages/misc-debs/config.sh index 19fa7c7d..32bb43bd 100644 --- a/packages/misc-debs/config.sh +++ b/packages/misc-debs/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL=none +SBOM_DEEP_SCAN="false" SKIP_COPYRIGHTS_CHECK=true # diff --git a/packages/nfs-utils/config.sh b/packages/nfs-utils/config.sh index 15e3c725..639097d6 100644 --- a/packages/nfs-utils/config.sh +++ b/packages/nfs-utils/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/nfs-utils.git" +SBOM_DEEP_SCAN="false" UPSTREAM_SOURCE_PACKAGE=nfs-utils diff --git a/packages/performance-diagnostics/config.sh b/packages/performance-diagnostics/config.sh index 4733a8c9..6d666c36 100644 --- a/packages/performance-diagnostics/config.sh +++ b/packages/performance-diagnostics/config.sh @@ -18,6 +18,7 @@ # DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/performance-diagnostics.git" +SBOM_DEEP_SCAN="false" function prepare() { logmust install_build_deps_from_control_file diff --git a/packages/ptools/config.sh b/packages/ptools/config.sh index 459cc4ca..54aef875 100644 --- a/packages/ptools/config.sh +++ b/packages/ptools/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/ptools.git" +SBOM_DEEP_SCAN="true" function prepare() { logmust install_pkgs cargo devscripts diff --git a/packages/python-rtslib-fb/config.sh b/packages/python-rtslib-fb/config.sh index 024c025e..99c334ef 100644 --- a/packages/python-rtslib-fb/config.sh +++ b/packages/python-rtslib-fb/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/python-rtslib-fb.git" +SBOM_DEEP_SCAN="false" UPSTREAM_SOURCE_PACKAGE=python-rtslib-fb diff --git a/packages/savedump/config.sh b/packages/savedump/config.sh index 1e9bdda2..ba291b44 100644 --- a/packages/savedump/config.sh +++ b/packages/savedump/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/savedump.git" +SBOM_DEEP_SCAN="false" function prepare() { logmust install_build_deps_from_control_file diff --git a/packages/sdb/config.sh b/packages/sdb/config.sh index 3b36fed4..119abc18 100644 --- a/packages/sdb/config.sh +++ b/packages/sdb/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/sdb.git" +SBOM_DEEP_SCAN="false" UPSTREAM_GIT_URL="https://github.com/sdimitro/sdb.git" UPSTREAM_GIT_BRANCH="develop" diff --git a/packages/syft/config.sh b/packages/syft/config.sh index 987ee8f6..d4d40573 100755 --- a/packages/syft/config.sh +++ b/packages/syft/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/syft.git" +SBOM_DEEP_SCAN="false" function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/targetcli-fb/config.sh b/packages/targetcli-fb/config.sh index abdc2ea3..3841b56b 100644 --- a/packages/targetcli-fb/config.sh +++ b/packages/targetcli-fb/config.sh @@ -17,6 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/targetcli-fb.git" +SBOM_DEEP_SCAN="false" PACKAGE_DEPENDENCIES="python-rtslib-fb" UPSTREAM_SOURCE_PACKAGE=targetcli-fb diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh index e7fe6b3a..171b111a 100644 --- a/packages/virtualization/config.sh +++ b/packages/virtualization/config.sh @@ -27,6 +27,7 @@ PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks" # PACKAGE_NEEDS_DOCKER="true" MEND_SCAN_APPLICABLE="true" +SBOM_DEEP_SCAN="true" function prepare() { logmust read_list "$WORKDIR/repo/appliance/packaging/build-dependencies" diff --git a/packages/windows-connector/config.sh b/packages/windows-connector/config.sh index 40479cd2..185083da 100644 --- a/packages/windows-connector/config.sh +++ b/packages/windows-connector/config.sh @@ -25,6 +25,7 @@ # DEFAULT_PACKAGE_GIT_URL="none" SKIP_COPYRIGHTS_CHECK=true +SBOM_DEEP_SCAN="true" function fetch() { PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index bd1edce5..de13dcb1 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -18,6 +18,7 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/zfs.git" PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust delphix-go dwarves" +SBOM_DEEP_SCAN="true" UPSTREAM_GIT_URL="https://github.com/openzfs/zfs.git" UPSTREAM_GIT_BRANCH="master" diff --git a/query-packages.sh b/query-packages.sh index d92c7c23..612f70ed 100755 --- a/query-packages.sh +++ b/query-packages.sh @@ -25,7 +25,7 @@ source "$TOP/lib/common.sh" # export LOGGING=false -ALL_OUTPUT_FIELDS=(name git-url dependencies can-update mend-scan mend-scan-images mend-scan-using-sudo) +ALL_OUTPUT_FIELDS=(name git-url dependencies can-update mend-scan mend-scan-images mend-scan-using-sudo sbom-deep-scan) function usage() { local output_fields="${ALL_OUTPUT_FIELDS[*]}" @@ -73,6 +73,7 @@ function print_package() { mend-scan) outarray+=("${MEND_SCAN_APPLICABLE:-none}") ;; mend-scan-images) outarray+=("${MEND_SCAN_IMAGES:-none}") ;; mend-scan-using-sudo) outarray+=("${MEND_SCAN_USING_SUDO:-none}") ;; + sbom-deep-scan) outarray+=("${SBOM_DEEP_SCAN:-none}") ;; dependencies) for dep in $PACKAGE_DEPENDENCIES; do check_package_exists "$dep" From 0b4e66b8a5249e1b48fe7db7caf6fd2732e5adb4 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Wed, 9 Sep 2026 17:17:00 +0530 Subject: [PATCH 02/17] fixed packages/docker-python-image/config.sh --- packages/docker-python-image/config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/docker-python-image/config.sh b/packages/docker-python-image/config.sh index 47f1f501..9379b26a 100644 --- a/packages/docker-python-image/config.sh +++ b/packages/docker-python-image/config.sh @@ -24,7 +24,7 @@ SBOM_DEEP_SCAN="false" # image it repackages, so the build needs a docker daemon. The build container # gets the host's daemon through its socket rather than running one of its own. # -PACKAGE_NEEDS_DOCKER="false" +PACKAGE_NEEDS_DOCKER="true" function prepare() { logmust install_build_deps_from_control_file From 48d68231062c61743ca24b4c41aa0346199dd43c Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Thu, 10 Sep 2026 00:57:24 +0530 Subject: [PATCH 03/17] DLPX-98872 fix: install syft/cyclonedx-cli before generate_sbom, fix version generate_sbom() assumed syft/cyclonedx-cli were already on PATH, but nothing installed them into the linux-pkg build container -- Phase 1 only installs them onto the appliance-build host. Confirmed via a real pre-push run: every SBOM_DEEP_SCAN package failed with "syft: command not found". Fixed by declaring syft/cyclonedx-cli as PACKAGE_DEPENDENCIES on all 8 flagged packages (masking, virtualization, delphix-sso-app, containerized-masking, windows-connector, zfs, ptools, delphix-rust), and having generate_sbom() install them from $DEPDIR before scanning -- the same pattern zfs already uses for delphix-rust. Also fixes --source-version showing up blank for packages (like masking) that don't set $PACKAGE_VERSION themselves: read the version back out of the built .deb via dpkg-deb instead of relying on a shell variable that doesn't reliably survive to this stage. verify-query-packages.sh's zfs dependency-list assertion updated to match the new PACKAGE_DEPENDENCIES. --- .github/scripts/verify-query-packages.sh | 2 +- lib/common.sh | 27 ++++++++++++++++++++++-- packages/containerized-masking/config.sh | 7 ++++++ packages/delphix-rust/config.sh | 7 ++++++ packages/delphix-sso-app/config.sh | 7 ++++++ packages/masking/config.sh | 7 ++++++ packages/ptools/config.sh | 7 ++++++ packages/virtualization/config.sh | 2 +- packages/windows-connector/config.sh | 7 ++++++ packages/zfs/config.sh | 2 +- 10 files changed, 70 insertions(+), 5 deletions(-) diff --git a/.github/scripts/verify-query-packages.sh b/.github/scripts/verify-query-packages.sh index c4051171..b6f8f64b 100755 --- a/.github/scripts/verify-query-packages.sh +++ b/.github/scripts/verify-query-packages.sh @@ -64,7 +64,7 @@ test "$(TARGET_KERNEL_FLAVORS=generic ./query-packages.sh list linux-kernel)" == # Check that when a package has multiple dependencies they are printed in the # expected format. test "$(TARGET_KERNEL_FLAVORS="generic aws" ./query-packages.sh single -o dependencies zfs)" == \ - "linux-kernel-generic,linux-kernel-aws,delphix-rust,delphix-go,dwarves" + "linux-kernel-generic,linux-kernel-aws,delphix-rust,delphix-go,dwarves,syft,cyclonedx-cli" # Check that the output from the appliance list contains zfs and # delphix-platform packages. Note, we explicitly do not use grep -q here as it diff --git a/lib/common.sh b/lib/common.sh index d4ed860c..5b4b4aed 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1483,6 +1483,18 @@ function generate_sbom() { "'$WORKDIR/artifacts'" fi + # + # syft/cyclonedx-cli are build-host-only tooling (never shipped in + # any product package), fetched the same way any other linux-pkg + # build dependency is: declared in PACKAGE_DEPENDENCIES, populated + # into $DEPDIR by the "fetch_dependencies" stage, installed here. + # This mirrors appliance-build's build-ancillary-repository.sh, + # which installs the same two packages onto the appliance-build host + # for the same reason. + # + check_env DEPDIR + logmust install_pkgs "$DEPDIR"/syft/*.deb "$DEPDIR"/cyclonedx-cli/*.deb + local sbom_file="$WORKDIR/artifacts/$PACKAGE.cdx.json" local sbom_scratch_dir sbom_scratch_dir="$(logmust mktemp -d)" @@ -1496,11 +1508,22 @@ function generate_sbom() { # local deb sbom_parts=() for deb in "${debs[@]}"; do - local part + local part deb_version part="$sbom_scratch_dir/$(basename "$deb").cdx.json" + # + # Read the version back out of the .deb itself, rather than + # relying on $PACKAGE_VERSION: by this point in the build, + # $PACKAGE_VERSION may no longer hold the final, + # revision-suffixed version set_changelog() wrote into the + # package (e.g. it's empty for packages that don't set it + # explicitly themselves, unlike syft/cyclonedx-cli's own + # config.sh). dpkg-deb reads the actual, authoritative + # version of the artifact being scanned. + # + deb_version="$(dpkg-deb -f "$deb" Version)" logmust syft scan "$deb" \ --source-name "$PACKAGE" \ - --source-version "$PACKAGE_VERSION" \ + --source-version "$deb_version" \ -o "cyclonedx-json@1.6=$part" sbom_parts+=("$part") done diff --git a/packages/containerized-masking/config.sh b/packages/containerized-masking/config.sh index a53a234e..ccff8400 100644 --- a/packages/containerized-masking/config.sh +++ b/packages/containerized-masking/config.sh @@ -41,6 +41,13 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" PACKAGE_NEEDS_DOCKER="true" MEND_SCAN_APPLICABLE="true" SBOM_DEEP_SCAN="true" + +# +# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() +# (lib/common.sh) to scan this package's own .deb -- never shipped in the +# built package itself. +# +PACKAGE_DEPENDENCIES="syft cyclonedx-cli" MEND_SCAN_IMAGES="'delphix-masking-proxy', 'delphix-masking-database', 'delphix-masking-app'" SKIP_COPYRIGHTS_CHECK=true diff --git a/packages/delphix-rust/config.sh b/packages/delphix-rust/config.sh index c1613a2b..581eada4 100755 --- a/packages/delphix-rust/config.sh +++ b/packages/delphix-rust/config.sh @@ -19,6 +19,13 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-rust.git" SBOM_DEEP_SCAN="true" +# +# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() +# (lib/common.sh) to scan this package's own .deb -- never shipped in the +# built package itself. +# +PACKAGE_DEPENDENCIES="syft cyclonedx-cli" + function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 480f79d0..175f3d61 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -21,6 +21,13 @@ MEND_SCAN_APPLICABLE="true" MEND_SCAN_USING_SUDO="true" SBOM_DEEP_SCAN="true" +# +# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() +# (lib/common.sh) to scan this package's own .deb -- never shipped in the +# built package itself. +# +PACKAGE_DEPENDENCIES="syft cyclonedx-cli" + function prepare() { logmust install_pkgs openjdk-17-jdk-headless: } diff --git a/packages/masking/config.sh b/packages/masking/config.sh index 2229e02d..2c42f4f9 100644 --- a/packages/masking/config.sh +++ b/packages/masking/config.sh @@ -22,6 +22,13 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" MEND_SCAN_APPLICABLE="true" SBOM_DEEP_SCAN="true" +# +# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() +# (lib/common.sh) to scan this package's own .deb -- never shipped in the +# built package itself. +# +PACKAGE_DEPENDENCIES="syft cyclonedx-cli" + function prepare() { logmust read_list "$WORKDIR/repo/packaging/build-dependencies" logmust install_pkgs "${_RET_LIST[@]}" diff --git a/packages/ptools/config.sh b/packages/ptools/config.sh index 54aef875..e6fefae1 100644 --- a/packages/ptools/config.sh +++ b/packages/ptools/config.sh @@ -19,6 +19,13 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/ptools.git" SBOM_DEEP_SCAN="true" +# +# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() +# (lib/common.sh) to scan this package's own .deb -- never shipped in the +# built package itself. +# +PACKAGE_DEPENDENCIES="syft cyclonedx-cli" + function prepare() { logmust install_pkgs cargo devscripts } diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh index 171b111a..f960df04 100644 --- a/packages/virtualization/config.sh +++ b/packages/virtualization/config.sh @@ -19,7 +19,7 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" -PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks" +PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks syft cyclonedx-cli" # # The build packages a docker image (see '-Ddockerize=true' below), which needs # a docker daemon. The build container gets the host's daemon through its diff --git a/packages/windows-connector/config.sh b/packages/windows-connector/config.sh index 185083da..4b96c987 100644 --- a/packages/windows-connector/config.sh +++ b/packages/windows-connector/config.sh @@ -27,6 +27,13 @@ DEFAULT_PACKAGE_GIT_URL="none" SKIP_COPYRIGHTS_CHECK=true SBOM_DEEP_SCAN="true" +# +# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() +# (lib/common.sh) to scan this package's own .deb -- never shipped in the +# built package itself. +# +PACKAGE_DEPENDENCIES="syft cyclonedx-cli" + function fetch() { PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" logmust fetch_repo_from_git diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index de13dcb1..36187eb0 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -17,7 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/zfs.git" -PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust delphix-go dwarves" +PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust delphix-go dwarves syft cyclonedx-cli" SBOM_DEEP_SCAN="true" UPSTREAM_GIT_URL="https://github.com/openzfs/zfs.git" From bdabf2bbdf0da2e32e7040c1b6ffc3482e0fa37d Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Thu, 10 Sep 2026 15:53:32 +0530 Subject: [PATCH 04/17] DLPX-98872: suppress Syft's file-metadata component in per-package SBOM generate_sbom() didn't set SYFT_FILE_METADATA_SELECTION=none, so every sidecar carried an extra CycloneDX "file" component for the .deb itself (SHA-1/SHA-256 hashes plus the absolute build-workspace path) alongside the real pkg:deb component -- confirmed in delphix-sso-app.cdx.json from a real pre-push build. Same fix appliance-build's 95-generate-sbom.binary hook already applies, for the same reason: a flat, per-package pkg:deb document shouldn't carry per-file noise or leak the build machine's local path. --- lib/common.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/common.sh b/lib/common.sh index 5b4b4aed..ae5fdf2e 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1521,7 +1521,14 @@ function generate_sbom() { # version of the artifact being scanned. # deb_version="$(dpkg-deb -f "$deb" Version)" - logmust syft scan "$deb" \ + # + # SYFT_FILE_METADATA_SELECTION=none suppresses Syft's default + # per-file "file" component (with SHA-1/SHA-256 hashes and the + # absolute build-workspace path baked in) -- noise that doesn't + # belong in a per-package pkg:deb sidecar. Same reasoning as + # appliance-build's 95-generate-sbom.binary hook. + # + SYFT_FILE_METADATA_SELECTION=none logmust syft scan "$deb" \ --source-name "$PACKAGE" \ --source-version "$deb_version" \ -o "cyclonedx-json@1.6=$part" From 703a4de7d2978acedc11112bf54d2403b4bf1574 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Thu, 10 Sep 2026 17:27:14 +0530 Subject: [PATCH 05/17] DLPX-98872: pin cyclonedx-cli merge's --output-version to v1_6 Multi-.deb packages (delphix-rust: delphix-rust + delphix-rust-src; zfs similarly) go through the cyclonedx-cli merge path in generate_sbom(), which had no --output-version pinned. cyclonedx-cli merge defaults to the newest spec version it supports (1.7), not the 1.6 Syft emitted, so the merged document then failed the very next validate step: Incorrect schema version: expected 1.6 actual 1.7 Confirmed via a real pre-push delphix-rust build. Single-.deb packages never hit this, since the merge step is skipped entirely for them (a plain cp). Co-Authored-By: Claude Sonnet 5 --- lib/common.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/common.sh b/lib/common.sh index ae5fdf2e..844c3e40 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1538,9 +1538,18 @@ function generate_sbom() { if [[ ${#sbom_parts[@]} -eq 1 ]]; then logmust cp "${sbom_parts[0]}" "$sbom_file" else + # + # --output-version must be pinned explicitly: cyclonedx-cli + # merge defaults to the newest spec version it supports (1.7), + # not the 1.6 that Syft emitted and that the validate call + # below (and every other producer in this pipeline) targets -- + # left unset, the merged doc fails validation with "Incorrect + # schema version: expected 1.6 actual 1.7". + # logmust cyclonedx-cli merge \ --input-files "${sbom_parts[@]}" \ --output-format json \ + --output-version v1_6 \ --output-file "$sbom_file" fi logmust rm -rf "$sbom_scratch_dir" From 8923f539e3aa2fd2a9fe904d0ca3caaec6d6e669 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Thu, 10 Sep 2026 18:52:16 +0530 Subject: [PATCH 06/17] DLPX-98872: strict 1:1 .deb <-> BOM mapping, drop cyclonedx-cli merge Reviewer feedback: each .deb should have its own distinct BOM file with a common filename prefix, not one merged SBOM per source package. The prior approach scanned each .deb separately then merged them via cyclonedx-cli merge into a single .cdx.json -- correct per the top-level design doc's stated intent ("one package-level SBOM... associated with all of that package's debs"), but that assumption didn't survive review. generate_sbom() now writes one .deb.cdx.json per .deb, independently scanned and validated, with no merge step at all. This also removes the cyclonedx-cli merge --output-version bug entirely, since there's no merge left to have a version mismatch in. Also reconfirms compliance with the reviewer's other point: syft, cyclonedx-cli, and all linux-kernel-* packages are SBOM_DEEP_SCAN=false (unaffected by this change, verified separately) -- no SBOM is generated for build-host tooling or 3rd-party kernel forks. Design doc updated to match: architecture diagram, the "multiple .debs per package" resolution, and the implementation-status/follow-ups sections now describe the 1:1 mapping and the real-build testing already done, instead of the merge approach and its since-superseded open question about cyclonedx-cli's --output-version default. Co-Authored-By: Claude Sonnet 5 --- ...6-09-08-sbom-per-package-sidecar-design.md | 140 +++++++++++------- lib/common.sh | 76 ++++------ 2 files changed, 110 insertions(+), 106 deletions(-) diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md index 7e97bccc..eb32f98a 100644 --- a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md +++ b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md @@ -115,41 +115,29 @@ function generate_sbom() { "'$WORKDIR/artifacts'" fi - local sbom_file="$WORKDIR/artifacts/$PACKAGE.cdx.json" - local sbom_scratch_dir - sbom_scratch_dir="$(logmust mktemp -d)" - - # A package can emit more than one .deb from a single build (e.g. - # "zfs" splits into zfs-dkms, zfsutils-linux, etc.) -- scan each one - # into its own document, then merge into a single sidecar so there's - # exactly one .cdx.json per package, matching how - # appliance-build associates a sidecar to a package via COMPONENTS. - local deb sbom_parts=() + check_env DEPDIR + logmust install_pkgs "$DEPDIR"/syft/*.deb "$DEPDIR"/cyclonedx-cli/*.deb + + # One sidecar per .deb, not per package: a package that emits more + # than one .deb (e.g. "zfs" splits into zfs-dkms, zfsutils-linux, + # etc.) gets one .deb.cdx.json per .deb -- a strict + # 1:1 mapping, no merging across a package's .deb(s). + local deb for deb in "${debs[@]}"; do - local part - part="$sbom_scratch_dir/$(basename "$deb").cdx.json" - logmust syft scan "$deb" \ + local sbom_file deb_version + sbom_file="$WORKDIR/artifacts/$(basename "$deb").cdx.json" + deb_version="$(dpkg-deb -f "$deb" Version)" + SYFT_FILE_METADATA_SELECTION=none logmust syft scan "$deb" \ --source-name "$PACKAGE" \ - --source-version "$PACKAGE_VERSION" \ - -o "cyclonedx-json@1.6=$part" - sbom_parts+=("$part") + --source-version "$deb_version" \ + -o "cyclonedx-json@1.6=$sbom_file" + + logmust cyclonedx-cli validate \ + --input-file "$sbom_file" \ + --input-format json \ + --input-version v1_6 \ + --fail-on-errors done - - if [[ ${#sbom_parts[@]} -eq 1 ]]; then - logmust cp "${sbom_parts[0]}" "$sbom_file" - else - logmust cyclonedx-cli merge \ - --input-files "${sbom_parts[@]}" \ - --output-format json \ - --output-file "$sbom_file" - fi - logmust rm -rf "$sbom_scratch_dir" - - logmust cyclonedx-cli validate \ - --input-file "$sbom_file" \ - --input-format json \ - --input-version v1_6 \ - --fail-on-errors } ``` @@ -170,15 +158,26 @@ flag, means no package needs its own override for the baseline case (only per-ec overrides, out of scope for Phase 2, would override the function in a package's `config.sh`, the same way packages already override `build()`). -**Resolved — multiple `.deb`s per package:** handled above via per-deb scan + `cyclonedx-cli -merge` into one sidecar. Verify against a real `zfs` build before merging (see *Follow-ups*). - -**Open — Syft's `.deb`-scan support:** not yet verified against the real `syft`/ -`cyclonedx-cli` binaries in this environment (neither is installed here). Confirm the pinned -`delphix-syft` version (`SYFT_VERSION` in the `syft` packaging repo) supports scanning a -standalone `.deb` file as a source (Phase 1 only exercises Syft against a directory/rootfs, -not a single `.deb`). If unsupported, fall back to extracting the `.deb`'s `data.tar.*` into -a temp dir first and scanning that directory instead. +**Resolved — multiple `.deb`s per package:** a reviewer flagged that the original +merge-into-one-sidecar approach (below, kept here for history) doesn't give a clean 1:1 +mapping between a `.deb` and its BOM. Changed to: one `.deb.cdx.json` per +`.deb`, no merge, no `cyclonedx-cli merge` step at all — each `.deb`'s sidecar is fully +independent and filename-matched to it. This is a real divergence from the top-level +design doc (CP-13456), which called for "one package-level SBOM... associated with all +of that package's debs via `COMPONENTS`" — that assumption didn't survive review. Phase 3 +(`appliance-build`'s consumer, not yet built) will need to associate each `.deb` with its +own sidecar directly by filename, not go through a package-level indirection. + +*(For reference, the approach this replaced: scan each `.deb` into its own document, then +`cyclonedx-cli merge --output-version v1_6` them into a single `.cdx.json`. Two +real bugs were found and fixed while that was still in place, both still relevant to the +current code since they're not specific to the merge step: `syft`/`cyclonedx-cli` need to +be installed from `$DEPDIR` before use — see §4's `install_pkgs` line — and Syft's default +file-metadata component needs suppressing via `SYFT_FILE_METADATA_SELECTION=none`.)* + +**Resolved — Syft's `.deb`-scan support:** confirmed working against the real `syft`/ +`cyclonedx-cli` binaries via an actual pre-push build (`delphix-sso-app`, `delphix-rust`) — +`syft scan ` scans a standalone `.deb` file directly, no extraction needed. ### 5. Package classification @@ -225,10 +224,11 @@ per-package artifact directories at all. | stage build --> $WORKDIR/artifacts/*.deb | | stage store_build_info --> GIT_HASH, BUILD_INFO, ... | | stage generate_sbom --> [only if SBOM_DEEP_SCAN="true"] | -| syft scan -o cyclonedx-json@1.6 | -| (merge via cyclonedx-cli if >1 .deb) | -| --> $WORKDIR/artifacts/.cdx.json | -| cyclonedx-cli validate --fail-on-errors | +| for each .deb: | +| syft scan -o cyclonedx-json@1.6 | +| --> .deb.cdx.json | +| cyclonedx-cli validate --fail-on-errors| +| (strict 1:1 .deb <-> BOM, no merging) | | stage post_build_checks | +------------------------------------+--------------------------------------+ | @@ -238,14 +238,14 @@ per-package artifact directories at all. v +---------------------------------------------------------------------------+ | S3: combined-packages/packages// | -| *.deb | -| .cdx.json <-- NEW: sidecar, sits beside the deb(s) | +| .deb | +| .deb.cdx.json <-- NEW: one per .deb, same prefix | | GIT_HASH, BUILD_INFO, ... (unchanged) | +---------------------------------------------------------------------------+ | | (Phase 3, not in scope here: - | appliance-build fetches + merges - | this sidecar into the Phase 1 base) + | appliance-build fetches each .deb's + | own sidecar by filename match) v [out of scope for Phase 2] ``` @@ -265,15 +265,41 @@ per-package artifact directories at all. - [x] `SBOM_DEEP_SCAN` classification added to all 40 packages' `config.sh` (§5). - [x] `sbom-deep-scan` field added to `query-packages.sh` (§2). - [x] `generate_sbom()` added to `lib/common.sh` and wired as `stage generate_sbom` in - `buildpkg.sh` (§4), including the multi-`.deb` merge case. + `buildpkg.sh` (§4), producing a strict 1:1 `.deb` → `.cdx.json` mapping (§4). - [x] CI lint added (§3), landed in the same change as the classification pass. - [x] Verified locally: `verify-sbom-scan-flag.sh` and the existing `verify-query-packages.sh` both pass; `shellcheck`/`shfmt` clean on every touched file. - -## Follow-ups before merging - -1. Validate against a real `syft`/`cyclonedx-cli` build host: confirm `syft scan ` - works as expected (the open question in §4), and run an actual `zfs` build (multi-`.deb` - package) and `masking` build (single-`.deb`, Java+npm) end-to-end — confirm - `$WORKDIR/artifacts/.cdx.json` is produced, schema-valid, and lands in S3. +- [x] Verified against a real build host: `delphix-sso-app` (single-`.deb`) and + `delphix-rust` (multi-`.deb`) pre-push builds both produce valid, schema-checked + CycloneDX 1.6 sidecars in S3, correctly named per `.deb`. +- [x] Confirmed compliant with reviewer feedback: `syft`, `cyclonedx-cli`, and all + `linux-kernel-*` packages are `SBOM_DEEP_SCAN="false"` (no SBOM generated for + build-host tooling or 3rd-party kernel forks), and the `.deb` ↔ `.cdx.json` mapping + is now strictly 1:1 with a shared filename prefix. + +## Bugs found and fixed during real-build testing + +Not caught by CI — only surfaced by actually running builds: + +1. **`syft`/`cyclonedx-cli` command not found.** Nothing installed them into the + `linux-pkg` build container (Phase 1 only solved this for the `appliance-build` host). + Fixed by declaring them as `PACKAGE_DEPENDENCIES` on all 8 flagged packages and + installing from `$DEPDIR` in `generate_sbom()`. +2. **Blank `--source-version`.** `$PACKAGE_VERSION` doesn't reliably survive to the + `generate_sbom` stage for packages that don't set it themselves (unlike + `syft`/`cyclonedx-cli`'s own `config.sh`). Fixed by reading the version back out of the + built `.deb` via `dpkg-deb -f "$deb" Version`. +3. **Stray Syft "file" component**, carrying file hashes and an absolute build-workspace + path, polluting the sidecar. Fixed with `SYFT_FILE_METADATA_SELECTION=none`, matching + `appliance-build`'s `95-generate-sbom.binary` hook. +4. **`cyclonedx-cli merge` defaulted to spec version 1.7**, failing the subsequent + `--input-version v1_6` validate call. Moot now that merging was removed entirely per + the 1:1-mapping change above, but the same lesson applies to any future + `cyclonedx-cli` invocation: pin `--output-version` explicitly, don't rely on its + default. + +## Follow-ups + +1. Phase 3 (`appliance-build`'s consumer) needs to key off the 1:1 `.deb` ↔ `.cdx.json` + filename mapping established here, not a package-level `COMPONENTS` indirection. 2. File as sub-tasks of DLPX-98872. \ No newline at end of file diff --git a/lib/common.sh b/lib/common.sh index 844c3e40..0877d6e6 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1462,15 +1462,16 @@ function store_build_info() { } # -# Generate a per-package CycloneDX SBOM sidecar by running Syft against -# this package's built .deb(s). Only packages that bundle third-party -# composition (jars, npm, wheels, Rust crates, ...) opt in via -# SBOM_DEEP_SCAN="true" in their config.sh -- everything else is left as -# a flat pkg:deb component by appliance-build's base chroot scan, so a -# deb scan here would add nothing. The resulting .cdx.json is -# dropped in $WORKDIR/artifacts/ alongside the .deb(s), where it's picked -# up by the same S3 sync as every other build artifact -- no separate -# upload path needed. +# Generate a CycloneDX SBOM sidecar for each of this package's built +# .deb(s) by running Syft against it. Only packages that bundle +# third-party composition (jars, npm, wheels, Rust crates, ...) opt in +# via SBOM_DEEP_SCAN="true" in their config.sh -- everything else is +# left as a flat pkg:deb component by appliance-build's base chroot +# scan, so a deb scan here would add nothing. Each .cdx.json +# is dropped in $WORKDIR/artifacts/ alongside the .deb it describes -- +# a strict 1:1 mapping, no merging across a package's .deb(s) -- where +# it's picked up by the same S3 sync as every other build artifact, no +# separate upload path needed. # function generate_sbom() { if [[ "$SBOM_DEEP_SCAN" != "true" ]]; then @@ -1495,21 +1496,18 @@ function generate_sbom() { check_env DEPDIR logmust install_pkgs "$DEPDIR"/syft/*.deb "$DEPDIR"/cyclonedx-cli/*.deb - local sbom_file="$WORKDIR/artifacts/$PACKAGE.cdx.json" - local sbom_scratch_dir - sbom_scratch_dir="$(logmust mktemp -d)" - # - # A package can emit more than one .deb from a single build (e.g. - # "zfs" splits into zfs-dkms, zfsutils-linux, etc.) -- scan each one - # into its own document, then merge them into a single sidecar so - # there's exactly one .cdx.json per package, matching how - # appliance-build associates a sidecar to a package via COMPONENTS. + # One sidecar per .deb, not per package: a package that emits more + # than one .deb (e.g. "zfs" splits into zfs-dkms, zfsutils-linux, + # etc.) gets one .deb.cdx.json per .deb, each a + # standalone document scoped to that .deb alone. No merging across + # .debs -- keeps a strict 1:1 mapping between a .deb and its BOM, + # with the .deb's own filename as the common prefix. # - local deb sbom_parts=() + local deb for deb in "${debs[@]}"; do - local part deb_version - part="$sbom_scratch_dir/$(basename "$deb").cdx.json" + local sbom_file deb_version + sbom_file="$WORKDIR/artifacts/$(basename "$deb").cdx.json" # # Read the version back out of the .deb itself, rather than # relying on $PACKAGE_VERSION: by this point in the build, @@ -1525,40 +1523,20 @@ function generate_sbom() { # SYFT_FILE_METADATA_SELECTION=none suppresses Syft's default # per-file "file" component (with SHA-1/SHA-256 hashes and the # absolute build-workspace path baked in) -- noise that doesn't - # belong in a per-package pkg:deb sidecar. Same reasoning as + # belong in a per-deb sidecar. Same reasoning as # appliance-build's 95-generate-sbom.binary hook. # SYFT_FILE_METADATA_SELECTION=none logmust syft scan "$deb" \ --source-name "$PACKAGE" \ --source-version "$deb_version" \ - -o "cyclonedx-json@1.6=$part" - sbom_parts+=("$part") - done - - if [[ ${#sbom_parts[@]} -eq 1 ]]; then - logmust cp "${sbom_parts[0]}" "$sbom_file" - else - # - # --output-version must be pinned explicitly: cyclonedx-cli - # merge defaults to the newest spec version it supports (1.7), - # not the 1.6 that Syft emitted and that the validate call - # below (and every other producer in this pipeline) targets -- - # left unset, the merged doc fails validation with "Incorrect - # schema version: expected 1.6 actual 1.7". - # - logmust cyclonedx-cli merge \ - --input-files "${sbom_parts[@]}" \ - --output-format json \ - --output-version v1_6 \ - --output-file "$sbom_file" - fi - logmust rm -rf "$sbom_scratch_dir" + -o "cyclonedx-json@1.6=$sbom_file" - logmust cyclonedx-cli validate \ - --input-file "$sbom_file" \ - --input-format json \ - --input-version v1_6 \ - --fail-on-errors + logmust cyclonedx-cli validate \ + --input-file "$sbom_file" \ + --input-format json \ + --input-version v1_6 \ + --fail-on-errors + done } function set_secret_build_args() { From 3d769d77915ea679a3777e052d142c1585ce8419 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Fri, 11 Sep 2026 15:59:57 +0530 Subject: [PATCH 07/17] DLPX-98872: provision syft/cyclonedx-cli as generic build tooling Review feedback: generate_sbom() is a default hook, inherited unmodified by every SBOM_DEEP_SCAN package, so the tooling it runs is the hook's concern -- a package's config.sh should not have to declare it. Those tools belong with the rest of linux-pkg's generic build dependencies, installed prior to any package build. All 8 flagged packages therefore drop PACKAGE_DEPENDENCIES="syft cyclonedx-cli" (virtualization and zfs are restored to their original dependency lists), and setup.sh gains install_sbom_tools() alongside install_awscli/install_shfmt. Unlike everything else installed there, delphix-syft/delphix-cyclonedx-cli are Delphix-built .debs with no apt source -- the container's apt sources are only the Ubuntu primary mirror and the PPA secondary mirror -- so they are fetched from the same S3 location fetch_dependencies() uses and installed by path. apt rather than dpkg, since delphix-cyclonedx-cli has real libicu dependencies. The install is best-effort: setup.sh is package-agnostic, so it cannot skip itself when the package being built is syft or cyclonedx-cli. A hard failure on a missing artifact would break every build on a branch where neither has been published yet, including their own, so a missing artifact warns and continues. generate_sbom() checks for the tools itself and fails loudly, affecting only the builds that need them. Trade-off worth noting: with no PACKAGE_DEPENDENCIES entry, Jenkins's static dependency graph no longer knows these packages relate to syft/cyclonedx-cli, so build-order batching and the rebuild-dependents- on-syft-change cascade no longer apply to them. Documented in the spec, along with the rejected alternative of deriving the dependency from SBOM_DEEP_SCAN in load_package_config(). verify-query-packages.sh's zfs assertion is restored accordingly. Co-Authored-By: Claude Opus 5 --- .github/scripts/verify-query-packages.sh | 2 +- ...6-09-08-sbom-per-package-sidecar-design.md | 57 ++++++++++-- lib/common.sh | 86 ++++++++++++++++--- packages/containerized-masking/config.sh | 6 -- packages/delphix-rust/config.sh | 7 -- packages/delphix-sso-app/config.sh | 7 -- packages/masking/config.sh | 7 -- packages/ptools/config.sh | 7 -- packages/virtualization/config.sh | 2 +- packages/windows-connector/config.sh | 7 -- packages/zfs/config.sh | 2 +- setup.sh | 6 ++ 12 files changed, 136 insertions(+), 60 deletions(-) diff --git a/.github/scripts/verify-query-packages.sh b/.github/scripts/verify-query-packages.sh index b6f8f64b..c4051171 100755 --- a/.github/scripts/verify-query-packages.sh +++ b/.github/scripts/verify-query-packages.sh @@ -64,7 +64,7 @@ test "$(TARGET_KERNEL_FLAVORS=generic ./query-packages.sh list linux-kernel)" == # Check that when a package has multiple dependencies they are printed in the # expected format. test "$(TARGET_KERNEL_FLAVORS="generic aws" ./query-packages.sh single -o dependencies zfs)" == \ - "linux-kernel-generic,linux-kernel-aws,delphix-rust,delphix-go,dwarves,syft,cyclonedx-cli" + "linux-kernel-generic,linux-kernel-aws,delphix-rust,delphix-go,dwarves" # Check that the output from the appliance list contains zfs and # delphix-platform packages. Note, we explicitly do not use grep -q here as it diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md index eb32f98a..2538ff65 100644 --- a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md +++ b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md @@ -115,8 +115,14 @@ function generate_sbom() { "'$WORKDIR/artifacts'" fi - check_env DEPDIR - logmust install_pkgs "$DEPDIR"/syft/*.deb "$DEPDIR"/cyclonedx-cli/*.deb + # syft/cyclonedx-cli come from setup.sh's install_sbom_tools(), not + # from anything this package declares (see below). That install is + # best-effort, so check here rather than letting "command not + # found" surface mid-scan. + local tool + for tool in syft cyclonedx-cli; do + command -v "$tool" >/dev/null || die "'$tool' is not installed..." + done # One sidecar per .deb, not per package: a package that emits more # than one .deb (e.g. "zfs" splits into zfs-dkms, zfsutils-linux, @@ -171,8 +177,8 @@ own sidecar directly by filename, not go through a package-level indirection. *(For reference, the approach this replaced: scan each `.deb` into its own document, then `cyclonedx-cli merge --output-version v1_6` them into a single `.cdx.json`. Two real bugs were found and fixed while that was still in place, both still relevant to the -current code since they're not specific to the merge step: `syft`/`cyclonedx-cli` need to -be installed from `$DEPDIR` before use — see §4's `install_pkgs` line — and Syft's default +current code since they're not specific to the merge step: `syft`/`cyclonedx-cli` have to +actually be installed in the build container before use — see §6 — and Syft's default file-metadata component needs suppressing via `SYFT_FILE_METADATA_SELECTION=none`.)* **Resolved — Syft's `.deb`-scan support:** confirmed working against the real `syft`/ @@ -203,6 +209,44 @@ the top-level design's Tooling section separately calls out a possible Go overri binary-build-info cataloger that may already cover it) — revisit under Phase 4's evaluation if gaps are found. +### 6. Provisioning syft/cyclonedx-cli — generic build tooling, not a package dependency + +`generate_sbom()` is a **default hook**: defined once in `lib/common.sh` and inherited +unmodified by all 8 flagged packages (none of them override it, unlike e.g. `zfs`'s own +`build()`). The tooling it runs is therefore the hook's concern, not its callers' — so no +package's `config.sh` declares `syft`/`cyclonedx-cli` anywhere. They are installed with the +rest of the generic build tooling in `setup.sh`, which runs before every package build: + +```bash +logmust install_awscli +logmust install_sbom_tools # must follow install_awscli -- fetches from S3 +logmust install_shfmt +``` + +`install_sbom_tools()` (`lib/common.sh`) is modelled on the existing `install_awscli()`/ +`install_shfmt()` non-apt installers, with one difference: `delphix-syft`/ +`delphix-cyclonedx-cli` are Delphix-built `.deb`s with no apt source (the container's apt +sources are only the Ubuntu primary mirror plus the PPA secondary mirror), so they're +fetched from the same S3 location `fetch_dependencies()` uses — +`get_package_dependency_s3_url()` → `aws s3 cp --recursive` → `apt-get install ` +(apt, not dpkg, because `delphix-cyclonedx-cli` has real `libicu` dependencies). + +**Best-effort by design.** `setup.sh` is package-agnostic — it has no idea which package is +about to be built — so it cannot skip itself when the package being built *is* `syft` or +`cyclonedx-cli`. Hard-failing on a missing artifact would therefore break *every* build on +a branch where those two haven't been published yet, including their own. Instead a missing +artifact warns and continues, and `generate_sbom()` checks for the tools itself and fails +loudly — so only the builds that actually need them are affected. + +**Rejected alternative:** deriving `PACKAGE_DEPENDENCIES += "syft cyclonedx-cli"` from +`SBOM_DEEP_SCAN` in `load_package_config()`. That also keeps it out of each `config.sh`, +is scoped to just the 8 packages, and keeps the relationship visible to Jenkins's static +dependency graph (so `syft` batches before its dependents and a `syft` rebuild cascades to +them). It was implemented and working, but sits in the per-package dependency layer rather +than the generic installed-prior layer, which is not what review asked for. Noting the +trade-off explicitly: with the `setup.sh` approach, Jenkins no longer knows these packages +relate to `syft`/`cyclonedx-cli`, so that batching/rebuild-cascade behaviour is lost. + ### S3 upload — no new plumbing needed Confirmed by reading `devops-gate/jenkins/jobs/pipelines/linux_pkg_build_package.groovy`'s @@ -283,8 +327,9 @@ Not caught by CI — only surfaced by actually running builds: 1. **`syft`/`cyclonedx-cli` command not found.** Nothing installed them into the `linux-pkg` build container (Phase 1 only solved this for the `appliance-build` host). - Fixed by declaring them as `PACKAGE_DEPENDENCIES` on all 8 flagged packages and - installing from `$DEPDIR` in `generate_sbom()`. + Fixed by installing them from `setup.sh` with the rest of the generic build tooling + (§6). An interim fix declared them as `PACKAGE_DEPENDENCIES` on each flagged package + instead; that was replaced per review feedback. 2. **Blank `--source-version`.** `$PACKAGE_VERSION` doesn't reliably survive to the `generate_sbom` stage for packages that don't set it themselves (unlike `syft`/`cyclonedx-cli`'s own `config.sh`). Fixed by reading the version back out of the diff --git a/lib/common.sh b/lib/common.sh index 0877d6e6..4d65ab99 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -619,6 +619,66 @@ function install_shfmt() { echo "shfmt version $(shfmt -version) is installed." } +# +# Install the tooling generate_sbom() needs. That is a default hook, shared +# unmodified by every package that sets SBOM_DEEP_SCAN, so the tools it runs +# belong with the rest of the generic build tooling installed here rather than +# being declared as a dependency by each of those packages individually. +# +# Unlike everything else installed from setup.sh these are Delphix-built +# packages with no apt source, so they are fetched from the same S3 location +# fetch_dependencies() pulls a package's dependencies from, then installed by +# path. apt rather than dpkg, because delphix-cyclonedx-cli has real +# dependencies (libicu and the usual shared libraries) that dpkg will not +# resolve. +# +# Best-effort by design: this runs before *every* package build, including the +# builds of syft and cyclonedx-cli themselves, and on a branch where neither +# has been published yet there is nothing to fetch. Failing hard would break +# every build on such a branch rather than just SBOM generation, so a missing +# artifact warns and moves on; generate_sbom() checks for the tools itself and +# fails loudly, for the only builds that actually need them. +# +function install_sbom_tools() { + local pkg s3url tmpdir + local debs=() + + tmpdir="$(mktemp -d)" || die "Failed to create a temporary directory" + + for pkg in syft cyclonedx-cli; do + # + # Run in a command substitution so that a failure to resolve + # the URL (get_package_dependency_s3_url dies when a package + # has no published artifacts) leaves $s3url empty here instead + # of aborting setup. + # + s3url="$( + get_package_dependency_s3_url "$pkg" >/dev/null 2>&1 + echo "$_RET" + )" + if [[ -z "$s3url" ]]; then + echo "WARNING: no published artifacts found for '$pkg';" \ + "skipping the SBOM tooling install. Builds of" \ + "packages that set SBOM_DEEP_SCAN will fail until" \ + "'$pkg' has been built for this branch." + logmust rm -rf "$tmpdir" + return 0 + fi + + [[ "$s3url" != */ ]] && s3url="$s3url/" + logmust mkdir -p "$tmpdir/$pkg" + logmust aws s3 cp --only-show-errors --recursive \ + "$s3url" "$tmpdir/$pkg/" + done + + debs=("$tmpdir"/*/*.deb) + [[ -e "${debs[0]}" ]] || + die "No .deb found in the fetched syft/cyclonedx-cli artifacts" + + logmust install_pkgs "${debs[@]}" + logmust rm -rf "$tmpdir" +} + # # Install kernel headers packages for all target kernels. # The kernel packages are fetched from S3. @@ -1485,16 +1545,22 @@ function generate_sbom() { fi # - # syft/cyclonedx-cli are build-host-only tooling (never shipped in - # any product package), fetched the same way any other linux-pkg - # build dependency is: declared in PACKAGE_DEPENDENCIES, populated - # into $DEPDIR by the "fetch_dependencies" stage, installed here. - # This mirrors appliance-build's build-ancillary-repository.sh, - # which installs the same two packages onto the appliance-build host - # for the same reason. - # - check_env DEPDIR - logmust install_pkgs "$DEPDIR"/syft/*.deb "$DEPDIR"/cyclonedx-cli/*.deb + # syft/cyclonedx-cli are part of the generic build tooling installed + # by setup.sh (install_sbom_tools()) before any package is built, + # rather than something each SBOM_DEEP_SCAN package declares for + # itself -- this is a default hook, so what it needs is its own + # concern, not its callers'. That install is best-effort, so check + # here rather than letting "command not found" surface from the + # middle of a scan. + # + local tool + for tool in syft cyclonedx-cli; do + command -v "$tool" >/dev/null || + die "'$tool' is not installed, so no SBOM can be" \ + "generated for '$PACKAGE'. It is provisioned by" \ + "install_sbom_tools() in setup.sh; check that" \ + "run's output for why it was skipped." + done # # One sidecar per .deb, not per package: a package that emits more diff --git a/packages/containerized-masking/config.sh b/packages/containerized-masking/config.sh index ccff8400..d1cff120 100644 --- a/packages/containerized-masking/config.sh +++ b/packages/containerized-masking/config.sh @@ -42,12 +42,6 @@ PACKAGE_NEEDS_DOCKER="true" MEND_SCAN_APPLICABLE="true" SBOM_DEEP_SCAN="true" -# -# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() -# (lib/common.sh) to scan this package's own .deb -- never shipped in the -# built package itself. -# -PACKAGE_DEPENDENCIES="syft cyclonedx-cli" MEND_SCAN_IMAGES="'delphix-masking-proxy', 'delphix-masking-database', 'delphix-masking-app'" SKIP_COPYRIGHTS_CHECK=true diff --git a/packages/delphix-rust/config.sh b/packages/delphix-rust/config.sh index 581eada4..c1613a2b 100755 --- a/packages/delphix-rust/config.sh +++ b/packages/delphix-rust/config.sh @@ -19,13 +19,6 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/delphix-rust.git" SBOM_DEEP_SCAN="true" -# -# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() -# (lib/common.sh) to scan this package's own .deb -- never shipped in the -# built package itself. -# -PACKAGE_DEPENDENCIES="syft cyclonedx-cli" - function build() { logmust mkdir -p "$WORKDIR/repo" diff --git a/packages/delphix-sso-app/config.sh b/packages/delphix-sso-app/config.sh index 175f3d61..480f79d0 100644 --- a/packages/delphix-sso-app/config.sh +++ b/packages/delphix-sso-app/config.sh @@ -21,13 +21,6 @@ MEND_SCAN_APPLICABLE="true" MEND_SCAN_USING_SUDO="true" SBOM_DEEP_SCAN="true" -# -# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() -# (lib/common.sh) to scan this package's own .deb -- never shipped in the -# built package itself. -# -PACKAGE_DEPENDENCIES="syft cyclonedx-cli" - function prepare() { logmust install_pkgs openjdk-17-jdk-headless: } diff --git a/packages/masking/config.sh b/packages/masking/config.sh index 2c42f4f9..2229e02d 100644 --- a/packages/masking/config.sh +++ b/packages/masking/config.sh @@ -22,13 +22,6 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dms-core-gate.git" MEND_SCAN_APPLICABLE="true" SBOM_DEEP_SCAN="true" -# -# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() -# (lib/common.sh) to scan this package's own .deb -- never shipped in the -# built package itself. -# -PACKAGE_DEPENDENCIES="syft cyclonedx-cli" - function prepare() { logmust read_list "$WORKDIR/repo/packaging/build-dependencies" logmust install_pkgs "${_RET_LIST[@]}" diff --git a/packages/ptools/config.sh b/packages/ptools/config.sh index e6fefae1..54aef875 100644 --- a/packages/ptools/config.sh +++ b/packages/ptools/config.sh @@ -19,13 +19,6 @@ DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/ptools.git" SBOM_DEEP_SCAN="true" -# -# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() -# (lib/common.sh) to scan this package's own .deb -- never shipped in the -# built package itself. -# -PACKAGE_DEPENDENCIES="syft cyclonedx-cli" - function prepare() { logmust install_pkgs cargo devscripts } diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh index f960df04..171b111a 100644 --- a/packages/virtualization/config.sh +++ b/packages/virtualization/config.sh @@ -19,7 +19,7 @@ source "$PWD/lib/common.sh" DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" -PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks syft cyclonedx-cli" +PACKAGE_DEPENDENCIES="crypt-blowfish host-jdks" # # The build packages a docker image (see '-Ddockerize=true' below), which needs # a docker daemon. The build container gets the host's daemon through its diff --git a/packages/windows-connector/config.sh b/packages/windows-connector/config.sh index 4b96c987..185083da 100644 --- a/packages/windows-connector/config.sh +++ b/packages/windows-connector/config.sh @@ -27,13 +27,6 @@ DEFAULT_PACKAGE_GIT_URL="none" SKIP_COPYRIGHTS_CHECK=true SBOM_DEEP_SCAN="true" -# -# syft/cyclonedx-cli are build-host-only tooling needed by generate_sbom() -# (lib/common.sh) to scan this package's own .deb -- never shipped in the -# built package itself. -# -PACKAGE_DEPENDENCIES="syft cyclonedx-cli" - function fetch() { PACKAGE_GIT_URL="https://github.com/delphix/dlpx-app-gate.git" logmust fetch_repo_from_git diff --git a/packages/zfs/config.sh b/packages/zfs/config.sh index 36187eb0..de13dcb1 100644 --- a/packages/zfs/config.sh +++ b/packages/zfs/config.sh @@ -17,7 +17,7 @@ # shellcheck disable=SC2034 DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/zfs.git" -PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust delphix-go dwarves syft cyclonedx-cli" +PACKAGE_DEPENDENCIES="@linux-kernel delphix-rust delphix-go dwarves" SBOM_DEEP_SCAN="true" UPSTREAM_GIT_URL="https://github.com/openzfs/zfs.git" diff --git a/setup.sh b/setup.sh index cb8248b5..c0a00815 100755 --- a/setup.sh +++ b/setup.sh @@ -263,6 +263,12 @@ function install_awscli() { logmust install_awscli +# +# Must follow install_awscli: the syft/cyclonedx-cli packages it installs are +# fetched from S3, so the aws CLI has to be present first. +# +logmust install_sbom_tools + logmust install_shfmt logmust git config --global user.email "eng@delphix.com" From 7934898c7f4e7b1ec9f93e340a49867726d4a79d Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Fri, 11 Sep 2026 16:00:05 +0530 Subject: [PATCH 08/17] DLPX-98872: explain the SBOM_DEEP_SCAN decision in the lint failure Review feedback: the lint told you that a package was unclassified but not how to classify it, leaving whoever adds the next package to go read lib/common.sh or the spec to find out. State the rule where it is actually needed -- 1st-party packages set "true" so the third-party components they package internally land in the product's aggregate SBOM; 3rd-party forks of Debian packages and anything not shipped in a product set "false", since appliance-build's image-level scan already covers those as a flat pkg:deb component. Co-Authored-By: Claude Opus 5 --- .github/scripts/verify-sbom-scan-flag.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/scripts/verify-sbom-scan-flag.sh b/.github/scripts/verify-sbom-scan-flag.sh index 94691ddd..7b97e587 100755 --- a/.github/scripts/verify-sbom-scan-flag.sh +++ b/.github/scripts/verify-sbom-scan-flag.sh @@ -18,6 +18,16 @@ if [[ -n "$unclassified" ]]; then echo "The following packages have not set SBOM_DEEP_SCAN (\"true\" or" \ "\"false\") in their config.sh:" echo "$unclassified" + echo + echo "Set it to \"true\" for 1st-party packages, so that the" \ + "third-party components they package internally (jars, npm" \ + "modules, Rust crates, ...) are included in the product's" \ + "aggregate SBOM." + echo "Set it to \"false\" for 3rd-party forks of Debian packages, and" \ + "for packages that are not included in a shipping product:" \ + "those are already covered as a flat pkg:deb component by" \ + "appliance-build's image-level scan, so a deep scan here would" \ + "add nothing." exit 1 fi From e5921bbf63deba4bc9d3437a11f29ef3971c8473 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Mon, 14 Sep 2026 13:42:13 +0530 Subject: [PATCH 09/17] DLPX-98872: extract each .deb before scanning it The sidecars produced so far were valid but empty. "syft scan " only identifies the archive -- its deb-archive-cataloger reads the control metadata and emits a single pkg:deb component, never descending into data.tar.* -- so none of the bundled jars, wheels or modules these sidecars exist to capture were ever found. A real delphix-virtualization sidecar contained exactly one component for a 1.17 GB Java + Angular application: no more than appliance-build's image-level dpkg scan already provides for free, and nothing for a vulnerability scanner to match against the bundled third-party code. Every flagged package was affected. This was prescribed in the spec as the fallback for exactly this case, and had been closed out as "confirmed working" purely because a build produced a schema-valid document; cyclonedx-cli validate passes an empty-but-well-formed BOM quite happily, so nothing failed. Only reading the output revealed it. generate_sbom() now runs dpkg-deb -x into a temp directory and scans that with "syft scan dir:...", using Syft's full catalogers rather than the dpkg-only restriction appliance-build's image-level hook applies. That restriction exists because a jar found somewhere on a whole rootfs cannot be attributed to the package that placed it; inside one package's own extracted payload everything found belongs to that package by construction. Shape change worth noting: the scanned source is now a directory, so the sidecar no longer carries a pkg:deb component for the package itself -- its identity is in metadata.component, and the flat pkg:deb entry still comes from the image-level scan. Phase 3 associates a sidecar with its .deb by filename, so nothing depends on it. The bundled npm frontend remains absent for masking/virtualization: those ship the built, minified Angular bundle with no package.json, so extraction cannot help. That is the known blind spot the top-level design flagged for Phase 4's method evaluation. Co-Authored-By: Claude Opus 5 --- ...6-09-08-sbom-per-package-sidecar-design.md | 62 ++++++++++++++++--- lib/common.sh | 35 +++++++++-- 2 files changed, 84 insertions(+), 13 deletions(-) diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md index 2538ff65..d8bd639e 100644 --- a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md +++ b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md @@ -130,14 +130,21 @@ function generate_sbom() { # 1:1 mapping, no merging across a package's .deb(s). local deb for deb in "${debs[@]}"; do - local sbom_file deb_version + local sbom_file deb_version extract_dir sbom_file="$WORKDIR/artifacts/$(basename "$deb").cdx.json" deb_version="$(dpkg-deb -f "$deb" Version)" - SYFT_FILE_METADATA_SELECTION=none logmust syft scan "$deb" \ + + # Scan the extracted payload, not the .deb -- see below. + extract_dir="$(logmust mktemp -d)" + logmust dpkg-deb -x "$deb" "$extract_dir" + + SYFT_FILE_METADATA_SELECTION=none logmust syft scan "dir:$extract_dir" \ --source-name "$PACKAGE" \ --source-version "$deb_version" \ -o "cyclonedx-json@1.6=$sbom_file" + logmust rm -rf "$extract_dir" + logmust cyclonedx-cli validate \ --input-file "$sbom_file" \ --input-format json \ @@ -181,9 +188,37 @@ current code since they're not specific to the merge step: `syft`/`cyclonedx-cli actually be installed in the build container before use — see §6 — and Syft's default file-metadata component needs suppressing via `SYFT_FILE_METADATA_SELECTION=none`.)* -**Resolved — Syft's `.deb`-scan support:** confirmed working against the real `syft`/ -`cyclonedx-cli` binaries via an actual pre-push build (`delphix-sso-app`, `delphix-rust`) — -`syft scan ` scans a standalone `.deb` file directly, no extraction needed. +**Resolved — Syft's `.deb`-scan support: the `.deb` must be extracted first.** This was +initially, and wrongly, closed as "confirmed working" after a `delphix-sso-app` pre-push +build produced a schema-valid document. It was valid but empty: `syft scan ` +only *identifies* the archive — its `deb-archive-cataloger` reads the control metadata and +emits a single `pkg:deb` component, never descending into `data.tar.*`. A +`delphix-virtualization` sidecar from a real build contained exactly **one** component for +a 1.17 GB Java + Angular application, i.e. no more information than the image-level dpkg +scan already provides for free, and nothing at all for a vulnerability scanner to match +against the bundled jars. + +`generate_sbom()` therefore does what this spec originally prescribed as the fallback: +`dpkg-deb -x` the `.deb` into a temp directory and scan that with `syft scan dir:...`, +with Syft's **full** catalogers rather than `--select-catalogers dpkg`. The dpkg-only +restriction in `appliance-build`'s image-level hook exists because a jar found somewhere +on a whole rootfs cannot be attributed to the package that placed it; inside a single +package's own extracted payload everything found belongs to that package by construction, +which is the whole reason for scanning here instead of at image level. + +Note the shape change this brings: because the scanned source is now a directory rather +than a `.deb`, the sidecar no longer carries a `pkg:deb` component for the package itself +— the package's identity lives in `metadata.component` (via `--source-name`/ +`--source-version`), and the flat `pkg:deb` entry continues to come from Phase 1's +image-level scan. Phase 3 associates a sidecar with its `.deb` by filename (§4), so +nothing depends on that component being present here. + +**Still open — the bundled npm frontend.** Extraction fixes the jars, but +`masking`/`virtualization` ship the *built*, minified Angular bundle with no +`package.json`/`node_modules`, so Syft has nothing to catalog for the frontend regardless +of extraction. That is the known blind spot the top-level design flagged for Phase 4's +method evaluation (per-language tooling vs. Xray vs. Mend), not something extraction can +address. ### 5. Package classification @@ -269,7 +304,9 @@ per-package artifact directories at all. | stage store_build_info --> GIT_HASH, BUILD_INFO, ... | | stage generate_sbom --> [only if SBOM_DEEP_SCAN="true"] | | for each .deb: | -| syft scan -o cyclonedx-json@1.6 | +| dpkg-deb -x | +| syft scan dir: (full | +| catalogers) -o cyclonedx-json@1.6 | | --> .deb.cdx.json | | cyclonedx-cli validate --fail-on-errors| | (strict 1:1 .deb <-> BOM, no merging) | @@ -323,8 +360,17 @@ per-package artifact directories at all. ## Bugs found and fixed during real-build testing -Not caught by CI — only surfaced by actually running builds: - +Not caught by CI — only surfaced by actually running builds, and in one case only by +inspecting the *contents* of a produced SBOM rather than its exit status: + +0. **Valid but empty SBOMs — the most serious of these.** `syft scan ` identifies the + archive without descending into it, so every sidecar contained a single `pkg:deb` + component and nothing else: no jars, no wheels, no modules. A `delphix-virtualization` + sidecar had one component for a 1.17 GB application. Every flagged package was affected. + Caught when the output was actually read, not when the build passed — the build had been + passing the whole time, and `cyclonedx-cli validate` passes an empty-but-well-formed + document quite happily. Fixed by extracting with `dpkg-deb -x` and scanning the + extracted tree with full catalogers (§4). 1. **`syft`/`cyclonedx-cli` command not found.** Nothing installed them into the `linux-pkg` build container (Phase 1 only solved this for the `appliance-build` host). Fixed by installing them from `setup.sh` with the rest of the generic build tooling diff --git a/lib/common.sh b/lib/common.sh index 4d65ab99..8e51a77a 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -1572,7 +1572,7 @@ function generate_sbom() { # local deb for deb in "${debs[@]}"; do - local sbom_file deb_version + local sbom_file deb_version extract_dir sbom_file="$WORKDIR/artifacts/$(basename "$deb").cdx.json" # # Read the version back out of the .deb itself, rather than @@ -1585,18 +1585,43 @@ function generate_sbom() { # version of the artifact being scanned. # deb_version="$(dpkg-deb -f "$deb" Version)" + + # + # Scan the .deb's extracted payload rather than the .deb file. + # "syft scan .deb" only identifies the archive: its + # deb-archive-cataloger reads the control metadata and emits a + # single pkg:deb component, never descending into data.tar.*, + # so none of the bundled jars/wheels/modules this sidecar + # exists to capture are found. That produced valid but empty + # documents -- one component for a 1.2GB application -- which + # is no more than appliance-build's image-level dpkg scan + # already gives for free. Extracting first is what the design + # spec prescribed for exactly this case. + # + extract_dir="$(logmust mktemp -d)" + logmust dpkg-deb -x "$deb" "$extract_dir" + + # + # Full catalogers here, deliberately unlike appliance-build's + # 95-generate-sbom.binary hook, which restricts to dpkg. That + # restriction exists because a jar sitting somewhere on a whole + # rootfs cannot be attributed to the package that placed it; + # within a single package's own extracted payload everything + # found belongs to that package by construction, which is the + # entire point of scanning here rather than at image level. # # SYFT_FILE_METADATA_SELECTION=none suppresses Syft's default # per-file "file" component (with SHA-1/SHA-256 hashes and the - # absolute build-workspace path baked in) -- noise that doesn't - # belong in a per-deb sidecar. Same reasoning as - # appliance-build's 95-generate-sbom.binary hook. + # scanned path baked in) -- with a whole extracted payload to + # walk that would otherwise emit an entry per file. # - SYFT_FILE_METADATA_SELECTION=none logmust syft scan "$deb" \ + SYFT_FILE_METADATA_SELECTION=none logmust syft scan "dir:$extract_dir" \ --source-name "$PACKAGE" \ --source-version "$deb_version" \ -o "cyclonedx-json@1.6=$sbom_file" + logmust rm -rf "$extract_dir" + logmust cyclonedx-cli validate \ --input-file "$sbom_file" \ --input-format json \ From 06fedec5c7af2a376ff45cabfa446043de0489a8 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Mon, 14 Sep 2026 13:50:44 +0530 Subject: [PATCH 10/17] DLPX-98872: enable SBOM deep scan for performance-diagnostics performance-diagnostics is built from Delphix source (github.com/delphix/performance-diagnostics), so by the same rule the lint now states -- 1st-party packages set "true" so that the third-party components they package internally are included in the product's aggregate SBOM -- it belongs in the deep-scan set. The initial classification pass grouped it with the single-ecosystem tools that need nothing beyond the image-level flat pkg:deb component, which was wrong. Brings the flagged set to 9 of 40 packages. Spec's classification table and package lists updated to match; the remaining hardcoded counts of "8 flagged packages" in prose are replaced with count-agnostic wording so they do not go stale again. Co-Authored-By: Claude Opus 5 --- .../2026-09-08-sbom-per-package-sidecar-design.md | 10 ++++++---- packages/performance-diagnostics/config.sh | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md index d8bd639e..6624f09b 100644 --- a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md +++ b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md @@ -33,7 +33,8 @@ dependency to add** — it only needs to *invoke* `syft`, which is already avail The Phase 1 base scan lists every installed `.deb` as a flat `pkg:deb` component. That is correct and sufficient for 3rd-party Debian packages, but wrong for Delphix's own first-party packages: `masking`, `virtualization`, `delphix-sso-app`, `containerized-masking`, -`windows-connector`, `zfs`, `ptools`, and `delphix-rust` all bundle third-party components +`windows-connector`, `zfs`, `ptools`, `delphix-rust`, and `performance-diagnostics` all +bundle third-party components across ecosystems (jars, npm, wheels, Rust crates) that `dpkg` cannot see — dpkg only knows the files *it* placed, not what's vendored inside a jar or statically linked into a Rust binary. Those components are invisible to a vulnerability scanner today. @@ -235,8 +236,9 @@ strictly "is it 1st-party" — see the `zfs` case), `SBOM_DEEP_SCAN="true"` is s | `zfs` | OpenZFS fork bundling Delphix's Rust object agent (crates invisible to dpkg) | | `ptools` | Rust | | `delphix-rust` | Rust | +| `performance-diagnostics` | 1st-party Delphix package built from its own source | -The remaining 32 packages (kernel packages, `misc-debs`, `syft`/`cyclonedx-cli` themselves, +The remaining 31 packages (kernel packages, `misc-debs`, `syft`/`cyclonedx-cli` themselves, etc.) get `SBOM_DEEP_SCAN="false"` — plain 3rd-party forks or single-ecosystem tools already fully represented by the Phase 1 flat `pkg:deb` component. `delphix-go` is a judgment call: the top-level design's Tooling section separately calls out a possible Go override @@ -247,7 +249,7 @@ if gaps are found. ### 6. Provisioning syft/cyclonedx-cli — generic build tooling, not a package dependency `generate_sbom()` is a **default hook**: defined once in `lib/common.sh` and inherited -unmodified by all 8 flagged packages (none of them override it, unlike e.g. `zfs`'s own +unmodified by every flagged package (none of them override it, unlike e.g. `zfs`'s own `build()`). The tooling it runs is therefore the hook's concern, not its callers' — so no package's `config.sh` declares `syft`/`cyclonedx-cli` anywhere. They are installed with the rest of the generic build tooling in `setup.sh`, which runs before every package build: @@ -275,7 +277,7 @@ loudly — so only the builds that actually need them are affected. **Rejected alternative:** deriving `PACKAGE_DEPENDENCIES += "syft cyclonedx-cli"` from `SBOM_DEEP_SCAN` in `load_package_config()`. That also keeps it out of each `config.sh`, -is scoped to just the 8 packages, and keeps the relationship visible to Jenkins's static +is scoped to just the flagged packages, and keeps the relationship visible to Jenkins's static dependency graph (so `syft` batches before its dependents and a `syft` rebuild cascades to them). It was implemented and working, but sits in the per-package dependency layer rather than the generic installed-prior layer, which is not what review asked for. Noting the diff --git a/packages/performance-diagnostics/config.sh b/packages/performance-diagnostics/config.sh index 6d666c36..f9e094f1 100644 --- a/packages/performance-diagnostics/config.sh +++ b/packages/performance-diagnostics/config.sh @@ -18,7 +18,7 @@ # DEFAULT_PACKAGE_GIT_URL="https://github.com/delphix/performance-diagnostics.git" -SBOM_DEEP_SCAN="false" +SBOM_DEEP_SCAN="true" function prepare() { logmust install_build_deps_from_control_file From 751943f3e2e7ae1bbd5d17cd8668efb294aae1df Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Fri, 18 Sep 2026 14:53:55 +0530 Subject: [PATCH 11/17] DLPX-98872: sanitize the CycloneDX sidecar before publishing it Raw Syft output is not publishable as-is. On delphix-virtualization it was 4.7 MB for 416 distinct components, three quarters of it repetition, and it recorded 470 directories under /opt/delphix -- including jar-internal paths such as resources.war:WEB-INF/lib/ST4-4.3.4.jar -- in a document destined for the customer-facing BOM. sanitize_sbom() applies resources/sanitize-sbom.jq to each sidecar between the Syft scan and the validate call, so what is validated is what is published. It drops the dependencies graph, drops every property except syft:cpe23, and merge-dedupes components. The dependencies graph went beyond sparse: for a directory scan Syft emits jar containment rather than resolved dependency edges, it covered 501 of 1693 components with no compositions element declaring it incomplete, and 90 of its 568 edges already pointed at bom-refs that do not exist -- the per-file components suppressed by SYFT_FILE_METADATA_SELECTION=none. De-duplication would have orphaned many more. Duplicates are merged rather than reduced to the first occurrence. Syft does not detect the same metadata at every location: licences differed across copies in 84 groups, externalReferences in 92 and cpe in 59, so keep-first would have silently lost licence data for 25 components and SHA-1 hashes for 31 from a document meant to replace the customer-facing licence CSV. A duplicate's differing primary CPE is demoted into syft:cpe23 so no matching coordinate is lost. syft:cpe23 is retained throughout because the schema permits a single top-level cpe while Syft derives several candidates per component. Result on that sidecar: 1693 -> 416 components, 43566 -> 6296 property entries, zero occurrences of /opt/delphix, 4.7 MB -> 0.8 MB, still schema-valid. Verified with Grype 0.119.0 that matching is unaffected -- raw and sanitized produce identical findings (40 matches, 21 CVEs), since Grype matches on purl and cpe. Location reporting is lost; detection is not. CYCLONEDX_FILTERING=false skips the pass and publishes the raw output, keeping syft:metadata:virtualPath so we can still determine where an unexpected component came from. Only the exact string "false" disables it, so a typo leaves us with a publishable document rather than one recording internal paths. Exposing it as a Jenkins build parameter is tracked as TOOL-31116. Co-Authored-By: Claude Opus 5 (1M context) --- ...6-09-08-sbom-per-package-sidecar-design.md | 67 ++++++++++++++++ lib/common.sh | 72 +++++++++++++++++ resources/sanitize-sbom.jq | 79 +++++++++++++++++++ 3 files changed, 218 insertions(+) create mode 100644 resources/sanitize-sbom.jq diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md index 6624f09b..36cf989c 100644 --- a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md +++ b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md @@ -284,6 +284,67 @@ than the generic installed-prior layer, which is not what review asked for. Noti trade-off explicitly: with the `setup.sh` approach, Jenkins no longer knows these packages relate to `syft`/`cyclonedx-cli`, so that batching/rebuild-cascade behaviour is lost. +### 7. Sanitizing the sidecar before publication + +Raw Syft output is not publishable as-is. Measured on a real `delphix-virtualization` +sidecar (4.7 MB, 1,693 components): + +| | raw | sanitized | +|---|---|---| +| components | 1,693 | 416 | +| property entries | 43,566 | 6,296 | +| `dependencies` entries | 26 | 0 | +| occurrences of `/opt/delphix` | 3,384 | 0 | +| size | 4.7 MB | 0.8 MB | + +`sanitize_sbom()` in `lib/common.sh` applies `resources/sanitize-sbom.jq` to each sidecar, +**between the Syft scan and the `cyclonedx-cli validate` call**, so that what is validated +is exactly what is published. It does three things: + +1. **Drops the `dependencies` graph.** For a directory scan Syft emits jar containment + ("this WAR bundles these jars"), not resolved dependency edges — it covered 501 of 1,693 + components, with no `compositions` element declaring it incomplete, so a consumer would + reasonably misread a missing entry as "this component has no dependencies". 90 of its 568 + edges were already dangling, pointing at the per-file components that + `SYFT_FILE_METADATA_SELECTION=none` suppresses (§4), and de-duplication below would + orphan many more. + +2. **Drops every property except `syft:cpe23`.** This removes the internal path leak — + `syft:location:*:path` and `syft:metadata:virtualPath` together exposed 470 directories + under `/opt/delphix`, including jar-internal structure such as + `resources.war:WEB-INF/lib/ST4-4.3.4.jar` — and metadata that merely restates the purl. + `syft:cpe23` is retained because the schema permits one top-level `cpe` while Syft derives + several candidates per component, and those are the fallback matching path when the + primary CPE guess is wrong. + +3. **Merge-dedupes components**, keyed on `purl` and falling back to `name+version+type`. A + purl-only key would drop the Windows binaries found by Syft's PE cataloger, which carry a + `cpe` but no `purl`; a name+version key would wrongly merge distinct components sharing a + name at version `UNKNOWN`. Duplicates are *merged* rather than reduced to the first + occurrence, because Syft does not detect the same metadata at every location — licences + differed across copies in 84 groups, `externalReferences` in 92, `cpe` in 59, and + keeping only the first would have silently lost licence data for 25 components and SHA-1 + hashes for 31. A duplicate's differing primary CPE is demoted into `syft:cpe23`, so no + matching coordinate is lost to the merge. + +The duplication is not a Syft defect: all eight `spring-core` entries had the identical +SHA-1, being one jar vendored into eight service directories. + +**`CYCLONEDX_FILTERING` — opting out for diagnostics.** The flag defaults to `true`, and +setting it to `false` skips this pass entirely, publishing the raw Syft output. That keeps +`syft:metadata:virtualPath`, the only field recording *where inside the package* a component +was found — of no use to a customer, but how we determine where an unexpected component came +from. Only the exact string `false` disables it: a typo should leave us with a publishable +document rather than one recording our internal paths. `cyclonedx-cli validate` runs in both +modes, and the sidecar filename is unchanged either way. + +Verified with Grype 0.119.0 that sanitizing does not weaken vulnerability matching: the raw +and sanitized documents produce identical findings (40 matches, 21 unique CVEs, same +severities), because Grype matches on `purl` and `cpe`, both standard top-level fields. What +is lost is location *reporting* — every match in the sanitized document carries zero +locations — not detection. Note this comparison did not exercise the de-duplication path, as +neither of the two vulnerable packages had duplicates. + ### S3 upload — no new plumbing needed Confirmed by reading `devops-gate/jenkins/jobs/pipelines/linux_pkg_build_package.groovy`'s @@ -359,6 +420,12 @@ per-package artifact directories at all. `linux-kernel-*` packages are `SBOM_DEEP_SCAN="false"` (no SBOM generated for build-host tooling or 3rd-party kernel forks), and the `.deb` ↔ `.cdx.json` mapping is now strictly 1:1 with a shared filename prefix. +- [x] `sanitize_sbom()` and `resources/sanitize-sbom.jq` added, gated on + `CYCLONEDX_FILTERING` (§7). Verified against a real `delphix-virtualization` sidecar: + 1,693 → 416 components, zero `/opt/delphix` occurrences, schema-valid, and identical + Grype findings before and after. +- [ ] `CYCLONEDX_FILTERING` exposed as a build parameter on the `build-package` and + `build-packages` Jenkins jobs — tracked separately as TOOL-31116 (`devops-gate`). ## Bugs found and fixed during real-build testing diff --git a/lib/common.sh b/lib/common.sh index 8e51a77a..64867ee0 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -36,6 +36,20 @@ export JENKINS_OPS_DIR="${JENKINS_OPS_DIR:-jenkins-ops}" # export UBUNTU_DISTRIBUTION="${UBUNTU_DISTRIBUTION:-noble}" +# +# Whether generate_sbom() post-processes each CycloneDX sidecar before it is +# validated and published (see sanitize_sbom() and resources/sanitize-sbom.jq). +# +# Set to "false" to publish the raw Syft output instead, which keeps +# syft:metadata:virtualPath and so records where inside the package each +# component was found. That is of no use to a customer, but is how we determine +# where an unexpected component came from. +# +# Only the exact string "false" disables it, so that a typo leaves us with a +# publishable document rather than one recording our internal paths. +# +export CYCLONEDX_FILTERING="${CYCLONEDX_FILTERING:-true}" + source "$(dirname "${BASH_SOURCE[0]}")/container.sh" # @@ -1533,6 +1547,35 @@ function store_build_info() { # it's picked up by the same S3 sync as every other build artifact, no # separate upload path needed. # +# +# Rewrite a Syft-generated CycloneDX document in place, dropping what should +# not be published and collapsing Syft's per-location duplicates. See +# resources/sanitize-sbom.jq for what is removed and why. +# +# Skipped entirely when CYCLONEDX_FILTERING is "false", which leaves the raw +# Syft output -- including syft:metadata:virtualPath, the only field that +# records where inside the package a component was found. That is of no use to +# a customer but is how we work out where an unexpected component came from, +# so it stays available on demand. +# +function sanitize_sbom() { + local sbom_file="$1" + local tmp + + check_env TOP + [[ -f "$sbom_file" ]] || die "sanitize_sbom: '$sbom_file' does not exist." + + tmp="$(logmust mktemp)" + + # + # jq cannot edit a file in place, so write to a temporary file and + # move it over only once jq has succeeded. Redirecting straight back + # into the input would truncate it before jq had read it. + # + logmust jq -f "$TOP/resources/sanitize-sbom.jq" "$sbom_file" >"$tmp" + logmust mv "$tmp" "$sbom_file" +} + function generate_sbom() { if [[ "$SBOM_DEEP_SCAN" != "true" ]]; then return 0 @@ -1562,6 +1605,19 @@ function generate_sbom() { "run's output for why it was skipped." done + # + # jq comes from setup.sh's baseline package install rather than + # install_sbom_tools(), but sanitize_sbom() is useless without it, so + # fail here rather than part-way through a package's .debs. + # + if [[ "$CYCLONEDX_FILTERING" != "false" ]]; then + command -v jq >/dev/null || + die "'jq' is not installed, so the SBOM for '$PACKAGE'" \ + "cannot be sanitized. Either install it (setup.sh" \ + "does so by default) or set CYCLONEDX_FILTERING" \ + "to 'false' to publish the raw Syft output." + fi + # # One sidecar per .deb, not per package: a package that emits more # than one .deb (e.g. "zfs" splits into zfs-dkms, zfsutils-linux, @@ -1622,6 +1678,22 @@ function generate_sbom() { logmust rm -rf "$extract_dir" + # + # Sanitize before validating, so that what is validated is + # exactly what is published. + # + # Only the exact string "false" disables this. Anything else, + # including unset, sanitizes: a typo should leave us with a + # publishable document rather than silently shipping one that + # records our internal paths. + # + if [[ "$CYCLONEDX_FILTERING" != "false" ]]; then + logmust sanitize_sbom "$sbom_file" + else + echo_bold "CYCLONEDX_FILTERING is 'false': publishing" \ + "the raw Syft output for $(basename "$deb")." + fi + logmust cyclonedx-cli validate \ --input-file "$sbom_file" \ --input-format json \ diff --git a/resources/sanitize-sbom.jq b/resources/sanitize-sbom.jq new file mode 100644 index 00000000..722af9e9 --- /dev/null +++ b/resources/sanitize-sbom.jq @@ -0,0 +1,79 @@ +# Sanitise a Syft-generated CycloneDX sidecar for external consumption. +# +# Applied by sanitize_sbom() in lib/common.sh to each .deb.cdx.json before +# it is validated and uploaded, unless CYCLONEDX_FILTERING is set to "false". +# +# Raw Syft output is not suitable to publish as-is: on delphix-virtualization it +# is 4.7 MB, of which roughly three quarters is repetition, and it records the +# absolute path of every component inside the package. +# +# 1. Drop the `dependencies` graph. +# +# Syft emits relationships it can observe, which for a directory scan means +# jar containment ("this WAR bundles these jars"), not resolved dependency +# edges. It covers about 30% of components, with no `compositions` element +# declaring it incomplete, so a consumer would reasonably misread a missing +# entry as "this component has no dependencies". It also holds dangling +# bom-refs pointing at the per-file components that +# SYFT_FILE_METADATA_SELECTION=none suppresses, and de-duplication below +# would orphan many more. +# +# 2. Drop every property except syft:cpe23. +# +# This removes the internal path leak (syft:location:*:path and +# syft:metadata:virtualPath, which together expose several hundred +# directories under /opt/delphix) and metadata that merely restates the +# purl (syft:metadata:-:groupID / :artifactID). +# +# syft:cpe23 is retained because the CycloneDX schema allows a single +# top-level `cpe`, while Syft derives several candidate CPEs per component +# to improve the odds of matching NVD, whose dictionary has no canonical +# naming convention. Those candidates are the fallback matching path for +# components whose primary CPE guess is wrong. +# +# 3. Merge-dedupe components. +# +# Keyed on purl, falling back to name+version+type: a purl-only key would +# silently drop the Windows binaries found by Syft's PE cataloger, which +# carry a cpe but no purl, and a name+version key would wrongly merge +# distinct components that share a name at version "UNKNOWN". +# +# Duplicates are merged rather than reduced to the first occurrence, since +# Syft does not detect the same metadata at every location -- keeping only +# the first silently loses licences and hashes that a later copy carried. +# A duplicate's differing primary CPE is demoted into syft:cpe23 so that no +# matching coordinate is lost to the merge. + +def keepprops: + [ (.properties // [])[] | select(.name == "syft:cpe23") ]; + +del(.dependencies) +| .components |= ( + group_by(.purl // ([.name, .version, .type] | tostring)) + | map( + reduce .[] as $c ({}; + if (. | length) == 0 then + ($c | .properties = keepprops) + else + .licenses = (((.licenses // []) + ($c.licenses // [])) | unique) + | .externalReferences = (((.externalReferences // []) + ($c.externalReferences // [])) | unique) + | .properties = ( + ( (.properties // []) + + ($c | keepprops) + + ( if ($c.cpe != null and $c.cpe != .cpe) + then [ { "name": "syft:cpe23", "value": $c.cpe } ] + else [] end ) + ) | unique + ) + end + ) + # A candidate identical to the component's own primary `cpe` adds + # nothing -- that value is already carried in the standard field. + | (.cpe // "") as $primary + | .properties = [ (.properties // [])[] + | select(.name != "syft:cpe23" or .value != $primary) ] + | if (.licenses | length) == 0 then del(.licenses) else . end + | if (.externalReferences | length) == 0 then del(.externalReferences) else . end + | if (.properties | length) == 0 then del(.properties) else . end + ) + ) From 2f05befced67151f3ade6e924288495944b3d49e Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Fri, 18 Sep 2026 16:28:29 +0530 Subject: [PATCH 12/17] DLPX-98872: retain syft:package:type in the sanitized sidecar Grype reported every matched component as "UnknownPackage" rather than "binary" once the properties were stripped. The type is not recoverable from anything else for a component whose purl carries no ecosystem: the six pkg:generic components and the eight PE binaries Syft finds without assigning a purl at all. Components with a pkg:maven purl were never affected, since Grype derives the type from the purl itself, which is why detection counts matched throughout and only the reported type was wrong. syft:package:metadataType and syft:package:foundBy were evaluated alongside it and changed nothing -- Grype's output was identical with and without them -- so they stay dropped. Retaining the one property costs roughly 3%. Also records in the spec what the Grype testing established about syft:cpe23: Grype sets match.java.using-cpes to false by default, so it ignores CPEs for the 405 Java components here and matches on the purl. Cutting syft:cpe23 from 6296 entries to 62 left Grype's output byte-identical. Those candidates are retained for Mend rather than Grype, which is worth stating explicitly since anyone measuring against Grype alone would conclude they are dead weight. Verified against a real delphix-virtualization sidecar: 1693 -> 416 components, 43566 -> 6712 property entries, zero occurrences of /opt/delphix, schema-valid, and Grype now reports 40 matches / 21 CVEs typed "binary", matching the unfiltered document exactly. Co-Authored-By: Claude Opus 5 (1M context) --- ...6-09-08-sbom-per-package-sidecar-design.md | 39 +++++++++++++++---- resources/sanitize-sbom.jq | 17 ++++++-- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md index 36cf989c..94cef410 100644 --- a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md +++ b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md @@ -309,13 +309,26 @@ is exactly what is published. It does three things: `SYFT_FILE_METADATA_SELECTION=none` suppresses (§4), and de-duplication below would orphan many more. -2. **Drops every property except `syft:cpe23`.** This removes the internal path leak — - `syft:location:*:path` and `syft:metadata:virtualPath` together exposed 470 directories - under `/opt/delphix`, including jar-internal structure such as +2. **Drops every property except `syft:cpe23` and `syft:package:type`.** This removes the + internal path leak — `syft:location:*:path` and `syft:metadata:virtualPath` together + exposed 470 directories under `/opt/delphix`, including jar-internal structure such as `resources.war:WEB-INF/lib/ST4-4.3.4.jar` — and metadata that merely restates the purl. + `syft:cpe23` is retained because the schema permits one top-level `cpe` while Syft derives several candidates per component, and those are the fallback matching path when the - primary CPE guess is wrong. + primary CPE guess is wrong. This is for **Mend's** benefit rather than Grype's: Grype sets + `match.java.using-cpes: false` by default, so for the 405 Java components here it matches + on the purl and ignores CPEs entirely. Reducing `syft:cpe23` from 6,296 entries to 62 left + Grype's output byte-identical — worth recording, since anyone measuring against Grype + alone would reasonably conclude these are dead weight. + + `syft:package:type` is retained because it is the only record of a component's ecosystem + for anything whose purl carries none: the six `pkg:generic` components and the eight PE + binaries Syft finds without assigning a purl. Without it Grype reports those as + `UnknownPackage` instead of `binary`. Components with a `pkg:maven` purl are unaffected, + as Grype derives the type from the purl. `syft:package:metadataType` and + `syft:package:foundBy` were evaluated alongside it and retained nothing — Grype's output + was unchanged with or without them — so they are dropped. 3. **Merge-dedupes components**, keyed on `purl` and falling back to `name+version+type`. A purl-only key would drop the Windows binaries found by Syft's PE cataloger, which carry a @@ -340,10 +353,20 @@ modes, and the sidecar filename is unchanged either way. Verified with Grype 0.119.0 that sanitizing does not weaken vulnerability matching: the raw and sanitized documents produce identical findings (40 matches, 21 unique CVEs, same -severities), because Grype matches on `purl` and `cpe`, both standard top-level fields. What -is lost is location *reporting* — every match in the sanitized document carries zero -locations — not detection. Note this comparison did not exercise the de-duplication path, as -neither of the two vulnerable packages had duplicates. +severities), because Grype matches on `purl` and the primary `cpe`, both standard top-level +fields. Detection was also confirmed for a component that *is* vulnerable, by injecting +`log4j-core 2.14.1` into each variant — all reported the same seven advisories, correctly +typed `java-archive` from the purl. + +What is lost is location *reporting*: every match in the sanitized document carries zero +locations, where the raw document names each path the component was found at. That is a +remediation-context loss, not a detection one, and it is precisely what +`CYCLONEDX_FILTERING=false` exists to recover. + +Two caveats on that testing. The comparison did not exercise the de-duplication path, since +neither vulnerable package had duplicates; and all 40 matches fell on a single component +(`jq`, at versions 1.4 and 1.6), so the Java matching path was only exercised by the injected +component rather than by a real finding. ### S3 upload — no new plumbing needed diff --git a/resources/sanitize-sbom.jq b/resources/sanitize-sbom.jq index 722af9e9..2b91e502 100644 --- a/resources/sanitize-sbom.jq +++ b/resources/sanitize-sbom.jq @@ -18,7 +18,7 @@ # SYFT_FILE_METADATA_SELECTION=none suppresses, and de-duplication below # would orphan many more. # -# 2. Drop every property except syft:cpe23. +# 2. Drop every property except syft:cpe23 and syft:package:type. # # This removes the internal path leak (syft:location:*:path and # syft:metadata:virtualPath, which together expose several hundred @@ -29,7 +29,17 @@ # top-level `cpe`, while Syft derives several candidate CPEs per component # to improve the odds of matching NVD, whose dictionary has no canonical # naming convention. Those candidates are the fallback matching path for -# components whose primary CPE guess is wrong. +# components whose primary CPE guess is wrong. Note this is for Mend's +# benefit rather than Grype's: Grype disables CPE matching for Java by +# default (match.java.using-cpes), so for the great majority of components +# here it matches on the purl instead. +# +# syft:package:type is retained because it is the only record of a +# component's ecosystem for anything whose purl does not carry one -- +# pkg:generic components and the PE binaries Syft finds without assigning +# a purl at all. Without it those are reported as "UnknownPackage" rather +# than "binary". Components with a pkg:maven purl are unaffected, as the +# type is derived from the purl itself. # # 3. Merge-dedupe components. # @@ -45,7 +55,8 @@ # matching coordinate is lost to the merge. def keepprops: - [ (.properties // [])[] | select(.name == "syft:cpe23") ]; + [ (.properties // [])[] + | select(.name | test("^syft:(cpe23|package:type)$")) ]; del(.dependencies) | .components |= ( From 40ed4405a0c3454b47a2f3a56f97a86aa1d87707 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Thu, 24 Sep 2026 12:20:27 +0530 Subject: [PATCH 13/17] DLPX-98872: type the SBOM root as application and declare a flat dependency graph Mend support analysed a sidecar we sent them and found two structural reasons it imports badly, neither of which the sanitisation addressed. First, metadata.component was typed "file" rather than "application". Syft emits that because generate_sbom() scans an extracted directory, and Mend's importer only treats metadata.component as the project root when it is application-typed -- so the root was being ignored outright. This affected the sanitized document as well as the raw one, since the filter never touched the metadata block. Second, Syft's dependencies array left roughly 70% of components with no declared relationship at all. Mend treats such a component as a direct dependency of the root rather than dropping it, so nothing was lost, but the result is a tree that is partly real and mostly inferred. That graph was already being discarded here for unrelated reasons: it encodes jar containment rather than resolved dependencies, covers ~30% of components, and carries dangling bom-refs. Discarding it without replacement still leaves the outcome to each consumer's default. It is now rebuilt as a single explicit edge -- the root depending on every component -- which says the same thing unambiguously, connects the root that Syft leaves unreferenced even in its own output, and takes the orphaned share from 70% to zero. compositions aggregate "incomplete" is what keeps that honest: a flat graph is not a resolved dependency tree and should not be read as one. This manufactures no relationships that are not there. Mend's closing point -- that scanning a packaged .deb cannot recover what only exists at build time, and that scanning the build or source with their CLI would -- matches what the top-level design already concluded, and is the subject of CP-13467. Verified on a real delphix-virtualization sidecar: root now application-typed, 0 of 416 components orphaned from the graph, still schema-valid, still zero occurrences of /opt/delphix, and Grype unchanged at 40 matches / 21 CVEs typed "binary". Co-Authored-By: Claude Opus 5 (1M context) --- ...6-09-08-sbom-per-package-sidecar-design.md | 48 +++++++++++++++---- resources/sanitize-sbom.jq | 33 ++++++++++++- 2 files changed, 69 insertions(+), 12 deletions(-) diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md index 94cef410..22bba27e 100644 --- a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md +++ b/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md @@ -292,22 +292,50 @@ sidecar (4.7 MB, 1,693 components): | | raw | sanitized | |---|---|---| | components | 1,693 | 416 | -| property entries | 43,566 | 6,296 | -| `dependencies` entries | 26 | 0 | +| property entries | 43,566 | 6,712 | +| `dependencies` entries | 26 | 1 | +| components orphaned from `dependencies` | 70% | 0% | +| `metadata.component.type` | `file` | `application` | | occurrences of `/opt/delphix` | 3,384 | 0 | -| size | 4.7 MB | 0.8 MB | +| size | 4.7 MB | 1.2 MB | `sanitize_sbom()` in `lib/common.sh` applies `resources/sanitize-sbom.jq` to each sidecar, **between the Syft scan and the `cyclonedx-cli validate` call**, so that what is validated is exactly what is published. It does three things: -1. **Drops the `dependencies` graph.** For a directory scan Syft emits jar containment - ("this WAR bundles these jars"), not resolved dependency edges — it covered 501 of 1,693 - components, with no `compositions` element declaring it incomplete, so a consumer would - reasonably misread a missing entry as "this component has no dependencies". 90 of its 568 - edges were already dangling, pointing at the per-file components that - `SYFT_FILE_METADATA_SELECTION=none` suppresses (§4), and de-duplication below would - orphan many more. +1. **Replaces the `dependencies` graph, and types the root component as `application`.** + + Syft's own graph is dropped. For a directory scan it emits jar containment ("this WAR + bundles these jars"), not resolved dependency edges — it covered 501 of 1,693 components, + with no `compositions` element declaring it incomplete, so a consumer would reasonably + misread a missing entry as "this component has no dependencies". 90 of its 568 edges were + already dangling, pointing at the per-file components that + `SYFT_FILE_METADATA_SELECTION=none` suppresses (§4), and de-duplication would orphan many + more. + + It is rebuilt as a single flat edge — the root depending on every component — and marked + `compositions: [{ aggregate: "incomplete" }]`. Both of these, and the `application` root + type, come from Mend support's analysis of a sidecar we sent them (2026-09-24): + + > The root component (virtualization 2026.09.15.08) is declared with type `file` rather + > than type `application`. Mend's importer only processes `metadata.component` as the + > project root when its type is `application`, so this root is not being recognized + > correctly. […] when a component has no defined relationship in `dependencies`, it is + > treated as a direct dependency of the root rather than being dropped or placed correctly + > in the hierarchy. + + Syft emits `type: file` because `generate_sbom()` scans an extracted directory, so the + root was being ignored entirely. Leaving `dependencies` absent is not harmful — nothing is + dropped, and Mend flattens to the root — but it leaves the outcome to each consumer's + default. Declaring the flat graph explicitly says the same thing unambiguously, connects + the root (which Syft leaves unreferenced even in its own output), and takes the share of + components orphaned from the graph from 70% to zero. `aggregate: "incomplete"` is what + keeps it honest: this is not a resolved dependency graph and should not be read as one. + + Note this does **not** manufacture real relationships. Mend's closing point — that a Syft + scan of the packaged `.deb` cannot recover what only exists at build time, and that + scanning the build or source with the Mend CLI would — is the same conclusion the + top-level design reached, and is CP-13467's subject. 2. **Drops every property except `syft:cpe23` and `syft:package:type`.** This removes the internal path leak — `syft:location:*:path` and `syft:metadata:virtualPath` together diff --git a/resources/sanitize-sbom.jq b/resources/sanitize-sbom.jq index 2b91e502..9bec4797 100644 --- a/resources/sanitize-sbom.jq +++ b/resources/sanitize-sbom.jq @@ -7,7 +7,7 @@ # is 4.7 MB, of which roughly three quarters is repetition, and it records the # absolute path of every component inside the package. # -# 1. Drop the `dependencies` graph. +# 1. Replace the `dependencies` graph, and type the root component. # # Syft emits relationships it can observe, which for a directory scan means # jar containment ("this WAR bundles these jars"), not resolved dependency @@ -16,7 +16,9 @@ # entry as "this component has no dependencies". It also holds dangling # bom-refs pointing at the per-file components that # SYFT_FILE_METADATA_SELECTION=none suppresses, and de-duplication below -# would orphan many more. +# would orphan many more. It is dropped and rebuilt at the end of this +# filter as a flat graph declared `incomplete`, alongside setting the root +# component's type to "application" -- see the comments there. # # 2. Drop every property except syft:cpe23 and syft:package:type. # @@ -88,3 +90,30 @@ del(.dependencies) | if (.properties | length) == 0 then del(.properties) else . end ) ) + +# Mend's CycloneDX importer only treats metadata.component as the project root +# when it is application-typed; Syft emits "file" because generate_sbom() scans +# an extracted directory. Without this the root is not recognised at all. +| .metadata.component.type = "application" + +# Re-state the dependency graph rather than leaving it absent. +# +# Syft's own graph was dropped above: it covered ~30% of components, encoded jar +# containment rather than resolved dependencies, and held dangling bom-refs. But +# leaving `dependencies` out entirely means each consumer falls back to its own +# default -- Mend, for one, treats a component with no declared relationship as a +# direct dependency of the root. Declaring that explicitly says the same thing +# unambiguously, and connects the root component, which Syft leaves unreferenced +# even in its own output. +# +# `compositions: incomplete` is what keeps this honest: it states that the +# relationships here are not a resolved dependency graph, so a flat tree is not +# mistaken for the real hierarchy. Recovering that would require scanning the +# build rather than the packaged .deb -- see CP-13467. +| .metadata.component["bom-ref"] as $root +| .dependencies = [ + { "ref": $root, "dependsOn": [ .components[]["bom-ref"] ] } + ] +| .compositions = [ + { "aggregate": "incomplete", "dependencies": [ $root ] } + ] From 15f355b7a9dd89646ac8eaebae41248cce82a619 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Fri, 25 Sep 2026 14:02:10 +0530 Subject: [PATCH 14/17] DLPX-98872: fix sanitize_sbom() crashing when Syft finds no components The performance-diagnostics pre-push build failed with jq: error (...cdx.json:1): Cannot iterate over null (null) Error: failed command 'jq -f .../resources/sanitize-sbom.jq ...' Syft omits `components` from the document altogether -- rather than emitting an empty array -- when it finds nothing it can catalog in the scanned payload. `.components |= (group_by(...))` then runs group_by against null and dies, taking the whole package build with it. An empty array was always handled correctly; only the absent key was not, which is why this survived testing against packages that do produce components. Normalising `.components` to an empty array before the pipeline fixes it. Verified against all three shapes -- key absent, key present but empty, and a real 416-component document -- with the output schema-valid in each case. For an empty scan the result is a well-formed document with no components, an empty dependsOn, and the root still typed application. Worth noting separately that performance-diagnostics producing no components at all means its sidecar conveys nothing beyond what appliance-build's image-level scan already records. That is the same class of problem as the earlier valid-but-empty sidecars, and its SBOM_DEEP_SCAN classification is worth revisiting on that basis. Co-Authored-By: Claude Opus 5 (1M context) --- resources/sanitize-sbom.jq | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/resources/sanitize-sbom.jq b/resources/sanitize-sbom.jq index 9bec4797..b587bd80 100644 --- a/resources/sanitize-sbom.jq +++ b/resources/sanitize-sbom.jq @@ -60,7 +60,13 @@ def keepprops: [ (.properties // [])[] | select(.name | test("^syft:(cpe23|package:type)$")) ]; -del(.dependencies) +# Syft omits `components` altogether -- rather than emitting an empty array -- +# when it finds nothing in the scanned payload, which happens for a package +# whose .deb holds only content no cataloger recognises. Normalise that to an +# empty array first, or every iteration below fails with "Cannot iterate over +# null" and takes the build with it. +.components //= [] +| del(.dependencies) | .components |= ( group_by(.purl // ([.name, .version, .type] | tostring)) | map( From eb9e15593205405488aa5fc6063b67dc32ddd690 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Mon, 28 Sep 2026 22:00:24 +0530 Subject: [PATCH 15/17] DLPX-98872 TEMPORARY DO NOT MERGE: default CYCLONEDX_FILTERING to false Flips the default so a pre-push build publishes the raw Syft output, to confirm the opt-out path produces the unsanitized document end to end -- roughly 4.9 MB and 1693 components with syft:metadata:virtualPath intact, rather than the 1.3 MB / 421 component sanitized form. This exists only because CYCLONEDX_FILTERING is not yet exposed as a Jenkins build parameter (TOOL-31116), so there is currently no way to pass it to a build without editing the default. REVERT THIS COMMIT BEFORE MERGE. Leaving it in would make every build publish sidecars carrying our internal /opt/delphix paths by default, which is the exact thing the sanitization was added to prevent. Co-Authored-By: Claude Opus 5 (1M context) --- lib/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common.sh b/lib/common.sh index 64867ee0..7dd37fe5 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -48,7 +48,7 @@ export UBUNTU_DISTRIBUTION="${UBUNTU_DISTRIBUTION:-noble}" # Only the exact string "false" disables it, so that a typo leaves us with a # publishable document rather than one recording our internal paths. # -export CYCLONEDX_FILTERING="${CYCLONEDX_FILTERING:-true}" +export CYCLONEDX_FILTERING="${CYCLONEDX_FILTERING:-false}" source "$(dirname "${BASH_SOURCE[0]}")/container.sh" From a4d72b43b21472b929f033fb74e698791e58d173 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Tue, 29 Sep 2026 18:07:44 +0530 Subject: [PATCH 16/17] post testing that flipping the flag CYCLONEDX_FILTERING to false is stopping the bom filtering, which is working fine. Changing back the CYCLONEDX_FILTERING to true --- lib/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common.sh b/lib/common.sh index 7dd37fe5..64867ee0 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -48,7 +48,7 @@ export UBUNTU_DISTRIBUTION="${UBUNTU_DISTRIBUTION:-noble}" # Only the exact string "false" disables it, so that a typo leaves us with a # publishable document rather than one recording our internal paths. # -export CYCLONEDX_FILTERING="${CYCLONEDX_FILTERING:-false}" +export CYCLONEDX_FILTERING="${CYCLONEDX_FILTERING:-true}" source "$(dirname "${BASH_SOURCE[0]}")/container.sh" From cc46d94086a2fb833cad35795e6fec8503b0bd43 Mon Sep 17 00:00:00 2001 From: Sanjeev Rohila Date: Wed, 30 Sep 2026 11:06:27 +0530 Subject: [PATCH 17/17] DLPX-98872: drop the "sidecar" jargon in favour of SBOM Review feedback: the term is meaningless in this context. Applied as asked in each of the three places it was raised -- the sentence "Generate a CycloneDX SBOM for each of this package's built .deb(s)" now stands on its own, "one sidecar per .deb" reads "one SBOM file per .deb", and the jq filter's header says simply "CycloneDX SBOM". Carried through the rest of the change for consistency, since leaving the word in twenty-odd other comments after calling it meaningless would be worse than not having started. That covers the remaining comments in lib/common.sh, the CI lint script, and the design doc, including its title and filename -- which now reads 2026-09-08-sbom-per-package-design.md. Where the word was doing grammatical work the sentence was reworded rather than substituted blindly, e.g. "Phase 3 associates a sidecar with its .deb" becomes "associates an SBOM with its .deb". No functional change: shellcheck and shfmt clean, and the jq filter produces a byte-identical document (416 components, schema-valid) on the same input. Co-Authored-By: Claude Opus 5 (1M context) --- .github/scripts/verify-sbom-scan-flag.sh | 2 +- ... => 2026-09-08-sbom-per-package-design.md} | 46 +++++++++---------- lib/common.sh | 8 ++-- resources/sanitize-sbom.jq | 2 +- 4 files changed, 29 insertions(+), 29 deletions(-) rename docs/specs/{2026-09-08-sbom-per-package-sidecar-design.md => 2026-09-08-sbom-per-package-design.md} (94%) diff --git a/.github/scripts/verify-sbom-scan-flag.sh b/.github/scripts/verify-sbom-scan-flag.sh index 7b97e587..e0976c13 100755 --- a/.github/scripts/verify-sbom-scan-flag.sh +++ b/.github/scripts/verify-sbom-scan-flag.sh @@ -8,7 +8,7 @@ cd "$(git rev-parse --show-toplevel)" # Every package must explicitly set SBOM_DEEP_SCAN to "true" or "false" in # its config.sh -- see docs on generate_sbom() in lib/common.sh. There is # no default: a package that hasn't been classified yet must fail CI -# rather than silently ship without a CycloneDX sidecar or without an +# rather than silently ship without a CycloneDX SBOM or without an # explicit decision that it doesn't need one. # unclassified=$(./query-packages.sh list -o name,sbom-deep-scan all | diff --git a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md b/docs/specs/2026-09-08-sbom-per-package-design.md similarity index 94% rename from docs/specs/2026-09-08-sbom-per-package-sidecar-design.md rename to docs/specs/2026-09-08-sbom-per-package-design.md index 22bba27e..3365ec60 100644 --- a/docs/specs/2026-09-08-sbom-per-package-sidecar-design.md +++ b/docs/specs/2026-09-08-sbom-per-package-design.md @@ -1,4 +1,4 @@ -# SBOM Per-Package Sidecar Generation — Design +# SBOM Per-Package Generation — Design - **Date:** 2026-09-08 - **Jira:** [DLPX-98872](https://perforce.atlassian.net/browse/DLPX-98872) @@ -98,7 +98,7 @@ fi The one-time classification of all 40 packages (§5) lands in the same change, so this lint never lands red. -### 4. `generate_sbom()` — sidecar generation +### 4. `generate_sbom()` — SBOM generation Modeled on `store_build_info()` — a default stage function in `lib/common.sh` that packages don't need to override, gated on the new flag: @@ -125,7 +125,7 @@ function generate_sbom() { command -v "$tool" >/dev/null || die "'$tool' is not installed..." done - # One sidecar per .deb, not per package: a package that emits more + # One SBOM file per .deb, not per package: a package that emits more # than one .deb (e.g. "zfs" splits into zfs-dkms, zfsutils-linux, # etc.) gets one .deb.cdx.json per .deb -- a strict # 1:1 mapping, no merging across a package's .deb(s). @@ -173,14 +173,14 @@ overrides, out of scope for Phase 2, would override the function in a package's the same way packages already override `build()`). **Resolved — multiple `.deb`s per package:** a reviewer flagged that the original -merge-into-one-sidecar approach (below, kept here for history) doesn't give a clean 1:1 +merge-into-one-SBOM approach (below, kept here for history) doesn't give a clean 1:1 mapping between a `.deb` and its BOM. Changed to: one `.deb.cdx.json` per -`.deb`, no merge, no `cyclonedx-cli merge` step at all — each `.deb`'s sidecar is fully +`.deb`, no merge, no `cyclonedx-cli merge` step at all — each `.deb`'s SBOM is fully independent and filename-matched to it. This is a real divergence from the top-level design doc (CP-13456), which called for "one package-level SBOM... associated with all of that package's debs via `COMPONENTS`" — that assumption didn't survive review. Phase 3 (`appliance-build`'s consumer, not yet built) will need to associate each `.deb` with its -own sidecar directly by filename, not go through a package-level indirection. +own SBOM directly by filename, not go through a package-level indirection. *(For reference, the approach this replaced: scan each `.deb` into its own document, then `cyclonedx-cli merge --output-version v1_6` them into a single `.cdx.json`. Two @@ -194,7 +194,7 @@ initially, and wrongly, closed as "confirmed working" after a `delphix-sso-app` build produced a schema-valid document. It was valid but empty: `syft scan ` only *identifies* the archive — its `deb-archive-cataloger` reads the control metadata and emits a single `pkg:deb` component, never descending into `data.tar.*`. A -`delphix-virtualization` sidecar from a real build contained exactly **one** component for +`delphix-virtualization` SBOM from a real build contained exactly **one** component for a 1.17 GB Java + Angular application, i.e. no more information than the image-level dpkg scan already provides for free, and nothing at all for a vulnerability scanner to match against the bundled jars. @@ -208,10 +208,10 @@ package's own extracted payload everything found belongs to that package by cons which is the whole reason for scanning here instead of at image level. Note the shape change this brings: because the scanned source is now a directory rather -than a `.deb`, the sidecar no longer carries a `pkg:deb` component for the package itself +than a `.deb`, the SBOM no longer carries a `pkg:deb` component for the package itself — the package's identity lives in `metadata.component` (via `--source-name`/ `--source-version`), and the flat `pkg:deb` entry continues to come from Phase 1's -image-level scan. Phase 3 associates a sidecar with its `.deb` by filename (§4), so +image-level scan. Phase 3 associates an SBOM with its `.deb` by filename (§4), so nothing depends on that component being present here. **Still open — the bundled npm frontend.** Extraction fixes the jars, but @@ -284,10 +284,10 @@ than the generic installed-prior layer, which is not what review asked for. Noti trade-off explicitly: with the `setup.sh` approach, Jenkins no longer knows these packages relate to `syft`/`cyclonedx-cli`, so that batching/rebuild-cascade behaviour is lost. -### 7. Sanitizing the sidecar before publication +### 7. Sanitizing the SBOM before publication Raw Syft output is not publishable as-is. Measured on a real `delphix-virtualization` -sidecar (4.7 MB, 1,693 components): +SBOM (4.7 MB, 1,693 components): | | raw | sanitized | |---|---|---| @@ -299,7 +299,7 @@ sidecar (4.7 MB, 1,693 components): | occurrences of `/opt/delphix` | 3,384 | 0 | | size | 4.7 MB | 1.2 MB | -`sanitize_sbom()` in `lib/common.sh` applies `resources/sanitize-sbom.jq` to each sidecar, +`sanitize_sbom()` in `lib/common.sh` applies `resources/sanitize-sbom.jq` to each SBOM, **between the Syft scan and the `cyclonedx-cli validate` call**, so that what is validated is exactly what is published. It does three things: @@ -315,7 +315,7 @@ is exactly what is published. It does three things: It is rebuilt as a single flat edge — the root depending on every component — and marked `compositions: [{ aggregate: "incomplete" }]`. Both of these, and the `application` root - type, come from Mend support's analysis of a sidecar we sent them (2026-09-24): + type, come from Mend support's analysis of an SBOM we sent them (2026-09-24): > The root component (virtualization 2026.09.15.08) is declared with type `file` rather > than type `application`. Mend's importer only processes `metadata.component` as the @@ -377,7 +377,7 @@ setting it to `false` skips this pass entirely, publishing the raw Syft output. was found — of no use to a customer, but how we determine where an unexpected component came from. Only the exact string `false` disables it: a typo should leave us with a publishable document rather than one recording our internal paths. `cyclonedx-cli validate` runs in both -modes, and the sidecar filename is unchanged either way. +modes, and the SBOM filename is unchanged either way. Verified with Grype 0.119.0 that sanitizing does not weaken vulnerability matching: the raw and sanitized documents produce identical findings (40 matches, 21 unique CVEs, same @@ -440,7 +440,7 @@ per-package artifact directories at all. | | (Phase 3, not in scope here: | appliance-build fetches each .deb's - | own sidecar by filename match) + | own SBOM by filename match) v [out of scope for Phase 2] ``` @@ -450,10 +450,10 @@ per-package artifact directories at all. - Per-ecosystem overrides (`cargo-cyclonedx` for Rust, `cyclonedx-gradle-plugin` for Java) — Phase 4's evaluation decides if the Syft-on-deb baseline here is good enough first. -- `appliance-build` fetching/merging these sidecars into the per-image document — that's +- `appliance-build` fetching/merging these SBOMs into the per-image document — that's Phase 3 (CP-13466), which explicitly depends on this phase completing. -- DCT/Hyperscale sidecars — deferred in the top-level design, not filed as a story yet. -- The `devops-gate` publishing switch (CSV → CycloneDX) — unrelated to sidecar generation. +- DCT/Hyperscale SBOMs — deferred in the top-level design, not filed as a story yet. +- The `devops-gate` publishing switch (CSV → CycloneDX) — unrelated to SBOM generation. ## Implementation status @@ -466,13 +466,13 @@ per-package artifact directories at all. `verify-query-packages.sh` both pass; `shellcheck`/`shfmt` clean on every touched file. - [x] Verified against a real build host: `delphix-sso-app` (single-`.deb`) and `delphix-rust` (multi-`.deb`) pre-push builds both produce valid, schema-checked - CycloneDX 1.6 sidecars in S3, correctly named per `.deb`. + CycloneDX 1.6 SBOMs in S3, correctly named per `.deb`. - [x] Confirmed compliant with reviewer feedback: `syft`, `cyclonedx-cli`, and all `linux-kernel-*` packages are `SBOM_DEEP_SCAN="false"` (no SBOM generated for build-host tooling or 3rd-party kernel forks), and the `.deb` ↔ `.cdx.json` mapping is now strictly 1:1 with a shared filename prefix. - [x] `sanitize_sbom()` and `resources/sanitize-sbom.jq` added, gated on - `CYCLONEDX_FILTERING` (§7). Verified against a real `delphix-virtualization` sidecar: + `CYCLONEDX_FILTERING` (§7). Verified against a real `delphix-virtualization` SBOM: 1,693 → 416 components, zero `/opt/delphix` occurrences, schema-valid, and identical Grype findings before and after. - [ ] `CYCLONEDX_FILTERING` exposed as a build parameter on the `build-package` and @@ -484,9 +484,9 @@ Not caught by CI — only surfaced by actually running builds, and in one case o inspecting the *contents* of a produced SBOM rather than its exit status: 0. **Valid but empty SBOMs — the most serious of these.** `syft scan ` identifies the - archive without descending into it, so every sidecar contained a single `pkg:deb` + archive without descending into it, so every SBOM contained a single `pkg:deb` component and nothing else: no jars, no wheels, no modules. A `delphix-virtualization` - sidecar had one component for a 1.17 GB application. Every flagged package was affected. + SBOM had one component for a 1.17 GB application. Every flagged package was affected. Caught when the output was actually read, not when the build passed — the build had been passing the whole time, and `cyclonedx-cli validate` passes an empty-but-well-formed document quite happily. Fixed by extracting with `dpkg-deb -x` and scanning the @@ -501,7 +501,7 @@ inspecting the *contents* of a produced SBOM rather than its exit status: `syft`/`cyclonedx-cli`'s own `config.sh`). Fixed by reading the version back out of the built `.deb` via `dpkg-deb -f "$deb" Version`. 3. **Stray Syft "file" component**, carrying file hashes and an absolute build-workspace - path, polluting the sidecar. Fixed with `SYFT_FILE_METADATA_SELECTION=none`, matching + path, polluting the SBOM. Fixed with `SYFT_FILE_METADATA_SELECTION=none`, matching `appliance-build`'s `95-generate-sbom.binary` hook. 4. **`cyclonedx-cli merge` defaulted to spec version 1.7**, failing the subsequent `--input-version v1_6` validate call. Moot now that merging was removed entirely per diff --git a/lib/common.sh b/lib/common.sh index 64867ee0..b28e91c5 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -37,7 +37,7 @@ export JENKINS_OPS_DIR="${JENKINS_OPS_DIR:-jenkins-ops}" export UBUNTU_DISTRIBUTION="${UBUNTU_DISTRIBUTION:-noble}" # -# Whether generate_sbom() post-processes each CycloneDX sidecar before it is +# Whether generate_sbom() post-processes each CycloneDX SBOM before it is # validated and published (see sanitize_sbom() and resources/sanitize-sbom.jq). # # Set to "false" to publish the raw Syft output instead, which keeps @@ -1536,7 +1536,7 @@ function store_build_info() { } # -# Generate a CycloneDX SBOM sidecar for each of this package's built +# Generate a CycloneDX SBOM for each of this package's built # .deb(s) by running Syft against it. Only packages that bundle # third-party composition (jars, npm, wheels, Rust crates, ...) opt in # via SBOM_DEEP_SCAN="true" in their config.sh -- everything else is @@ -1619,7 +1619,7 @@ function generate_sbom() { fi # - # One sidecar per .deb, not per package: a package that emits more + # One SBOM file per .deb, not per package: a package that emits more # than one .deb (e.g. "zfs" splits into zfs-dkms, zfsutils-linux, # etc.) gets one .deb.cdx.json per .deb, each a # standalone document scoped to that .deb alone. No merging across @@ -1647,7 +1647,7 @@ function generate_sbom() { # "syft scan .deb" only identifies the archive: its # deb-archive-cataloger reads the control metadata and emits a # single pkg:deb component, never descending into data.tar.*, - # so none of the bundled jars/wheels/modules this sidecar + # so none of the bundled jars/wheels/modules this SBOM # exists to capture are found. That produced valid but empty # documents -- one component for a 1.2GB application -- which # is no more than appliance-build's image-level dpkg scan diff --git a/resources/sanitize-sbom.jq b/resources/sanitize-sbom.jq index b587bd80..2e49f881 100644 --- a/resources/sanitize-sbom.jq +++ b/resources/sanitize-sbom.jq @@ -1,4 +1,4 @@ -# Sanitise a Syft-generated CycloneDX sidecar for external consumption. +# Sanitise a Syft-generated CycloneDX SBOM for external consumption. # # Applied by sanitize_sbom() in lib/common.sh to each .deb.cdx.json before # it is validated and uploaded, unless CYCLONEDX_FILTERING is set to "false".