From c2b05d3c86d5c16111c28e8ce81294df645c30a1 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 02:40:30 -0700 Subject: [PATCH 01/23] feat(sbom): emit package origin and preserve ingested assertions Issue #380 asked for per-component supplier and description sourced from deps.dev during --enrich. Probing the live API across npm, PyPI, Maven, NuGet, Cargo, and Go shows GetVersion asserts neither field: it carries only links[] and registries[]. A description exists only on the separate project endpoint, where it is the source repository's description, so every package in a monorepo would receive the same text. Deriving supplier or description for third-party packages needs registry-native metadata Bomly does not fetch, which would mean new allowed network hosts. Those two fields are therefore deferred, and this change ships what needs no new source: metadata Bomly already collects and then dropped at the export boundary, plus assertions a third party made in an SBOM Bomly ingests. Nothing is invented. Package origin. ResolvedURL is not a URL. npm/pnpm/yarn/bun record the exact tarball, Bundler records the GEM registry root, Cargo records a registry+/sparse+/git+ prefixed index or repo, swiftpm records a repository, and uv, pipenv, pub, and npm link entries can all record a local filesystem path. internal/sbom/locator.go classifies each value into artifact / VCS / registry-root / nothing before it can reach a document. Two rules are load-bearing: an http(s) scheme is required and userinfo is rejected, which keeps build-machine paths and private registry tokens out of published output; and a registry root never becomes a download location, because that failure would be silent and plausible rather than caught by a validator. Round-trip preservation. Ingest is not decode-then-encode: ToGraph converts to an sdk.Graph and export rebuilds a fresh document, so supplier, description, and external references were lost by a format conversion even though both decoders could see them. They now ride Dependency.Metadata under bomly.sbom.* keys, following the SetDetectionLicenses precedent, with no SDK contract change. ToGraph deliberately does not set Dependency.Source, which feeds RegistryMatchEligible and would quietly break scan --sbom --enrich. Also fixes two pre-existing gaps found by the new round-trip tests: FromDepGraph never read pkg.CPEs, and ToGraph dropped CPEs and digests. The two new fuzz targets found four real bugs while being written: an "@revision" suffix re-parsing as userinfo on a path-less URL, a control character reaching that suffix, an invalid percent-escape from a dot-containing non-hostname, and a scheme with no host. Verified with make test, make generate (no drift), make fuzz, and a new smoke case. Both formats validate against spdxlib.ValidateDocument and the CycloneDX 1.7 schema; real scans across npm/ruby/rust/python/go leak no local paths. Co-Authored-By: Claude Opus 5 --- .github/workflows/smoke.yml | 2 +- .github/workflows/update-smoke-goldens.yml | 2 +- dev-docs/ARCHITECTURE.md | 21 ++ docs/SBOM.md | 67 +++++- internal/sbom/cyclonedx.go | 122 ++++++++-- internal/sbom/distribution_test.go | 256 ++++++++++++++++++++ internal/sbom/graph.go | 28 ++- internal/sbom/ingest_metadata.go | 133 +++++++++++ internal/sbom/locator.go | 257 +++++++++++++++++++++ internal/sbom/locator_fuzz_test.go | 119 ++++++++++ internal/sbom/locator_test.go | 149 ++++++++++++ internal/sbom/model.go | 30 +++ internal/sbom/roundtrip_test.go | 224 ++++++++++++++++++ internal/sbom/spdx23.go | 142 +++++++++++- internal/sbom/transform.go | 31 +++ scripts/run-fuzz.sh | 2 + test/smoke/smoke_test.go | 103 +++++++++ 17 files changed, 1655 insertions(+), 33 deletions(-) create mode 100644 internal/sbom/distribution_test.go create mode 100644 internal/sbom/ingest_metadata.go create mode 100644 internal/sbom/locator.go create mode 100644 internal/sbom/locator_fuzz_test.go create mode 100644 internal/sbom/locator_test.go create mode 100644 internal/sbom/roundtrip_test.go diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 4ab67556..6e3a2d6a 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -97,7 +97,7 @@ jobs: - name: ruby run: 'TestScan$/scan-bundler' - name: sbom - run: 'TestScan$/scan-sbom|TestDiff/(diff-sbom$|diff-sbom-detail-change$)|TestLiteScan/lite-scan-sbom|TestScanSBOMSyftJSONRejected$' + run: 'TestScan$/scan-sbom|TestDiff/(diff-sbom$|diff-sbom-detail-change$)|TestLiteScan/lite-scan-sbom|TestScanSBOMSyftJSONRejected$|TestScanSBOMExportDistribution$' - name: dotnet run: 'TestScan$/scan-nuget' dotnet: true diff --git a/.github/workflows/update-smoke-goldens.yml b/.github/workflows/update-smoke-goldens.yml index ade2b7d8..f58e9315 100644 --- a/.github/workflows/update-smoke-goldens.yml +++ b/.github/workflows/update-smoke-goldens.yml @@ -121,7 +121,7 @@ jobs: - name: cpp run: 'TestScan$/scan-cpp-conan' - name: sbom - run: 'TestScan$/scan-sbom|TestDiff/(diff-sbom$|diff-sbom-detail-change$)|TestLiteScan/lite-scan-sbom|TestScanSBOMSyftJSONRejected$' + run: 'TestScan$/scan-sbom|TestDiff/(diff-sbom$|diff-sbom-detail-change$)|TestLiteScan/lite-scan-sbom|TestScanSBOMSyftJSONRejected$|TestScanSBOMExportDistribution$' - name: plugin run: 'TestPluginWorkflows' - name: container diff --git a/dev-docs/ARCHITECTURE.md b/dev-docs/ARCHITECTURE.md index b556d462..52606175 100644 --- a/dev-docs/ARCHITECTURE.md +++ b/dev-docs/ARCHITECTURE.md @@ -166,6 +166,27 @@ inherit these checkout validation controls. Reachability data lives on `sdk.Vulnerability.Reachability` rather than on `Finding.Reachability` because `--analyze` must be useful without `--audit`. Matchers populate the OSV-aligned `Vulnerability` record on the PURL-keyed registry package; the analyzer enriches it in place; the output layer resolves the analyzer's annotation by `(Finding.PackageRef, Finding.VulnerabilityID)` when emitting SARIF and the JSON `Finding` projection. This keeps a single source of truth (the registry) and removes the per-manifest sync that the old graph-mutating model required. +### Decision: SBOM distribution data is classified, not passed through + +`sdk.Dependency.ResolvedURL` is not a URL. Detectors write whatever their lockfile records: npm/pnpm/yarn/bun write the exact tarball, Bundler writes the `GEM remote:` registry root, Cargo writes a `registry+`/`sparse+`/`git+`-prefixed index or repo, swiftpm writes a repository, and uv, pipenv, pub, and npm link entries can all write a **local filesystem path**. Some private-registry URLs embed a token. + +`internal/sbom/locator.go` therefore classifies each value into artifact / VCS / registry-root / nothing before it can reach an SBOM, rather than mapping the field straight onto `PackageDownloadLocation`. Two rules are load-bearing: + +1. **An http(s) scheme is required, and userinfo is rejected.** This is what keeps build-machine directory layout and private-registry credentials out of a published document. The check is on the value, never on `Source` — uv's `path`/`editable` values arrive under non-`file` sources. +2. **A registry root never becomes a download location.** `https://rubygems.org/` is schema-valid and both validators accept it, so the failure would be silent and plausible: every consumer would read it as the artifact's origin. `NOASSERTION` is the honest answer. + +Unrecognized shapes degrade toward the weaker claim (registry root, then nothing) rather than toward the stronger one. `FuzzClassifyResolvedURL` asserts the safety property directly: a classified value is either empty or an absolute http(s)/`git+http(s)` URL with no userinfo. + +### Decision: ingested SBOM assertions ride `Dependency.Metadata` + +SBOM ingest is not decode-then-encode. `internal/detectors/sbom` decodes to a neutral `Document`, `sbom.ToGraph` converts it to an `sdk.Graph`, the graph flows through the whole pipeline, and export rebuilds a *fresh* document via `FromDepGraph`. Anything not carried onto the `sdk.Dependency` is lost before export — which is why supplier, description, and external references were previously dropped by a format conversion even though both decoders could see them. + +They are carried on `Dependency.Metadata` under `bomly.sbom.*` keys, following the precedent of `sdk.SetDetectionLicenses`. This needs no SDK contract change, and consolidation preserves the keys because it clones nodes rather than rebuilding them. + +`ToGraph` deliberately does **not** set `Dependency.Source` from an ingested document. `Source` feeds `RegistryMatchEligible()`, so classifying an ingested component as `git` or `url` would quietly make it ineligible for enrichment and break `scan --sbom --enrich`. Setting `ResolvedURL` alone is safe; eligibility never reads it. + +Precedence is *detection classifies, ingest corrects, enrichment fills gaps*. Ingested values win over Bomly's own derivation because re-exporting must not silently rewrite another producer's assertion — and because `ToGraph` drops `Source`, re-deriving the bucket would be strictly worse information than the one the source document already chose. + ### Decision: external lookups use `Coordinates.EcosystemName()`, never the bare `Name` `Coordinates` stores identity as `Org` + `Name` following the PURL namespace/name split, so `Name` alone is `postcss` for both `postcss` and `@tailwindcss/postcss`. Anything that leaves the process under a name — Grype's DB search, the OSV name-keyed query, name-derived cache keys, SBOM component names, the bare specifiers `jsreach` matches imports against — must use `EcosystemName()`, which rebuilds the ecosystem-native form (`@org/name` for npm, `org:name` for the Maven family, `org/name` for Go, Composer, Swift, and GitHub Actions). diff --git a/docs/SBOM.md b/docs/SBOM.md index 7afca480..dc2bbbb3 100644 --- a/docs/SBOM.md +++ b/docs/SBOM.md @@ -99,6 +99,38 @@ Both formats carry: knows fixed versions, each vulnerability carries a `recommendation` ("Upgrade to "). No guidance is invented when no fix is known. SPDX 2.3 has no equivalent field. +- Where each package came from, when the lockfile records it (see below). + +### Where a package came from + +Lockfiles record very different things in the same field. Some name the exact +file that was downloaded, some name only the registry the ecosystem fetches +from, and some name a directory on the machine that ran the scan. Bomly sorts +each value into one of three kinds and emits it accordingly: + +| What the lockfile recorded | SPDX | CycloneDX | +| --- | --- | --- | +| The exact package file | `downloadLocation` | `distribution` reference | +| A source repository | `downloadLocation` (`git+` form) | `vcs` reference | +| Only a registry root | `NOASSERTION` | `distribution` reference, marked as a registry root | +| A local path, or nothing usable | `NOASSERTION` | nothing | + +Two rules matter here: + +- **A registry root is never used as a download location.** `https://rubygems.org/` + is a valid URL, so a validator would accept it, but it is not where that gem + came from. Saying nothing is better than saying something false. +- **Local filesystem paths are never written to an SBOM**, and neither are URLs + carrying a username or password. Several lockfile formats record a directory + on the build machine, or a private-registry URL with a token embedded in it. + Publishing either one would leak information about the machine that ran the + scan. + +Coverage follows what each ecosystem actually records. npm, pnpm, yarn, and bun +lockfiles name the exact package archive, so those get a real download location. +Bundler, Cargo, pub, and most Python lockfiles record only a registry or index +root, so those get a registry reference and `NOASSERTION`. Go modules, Maven, +Gradle, NuGet, and Composer record no location at all. ### Document identity @@ -151,11 +183,24 @@ and the contact fields in the creation-info comment. When `manufacturer` is set, it becomes the supplier of the primary component in both formats (CycloneDX `metadata.manufacturer`, SPDX `PackageSupplier` on the package the document DESCRIBES). Supplier is not defaulted to anything -when the field is unset, and per-component supplier and description data is -never invented: those fields stay absent unless a data source actually -provides them. Third-party CRA profile checks will flag the missing +when the field is unset. Third-party CRA profile checks will flag the missing manufacturer/contact metadata until the `sbom` section is configured. +Per-component supplier and description are never invented. Bomly writes them +in exactly one case: when you scan an SBOM that already contains them, they are +carried through to the output so that converting between formats does not throw +away another producer's assertions. On the primary component, a configured +`manufacturer` takes precedence, because that is your own claim about your own +product. + +Bomly does not derive supplier or description for third-party packages. Doing so +would need registry metadata that Bomly does not fetch — deps.dev, its enrichment +source for package facts, asserts neither field. Guessing (for example treating a +PURL namespace as a supplier) would put invented claims into a compliance +document, so those fields stay absent instead. This means an enriched scan of a +project does not by itself satisfy the CRA profile's mandatory direct-dependency +supplier check. + When `--enrich` is set, components are enriched from the matching-stage package registry (keyed by PURL): @@ -165,18 +210,28 @@ registry (keyed by PURL): - Vulnerabilities — CycloneDX as a first-class `vulnerabilities` array (ratings, CWEs, advisories, `affects`); SPDX as `SECURITY`/`advisory` external references. - End-of-life status (CycloneDX `bomly:eol*` properties, SPDX package comment). +- The source repository resolved by the OpenSSF Scorecard matcher (CycloneDX + `vcs` external reference, SPDX `PackageSourceInfo`). A repository recorded by + the detector itself is more precise, so it wins when both are known. Reachability annotations and other Bomly-specific metadata are emitted in the JSON output (`--json` or `--format json`), not in the standard SBOM formats. See [Output formats](OUTPUT_FORMATS.md). ### Preservation and conversion limits Bomly preserves component identity (including PURL), dependency edges, roots, -scope, package type, licenses, digests, CPEs, and the enrichment fields described -above when the destination format has an equivalent representation. Encoding is -deterministic when the scan timestamp and document identifiers are fixed. +scope, package type, licenses, digests, CPEs, supplier, originator/publisher, +description, download and repository locations, and the enrichment fields +described above when the destination format has an equivalent representation. +Encoding is deterministic when the scan timestamp and document identifiers are +fixed. Some information necessarily becomes less specific during conversion: +- SPDX 2.3 has no external-reference category for arbitrary links, so CycloneDX + references other than `distribution` and `vcs` (for example `documentation` + or `issue-tracker`) are dropped when converting to SPDX. They survive a + CycloneDX-to-CycloneDX pass. + - CycloneDX vulnerability records preserve ratings, CWEs, affected component references, descriptions, and advisory URLs. SPDX 2.3 represents each vulnerability as a package security advisory reference, so ratings, affected diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 27fbe393..6977b031 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -47,6 +47,14 @@ func (c cycloneDXCodec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, e if props := cycloneDXEOLProperties(comp.EOL); len(props) > 0 { component.Properties = &props } + if comp.Supplier != "" { + component.Supplier = &cdx.OrganizationalEntity{Name: comp.Supplier} + } + component.Publisher = comp.Originator + component.Description = comp.Description + if refs := cycloneDXComponentReferences(comp); len(refs) > 0 { + component.ExternalReferences = &refs + } components = append(components, component) } bom.Components = &components @@ -122,16 +130,7 @@ func (c cycloneDXCodec) decodeJSON(data []byte) (*Document, error) { componentByID := make(map[string]Component) if bom.Components != nil { for _, comp := range *bom.Components { - componentByID[comp.BOMRef] = Component{ - ID: comp.BOMRef, - Name: comp.Name, - Type: string(comp.Type), - Scope: string(comp.Scope), - Version: comp.Version, - PURL: comp.PackageURL, - Copyright: comp.Copyright, - Licenses: parseCycloneDXLicenses(comp.Licenses), - } + componentByID[comp.BOMRef] = componentFromCycloneDX(comp) } } @@ -169,16 +168,7 @@ func (c cycloneDXCodec) decodeJSON(data []byte) (*Document, error) { if len(componentByID) == 0 && bom.Metadata != nil && bom.Metadata.Component != nil { root := bom.Metadata.Component - componentByID[root.BOMRef] = Component{ - ID: root.BOMRef, - Name: root.Name, - Type: string(root.Type), - Scope: string(root.Scope), - Version: root.Version, - PURL: root.PackageURL, - Copyright: root.Copyright, - Licenses: parseCycloneDXLicenses(root.Licenses), - } + componentByID[root.BOMRef] = componentFromCycloneDX(*root) } components := make([]Component, 0, len(componentByID)) @@ -265,6 +255,98 @@ func cycloneDXSecurityReferences(p Provenance) []cdx.ExternalReference { return refs } +// componentFromCycloneDX projects one CycloneDX component onto the neutral +// model. Both decode paths (the component inventory and the metadata-only +// fallback) share it so they cannot drift apart. +func componentFromCycloneDX(comp cdx.Component) Component { + component := Component{ + ID: comp.BOMRef, + Name: comp.Name, + Type: string(comp.Type), + Scope: string(comp.Scope), + Version: comp.Version, + PURL: comp.PackageURL, + Copyright: comp.Copyright, + Licenses: parseCycloneDXLicenses(comp.Licenses), + Description: comp.Description, + Originator: comp.Publisher, + } + if comp.CPE != "" { + component.CPEs = []string{comp.CPE} + } + if comp.Supplier != nil && comp.Supplier.Name != "" { + component.Supplier = comp.Supplier.Name + component.SupplierType = "Organization" + } + if comp.Hashes != nil { + for _, hash := range *comp.Hashes { + component.Digests = append(component.Digests, Digest{ + Algorithm: strings.ToLower(strings.ReplaceAll(string(hash.Algorithm), "-", "")), + Value: hash.Value, + }) + } + } + if comp.ExternalReferences != nil { + for _, ref := range *comp.ExternalReferences { + switch ref.Type { + case cdx.ERTypeDistribution: + applyLocator(&component, classifyResolvedURL(ref.URL, "", "")) + case cdx.ERTypeVCS: + component.VCSURL = ref.URL + default: + component.ExternalRefs = append(component.ExternalRefs, ExternalRef{ + Type: string(ref.Type), + URL: ref.URL, + Comment: ref.Comment, + }) + } + } + } + return component +} + +// cycloneDXComponentReferences builds the per-component external references: +// where the package came from, its source repository, and anything preserved +// from an ingested document. +// +// A registry root is emitted only when no exact artifact URL is known, and it +// carries a comment saying so — it names where the ecosystem fetches from, not +// where this package came from. +func cycloneDXComponentReferences(component Component) []cdx.ExternalReference { + refs := make([]cdx.ExternalReference, 0, 3+len(component.ExternalRefs)) + + switch { + case component.ArtifactURL != "": + refs = append(refs, cdx.ExternalReference{Type: cdx.ERTypeDistribution, URL: component.ArtifactURL}) + case component.RegistryURL != "": + refs = append(refs, cdx.ExternalReference{ + Type: cdx.ERTypeDistribution, + URL: component.RegistryURL, + Comment: "Registry root; not the exact artifact location", + }) + } + + // VCSURL is detector-supplied and version-exact, so it wins over the + // scorecard repository. Emitting both would assert the same repository + // twice from two sources. + if vcs := firstNonEmpty(component.VCSURL, component.Repository); vcs != "" { + refs = append(refs, cdx.ExternalReference{Type: cdx.ERTypeVCS, URL: vcs}) + } + + for _, ref := range component.ExternalRefs { + refs = append(refs, cdx.ExternalReference{ + Type: cdx.ExternalReferenceType(ref.Type), + URL: ref.URL, + Comment: ref.Comment, + }) + } + + if len(refs) == 0 { + return nil + } + return refs +} + // bareEmail returns value when it looks like a plain email address (no URI // scheme), otherwise "". func bareEmail(value string) string { diff --git a/internal/sbom/distribution_test.go b/internal/sbom/distribution_test.go new file mode 100644 index 00000000..7d581165 --- /dev/null +++ b/internal/sbom/distribution_test.go @@ -0,0 +1,256 @@ +package sbom + +import ( + "encoding/json" + "strings" + "testing" + + cdx "github.com/CycloneDX/cyclonedx-go" + "github.com/bomly-dev/bomly-sdk" + v23 "github.com/spdx/tools-golang/spdx/v2/v2_3" +) + +// graphWithResolvedURL builds a one-node graph carrying a detector-supplied +// resolved URL, the shape every distribution assertion is derived from. +func graphWithResolvedURL(t *testing.T, resolved string, source sdk.DependencySource, ecosystem sdk.Ecosystem) *sdk.Graph { + t.Helper() + g := sdk.New() + node := sdk.NewDependencyWithID("pkg@1.0.0", sdk.Dependency{ + Coordinates: sdk.Coordinates{ + Name: "pkg", + Version: "1.0.0", + PURL: "pkg:npm/pkg@1.0.0", + Ecosystem: ecosystem, + }, + Source: source, + ResolvedURL: resolved, + }) + if err := g.AddNode(node); err != nil { + t.Fatalf("add node: %v", err) + } + return g +} + +func spdxPackageFor(t *testing.T, g *sdk.Graph, opts BuildOptions) *v23.Package { + t.Helper() + out, err := MarshalDepGraphJSON(g, TargetSPDX23JSON, opts, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal spdx: %v", err) + } + var doc v23.Document + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("unmarshal spdx: %v", err) + } + for _, p := range doc.Packages { + if p != nil && p.PackageName == "pkg" { + return p + } + } + t.Fatal("package not found in spdx output") + return nil +} + +func cycloneDXComponentFor(t *testing.T, g *sdk.Graph, opts BuildOptions) cdx.Component { + t.Helper() + out, err := MarshalDepGraphJSON(g, TargetCycloneDX17JSON, opts, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal cyclonedx: %v", err) + } + var bom cdx.BOM + if err := json.Unmarshal(out, &bom); err != nil { + t.Fatalf("unmarshal cyclonedx: %v", err) + } + if bom.Components == nil { + t.Fatal("no components in cyclonedx output") + } + for _, comp := range *bom.Components { + if comp.Name == "pkg" { + return comp + } + } + t.Fatal("component not found in cyclonedx output") + return cdx.Component{} +} + +func externalRefURL(comp cdx.Component, refType cdx.ExternalReferenceType) string { + if comp.ExternalReferences == nil { + return "" + } + for _, ref := range *comp.ExternalReferences { + if ref.Type == refType { + return ref.URL + } + } + return "" +} + +func TestDistributionProjection(t *testing.T) { + cases := []struct { + name string + resolved string + source sdk.DependencySource + ecosystem sdk.Ecosystem + wantDownload string + wantDistRef string + wantVCSRef string + }{ + { + name: "npm tarball becomes a download location", + resolved: "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz", + source: sdk.DependencySourceRegistry, + ecosystem: sdk.EcosystemNPM, + wantDownload: "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz", + wantDistRef: "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz", + }, + { + // The single most important regression case: a registry root is + // a valid URL that both validators accept, and reading it as the + // artifact's origin would be wrong. + name: "rubygems registry root never becomes a download location", + resolved: "https://rubygems.org/", + source: sdk.DependencySourceRegistry, + ecosystem: sdk.EcosystemRuby, + wantDownload: "NOASSERTION", + wantDistRef: "https://rubygems.org/", + }, + { + name: "cargo registry index never becomes a download location", + resolved: "registry+https://github.com/rust-lang/crates.io-index", + source: sdk.DependencySourceRegistry, + ecosystem: sdk.EcosystemRust, + wantDownload: "NOASSERTION", + wantDistRef: "https://github.com/rust-lang/crates.io-index", + }, + { + name: "cargo git pin normalizes to the spdx vcs form", + resolved: "git+https://github.com/a/b?rev=deadbeef", + source: sdk.DependencySourceGit, + ecosystem: sdk.EcosystemRust, + wantDownload: "git+https://github.com/a/b@deadbeef", + wantVCSRef: "git+https://github.com/a/b@deadbeef", + }, + { + name: "uv editable local path is never emitted", + resolved: "/Users/ahmed/dev/mylib", + source: sdk.DependencySourceFile, + ecosystem: sdk.EcosystemPython, + wantDownload: "NOASSERTION", + }, + { + name: "credential bearing url is never emitted", + resolved: "https://tok:s3cret@nexus.corp/pkg-1.0.0.tgz", + source: sdk.DependencySourceRegistry, + ecosystem: sdk.EcosystemNPM, + wantDownload: "NOASSERTION", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + g := graphWithResolvedURL(t, tc.resolved, tc.source, tc.ecosystem) + + pkg := spdxPackageFor(t, g, BuildOptions{}) + if pkg.PackageDownloadLocation != tc.wantDownload { + t.Fatalf("downloadLocation = %q, want %q", pkg.PackageDownloadLocation, tc.wantDownload) + } + + comp := cycloneDXComponentFor(t, g, BuildOptions{}) + if got := externalRefURL(comp, cdx.ERTypeDistribution); got != tc.wantDistRef { + t.Fatalf("distribution ref = %q, want %q", got, tc.wantDistRef) + } + if got := externalRefURL(comp, cdx.ERTypeVCS); got != tc.wantVCSRef { + t.Fatalf("vcs ref = %q, want %q", got, tc.wantVCSRef) + } + }) + } +} + +// TestDistributionNeverLeaksLocalPaths asserts on the encoded bytes, the way a +// consumer would see them. A path leak is the one genuinely dangerous failure +// mode of this feature. +func TestDistributionNeverLeaksLocalPaths(t *testing.T) { + secrets := []string{ + "/Users/ahmed/secret-project", + "../../etc/passwd", + "file:///Users/ahmed/x", + "https://tok:s3cret@nexus.corp/a-1.0.tgz", + } + for _, secret := range secrets { + g := graphWithResolvedURL(t, secret, sdk.DependencySourceRegistry, sdk.EcosystemNPM) + for _, target := range []Target{TargetSPDX23JSON, TargetCycloneDX17JSON} { + out, err := MarshalDepGraphJSON(g, target, BuildOptions{}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal %s: %v", target, err) + } + needle := secret + if idx := strings.Index(needle, "://"); idx >= 0 { + needle = needle[idx+3:] + } + if strings.Contains(string(out), needle) { + t.Fatalf("%s output leaked %q", target, secret) + } + } + } +} + +func TestScorecardRepositoryProjection(t *testing.T) { + const purl = "pkg:npm/pkg@1.0.0" + g := graphWithResolvedURL(t, "", sdk.DependencySourceRegistry, sdk.EcosystemNPM) + + registry := sdk.NewPackageRegistry() + pkg := registry.Ensure(purl) + pkg.Name, pkg.Version = "pkg", "1.0.0" + pkg.Scorecard = &sdk.PackageScorecard{Repository: "github.com/owner/repo"} + + opts := BuildOptions{Registry: registry} + comp := cycloneDXComponentFor(t, g, opts) + if got := externalRefURL(comp, cdx.ERTypeVCS); got != "https://github.com/owner/repo" { + t.Fatalf("vcs ref = %q, want the normalized scorecard repository", got) + } + + spdxPkg := spdxPackageFor(t, g, opts) + if want := "Source repository: https://github.com/owner/repo"; spdxPkg.PackageSourceInfo != want { + t.Fatalf("sourceInfo = %q, want %q", spdxPkg.PackageSourceInfo, want) + } +} + +// TestScorecardRepositoryAbsentWithoutEnrichment covers the registry-less call +// shape used by the benchmark, where every registry-sourced field must degrade +// to omitted rather than panic or invent a value. +func TestScorecardRepositoryAbsentWithoutEnrichment(t *testing.T) { + g := graphWithResolvedURL(t, "", sdk.DependencySourceRegistry, sdk.EcosystemNPM) + + comp := cycloneDXComponentFor(t, g, BuildOptions{}) + if comp.ExternalReferences != nil { + t.Fatalf("expected no external references, got %+v", *comp.ExternalReferences) + } + if spdxPkg := spdxPackageFor(t, g, BuildOptions{}); spdxPkg.PackageSourceInfo != "" { + t.Fatalf("expected no sourceInfo, got %q", spdxPkg.PackageSourceInfo) + } +} + +// TestDetectorVCSWinsOverScorecardRepository keeps the two sources from each +// asserting the same repository twice. +func TestDetectorVCSWinsOverScorecardRepository(t *testing.T) { + const purl = "pkg:npm/pkg@1.0.0" + g := graphWithResolvedURL(t, "git+https://github.com/a/b?rev=deadbeef", sdk.DependencySourceGit, sdk.EcosystemNPM) + + registry := sdk.NewPackageRegistry() + pkg := registry.Ensure(purl) + pkg.Name, pkg.Version = "pkg", "1.0.0" + pkg.Scorecard = &sdk.PackageScorecard{Repository: "github.com/owner/repo"} + + comp := cycloneDXComponentFor(t, g, BuildOptions{Registry: registry}) + vcsRefs := 0 + for _, ref := range *comp.ExternalReferences { + if ref.Type == cdx.ERTypeVCS { + vcsRefs++ + if ref.URL != "git+https://github.com/a/b@deadbeef" { + t.Fatalf("vcs ref = %q, want the detector-supplied pin", ref.URL) + } + } + } + if vcsRefs != 1 { + t.Fatalf("expected exactly one vcs reference, got %d", vcsRefs) + } +} diff --git a/internal/sbom/graph.go b/internal/sbom/graph.go index 3944f3c2..e905dd74 100644 --- a/internal/sbom/graph.go +++ b/internal/sbom/graph.go @@ -38,6 +38,11 @@ func ToGraph(doc *Document) (*sdk.Graph, error) { if purl := strings.TrimSpace(component.PURL); purl != "" { packageID = purl } + // Deliberately no Source: sdk.Dependency.Source feeds + // RegistryMatchEligible, so classifying an ingested component as git + // or url would quietly make it ineligible for enrichment and break + // `scan --sbom --enrich`. ResolvedURL alone is safe — eligibility + // never reads it. pkg := sdk.NewDependencyWithID(packageID, sdk.Dependency{Coordinates: sdk.Coordinates{Name: component.Name, Version: component.Version, @@ -46,9 +51,13 @@ func ToGraph(doc *Document) (*sdk.Graph, error) { Type: sdk.ParsePackageType(component.Type), PURL: strings.TrimSpace(component.PURL)}, Scopes: sdk.ScopesOf(sdk.Scope(component.Scope)), - Copyright: component.Copyright, + Copyright: component.Copyright, + CPEs: append([]string(nil), component.CPEs...), + Digests: graphDigests(component.Digests), + ResolvedURL: firstNonEmpty(component.ArtifactURL, component.VCSURL, component.RegistryURL), }) sdk.SetDetectionLicenses(pkg, graphLicenses(component.Licenses)) + setIngestedMetadata(pkg, component) if _, exists := depsGraph.Node(packageID); !exists { if err := depsGraph.AddNode(pkg); err != nil { @@ -104,6 +113,23 @@ func isDocumentRootPseudoPackage(component Component) bool { return false } +// graphDigests projects component digests onto a graph node. Ingest dropped +// these previously, so an incoming document's hashes did not survive a format +// conversion. +func graphDigests(digests []Digest) []sdk.Digest { + if len(digests) == 0 { + return nil + } + out := make([]sdk.Digest, 0, len(digests)) + for _, digest := range digests { + out = append(out, sdk.Digest{ + Algorithm: sdk.DigestAlgorithm(digest.Algorithm), + Value: digest.Value, + }) + } + return out +} + func graphLicenses(licenses []License) []sdk.PackageLicense { if len(licenses) == 0 { return nil diff --git a/internal/sbom/ingest_metadata.go b/internal/sbom/ingest_metadata.go new file mode 100644 index 00000000..a3ec2c40 --- /dev/null +++ b/internal/sbom/ingest_metadata.go @@ -0,0 +1,133 @@ +package sbom + +import ( + "github.com/bomly-dev/bomly-sdk" +) + +// Well-known Dependency.Metadata keys used to carry assertions from an +// ingested SBOM across the graph hop. +// +// Ingest is not a decode-then-encode pass: a document becomes an sdk.Graph +// (ToGraph), flows through the whole pipeline, and a fresh document is rebuilt +// from that graph (FromDepGraph). Anything not placed on the sdk.Dependency +// is therefore lost before export. Riding Dependency.Metadata follows the +// precedent set by sdk.SetDetectionLicenses and needs no SDK contract change. +const ( + metadataKeySupplier = "bomly.sbom.supplier" + metadataKeySupplierType = "bomly.sbom.supplier_type" + metadataKeyOriginator = "bomly.sbom.originator" + metadataKeyOriginatorType = "bomly.sbom.originator_type" + metadataKeyDescription = "bomly.sbom.description" + metadataKeyRepository = "bomly.sbom.repository" + metadataKeyArtifactURL = "bomly.sbom.artifact_url" + metadataKeyVCSURL = "bomly.sbom.vcs_url" + metadataKeyRegistryURL = "bomly.sbom.registry_url" + metadataKeyExternalRefs = "bomly.sbom.external_refs" +) + +// setIngestedMetadata stashes the assertions an ingested SBOM made about a +// component onto the graph node that replaces it. +func setIngestedMetadata(dep *sdk.Dependency, component Component) { + if dep == nil { + return + } + values := map[string]string{ + metadataKeySupplier: component.Supplier, + metadataKeySupplierType: component.SupplierType, + metadataKeyOriginator: component.Originator, + metadataKeyOriginatorType: component.OriginatorType, + metadataKeyDescription: component.Description, + metadataKeyRepository: component.Repository, + metadataKeyArtifactURL: component.ArtifactURL, + metadataKeyVCSURL: component.VCSURL, + metadataKeyRegistryURL: component.RegistryURL, + } + for key, value := range values { + if value == "" { + continue + } + if dep.Metadata == nil { + dep.Metadata = make(map[string]any) + } + dep.Metadata[key] = value + } + + if len(component.ExternalRefs) == 0 { + return + } + refs := make([]any, 0, len(component.ExternalRefs)) + for _, ref := range component.ExternalRefs { + refs = append(refs, map[string]any{ + "type": ref.Type, + "url": ref.URL, + "comment": ref.Comment, + }) + } + if dep.Metadata == nil { + dep.Metadata = make(map[string]any) + } + dep.Metadata[metadataKeyExternalRefs] = refs +} + +// applyIngestedMetadata restores assertions stashed by setIngestedMetadata. +// +// The ingested values win over anything Bomly derived: they are the source +// document's own claims, and re-exporting must not silently rewrite another +// producer's assertion. For the distribution fields this also repairs a +// deliberate loss of fidelity — ToGraph does not record DependencySource (see +// its comment), so re-classifying the round-tripped URL would be strictly +// worse information than the bucket the source document already chose. +// +// Values are read defensively: a future plugin hop would JSON-round-trip this +// map, so anything that does not type-assert is skipped rather than trusted. +func applyIngestedMetadata(component *Component, metadata map[string]any) { + if component == nil || len(metadata) == 0 { + return + } + + targets := map[string]*string{ + metadataKeySupplier: &component.Supplier, + metadataKeySupplierType: &component.SupplierType, + metadataKeyOriginator: &component.Originator, + metadataKeyOriginatorType: &component.OriginatorType, + metadataKeyDescription: &component.Description, + metadataKeyRepository: &component.Repository, + } + for key, target := range targets { + if value, ok := metadata[key].(string); ok && value != "" { + *target = value + } + } + + // The three distribution fields move as a set: a partial overwrite could + // leave a component claiming both an ingested artifact URL and a stale + // derived registry root. + artifact, _ := metadata[metadataKeyArtifactURL].(string) + vcs, _ := metadata[metadataKeyVCSURL].(string) + registry, _ := metadata[metadataKeyRegistryURL].(string) + if artifact != "" || vcs != "" || registry != "" { + component.ArtifactURL, component.VCSURL, component.RegistryURL = artifact, vcs, registry + } + + raw, ok := metadata[metadataKeyExternalRefs].([]any) + if !ok { + return + } + refs := make([]ExternalRef, 0, len(raw)) + for _, entry := range raw { + fields, ok := entry.(map[string]any) + if !ok { + continue + } + refType, _ := fields["type"].(string) + refURL, _ := fields["url"].(string) + comment, _ := fields["comment"].(string) + if refType == "" || refURL == "" { + continue + } + refs = append(refs, ExternalRef{Type: refType, URL: refURL, Comment: comment}) + } + if len(refs) > 0 { + component.ExternalRefs = refs + } +} diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go new file mode 100644 index 00000000..45f2f1a9 --- /dev/null +++ b/internal/sbom/locator.go @@ -0,0 +1,257 @@ +package sbom + +import ( + "net/url" + "strings" + + "github.com/bomly-dev/bomly-sdk" +) + +// LocatorKind classifies what a detector-supplied resolved URL actually points +// at. Detectors record wildly different things in the same field — an npm +// lockfile records a tarball URL, a Gemfile.lock records the registry root, a +// uv lock can record a local directory — so the kind must be decided per value +// rather than per ecosystem. +type LocatorKind int + +const ( + // LocatorNone means nothing publishable could be asserted. Local + // filesystem paths and credential-bearing URLs land here. + LocatorNone LocatorKind = iota + // LocatorArtifact is a concrete downloadable file. + LocatorArtifact + // LocatorVCS is a source-control location. + LocatorVCS + // LocatorRegistryRoot is a registry or index root: it says where the + // ecosystem fetches from, not where this package came from. + LocatorRegistryRoot +) + +// Locator is the classified form of a resolved URL, ready for projection into +// an SBOM. URL is empty whenever Kind is LocatorNone. +type Locator struct { + Kind LocatorKind + URL string +} + +// artifactExtensions are the archive suffixes that mark a URL path as a +// concrete package artifact rather than a registry endpoint. The list is an +// allowlist on purpose: an unrecognized path shape degrades to a registry +// root, which is never used as a download location. +var artifactExtensions = []string{ + ".tgz", ".tar.gz", ".tar.bz2", ".tar.xz", ".tar", ".zip", + ".whl", ".gem", ".crate", ".jar", ".nupkg", ".egg", ".conda", +} + +// classifyResolvedURL decides what raw points at, given the detector's own +// source classification and the package ecosystem. +// +// The function is deliberately conservative: it emits nothing unless the value +// is an unambiguous http(s) URL, and it prefers the weaker LocatorRegistryRoot +// over LocatorArtifact whenever the path shape is not recognizably an archive. +// An SBOM that omits a download location is correct; one that points at the +// wrong place, or at the developer's home directory, is not. +func classifyResolvedURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosystem) Locator { + raw = strings.TrimSpace(raw) + if raw == "" { + return Locator{} + } + + // Cargo records the raw Cargo.lock `source` string, which carries its own + // scheme prefix. The prefix is a stronger signal than anything else here. + hint := LocatorNone + switch { + case strings.HasPrefix(raw, "registry+"): + raw, hint = strings.TrimPrefix(raw, "registry+"), LocatorRegistryRoot + case strings.HasPrefix(raw, "sparse+"): + raw, hint = strings.TrimPrefix(raw, "sparse+"), LocatorRegistryRoot + case strings.HasPrefix(raw, "git+"): + raw, hint = strings.TrimPrefix(raw, "git+"), LocatorVCS + } + + parsed, err := url.Parse(raw) + if err != nil { + return Locator{} + } + + // The scheme gate is what keeps local filesystem layout out of published + // SBOMs. Several detectors put bare paths in this field (uv `editable` + // and `path`, pipenv `path`, pub `path`, npm link entries), and they do + // not consistently carry DependencySourceFile, so the check must be on + // the value rather than on source. + switch strings.ToLower(parsed.Scheme) { + case "http", "https": + default: + return Locator{} + } + if parsed.Host == "" { + return Locator{} + } + + // A lockfile pointing at a private registry can embed a token. Publishing + // it in an SBOM would leak a live credential, so drop the value entirely + // rather than try to strip the userinfo and emit the rest. + if parsed.User != nil { + return Locator{} + } + + switch source { + case sdk.DependencySourceFile, sdk.DependencySourceProject, sdk.DependencySourceWorkspace: + return Locator{} + } + + if hint == LocatorVCS { + return normalizeVCS(parsed) + } + if hint == LocatorRegistryRoot { + return Locator{Kind: LocatorRegistryRoot, URL: parsed.String()} + } + + // Swift package identity is the repository URL, so swiftpm records a repo + // even for `kind: registry` pins. Without this the extension check below + // would demote them to registry roots. + if ecosystem == sdk.EcosystemSwift { + return normalizeVCS(parsed) + } + + if source == sdk.DependencySourceGit || strings.HasSuffix(parsed.Path, ".git") { + return normalizeVCS(parsed) + } + + if isConcreteArtifactPath(parsed.Path) { + return Locator{Kind: LocatorArtifact, URL: parsed.String()} + } + return Locator{Kind: LocatorRegistryRoot, URL: parsed.String()} +} + +// isConcreteArtifactPath reports whether the final path segment names a +// package archive. +func isConcreteArtifactPath(path string) bool { + idx := strings.LastIndex(path, "/") + last := strings.ToLower(path[idx+1:]) + if last == "" { + return false + } + for _, ext := range artifactExtensions { + if strings.HasSuffix(last, ext) { + return true + } + } + return false +} + +// normalizeVCS renders parsed in the SPDX 2.3 version-control form, +// "+://[@]". +// +// That grammar has no query component, so a cargo value such as +// "git+https://host/a/b?rev=abc" cannot be passed through as-is: the revision +// is moved to the "@" suffix and the query is dropped. +func normalizeVCS(parsed *url.URL) Locator { + revision := "" + for _, key := range []string{"rev", "tag", "branch"} { + if value := strings.TrimSpace(parsed.Query().Get(key)); value != "" { + revision = value + break + } + } + if revision == "" { + revision = strings.TrimSpace(parsed.Fragment) + } + + clean := *parsed + clean.RawQuery = "" + clean.Fragment = "" + + // A repository location needs a path: "https://host" alone identifies no + // repository. Requiring one also removes an ambiguity in the SPDX form — + // with an empty path, the "@" suffix would re-parse as URL + // userinfo ("git+https://host@rev" reads as user "host", host "rev"). + if path := strings.Trim(clean.Path, "/"); path == "" { + return Locator{} + } + + base := strings.TrimSuffix(clean.String(), "/") + out := "git+" + base + if isSafeRevision(revision) { + out += "@" + revision + } + return Locator{Kind: LocatorVCS, URL: out} +} + +// isSafeRevision reports whether revision is a plausible git revision that can +// be appended to an SPDX version-control locator verbatim. +// +// The value originates in a lockfile, so it may contain anything at all; a +// control character or a space would produce an unparseable locator. An +// unsafe revision is dropped rather than escaped, leaving the still-correct +// repository URL without a pinned revision. +func isSafeRevision(revision string) bool { + if revision == "" || len(revision) > 256 { + return false + } + for _, r := range revision { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '.', r == '_', r == '-', r == '/', r == '+': + default: + return false + } + } + return true +} + +// normalizeRepositoryURL renders a scheme-less canonical repository +// identifier, such as the "github.com/owner/repo" the OpenSSF Scorecard +// matcher records, as an absolute https URL. +// +// CycloneDX external reference URLs are iri-reference typed, so a scheme-less +// value would validate but be read as a relative reference. Returns "" when +// the value does not look like a host-qualified repository. +func normalizeRepositoryURL(repo string) string { + repo = strings.TrimSpace(repo) + if repo == "" || strings.ContainsAny(repo, " \t\r\n") { + return "" + } + if strings.Contains(repo, "://") { + parsed, err := url.Parse(repo) + if err != nil || parsed.User != nil || parsed.Host == "" { + return "" + } + switch strings.ToLower(parsed.Scheme) { + case "http", "https": + return parsed.String() + default: + return "" + } + } + + host, rest, ok := strings.Cut(repo, "/") + if !ok || !isHostname(host) || strings.TrimSpace(rest) == "" { + return "" + } + + // Re-parse rather than trusting the shape check: the path may still hold + // something url.Parse rejects, such as an invalid percent-escape. + candidate := "https://" + repo + parsed, err := url.Parse(candidate) + if err != nil || parsed.Host != host || parsed.User != nil { + return "" + } + return candidate +} + +// isHostname reports whether value looks like a dotted DNS hostname. +func isHostname(value string) bool { + if !strings.Contains(value, ".") || strings.HasPrefix(value, ".") || strings.HasSuffix(value, ".") { + return false + } + for _, r := range value { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '.', r == '-': + default: + return false + } + } + return true +} diff --git a/internal/sbom/locator_fuzz_test.go b/internal/sbom/locator_fuzz_test.go new file mode 100644 index 00000000..b317a3c9 --- /dev/null +++ b/internal/sbom/locator_fuzz_test.go @@ -0,0 +1,119 @@ +package sbom + +import ( + "testing" + + "github.com/bomly-dev/bomly-sdk" + "github.com/bomly-dev/bomly-sdk/testkit" +) + +// FuzzClassifyResolvedURL exercises the resolved-URL classifier, which runs +// over values taken verbatim from untrusted lockfiles (npm `resolved`, uv +// `editable`/`path`, Cargo.lock `source`, Gemfile.lock `GEM remote:`, and so +// on). +// +// The invariant under test is a safety property, not just an absence of +// panics: a classified value is either nothing at all or an absolute http(s) +// URL with no embedded credentials. That is what keeps local filesystem paths +// and private-registry tokens out of published SBOMs. +func FuzzClassifyResolvedURL(f *testing.F) { + for _, seed := range []string{ + "https://registry.npmjs.org/react/-/react-18.2.0.tgz", + "https://files.pythonhosted.org/packages/x/req-2.31.0-py3-none-any.whl", + "https://rubygems.org/", + "https://pypi.org/simple", + "registry+https://github.com/rust-lang/crates.io-index", + "sparse+https://index.crates.io/", + "git+https://github.com/a/b?rev=deadbeef", + "git+https://github.com/a/b#cafebabe", + "https://github.com/apple/swift-nio.git", + "https://registry.npmjs.org/@babel/core/-/core-7.0.0.tgz", + "https://tok:s3cret@nexus.corp/a/b-1.0.tgz", + "/Users/ahmed/dev/mylib", + "../vendor/foo", + ".", + `C:\src\lib`, + "file:///tmp/x.tgz", + "link:../pkg", + "workspace:*", + "git@github.com:a/b.git", + "https://", + // Regression: a host-only URL with a fragment once produced + // "git+http://0@0", whose "@" suffix re-parses as userinfo. + "http://0#0", + // Regression: a control character in the fragment once reached the + // revision suffix and produced an unparseable locator. + "http://0/0#\x02", + "::::", + "", + " ", + "\x00\uFFFD", + "%%%%", + "https://host/%2e%2e/%2e%2e/etc/passwd", + } { + f.Add(seed) + } + + sources := []sdk.DependencySource{ + sdk.DependencySourceRegistry, sdk.DependencySourceGit, sdk.DependencySourceURL, + sdk.DependencySourceFile, sdk.DependencySourceProject, sdk.DependencySourceWorkspace, "", + } + ecosystems := []sdk.Ecosystem{ + sdk.EcosystemNPM, sdk.EcosystemPython, sdk.EcosystemRust, + sdk.EcosystemRuby, sdk.EcosystemSwift, sdk.EcosystemUnknown, + } + + f.Fuzz(func(t *testing.T, raw string) { + if len(raw) > testkit.MaxFuzzInputSize { + return + } + for _, source := range sources { + for _, ecosystem := range ecosystems { + got := classifyResolvedURL(raw, source, ecosystem) + assertPublishableLocator(t, got, raw) + + if again := classifyResolvedURL(raw, source, ecosystem); again != got { + t.Fatalf("nondeterministic classification of %q: %+v vs %+v", raw, got, again) + } + } + } + }) +} + +// FuzzNormalizeRepositoryURL exercises the scorecard repository renderer, +// whose input is an API-supplied repository identifier. +func FuzzNormalizeRepositoryURL(f *testing.F) { + for _, seed := range []string{ + "github.com/kubernetes/kubernetes", + "https://github.com/a/b", + "github.com", + "not-a-host", + "/Users/ahmed/repo", + "has space/repo", + "ftp://github.com/a/b", + "https://tok:sec@github.com/a/b", + "", + "\x00", + // Regression: a dot-containing but non-hostname prefix once produced + // "https://%./0", which is an invalid percent-escape. + "%./0", + // Regression: a scheme with no host once produced the bare "http:". + "http://", + } { + f.Add(seed) + } + + f.Fuzz(func(t *testing.T, raw string) { + if len(raw) > testkit.MaxFuzzInputSize { + return + } + got := normalizeRepositoryURL(raw) + if got == "" { + return + } + assertPublishableLocator(t, Locator{Kind: LocatorVCS, URL: got}, raw) + if again := normalizeRepositoryURL(raw); again != got { + t.Fatalf("nondeterministic normalization of %q: %q vs %q", raw, got, again) + } + }) +} diff --git a/internal/sbom/locator_test.go b/internal/sbom/locator_test.go new file mode 100644 index 00000000..a7a55cb6 --- /dev/null +++ b/internal/sbom/locator_test.go @@ -0,0 +1,149 @@ +package sbom + +import ( + "net/url" + "strings" + "testing" + + "github.com/bomly-dev/bomly-sdk" +) + +func TestClassifyResolvedURL(t *testing.T) { + cases := []struct { + name string + raw string + source sdk.DependencySource + ecosystem sdk.Ecosystem + wantKind LocatorKind + wantURL string + }{ + // Artifacts: the npm family is the cleanest case. + {"npm tarball", "https://registry.npmjs.org/react/-/react-18.2.0.tgz", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorArtifact, "https://registry.npmjs.org/react/-/react-18.2.0.tgz"}, + {"python wheel", "https://files.pythonhosted.org/packages/x/req-2.31.0-py3-none-any.whl", sdk.DependencySourceURL, sdk.EcosystemPython, LocatorArtifact, "https://files.pythonhosted.org/packages/x/req-2.31.0-py3-none-any.whl"}, + {"python sdist", "https://files.pythonhosted.org/packages/x/req-2.31.0.tar.gz", sdk.DependencySourceURL, sdk.EcosystemPython, LocatorArtifact, "https://files.pythonhosted.org/packages/x/req-2.31.0.tar.gz"}, + {"gem artifact", "https://rubygems.org/gems/rake-13.0.6.gem", sdk.DependencySourceRegistry, sdk.EcosystemRuby, LocatorArtifact, "https://rubygems.org/gems/rake-13.0.6.gem"}, + + // Registry roots must never become a download location. + {"rubygems root", "https://rubygems.org/", sdk.DependencySourceRegistry, sdk.EcosystemRuby, LocatorRegistryRoot, "https://rubygems.org/"}, + {"pypi simple index", "https://pypi.org/simple", sdk.DependencySourceRegistry, sdk.EcosystemPython, LocatorRegistryRoot, "https://pypi.org/simple"}, + {"pub root", "https://pub.dev", sdk.DependencySourceRegistry, sdk.EcosystemDart, LocatorRegistryRoot, "https://pub.dev"}, + {"cargo registry prefix", "registry+https://github.com/rust-lang/crates.io-index", sdk.DependencySourceRegistry, sdk.EcosystemRust, LocatorRegistryRoot, "https://github.com/rust-lang/crates.io-index"}, + {"cargo sparse prefix", "sparse+https://index.crates.io/", sdk.DependencySourceRegistry, sdk.EcosystemRust, LocatorRegistryRoot, "https://index.crates.io/"}, + + // VCS, including the SPDX grammar normalization. + {"cargo git with rev", "git+https://github.com/a/b?rev=deadbeef", sdk.DependencySourceGit, sdk.EcosystemRust, LocatorVCS, "git+https://github.com/a/b@deadbeef"}, + {"cargo git with fragment", "git+https://github.com/a/b#cafebabe", sdk.DependencySourceGit, sdk.EcosystemRust, LocatorVCS, "git+https://github.com/a/b@cafebabe"}, + {"cargo git with tag", "git+https://github.com/a/b?tag=v1.2.3", sdk.DependencySourceGit, sdk.EcosystemRust, LocatorVCS, "git+https://github.com/a/b@v1.2.3"}, + {"git suffix path", "https://github.com/apple/swift-nio.git", sdk.DependencySourceGit, sdk.EcosystemSwift, LocatorVCS, "git+https://github.com/apple/swift-nio.git"}, + {"swiftpm registry kind still vcs", "https://github.com/apple/swift-nio", sdk.DependencySourceRegistry, sdk.EcosystemSwift, LocatorVCS, "git+https://github.com/apple/swift-nio"}, + + // Local filesystem paths must never be emitted. + {"absolute posix path", "/Users/ahmed/dev/mylib", sdk.DependencySourceFile, sdk.EcosystemPython, LocatorNone, ""}, + {"absolute path with non-file source", "/home/runner/work/x/y", sdk.DependencySourceURL, sdk.EcosystemPython, LocatorNone, ""}, + {"relative path", "../vendor/foo", sdk.DependencySourceFile, sdk.EcosystemPython, LocatorNone, ""}, + {"uv editable dot", ".", sdk.DependencySourceFile, sdk.EcosystemPython, LocatorNone, ""}, + {"windows path", `C:\src\lib`, sdk.DependencySourceFile, sdk.EcosystemNPM, LocatorNone, ""}, + {"file scheme", "file:///tmp/x.tgz", sdk.DependencySourceFile, sdk.EcosystemNPM, LocatorNone, ""}, + {"npm link specifier", "link:../pkg", sdk.DependencySourceWorkspace, sdk.EcosystemNPM, LocatorNone, ""}, + {"npm workspace specifier", "workspace:*", sdk.DependencySourceWorkspace, sdk.EcosystemNPM, LocatorNone, ""}, + {"npm link local dir", "https://registry.npmjs.org/a/-/a-1.0.0.tgz", sdk.DependencySourceWorkspace, sdk.EcosystemNPM, LocatorNone, ""}, + {"scp style git", "git@github.com:a/b.git", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorNone, ""}, + + // Credentials must never reach an SBOM. + {"userinfo token", "https://tok:s3cret@nexus.corp/a/b-1.0.tgz", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, + {"username only", "https://tok@nexus.corp/a/b-1.0.tgz", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, + + // Degenerate input. + {"empty", "", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, + {"whitespace", " ", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, + {"scheme only", "https://", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, + {"colons", "::::", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := classifyResolvedURL(tc.raw, tc.source, tc.ecosystem) + if got.Kind != tc.wantKind { + t.Fatalf("kind = %v, want %v (url %q)", got.Kind, tc.wantKind, got.URL) + } + if got.URL != tc.wantURL { + t.Fatalf("url = %q, want %q", got.URL, tc.wantURL) + } + }) + } +} + +// TestClassifyResolvedURLNeverEmitsLocalPaths is the invariant that matters +// most: whatever a lockfile contains, a classified value is either empty or an +// absolute http(s)/git+http(s) URL. A regression here leaks the developer's or +// CI runner's directory layout into a published SBOM. +func TestClassifyResolvedURLNeverEmitsLocalPaths(t *testing.T) { + inputs := []string{ + "/Users/ahmed/secret-project/lib", "../../etc/passwd", "./local", ".", + `C:\Users\ahmed\proj`, "file:///Users/ahmed/x", "link:../a", "workspace:^1", + "git@github.com:a/b.git", "ssh://git@github.com/a/b.git", "", " ", + "https://user:pass@host/a.tgz", + } + for _, in := range inputs { + for _, source := range []sdk.DependencySource{ + sdk.DependencySourceRegistry, sdk.DependencySourceGit, sdk.DependencySourceURL, + sdk.DependencySourceFile, sdk.DependencySourceProject, sdk.DependencySourceWorkspace, "", + } { + got := classifyResolvedURL(in, source, sdk.EcosystemNPM) + assertPublishableLocator(t, got, in) + } + } +} + +// assertPublishableLocator enforces the emit-or-nothing contract shared by the +// table test and the fuzz target. +func assertPublishableLocator(t *testing.T, got Locator, input string) { + t.Helper() + if got.Kind == LocatorNone { + if got.URL != "" { + t.Fatalf("LocatorNone carried url %q for input %q", got.URL, input) + } + return + } + if got.URL == "" { + t.Fatalf("kind %v carried an empty url for input %q", got.Kind, input) + } + switch { + case strings.HasPrefix(got.URL, "https://"), strings.HasPrefix(got.URL, "http://"), + strings.HasPrefix(got.URL, "git+https://"), strings.HasPrefix(got.URL, "git+http://"): + default: + t.Fatalf("emitted non-http(s) locator %q for input %q", got.URL, input) + } + // Check userinfo structurally rather than by looking for "@": scoped npm + // packages ("/@babel/core/") and VCS revisions ("@deadbeef") both contain + // one legitimately. + parsed, err := url.Parse(strings.TrimPrefix(got.URL, "git+")) + if err != nil { + t.Fatalf("emitted unparseable locator %q for input %q", got.URL, input) + } + if parsed.User != nil { + t.Fatalf("emitted locator with userinfo %q for input %q", got.URL, input) + } +} + +func TestNormalizeRepositoryURL(t *testing.T) { + cases := []struct{ in, want string }{ + {"github.com/kubernetes/kubernetes", "https://github.com/kubernetes/kubernetes"}, + {"gitlab.com/org/repo", "https://gitlab.com/org/repo"}, + {"https://github.com/a/b", "https://github.com/a/b"}, + {"", ""}, + {" ", ""}, + {"not-a-host", ""}, + {"github.com", ""}, + {"github.com/", ""}, + {"/Users/ahmed/repo", ""}, + {"has space/repo", ""}, + {"ftp://github.com/a/b", ""}, + {"https://tok:sec@github.com/a/b", ""}, + } + for _, tc := range cases { + if got := normalizeRepositoryURL(tc.in); got != tc.want { + t.Fatalf("normalizeRepositoryURL(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/internal/sbom/model.go b/internal/sbom/model.go index 31607005..9cefab48 100644 --- a/internal/sbom/model.go +++ b/internal/sbom/model.go @@ -135,6 +135,36 @@ type Component struct { Digests []Digest Vulnerabilities []Vulnerability EOL *EOL + + // Where the package came from, classified from the detector-supplied + // resolved URL. At most one of ArtifactURL and VCSURL is set; + // RegistryURL is the weaker fallback naming a registry or index root, + // and is deliberately never used as a download location. + ArtifactURL string + VCSURL string + RegistryURL string + + // Repository is a canonical "github.com/owner/repo" source repository + // supplied by the OpenSSF Scorecard matcher during enrichment. + Repository string + + // Assertions preserved verbatim from an ingested SBOM. Bomly never + // invents these; they are present only when the source document, or a + // matcher, actually asserted them. + Supplier string + SupplierType string + Originator string + OriginatorType string + Description string + ExternalRefs []ExternalRef +} + +// ExternalRef is an external reference carried through from an ingested SBOM +// so that a format conversion does not silently discard it. +type ExternalRef struct { + Type string + URL string + Comment string } // Dependency describes one package relationship list in the intermediate SBOM model. diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go new file mode 100644 index 00000000..0ede3c38 --- /dev/null +++ b/internal/sbom/roundtrip_test.go @@ -0,0 +1,224 @@ +package sbom + +import ( + "encoding/json" + "testing" + + cdx "github.com/CycloneDX/cyclonedx-go" + v23 "github.com/spdx/tools-golang/spdx/v2/v2_3" +) + +// supplierRichCycloneDX is an ingested document asserting the fields Bomly +// itself never invents. A third party asserted them, so a format conversion +// must carry them through rather than silently drop them. +const supplierRichCycloneDX = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "metadata": {"component": {"bom-ref": "root", "type": "application", "name": "app", "version": "1.0.0"}}, + "components": [ + { + "bom-ref": "root", + "type": "application", + "name": "app", + "version": "1.0.0", + "purl": "pkg:npm/app@1.0.0" + }, + { + "bom-ref": "pkg:npm/left-pad@1.3.0", + "type": "library", + "name": "left-pad", + "version": "1.3.0", + "purl": "pkg:npm/left-pad@1.3.0", + "description": "String left padding", + "publisher": "azer", + "supplier": {"name": "Example Supplier Inc."}, + "cpe": "cpe:2.3:a:example:left-pad:1.3.0:*:*:*:*:*:*:*", + "hashes": [{"alg": "SHA-256", "content": "abc123"}], + "externalReferences": [ + {"type": "distribution", "url": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"}, + {"type": "vcs", "url": "https://github.com/stevemao/left-pad"}, + {"type": "documentation", "url": "https://example.com/docs"} + ] + } + ], + "dependencies": [{"ref": "root", "dependsOn": ["pkg:npm/left-pad@1.3.0"]}] +}` + +// ingestAndReexport runs the real ingest path: decode, convert to a graph the +// way the SBOM detector does, then rebuild a document from that graph. This is +// what `bomly scan --sbom --path in.cdx.json --format spdx` performs, and it +// is why decoder changes alone are not enough. +func ingestAndReexport(t *testing.T, in []byte, target Target) []byte { + t.Helper() + doc, _, err := UnmarshalAutoJSON(in) + if err != nil { + t.Fatalf("unmarshal: %v", err) + } + graph, err := ToGraph(doc) + if err != nil { + t.Fatalf("to graph: %v", err) + } + out, err := MarshalDepGraphJSON(graph, target, BuildOptions{}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal %s: %v", target, err) + } + return out +} + +func TestIngestedAssertionsSurviveConversionToSPDX(t *testing.T) { + out := ingestAndReexport(t, []byte(supplierRichCycloneDX), TargetSPDX23JSON) + + var doc v23.Document + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("unmarshal spdx: %v", err) + } + + var pkg *v23.Package + for _, p := range doc.Packages { + if p != nil && p.PackageName == "left-pad" { + pkg = p + } + } + if pkg == nil { + t.Fatal("left-pad missing from spdx output") + } + + if pkg.PackageSupplier == nil || pkg.PackageSupplier.Supplier != "Example Supplier Inc." { + t.Fatalf("supplier = %+v, want the ingested value", pkg.PackageSupplier) + } + if pkg.PackageDescription != "String left padding" { + t.Fatalf("description = %q, want the ingested value", pkg.PackageDescription) + } + if pkg.PackageOriginator == nil || pkg.PackageOriginator.Originator != "azer" { + t.Fatalf("originator = %+v, want the ingested publisher", pkg.PackageOriginator) + } + if want := "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"; pkg.PackageDownloadLocation != want { + t.Fatalf("downloadLocation = %q, want %q", pkg.PackageDownloadLocation, want) + } + if len(parseSPDXCPEs(pkg.PackageExternalReferences)) != 1 { + t.Fatal("ingested CPE did not survive conversion") + } + if len(pkg.PackageChecksums) != 1 { + t.Fatalf("ingested checksum did not survive conversion: %+v", pkg.PackageChecksums) + } +} + +func TestIngestedAssertionsSurviveCycloneDXRoundTrip(t *testing.T) { + out := ingestAndReexport(t, []byte(supplierRichCycloneDX), TargetCycloneDX17JSON) + + var bom cdx.BOM + if err := json.Unmarshal(out, &bom); err != nil { + t.Fatalf("unmarshal cyclonedx: %v", err) + } + if bom.Components == nil { + t.Fatal("no components in output") + } + + var comp *cdx.Component + for i := range *bom.Components { + if (*bom.Components)[i].Name == "left-pad" { + comp = &(*bom.Components)[i] + } + } + if comp == nil { + t.Fatal("left-pad missing from cyclonedx output") + } + + if comp.Supplier == nil || comp.Supplier.Name != "Example Supplier Inc." { + t.Fatalf("supplier = %+v, want the ingested value", comp.Supplier) + } + if comp.Description != "String left padding" { + t.Fatalf("description = %q, want the ingested value", comp.Description) + } + if comp.Publisher != "azer" { + t.Fatalf("publisher = %q, want the ingested value", comp.Publisher) + } + if got := externalRefURL(*comp, cdx.ERTypeDistribution); got != "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz" { + t.Fatalf("distribution ref = %q, want the ingested value", got) + } + if got := externalRefURL(*comp, cdx.ERTypeVCS); got != "https://github.com/stevemao/left-pad" { + t.Fatalf("vcs ref = %q, want the ingested value", got) + } + // An external reference type Bomly has no opinion about must pass through + // rather than be dropped as unrecognized. + if got := externalRefURL(*comp, cdx.ERTypeDocumentation); got != "https://example.com/docs" { + t.Fatalf("documentation ref = %q, want it preserved verbatim", got) + } +} + +// TestManufacturerWinsOnRootIngestedSupplierSurvivesElsewhere pins the +// precedence rule: the user's configured claim about their own product beats +// an ingested claim on the primary package, and does not erase ingested +// suppliers on dependencies. +func TestManufacturerWinsOnRootIngestedSupplierSurvivesElsewhere(t *testing.T) { + doc, _, err := UnmarshalAutoJSON([]byte(supplierRichCycloneDX)) + if err != nil { + t.Fatalf("unmarshal: %v", err) + } + graph, err := ToGraph(doc) + if err != nil { + t.Fatalf("to graph: %v", err) + } + out, err := MarshalDepGraphJSON(graph, TargetSPDX23JSON, BuildOptions{ + ProjectRoot: &ProjectRoot{Name: "demo-project"}, + Provenance: Provenance{Manufacturer: "Example Org"}, + }, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal spdx: %v", err) + } + + var spdxDoc v23.Document + if err := json.Unmarshal(out, &spdxDoc); err != nil { + t.Fatalf("unmarshal spdx: %v", err) + } + + for _, p := range spdxDoc.Packages { + if p == nil || p.PackageSupplier == nil { + continue + } + switch p.PackageName { + case "left-pad": + if p.PackageSupplier.Supplier != "Example Supplier Inc." { + t.Fatalf("dependency supplier = %q, want the ingested value", p.PackageSupplier.Supplier) + } + default: + if p.PackageSupplier.Supplier != "Example Org" { + t.Fatalf("root supplier = %q, want the configured manufacturer", p.PackageSupplier.Supplier) + } + } + } +} + +// TestSPDXNoAssertionSupplierDecodesToAbsent keeps the reserved marker from +// being re-emitted as if it were a real supplier name. +func TestSPDXNoAssertionSupplierDecodesToAbsent(t *testing.T) { + const in = `{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [{ + "name": "left-pad", + "SPDXID": "SPDXRef-Package-left-pad", + "versionInfo": "1.3.0", + "downloadLocation": "NOASSERTION", + "supplier": "NOASSERTION", + "filesAnalyzed": false + }] + }` + + doc, _, err := UnmarshalAutoJSON([]byte(in)) + if err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, component := range doc.Components { + if component.Supplier != "" { + t.Fatalf("NOASSERTION decoded to supplier %q, want absent", component.Supplier) + } + if component.ArtifactURL != "" || component.VCSURL != "" || component.RegistryURL != "" { + t.Fatalf("NOASSERTION decoded to a distribution locator: %+v", component) + } + } +} diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index cff750e4..a4aad463 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -44,7 +44,7 @@ func (spdx23Codec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, error) PackageName: c.NameOrID(), PackageSPDXIdentifier: spdxID, PackageVersion: c.Version, - PackageDownloadLocation: "NOASSERTION", + PackageDownloadLocation: spdxDownloadLocation(c), FilesAnalyzed: false, PackageComment: spdxPackageComment(c), PackageLicenseDeclared: spdxLicenseValue(c.Licenses), @@ -53,12 +53,27 @@ func (spdx23Codec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, error) PackageChecksums: spdxChecksums(c.Digests), PackageExternalReferences: spdxExternalReferences(c), PrimaryPackagePurpose: spdxPrimaryPackagePurpose(c.Type), + PackageDescription: c.Description, + PackageSourceInfo: spdxSourceInfo(c), } + if c.Originator != "" { + pkg.PackageOriginator = &common.Originator{ + OriginatorType: spdxEntityType(c.OriginatorType), + Originator: c.Originator, + } + } + // The configured manufacturer is the user's own claim about their + // product, so it wins on the document's primary package. Everywhere + // else a supplier appears only when a source document asserted one. + supplier, supplierType := c.Supplier, c.SupplierType if _, isRoot := rootComponents[c.ID]; isRoot || IsProjectRootComponent(c) { if doc.Provenance.Manufacturer != "" { - pkg.PackageSupplier = &common.Supplier{SupplierType: "Organization", Supplier: doc.Provenance.Manufacturer} + supplier, supplierType = doc.Provenance.Manufacturer, "Organization" } } + if supplier != "" { + pkg.PackageSupplier = &common.Supplier{SupplierType: spdxEntityType(supplierType), Supplier: supplier} + } packages = append(packages, pkg) } @@ -143,7 +158,7 @@ func (spdx23Codec) decodeJSON(data []byte) (*Document, error) { continue } id := common.RenderElementID(p.PackageSPDXIdentifier) - components = append(components, Component{ + component := Component{ ID: id, Name: p.PackageName, Version: p.PackageVersion, @@ -154,7 +169,24 @@ func (spdx23Codec) decodeJSON(data []byte) (*Document, error) { PackageManager: parseSPDXPackageManager(p.PackageExternalReferences), Copyright: parseSPDXCopyright(p.PackageCopyrightText), Licenses: parseSPDXLicenses(p.PackageLicenseConcluded, p.PackageLicenseDeclared), - }) + Description: firstNonEmpty(parseSPDXEntity(p.PackageDescription), parseSPDXEntity(p.PackageSummary)), + Repository: parseSPDXSourceInfo(p.PackageSourceInfo), + CPEs: parseSPDXCPEs(p.PackageExternalReferences), + Digests: parseSPDXChecksums(p.PackageChecksums), + } + if p.PackageSupplier != nil { + component.Supplier = parseSPDXEntity(p.PackageSupplier.Supplier) + component.SupplierType = p.PackageSupplier.SupplierType + } + if p.PackageOriginator != nil { + component.Originator = parseSPDXEntity(p.PackageOriginator.Originator) + component.OriginatorType = p.PackageOriginator.OriginatorType + } + applyLocator(&component, classifyResolvedURL(parseSPDXEntity(p.PackageDownloadLocation), "", "")) + if home := strings.TrimSpace(p.PackageHomePage); home != "" && parseSPDXEntity(home) != "" { + component.ExternalRefs = append(component.ExternalRefs, ExternalRef{Type: "website", URL: home}) + } + components = append(components, component) } depsByRef := make(map[string][]string, len(components)) @@ -428,6 +460,72 @@ func spdxCopyrightValue(value string) string { return value } +// parseSPDXEntity normalizes an SPDX free-text field, treating the reserved +// NOASSERTION and NONE markers as absent rather than as literal values. +func parseSPDXEntity(value string) string { + value = strings.TrimSpace(value) + switch strings.ToUpper(value) { + case "", "NOASSERTION", "NONE": + return "" + } + return value +} + +// parseSPDXSourceInfo recovers a source repository written by spdxSourceInfo. +// Free text without the marker prefix is another producer's prose and is left +// alone rather than guessed at. +func parseSPDXSourceInfo(value string) string { + value = strings.TrimSpace(value) + if !strings.HasPrefix(value, spdxSourceInfoPrefix) { + return "" + } + return strings.TrimSpace(strings.TrimPrefix(value, spdxSourceInfoPrefix)) +} + +// spdxSourceInfoPrefix marks a source repository inside the free-text +// PackageSourceInfo field so the decoder can recover it deterministically. +const spdxSourceInfoPrefix = "Source repository: " + +// spdxDownloadLocation renders where the package was obtained. +// +// A registry root is deliberately not used: "https://rubygems.org/" is a +// syntactically valid URL that both validators accept, and every consumer +// would read it as the artifact's origin, which is false. NOASSERTION is the +// honest answer when only a registry root is known. +func spdxDownloadLocation(component Component) string { + if location := firstNonEmpty(component.ArtifactURL, component.VCSURL); location != "" { + return location + } + return "NOASSERTION" +} + +// spdxSourceInfo renders the source repository into SPDX's free-text origin +// field. +// +// PackageSourceInfo is defined as free text about the origin of the package, +// which is what a source repository is. PackageHomePage is a different claim +// (the package's home page) that Bomly does not know, and SPDX 2.3 has no +// version-control external-reference category. +func spdxSourceInfo(component Component) string { + if repo := strings.TrimSpace(component.Repository); repo != "" { + return spdxSourceInfoPrefix + repo + } + return "" +} + +// spdxEntityType normalizes a supplier or originator type to one SPDX accepts, +// defaulting to Organization. +func spdxEntityType(value string) string { + switch strings.ToLower(strings.TrimSpace(value)) { + case "person": + return "Person" + case "noassertion": + return "NOASSERTION" + default: + return "Organization" + } +} + func spdxExternalReferences(component Component) []*v23.PackageExternalReference { refs := make([]*v23.PackageExternalReference, 0, 1+len(component.CPEs)+len(component.Vulnerabilities)) if purl := strings.TrimSpace(component.PURL); purl != "" { @@ -489,6 +587,42 @@ func parseSPDXLicenses(values ...string) []License { return nil } +// parseSPDXCPEs recovers CPE identifiers from a package's security external +// references. +func parseSPDXCPEs(refs []*v23.PackageExternalReference) []string { + var cpes []string + for _, ref := range refs { + if ref == nil { + continue + } + if strings.EqualFold(strings.TrimSpace(ref.RefType), "cpe23Type") { + if locator := strings.TrimSpace(ref.Locator); locator != "" { + cpes = append(cpes, locator) + } + } + } + return cpes +} + +// parseSPDXChecksums projects SPDX checksums onto neutral digests. +func parseSPDXChecksums(checksums []common.Checksum) []Digest { + if len(checksums) == 0 { + return nil + } + digests := make([]Digest, 0, len(checksums)) + for _, checksum := range checksums { + algorithm := strings.ToLower(strings.ReplaceAll(string(checksum.Algorithm), "-", "")) + if algorithm == "" || strings.TrimSpace(checksum.Value) == "" { + continue + } + digests = append(digests, Digest{Algorithm: algorithm, Value: strings.TrimSpace(checksum.Value)}) + } + if len(digests) == 0 { + return nil + } + return digests +} + func parseSPDXPURL(refs []*v23.PackageExternalReference) string { for _, ref := range refs { if ref == nil { diff --git a/internal/sbom/transform.go b/internal/sbom/transform.go index 891d4f4a..daded4b5 100644 --- a/internal/sbom/transform.go +++ b/internal/sbom/transform.go @@ -46,7 +46,11 @@ func FromDepGraph(g *sdk.Graph, opts BuildOptions) (*Document, error) { Copyright: pkg.Copyright, Licenses: componentLicenses(sdk.DetectionLicenses(pkg)), Digests: componentDigests(pkg.Digests), + CPEs: append([]string(nil), pkg.CPEs...), } + // Detection classifies, ingest corrects, enrichment fills gaps. + applyLocator(&component, classifyResolvedURL(pkg.ResolvedURL, pkg.Source, pkg.Ecosystem)) + applyIngestedMetadata(&component, pkg.Metadata) enrichComponentFromRegistry(&component, opts.Registry, pkg.PURL) components = append(components, component) depsByRef[pkg.ID] = nil @@ -291,6 +295,33 @@ func enrichComponentFromRegistry(component *Component, registry *sdk.PackageRegi LatestVersion: pkg.EOL.LatestVersion, } } + if pkg.Scorecard != nil { + // Only fill, never clear: an unnormalizable scorecard value must not + // erase a repository an ingested document asserted. + if repo := normalizeRepositoryURL(pkg.Scorecard.Repository); repo != "" { + component.Repository = repo + } + } + // The registry copy of ResolvedURL is normally the detection value echoed + // back, so it only fills a gap a matcher supplied. Source is not carried + // on Package, so classification here is shape-driven only. + if component.ArtifactURL == "" && component.VCSURL == "" && component.RegistryURL == "" { + applyLocator(component, classifyResolvedURL(pkg.ResolvedURL, "", pkg.Ecosystem)) + } +} + +// applyLocator projects a classified resolved URL onto a component. A +// LocatorNone leaves the component untouched, so an unpublishable value simply +// results in no assertion. +func applyLocator(component *Component, locator Locator) { + switch locator.Kind { + case LocatorArtifact: + component.ArtifactURL = locator.URL + case LocatorVCS: + component.VCSURL = locator.URL + case LocatorRegistryRoot: + component.RegistryURL = locator.URL + } } // vulnerabilitiesFromPackage projects matching-stage advisories into the diff --git a/scripts/run-fuzz.sh b/scripts/run-fuzz.sh index 36bf6147..36476e5b 100755 --- a/scripts/run-fuzz.sh +++ b/scripts/run-fuzz.sh @@ -31,6 +31,8 @@ targets=( "github.com/bomly-dev/bomly-cli/internal/detectors/swiftpm FuzzDepGraphFromSwiftResolved" "github.com/bomly-dev/bomly-cli/internal/sbom FuzzUnmarshalAutoJSON" "github.com/bomly-dev/bomly-cli/internal/sbom FuzzNormalizeSPDXLicenseExpression" + "github.com/bomly-dev/bomly-cli/internal/sbom FuzzClassifyResolvedURL" + "github.com/bomly-dev/bomly-cli/internal/sbom FuzzNormalizeRepositoryURL" "github.com/bomly-dev/bomly-cli/internal/baseline FuzzLoad" "github.com/bomly-dev/bomly-cli/internal/engine FuzzConsolidateVulnerabilities" "github.com/bomly-dev/bomly-cli/internal/plugin FuzzPluginPathSanitizers" diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index bcd0d6ab..a5059387 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -511,6 +511,109 @@ func TestExplain(t *testing.T) { } } +// TestScanSBOMExportDistribution covers SBOM *export* end to end, which the +// golden-backed cases above do not: they all emit `--format json`, so nothing +// asserted the SPDX/CycloneDX bytes themselves. +// +// The case is deliberately property-asserting rather than golden-backed. An +// SBOM document carries four volatile fields (documentNamespace, serialNumber, +// created/timestamp, tool version) that normalizeJSON is not written for, so a +// golden would flap on every run. +// +// It is pinned to bomly-dev/example-javascript-npm because npm lockfiles record +// the exact package archive — the one ecosystem family where the artifact path +// actually fires. The leak assertion is the important one: no scan-machine path +// may ever reach a published SBOM. +func TestScanSBOMExportDistribution(t *testing.T) { + t.Parallel() + requireTool(t, "npm") + + dir := t.TempDir() + spdxPath := filepath.Join(dir, "out.spdx.json") + cdxPath := filepath.Join(dir, "out.cdx.json") + + _, stderr, code := runBomly(t, + "scan", "--url", "https://github.com/bomly-dev/example-javascript-npm", "--ref", "v1.0.0", + "--detectors", "npm", + "-o", "spdx="+spdxPath, "-o", "cyclonedx="+cdxPath, + ) + if code != 0 { + t.Fatalf("bomly exited %d\nstderr:\n%s", code, stderr) + } + + spdxRaw, err := os.ReadFile(spdxPath) + if err != nil { + t.Fatalf("read spdx: %v", err) + } + cdxRaw, err := os.ReadFile(cdxPath) + if err != nil { + t.Fatalf("read cyclonedx: %v", err) + } + + var spdxDoc struct { + Packages []struct { + Name string `json:"name"` + DownloadLocation string `json:"downloadLocation"` + } `json:"packages"` + } + if err := json.Unmarshal(spdxRaw, &spdxDoc); err != nil { + t.Fatalf("unmarshal spdx: %v", err) + } + + registryDownloads := 0 + for _, pkg := range spdxDoc.Packages { + switch { + case pkg.DownloadLocation == "NOASSERTION": + case strings.HasPrefix(pkg.DownloadLocation, "https://"): + registryDownloads++ + default: + t.Fatalf("package %q has a download location that is neither NOASSERTION nor https: %q", + pkg.Name, pkg.DownloadLocation) + } + } + if registryDownloads == 0 { + t.Fatal("expected npm packages to carry registry download locations, got none") + } + + var cdxDoc struct { + Components []struct { + Name string `json:"name"` + ExternalReferences []struct { + Type string `json:"type"` + URL string `json:"url"` + } `json:"externalReferences"` + } `json:"components"` + } + if err := json.Unmarshal(cdxRaw, &cdxDoc); err != nil { + t.Fatalf("unmarshal cyclonedx: %v", err) + } + distributions := 0 + for _, comp := range cdxDoc.Components { + for _, ref := range comp.ExternalReferences { + if ref.Type == "distribution" { + distributions++ + if !strings.HasPrefix(ref.URL, "https://") { + t.Fatalf("component %q distribution reference is not https: %q", comp.Name, ref.URL) + } + } + } + } + if distributions == 0 { + t.Fatal("expected npm components to carry distribution references, got none") + } + + // Nothing about the machine that ran the scan may appear in either + // document. The temp dir stands in for any local path the detectors saw. + for name, raw := range map[string][]byte{"spdx": spdxRaw, "cyclonedx": cdxRaw} { + if strings.Contains(string(raw), dir) { + t.Fatalf("%s output leaked the scan working directory %q", name, dir) + } + if strings.Contains(string(raw), "file://") { + t.Fatalf("%s output contains a file:// locator", name) + } + } +} + // TestScanSBOMSyftJSONRejected locks in the syft-JSON ingest removal end to // end: `--sbom` on a syft-format JSON file must exit 3 (resolution failure — // the detector cannot produce a graph) with the actionable conversion hint, From f60fae685905af3ba94e32d58a299b6f2c41b94d Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 09:13:35 -0700 Subject: [PATCH 02/23] fix(sbom): gate ingested URLs and reject non-userinfo credentials Review found the safety gate had two holes, both of which could put a credential or a local path into a published document. Credentials outside userinfo. Signed and private-registry links carry them as query parameters ("?token=", "?X-Amz-Signature="), which the userinfo check did not see and parsed.String() re-emitted verbatim. A benign query parameter cannot be told apart from a credential, so any query or fragment now disqualifies a non-VCS locator. VCS locators stay exempt because normalizeVCS already discards both and keeps only a character-checked revision. Ingested URLs bypassed the gate entirely. A CycloneDX `vcs` reference, an SPDX PackageHomePage, and every passthrough external reference were stored raw and re-emitted, so a hostile or careless source document could launder a file:// path or an embedded credential into Bomly's output. Ingest is untrusted input and now passes the same checks a lockfile value does. Also from review: - Ingested VCS URLs are normalized to the SPDX git+ form. They become PackageDownloadLocation directly, where a bare https URL made a repository look like an ordinary package download. - SPDX PackageDownloadLocation is no longer re-classified by path shape. The field already asserts a download location, so an exact endpoint without an archive suffix was being demoted to a registry root and re-exported as NOASSERTION. - SHA3-256/384/512 survive ingest. Stripping every hyphen produced "sha3256", which neither encoder matches, silently dropping the checksum. - SPDX PackageHomePage is restored on export; it was decoded but never re-emitted, so an SPDX-to-SPDX pass dropped a field SPDX represents exactly. - Scorecard repository no longer overwrites an ingested one, matching the documented ingest-wins precedence, and is suppressed in SPDX PackageSourceInfo when a detector VCS location is already emitted, so one package cannot assert two different source repositories. - CycloneDX supplier URLs are preserved rather than flattened to a name. - cpe22Type references are accepted on ingest alongside cpe23Type. - The Component locator comment no longer claims at most one of ArtifactURL/VCSURL is set; an ingested document can assert both. - The smoke leak check no longer relies on the output directory, which would miss a leak of the separate bomly-git-* clone path. Co-Authored-By: Claude Opus 5 --- dev-docs/ARCHITECTURE.md | 4 +- docs/SBOM.md | 10 +-- internal/sbom/cyclonedx.go | 25 ++++++- internal/sbom/ingest_metadata.go | 3 + internal/sbom/locator.go | 88 ++++++++++++++++++++++++ internal/sbom/locator_test.go | 9 ++- internal/sbom/model.go | 11 +-- internal/sbom/roundtrip_test.go | 111 ++++++++++++++++++++++++++++++- internal/sbom/spdx23.go | 31 +++++++-- internal/sbom/transform.go | 38 +++++++++-- test/smoke/smoke_test.go | 26 ++++++-- 11 files changed, 325 insertions(+), 31 deletions(-) diff --git a/dev-docs/ARCHITECTURE.md b/dev-docs/ARCHITECTURE.md index 52606175..3f46e839 100644 --- a/dev-docs/ARCHITECTURE.md +++ b/dev-docs/ARCHITECTURE.md @@ -172,7 +172,9 @@ Reachability data lives on `sdk.Vulnerability.Reachability` rather than on `Find `internal/sbom/locator.go` therefore classifies each value into artifact / VCS / registry-root / nothing before it can reach an SBOM, rather than mapping the field straight onto `PackageDownloadLocation`. Two rules are load-bearing: -1. **An http(s) scheme is required, and userinfo is rejected.** This is what keeps build-machine directory layout and private-registry credentials out of a published document. The check is on the value, never on `Source` — uv's `path`/`editable` values arrive under non-`file` sources. +1. **An http(s) scheme is required, and credentials are rejected.** This is what keeps build-machine directory layout and private-registry credentials out of a published document. Credentials travel in two places: `user:password@host` userinfo, and query or fragment values on signed and private-registry links (`?token=`, `?X-Amz-Signature=`). A benign query parameter cannot be told apart from a credential, so any query or fragment disqualifies a non-VCS locator; VCS locators are exempt only because `normalizeVCS` discards both and keeps a character-checked revision. The check is on the value, never on `Source` — uv's `path`/`editable` values arrive under non-`file` sources. + + The same gate applies to URLs read out of an **ingested** SBOM. Those are untrusted input that gets re-emitted verbatim, so a hostile or careless document could otherwise launder a `file://` path or a credential into output Bomly publishes. 2. **A registry root never becomes a download location.** `https://rubygems.org/` is schema-valid and both validators accept it, so the failure would be silent and plausible: every consumer would read it as the artifact's origin. `NOASSERTION` is the honest answer. Unrecognized shapes degrade toward the weaker claim (registry root, then nothing) rather than toward the stronger one. `FuzzClassifyResolvedURL` asserts the safety property directly: a classified value is either empty or an absolute http(s)/`git+http(s)` URL with no userinfo. diff --git a/docs/SBOM.md b/docs/SBOM.md index dc2bbbb3..4a5670d0 100644 --- a/docs/SBOM.md +++ b/docs/SBOM.md @@ -121,10 +121,12 @@ Two rules matter here: is a valid URL, so a validator would accept it, but it is not where that gem came from. Saying nothing is better than saying something false. - **Local filesystem paths are never written to an SBOM**, and neither are URLs - carrying a username or password. Several lockfile formats record a directory - on the build machine, or a private-registry URL with a token embedded in it. - Publishing either one would leak information about the machine that ran the - scan. + carrying a credential. Several lockfile formats record a directory on the + build machine, or a private-registry URL with a token in it — either in the + `user:password@host` position or as a query parameter such as `?token=` on a + signed download link. Any of these is dropped rather than published. The same + check applies to URLs read out of an SBOM Bomly ingests, since those are + untrusted input too. Coverage follows what each ecosystem actually records. npm, pnpm, yarn, and bun lockfiles name the exact package archive, so those get a real download location. diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 6977b031..94d1b5bd 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -48,7 +48,12 @@ func (c cycloneDXCodec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, e component.Properties = &props } if comp.Supplier != "" { - component.Supplier = &cdx.OrganizationalEntity{Name: comp.Supplier} + supplier := &cdx.OrganizationalEntity{Name: comp.Supplier} + if comp.SupplierURL != "" { + urls := []string{comp.SupplierURL} + supplier.URL = &urls + } + component.Supplier = supplier } component.Publisher = comp.Originator component.Description = comp.Description @@ -277,11 +282,19 @@ func componentFromCycloneDX(comp cdx.Component) Component { if comp.Supplier != nil && comp.Supplier.Name != "" { component.Supplier = comp.Supplier.Name component.SupplierType = "Organization" + if comp.Supplier.URL != nil { + for _, u := range *comp.Supplier.URL { + if isPublishableReferenceURL(u) { + component.SupplierURL = u + break + } + } + } } if comp.Hashes != nil { for _, hash := range *comp.Hashes { component.Digests = append(component.Digests, Digest{ - Algorithm: strings.ToLower(strings.ReplaceAll(string(hash.Algorithm), "-", "")), + Algorithm: normalizeDigestAlgorithm(string(hash.Algorithm)), Value: hash.Value, }) } @@ -292,8 +305,14 @@ func componentFromCycloneDX(comp cdx.Component) Component { case cdx.ERTypeDistribution: applyLocator(&component, classifyResolvedURL(ref.URL, "", "")) case cdx.ERTypeVCS: - component.VCSURL = ref.URL + // Untrusted input: gate and normalize it exactly like a + // detector-supplied value, because it is re-emitted and also + // becomes the SPDX download location. + component.VCSURL = classifyIngestedVCS(ref.URL) default: + if !isPublishableReferenceURL(ref.URL) { + continue + } component.ExternalRefs = append(component.ExternalRefs, ExternalRef{ Type: string(ref.Type), URL: ref.URL, diff --git a/internal/sbom/ingest_metadata.go b/internal/sbom/ingest_metadata.go index a3ec2c40..12da4d40 100644 --- a/internal/sbom/ingest_metadata.go +++ b/internal/sbom/ingest_metadata.go @@ -15,6 +15,7 @@ import ( const ( metadataKeySupplier = "bomly.sbom.supplier" metadataKeySupplierType = "bomly.sbom.supplier_type" + metadataKeySupplierURL = "bomly.sbom.supplier_url" metadataKeyOriginator = "bomly.sbom.originator" metadataKeyOriginatorType = "bomly.sbom.originator_type" metadataKeyDescription = "bomly.sbom.description" @@ -34,6 +35,7 @@ func setIngestedMetadata(dep *sdk.Dependency, component Component) { values := map[string]string{ metadataKeySupplier: component.Supplier, metadataKeySupplierType: component.SupplierType, + metadataKeySupplierURL: component.SupplierURL, metadataKeyOriginator: component.Originator, metadataKeyOriginatorType: component.OriginatorType, metadataKeyDescription: component.Description, @@ -88,6 +90,7 @@ func applyIngestedMetadata(component *Component, metadata map[string]any) { targets := map[string]*string{ metadataKeySupplier: &component.Supplier, metadataKeySupplierType: &component.SupplierType, + metadataKeySupplierURL: &component.SupplierURL, metadataKeyOriginator: &component.Originator, metadataKeyOriginatorType: &component.OriginatorType, metadataKeyDescription: &component.Description, diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index 45f2f1a9..cff025ef 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -103,6 +103,18 @@ func classifyResolvedURL(raw string, source sdk.DependencySource, ecosystem sdk. if hint == LocatorVCS { return normalizeVCS(parsed) } + + // Credentials also travel outside userinfo: signed and private-registry + // URLs carry them as query parameters or fragments + // ("...?token=", "...?X-Amz-Signature=..."). A benign query + // parameter cannot be told apart from a credential here, so any query or + // fragment disqualifies a non-VCS locator. VCS locators are exempt + // because normalizeVCS discards both, keeping only a character-checked + // revision. + if parsed.RawQuery != "" || parsed.Fragment != "" { + return Locator{} + } + if hint == LocatorRegistryRoot { return Locator{Kind: LocatorRegistryRoot, URL: parsed.String()} } @@ -217,6 +229,9 @@ func normalizeRepositoryURL(repo string) string { if err != nil || parsed.User != nil || parsed.Host == "" { return "" } + if parsed.RawQuery != "" || parsed.Fragment != "" { + return "" + } switch strings.ToLower(parsed.Scheme) { case "http", "https": return parsed.String() @@ -224,6 +239,9 @@ func normalizeRepositoryURL(repo string) string { return "" } } + if strings.ContainsAny(repo, "?#") { + return "" + } host, rest, ok := strings.Cut(repo, "/") if !ok || !isHostname(host) || strings.TrimSpace(rest) == "" { @@ -240,6 +258,76 @@ func normalizeRepositoryURL(repo string) string { return candidate } +// classifyIngestedVCS renders a repository URL taken from an ingested SBOM in +// the SPDX version-control form. +// +// CycloneDX permits a plain https repository URL in a `vcs` reference, but +// SPDX 2.3 uses the `git+` notation and `spdxDownloadLocation` +// emits this value directly — so an unnormalized value would make a repository +// look like an ordinary package download. The value is untrusted, so it goes +// through the same scheme, host, and userinfo gate as a detector-supplied one. +func classifyIngestedVCS(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + raw = strings.TrimPrefix(raw, "git+") + parsed, err := url.Parse(raw) + if err != nil || parsed.User != nil || parsed.Host == "" { + return "" + } + switch strings.ToLower(parsed.Scheme) { + case "http", "https": + default: + return "" + } + return normalizeVCS(parsed).URL +} + +// classifyAssertedDownloadLocation classifies a value that its source document +// already declared to be a download location, such as SPDX +// PackageDownloadLocation. +// +// The path-shape heuristic that guards detector-supplied values does not apply +// here: an exact endpoint without a recognizable archive suffix +// ("https://repo.example/download?id=123" reduced to its path) would be demoted +// to a registry root and re-exported as NOASSERTION, discarding an assertion +// the source document actually made. The safety gate still applies in full, so +// local paths and credential-bearing URLs are still dropped. +func classifyAssertedDownloadLocation(raw string) Locator { + locator := classifyResolvedURL(raw, "", "") + if locator.Kind == LocatorRegistryRoot { + locator.Kind = LocatorArtifact + } + return locator +} + +// isPublishableReferenceURL reports whether a URL carried on an ingested +// external reference is safe to re-emit. +// +// Ingested references are re-emitted verbatim, so they need the same gate a +// detector-supplied value gets: no local paths, no credentials in userinfo, +// and no credential-bearing query or fragment. `mailto:` is allowed because a +// security-contact reference is a legitimate, path-free reference target. +func isPublishableReferenceURL(raw string) bool { + raw = strings.TrimSpace(raw) + if raw == "" { + return false + } + parsed, err := url.Parse(raw) + if err != nil || parsed.User != nil { + return false + } + switch strings.ToLower(parsed.Scheme) { + case "http", "https": + return parsed.Host != "" && parsed.RawQuery == "" && parsed.Fragment == "" + case "mailto": + return parsed.Opaque != "" && parsed.RawQuery == "" + default: + return false + } +} + // isHostname reports whether value looks like a dotted DNS hostname. func isHostname(value string) bool { if !strings.Contains(value, ".") || strings.HasPrefix(value, ".") || strings.HasSuffix(value, ".") { diff --git a/internal/sbom/locator_test.go b/internal/sbom/locator_test.go index a7a55cb6..860899f1 100644 --- a/internal/sbom/locator_test.go +++ b/internal/sbom/locator_test.go @@ -49,9 +49,16 @@ func TestClassifyResolvedURL(t *testing.T) { {"npm link local dir", "https://registry.npmjs.org/a/-/a-1.0.0.tgz", sdk.DependencySourceWorkspace, sdk.EcosystemNPM, LocatorNone, ""}, {"scp style git", "git@github.com:a/b.git", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorNone, ""}, - // Credentials must never reach an SBOM. + // Credentials must never reach an SBOM, in userinfo or anywhere else. {"userinfo token", "https://tok:s3cret@nexus.corp/a/b-1.0.tgz", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, {"username only", "https://tok@nexus.corp/a/b-1.0.tgz", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, + {"query token", "https://nexus.corp/a/b-1.0.tgz?token=s3cret", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, + {"presigned signature", "https://s3.amazonaws.com/b/a-1.0.tgz?X-Amz-Signature=deadbeef", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, + {"fragment secret", "https://nexus.corp/a/b-1.0.tgz#s3cret", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, + {"registry root with query", "https://nexus.corp/?apiKey=s3cret", sdk.DependencySourceRegistry, sdk.EcosystemRuby, LocatorNone, ""}, + // A VCS locator is exempt only because normalizeVCS discards the query + // and keeps a character-checked revision. + {"vcs query keeps only the revision", "git+https://github.com/a/b?rev=deadbeef&token=s3cret", sdk.DependencySourceGit, sdk.EcosystemRust, LocatorVCS, "git+https://github.com/a/b@deadbeef"}, // Degenerate input. {"empty", "", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, diff --git a/internal/sbom/model.go b/internal/sbom/model.go index 9cefab48..b7f01667 100644 --- a/internal/sbom/model.go +++ b/internal/sbom/model.go @@ -136,10 +136,12 @@ type Component struct { Vulnerabilities []Vulnerability EOL *EOL - // Where the package came from, classified from the detector-supplied - // resolved URL. At most one of ArtifactURL and VCSURL is set; - // RegistryURL is the weaker fallback naming a registry or index root, - // and is deliberately never used as a download location. + // Where the package came from. Detection-time classification of a single + // resolved URL sets exactly one of these, but an ingested document may + // assert several independently — CycloneDX can carry both a + // `distribution` and a `vcs` reference — so more than one may be set. + // RegistryURL is the weakest of the three: it names a registry or index + // root, and is deliberately never used as a download location. ArtifactURL string VCSURL string RegistryURL string @@ -153,6 +155,7 @@ type Component struct { // matcher, actually asserted them. Supplier string SupplierType string + SupplierURL string Originator string OriginatorType string Description string diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 0ede3c38..c7d8f921 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -2,6 +2,7 @@ package sbom import ( "encoding/json" + "strings" "testing" cdx "github.com/CycloneDX/cyclonedx-go" @@ -32,7 +33,7 @@ const supplierRichCycloneDX = `{ "purl": "pkg:npm/left-pad@1.3.0", "description": "String left padding", "publisher": "azer", - "supplier": {"name": "Example Supplier Inc."}, + "supplier": {"name": "Example Supplier Inc.", "url": ["https://supplier.example.com"]}, "cpe": "cpe:2.3:a:example:left-pad:1.3.0:*:*:*:*:*:*:*", "hashes": [{"alg": "SHA-256", "content": "abc123"}], "externalReferences": [ @@ -128,6 +129,11 @@ func TestIngestedAssertionsSurviveCycloneDXRoundTrip(t *testing.T) { if comp.Supplier == nil || comp.Supplier.Name != "Example Supplier Inc." { t.Fatalf("supplier = %+v, want the ingested value", comp.Supplier) } + // A supplier's URL is part of the compliance assertion, so it must not be + // flattened away to a bare name. + if comp.Supplier.URL == nil || len(*comp.Supplier.URL) != 1 || (*comp.Supplier.URL)[0] != "https://supplier.example.com" { + t.Fatalf("supplier url = %+v, want the ingested value preserved", comp.Supplier.URL) + } if comp.Description != "String left padding" { t.Fatalf("description = %q, want the ingested value", comp.Description) } @@ -137,8 +143,11 @@ func TestIngestedAssertionsSurviveCycloneDXRoundTrip(t *testing.T) { if got := externalRefURL(*comp, cdx.ERTypeDistribution); got != "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz" { t.Fatalf("distribution ref = %q, want the ingested value", got) } - if got := externalRefURL(*comp, cdx.ERTypeVCS); got != "https://github.com/stevemao/left-pad" { - t.Fatalf("vcs ref = %q, want the ingested value", got) + // Stored in the SPDX version-control form: this same value becomes the + // SPDX PackageDownloadLocation, where a bare https URL would make a + // repository look like an ordinary package download. + if got := externalRefURL(*comp, cdx.ERTypeVCS); got != "git+https://github.com/stevemao/left-pad" { + t.Fatalf("vcs ref = %q, want the normalized ingested value", got) } // An external reference type Bomly has no opinion about must pass through // rather than be dropped as unrecognized. @@ -190,6 +199,102 @@ func TestManufacturerWinsOnRootIngestedSupplierSurvivesElsewhere(t *testing.T) { } } +// hostileCycloneDX asserts URLs that must never be re-published: a local path, +// embedded credentials in userinfo and in a query parameter, and a home-page +// style reference pointing at the filesystem. +const hostileCycloneDX = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [ + { + "bom-ref": "pkg:npm/evil@1.0.0", + "type": "library", + "name": "evil", + "version": "1.0.0", + "purl": "pkg:npm/evil@1.0.0", + "externalReferences": [ + {"type": "distribution", "url": "file:///Users/victim/secret/evil-1.0.0.tgz"}, + {"type": "vcs", "url": "https://tok:s3cret@github.com/a/b"}, + {"type": "website", "url": "file:///Users/victim/secret/index.html"}, + {"type": "documentation", "url": "https://docs.example.com/x?token=s3cret"}, + {"type": "issue-tracker", "url": "https://issues.example.com/evil"} + ] + } + ] +}` + +// TestIngestedUnsafeURLsAreNotRepublished is the counterpart to the +// detector-side leak test. An ingested document is untrusted input, so its +// URLs must pass the same gate a lockfile value does — otherwise a hostile or +// merely careless SBOM could launder a credential or a local path into output +// Bomly publishes. +func TestIngestedUnsafeURLsAreNotRepublished(t *testing.T) { + for _, target := range []Target{TargetSPDX23JSON, TargetCycloneDX17JSON} { + out := ingestAndReexport(t, []byte(hostileCycloneDX), target) + rendered := string(out) + + for _, forbidden := range []string{ + "file://", "/Users/victim", "s3cret", "tok:", "token=", + } { + if strings.Contains(rendered, forbidden) { + t.Fatalf("%s output republished %q from an ingested document:\n%s", target, forbidden, rendered) + } + } + + // The one safe reference must still survive, so the gate is filtering + // rather than discarding everything. + if target == TargetCycloneDX17JSON && !strings.Contains(rendered, "https://issues.example.com/evil") { + t.Fatalf("cyclonedx output dropped the safe reference:\n%s", rendered) + } + } +} + +// TestSPDXHomePageSurvivesSPDXRoundTrip covers a field SPDX represents +// exactly, which an earlier revision decoded but never re-emitted. +func TestSPDXHomePageSurvivesSPDXRoundTrip(t *testing.T) { + const in = `{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [{ + "name": "left-pad", + "SPDXID": "SPDXRef-Package-left-pad", + "versionInfo": "1.3.0", + "downloadLocation": "https://repo.example/download/left-pad", + "homepage": "https://left-pad.example.com", + "checksums": [{"algorithm": "SHA3-256", "checksumValue": "abc123"}], + "filesAnalyzed": false + }] + }` + + out := ingestAndReexport(t, []byte(in), TargetSPDX23JSON) + var doc v23.Document + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("unmarshal spdx: %v", err) + } + for _, p := range doc.Packages { + if p == nil || p.PackageName != "left-pad" { + continue + } + if p.PackageHomePage != "https://left-pad.example.com" { + t.Fatalf("homepage = %q, want it preserved", p.PackageHomePage) + } + // An exact endpoint with no archive suffix: the source document + // declared it a download location, so it must not be demoted. + if p.PackageDownloadLocation != "https://repo.example/download/left-pad" { + t.Fatalf("downloadLocation = %q, want the asserted value", p.PackageDownloadLocation) + } + if len(p.PackageChecksums) != 1 { + t.Fatalf("SHA3-256 checksum did not survive: %+v", p.PackageChecksums) + } + return + } + t.Fatal("left-pad missing from output") +} + // TestSPDXNoAssertionSupplierDecodesToAbsent keeps the reserved marker from // being re-emitted as if it were a real supplier name. func TestSPDXNoAssertionSupplierDecodesToAbsent(t *testing.T) { diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index a4aad463..22ab1344 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -55,6 +55,7 @@ func (spdx23Codec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, error) PrimaryPackagePurpose: spdxPrimaryPackagePurpose(c.Type), PackageDescription: c.Description, PackageSourceInfo: spdxSourceInfo(c), + PackageHomePage: spdxHomePage(c), } if c.Originator != "" { pkg.PackageOriginator = &common.Originator{ @@ -182,8 +183,8 @@ func (spdx23Codec) decodeJSON(data []byte) (*Document, error) { component.Originator = parseSPDXEntity(p.PackageOriginator.Originator) component.OriginatorType = p.PackageOriginator.OriginatorType } - applyLocator(&component, classifyResolvedURL(parseSPDXEntity(p.PackageDownloadLocation), "", "")) - if home := strings.TrimSpace(p.PackageHomePage); home != "" && parseSPDXEntity(home) != "" { + applyLocator(&component, classifyAssertedDownloadLocation(parseSPDXEntity(p.PackageDownloadLocation))) + if home := parseSPDXEntity(p.PackageHomePage); isPublishableReferenceURL(home) { component.ExternalRefs = append(component.ExternalRefs, ExternalRef{Type: "website", URL: home}) } components = append(components, component) @@ -507,12 +508,31 @@ func spdxDownloadLocation(component Component) string { // (the package's home page) that Bomly does not know, and SPDX 2.3 has no // version-control external-reference category. func spdxSourceInfo(component Component) string { + // A detector-supplied VCS location is version-exact and already rendered + // as the download location. Emitting the matcher-derived repository too + // would make one package assert two different source repositories. + if component.VCSURL != "" { + return "" + } if repo := strings.TrimSpace(component.Repository); repo != "" { return spdxSourceInfoPrefix + repo } return "" } +// spdxHomePage restores a home page carried on an ingested website reference. +// +// SPDX has a first-class field for this, so an SPDX-to-SPDX pass would +// otherwise drop a value the destination format represents exactly. +func spdxHomePage(component Component) string { + for _, ref := range component.ExternalRefs { + if strings.EqualFold(ref.Type, "website") && isPublishableReferenceURL(ref.URL) { + return ref.URL + } + } + return "" +} + // spdxEntityType normalizes a supplier or originator type to one SPDX accepts, // defaulting to Organization. func spdxEntityType(value string) string { @@ -595,7 +615,10 @@ func parseSPDXCPEs(refs []*v23.PackageExternalReference) []string { if ref == nil { continue } - if strings.EqualFold(strings.TrimSpace(ref.RefType), "cpe23Type") { + // SPDX 2.3 defines both CPE reference types. Bomly writes cpe23Type, + // but an ingested third-party document may use either. + switch strings.ToLower(strings.TrimSpace(ref.RefType)) { + case "cpe23type", "cpe22type": if locator := strings.TrimSpace(ref.Locator); locator != "" { cpes = append(cpes, locator) } @@ -611,7 +634,7 @@ func parseSPDXChecksums(checksums []common.Checksum) []Digest { } digests := make([]Digest, 0, len(checksums)) for _, checksum := range checksums { - algorithm := strings.ToLower(strings.ReplaceAll(string(checksum.Algorithm), "-", "")) + algorithm := normalizeDigestAlgorithm(string(checksum.Algorithm)) if algorithm == "" || strings.TrimSpace(checksum.Value) == "" { continue } diff --git a/internal/sbom/transform.go b/internal/sbom/transform.go index daded4b5..953a140a 100644 --- a/internal/sbom/transform.go +++ b/internal/sbom/transform.go @@ -295,12 +295,11 @@ func enrichComponentFromRegistry(component *Component, registry *sdk.PackageRegi LatestVersion: pkg.EOL.LatestVersion, } } - if pkg.Scorecard != nil { - // Only fill, never clear: an unnormalizable scorecard value must not - // erase a repository an ingested document asserted. - if repo := normalizeRepositoryURL(pkg.Scorecard.Repository); repo != "" { - component.Repository = repo - } + if pkg.Scorecard != nil && component.Repository == "" { + // Fill a gap only. The scorecard repository is matcher-derived, so an + // ingested document's own assertion outranks it under the + // detection-classifies / ingest-corrects / enrichment-fills-gaps rule. + component.Repository = normalizeRepositoryURL(pkg.Scorecard.Repository) } // The registry copy of ResolvedURL is normally the detection value echoed // back, so it only fills a gap a matcher supplied. Source is not carried @@ -310,6 +309,33 @@ func enrichComponentFromRegistry(component *Component, registry *sdk.PackageRegi } } +// normalizeDigestAlgorithm renders a decoded checksum algorithm in the form +// both encoders recognize. +// +// Stripping every separator is wrong for the SHA-3 family: "SHA3-256" would +// become "sha3256", which neither spdxChecksumAlgorithm nor +// cycloneDXHashAlgorithm matches, so the checksum would be silently dropped on +// re-export. The SHA-3 names keep their hyphen; the others lose theirs. +func normalizeDigestAlgorithm(algorithm string) string { + normalized := strings.ToLower(strings.TrimSpace(algorithm)) + normalized = strings.ReplaceAll(normalized, "_", "-") + switch normalized { + case "sha3-256", "sha3256": + return "sha3-256" + case "sha3-384", "sha3384": + return "sha3-384" + case "sha3-512", "sha3512": + return "sha3-512" + case "blake2b-256", "blake2b256": + return "blake2b-256" + case "blake2b-384", "blake2b384": + return "blake2b-384" + case "blake2b-512", "blake2b512": + return "blake2b-512" + } + return strings.ReplaceAll(normalized, "-", "") +} + // applyLocator projects a classified resolved URL onto a component. A // LocatorNone leaves the component untouched, so an unpublishable value simply // results in no assertion. diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index a5059387..39c13ac0 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -603,13 +603,29 @@ func TestScanSBOMExportDistribution(t *testing.T) { } // Nothing about the machine that ran the scan may appear in either - // document. The temp dir stands in for any local path the detectors saw. + // document. + // + // Checking only `dir` would be too weak: that is the output directory + // passed to -o, while a --url scan clones the repository into a separate + // bomly-git-* temp directory. A detector leaking that clone path would go + // unnoticed. Assert on the shape of any local path instead, so the check + // covers paths this test never sees. for name, raw := range map[string][]byte{"spdx": spdxRaw, "cyclonedx": cdxRaw} { - if strings.Contains(string(raw), dir) { - t.Fatalf("%s output leaked the scan working directory %q", name, dir) + rendered := string(raw) + if strings.Contains(rendered, dir) { + t.Fatalf("%s output leaked the output directory %q", name, dir) } - if strings.Contains(string(raw), "file://") { - t.Fatalf("%s output contains a file:// locator", name) + for _, marker := range []string{ + "file://", "bomly-git-", os.TempDir(), + `"/Users/`, `"/home/`, `"/var/folders/`, `"/private/`, + `:"/tmp/`, `":/tmp/`, + } { + if marker == "" { + continue + } + if strings.Contains(rendered, marker) { + t.Fatalf("%s output leaked a local path or clone directory (%q)", name, marker) + } } } } From 7ca8a188773fefdde03b5573a41fde8e1b2c9d47 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 09:24:21 -0700 Subject: [PATCH 03/23] fix(sbom): classify VCS locators before the credential gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The credential gate added in the previous commit was placed above the source-based VCS classification, so only cargo's explicit "git+" prefix reached normalizeVCS. A git dependency that pins its revision the usual way — "https://host/repo.git?rev=", a DependencySourceGit URL, or a swiftpm pin — was dropped by the gate instead of normalized, losing a valid revision assertion. The code comment claimed VCS locators were exempt while the ordering made that false for most of them. Every VCS form is now classified first. That is safe because normalizeVCS discards the query and fragment outright and keeps only a character-checked revision, so a credential in either position still cannot survive: "?token=s3cret" on a git URL yields the bare repository locator. Added table cases for all four VCS shapes, including the credential case, so the ordering cannot silently regress. Also from review: - mailto references rejected fragments nowhere, so "mailto:security@example.com#token=" was re-emitted verbatim as a supplier URL or external reference. isPublishableReferenceURL now requires an empty fragment, with a dedicated unit table. - The hostile-ingest fixture carried its secrets only in a query, where the query gate could mask a missing fragment check. It now carries a distinct unique secret per position — query, fragment, and mailto fragment — so no one gate can hide another's absence. Verified by reverting the mailto guard and confirming the test fails. - The SHA3-256 round-trip assertion checked only the checksum count, which would not notice the algorithm degrading to another family. It now asserts algorithm and value. Co-Authored-By: Claude Opus 5 --- internal/sbom/locator.go | 34 +++++++++++++++++-------------- internal/sbom/locator_test.go | 36 ++++++++++++++++++++++++++++++++- internal/sbom/roundtrip_test.go | 13 +++++++++++- 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index cff025ef..a075d6c5 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -100,17 +100,32 @@ func classifyResolvedURL(raw string, source sdk.DependencySource, ecosystem sdk. return Locator{} } + // Every VCS form is classified before the query and fragment gate below. + // normalizeVCS discards both and keeps only a character-checked revision, + // so it is safe on values the gate would otherwise reject — and a git + // dependency legitimately pins its revision that way + // ("https://host/repo.git?rev="). Gating first would silently drop + // those pins. if hint == LocatorVCS { return normalizeVCS(parsed) } + if hint != LocatorRegistryRoot { + // Swift package identity is the repository URL, so swiftpm records a + // repo even for `kind: registry` pins. Without this the extension + // check below would demote them to registry roots. + if ecosystem == sdk.EcosystemSwift { + return normalizeVCS(parsed) + } + if source == sdk.DependencySourceGit || strings.HasSuffix(parsed.Path, ".git") { + return normalizeVCS(parsed) + } + } // Credentials also travel outside userinfo: signed and private-registry // URLs carry them as query parameters or fragments // ("...?token=", "...?X-Amz-Signature=..."). A benign query // parameter cannot be told apart from a credential here, so any query or - // fragment disqualifies a non-VCS locator. VCS locators are exempt - // because normalizeVCS discards both, keeping only a character-checked - // revision. + // fragment disqualifies a non-VCS locator. if parsed.RawQuery != "" || parsed.Fragment != "" { return Locator{} } @@ -119,17 +134,6 @@ func classifyResolvedURL(raw string, source sdk.DependencySource, ecosystem sdk. return Locator{Kind: LocatorRegistryRoot, URL: parsed.String()} } - // Swift package identity is the repository URL, so swiftpm records a repo - // even for `kind: registry` pins. Without this the extension check below - // would demote them to registry roots. - if ecosystem == sdk.EcosystemSwift { - return normalizeVCS(parsed) - } - - if source == sdk.DependencySourceGit || strings.HasSuffix(parsed.Path, ".git") { - return normalizeVCS(parsed) - } - if isConcreteArtifactPath(parsed.Path) { return Locator{Kind: LocatorArtifact, URL: parsed.String()} } @@ -322,7 +326,7 @@ func isPublishableReferenceURL(raw string) bool { case "http", "https": return parsed.Host != "" && parsed.RawQuery == "" && parsed.Fragment == "" case "mailto": - return parsed.Opaque != "" && parsed.RawQuery == "" + return parsed.Opaque != "" && parsed.RawQuery == "" && parsed.Fragment == "" default: return false } diff --git a/internal/sbom/locator_test.go b/internal/sbom/locator_test.go index 860899f1..151add10 100644 --- a/internal/sbom/locator_test.go +++ b/internal/sbom/locator_test.go @@ -57,8 +57,13 @@ func TestClassifyResolvedURL(t *testing.T) { {"fragment secret", "https://nexus.corp/a/b-1.0.tgz#s3cret", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, {"registry root with query", "https://nexus.corp/?apiKey=s3cret", sdk.DependencySourceRegistry, sdk.EcosystemRuby, LocatorNone, ""}, // A VCS locator is exempt only because normalizeVCS discards the query - // and keeps a character-checked revision. + // and keeps a character-checked revision. Every VCS form must be + // classified before the gate, not just the cargo "git+" prefix. {"vcs query keeps only the revision", "git+https://github.com/a/b?rev=deadbeef&token=s3cret", sdk.DependencySourceGit, sdk.EcosystemRust, LocatorVCS, "git+https://github.com/a/b@deadbeef"}, + {"git source pin without prefix", "https://github.com/a/b?rev=deadbeef", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/a/b@deadbeef"}, + {"dot-git path pin", "https://host/repo.git?rev=deadbeef", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorVCS, "git+https://host/repo.git@deadbeef"}, + {"swift pin with revision", "https://github.com/apple/swift-nio?tag=2.0.0", sdk.DependencySourceRegistry, sdk.EcosystemSwift, LocatorVCS, "git+https://github.com/apple/swift-nio@2.0.0"}, + {"git source drops a credential query", "https://github.com/a/b?token=s3cret", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/a/b"}, // Degenerate input. {"empty", "", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, @@ -133,6 +138,35 @@ func assertPublishableLocator(t *testing.T, got Locator, input string) { } } +func TestIsPublishableReferenceURL(t *testing.T) { + cases := []struct { + in string + want bool + }{ + {"https://example.com/docs", true}, + {"http://example.com/docs", true}, + {"mailto:security@example.com", true}, + // Credentials hide in every part of a reference, not just userinfo. + {"mailto:security@example.com#token=s3cret", false}, + {"mailto:security@example.com?subject=s3cret", false}, + {"https://example.com/docs?token=s3cret", false}, + {"https://example.com/docs#s3cret", false}, + {"https://tok:s3cret@example.com/docs", false}, + {"file:///Users/victim/secret.html", false}, + {"/Users/victim/secret.html", false}, + {"javascript:alert(1)", false}, + {"https://", false}, + {"mailto:", false}, + {"", false}, + {" ", false}, + } + for _, tc := range cases { + if got := isPublishableReferenceURL(tc.in); got != tc.want { + t.Fatalf("isPublishableReferenceURL(%q) = %v, want %v", tc.in, got, tc.want) + } + } +} + func TestNormalizeRepositoryURL(t *testing.T) { cases := []struct{ in, want string }{ {"github.com/kubernetes/kubernetes", "https://github.com/kubernetes/kubernetes"}, diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index c7d8f921..bd4a07bc 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -6,6 +6,7 @@ import ( "testing" cdx "github.com/CycloneDX/cyclonedx-go" + "github.com/spdx/tools-golang/spdx/v2/common" v23 "github.com/spdx/tools-golang/spdx/v2/v2_3" ) @@ -217,7 +218,9 @@ const hostileCycloneDX = `{ {"type": "distribution", "url": "file:///Users/victim/secret/evil-1.0.0.tgz"}, {"type": "vcs", "url": "https://tok:s3cret@github.com/a/b"}, {"type": "website", "url": "file:///Users/victim/secret/index.html"}, - {"type": "documentation", "url": "https://docs.example.com/x?token=s3cret"}, + {"type": "documentation", "url": "https://docs.example.com/x?token=qu3rysecret"}, + {"type": "chat", "url": "https://chat.example.com/x#fr4gsecret"}, + {"type": "support", "url": "mailto:help@example.com#m41lsecret"}, {"type": "issue-tracker", "url": "https://issues.example.com/evil"} ] } @@ -234,8 +237,11 @@ func TestIngestedUnsafeURLsAreNotRepublished(t *testing.T) { out := ingestAndReexport(t, []byte(hostileCycloneDX), target) rendered := string(out) + // Each secret is unique so a gate that rejects one form cannot mask + // another: the query case must not be what stops the fragment case. for _, forbidden := range []string{ "file://", "/Users/victim", "s3cret", "tok:", "token=", + "qu3rysecret", "fr4gsecret", "m41lsecret", } { if strings.Contains(rendered, forbidden) { t.Fatalf("%s output republished %q from an ingested document:\n%s", target, forbidden, rendered) @@ -290,6 +296,11 @@ func TestSPDXHomePageSurvivesSPDXRoundTrip(t *testing.T) { if len(p.PackageChecksums) != 1 { t.Fatalf("SHA3-256 checksum did not survive: %+v", p.PackageChecksums) } + // Assert both fields: a length check alone would not notice the + // algorithm degrading to a different family on the way through. + if got := p.PackageChecksums[0]; got.Algorithm != common.SHA3_256 || got.Value != "abc123" { + t.Fatalf("checksum = %+v, want SHA3-256/abc123", got) + } return } t.Fatal("left-pad missing from output") From 9ee7d64f8bdcd5cacfde2a0fe64e6bd530ec999b Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 09:36:23 -0700 Subject: [PATCH 04/23] fix(sbom): keep yarn checksum fragments, gate SPDX source info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The blanket fragment rejection from the credential fix dropped a whole ecosystem's download locations. Yarn appends the artifact's own sha1 to every `resolved` URL ("...-1.4.0.tgz#71ee51fa..."), so scanning a Yarn project produced 387 NOASSERTION and 1 real location — while the docs this PR added promise Yarn gets real download locations. A fixed-length hex digest is now recognized and stripped rather than treated as a secret; that project is back to 379. Any other fragment shape is still rejected, so the credential gate is unchanged for arbitrary values. Repositories recovered from SPDX PackageSourceInfo skipped the ingest gate. A source document asserting "Source repository: https://user:secret@github.com/org/repo", or a file:// path, was re-published verbatim in both formats — SPDX as PackageSourceInfo, CycloneDX as a vcs reference. The value now goes through normalizeRepositoryURL like every other ingested URL. Verified load-bearing by reverting the call and confirming the test fails. Two more from review: - Revision precedence was backwards. For "?rev=main#abc123" the fragment is the resolved commit and the query is what was requested, so the old order recorded a moving branch instead of the commit that was locked. This also contradicted uvSourceRevision, which prefers the fragment when the same lockfile value is parsed for detection. - A CycloneDX `publisher` is a free string the spec defines as a person or an organization, and SPDX has no untyped originator. Defaulting to "Organization:" asserted an entity type the source never made, which is the exact class of invented claim this PR exists to avoid. The field is now omitted on CycloneDX-to-SPDX and recorded as a conversion limit; it still round-trips through CycloneDX, and a typed SPDX originator is preserved. Supplier keeps its Organization type, since CycloneDX types that field as an organizational entity. Co-Authored-By: Claude Opus 5 --- docs/SBOM.md | 15 ++++++++ internal/sbom/locator.go | 64 +++++++++++++++++++++++++++------ internal/sbom/locator_test.go | 12 +++++++ internal/sbom/roundtrip_test.go | 57 +++++++++++++++++++++++++++-- internal/sbom/spdx23.go | 15 ++++++-- 5 files changed, 149 insertions(+), 14 deletions(-) diff --git a/docs/SBOM.md b/docs/SBOM.md index 4a5670d0..25884186 100644 --- a/docs/SBOM.md +++ b/docs/SBOM.md @@ -128,6 +128,15 @@ Two rules matter here: check applies to URLs read out of an SBOM Bomly ingests, since those are untrusted input too. + One fragment is recognized rather than dropped: Yarn appends the artifact's + own checksum to each `resolved` URL (`...-1.4.0.tgz#71ee51fa...`). That is a + fixed-length digest, not a secret, so it is stripped and the download + location is kept. + +For a dependency pinned to a source repository, Bomly records the resolved +commit when the lockfile has one, in preference to the branch or tag that was +requested. A branch moves; the commit is what was actually locked. + Coverage follows what each ecosystem actually records. npm, pnpm, yarn, and bun lockfiles name the exact package archive, so those get a real download location. Bundler, Cargo, pub, and most Python lockfiles record only a registry or index @@ -233,6 +242,12 @@ Some information necessarily becomes less specific during conversion: references other than `distribution` and `vcs` (for example `documentation` or `issue-tracker`) are dropped when converting to SPDX. They survive a CycloneDX-to-CycloneDX pass. +- A CycloneDX `publisher` is a plain string that the format defines as either a + person or an organization. SPDX has no untyped equivalent — it requires the + value to be declared one or the other — so the publisher is dropped when + converting to SPDX rather than labelled with a guess. It survives a + CycloneDX-to-CycloneDX pass, and an SPDX originator that already states its + type is preserved. - CycloneDX vulnerability records preserve ratings, CWEs, affected component references, descriptions, and advisory URLs. SPDX 2.3 represents each diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index a075d6c5..2bd4fe79 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -124,12 +124,27 @@ func classifyResolvedURL(raw string, source sdk.DependencySource, ecosystem sdk. // Credentials also travel outside userinfo: signed and private-registry // URLs carry them as query parameters or fragments // ("...?token=", "...?X-Amz-Signature=..."). A benign query - // parameter cannot be told apart from a credential here, so any query or - // fragment disqualifies a non-VCS locator. - if parsed.RawQuery != "" || parsed.Fragment != "" { + // parameter cannot be told apart from a credential here, so any query + // disqualifies a non-VCS locator. + if parsed.RawQuery != "" { return Locator{} } + // A fragment is rejected on the same grounds, with one exception: Yarn + // v1 appends the artifact's own checksum to every `resolved` URL + // ("...-1.4.0.tgz#71ee51fa..."). That is a fixed-format digest, not a + // secret, and rejecting it would drop the download location for every + // package in a Yarn lockfile. Strip it and keep the URL; anything that + // is not digest-shaped is still treated as a secret. + if parsed.Fragment != "" { + if !isChecksumFragment(parsed.Fragment) { + return Locator{} + } + clean := *parsed + clean.Fragment = "" + parsed = &clean + } + if hint == LocatorRegistryRoot { return Locator{Kind: LocatorRegistryRoot, URL: parsed.String()} } @@ -140,6 +155,28 @@ func classifyResolvedURL(raw string, source sdk.DependencySource, ecosystem sdk. return Locator{Kind: LocatorRegistryRoot, URL: parsed.String()} } +// isChecksumFragment reports whether a URL fragment is a bare hex digest of a +// standard length, the form Yarn v1 and some registries append to an artifact +// URL. +// +// The length allowlist is what keeps this from becoming a hole: an arbitrary +// hex-looking secret of some other length is still rejected as a credential. +func isChecksumFragment(fragment string) bool { + switch len(fragment) { + case 32, 40, 64, 96, 128: // md5, sha1, sha256, sha384, sha512 + default: + return false + } + for _, r := range fragment { + switch { + case r >= '0' && r <= '9', r >= 'a' && r <= 'f', r >= 'A' && r <= 'F': + default: + return false + } + } + return true +} + // isConcreteArtifactPath reports whether the final path segment names a // package archive. func isConcreteArtifactPath(path string) bool { @@ -163,15 +200,22 @@ func isConcreteArtifactPath(path string) bool { // "git+https://host/a/b?rev=abc" cannot be passed through as-is: the revision // is moved to the "@" suffix and the query is dropped. func normalizeVCS(parsed *url.URL) Locator { - revision := "" - for _, key := range []string{"rev", "tag", "branch"} { - if value := strings.TrimSpace(parsed.Query().Get(key)); value != "" { - revision = value - break - } + // The fragment carries the resolved commit and the query carries what was + // requested, so "?branch=main#abc123" locked abc123. Preferring the + // fragment records the immutable commit rather than a moving branch, and + // matches the precedence uvSourceRevision already applies when the same + // lockfile value is parsed for detection. + revision := strings.TrimSpace(parsed.Fragment) + if !isSafeRevision(revision) { + revision = "" } if revision == "" { - revision = strings.TrimSpace(parsed.Fragment) + for _, key := range []string{"rev", "tag", "branch"} { + if value := strings.TrimSpace(parsed.Query().Get(key)); value != "" { + revision = value + break + } + } } clean := *parsed diff --git a/internal/sbom/locator_test.go b/internal/sbom/locator_test.go index 151add10..f10cf369 100644 --- a/internal/sbom/locator_test.go +++ b/internal/sbom/locator_test.go @@ -55,6 +55,12 @@ func TestClassifyResolvedURL(t *testing.T) { {"query token", "https://nexus.corp/a/b-1.0.tgz?token=s3cret", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, {"presigned signature", "https://s3.amazonaws.com/b/a-1.0.tgz?X-Amz-Signature=deadbeef", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, {"fragment secret", "https://nexus.corp/a/b-1.0.tgz#s3cret", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, + {"hex fragment of odd length is still a secret", "https://nexus.corp/a/b-1.0.tgz#deadbeef", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, + // Yarn v1 appends the artifact's own sha1 to every resolved URL. + // Treating that as a secret would drop the download location for + // every package in a Yarn lockfile. + {"yarn checksum fragment is stripped", "https://registry.yarnpkg.com/react/-/react-18.2.0.tgz#ceeba79ee36dfa7612b1ede82a3e37b2a30def1c", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorArtifact, "https://registry.yarnpkg.com/react/-/react-18.2.0.tgz"}, + {"sha256 fragment is stripped", "https://reg.example/a/b-1.0.tgz#" + strings.Repeat("a", 64), sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorArtifact, "https://reg.example/a/b-1.0.tgz"}, {"registry root with query", "https://nexus.corp/?apiKey=s3cret", sdk.DependencySourceRegistry, sdk.EcosystemRuby, LocatorNone, ""}, // A VCS locator is exempt only because normalizeVCS discards the query // and keeps a character-checked revision. Every VCS form must be @@ -64,6 +70,12 @@ func TestClassifyResolvedURL(t *testing.T) { {"dot-git path pin", "https://host/repo.git?rev=deadbeef", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorVCS, "git+https://host/repo.git@deadbeef"}, {"swift pin with revision", "https://github.com/apple/swift-nio?tag=2.0.0", sdk.DependencySourceRegistry, sdk.EcosystemSwift, LocatorVCS, "git+https://github.com/apple/swift-nio@2.0.0"}, {"git source drops a credential query", "https://github.com/a/b?token=s3cret", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/a/b"}, + // The fragment is the resolved commit and the query is what was + // requested, so the immutable commit wins over a moving branch. This + // is the shape uv records: "?rev=main#abc123". + {"resolved commit beats requested branch", "https://github.com/a/b?branch=main#abc123", sdk.DependencySourceGit, sdk.EcosystemPython, LocatorVCS, "git+https://github.com/a/b@abc123"}, + {"resolved commit beats requested rev", "https://github.com/example/git-helper?rev=main#abc123", sdk.DependencySourceGit, sdk.EcosystemPython, LocatorVCS, "git+https://github.com/example/git-helper@abc123"}, + {"requested rev used when no commit resolved", "https://github.com/a/b?rev=v1.2.3", sdk.DependencySourceGit, sdk.EcosystemPython, LocatorVCS, "git+https://github.com/a/b@v1.2.3"}, // Degenerate input. {"empty", "", sdk.DependencySourceRegistry, sdk.EcosystemNPM, LocatorNone, ""}, diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index bd4a07bc..474797c8 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -92,8 +92,12 @@ func TestIngestedAssertionsSurviveConversionToSPDX(t *testing.T) { if pkg.PackageDescription != "String left padding" { t.Fatalf("description = %q, want the ingested value", pkg.PackageDescription) } - if pkg.PackageOriginator == nil || pkg.PackageOriginator.Originator != "azer" { - t.Fatalf("originator = %+v, want the ingested publisher", pkg.PackageOriginator) + // A CycloneDX publisher is a free string the spec defines as a person or + // an organization, and SPDX has no untyped originator. Emitting one would + // assert an entity type the source never made, so the field is omitted + // rather than guessed. It still round-trips through CycloneDX. + if pkg.PackageOriginator != nil { + t.Fatalf("originator = %+v, want it omitted rather than typed by guess", pkg.PackageOriginator) } if want := "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"; pkg.PackageDownloadLocation != want { t.Fatalf("downloadLocation = %q, want %q", pkg.PackageDownloadLocation, want) @@ -256,6 +260,55 @@ func TestIngestedUnsafeURLsAreNotRepublished(t *testing.T) { } } +// TestSPDXSourceInfoRepositoryIsGated covers the other direction of the +// ingest gate: a repository recovered from SPDX PackageSourceInfo is +// re-published in both formats, so it needs the same validation as any other +// ingested URL. +func TestSPDXSourceInfoRepositoryIsGated(t *testing.T) { + cases := []struct { + name string + sourceInfo string + wantRepo string + }{ + {"credentials", "Source repository: https://user:sp3csecret@github.com/org/repo", ""}, + {"local path", "Source repository: file:///Users/victim/repo", ""}, + {"query", "Source repository: https://github.com/org/repo?token=sp3csecret", ""}, + {"safe", "Source repository: https://github.com/org/repo", "https://github.com/org/repo"}, + {"unmarked prose", "Built from an internal mirror", ""}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := parseSPDXSourceInfo(tc.sourceInfo); got != tc.wantRepo { + t.Fatalf("parseSPDXSourceInfo(%q) = %q, want %q", tc.sourceInfo, got, tc.wantRepo) + } + }) + } + + const hostile = `{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [{ + "name": "evil", + "SPDXID": "SPDXRef-Package-evil", + "versionInfo": "1.0.0", + "downloadLocation": "NOASSERTION", + "sourceInfo": "Source repository: https://user:sp3csecret@github.com/org/repo", + "filesAnalyzed": false + }] + }` + + for _, target := range []Target{TargetSPDX23JSON, TargetCycloneDX17JSON} { + out := ingestAndReexport(t, []byte(hostile), target) + if strings.Contains(string(out), "sp3csecret") { + t.Fatalf("%s republished a credential from SPDX sourceInfo:\n%s", target, out) + } + } +} + // TestSPDXHomePageSurvivesSPDXRoundTrip covers a field SPDX represents // exactly, which an earlier revision decoded but never re-emitted. func TestSPDXHomePageSurvivesSPDXRoundTrip(t *testing.T) { diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 22ab1344..894ad655 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -57,7 +57,12 @@ func (spdx23Codec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, error) PackageSourceInfo: spdxSourceInfo(c), PackageHomePage: spdxHomePage(c), } - if c.Originator != "" { + // SPDX has no untyped originator: the value must be declared a Person + // or an Organization. A CycloneDX `publisher` is a free string that + // the spec defines as either, so emitting one here would assert an + // entity type the source document never made. Emit only when the + // source established the type. + if c.Originator != "" && c.OriginatorType != "" { pkg.PackageOriginator = &common.Originator{ OriginatorType: spdxEntityType(c.OriginatorType), Originator: c.Originator, @@ -475,12 +480,18 @@ func parseSPDXEntity(value string) string { // parseSPDXSourceInfo recovers a source repository written by spdxSourceInfo. // Free text without the marker prefix is another producer's prose and is left // alone rather than guessed at. +// +// The recovered value is untrusted and is re-published in both formats — SPDX +// PackageSourceInfo and a CycloneDX `vcs` reference — so it goes through the +// same gate as every other ingested URL. A source document asserting +// "Source repository: https://user:secret@github.com/org/repo", or a local +// file:// path, yields nothing. func parseSPDXSourceInfo(value string) string { value = strings.TrimSpace(value) if !strings.HasPrefix(value, spdxSourceInfoPrefix) { return "" } - return strings.TrimSpace(strings.TrimPrefix(value, spdxSourceInfoPrefix)) + return normalizeRepositoryURL(strings.TrimPrefix(value, spdxSourceInfoPrefix)) } // spdxSourceInfoPrefix marks a source repository inside the free-text From e39234ea1b922fb6eb1dda72d567da1c6df4ad02 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 09:49:47 -0700 Subject: [PATCH 05/23] fix(sbom): validate reference types, merge duplicate PURLs, pin revisions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five findings from review, all confirmed against the code before fixing. Unknown external-reference types could produce schema-invalid output. The CycloneDX library's version-downgrade pass rewrites unsupported types it recognizes, but its fallthrough returns true for anything else, so an arbitrary string from an ingested document was emitted verbatim against a closed enum. Preserved types are now checked against the encoder vocabulary and remapped to "other", which keeps the link and stays valid; per-version narrowing is still the library's job. BLAKE2b checksums were normalized to a canonical name neither encoder recognized, so they were silently discarded after the graph hop even on a same-format round trip. Both formats define BLAKE2b-256/384/512; the mappings were simply missing. This was self-inflicted: the previous commit added BLAKE2b to the normalizer without checking the encoders. Duplicate-PURL components lost their assertions. Several component IDs mapping to one PURL is a supported ingest shape, but only the first became a graph node and later duplicates were discarded whole, taking their supplier, description, distribution, digest, and CPE assertions with them. They are now merged with fill-gaps semantics and unioned set-valued fields. A person supplier was recast as an organization. CycloneDX has no person-valued supplier, so an SPDX "Person: Alice" became an organizational entity — the same invented-type problem as the publisher case, in the other direction. That conversion is now omitted; SPDX represents the type natively and still round-trips it. Detector-resolved commits were ignored. Bundler, pub, and the python detectors record a git dependency's resolved revision in Metadata["source_revision"] while leaving ResolvedURL as the bare remote, so those packages exported an unpinned, moving locator even though the commit was known. A safe revision is now folded into the locator. The two highest-risk fixes were verified load-bearing by reverting each and confirming its test fails. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 82 +++++++++++++++- internal/sbom/distribution_test.go | 79 ++++++++++++++++ internal/sbom/graph.go | 79 +++++++++++++++- internal/sbom/locator.go | 28 ++++++ internal/sbom/roundtrip_test.go | 145 +++++++++++++++++++++++++++++ internal/sbom/spdx23.go | 6 ++ internal/sbom/transform.go | 5 +- 7 files changed, 417 insertions(+), 7 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 94d1b5bd..17f56a72 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -47,7 +47,11 @@ func (c cycloneDXCodec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, e if props := cycloneDXEOLProperties(comp.EOL); len(props) > 0 { component.Properties = &props } - if comp.Supplier != "" { + // CycloneDX has no person-valued supplier: the field is an + // organizational entity. An SPDX document that explicitly said + // "Person: Alice" would be recast as an organization, changing a + // compliance-relevant assertion, so that case is omitted instead. + if comp.Supplier != "" && !strings.EqualFold(comp.SupplierType, "Person") { supplier := &cdx.OrganizationalEntity{Name: comp.Supplier} if comp.SupplierURL != "" { urls := []string{comp.SupplierURL} @@ -260,6 +264,74 @@ func cycloneDXSecurityReferences(p Provenance) []cdx.ExternalReference { return refs } +// knownExternalReferenceTypes is the CycloneDX externalReference type +// vocabulary understood by the encoder. +// +// An ingested document can carry any string here, and the library's +// version-downgrade pass only rewrites types it recognizes — an unknown string +// falls through and is emitted verbatim, producing a schema-invalid document. +// A type outside this set is rewritten to "other", which keeps the link while +// staying valid. Per-version narrowing (a 1.7 type emitted as 1.4) is then +// handled by the library. +var knownExternalReferenceTypes = map[string]struct{}{ + "adversary-model": {}, + "advisories": {}, + "attestation": {}, + "bom": {}, + "build-meta": {}, + "build-system": {}, + "certification-report": {}, + "chat": {}, + "citation": {}, + "codified-infrastructure": {}, + "component-analysis-report": {}, + "configuration": {}, + "digital-signature": {}, + "distribution": {}, + "distribution-intake": {}, + "documentation": {}, + "dynamic-analysis-report": {}, + "electronic-signature": {}, + "evidence": {}, + "exploitability-statement": {}, + "formulation": {}, + "issue-tracker": {}, + "license": {}, + "log": {}, + "mailing-list": {}, + "maturity-report": {}, + "model-card": {}, + "other": {}, + "patent": {}, + "patent-assertion": {}, + "patent-family": {}, + "pentest-report": {}, + "poam": {}, + "quality-metrics": {}, + "release-notes": {}, + "rfc-9116": {}, + "risk-assessment": {}, + "runtime-analysis-report": {}, + "security-contact": {}, + "social": {}, + "source-distribution": {}, + "static-analysis-report": {}, + "support": {}, + "threat-model": {}, + "vcs": {}, + "vulnerability-assertion": {}, + "website": {}, +} + +// externalReferenceType maps a preserved reference type onto the encoder +// vocabulary, falling back to "other" for anything unrecognized. +func externalReferenceType(value string) cdx.ExternalReferenceType { + if _, ok := knownExternalReferenceTypes[strings.ToLower(strings.TrimSpace(value))]; ok { + return cdx.ExternalReferenceType(strings.ToLower(strings.TrimSpace(value))) + } + return cdx.ERTypeOther +} + // componentFromCycloneDX projects one CycloneDX component onto the neutral // model. Both decode paths (the component inventory and the metadata-only // fallback) share it so they cannot drift apart. @@ -354,7 +426,7 @@ func cycloneDXComponentReferences(component Component) []cdx.ExternalReference { for _, ref := range component.ExternalRefs { refs = append(refs, cdx.ExternalReference{ - Type: cdx.ExternalReferenceType(ref.Type), + Type: externalReferenceType(ref.Type), URL: ref.URL, Comment: ref.Comment, }) @@ -556,6 +628,12 @@ func cycloneDXHashAlgorithm(algorithm string) cdx.HashAlgorithm { return cdx.HashAlgoSHA384 case "sha512", "sha-512": return cdx.HashAlgoSHA512 + case "blake2b-256": + return cdx.HashAlgoBlake2b_256 + case "blake2b-384": + return cdx.HashAlgoBlake2b_384 + case "blake2b-512": + return cdx.HashAlgoBlake2b_512 case "sha3-256": return cdx.HashAlgoSHA3_256 case "sha3-384": diff --git a/internal/sbom/distribution_test.go b/internal/sbom/distribution_test.go index 7d581165..1a6d64a3 100644 --- a/internal/sbom/distribution_test.go +++ b/internal/sbom/distribution_test.go @@ -165,6 +165,85 @@ func TestDistributionProjection(t *testing.T) { } } +// TestDetectorRevisionPinsVCSLocator covers the detectors (bundler, pub, the +// python family) that keep a git dependency's resolved commit in metadata +// while leaving ResolvedURL as the bare remote. Without folding it in, the +// export names a moving branch for a package whose commit is known. +func TestDetectorRevisionPinsVCSLocator(t *testing.T) { + g := sdk.New() + node := sdk.NewDependencyWithID("pkg@1.0.0", sdk.Dependency{ + Coordinates: sdk.Coordinates{ + Name: "pkg", Version: "1.0.0", + PURL: "pkg:gem/pkg@1.0.0", Ecosystem: sdk.EcosystemRuby, + }, + Source: sdk.DependencySourceGit, + ResolvedURL: "https://github.com/owner/repo", + Metadata: map[string]any{"source_revision": "9f8e7d6c5b4a"}, + }) + if err := g.AddNode(node); err != nil { + t.Fatalf("add node: %v", err) + } + + pkg := spdxPackageFor(t, g, BuildOptions{}) + if want := "git+https://github.com/owner/repo@9f8e7d6c5b4a"; pkg.PackageDownloadLocation != want { + t.Fatalf("downloadLocation = %q, want the detector-resolved commit pinned (%q)", + pkg.PackageDownloadLocation, want) + } +} + +// TestUnsafeDetectorRevisionIsIgnored keeps an unusable metadata value from +// producing a malformed locator. +func TestUnsafeDetectorRevisionIsIgnored(t *testing.T) { + g := sdk.New() + node := sdk.NewDependencyWithID("pkg@1.0.0", sdk.Dependency{ + Coordinates: sdk.Coordinates{ + Name: "pkg", Version: "1.0.0", + PURL: "pkg:gem/pkg@1.0.0", Ecosystem: sdk.EcosystemRuby, + }, + Source: sdk.DependencySourceGit, + ResolvedURL: "https://github.com/owner/repo", + Metadata: map[string]any{"source_revision": "bad revision\x00"}, + }) + if err := g.AddNode(node); err != nil { + t.Fatalf("add node: %v", err) + } + + pkg := spdxPackageFor(t, g, BuildOptions{}) + if want := "git+https://github.com/owner/repo"; pkg.PackageDownloadLocation != want { + t.Fatalf("downloadLocation = %q, want the unpinned repository (%q)", + pkg.PackageDownloadLocation, want) + } +} + +// TestBlake2bDigestsSurviveRoundTrip pairs with normalizeDigestAlgorithm: the +// canonical names it preserves must exist in both encoders, or the checksum is +// silently discarded after the graph hop. +func TestBlake2bDigestsSurviveRoundTrip(t *testing.T) { + for _, algorithm := range []string{"blake2b-256", "blake2b-384", "blake2b-512", "sha3-256"} { + t.Run(algorithm, func(t *testing.T) { + g := sdk.New() + node := sdk.NewDependencyWithID("pkg@1.0.0", sdk.Dependency{ + Coordinates: sdk.Coordinates{ + Name: "pkg", Version: "1.0.0", + PURL: "pkg:npm/pkg@1.0.0", Ecosystem: sdk.EcosystemNPM, + }, + Digests: []sdk.Digest{{Algorithm: sdk.DigestAlgorithm(algorithm), Value: "abc123"}}, + }) + if err := g.AddNode(node); err != nil { + t.Fatalf("add node: %v", err) + } + + if pkg := spdxPackageFor(t, g, BuildOptions{}); len(pkg.PackageChecksums) != 1 { + t.Fatalf("spdx dropped a %s checksum: %+v", algorithm, pkg.PackageChecksums) + } + comp := cycloneDXComponentFor(t, g, BuildOptions{}) + if comp.Hashes == nil || len(*comp.Hashes) != 1 { + t.Fatalf("cyclonedx dropped a %s hash: %+v", algorithm, comp.Hashes) + } + }) + } +} + // TestDistributionNeverLeaksLocalPaths asserts on the encoded bytes, the way a // consumer would see them. A path leak is the one genuinely dangerous failure // mode of this feature. diff --git a/internal/sbom/graph.go b/internal/sbom/graph.go index e905dd74..4cbbf468 100644 --- a/internal/sbom/graph.go +++ b/internal/sbom/graph.go @@ -59,10 +59,14 @@ func ToGraph(doc *Document) (*sdk.Graph, error) { sdk.SetDetectionLicenses(pkg, graphLicenses(component.Licenses)) setIngestedMetadata(pkg, component) - if _, exists := depsGraph.Node(packageID); !exists { - if err := depsGraph.AddNode(pkg); err != nil { - return nil, fmt.Errorf("add package %q: %w", component.ID, err) - } + if existing, exists := depsGraph.Node(packageID); exists { + // Several component IDs can share one PURL (a lockfile entry and + // an installed-metadata entry for the same package). Only the + // first becomes a graph node, so the duplicate's assertions have + // to be folded in rather than dropped with the discarded object. + mergeIngestedNode(existing, pkg) + } else if err := depsGraph.AddNode(pkg); err != nil { + return nil, fmt.Errorf("add package %q: %w", component.ID, err) } idMap[component.ID] = packageID } @@ -113,6 +117,73 @@ func isDocumentRootPseudoPackage(component Component) bool { return false } +// mergeIngestedNode folds a duplicate-PURL component's assertions into the +// graph node that already represents that package. +// +// Fill-gaps semantics: the first component wins any conflict, and later ones +// only supply what is still missing. Set-valued fields are unioned, since two +// entries for the same package may each carry a digest or CPE the other does +// not. +func mergeIngestedNode(existing, incoming *sdk.Dependency) { + if existing == nil || incoming == nil { + return + } + if existing.Copyright == "" { + existing.Copyright = incoming.Copyright + } + if existing.ResolvedURL == "" { + existing.ResolvedURL = incoming.ResolvedURL + } + existing.CPEs = unionStrings(existing.CPEs, incoming.CPEs) + existing.Digests = unionDigests(existing.Digests, incoming.Digests) + + if len(sdk.DetectionLicenses(existing)) == 0 { + if licenses := sdk.DetectionLicenses(incoming); len(licenses) > 0 { + sdk.SetDetectionLicenses(existing, licenses) + } + } + for key, value := range incoming.Metadata { + if existing.Metadata == nil { + existing.Metadata = make(map[string]any, len(incoming.Metadata)) + } + if _, present := existing.Metadata[key]; !present { + existing.Metadata[key] = value + } + } +} + +// unionStrings appends values from extra that are not already in base. +func unionStrings(base, extra []string) []string { + seen := make(map[string]struct{}, len(base)) + for _, value := range base { + seen[value] = struct{}{} + } + for _, value := range extra { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + base = append(base, value) + } + return base +} + +// unionDigests appends digests from extra that are not already in base. +func unionDigests(base, extra []sdk.Digest) []sdk.Digest { + seen := make(map[sdk.Digest]struct{}, len(base)) + for _, digest := range base { + seen[digest] = struct{}{} + } + for _, digest := range extra { + if _, ok := seen[digest]; ok { + continue + } + seen[digest] = struct{}{} + base = append(base, digest) + } + return base +} + // graphDigests projects component digests onto a graph node. Ingest dropped // these previously, so an incoming document's hashes did not survive a format // conversion. diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index 2bd4fe79..43453df3 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -51,6 +51,34 @@ var artifactExtensions = []string{ // over LocatorArtifact whenever the path shape is not recognizably an archive. // An SBOM that omits a download location is correct; one that points at the // wrong place, or at the developer's home directory, is not. +// metadataKeySourceRevision is the Dependency.Metadata key several detectors +// (ruby, pub, and the python family) use to record the commit a git +// dependency resolved to, separately from the repository URL. +const metadataKeySourceRevision = "source_revision" + +// sourceRevisionFrom returns a detector-recorded resolved commit, or "". +func sourceRevisionFrom(metadata map[string]any) string { + revision, _ := metadata[metadataKeySourceRevision].(string) + revision = strings.TrimSpace(revision) + if !isSafeRevision(revision) { + return "" + } + return revision +} + +// pinLocator attaches a detector-recorded revision to a VCS locator that does +// not already carry one. +// +// Bundler, pub, and the python detectors keep the resolved commit in metadata +// while leaving ResolvedURL as the bare remote, so without this the export +// would name a moving branch for a dependency whose commit is actually known. +func pinLocator(locator Locator, revision string) Locator { + if locator.Kind != LocatorVCS || revision == "" || strings.Contains(locator.URL, "@") { + return locator + } + return Locator{Kind: LocatorVCS, URL: locator.URL + "@" + revision} +} + func classifyResolvedURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosystem) Locator { raw = strings.TrimSpace(raw) if raw == "" { diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 474797c8..e373a419 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -309,6 +309,151 @@ func TestSPDXSourceInfoRepositoryIsGated(t *testing.T) { } } +// TestUnknownExternalReferenceTypeIsMappedToOther keeps an ingested document +// from producing schema-invalid output. The CycloneDX library's +// version-downgrade pass only rewrites types it recognizes, so an arbitrary +// string would otherwise be emitted verbatim against a closed enum. +func TestUnknownExternalReferenceTypeIsMappedToOther(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", + "type": "library", + "name": "a", + "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "externalReferences": [ + {"type": "totally-made-up", "url": "https://example.com/x"}, + {"type": "issue-tracker", "url": "https://example.com/issues"} + ] + }] + }` + + out := ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON) + if strings.Contains(string(out), "totally-made-up") { + t.Fatalf("unknown external reference type was emitted verbatim:\n%s", out) + } + + var bom cdx.BOM + if err := json.Unmarshal(out, &bom); err != nil { + t.Fatalf("unmarshal: %v", err) + } + comp := (*bom.Components)[0] + if got := externalRefURL(comp, cdx.ERTypeOther); got != "https://example.com/x" { + t.Fatalf("expected the unknown type remapped to \"other\" with its URL kept, got %q", got) + } + if got := externalRefURL(comp, cdx.ERTypeIssueTracker); got != "https://example.com/issues" { + t.Fatalf("known type was not preserved, got %q", got) + } +} + +// TestPersonSupplierIsNotRecastAsOrganization covers the counterpart of the +// publisher rule: CycloneDX has no person-valued supplier, so an explicit +// SPDX "Person:" supplier must not be relabelled. +func TestPersonSupplierIsNotRecastAsOrganization(t *testing.T) { + const in = `{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [{ + "name": "left-pad", + "SPDXID": "SPDXRef-Package-left-pad", + "versionInfo": "1.3.0", + "downloadLocation": "NOASSERTION", + "supplier": "Person: Alice", + "filesAnalyzed": false + }] + }` + + cdxOut := ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON) + if strings.Contains(string(cdxOut), "Alice") { + t.Fatalf("a person supplier was recast as a CycloneDX organization:\n%s", cdxOut) + } + + // SPDX represents the type natively, so it must survive there. + spdxOut := ingestAndReexport(t, []byte(in), TargetSPDX23JSON) + var doc v23.Document + if err := json.Unmarshal(spdxOut, &doc); err != nil { + t.Fatalf("unmarshal spdx: %v", err) + } + for _, p := range doc.Packages { + if p == nil || p.PackageName != "left-pad" { + continue + } + if p.PackageSupplier == nil || p.PackageSupplier.SupplierType != "Person" || p.PackageSupplier.Supplier != "Alice" { + t.Fatalf("spdx supplier = %+v, want Person: Alice preserved", p.PackageSupplier) + } + return + } + t.Fatal("left-pad missing from spdx output") +} + +// TestDuplicatePURLAssertionsAreMerged covers an ingest shape the codec +// already supports: several component IDs mapping to one PURL. Only the first +// becomes a graph node, so a later duplicate's assertions must be folded in +// rather than discarded with it. +func TestDuplicatePURLAssertionsAreMerged(t *testing.T) { + doc := &Document{ + Components: []Component{ + { + ID: "first", + Name: "certifi", + Version: "2026.4.22", + PURL: "pkg:pypi/certifi@2026.4.22", + CPEs: []string{"cpe:2.3:a:x:certifi:2026.4.22:*:*:*:*:*:*:*"}, + }, + { + ID: "second", + Name: "certifi", + Version: "2026.4.22", + PURL: "pkg:pypi/certifi@2026.4.22", + Supplier: "Example Supplier Inc.", + Description: "Root certificates", + ArtifactURL: "https://files.pythonhosted.org/x/certifi-2026.4.22-py3-none-any.whl", + Digests: []Digest{{Algorithm: "sha256", Value: "abc123"}}, + }, + }, + Dependencies: []Dependency{{Ref: "first", DependsOn: []string{"second"}}}, + } + + graph, err := ToGraph(doc) + if err != nil { + t.Fatalf("to graph: %v", err) + } + out, err := MarshalDepGraphJSON(graph, TargetSPDX23JSON, BuildOptions{}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var spdxDoc v23.Document + if err := json.Unmarshal(out, &spdxDoc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(spdxDoc.Packages) != 1 { + t.Fatalf("expected one merged package, got %d", len(spdxDoc.Packages)) + } + pkg := spdxDoc.Packages[0] + if pkg.PackageSupplier == nil || pkg.PackageSupplier.Supplier != "Example Supplier Inc." { + t.Fatalf("supplier from the duplicate was dropped: %+v", pkg.PackageSupplier) + } + if pkg.PackageDescription != "Root certificates" { + t.Fatalf("description from the duplicate was dropped: %q", pkg.PackageDescription) + } + if !strings.HasSuffix(pkg.PackageDownloadLocation, ".whl") { + t.Fatalf("download location from the duplicate was dropped: %q", pkg.PackageDownloadLocation) + } + if len(pkg.PackageChecksums) != 1 { + t.Fatalf("digest from the duplicate was dropped: %+v", pkg.PackageChecksums) + } + if len(parseSPDXCPEs(pkg.PackageExternalReferences)) != 1 { + t.Fatal("CPE from the first component was lost in the merge") + } +} + // TestSPDXHomePageSurvivesSPDXRoundTrip covers a field SPDX represents // exactly, which an earlier revision decoded but never re-emitted. func TestSPDXHomePageSurvivesSPDXRoundTrip(t *testing.T) { diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 894ad655..2ff0b65c 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -407,6 +407,12 @@ func spdxChecksumAlgorithm(algorithm string) common.ChecksumAlgorithm { return common.SHA384 case "sha512", "sha-512": return common.SHA512 + case "blake2b-256": + return common.BLAKE2b_256 + case "blake2b-384": + return common.BLAKE2b_384 + case "blake2b-512": + return common.BLAKE2b_512 case "sha3-256": return common.SHA3_256 case "sha3-384": diff --git a/internal/sbom/transform.go b/internal/sbom/transform.go index 953a140a..61a9f37e 100644 --- a/internal/sbom/transform.go +++ b/internal/sbom/transform.go @@ -49,7 +49,10 @@ func FromDepGraph(g *sdk.Graph, opts BuildOptions) (*Document, error) { CPEs: append([]string(nil), pkg.CPEs...), } // Detection classifies, ingest corrects, enrichment fills gaps. - applyLocator(&component, classifyResolvedURL(pkg.ResolvedURL, pkg.Source, pkg.Ecosystem)) + applyLocator(&component, pinLocator( + classifyResolvedURL(pkg.ResolvedURL, pkg.Source, pkg.Ecosystem), + sourceRevisionFrom(pkg.Metadata), + )) applyIngestedMetadata(&component, pkg.Metadata) enrichComponentFromRegistry(&component, opts.Registry, pkg.PURL) components = append(components, component) From 973b0e32e60052a1bce3a74f2abd7106d14fdc2b Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 10:01:05 -0700 Subject: [PATCH 06/23] fix(sbom): require commit-shaped VCS fragments, keep repo alongside artifact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A credential could still reach output through the version-control path. That path is exempt from the query/fragment gate because normalizeVCS discards both and keeps a revision, but the revision itself was only checked with isSafeRevision, which permits letters, digits, and underscores — exactly the shape of an access token. An ingested "https://github.com/org/repo#ghp_abcd1234" was republished after the "@" in both formats. Confirmed by running the predicate over real token shapes before changing anything. Fragments must now be commit-shaped: bare hex, 4 to 64 characters. That is faithful to the format, since uv and cargo both write the resolved commit there, and it excludes token shapes on their underscores and non-hex letters. Query values keep the looser rule, because the key names them a revision and tags like "v1.2.3" are legitimate there. Two more from review: - A component asserting both a distribution and a vcs reference lost the repository. SPDX has one download-location field, the artifact takes it, and the source-info suppression was unconditional on VCSURL, so the repository was dropped even though PackageSourceInfo can hold it. Suppression now applies only when the VCS locator actually became the download location. - A primary component described in both metadata.component and the inventory kept only the inventory copy, so assertions placed on the metadata copy were discarded. They are now merged, inventory wins. The metadata-only primary component is deliberately still not promoted to a graph node: that is documented behavior (docs/SBOM.md), and adding it would demote the real graph roots on re-ingestion. Noted in code rather than changed silently. Verified the fragment guard is load-bearing by reverting it to isSafeRevision and confirming the test fails. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 17 ++++- internal/sbom/locator.go | 29 +++++++- internal/sbom/locator_test.go | 6 ++ internal/sbom/model.go | 41 +++++++++++ internal/sbom/roundtrip_test.go | 117 ++++++++++++++++++++++++++++++++ internal/sbom/spdx23.go | 21 ++++-- 6 files changed, 221 insertions(+), 10 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 17f56a72..a0456bba 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -175,9 +175,22 @@ func (c cycloneDXCodec) decodeJSON(data []byte) (*Document, error) { } } - if len(componentByID) == 0 && bom.Metadata != nil && bom.Metadata.Component != nil { + if bom.Metadata != nil && bom.Metadata.Component != nil { root := bom.Metadata.Component - componentByID[root.BOMRef] = componentFromCycloneDX(*root) + switch existing, listed := componentByID[root.BOMRef]; { + case listed: + // A producer may describe the primary component in both places + // and put assertions only on the metadata copy. Fold those in; + // the inventory entry stays authoritative for anything it set. + mergeComponentAssertions(&existing, componentFromCycloneDX(*root)) + componentByID[root.BOMRef] = existing + case len(componentByID) == 0: + componentByID[root.BOMRef] = componentFromCycloneDX(*root) + } + // A primary component that appears only in metadata.component while + // an inventory exists is deliberately not promoted to a node: it is + // the document's subject, re-synthesized on export, and adding it + // here would demote the real graph roots on re-ingestion. } components := make([]Component, 0, len(componentByID)) diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index 43453df3..a8fca346 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -233,8 +233,13 @@ func normalizeVCS(parsed *url.URL) Locator { // fragment records the immutable commit rather than a moving branch, and // matches the precedence uvSourceRevision already applies when the same // lockfile value is parsed for detection. + // + // A fragment must be commit-shaped, not merely character-safe: an access + // token such as "#ghp_abcd1234" passes isSafeRevision and would then be + // republished after the "@". A query value is held to the looser rule + // because its key names it a revision. revision := strings.TrimSpace(parsed.Fragment) - if !isSafeRevision(revision) { + if !isCommitFragment(revision) { revision = "" } if revision == "" { @@ -266,6 +271,28 @@ func normalizeVCS(parsed *url.URL) Locator { return Locator{Kind: LocatorVCS, URL: out} } +// isCommitFragment reports whether a URL fragment looks like a git object +// name. +// +// Fragments on a version-control URL carry the resolved commit by convention +// (uv and cargo both write one), so requiring bare hex is faithful to the +// format and, unlike the looser isSafeRevision, excludes credential shapes: +// "ghp_abcd1234" and "github_pat_11ABC_xyz" both fail on their underscores +// and non-hex letters. +func isCommitFragment(fragment string) bool { + if len(fragment) < 4 || len(fragment) > 64 { + return false + } + for _, r := range fragment { + switch { + case r >= '0' && r <= '9', r >= 'a' && r <= 'f', r >= 'A' && r <= 'F': + default: + return false + } + } + return true +} + // isSafeRevision reports whether revision is a plausible git revision that can // be appended to an SPDX version-control locator verbatim. // diff --git a/internal/sbom/locator_test.go b/internal/sbom/locator_test.go index f10cf369..4abedc48 100644 --- a/internal/sbom/locator_test.go +++ b/internal/sbom/locator_test.go @@ -75,6 +75,12 @@ func TestClassifyResolvedURL(t *testing.T) { // is the shape uv records: "?rev=main#abc123". {"resolved commit beats requested branch", "https://github.com/a/b?branch=main#abc123", sdk.DependencySourceGit, sdk.EcosystemPython, LocatorVCS, "git+https://github.com/a/b@abc123"}, {"resolved commit beats requested rev", "https://github.com/example/git-helper?rev=main#abc123", sdk.DependencySourceGit, sdk.EcosystemPython, LocatorVCS, "git+https://github.com/example/git-helper@abc123"}, + // A fragment must be commit-shaped, not merely character-safe: an + // access token passes isSafeRevision and would be republished. + {"pat fragment is not a revision", "https://github.com/org/repo#ghp_abcd1234", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, + {"fine-grained pat fragment is not a revision", "https://github.com/org/repo#github_pat_11ABCDE", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, + {"non-hex fragment is not a revision", "https://github.com/org/repo#release-candidate", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, + {"named query revision still allows tags", "https://github.com/org/repo?tag=v1.2.3", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo@v1.2.3"}, {"requested rev used when no commit resolved", "https://github.com/a/b?rev=v1.2.3", sdk.DependencySourceGit, sdk.EcosystemPython, LocatorVCS, "git+https://github.com/a/b@v1.2.3"}, // Degenerate input. diff --git a/internal/sbom/model.go b/internal/sbom/model.go index b7f01667..5df3e969 100644 --- a/internal/sbom/model.go +++ b/internal/sbom/model.go @@ -162,6 +162,47 @@ type Component struct { ExternalRefs []ExternalRef } +// mergeComponentAssertions fills gaps in dst from src, leaving anything dst +// already asserts untouched. Used when one component is described in more than +// one place in a source document. +func mergeComponentAssertions(dst *Component, src Component) { + if dst == nil { + return + } + for _, field := range []struct { + target *string + value string + }{ + {&dst.Supplier, src.Supplier}, + {&dst.SupplierType, src.SupplierType}, + {&dst.SupplierURL, src.SupplierURL}, + {&dst.Originator, src.Originator}, + {&dst.OriginatorType, src.OriginatorType}, + {&dst.Description, src.Description}, + {&dst.Repository, src.Repository}, + {&dst.ArtifactURL, src.ArtifactURL}, + {&dst.VCSURL, src.VCSURL}, + {&dst.RegistryURL, src.RegistryURL}, + {&dst.Copyright, src.Copyright}, + } { + if *field.target == "" { + *field.target = field.value + } + } + if len(dst.CPEs) == 0 { + dst.CPEs = src.CPEs + } + if len(dst.Digests) == 0 { + dst.Digests = src.Digests + } + if len(dst.Licenses) == 0 { + dst.Licenses = src.Licenses + } + if len(dst.ExternalRefs) == 0 { + dst.ExternalRefs = src.ExternalRefs + } +} + // ExternalRef is an external reference carried through from an ingested SBOM // so that a format conversion does not silently discard it. type ExternalRef struct { diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index e373a419..9a168765 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -454,6 +454,123 @@ func TestDuplicatePURLAssertionsAreMerged(t *testing.T) { } } +// TestIngestedVCSFragmentCredentialIsNotRepublished covers the version-control +// normalization path, which is exempt from the query/fragment gate. A fragment +// there is treated as a revision, so it must be commit-shaped rather than +// merely character-safe — an access token satisfies the looser rule. +func TestIngestedVCSFragmentCredentialIsNotRepublished(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", + "type": "library", + "name": "a", + "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "externalReferences": [ + {"type": "vcs", "url": "https://github.com/org/repo#ghp_abcd1234"} + ] + }] + }` + + for _, target := range []Target{TargetSPDX23JSON, TargetCycloneDX17JSON} { + out := ingestAndReexport(t, []byte(in), target) + rendered := string(out) + if strings.Contains(rendered, "ghp_abcd1234") { + t.Fatalf("%s republished a token from a vcs fragment:\n%s", target, rendered) + } + // The repository itself is still a legitimate assertion. + if !strings.Contains(rendered, "github.com/org/repo") { + t.Fatalf("%s dropped the repository along with the token:\n%s", target, rendered) + } + } +} + +// TestVCSSurvivesWhenArtifactOwnsDownloadLocation covers a component asserting +// both a distribution and a vcs reference. The artifact takes SPDX's single +// download-location field, so the repository has to land in PackageSourceInfo +// rather than being dropped. +func TestVCSSurvivesWhenArtifactOwnsDownloadLocation(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", + "type": "library", + "name": "a", + "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "externalReferences": [ + {"type": "distribution", "url": "https://registry.npmjs.org/a/-/a-1.0.0.tgz"}, + {"type": "vcs", "url": "https://github.com/org/repo"} + ] + }] + }` + + out := ingestAndReexport(t, []byte(in), TargetSPDX23JSON) + var doc v23.Document + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, p := range doc.Packages { + if p == nil || p.PackageName != "a" { + continue + } + if p.PackageDownloadLocation != "https://registry.npmjs.org/a/-/a-1.0.0.tgz" { + t.Fatalf("downloadLocation = %q, want the artifact", p.PackageDownloadLocation) + } + if !strings.Contains(p.PackageSourceInfo, "github.com/org/repo") { + t.Fatalf("sourceInfo = %q, want the repository preserved", p.PackageSourceInfo) + } + return + } + t.Fatal("component missing from output") +} + +// TestPrimaryComponentAssertionsMergeFromMetadata covers a producer that lists +// the primary component in the inventory but puts its assertions only on +// metadata.component. +func TestPrimaryComponentAssertionsMergeFromMetadata(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "metadata": {"component": { + "bom-ref": "root", "type": "application", "name": "app", "version": "1.0.0", + "purl": "pkg:npm/app@1.0.0", + "description": "The scanned application", + "supplier": {"name": "Example Supplier Inc."} + }}, + "components": [ + {"bom-ref": "root", "type": "application", "name": "app", "version": "1.0.0", "purl": "pkg:npm/app@1.0.0"}, + {"bom-ref": "pkg:npm/dep@1.0.0", "type": "library", "name": "dep", "version": "1.0.0", "purl": "pkg:npm/dep@1.0.0"} + ], + "dependencies": [{"ref": "root", "dependsOn": ["pkg:npm/dep@1.0.0"]}] + }` + + out := ingestAndReexport(t, []byte(in), TargetSPDX23JSON) + var doc v23.Document + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, p := range doc.Packages { + if p == nil || p.PackageName != "app" { + continue + } + if p.PackageDescription != "The scanned application" { + t.Fatalf("description = %q, want it merged from metadata.component", p.PackageDescription) + } + if p.PackageSupplier == nil || p.PackageSupplier.Supplier != "Example Supplier Inc." { + t.Fatalf("supplier = %+v, want it merged from metadata.component", p.PackageSupplier) + } + return + } + t.Fatal("app missing from output") +} + // TestSPDXHomePageSurvivesSPDXRoundTrip covers a field SPDX represents // exactly, which an earlier revision decoded but never re-emitted. func TestSPDXHomePageSurvivesSPDXRoundTrip(t *testing.T) { diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 2ff0b65c..1da703f0 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -525,16 +525,23 @@ func spdxDownloadLocation(component Component) string { // (the package's home page) that Bomly does not know, and SPDX 2.3 has no // version-control external-reference category. func spdxSourceInfo(component Component) string { - // A detector-supplied VCS location is version-exact and already rendered - // as the download location. Emitting the matcher-derived repository too - // would make one package assert two different source repositories. - if component.VCSURL != "" { + // Suppress only when the VCS locator actually became the download + // location, which is the case that would make one package assert two + // different source repositories. An ingested component may carry both a + // distribution and a vcs reference; there the artifact owns + // downloadLocation, and dropping the repository as well would lose it + // entirely even though PackageSourceInfo can represent it. + if component.VCSURL != "" && component.ArtifactURL == "" { return "" } - if repo := strings.TrimSpace(component.Repository); repo != "" { - return spdxSourceInfoPrefix + repo + repo := strings.TrimSpace(component.Repository) + if repo == "" && component.ArtifactURL != "" { + repo = strings.TrimPrefix(component.VCSURL, "git+") } - return "" + if repo == "" { + return "" + } + return spdxSourceInfoPrefix + repo } // spdxHomePage restores a home page carried on an ingested website reference. From 54f4c52dfcbafdc7188f41dcdb3131842079a3e9 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 10:12:39 -0700 Subject: [PATCH 07/23] fix(sbom): validate ingested digests, keep reference comments and summaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Malformed digests were copied verbatim from ingested documents. The encoders filter on algorithm only, so {"alg":"SHA-256","content": "not-a-hash"} survived the graph hop and was re-emitted: schema-invalid in both formats, and a false integrity assertion about the package. Ingested digests now go through the existing hex/SRI normalization and are checked against the algorithm's expected length, dropping anything that cannot be a digest. This exposed unrealistic test data of my own: several fixtures used "abc123" as a SHA-256, which is six hex characters rather than sixty-four. The validator was right and the fixtures were wrong; they now carry full-length values. Comments on classified references were lost, and worse, replaced. A distribution or vcs reference kept only its URL, so a producer's comment disappeared on a CycloneDX round trip — and for a registry-shaped URL, Bomly's fixed "Registry root" text was substituted, which can flatly contradict an assertion such as "Exact authenticated download endpoint". Comments now ride alongside each locator, and Bomly's explanatory text is used only when the producer supplied none. SPDX summary and description are no longer conflated. The decoder folded PackageSummary into Description, which moves a summary into a semantically different field and drops it outright when both are present; SPDX 2.3 represents them distinctly. Summary is now carried separately and restored on export, and is used in CycloneDX only as a fallback, since that format has a single field. Verified the digest validator is load-bearing by disabling its checks and confirming the test fails. Real scans are unchanged: the detection path does not route through this validation, and yarn/npm checksum counts match the previous commit exactly. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 27 ++++--- internal/sbom/ingest_metadata.go | 12 +++ internal/sbom/model.go | 17 ++++- internal/sbom/roundtrip_test.go | 125 ++++++++++++++++++++++++++++++- internal/sbom/spdx23.go | 10 ++- internal/sbom/transform.go | 38 +++++++++- 6 files changed, 207 insertions(+), 22 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index a0456bba..b183d928 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -60,7 +60,7 @@ func (c cycloneDXCodec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, e component.Supplier = supplier } component.Publisher = comp.Originator - component.Description = comp.Description + component.Description = firstNonEmpty(comp.Description, comp.Summary) if refs := cycloneDXComponentReferences(comp); len(refs) > 0 { component.ExternalReferences = &refs } @@ -378,22 +378,23 @@ func componentFromCycloneDX(comp cdx.Component) Component { } if comp.Hashes != nil { for _, hash := range *comp.Hashes { - component.Digests = append(component.Digests, Digest{ - Algorithm: normalizeDigestAlgorithm(string(hash.Algorithm)), - Value: hash.Value, - }) + if digest, ok := ingestedDigest(string(hash.Algorithm), hash.Value); ok { + component.Digests = append(component.Digests, digest) + } } } if comp.ExternalReferences != nil { for _, ref := range *comp.ExternalReferences { switch ref.Type { case cdx.ERTypeDistribution: - applyLocator(&component, classifyResolvedURL(ref.URL, "", "")) + applyLocatorComment(&component, classifyResolvedURL(ref.URL, "", ""), ref.Comment) case cdx.ERTypeVCS: // Untrusted input: gate and normalize it exactly like a // detector-supplied value, because it is re-emitted and also // becomes the SPDX download location. - component.VCSURL = classifyIngestedVCS(ref.URL) + if vcs := classifyIngestedVCS(ref.URL); vcs != "" { + component.VCSURL, component.VCSComment = vcs, ref.Comment + } default: if !isPublishableReferenceURL(ref.URL) { continue @@ -421,12 +422,18 @@ func cycloneDXComponentReferences(component Component) []cdx.ExternalReference { switch { case component.ArtifactURL != "": - refs = append(refs, cdx.ExternalReference{Type: cdx.ERTypeDistribution, URL: component.ArtifactURL}) + refs = append(refs, cdx.ExternalReference{ + Type: cdx.ERTypeDistribution, + URL: component.ArtifactURL, + Comment: component.ArtifactComment, + }) case component.RegistryURL != "": + // Only explain the value when the producer said nothing: replacing a + // source document's own comment could contradict what it asserted. refs = append(refs, cdx.ExternalReference{ Type: cdx.ERTypeDistribution, URL: component.RegistryURL, - Comment: "Registry root; not the exact artifact location", + Comment: firstNonEmpty(component.RegistryComment, "Registry root; not the exact artifact location"), }) } @@ -434,7 +441,7 @@ func cycloneDXComponentReferences(component Component) []cdx.ExternalReference { // scorecard repository. Emitting both would assert the same repository // twice from two sources. if vcs := firstNonEmpty(component.VCSURL, component.Repository); vcs != "" { - refs = append(refs, cdx.ExternalReference{Type: cdx.ERTypeVCS, URL: vcs}) + refs = append(refs, cdx.ExternalReference{Type: cdx.ERTypeVCS, URL: vcs, Comment: component.VCSComment}) } for _, ref := range component.ExternalRefs { diff --git a/internal/sbom/ingest_metadata.go b/internal/sbom/ingest_metadata.go index 12da4d40..c8cf6e37 100644 --- a/internal/sbom/ingest_metadata.go +++ b/internal/sbom/ingest_metadata.go @@ -19,6 +19,10 @@ const ( metadataKeyOriginator = "bomly.sbom.originator" metadataKeyOriginatorType = "bomly.sbom.originator_type" metadataKeyDescription = "bomly.sbom.description" + metadataKeySummary = "bomly.sbom.summary" + metadataKeyArtifactNote = "bomly.sbom.artifact_comment" + metadataKeyVCSNote = "bomly.sbom.vcs_comment" + metadataKeyRegistryNote = "bomly.sbom.registry_comment" metadataKeyRepository = "bomly.sbom.repository" metadataKeyArtifactURL = "bomly.sbom.artifact_url" metadataKeyVCSURL = "bomly.sbom.vcs_url" @@ -39,6 +43,10 @@ func setIngestedMetadata(dep *sdk.Dependency, component Component) { metadataKeyOriginator: component.Originator, metadataKeyOriginatorType: component.OriginatorType, metadataKeyDescription: component.Description, + metadataKeySummary: component.Summary, + metadataKeyArtifactNote: component.ArtifactComment, + metadataKeyVCSNote: component.VCSComment, + metadataKeyRegistryNote: component.RegistryComment, metadataKeyRepository: component.Repository, metadataKeyArtifactURL: component.ArtifactURL, metadataKeyVCSURL: component.VCSURL, @@ -94,6 +102,10 @@ func applyIngestedMetadata(component *Component, metadata map[string]any) { metadataKeyOriginator: &component.Originator, metadataKeyOriginatorType: &component.OriginatorType, metadataKeyDescription: &component.Description, + metadataKeySummary: &component.Summary, + metadataKeyArtifactNote: &component.ArtifactComment, + metadataKeyVCSNote: &component.VCSComment, + metadataKeyRegistryNote: &component.RegistryComment, metadataKeyRepository: &component.Repository, } for key, target := range targets { diff --git a/internal/sbom/model.go b/internal/sbom/model.go index 5df3e969..12abf619 100644 --- a/internal/sbom/model.go +++ b/internal/sbom/model.go @@ -146,6 +146,13 @@ type Component struct { VCSURL string RegistryURL string + // Comments a source document attached to the references above. Kept + // separately so an ingested comment is preserved rather than replaced by + // Bomly's own, which could contradict what the producer asserted. + ArtifactComment string + VCSComment string + RegistryComment string + // Repository is a canonical "github.com/owner/repo" source repository // supplied by the OpenSSF Scorecard matcher during enrichment. Repository string @@ -159,7 +166,11 @@ type Component struct { Originator string OriginatorType string Description string - ExternalRefs []ExternalRef + // Summary is SPDX's short-form description. SPDX 2.3 represents it and + // PackageDescription as distinct fields; CycloneDX has only one, so the + // summary is used there just as a fallback. + Summary string + ExternalRefs []ExternalRef } // mergeComponentAssertions fills gaps in dst from src, leaving anything dst @@ -179,10 +190,14 @@ func mergeComponentAssertions(dst *Component, src Component) { {&dst.Originator, src.Originator}, {&dst.OriginatorType, src.OriginatorType}, {&dst.Description, src.Description}, + {&dst.Summary, src.Summary}, {&dst.Repository, src.Repository}, {&dst.ArtifactURL, src.ArtifactURL}, {&dst.VCSURL, src.VCSURL}, {&dst.RegistryURL, src.RegistryURL}, + {&dst.ArtifactComment, src.ArtifactComment}, + {&dst.VCSComment, src.VCSComment}, + {&dst.RegistryComment, src.RegistryComment}, {&dst.Copyright, src.Copyright}, } { if *field.target == "" { diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 9a168765..413cf8b0 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -36,7 +36,7 @@ const supplierRichCycloneDX = `{ "publisher": "azer", "supplier": {"name": "Example Supplier Inc.", "url": ["https://supplier.example.com"]}, "cpe": "cpe:2.3:a:example:left-pad:1.3.0:*:*:*:*:*:*:*", - "hashes": [{"alg": "SHA-256", "content": "abc123"}], + "hashes": [{"alg": "SHA-256", "content": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}], "externalReferences": [ {"type": "distribution", "url": "https://registry.npmjs.org/left-pad/-/left-pad-1.3.0.tgz"}, {"type": "vcs", "url": "https://github.com/stevemao/left-pad"}, @@ -414,7 +414,7 @@ func TestDuplicatePURLAssertionsAreMerged(t *testing.T) { Supplier: "Example Supplier Inc.", Description: "Root certificates", ArtifactURL: "https://files.pythonhosted.org/x/certifi-2026.4.22-py3-none-any.whl", - Digests: []Digest{{Algorithm: "sha256", Value: "abc123"}}, + Digests: []Digest{{Algorithm: "sha256", Value: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}, }, }, Dependencies: []Dependency{{Ref: "first", DependsOn: []string{"second"}}}, @@ -571,6 +571,123 @@ func TestPrimaryComponentAssertionsMergeFromMetadata(t *testing.T) { t.Fatal("app missing from output") } +// TestMalformedIngestedDigestIsDropped keeps a bogus hash out of the output. +// The encoders filter on algorithm only, so an unvalidated value would be +// re-emitted verbatim: schema-invalid, and a false integrity assertion. +func TestMalformedIngestedDigestIsDropped(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", + "type": "library", + "name": "a", + "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "hashes": [ + {"alg": "SHA-256", "content": "not-a-hash"}, + {"alg": "SHA-256", "content": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"} + ] + }] + }` + + for _, target := range []Target{TargetSPDX23JSON, TargetCycloneDX17JSON} { + out := ingestAndReexport(t, []byte(in), target) + if strings.Contains(string(out), "not-a-hash") { + t.Fatalf("%s republished a malformed digest:\n%s", target, out) + } + if !strings.Contains(string(out), "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc") { + t.Fatalf("%s dropped the valid digest alongside the malformed one:\n%s", target, out) + } + } +} + +// TestIngestedReferenceCommentsArePreserved covers comments on the two +// reference kinds Bomly classifies. Replacing a producer's comment with +// Bomly's own could contradict what the source asserted. +func TestIngestedReferenceCommentsArePreserved(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", + "type": "library", + "name": "a", + "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "externalReferences": [ + {"type": "distribution", "url": "https://reg.example/", "comment": "Exact authenticated download endpoint"}, + {"type": "vcs", "url": "https://github.com/org/repo", "comment": "Mirror of the upstream repository"} + ] + }] + }` + + out := ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON) + var bom cdx.BOM + if err := json.Unmarshal(out, &bom); err != nil { + t.Fatalf("unmarshal: %v", err) + } + comp := (*bom.Components)[0] + if comp.ExternalReferences == nil { + t.Fatal("no external references in output") + } + for _, ref := range *comp.ExternalReferences { + switch ref.Type { + case cdx.ERTypeDistribution: + if ref.Comment != "Exact authenticated download endpoint" { + t.Fatalf("distribution comment = %q, want the producer's own text", ref.Comment) + } + case cdx.ERTypeVCS: + if ref.Comment != "Mirror of the upstream repository" { + t.Fatalf("vcs comment = %q, want the producer's own text", ref.Comment) + } + } + } +} + +// TestSPDXSummaryAndDescriptionStaySeparate covers two fields SPDX represents +// distinctly. Folding the summary into the description moves it to a +// semantically different field and loses it entirely when both are present. +func TestSPDXSummaryAndDescriptionStaySeparate(t *testing.T) { + const in = `{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [{ + "name": "left-pad", + "SPDXID": "SPDXRef-Package-left-pad", + "versionInfo": "1.3.0", + "downloadLocation": "NOASSERTION", + "summary": "Pads a string", + "description": "A longer explanation of the padding behaviour", + "filesAnalyzed": false + }] + }` + + out := ingestAndReexport(t, []byte(in), TargetSPDX23JSON) + var doc v23.Document + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, p := range doc.Packages { + if p == nil || p.PackageName != "left-pad" { + continue + } + if p.PackageSummary != "Pads a string" { + t.Fatalf("summary = %q, want it preserved separately", p.PackageSummary) + } + if p.PackageDescription != "A longer explanation of the padding behaviour" { + t.Fatalf("description = %q, want it preserved separately", p.PackageDescription) + } + return + } + t.Fatal("left-pad missing from output") +} + // TestSPDXHomePageSurvivesSPDXRoundTrip covers a field SPDX represents // exactly, which an earlier revision decoded but never re-emitted. func TestSPDXHomePageSurvivesSPDXRoundTrip(t *testing.T) { @@ -586,7 +703,7 @@ func TestSPDXHomePageSurvivesSPDXRoundTrip(t *testing.T) { "versionInfo": "1.3.0", "downloadLocation": "https://repo.example/download/left-pad", "homepage": "https://left-pad.example.com", - "checksums": [{"algorithm": "SHA3-256", "checksumValue": "abc123"}], + "checksums": [{"algorithm": "SHA3-256", "checksumValue": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}], "filesAnalyzed": false }] }` @@ -613,7 +730,7 @@ func TestSPDXHomePageSurvivesSPDXRoundTrip(t *testing.T) { } // Assert both fields: a length check alone would not notice the // algorithm degrading to a different family on the way through. - if got := p.PackageChecksums[0]; got.Algorithm != common.SHA3_256 || got.Value != "abc123" { + if got := p.PackageChecksums[0]; got.Algorithm != common.SHA3_256 || got.Value != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { t.Fatalf("checksum = %+v, want SHA3-256/abc123", got) } return diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 1da703f0..8555d9fc 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -54,6 +54,7 @@ func (spdx23Codec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, error) PackageExternalReferences: spdxExternalReferences(c), PrimaryPackagePurpose: spdxPrimaryPackagePurpose(c.Type), PackageDescription: c.Description, + PackageSummary: c.Summary, PackageSourceInfo: spdxSourceInfo(c), PackageHomePage: spdxHomePage(c), } @@ -175,7 +176,8 @@ func (spdx23Codec) decodeJSON(data []byte) (*Document, error) { PackageManager: parseSPDXPackageManager(p.PackageExternalReferences), Copyright: parseSPDXCopyright(p.PackageCopyrightText), Licenses: parseSPDXLicenses(p.PackageLicenseConcluded, p.PackageLicenseDeclared), - Description: firstNonEmpty(parseSPDXEntity(p.PackageDescription), parseSPDXEntity(p.PackageSummary)), + Description: parseSPDXEntity(p.PackageDescription), + Summary: parseSPDXEntity(p.PackageSummary), Repository: parseSPDXSourceInfo(p.PackageSourceInfo), CPEs: parseSPDXCPEs(p.PackageExternalReferences), Digests: parseSPDXChecksums(p.PackageChecksums), @@ -658,11 +660,11 @@ func parseSPDXChecksums(checksums []common.Checksum) []Digest { } digests := make([]Digest, 0, len(checksums)) for _, checksum := range checksums { - algorithm := normalizeDigestAlgorithm(string(checksum.Algorithm)) - if algorithm == "" || strings.TrimSpace(checksum.Value) == "" { + digest, ok := ingestedDigest(string(checksum.Algorithm), checksum.Value) + if !ok { continue } - digests = append(digests, Digest{Algorithm: algorithm, Value: strings.TrimSpace(checksum.Value)}) + digests = append(digests, digest) } if len(digests) == 0 { return nil diff --git a/internal/sbom/transform.go b/internal/sbom/transform.go index 61a9f37e..211b42f2 100644 --- a/internal/sbom/transform.go +++ b/internal/sbom/transform.go @@ -251,6 +251,32 @@ func normalizeDigestValue(algorithm, value string) string { return value } +// ingestedDigest normalizes and validates a digest taken from an untrusted +// SBOM, returning ok=false when the value cannot be a digest of the stated +// algorithm. +// +// The encoders filter on algorithm only, so a malformed value such as +// {"alg":"SHA-256","content":"not-a-hash"} would otherwise be re-emitted +// verbatim: schema-invalid in both formats, and a false integrity assertion +// about the package. An algorithm with no known length is accepted only when +// the value is plausible hex, since there is nothing else to check it against. +func ingestedDigest(algorithm, value string) (Digest, bool) { + algorithm = normalizeDigestAlgorithm(algorithm) + value = strings.TrimSpace(value) + if algorithm == "" || value == "" { + return Digest{}, false + } + + normalized := normalizeDigestValue(algorithm, value) + if size, known := digestHexSizes[algorithm]; known && len(normalized) != size*2 { + return Digest{}, false + } + if _, err := hex.DecodeString(normalized); err != nil { + return Digest{}, false + } + return Digest{Algorithm: algorithm, Value: strings.ToLower(normalized)}, true +} + func uniqueToolNames(values []string) []string { out := make([]string, 0, len(values)) seen := make(map[string]struct{}, len(values)) @@ -343,13 +369,19 @@ func normalizeDigestAlgorithm(algorithm string) string { // LocatorNone leaves the component untouched, so an unpublishable value simply // results in no assertion. func applyLocator(component *Component, locator Locator) { + applyLocatorComment(component, locator, "") +} + +// applyLocatorComment is applyLocator for an ingested reference, carrying the +// producer's own comment alongside the classified URL. +func applyLocatorComment(component *Component, locator Locator, comment string) { switch locator.Kind { case LocatorArtifact: - component.ArtifactURL = locator.URL + component.ArtifactURL, component.ArtifactComment = locator.URL, comment case LocatorVCS: - component.VCSURL = locator.URL + component.VCSURL, component.VCSComment = locator.URL, comment case LocatorRegistryRoot: - component.RegistryURL = locator.URL + component.RegistryURL, component.RegistryComment = locator.URL, comment } } From 11a5e6776a7088d7545d4dc483f35f6c6ba2b0dd Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 10:23:01 -0700 Subject: [PATCH 08/23] fix(sbom): keep CPE 2.2 labels, urn references, and unioned reference sets Four fidelity findings from review. CPE 2.2 locators were relabelled as 2.3. The previous commit taught the decoder to accept cpe22Type but the encoder still wrote every CPE as cpe23Type, so an ingested "cpe:/a:vendor:product:1.0" came back claiming a syntax it does not use. The two forms are self-describing, so the type is now derived from the locator rather than carried as extra state. Valid non-HTTP external references were dropped. CycloneDX external-reference URLs are IRI references, and rejecting every scheme outside http(s) and mailto discarded legitimate values such as a `bom` reference to "urn:uuid:...". urn, git, ftp, and ftps are now accepted; file:, data:, javascript:, and unknown schemes stay rejected, since these values are republished verbatim and denying the unknown is the safe default. External references were treated as a scalar in both merges. They are a set: each copy of a component may name a different link, so a fill-gaps copy dropped the second list whenever the first had any. This affected duplicate-PURL components at the graph level, where the whole list rides under one metadata key, and a primary component described in both metadata.component and the inventory at the model level. Both now union by type and URL. Both union fixes were verified load-bearing by reverting each to fill-gaps and confirming the tests fail. Co-Authored-By: Claude Opus 5 --- internal/sbom/graph.go | 51 ++++++++++++ internal/sbom/locator.go | 12 ++- internal/sbom/locator_test.go | 7 ++ internal/sbom/model.go | 27 ++++++- internal/sbom/roundtrip_test.go | 133 ++++++++++++++++++++++++++++++++ internal/sbom/spdx23.go | 16 +++- 6 files changed, 242 insertions(+), 4 deletions(-) diff --git a/internal/sbom/graph.go b/internal/sbom/graph.go index 4cbbf468..f5baaf4d 100644 --- a/internal/sbom/graph.go +++ b/internal/sbom/graph.go @@ -146,12 +146,63 @@ func mergeIngestedNode(existing, incoming *sdk.Dependency) { if existing.Metadata == nil { existing.Metadata = make(map[string]any, len(incoming.Metadata)) } + // External references are a set carried under one key, so a key-level + // fill-gaps merge would drop the duplicate's whole list whenever the + // first component had any. Union them instead. + if key == metadataKeyExternalRefs { + existing.Metadata[key] = unionExternalRefValues(existing.Metadata[key], value) + continue + } if _, present := existing.Metadata[key]; !present { existing.Metadata[key] = value } } } +// unionExternalRefValues merges two serialized external-reference lists, +// keyed by type and URL. Entries that are not the expected shape are kept +// as-is on the base side and skipped on the incoming side. +func unionExternalRefValues(base, extra any) any { + baseList, _ := base.([]any) + extraList, _ := extra.([]any) + if len(extraList) == 0 { + return base + } + if len(baseList) == 0 { + return extra + } + + type key struct{ refType, url string } + refKey := func(entry any) (key, bool) { + fields, ok := entry.(map[string]any) + if !ok { + return key{}, false + } + refType, _ := fields["type"].(string) + url, _ := fields["url"].(string) + return key{refType, url}, true + } + + seen := make(map[key]struct{}, len(baseList)) + for _, entry := range baseList { + if k, ok := refKey(entry); ok { + seen[k] = struct{}{} + } + } + for _, entry := range extraList { + k, ok := refKey(entry) + if !ok { + continue + } + if _, present := seen[k]; present { + continue + } + seen[k] = struct{}{} + baseList = append(baseList, entry) + } + return baseList +} + // unionStrings appends values from extra that are not already in base. func unionStrings(base, extra []string) []string { seen := make(map[string]struct{}, len(base)) diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index a8fca346..956d870b 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -424,9 +424,19 @@ func isPublishableReferenceURL(raw string) bool { switch strings.ToLower(parsed.Scheme) { case "http", "https": return parsed.Host != "" && parsed.RawQuery == "" && parsed.Fragment == "" - case "mailto": + case "git", "ftp", "ftps": + // Network transports CycloneDX legitimately uses for distribution and + // version-control references. + return parsed.Host != "" && parsed.RawQuery == "" && parsed.Fragment == "" + case "mailto", "urn": + // Opaque identifiers with no host and no filesystem reach. A `bom` + // reference to "urn:uuid:..." is the common CycloneDX case. return parsed.Opaque != "" && parsed.RawQuery == "" && parsed.Fragment == "" default: + // Anything else — notably file:, data:, and javascript: — stays + // rejected. Denying unknown schemes is the safe default: these values + // are republished verbatim, and a scheme Bomly cannot reason about + // may reference the local machine or execute in a consumer. return false } } diff --git a/internal/sbom/locator_test.go b/internal/sbom/locator_test.go index 4abedc48..9ac74362 100644 --- a/internal/sbom/locator_test.go +++ b/internal/sbom/locator_test.go @@ -170,9 +170,16 @@ func TestIsPublishableReferenceURL(t *testing.T) { {"https://example.com/docs?token=s3cret", false}, {"https://example.com/docs#s3cret", false}, {"https://tok:s3cret@example.com/docs", false}, + // CycloneDX external-reference URLs are IRI references, so safe + // non-HTTP identifiers must survive a round trip. + {"urn:uuid:3f2504e0-4f89-41d3-9a0c-0305e82c3301", true}, + {"git://github.com/org/repo", true}, + {"ftp://files.example.com/pkg.tgz", true}, + {"urn:uuid:3f2504e0#s3cret", false}, {"file:///Users/victim/secret.html", false}, {"/Users/victim/secret.html", false}, {"javascript:alert(1)", false}, + {"data:text/html,", false}, {"https://", false}, {"mailto:", false}, {"", false}, diff --git a/internal/sbom/model.go b/internal/sbom/model.go index 12abf619..3857f3bc 100644 --- a/internal/sbom/model.go +++ b/internal/sbom/model.go @@ -213,9 +213,32 @@ func mergeComponentAssertions(dst *Component, src Component) { if len(dst.Licenses) == 0 { dst.Licenses = src.Licenses } - if len(dst.ExternalRefs) == 0 { - dst.ExternalRefs = src.ExternalRefs + // External references are a set, not a single assertion: each copy may + // name a different link, so a fill-gaps copy would drop the second one + // entirely whenever the first had any. + dst.ExternalRefs = unionExternalRefs(dst.ExternalRefs, src.ExternalRefs) +} + +// unionExternalRefs appends references from extra that base does not already +// carry, keyed by type and URL. +func unionExternalRefs(base, extra []ExternalRef) []ExternalRef { + if len(extra) == 0 { + return base + } + type key struct{ refType, url string } + seen := make(map[key]struct{}, len(base)) + for _, ref := range base { + seen[key{ref.Type, ref.URL}] = struct{}{} + } + for _, ref := range extra { + k := key{ref.Type, ref.URL} + if _, ok := seen[k]; ok { + continue + } + seen[k] = struct{}{} + base = append(base, ref) } + return base } // ExternalRef is an external reference carried through from an ingested SBOM diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 413cf8b0..2b8338c8 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -688,6 +688,139 @@ func TestSPDXSummaryAndDescriptionStaySeparate(t *testing.T) { t.Fatal("left-pad missing from output") } +// TestCPE22ReferenceTypeIsPreserved keeps an SPDX round trip from relabelling +// a CPE 2.2 locator as 2.3 without converting its syntax. +func TestCPE22ReferenceTypeIsPreserved(t *testing.T) { + const in = `{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [{ + "name": "left-pad", + "SPDXID": "SPDXRef-Package-left-pad", + "versionInfo": "1.3.0", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": false, + "externalRefs": [ + {"referenceCategory": "SECURITY", "referenceType": "cpe22Type", "referenceLocator": "cpe:/a:vendor:product:1.0"}, + {"referenceCategory": "SECURITY", "referenceType": "cpe23Type", "referenceLocator": "cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*"} + ] + }] + }` + + out := ingestAndReexport(t, []byte(in), TargetSPDX23JSON) + var doc v23.Document + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + byLocator := map[string]string{} + for _, p := range doc.Packages { + if p == nil { + continue + } + for _, ref := range p.PackageExternalReferences { + if ref != nil { + byLocator[ref.Locator] = ref.RefType + } + } + } + if got := byLocator["cpe:/a:vendor:product:1.0"]; got != "cpe22Type" { + t.Fatalf("cpe 2.2 locator emitted as %q, want cpe22Type", got) + } + if got := byLocator["cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*"]; got != "cpe23Type" { + t.Fatalf("cpe 2.3 locator emitted as %q, want cpe23Type", got) + } +} + +// TestUrnExternalReferenceSurvives covers a valid non-HTTP IRI. CycloneDX +// external-reference URLs are IRI references, so a "bom" reference to a +// urn:uuid must not be dropped as an unknown scheme. +func TestUrnExternalReferenceSurvives(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", + "type": "library", + "name": "a", + "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "externalReferences": [ + {"type": "bom", "url": "urn:uuid:3f2504e0-4f89-41d3-9a0c-0305e82c3301"} + ] + }] + }` + + out := ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON) + if !strings.Contains(string(out), "urn:uuid:3f2504e0-4f89-41d3-9a0c-0305e82c3301") { + t.Fatalf("a valid urn external reference was dropped:\n%s", out) + } +} + +// TestDuplicatePURLExternalRefsAreUnioned covers reference sets on two +// components sharing a PURL. They are a set, not a single assertion, so a +// fill-gaps merge would drop the second list whenever the first had any. +func TestDuplicatePURLExternalRefsAreUnioned(t *testing.T) { + doc := &Document{ + Components: []Component{ + { + ID: "first", Name: "a", Version: "1.0.0", PURL: "pkg:npm/a@1.0.0", + ExternalRefs: []ExternalRef{{Type: "website", URL: "https://example.com/site"}}, + }, + { + ID: "second", Name: "a", Version: "1.0.0", PURL: "pkg:npm/a@1.0.0", + ExternalRefs: []ExternalRef{{Type: "documentation", URL: "https://example.com/docs"}}, + }, + }, + Dependencies: []Dependency{{Ref: "first", DependsOn: []string{"second"}}}, + } + + graph, err := ToGraph(doc) + if err != nil { + t.Fatalf("to graph: %v", err) + } + out, err := MarshalDepGraphJSON(graph, TargetCycloneDX17JSON, BuildOptions{}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + for _, want := range []string{"https://example.com/site", "https://example.com/docs"} { + if !strings.Contains(string(out), want) { + t.Fatalf("merged component lost %q:\n%s", want, out) + } + } +} + +// TestPrimaryComponentExternalRefsAreUnioned is the same set semantics for a +// primary component described in both metadata.component and the inventory. +func TestPrimaryComponentExternalRefsAreUnioned(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "metadata": {"component": { + "bom-ref": "root", "type": "application", "name": "app", "version": "1.0.0", + "purl": "pkg:npm/app@1.0.0", + "externalReferences": [{"type": "documentation", "url": "https://example.com/docs"}] + }}, + "components": [ + {"bom-ref": "root", "type": "application", "name": "app", "version": "1.0.0", "purl": "pkg:npm/app@1.0.0", + "externalReferences": [{"type": "website", "url": "https://example.com/site"}]}, + {"bom-ref": "pkg:npm/dep@1.0.0", "type": "library", "name": "dep", "version": "1.0.0", "purl": "pkg:npm/dep@1.0.0"} + ], + "dependencies": [{"ref": "root", "dependsOn": ["pkg:npm/dep@1.0.0"]}] + }` + + out := ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON) + for _, want := range []string{"https://example.com/site", "https://example.com/docs"} { + if !strings.Contains(string(out), want) { + t.Fatalf("primary component lost %q:\n%s", want, out) + } + } +} + // TestSPDXHomePageSurvivesSPDXRoundTrip covers a field SPDX represents // exactly, which an earlier revision decoded but never re-emitted. func TestSPDXHomePageSurvivesSPDXRoundTrip(t *testing.T) { diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 8555d9fc..12ad59b9 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -588,7 +588,7 @@ func spdxExternalReferences(component Component) []*v23.PackageExternalReference } refs = append(refs, &v23.PackageExternalReference{ Category: "SECURITY", - RefType: "cpe23Type", + RefType: spdxCPERefType(cpe), Locator: cpe, }) } @@ -633,6 +633,20 @@ func parseSPDXLicenses(values ...string) []License { return nil } +// spdxCPERefType labels a CPE locator with the reference type matching its own +// syntax. +// +// The two forms are self-describing — 2.3 is colon-delimited and starts +// "cpe:2.3:", 2.2 starts "cpe:/" — so the type is derived rather than carried. +// Emitting a 2.2 locator as cpe23Type would relabel it without converting it, +// asserting a syntax the value does not use. +func spdxCPERefType(cpe string) string { + if strings.HasPrefix(strings.TrimSpace(cpe), "cpe:/") { + return "cpe22Type" + } + return "cpe23Type" +} + // parseSPDXCPEs recovers CPE identifiers from a package's security external // references. func parseSPDXCPEs(refs []*v23.PackageExternalReference) []string { From 8ef7310b3fa76b476ca416d2f53ce1587c215b4a Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 10:35:55 -0700 Subject: [PATCH 09/23] test(sbom): assert digest algorithms, not just counts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review flagged that TestBlake2bDigestsSurviveRoundTrip only counted checksums. The stated mechanism does not hold — both encoders skip an algorithm they cannot map, so a missing case already fails the count, which I confirmed by deleting the BLAKE2b cases and watching the test go red. But the suggestion closes a real gap in the other direction: a case mapped to the *wrong* constant keeps the count at one while relabelling the digest as a different family. Verified by pointing blake2b-256 at SHA256, which the old assertion accepted and the new one rejects. The test now asserts the exact emitted algorithm and value in both formats, and covers sha256 alongside the sha3 and blake2b families. Also corrects two stale statements: a failure message still naming the old "abc123" fixture value, and a docs sentence saying "three kinds" above a four-row table. The three kinds are right — an exact file, a source repository, a registry root — so the prose now names them and says plainly that a value fitting none of them is not published. Co-Authored-By: Claude Opus 5 --- docs/SBOM.md | 4 ++- internal/sbom/distribution_test.go | 40 +++++++++++++++++++++++++----- internal/sbom/roundtrip_test.go | 2 +- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/docs/SBOM.md b/docs/SBOM.md index 25884186..3979af0d 100644 --- a/docs/SBOM.md +++ b/docs/SBOM.md @@ -106,7 +106,9 @@ Both formats carry: Lockfiles record very different things in the same field. Some name the exact file that was downloaded, some name only the registry the ecosystem fetches from, and some name a directory on the machine that ran the scan. Bomly sorts -each value into one of three kinds and emits it accordingly: +each value into one of three kinds — an exact package file, a source +repository, or a registry root — and emits it accordingly. A value that fits +none of them, such as a local path, is not published at all: | What the lockfile recorded | SPDX | CycloneDX | | --- | --- | --- | diff --git a/internal/sbom/distribution_test.go b/internal/sbom/distribution_test.go index 1a6d64a3..f8dae487 100644 --- a/internal/sbom/distribution_test.go +++ b/internal/sbom/distribution_test.go @@ -7,6 +7,7 @@ import ( cdx "github.com/CycloneDX/cyclonedx-go" "github.com/bomly-dev/bomly-sdk" + "github.com/spdx/tools-golang/spdx/v2/common" v23 "github.com/spdx/tools-golang/spdx/v2/v2_3" ) @@ -218,27 +219,54 @@ func TestUnsafeDetectorRevisionIsIgnored(t *testing.T) { // TestBlake2bDigestsSurviveRoundTrip pairs with normalizeDigestAlgorithm: the // canonical names it preserves must exist in both encoders, or the checksum is // silently discarded after the graph hop. +// +// Both encoders skip an algorithm they cannot map, so a count check already +// catches a missing case. The algorithm is asserted as well to catch the other +// direction — a case mapped to the wrong constant, which would keep the count +// at one while relabelling the digest as a different family. func TestBlake2bDigestsSurviveRoundTrip(t *testing.T) { - for _, algorithm := range []string{"blake2b-256", "blake2b-384", "blake2b-512", "sha3-256"} { - t.Run(algorithm, func(t *testing.T) { + const value = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + + cases := []struct { + algorithm string + wantSPDX common.ChecksumAlgorithm + wantCDX cdx.HashAlgorithm + }{ + {"blake2b-256", common.BLAKE2b_256, cdx.HashAlgoBlake2b_256}, + {"blake2b-384", common.BLAKE2b_384, cdx.HashAlgoBlake2b_384}, + {"blake2b-512", common.BLAKE2b_512, cdx.HashAlgoBlake2b_512}, + {"sha3-256", common.SHA3_256, cdx.HashAlgoSHA3_256}, + {"sha256", common.SHA256, cdx.HashAlgoSHA256}, + } + + for _, tc := range cases { + t.Run(tc.algorithm, func(t *testing.T) { g := sdk.New() node := sdk.NewDependencyWithID("pkg@1.0.0", sdk.Dependency{ Coordinates: sdk.Coordinates{ Name: "pkg", Version: "1.0.0", PURL: "pkg:npm/pkg@1.0.0", Ecosystem: sdk.EcosystemNPM, }, - Digests: []sdk.Digest{{Algorithm: sdk.DigestAlgorithm(algorithm), Value: "abc123"}}, + Digests: []sdk.Digest{{Algorithm: sdk.DigestAlgorithm(tc.algorithm), Value: value}}, }) if err := g.AddNode(node); err != nil { t.Fatalf("add node: %v", err) } - if pkg := spdxPackageFor(t, g, BuildOptions{}); len(pkg.PackageChecksums) != 1 { - t.Fatalf("spdx dropped a %s checksum: %+v", algorithm, pkg.PackageChecksums) + pkg := spdxPackageFor(t, g, BuildOptions{}) + if len(pkg.PackageChecksums) != 1 { + t.Fatalf("spdx dropped a %s checksum: %+v", tc.algorithm, pkg.PackageChecksums) + } + if got := pkg.PackageChecksums[0]; got.Algorithm != tc.wantSPDX || got.Value != value { + t.Fatalf("spdx checksum = %+v, want %s with the asserted value", got, tc.wantSPDX) } + comp := cycloneDXComponentFor(t, g, BuildOptions{}) if comp.Hashes == nil || len(*comp.Hashes) != 1 { - t.Fatalf("cyclonedx dropped a %s hash: %+v", algorithm, comp.Hashes) + t.Fatalf("cyclonedx dropped a %s hash: %+v", tc.algorithm, comp.Hashes) + } + if got := (*comp.Hashes)[0]; got.Algorithm != tc.wantCDX || got.Value != value { + t.Fatalf("cyclonedx hash = %+v, want %s with the asserted value", got, tc.wantCDX) } }) } diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 2b8338c8..921542c9 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -864,7 +864,7 @@ func TestSPDXHomePageSurvivesSPDXRoundTrip(t *testing.T) { // Assert both fields: a length check alone would not notice the // algorithm degrading to a different family on the way through. if got := p.PackageChecksums[0]; got.Algorithm != common.SHA3_256 || got.Value != "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" { - t.Fatalf("checksum = %+v, want SHA3-256/abc123", got) + t.Fatalf("checksum = %+v, want SHA3-256 with the asserted 64-character hex value", got) } return } From 1cb259d91c25d670080ae022326cd9acad815f2f Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 10:45:56 -0700 Subject: [PATCH 10/23] fix(sbom): reject token-shaped revisions, validate BLAKE2b lengths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps, both mine, both the same shape: I added a guard on one path and left its siblings alone. The commit-shaped check only covered URL fragments. A revision also reaches the output verbatim from Metadata["source_revision"], which bundler, pub, and the python detectors populate from a lockfile, and from the rev/tag/branch query keys. Both still used isSafeRevision, whose own comment admitted it accepts access-token shapes. Those paths cannot simply require hex: a tag or branch is legitimate there and uses the same characters a token does. isSafeRevision now rejects known issuer prefixes instead — GitHub, GitLab, npm, PyPI, Slack, Stripe/OpenAI, AWS, Google, Hugging Face, DigitalOcean, Shopify. The doc comment states the limit plainly: this is a narrow check, a bespoke secret format would still pass, and the strong guarantee remains on the fragment path where bare hex is required. Tests cover both directions, including near-misses like "AKIRA" and "release_candidate". BLAKE2b was absent from digestHexSizes, so the ingest length check was skipped for exactly the algorithms the previous commit taught the encoders to emit: "ab" passed as a BLAKE2b-256 digest. Adding the encoder mappings without the paired size entries is the same omission as the BLAKE2b encoder gap two commits ago; the table now carries a note that every accepted algorithm needs an entry. Adding those sizes also exposed a wrong fixture of my own. The BLAKE2b test reused one 64-character value for every algorithm, and for BLAKE2b-384 that string is valid base64 decoding to exactly 48 bytes, so normalizeDigestValue rewrote it as an SRI digest and the assertion compared against a value the encoder never saw. Each case now uses its real hex width. All three fixes verified load-bearing by reverting each in turn. Co-Authored-By: Claude Opus 5 --- internal/sbom/distribution_test.go | 22 ++++++++++------- internal/sbom/locator.go | 38 ++++++++++++++++++++++++++++++ internal/sbom/locator_test.go | 33 ++++++++++++++++++++++++++ internal/sbom/roundtrip_test.go | 6 +++++ internal/sbom/transform.go | 5 ++++ 5 files changed, 95 insertions(+), 9 deletions(-) diff --git a/internal/sbom/distribution_test.go b/internal/sbom/distribution_test.go index f8dae487..85d8335d 100644 --- a/internal/sbom/distribution_test.go +++ b/internal/sbom/distribution_test.go @@ -225,22 +225,26 @@ func TestUnsafeDetectorRevisionIsIgnored(t *testing.T) { // direction — a case mapped to the wrong constant, which would keep the count // at one while relabelling the digest as a different family. func TestBlake2bDigestsSurviveRoundTrip(t *testing.T) { - const value = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" - cases := []struct { algorithm string - wantSPDX common.ChecksumAlgorithm - wantCDX cdx.HashAlgorithm + // hexLen is the digest's real hex width. A wrong-width value is not + // merely unrealistic: normalizeDigestValue would read it as an + // npm-style base64 SRI digest and rewrite it, so the assertion would + // compare against something the encoder never saw. + hexLen int + wantSPDX common.ChecksumAlgorithm + wantCDX cdx.HashAlgorithm }{ - {"blake2b-256", common.BLAKE2b_256, cdx.HashAlgoBlake2b_256}, - {"blake2b-384", common.BLAKE2b_384, cdx.HashAlgoBlake2b_384}, - {"blake2b-512", common.BLAKE2b_512, cdx.HashAlgoBlake2b_512}, - {"sha3-256", common.SHA3_256, cdx.HashAlgoSHA3_256}, - {"sha256", common.SHA256, cdx.HashAlgoSHA256}, + {"blake2b-256", 64, common.BLAKE2b_256, cdx.HashAlgoBlake2b_256}, + {"blake2b-384", 96, common.BLAKE2b_384, cdx.HashAlgoBlake2b_384}, + {"blake2b-512", 128, common.BLAKE2b_512, cdx.HashAlgoBlake2b_512}, + {"sha3-256", 64, common.SHA3_256, cdx.HashAlgoSHA3_256}, + {"sha256", 64, common.SHA256, cdx.HashAlgoSHA256}, } for _, tc := range cases { t.Run(tc.algorithm, func(t *testing.T) { + value := strings.Repeat("a", tc.hexLen) g := sdk.New() node := sdk.NewDependencyWithID("pkg@1.0.0", sdk.Dependency{ Coordinates: sdk.Coordinates{ diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index 956d870b..c1776104 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -293,6 +293,41 @@ func isCommitFragment(fragment string) bool { return true } +// credentialPrefixes are issuer prefixes used by common access-token formats. +// No legitimate git tag, branch, or commit begins with one. +var credentialPrefixes = []string{ + "ghp_", "gho_", "ghu_", "ghs_", "ghr_", "github_pat_", // GitHub + "glpat-", "gldt-", // GitLab + "npm_", // npm + "pypi-", // PyPI + "xoxb-", "xoxp-", "xoxa-", "xoxr-", "xoxs-", // Slack + "sk_live_", "pk_live_", "sk-", // Stripe, OpenAI + "akia", "asia", // AWS access key ids + "aiza", // Google + "hf_", // Hugging Face + "dop_v1_", "doo_v1_", // DigitalOcean + "shpat_", "shpss_", // Shopify +} + +// looksLikeCredential reports whether a value carries a recognizable +// access-token prefix. +// +// A revision reaches an SBOM verbatim after the "@", and unlike a URL fragment +// it cannot simply be required to be hex: tags and branches are legitimate +// here, and they use the same character set a token does. Matching known +// issuer prefixes is therefore the check available, and it is deliberately +// narrow — a bespoke or unrecognized secret format would still pass. The +// stronger guarantee lives on the fragment path, which requires bare hex. +func looksLikeCredential(value string) bool { + lowered := strings.ToLower(strings.TrimSpace(value)) + for _, prefix := range credentialPrefixes { + if strings.HasPrefix(lowered, prefix) { + return true + } + } + return false +} + // isSafeRevision reports whether revision is a plausible git revision that can // be appended to an SPDX version-control locator verbatim. // @@ -304,6 +339,9 @@ func isSafeRevision(revision string) bool { if revision == "" || len(revision) > 256 { return false } + if looksLikeCredential(revision) { + return false + } for _, r := range revision { switch { case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': diff --git a/internal/sbom/locator_test.go b/internal/sbom/locator_test.go index 9ac74362..9360affd 100644 --- a/internal/sbom/locator_test.go +++ b/internal/sbom/locator_test.go @@ -81,6 +81,11 @@ func TestClassifyResolvedURL(t *testing.T) { {"fine-grained pat fragment is not a revision", "https://github.com/org/repo#github_pat_11ABCDE", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, {"non-hex fragment is not a revision", "https://github.com/org/repo#release-candidate", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, {"named query revision still allows tags", "https://github.com/org/repo?tag=v1.2.3", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo@v1.2.3"}, + // A query key names its value a revision, so tags and branches are + // legitimate there — but a recognizable token is not. + {"token in rev query is rejected", "https://github.com/org/repo?rev=ghp_abcd1234", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, + {"token in branch query is rejected", "https://github.com/org/repo?branch=glpat-Abc123", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, + {"underscored branch still allowed", "https://github.com/org/repo?branch=release_candidate", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo@release_candidate"}, {"requested rev used when no commit resolved", "https://github.com/a/b?rev=v1.2.3", sdk.DependencySourceGit, sdk.EcosystemPython, LocatorVCS, "git+https://github.com/a/b@v1.2.3"}, // Degenerate input. @@ -192,6 +197,34 @@ func TestIsPublishableReferenceURL(t *testing.T) { } } +func TestIsSafeRevisionRejectsCredentialShapes(t *testing.T) { + rejected := []string{ + "ghp_abcd1234", "github_pat_11ABCDE_xyz", "gho_abc", "ghs_abc", + "glpat-Abc123", "npm_abcdef", "pypi-AgEIcHlwaS5vcmc", + "xoxb-123-456-abc", "sk-abcdef123456", "sk_live_abc", + "AKIAIOSFODNN7EXAMPLE", "AIzaSyA-abc123", "hf_abcDEF", + "dop_v1_abc", "shpat_abc123", + } + for _, value := range rejected { + if isSafeRevision(value) { + t.Fatalf("isSafeRevision(%q) = true, want it rejected as a credential shape", value) + } + } + + // Real refs must keep working, including ones using the same characters a + // token does. + allowed := []string{ + "main", "v1.2.3", "2.0.0-beta.1", "release_candidate", "feature/foo", + "9f8e7d6c5b4a", "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + "skip-ci", "package_name", "AKIRA", + } + for _, value := range allowed { + if !isSafeRevision(value) { + t.Fatalf("isSafeRevision(%q) = false, want a legitimate ref accepted", value) + } + } +} + func TestNormalizeRepositoryURL(t *testing.T) { cases := []struct{ in, want string }{ {"github.com/kubernetes/kubernetes", "https://github.com/kubernetes/kubernetes"}, diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 921542c9..39b005e9 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -587,6 +587,7 @@ func TestMalformedIngestedDigestIsDropped(t *testing.T) { "purl": "pkg:npm/a@1.0.0", "hashes": [ {"alg": "SHA-256", "content": "not-a-hash"}, + {"alg": "BLAKE2b-256", "content": "ab"}, {"alg": "SHA-256", "content": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"} ] }] @@ -597,6 +598,11 @@ func TestMalformedIngestedDigestIsDropped(t *testing.T) { if strings.Contains(string(out), "not-a-hash") { t.Fatalf("%s republished a malformed digest:\n%s", target, out) } + // Hex but far too short. Every algorithm the encoders accept needs a + // length entry, or this passes validation unchecked. + if strings.Contains(string(out), `"ab"`) { + t.Fatalf("%s republished a short BLAKE2b digest:\n%s", target, out) + } if !strings.Contains(string(out), "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc") { t.Fatalf("%s dropped the valid digest alongside the malformed one:\n%s", target, out) } diff --git a/internal/sbom/transform.go b/internal/sbom/transform.go index 211b42f2..03a6a7f7 100644 --- a/internal/sbom/transform.go +++ b/internal/sbom/transform.go @@ -232,6 +232,11 @@ var digestHexSizes = map[string]int{ "sha3-256": 32, "sha3-384": 48, "sha3-512": 64, + // Every algorithm the encoders accept needs an entry here, or an ingested + // value of the wrong length passes validation unchecked. + "blake2b-256": 32, + "blake2b-384": 48, + "blake2b-512": 64, } func normalizeDigestValue(algorithm, value string) string { From c749a9bb351082fd9c0b0e5ae16081de5e610e34 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 10:57:53 -0700 Subject: [PATCH 11/23] fix(sbom): allow benign queries on asserted downloads, union set fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings, all cases where a guard was applied too bluntly. classifyAssertedDownloadLocation rejected every query, so an SPDX downloadLocation such as "https://repo.example/download?id=123" was re-exported as NOASSERTION. That example is the one my own doc comment used to describe the behavior, which the code did not actually produce — the comment stated the intent rather than the effect. The classifier now takes an explicit flag: a detector-supplied value still loses any query, because nothing asserts it is a download location and omitting is cheap, while a value the source document declared a download location keeps a benign query and is rejected only when a parameter name or value looks like a credential. git:// version-control references were dropped. isPublishableReferenceURL learned that transport last commit, but the narrower VCS path still permitted only HTTP(S), so a repository disappeared on a CycloneDX round trip. SPDX renders it "git+git://host/path", which its grammar allows. CPEs and digests were merged as scalars. External references were unioned last commit but their neighbours were not, so a component described in both metadata.component and the inventory lost whichever identifiers or hashes the second copy carried alone. Multiple distribution references overwrote one another. The neutral model has a single artifact slot, so listing mirrors meant only the last survived; the first classified value now takes the slot and the rest are preserved verbatim. Also re-attaches the classifyResolvedURL doc comment, which an earlier edit stranded above an unrelated const. Verified the query relaxation does not leak into the detector path, and mutation-checked it and the digest union. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 19 ++++++- internal/sbom/locator.go | 97 ++++++++++++++++++++++++++------- internal/sbom/locator_test.go | 38 +++++++++++++ internal/sbom/model.go | 30 ++++++++-- internal/sbom/roundtrip_test.go | 91 +++++++++++++++++++++++++++++++ 5 files changed, 248 insertions(+), 27 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index b183d928..f35b59ad 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -387,7 +387,24 @@ func componentFromCycloneDX(comp cdx.Component) Component { for _, ref := range *comp.ExternalReferences { switch ref.Type { case cdx.ERTypeDistribution: - applyLocatorComment(&component, classifyResolvedURL(ref.URL, "", ""), ref.Comment) + // A component may list several distribution URLs (mirrors). + // The neutral model has one artifact slot, so the first + // classified value takes it and the rest are preserved + // verbatim rather than overwriting the earlier assertion. + if component.ArtifactURL == "" && component.RegistryURL == "" { + applyLocatorComment(&component, classifyResolvedURL(ref.URL, "", ""), ref.Comment) + if component.ArtifactURL != "" || component.RegistryURL != "" { + continue + } + } + if !isPublishableReferenceURL(ref.URL) { + continue + } + component.ExternalRefs = append(component.ExternalRefs, ExternalRef{ + Type: string(ref.Type), + URL: ref.URL, + Comment: ref.Comment, + }) case cdx.ERTypeVCS: // Untrusted input: gate and normalize it exactly like a // detector-supplied value, because it is re-emitted and also diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index c1776104..61fadba5 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -43,14 +43,6 @@ var artifactExtensions = []string{ ".whl", ".gem", ".crate", ".jar", ".nupkg", ".egg", ".conda", } -// classifyResolvedURL decides what raw points at, given the detector's own -// source classification and the package ecosystem. -// -// The function is deliberately conservative: it emits nothing unless the value -// is an unambiguous http(s) URL, and it prefers the weaker LocatorRegistryRoot -// over LocatorArtifact whenever the path shape is not recognizably an archive. -// An SBOM that omits a download location is correct; one that points at the -// wrong place, or at the developer's home directory, is not. // metadataKeySourceRevision is the Dependency.Metadata key several detectors // (ruby, pub, and the python family) use to record the commit a git // dependency resolved to, separately from the repository URL. @@ -79,7 +71,58 @@ func pinLocator(locator Locator, revision string) Locator { return Locator{Kind: LocatorVCS, URL: locator.URL + "@" + revision} } +// credentialQueryKeys are query parameter names that carry a secret. A +// download URL using one of them must not be published. +var credentialQueryKeys = []string{ + "token", "access_token", "auth", "authorization", "apikey", "api_key", + "key", "secret", "password", "passwd", "pwd", "credential", "sig", + "signature", "x-amz-signature", "x-amz-credential", "x-amz-security-token", + "x-goog-signature", "se", "sp", "sv", "sr", // Azure SAS parameters +} + +// hasCredentialQuery reports whether a URL's query carries something that +// looks like a secret, by parameter name or by value shape. +func hasCredentialQuery(parsed *url.URL) bool { + if parsed.RawQuery == "" { + return false + } + values, err := url.ParseQuery(parsed.RawQuery) + if err != nil { + // Unparseable query: cannot be inspected, so assume the worst. + return true + } + for key, vals := range values { + lowered := strings.ToLower(strings.TrimSpace(key)) + for _, candidate := range credentialQueryKeys { + if lowered == candidate { + return true + } + } + for _, value := range vals { + if looksLikeCredential(value) { + return true + } + } + } + return false +} + +// classifyResolvedURL decides what raw points at, given the detector's own +// source classification and the package ecosystem. +// +// The function is deliberately conservative: it emits nothing unless the value +// is an unambiguous http(s) URL, and it prefers the weaker LocatorRegistryRoot +// over LocatorArtifact whenever the path shape is not recognizably an archive. +// An SBOM that omits a download location is correct; one that points at the +// wrong place, or at the developer's home directory, is not. func classifyResolvedURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosystem) Locator { + return classifyURL(raw, source, ecosystem, false) +} + +// classifyURL is the shared classifier. allowBenignQuery relaxes the blanket +// query rejection to a credential-shape check, which is appropriate only when +// the source document itself declared the value a download location. +func classifyURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosystem, allowBenignQuery bool) Locator { raw = strings.TrimSpace(raw) if raw == "" { return Locator{} @@ -151,11 +194,19 @@ func classifyResolvedURL(raw string, source sdk.DependencySource, ecosystem sdk. // Credentials also travel outside userinfo: signed and private-registry // URLs carry them as query parameters or fragments - // ("...?token=", "...?X-Amz-Signature=..."). A benign query - // parameter cannot be told apart from a credential here, so any query - // disqualifies a non-VCS locator. + // ("...?token=", "...?X-Amz-Signature=..."). + // + // For a detector-supplied value there is nothing asserting the URL is a + // download location, so any query disqualifies it — a benign parameter + // cannot be told apart from a credential, and omitting is cheap. When the + // source document itself declared the value a download location, dropping + // every query would discard a real assertion such as + // "https://repo.example/download?id=123", so the check narrows to + // credential-shaped parameters. if parsed.RawQuery != "" { - return Locator{} + if !allowBenignQuery || hasCredentialQuery(parsed) { + return Locator{} + } } // A fragment is rejected on the same grounds, with one exception: Yarn @@ -417,8 +468,12 @@ func classifyIngestedVCS(raw string) string { if err != nil || parsed.User != nil || parsed.Host == "" { return "" } + // git:// is a legitimate transport for a version-control reference, and + // isPublishableReferenceURL already accepts it; rejecting it here would + // drop the repository on a CycloneDX round trip. SPDX renders it as + // "git+git://host/path", which its version-control grammar allows. switch strings.ToLower(parsed.Scheme) { - case "http", "https": + case "http", "https", "git": default: return "" } @@ -429,14 +484,16 @@ func classifyIngestedVCS(raw string) string { // already declared to be a download location, such as SPDX // PackageDownloadLocation. // -// The path-shape heuristic that guards detector-supplied values does not apply -// here: an exact endpoint without a recognizable archive suffix -// ("https://repo.example/download?id=123" reduced to its path) would be demoted -// to a registry root and re-exported as NOASSERTION, discarding an assertion -// the source document actually made. The safety gate still applies in full, so -// local paths and credential-bearing URLs are still dropped. +// Two guards that suit a detector-supplied value are relaxed here, because +// they would discard an assertion the source document actually made: +// the path-shape heuristic, which would demote an exact endpoint with no +// archive suffix to a registry root, and the blanket query rejection, which +// would drop "https://repo.example/download?id=123" entirely. Queries are +// still rejected when a parameter looks like a credential, and the rest of the +// safety gate — scheme, host, userinfo, fragment — applies unchanged, so local +// paths and secrets are still dropped. func classifyAssertedDownloadLocation(raw string) Locator { - locator := classifyResolvedURL(raw, "", "") + locator := classifyURL(raw, "", "", true) if locator.Kind == LocatorRegistryRoot { locator.Kind = LocatorArtifact } diff --git a/internal/sbom/locator_test.go b/internal/sbom/locator_test.go index 9360affd..6772d121 100644 --- a/internal/sbom/locator_test.go +++ b/internal/sbom/locator_test.go @@ -197,6 +197,44 @@ func TestIsPublishableReferenceURL(t *testing.T) { } } +// TestClassifyAssertedDownloadLocation covers the relaxed path used when a +// source document itself declared a value to be a download location. A benign +// query survives; a credential-shaped one still does not. +func TestClassifyAssertedDownloadLocation(t *testing.T) { + cases := []struct { + in string + wantKind LocatorKind + wantURL string + }{ + // An exact endpoint with no archive suffix must not be demoted. + {"https://repo.example/download?id=123", LocatorArtifact, "https://repo.example/download?id=123"}, + {"https://repo.example/download/left-pad", LocatorArtifact, "https://repo.example/download/left-pad"}, + {"https://reg.example/a/b-1.0.tgz", LocatorArtifact, "https://reg.example/a/b-1.0.tgz"}, + // Credentials stay rejected, by parameter name and by value shape. + {"https://repo.example/download?token=s3cret", LocatorNone, ""}, + {"https://repo.example/download?X-Amz-Signature=deadbeef", LocatorNone, ""}, + {"https://repo.example/download?id=ghp_abcd1234", LocatorNone, ""}, + {"https://tok:s3cret@repo.example/download?id=123", LocatorNone, ""}, + {"file:///Users/victim/pkg.tgz", LocatorNone, ""}, + {"/Users/victim/pkg.tgz", LocatorNone, ""}, + {"NOASSERTION", LocatorNone, ""}, + {"", LocatorNone, ""}, + } + for _, tc := range cases { + got := classifyAssertedDownloadLocation(tc.in) + if got.Kind != tc.wantKind || got.URL != tc.wantURL { + t.Fatalf("classifyAssertedDownloadLocation(%q) = %+v, want kind %v url %q", + tc.in, got, tc.wantKind, tc.wantURL) + } + } + + // The relaxation must not leak into the detector path, which has no such + // assertion behind it. + if got := classifyResolvedURL("https://repo.example/download?id=123", "", ""); got.Kind != LocatorNone { + t.Fatalf("detector path accepted a query: %+v", got) + } +} + func TestIsSafeRevisionRejectsCredentialShapes(t *testing.T) { rejected := []string{ "ghp_abcd1234", "github_pat_11ABCDE_xyz", "gho_abc", "ghs_abc", diff --git a/internal/sbom/model.go b/internal/sbom/model.go index 3857f3bc..fe73c1ff 100644 --- a/internal/sbom/model.go +++ b/internal/sbom/model.go @@ -204,12 +204,11 @@ func mergeComponentAssertions(dst *Component, src Component) { *field.target = field.value } } - if len(dst.CPEs) == 0 { - dst.CPEs = src.CPEs - } - if len(dst.Digests) == 0 { - dst.Digests = src.Digests - } + // CPEs and digests are set-valued too: each copy of a component may carry + // an identifier or a hash the other lacks, so a fill-gaps copy would + // discard the second set whenever the first had any. + dst.CPEs = unionStrings(dst.CPEs, src.CPEs) + dst.Digests = unionComponentDigests(dst.Digests, src.Digests) if len(dst.Licenses) == 0 { dst.Licenses = src.Licenses } @@ -219,6 +218,25 @@ func mergeComponentAssertions(dst *Component, src Component) { dst.ExternalRefs = unionExternalRefs(dst.ExternalRefs, src.ExternalRefs) } +// unionComponentDigests appends digests from extra that base does not carry. +func unionComponentDigests(base, extra []Digest) []Digest { + if len(extra) == 0 { + return base + } + seen := make(map[Digest]struct{}, len(base)) + for _, digest := range base { + seen[digest] = struct{}{} + } + for _, digest := range extra { + if _, ok := seen[digest]; ok { + continue + } + seen[digest] = struct{}{} + base = append(base, digest) + } + return base +} + // unionExternalRefs appends references from extra that base does not already // carry, keyed by type and URL. func unionExternalRefs(base, extra []ExternalRef) []ExternalRef { diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 39b005e9..803b76e6 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -827,6 +827,97 @@ func TestPrimaryComponentExternalRefsAreUnioned(t *testing.T) { } } +// TestGitTransportVCSReferenceSurvives covers a git:// repository reference. +// isPublishableReferenceURL already accepts that transport, so the narrower +// VCS path must not reject it and drop the repository. +func TestGitTransportVCSReferenceSurvives(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", + "type": "library", + "name": "a", + "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "externalReferences": [{"type": "vcs", "url": "git://github.com/org/repo"}] + }] + }` + + for _, target := range []Target{TargetSPDX23JSON, TargetCycloneDX17JSON} { + out := ingestAndReexport(t, []byte(in), target) + if !strings.Contains(string(out), "github.com/org/repo") { + t.Fatalf("%s dropped a git:// repository reference:\n%s", target, out) + } + } +} + +// TestMultipleDistributionReferencesArePreserved covers a component listing +// several download mirrors. The neutral model has one artifact slot, so the +// extras have to be kept rather than overwriting each other. +func TestMultipleDistributionReferencesArePreserved(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", + "type": "library", + "name": "a", + "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "externalReferences": [ + {"type": "distribution", "url": "https://primary.example/a-1.0.0.tgz"}, + {"type": "distribution", "url": "https://mirror.example/a-1.0.0.tgz"} + ] + }] + }` + + out := ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON) + for _, want := range []string{"https://primary.example/a-1.0.0.tgz", "https://mirror.example/a-1.0.0.tgz"} { + if !strings.Contains(string(out), want) { + t.Fatalf("distribution mirror %q was overwritten:\n%s", want, out) + } + } +} + +// TestPrimaryComponentDigestsAreUnioned covers set-valued integrity data on a +// component described in both metadata.component and the inventory. +func TestPrimaryComponentDigestsAreUnioned(t *testing.T) { + // Correct hex widths for each algorithm: a wrong-width value would be + // dropped by digest validation, and the test would then pass without ever + // exercising the union. + inventoryHash := strings.Repeat("a", 64) // SHA-256 + metadataHash := strings.Repeat("b", 128) // SHA-512 + in := `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "metadata": {"component": { + "bom-ref": "root", "type": "application", "name": "app", "version": "1.0.0", + "purl": "pkg:npm/app@1.0.0", + "hashes": [{"alg": "SHA-512", "content": "` + metadataHash + `"}] + }}, + "components": [ + {"bom-ref": "root", "type": "application", "name": "app", "version": "1.0.0", "purl": "pkg:npm/app@1.0.0", + "hashes": [{"alg": "SHA-256", "content": "` + inventoryHash + `"}]}, + {"bom-ref": "pkg:npm/dep@1.0.0", "type": "library", "name": "dep", "version": "1.0.0", "purl": "pkg:npm/dep@1.0.0"} + ], + "dependencies": [{"ref": "root", "dependsOn": ["pkg:npm/dep@1.0.0"]}] + }` + + out := ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON) + for name, want := range map[string]string{ + "inventory": inventoryHash, + "metadata": metadataHash, + } { + if !strings.Contains(string(out), want) { + t.Fatalf("%s hash was discarded by the merge:\n%s", name, out) + } + } +} + // TestSPDXHomePageSurvivesSPDXRoundTrip covers a field SPDX represents // exactly, which an earlier revision decoded but never re-emitted. func TestSPDXHomePageSurvivesSPDXRoundTrip(t *testing.T) { From 03d0f44812b74be643e9bbbf5ba448f08f230069 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 11:09:25 -0700 Subject: [PATCH 12/23] fix(sbom): keep VCS syntax in source info, validate CPEs, map BLAKE3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings, three of them the same failure repeated: a value is accepted on the way in and then quietly mangled or dropped on the way out. spdxSourceInfo stripped "git+" from a pinned locator, so "git+https://github.com/org/repo@deadbeef" became "https://github.com/org/repo@deadbeef" — an address where the revision is now part of the path, pointing at a repository that does not exist, and one that re-ingests cleanly as though it did. The version-control form is kept intact, and parseSPDXSourceInfo now recognizes it and restores it to VCSURL rather than reading it as a plain repository URL. CPE locators were preserved unchecked, so a document labelling "not-a-cpe" as cpe23Type had that republished as an SPDX security reference and as CycloneDX's cpe field: a false package-identity assertion, and non-conformant output. Both bindings are now validated — the 2.3 formatted string by component count and part, the 2.2 URI by component bound and part — with escaped colons treated as literals. Applied on both ingest paths, not just SPDX. BLAKE3 was accepted by parseSPDXChecksums and normalized, but neither encoder knew it, so an SPDX-to-SPDX round trip dropped it. Both formats define it; the mappings and the length entry were simply missing. That is the third algorithm to hit this, so the omission is systematic rather than incidental. Enrichment overwrote ingested CPEs and digests. Those are set-valued, so a matcher-supplied identifier or hash deleted the source document's own assertions, contradicting the documented ingest-wins ordering. Both are unioned now, matching what the two merge paths already do. The enrichment union and the CPE validation were mutation-checked. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 4 +- internal/sbom/distribution_test.go | 67 +++++++++++++++++ internal/sbom/locator.go | 26 +++++++ internal/sbom/roundtrip_test.go | 112 ++++++++++++++++++++++++++++- internal/sbom/spdx23.go | 91 +++++++++++++++++++++-- internal/sbom/transform.go | 11 ++- 6 files changed, 297 insertions(+), 14 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index f35b59ad..4f89ad91 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -361,7 +361,7 @@ func componentFromCycloneDX(comp cdx.Component) Component { Description: comp.Description, Originator: comp.Publisher, } - if comp.CPE != "" { + if isValidCPE(comp.CPE) { component.CPEs = []string{comp.CPE} } if comp.Supplier != nil && comp.Supplier.Name != "" { @@ -665,6 +665,8 @@ func cycloneDXHashAlgorithm(algorithm string) cdx.HashAlgorithm { return cdx.HashAlgoSHA384 case "sha512", "sha-512": return cdx.HashAlgoSHA512 + case "blake3": + return cdx.HashAlgoBlake3 case "blake2b-256": return cdx.HashAlgoBlake2b_256 case "blake2b-384": diff --git a/internal/sbom/distribution_test.go b/internal/sbom/distribution_test.go index 85d8335d..bef31ee6 100644 --- a/internal/sbom/distribution_test.go +++ b/internal/sbom/distribution_test.go @@ -365,3 +365,70 @@ func TestDetectorVCSWinsOverScorecardRepository(t *testing.T) { t.Fatalf("expected exactly one vcs reference, got %d", vcsRefs) } } + +// TestEnrichmentDoesNotClobberIngestedSets covers `scan --sbom --enrich`. +// CPEs and digests are set-valued, so a matcher-supplied value must be added +// to the source document's assertions rather than replacing them — the +// ingest-wins / enrichment-fills-gaps ordering applies to sets too. +func TestEnrichmentDoesNotClobberIngestedSets(t *testing.T) { + const purl = "pkg:npm/a@1.0.0" + const ingestedCPE = "cpe:2.3:a:ingested:a:1.0.0:*:*:*:*:*:*:*" + const matcherCPE = "cpe:2.3:a:matcher:a:1.0.0:*:*:*:*:*:*:*" + ingestedHash := strings.Repeat("a", 64) + matcherHash := strings.Repeat("b", 64) + + g := sdk.New() + node := sdk.NewDependencyWithID(purl, sdk.Dependency{ + Coordinates: sdk.Coordinates{Name: "a", Version: "1.0.0", PURL: purl, Ecosystem: sdk.EcosystemNPM}, + CPEs: []string{ingestedCPE}, + Digests: []sdk.Digest{{Algorithm: sdk.DigestAlgorithmSHA256, Value: ingestedHash}}, + }) + if err := g.AddNode(node); err != nil { + t.Fatalf("add node: %v", err) + } + + registry := sdk.NewPackageRegistry() + pkg := registry.Ensure(purl) + pkg.Name, pkg.Version = "a", "1.0.0" + pkg.CPEs = []string{matcherCPE} + pkg.Digests = []sdk.Digest{{Algorithm: sdk.DigestAlgorithmSHA256, Value: matcherHash}} + + out, err := MarshalDepGraphJSON(g, TargetSPDX23JSON, BuildOptions{Registry: registry}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + for name, want := range map[string]string{ + "ingested CPE": ingestedCPE, + "matcher CPE": matcherCPE, + "ingested hash": ingestedHash, + "matcher hash": matcherHash, + } { + if !strings.Contains(string(out), want) { + t.Fatalf("%s was discarded by enrichment:\n%s", name, out) + } + } +} + +// TestBlake3DigestSurvivesRoundTrip pairs with the BLAKE3 encoder mappings: +// parseSPDXChecksums accepts the algorithm, so both encoders must know it or +// the checksum is silently dropped after the graph hop. +func TestBlake3DigestSurvivesRoundTrip(t *testing.T) { + value := strings.Repeat("d", 64) + g := sdk.New() + node := sdk.NewDependencyWithID("pkg@1.0.0", sdk.Dependency{ + Coordinates: sdk.Coordinates{Name: "pkg", Version: "1.0.0", PURL: "pkg:npm/pkg@1.0.0", Ecosystem: sdk.EcosystemNPM}, + Digests: []sdk.Digest{{Algorithm: "blake3", Value: value}}, + }) + if err := g.AddNode(node); err != nil { + t.Fatalf("add node: %v", err) + } + + pkg := spdxPackageFor(t, g, BuildOptions{}) + if len(pkg.PackageChecksums) != 1 || pkg.PackageChecksums[0].Algorithm != common.BLAKE3 { + t.Fatalf("spdx blake3 checksum = %+v, want it preserved", pkg.PackageChecksums) + } + comp := cycloneDXComponentFor(t, g, BuildOptions{}) + if comp.Hashes == nil || len(*comp.Hashes) != 1 || (*comp.Hashes)[0].Algorithm != cdx.HashAlgoBlake3 { + t.Fatalf("cyclonedx blake3 hash = %+v, want it preserved", comp.Hashes) + } +} diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index 61fadba5..b7f89ee3 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -500,6 +500,32 @@ func classifyAssertedDownloadLocation(raw string) Locator { return locator } +// validatedVCSLocator checks an already-rendered "git+://…" +// locator and returns it unchanged when it is safe to republish, or "". +// +// The locator is kept verbatim rather than reparsed and rebuilt: any pinned +// revision after "@" is part of the value's meaning, and re-deriving it risks +// changing what it says. +func validatedVCSLocator(locator string) string { + locator = strings.TrimSpace(locator) + if !strings.HasPrefix(locator, "git+") { + return "" + } + parsed, err := url.Parse(strings.TrimPrefix(locator, "git+")) + if err != nil || parsed.User != nil || parsed.Host == "" { + return "" + } + switch strings.ToLower(parsed.Scheme) { + case "http", "https", "git": + default: + return "" + } + if hasCredentialQuery(parsed) || parsed.Fragment != "" { + return "" + } + return locator +} + // isPublishableReferenceURL reports whether a URL carried on an ingested // external reference is safe to re-emit. // diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 803b76e6..62d38d95 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -6,6 +6,7 @@ import ( "testing" cdx "github.com/CycloneDX/cyclonedx-go" + "github.com/bomly-dev/bomly-sdk" "github.com/spdx/tools-golang/spdx/v2/common" v23 "github.com/spdx/tools-golang/spdx/v2/v2_3" ) @@ -279,7 +280,7 @@ func TestSPDXSourceInfoRepositoryIsGated(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - if got := parseSPDXSourceInfo(tc.sourceInfo); got != tc.wantRepo { + if got, _ := parseSPDXSourceInfo(tc.sourceInfo); got != tc.wantRepo { t.Fatalf("parseSPDXSourceInfo(%q) = %q, want %q", tc.sourceInfo, got, tc.wantRepo) } }) @@ -1000,3 +1001,112 @@ func TestSPDXNoAssertionSupplierDecodesToAbsent(t *testing.T) { } } } + +// TestIsValidCPE guards the package-identity assertion. An ingested document +// can label any string cpe23Type, and an unchecked value is republished as an +// SPDX security reference and CycloneDX's `cpe` field. +func TestIsValidCPE(t *testing.T) { + valid := []string{ + "cpe:2.3:a:example:left-pad:1.3.0:*:*:*:*:*:*:*", + "cpe:2.3:o:vendor:os:1.0:-:*:*:*:*:*:*", + `cpe:2.3:a:ven\:dor:prod:1.0:*:*:*:*:*:*:*`, + "cpe:/a:vendor:product:1.0", + "cpe:/o:vendor:os", + } + for _, value := range valid { + if !isValidCPE(value) { + t.Fatalf("isValidCPE(%q) = false, want a well-formed CPE accepted", value) + } + } + + invalid := []string{ + "not-a-cpe", "", " ", + "cpe:2.3:a:example:left-pad", // too few components + "cpe:2.3:a:example:left-pad:1.3.0:*:*:*:*:*:*:*:extra", // too many + "cpe:2.3:z:vendor:product:1.0:*:*:*:*:*:*:*", // bad part + "cpe:/z:vendor:product", // bad part + "cpe:/a:b:c:d:e:f:g:h", // too many + "https://example.com", + } + for _, value := range invalid { + if isValidCPE(value) { + t.Fatalf("isValidCPE(%q) = true, want it rejected", value) + } + } +} + +// TestMalformedCPEIsNotRepublished is the end-to-end counterpart. +func TestMalformedCPEIsNotRepublished(t *testing.T) { + const in = `{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [{ + "name": "a", "SPDXID": "SPDXRef-Package-a", "versionInfo": "1.0.0", + "downloadLocation": "NOASSERTION", "filesAnalyzed": false, + "externalRefs": [ + {"referenceCategory": "SECURITY", "referenceType": "cpe23Type", "referenceLocator": "not-a-cpe"}, + {"referenceCategory": "SECURITY", "referenceType": "cpe23Type", "referenceLocator": "cpe:2.3:a:v:p:1.0:*:*:*:*:*:*:*"} + ] + }] + }` + + for _, target := range []Target{TargetSPDX23JSON, TargetCycloneDX17JSON} { + out := ingestAndReexport(t, []byte(in), target) + if strings.Contains(string(out), "not-a-cpe") { + t.Fatalf("%s republished a malformed CPE:\n%s", target, out) + } + if !strings.Contains(string(out), "cpe:2.3:a:v:p:1.0") { + t.Fatalf("%s dropped the valid CPE alongside it:\n%s", target, out) + } + } +} + +// TestPinnedVCSSourceInfoKeepsItsSyntax covers a component with both an +// artifact and a pinned repository. Stripping "git+" would turn the revision +// into part of the URL path, yielding an address that does not exist and that +// re-ingests cleanly as if it did. +func TestPinnedVCSSourceInfoKeepsItsSyntax(t *testing.T) { + doc := &Document{ + Components: []Component{{ + ID: "a", Name: "a", Version: "1.0.0", PURL: "pkg:npm/a@1.0.0", + ArtifactURL: "https://reg.example/a-1.0.0.tgz", + VCSURL: "git+https://github.com/org/repo@deadbeef", + }}, + } + out, err := MarshalDepGraphJSON(mustGraphFromDoc(t, doc), TargetSPDX23JSON, BuildOptions{}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var spdxDoc v23.Document + if err := json.Unmarshal(out, &spdxDoc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, p := range spdxDoc.Packages { + if p == nil || p.PackageName != "a" { + continue + } + if !strings.Contains(p.PackageSourceInfo, "git+https://github.com/org/repo@deadbeef") { + t.Fatalf("sourceInfo = %q, want the version-control form intact", p.PackageSourceInfo) + } + // And it must survive a re-ingest rather than decoding as a bare URL. + if _, vcs := parseSPDXSourceInfo(p.PackageSourceInfo); vcs != "git+https://github.com/org/repo@deadbeef" { + t.Fatalf("re-ingested vcs = %q, want the pinned locator", vcs) + } + return + } + t.Fatal("component missing from output") +} + +// mustGraphFromDoc converts a neutral document to a graph for export tests. +func mustGraphFromDoc(t *testing.T, doc *Document) *sdk.Graph { + t.Helper() + graph, err := ToGraph(doc) + if err != nil { + t.Fatalf("to graph: %v", err) + } + return graph +} diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 12ad59b9..ec2b6176 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -178,10 +178,10 @@ func (spdx23Codec) decodeJSON(data []byte) (*Document, error) { Licenses: parseSPDXLicenses(p.PackageLicenseConcluded, p.PackageLicenseDeclared), Description: parseSPDXEntity(p.PackageDescription), Summary: parseSPDXEntity(p.PackageSummary), - Repository: parseSPDXSourceInfo(p.PackageSourceInfo), CPEs: parseSPDXCPEs(p.PackageExternalReferences), Digests: parseSPDXChecksums(p.PackageChecksums), } + component.Repository, component.VCSURL = parseSPDXSourceInfo(p.PackageSourceInfo) if p.PackageSupplier != nil { component.Supplier = parseSPDXEntity(p.PackageSupplier.Supplier) component.SupplierType = p.PackageSupplier.SupplierType @@ -409,6 +409,8 @@ func spdxChecksumAlgorithm(algorithm string) common.ChecksumAlgorithm { return common.SHA384 case "sha512", "sha-512": return common.SHA512 + case "blake3": + return common.BLAKE3 case "blake2b-256": return common.BLAKE2b_256 case "blake2b-384": @@ -494,12 +496,19 @@ func parseSPDXEntity(value string) string { // same gate as every other ingested URL. A source document asserting // "Source repository: https://user:secret@github.com/org/repo", or a local // file:// path, yields nothing. -func parseSPDXSourceInfo(value string) string { +func parseSPDXSourceInfo(value string) (repository, vcs string) { value = strings.TrimSpace(value) if !strings.HasPrefix(value, spdxSourceInfoPrefix) { - return "" + return "", "" + } + locator := strings.TrimSpace(strings.TrimPrefix(value, spdxSourceInfoPrefix)) + if strings.HasPrefix(locator, "git+") { + // spdxSourceInfo writes the version-control form when the artifact + // owns downloadLocation, so it has to be recognized here or the + // repository is lost on the way back. + return "", validatedVCSLocator(locator) } - return normalizeRepositoryURL(strings.TrimPrefix(value, spdxSourceInfoPrefix)) + return normalizeRepositoryURL(locator), "" } // spdxSourceInfoPrefix marks a source repository inside the free-text @@ -538,7 +547,11 @@ func spdxSourceInfo(component Component) string { } repo := strings.TrimSpace(component.Repository) if repo == "" && component.ArtifactURL != "" { - repo = strings.TrimPrefix(component.VCSURL, "git+") + // Keep the version-control form intact. Stripping "git+" from + // "git+https://host/org/repo@deadbeef" turns the revision into part + // of the URL path, producing a repository address that does not + // exist — and one that re-ingests cleanly as if it did. + repo = component.VCSURL } if repo == "" { return "" @@ -633,6 +646,72 @@ func parseSPDXLicenses(values ...string) []License { return nil } +// isValidCPE reports whether a locator is a well-formed CPE in either the 2.3 +// formatted-string or the 2.2 URI binding. +// +// An ingested document can label any string cpe23Type. Preserving one +// unchecked republishes a false package-identity assertion — it becomes an +// SPDX security reference and CycloneDX's `cpe` field — and can make the +// output non-conformant, so the shape is verified before it is stored. +func isValidCPE(value string) bool { + value = strings.TrimSpace(value) + switch { + case strings.HasPrefix(value, "cpe:2.3:"): + // "cpe:2.3:" plus 11 colon-separated components: part, vendor, + // product, version, update, edition, language, sw_edition, + // target_sw, target_hw, other. Escaped colons ("\:") are literals. + parts := splitUnescaped(value, ':') + if len(parts) != 13 { + return false + } + return isCPEPart(parts[2]) + case strings.HasPrefix(value, "cpe:/"): + // URI binding: "cpe:/" plus up to 7 components, the first the part. + parts := splitUnescaped(strings.TrimPrefix(value, "cpe:/"), ':') + if len(parts) == 0 || len(parts) > 7 { + return false + } + return isCPEPart(parts[0]) + default: + return false + } +} + +// isCPEPart reports whether a CPE part component is one of the defined values. +func isCPEPart(part string) bool { + switch part { + case "a", "o", "h", "*", "-", "": + return true + default: + return false + } +} + +// splitUnescaped splits on sep, treating a backslash-escaped separator as a +// literal character rather than a delimiter. +func splitUnescaped(value string, sep rune) []string { + var parts []string + var current strings.Builder + escaped := false + for _, r := range value { + switch { + case escaped: + current.WriteRune(r) + escaped = false + case r == '\\': + current.WriteRune(r) + escaped = true + case r == sep: + parts = append(parts, current.String()) + current.Reset() + default: + current.WriteRune(r) + } + } + parts = append(parts, current.String()) + return parts +} + // spdxCPERefType labels a CPE locator with the reference type matching its own // syntax. // @@ -659,7 +738,7 @@ func parseSPDXCPEs(refs []*v23.PackageExternalReference) []string { // but an ingested third-party document may use either. switch strings.ToLower(strings.TrimSpace(ref.RefType)) { case "cpe23type", "cpe22type": - if locator := strings.TrimSpace(ref.Locator); locator != "" { + if locator := strings.TrimSpace(ref.Locator); isValidCPE(locator) { cpes = append(cpes, locator) } } diff --git a/internal/sbom/transform.go b/internal/sbom/transform.go index 03a6a7f7..d3e5729a 100644 --- a/internal/sbom/transform.go +++ b/internal/sbom/transform.go @@ -234,6 +234,7 @@ var digestHexSizes = map[string]int{ "sha3-512": 64, // Every algorithm the encoders accept needs an entry here, or an ingested // value of the wrong length passes validation unchecked. + "blake3": 32, "blake2b-256": 32, "blake2b-384": 48, "blake2b-512": 64, @@ -312,12 +313,10 @@ func enrichComponentFromRegistry(component *Component, registry *sdk.PackageRegi if len(pkg.Licenses) > 0 { component.Licenses = componentLicenses(pkg.Licenses) } - if len(pkg.CPEs) > 0 { - component.CPEs = append([]string(nil), pkg.CPEs...) - } - if digests := componentDigests(pkg.Digests); len(digests) > 0 { - component.Digests = digests - } + // Union rather than replace: an ingested document's identifiers are its + // own assertions, and enrichment fills gaps rather than overwriting. + component.CPEs = unionStrings(component.CPEs, pkg.CPEs) + component.Digests = unionComponentDigests(component.Digests, componentDigests(pkg.Digests)) if len(pkg.Vulnerabilities) > 0 { component.Vulnerabilities = vulnerabilitiesFromPackage(pkg.EcosystemName(), pkg.Vulnerabilities) } From c534b7db50199f692023bcac57941d279d53684a Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 11:35:05 -0700 Subject: [PATCH 13/23] fix(sbom): sweep the set-valued and algorithm-mapping classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review surfaced five more instances of two bug classes I had been fixing one at a time. Rather than patch these individually again, this closes both classes and adds invariants so the next instance fails a test instead of a review. Set-valued fields treated as scalars. Audited every Component field against each merge path. Licenses and Vulnerabilities were still fill-gaps in mergeComponentAssertions, SupplierURL was a scalar where CycloneDX types an array, and multiple vcs references overwrote one another the way distribution mirrors used to. SupplierURL is now SupplierURLs, every set unions, and one test asserts licenses, supplier URLs, and external references all survive a duplicate-component merge. Algorithms accepted on ingest but unknown to an encoder. The audit found a fourth and fifth case beyond the three fixed one-by-one: SHA-224 is SPDX-only, and CycloneDX-only Streebog-256/512 were unmapped and unvalidated. Streebog is now wired; SHA-224 is a genuine format difference, so the invariant is that a validated algorithm reaches at least one encoder rather than both. Two table-driven tests now walk digestHexSizes and the encoder sets in each direction, so an algorithm added to one side without the other fails immediately. Also from review: - An ingested CycloneDX distribution reference is a source-declared download location, so it now uses the credential-aware classifier instead of the detector-oriented one, which rejects every query. A benign "?id=123" survives; "?token=" still does not. - SPDX NONE collapsed to NOASSERTION. Those are different assertions — not downloadable versus no claim made — so the marker is carried and re-emitted. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 29 +++++-- internal/sbom/distribution_test.go | 49 +++++++++++ internal/sbom/graph.go | 35 ++++++++ internal/sbom/ingest_metadata.go | 33 ++++++- internal/sbom/model.go | 38 ++++++++- internal/sbom/roundtrip_test.go | 133 +++++++++++++++++++++++++++++ internal/sbom/spdx23.go | 9 ++ internal/sbom/transform.go | 14 ++- 8 files changed, 322 insertions(+), 18 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 4f89ad91..323d03aa 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -53,8 +53,8 @@ func (c cycloneDXCodec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, e // compliance-relevant assertion, so that case is omitted instead. if comp.Supplier != "" && !strings.EqualFold(comp.SupplierType, "Person") { supplier := &cdx.OrganizationalEntity{Name: comp.Supplier} - if comp.SupplierURL != "" { - urls := []string{comp.SupplierURL} + if len(comp.SupplierURLs) > 0 { + urls := append([]string(nil), comp.SupplierURLs...) supplier.URL = &urls } component.Supplier = supplier @@ -370,8 +370,7 @@ func componentFromCycloneDX(comp cdx.Component) Component { if comp.Supplier.URL != nil { for _, u := range *comp.Supplier.URL { if isPublishableReferenceURL(u) { - component.SupplierURL = u - break + component.SupplierURLs = append(component.SupplierURLs, u) } } } @@ -392,7 +391,7 @@ func componentFromCycloneDX(comp cdx.Component) Component { // classified value takes it and the rest are preserved // verbatim rather than overwriting the earlier assertion. if component.ArtifactURL == "" && component.RegistryURL == "" { - applyLocatorComment(&component, classifyResolvedURL(ref.URL, "", ""), ref.Comment) + applyLocatorComment(&component, classifyAssertedDownloadLocation(ref.URL), ref.Comment) if component.ArtifactURL != "" || component.RegistryURL != "" { continue } @@ -408,10 +407,22 @@ func componentFromCycloneDX(comp cdx.Component) Component { case cdx.ERTypeVCS: // Untrusted input: gate and normalize it exactly like a // detector-supplied value, because it is re-emitted and also - // becomes the SPDX download location. - if vcs := classifyIngestedVCS(ref.URL); vcs != "" { + // becomes the SPDX download location. CycloneDX types this as + // an array, so additional repositories are kept rather than + // overwriting the first. + vcs := classifyIngestedVCS(ref.URL) + if vcs == "" { + continue + } + if component.VCSURL == "" { component.VCSURL, component.VCSComment = vcs, ref.Comment + continue } + component.ExternalRefs = append(component.ExternalRefs, ExternalRef{ + Type: string(ref.Type), + URL: vcs, + Comment: ref.Comment, + }) default: if !isPublishableReferenceURL(ref.URL) { continue @@ -665,6 +676,10 @@ func cycloneDXHashAlgorithm(algorithm string) cdx.HashAlgorithm { return cdx.HashAlgoSHA384 case "sha512", "sha-512": return cdx.HashAlgoSHA512 + case "streebog-256": + return cdx.HashAlgoStreebog256 + case "streebog-512": + return cdx.HashAlgoStreebog512 case "blake3": return cdx.HashAlgoBlake3 case "blake2b-256": diff --git a/internal/sbom/distribution_test.go b/internal/sbom/distribution_test.go index bef31ee6..a508d975 100644 --- a/internal/sbom/distribution_test.go +++ b/internal/sbom/distribution_test.go @@ -432,3 +432,52 @@ func TestBlake3DigestSurvivesRoundTrip(t *testing.T) { t.Fatalf("cyclonedx blake3 hash = %+v, want it preserved", comp.Hashes) } } + +// TestEveryAcceptedDigestAlgorithmRoundTrips closes the class of bug that +// produced three separate one-line fixes (BLAKE2b, BLAKE3, and the SHA-3 +// hyphen): an algorithm the decoder normalizes but an encoder does not know is +// silently dropped, and an algorithm without a length entry skips validation. +// +// Rather than another per-algorithm test, this asserts the invariant directly +// over the whole table, so a future addition cannot land half-wired. +func TestEveryAcceptedDigestAlgorithmRoundTrips(t *testing.T) { + for algorithm, size := range digestHexSizes { + canonical := normalizeDigestAlgorithm(algorithm) + t.Run(algorithm, func(t *testing.T) { + // The formats do not agree on their algorithm sets — SHA-224 is + // SPDX-only, Streebog is CycloneDX-only — so the invariant is + // that a validated algorithm reaches at least one encoder. An + // entry reaching neither is dead weight that silently drops + // every value it accepts. + if spdxChecksumAlgorithm(canonical) == "" && cycloneDXHashAlgorithm(canonical) == "" { + t.Fatalf("%q has a length entry but no encoder mapping in either format", algorithm) + } + + // A value of the declared width must survive; a short one must not. + value := strings.Repeat("a", size*2) + if _, ok := ingestedDigest(algorithm, value); !ok { + t.Fatalf("a correctly sized %q digest was rejected", algorithm) + } + if _, ok := ingestedDigest(algorithm, "ab"); ok { + t.Fatalf("a short %q digest was accepted", algorithm) + } + }) + } +} + +// TestNormalizedAlgorithmsHaveLengthEntries is the other direction: an +// algorithm an encoder can emit must also be length-validated on ingest, or a +// malformed value reaches the output unchecked. +func TestNormalizedAlgorithmsHaveLengthEntries(t *testing.T) { + for _, algorithm := range []string{ + "md5", "sha1", "sha256", "sha384", "sha512", + "sha3-256", "sha3-384", "sha3-512", + "blake2b-256", "blake2b-384", "blake2b-512", "blake3", + "streebog-256", "streebog-512", + } { + canonical := normalizeDigestAlgorithm(algorithm) + if _, ok := digestHexSizes[canonical]; !ok { + t.Fatalf("%q is emitted by the encoders but has no length entry, so ingest cannot validate it", algorithm) + } + } +} diff --git a/internal/sbom/graph.go b/internal/sbom/graph.go index f5baaf4d..2dd63ed0 100644 --- a/internal/sbom/graph.go +++ b/internal/sbom/graph.go @@ -149,6 +149,10 @@ func mergeIngestedNode(existing, incoming *sdk.Dependency) { // External references are a set carried under one key, so a key-level // fill-gaps merge would drop the duplicate's whole list whenever the // first component had any. Union them instead. + if key == metadataKeySupplierURLs { + existing.Metadata[key] = unionAnyStrings(existing.Metadata[key], value) + continue + } if key == metadataKeyExternalRefs { existing.Metadata[key] = unionExternalRefValues(existing.Metadata[key], value) continue @@ -159,6 +163,37 @@ func mergeIngestedNode(existing, incoming *sdk.Dependency) { } } +// unionAnyStrings merges two serialized string lists, preserving order and +// dropping duplicates. +func unionAnyStrings(base, extra any) any { + baseList, _ := base.([]any) + extraList, _ := extra.([]any) + if len(extraList) == 0 { + return base + } + if len(baseList) == 0 { + return extra + } + seen := make(map[string]struct{}, len(baseList)) + for _, entry := range baseList { + if value, ok := entry.(string); ok { + seen[value] = struct{}{} + } + } + for _, entry := range extraList { + value, ok := entry.(string) + if !ok { + continue + } + if _, present := seen[value]; present { + continue + } + seen[value] = struct{}{} + baseList = append(baseList, entry) + } + return baseList +} + // unionExternalRefValues merges two serialized external-reference lists, // keyed by type and URL. Entries that are not the expected shape are kept // as-is on the base side and skipped on the incoming side. diff --git a/internal/sbom/ingest_metadata.go b/internal/sbom/ingest_metadata.go index c8cf6e37..fc811827 100644 --- a/internal/sbom/ingest_metadata.go +++ b/internal/sbom/ingest_metadata.go @@ -15,7 +15,8 @@ import ( const ( metadataKeySupplier = "bomly.sbom.supplier" metadataKeySupplierType = "bomly.sbom.supplier_type" - metadataKeySupplierURL = "bomly.sbom.supplier_url" + metadataKeySupplierURLs = "bomly.sbom.supplier_urls" + metadataKeyNoDownload = "bomly.sbom.no_download_location" metadataKeyOriginator = "bomly.sbom.originator" metadataKeyOriginatorType = "bomly.sbom.originator_type" metadataKeyDescription = "bomly.sbom.description" @@ -39,7 +40,6 @@ func setIngestedMetadata(dep *sdk.Dependency, component Component) { values := map[string]string{ metadataKeySupplier: component.Supplier, metadataKeySupplierType: component.SupplierType, - metadataKeySupplierURL: component.SupplierURL, metadataKeyOriginator: component.Originator, metadataKeyOriginatorType: component.OriginatorType, metadataKeyDescription: component.Description, @@ -62,6 +62,23 @@ func setIngestedMetadata(dep *sdk.Dependency, component Component) { dep.Metadata[key] = value } + if len(component.SupplierURLs) > 0 { + urls := make([]any, 0, len(component.SupplierURLs)) + for _, u := range component.SupplierURLs { + urls = append(urls, u) + } + if dep.Metadata == nil { + dep.Metadata = make(map[string]any) + } + dep.Metadata[metadataKeySupplierURLs] = urls + } + if component.NoDownloadLocation { + if dep.Metadata == nil { + dep.Metadata = make(map[string]any) + } + dep.Metadata[metadataKeyNoDownload] = true + } + if len(component.ExternalRefs) == 0 { return } @@ -98,7 +115,6 @@ func applyIngestedMetadata(component *Component, metadata map[string]any) { targets := map[string]*string{ metadataKeySupplier: &component.Supplier, metadataKeySupplierType: &component.SupplierType, - metadataKeySupplierURL: &component.SupplierURL, metadataKeyOriginator: &component.Originator, metadataKeyOriginatorType: &component.OriginatorType, metadataKeyDescription: &component.Description, @@ -124,6 +140,17 @@ func applyIngestedMetadata(component *Component, metadata map[string]any) { component.ArtifactURL, component.VCSURL, component.RegistryURL = artifact, vcs, registry } + if urls, ok := metadata[metadataKeySupplierURLs].([]any); ok { + for _, entry := range urls { + if value, ok := entry.(string); ok && value != "" { + component.SupplierURLs = unionStrings(component.SupplierURLs, []string{value}) + } + } + } + if none, ok := metadata[metadataKeyNoDownload].(bool); ok && none { + component.NoDownloadLocation = true + } + raw, ok := metadata[metadataKeyExternalRefs].([]any) if !ok { return diff --git a/internal/sbom/model.go b/internal/sbom/model.go index fe73c1ff..846556d5 100644 --- a/internal/sbom/model.go +++ b/internal/sbom/model.go @@ -149,6 +149,12 @@ type Component struct { // Comments a source document attached to the references above. Kept // separately so an ingested comment is preserved rather than replaced by // Bomly's own, which could contradict what the producer asserted. + // NoDownloadLocation records SPDX's "NONE" marker, which asserts the + // package is not downloadable. That is a different claim from + // NOASSERTION, which says the producer made none, so the two cannot + // collapse into an empty locator. + NoDownloadLocation bool + ArtifactComment string VCSComment string RegistryComment string @@ -162,7 +168,7 @@ type Component struct { // matcher, actually asserted them. Supplier string SupplierType string - SupplierURL string + SupplierURLs []string Originator string OriginatorType string Description string @@ -186,7 +192,6 @@ func mergeComponentAssertions(dst *Component, src Component) { }{ {&dst.Supplier, src.Supplier}, {&dst.SupplierType, src.SupplierType}, - {&dst.SupplierURL, src.SupplierURL}, {&dst.Originator, src.Originator}, {&dst.OriginatorType, src.OriginatorType}, {&dst.Description, src.Description}, @@ -209,8 +214,13 @@ func mergeComponentAssertions(dst *Component, src Component) { // discard the second set whenever the first had any. dst.CPEs = unionStrings(dst.CPEs, src.CPEs) dst.Digests = unionComponentDigests(dst.Digests, src.Digests) - if len(dst.Licenses) == 0 { - dst.Licenses = src.Licenses + dst.Licenses = unionLicenses(dst.Licenses, src.Licenses) + dst.SupplierURLs = unionStrings(dst.SupplierURLs, src.SupplierURLs) + if len(dst.Vulnerabilities) == 0 { + dst.Vulnerabilities = src.Vulnerabilities + } + if !dst.NoDownloadLocation { + dst.NoDownloadLocation = src.NoDownloadLocation } // External references are a set, not a single assertion: each copy may // name a different link, so a fill-gaps copy would drop the second one @@ -218,6 +228,26 @@ func mergeComponentAssertions(dst *Component, src Component) { dst.ExternalRefs = unionExternalRefs(dst.ExternalRefs, src.ExternalRefs) } +// unionLicenses appends licenses from extra that base does not carry, keyed +// by the normalized expression. +func unionLicenses(base, extra []License) []License { + if len(extra) == 0 { + return base + } + seen := make(map[License]struct{}, len(base)) + for _, license := range base { + seen[license] = struct{}{} + } + for _, license := range extra { + if _, ok := seen[license]; ok { + continue + } + seen[license] = struct{}{} + base = append(base, license) + } + return base +} + // unionComponentDigests appends digests from extra that base does not carry. func unionComponentDigests(base, extra []Digest) []Digest { if len(extra) == 0 { diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 62d38d95..0fecb413 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -1110,3 +1110,136 @@ func mustGraphFromDoc(t *testing.T, doc *Document) *sdk.Graph { } return graph } + +// TestSetValuedAssertionsSurviveDuplicateMerge is the sweep's regression net. +// Every set-valued field on a component described twice must union rather than +// let one copy win; this repeatedly turned up one field at a time in review. +func TestSetValuedAssertionsSurviveDuplicateMerge(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "metadata": {"component": { + "bom-ref": "root", "type": "application", "name": "app", "version": "1.0.0", + "purl": "pkg:npm/app@1.0.0", + "licenses": [{"license": {"id": "Apache-2.0"}}], + "supplier": {"name": "Acme", "url": ["https://second.example"]}, + "externalReferences": [{"type": "documentation", "url": "https://docs.example"}] + }}, + "components": [ + {"bom-ref": "root", "type": "application", "name": "app", "version": "1.0.0", "purl": "pkg:npm/app@1.0.0", + "licenses": [{"license": {"id": "MIT"}}], + "supplier": {"name": "Acme", "url": ["https://first.example"]}, + "externalReferences": [{"type": "website", "url": "https://site.example"}]}, + {"bom-ref": "pkg:npm/dep@1.0.0", "type": "library", "name": "dep", "version": "1.0.0", "purl": "pkg:npm/dep@1.0.0"} + ], + "dependencies": [{"ref": "root", "dependsOn": ["pkg:npm/dep@1.0.0"]}] + }` + + out := string(ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON)) + for name, want := range map[string]string{ + "inventory license": "MIT", + "metadata license": "Apache-2.0", + "inventory supplier url": "https://first.example", + "metadata supplier url": "https://second.example", + "inventory reference": "https://site.example", + "metadata reference": "https://docs.example", + } { + if !strings.Contains(out, want) { + t.Fatalf("%s was discarded by the merge:\n%s", name, out) + } + } +} + +// TestMultipleVCSReferencesArePreserved mirrors the distribution-mirror case: +// CycloneDX types vcs as an array, so extra repositories must not overwrite. +func TestMultipleVCSReferencesArePreserved(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", "type": "library", "name": "a", "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "externalReferences": [ + {"type": "vcs", "url": "https://github.com/org/primary"}, + {"type": "vcs", "url": "https://gitlab.com/org/mirror"} + ] + }] + }` + + out := string(ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON)) + for _, want := range []string{"github.com/org/primary", "gitlab.com/org/mirror"} { + if !strings.Contains(out, want) { + t.Fatalf("vcs mirror %q was overwritten:\n%s", want, out) + } + } +} + +// TestIngestedDistributionKeepsBenignQuery covers a source-declared download +// with a benign query. The detector-oriented classifier drops every query, +// which is right for a lockfile value and wrong for an asserted one. +func TestIngestedDistributionKeepsBenignQuery(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", "type": "library", "name": "a", "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "externalReferences": [ + {"type": "distribution", "url": "https://repo.example/download?id=123"} + ] + }] + }` + + for _, target := range []Target{TargetSPDX23JSON, TargetCycloneDX17JSON} { + out := string(ingestAndReexport(t, []byte(in), target)) + if !strings.Contains(out, "https://repo.example/download?id=123") { + t.Fatalf("%s dropped an asserted download with a benign query:\n%s", target, out) + } + } + + // A credential-bearing query is still rejected on the same path. + hostile := strings.Replace(in, "?id=123", "?token=s3cret", 1) + for _, target := range []Target{TargetSPDX23JSON, TargetCycloneDX17JSON} { + if out := string(ingestAndReexport(t, []byte(hostile), target)); strings.Contains(out, "s3cret") { + t.Fatalf("%s republished a credential query:\n%s", target, out) + } + } +} + +// TestSPDXNoneDownloadLocationIsPreserved keeps two distinct SPDX assertions +// apart: NONE says the package is not downloadable, NOASSERTION says the +// producer made no claim. +func TestSPDXNoneDownloadLocationIsPreserved(t *testing.T) { + const in = `{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [ + {"name": "none-pkg", "SPDXID": "SPDXRef-a", "versionInfo": "1.0.0", "downloadLocation": "NONE", "filesAnalyzed": false}, + {"name": "noassert-pkg", "SPDXID": "SPDXRef-b", "versionInfo": "1.0.0", "downloadLocation": "NOASSERTION", "filesAnalyzed": false} + ] + }` + + out := ingestAndReexport(t, []byte(in), TargetSPDX23JSON) + var doc v23.Document + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + got := map[string]string{} + for _, p := range doc.Packages { + if p != nil { + got[p.PackageName] = p.PackageDownloadLocation + } + } + if got["none-pkg"] != "NONE" { + t.Fatalf("NONE became %q, want the assertion preserved", got["none-pkg"]) + } + if got["noassert-pkg"] != "NOASSERTION" { + t.Fatalf("NOASSERTION became %q", got["noassert-pkg"]) + } +} diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index ec2b6176..01e371f9 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -190,6 +190,9 @@ func (spdx23Codec) decodeJSON(data []byte) (*Document, error) { component.Originator = parseSPDXEntity(p.PackageOriginator.Originator) component.OriginatorType = p.PackageOriginator.OriginatorType } + if strings.EqualFold(strings.TrimSpace(p.PackageDownloadLocation), "NONE") { + component.NoDownloadLocation = true + } applyLocator(&component, classifyAssertedDownloadLocation(parseSPDXEntity(p.PackageDownloadLocation))) if home := parseSPDXEntity(p.PackageHomePage); isPublishableReferenceURL(home) { component.ExternalRefs = append(component.ExternalRefs, ExternalRef{Type: "website", URL: home}) @@ -525,6 +528,12 @@ func spdxDownloadLocation(component Component) string { if location := firstNonEmpty(component.ArtifactURL, component.VCSURL); location != "" { return location } + // SPDX separates "there is no download location" from "no claim was + // made". Collapsing NONE to NOASSERTION would weaken an explicit + // assertion the source document made. + if component.NoDownloadLocation { + return "NONE" + } return "NOASSERTION" } diff --git a/internal/sbom/transform.go b/internal/sbom/transform.go index d3e5729a..09a9fbd4 100644 --- a/internal/sbom/transform.go +++ b/internal/sbom/transform.go @@ -234,10 +234,12 @@ var digestHexSizes = map[string]int{ "sha3-512": 64, // Every algorithm the encoders accept needs an entry here, or an ingested // value of the wrong length passes validation unchecked. - "blake3": 32, - "blake2b-256": 32, - "blake2b-384": 48, - "blake2b-512": 64, + "blake3": 32, + "streebog-256": 32, + "streebog-512": 64, + "blake2b-256": 32, + "blake2b-384": 48, + "blake2b-512": 64, } func normalizeDigestValue(algorithm, value string) string { @@ -359,6 +361,10 @@ func normalizeDigestAlgorithm(algorithm string) string { return "sha3-384" case "sha3-512", "sha3512": return "sha3-512" + case "streebog-256", "streebog256": + return "streebog-256" + case "streebog-512", "streebog512": + return "streebog-512" case "blake2b-256", "blake2b256": return "blake2b-256" case "blake2b-384", "blake2b384": From 4dd09b4f8dcfc348d759443eedcbc5bb9b3d5c0c Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 12:02:01 -0700 Subject: [PATCH 14/23] fix(sbom): validate rendered VCS revisions and every CPE component MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A credential path I created by design decision. validatedVCSLocator deliberately returned the locator verbatim so the pinned revision would not be altered, and that choice bypassed revision validation entirely: in "git+https://host/org/repo@ghp_abcd1234" the suffix parses as part of the URL path, so neither the userinfo check nor isSafeRevision ever saw it. The suffix is now split off and validated; an unsafe one is dropped and the repository kept, matching normalizeVCS. CPE validation checked only the field count and the part, so "cpe:2.3:a:vendor with space:product:…" passed and was republished as a package-identity assertion. Every component is now checked as an avstring: printable ASCII, no whitespace, reserved punctuation escaped, no trailing lone backslash, and no empty part. Deliberately not over-strict: "*" and "?" stay legal unescaped, because partial values like "1.3.*" are common and rejecting them would drop real identity data — the same kind of harm as preserving a fabricated one. A corpus of thirteen NVD-style CPEs guards that direction. Note on method: my first mutation check of the CPE fix reported zero failures, which I nearly accepted. The revert had silently failed to match, so it measured nothing — and the test itself was also vacuous, because the same scripted edit had dropped the malformed cases while landing the well-formed ones. Both are now applied and verified: with the component loop removed the test fails on the whitespace case. Co-Authored-By: Claude Opus 5 --- internal/sbom/locator.go | 27 +++++++++-- internal/sbom/roundtrip_test.go | 84 +++++++++++++++++++++++++++++++++ internal/sbom/spdx23.go | 65 +++++++++++++++++++++++-- 3 files changed, 168 insertions(+), 8 deletions(-) diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index b7f89ee3..a37ef3cd 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -503,15 +503,27 @@ func classifyAssertedDownloadLocation(raw string) Locator { // validatedVCSLocator checks an already-rendered "git+://…" // locator and returns it unchanged when it is safe to republish, or "". // -// The locator is kept verbatim rather than reparsed and rebuilt: any pinned -// revision after "@" is part of the value's meaning, and re-deriving it risks -// changing what it says. +// The pinned revision has to be split off and checked explicitly. In +// "git+https://host/org/repo@ghp_abcd1234" the suffix parses as part of the +// URL path, not as userinfo, so the scheme and userinfo checks never see it — +// an earlier version returned that locator unchanged and republished the +// token. An unsafe revision is dropped and the repository kept, matching what +// normalizeVCS does. func validatedVCSLocator(locator string) string { locator = strings.TrimSpace(locator) if !strings.HasPrefix(locator, "git+") { return "" } - parsed, err := url.Parse(strings.TrimPrefix(locator, "git+")) + base := strings.TrimPrefix(locator, "git+") + + revision := "" + if scheme, rest, ok := strings.Cut(base, "://"); ok { + if path, suffix, found := strings.Cut(rest, "@"); found { + base, revision = scheme+"://"+path, suffix + } + } + + parsed, err := url.Parse(base) if err != nil || parsed.User != nil || parsed.Host == "" { return "" } @@ -523,7 +535,12 @@ func validatedVCSLocator(locator string) string { if hasCredentialQuery(parsed) || parsed.Fragment != "" { return "" } - return locator + + out := "git+" + base + if isSafeRevision(revision) { + out += "@" + revision + } + return out } // isPublishableReferenceURL reports whether a URL carried on an ingested diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 0fecb413..565ca9f3 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -1012,6 +1012,10 @@ func TestIsValidCPE(t *testing.T) { `cpe:2.3:a:ven\:dor:prod:1.0:*:*:*:*:*:*:*`, "cpe:/a:vendor:product:1.0", "cpe:/o:vendor:os", + // Wildcards inside a value are legal and common; rejecting them + // would drop real identity data. + "cpe:2.3:a:vendor:product:1.3.*:*:*:*:*:*:*:*", + "cpe:2.3:a:vendor:node.js:10.0:*:*:*:*:*:*:*", } for _, value := range valid { if !isValidCPE(value) { @@ -1027,6 +1031,13 @@ func TestIsValidCPE(t *testing.T) { "cpe:/z:vendor:product", // bad part "cpe:/a:b:c:d:e:f:g:h", // too many "https://example.com", + // Component-level checks: a correct field count is not enough. + "cpe:2.3:a:vendor with space:product:*:*:*:*:*:*:*:*", // unescaped whitespace + "cpe:2.3::vendor:product:*:*:*:*:*:*:*:*", // empty part + "cpe:2.3:a:vendor:prod" locator. The suffix parses as part of the URL path, so +// the scheme and userinfo checks never see it — an earlier version returned +// such a locator unchanged and republished the token in it. +func TestVCSLocatorRevisionIsValidated(t *testing.T) { + cases := []struct{ in, want string }{ + {"git+https://github.com/org/repo@deadbeef", "git+https://github.com/org/repo@deadbeef"}, + {"git+https://github.com/org/repo@v1.2.3", "git+https://github.com/org/repo@v1.2.3"}, + // Token-shaped revision: the repository survives, the secret does not. + {"git+https://github.com/org/repo@ghp_abcd1234", "git+https://github.com/org/repo"}, + {"git+https://github.com/org/repo@glpat-Abc123", "git+https://github.com/org/repo"}, + {"git+https://tok:s3cret@github.com/org/repo", ""}, + {"git+file:///Users/victim/repo", ""}, + {"https://github.com/org/repo", ""}, + {"", ""}, + } + for _, tc := range cases { + if got := validatedVCSLocator(tc.in); got != tc.want { + t.Fatalf("validatedVCSLocator(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// TestVCSSourceInfoTokenIsNotRepublished is the end-to-end counterpart. +func TestVCSSourceInfoTokenIsNotRepublished(t *testing.T) { + const in = `{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [{ + "name": "a", "SPDXID": "SPDXRef-a", "versionInfo": "1.0.0", + "downloadLocation": "https://reg.example/a-1.0.0.tgz", "filesAnalyzed": false, + "sourceInfo": "Source repository: git+https://github.com/org/repo@ghp_abcd1234" + }] + }` + + for _, target := range []Target{TargetSPDX23JSON, TargetCycloneDX17JSON} { + out := string(ingestAndReexport(t, []byte(in), target)) + if strings.Contains(out, "ghp_abcd1234") { + t.Fatalf("%s republished a token from a rendered VCS locator:\n%s", target, out) + } + if !strings.Contains(out, "github.com/org/repo") { + t.Fatalf("%s dropped the repository along with the token:\n%s", target, out) + } + } +} + +// Realistic CPEs drawn from NVD-style identifiers, to confirm the stricter +// component validation does not reject genuine identity data. +func TestRealWorldCPEsAccepted(t *testing.T) { + for _, value := range []string{ + "cpe:2.3:a:apache:log4j:2.14.1:*:*:*:*:*:*:*", + "cpe:2.3:a:openbsd:openssh:8.9:p1:*:*:*:*:*:*", + "cpe:2.3:a:nodejs:node.js:16.13.0:*:*:*:*:*:*:*", + "cpe:2.3:o:linux:linux_kernel:5.15.0:*:*:*:*:*:*:*", + "cpe:2.3:a:facebook:react:18.2.0:*:*:*:*:*:*:*", + "cpe:2.3:a:python:cpython:3.11.0:rc1:*:*:*:*:*:*", + "cpe:2.3:a:microsoft:.net:6.0:*:*:*:*:*:*:*", + "cpe:2.3:a:vendor:product:1.3.*:*:*:*:*:*:*:*", + "cpe:2.3:h:cisco:asr_9000:-:*:*:*:*:*:*:*", + "cpe:2.3:a:gnu:glibc:2.35:*:*:*:*:*:*:*", + `cpe:2.3:a:acme:c\+\+_library:1.0:*:*:*:*:*:*:*`, + "cpe:/a:apache:log4j:2.14.1", + "cpe:/o:linux:linux_kernel:5.15.0", + } { + if !isValidCPE(value) { + t.Errorf("real-world CPE rejected: %q", value) + } + } +} diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 01e371f9..33913e80 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -673,29 +673,88 @@ func isValidCPE(value string) bool { if len(parts) != 13 { return false } - return isCPEPart(parts[2]) + if !isCPEPart(parts[2]) { + return false + } + // Every remaining component must be a well-formed avstring. Checking + // only the field count let values such as + // "cpe:2.3:a:vendor with space:product:…" through. + for _, part := range parts[3:] { + if !isCPEComponent(part) { + return false + } + } + return true case strings.HasPrefix(value, "cpe:/"): // URI binding: "cpe:/" plus up to 7 components, the first the part. parts := splitUnescaped(strings.TrimPrefix(value, "cpe:/"), ':') if len(parts) == 0 || len(parts) > 7 { return false } - return isCPEPart(parts[0]) + if !isCPEPart(parts[0]) { + return false + } + for _, part := range parts[1:] { + if !isCPEComponent(part) { + return false + } + } + return true default: return false } } // isCPEPart reports whether a CPE part component is one of the defined values. +// +// An empty part is not among them: "cpe:2.3::vendor:…" names no component +// class, so accepting it would preserve an identity assertion that identifies +// nothing. func isCPEPart(part string) bool { switch part { - case "a", "o", "h", "*", "-", "": + case "a", "o", "h", "*", "-": return true default: return false } } +// isCPEComponent reports whether a CPE attribute value is well formed. +// +// The logical values "*" and "-" stand alone. Otherwise the value is a quoted +// string of printable ASCII: whitespace and control characters are not +// permitted, and the punctuation CPE reserves must be backslash-escaped. A +// trailing lone backslash escapes nothing and is malformed. +// +// The wildcards "*" and "?" are allowed unescaped, because partial values such +// as "1.3.*" are legal and common. Rejecting them would drop real identity +// data, which is the same kind of harm as preserving a fabricated value. +func isCPEComponent(value string) bool { + switch value { + case "*", "-": + return true + case "": + return false + } + + escaped := false + for _, r := range value { + switch { + case escaped: + escaped = false + case r == '\\': + escaped = true + case r < '!' || r > '~': + // Space, control characters, and anything non-ASCII. + return false + case strings.ContainsRune(`!"#$%&'()+,/:;<=>@[]^`+"`"+`{|}~`, r): + // Reserved punctuation must be escaped to appear literally. + return false + } + } + return !escaped +} + // splitUnescaped splits on sep, treating a backslash-escaped separator as a // literal character rather than a delimiter. func splitUnescaped(value string, sep rune) []string { From 09e59b846dcb690289255e6e0034ad9e2393d95f Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 12:39:38 -0700 Subject: [PATCH 15/23] fix(sbom): close the remaining gate, union, and vocabulary gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twelve findings across both reviewers, nearly all in classes this PR has already fixed once. Grouping them by cause rather than by symptom. Credential detection. client_secret matched neither the parameter denylist nor the token-prefix check, so an asserted download carrying it was published. The list now covers the standard OAuth, session, and signature parameter names. Precedence. An SPDX package can declare downloadLocation NONE and still record a source repository; the recovered locator was winning, replacing an explicit "not downloadable" with a repository URL. NONE now outranks both locators. Gate consistency. Only the distribution reference used the credential-aware asserted-URL path, so "?version=1" on a documentation reference was dropped by the blanket query rejection. All source-declared references now share that gate, and ftp/ftps — already accepted for references — are allowed on the asserted-download path while detector-derived values stay HTTP-only. Set-valued fields. Vulnerabilities at the model level and licenses at the graph level were the last two still merging as scalars. Classified locators also crossed the graph as independent metadata keys, so a later locator's comment could attach to an earlier locator's URL; URL and comment now move as one unit. Algorithm vocabulary. ADLER32, MD2, MD4, and MD6 were accepted on ingest with no encoder mapping, the same shape as BLAKE2b and BLAKE3 before them. MD6 has no fixed width, so it is recorded as explicitly variable-length rather than given a wrong constant, and the coverage invariant now distinguishes "unmeasurable" from "forgotten". Fidelity. Supplier contacts and external-reference hashes are now preserved. I declined both earlier — contacts on privacy grounds — but that reasoning does not hold: the producer already published them, a format conversion is not a new disclosure, and the same principle already applied to supplier URLs. Two notes on method. Serializing contacts as maps and merging them with the string union would have silently dropped every one; the mutation check caught it. A second mutation check then reported no failures with the pattern confirmed applied, which showed the contact test used a single component and never reached the graph merge at all — that path now has its own duplicate-component test. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 45 +++++++- internal/sbom/distribution_test.go | 18 +++- internal/sbom/graph.go | 111 ++++++++++++++++++- internal/sbom/ingest_metadata.go | 57 +++++++++- internal/sbom/locator.go | 33 ++++-- internal/sbom/model.go | 73 +++++++++++-- internal/sbom/roundtrip_test.go | 164 +++++++++++++++++++++++++++++ internal/sbom/spdx23.go | 20 +++- internal/sbom/transform.go | 11 ++ 9 files changed, 495 insertions(+), 37 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 323d03aa..99e40382 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -57,6 +57,15 @@ func (c cycloneDXCodec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, e urls := append([]string(nil), comp.SupplierURLs...) supplier.URL = &urls } + if len(comp.SupplierContacts) > 0 { + contacts := make([]cdx.OrganizationalContact, 0, len(comp.SupplierContacts)) + for _, c := range comp.SupplierContacts { + contacts = append(contacts, cdx.OrganizationalContact{ + Name: c.Name, Email: c.Email, Phone: c.Phone, + }) + } + supplier.Contact = &contacts + } component.Supplier = supplier } component.Publisher = comp.Originator @@ -345,6 +354,22 @@ func externalReferenceType(value string) cdx.ExternalReferenceType { return cdx.ERTypeOther } +// referenceDigests validates the integrity assertion carried on an external +// reference. It uses the same gate as component hashes, so a malformed value +// cannot ride in on a reference instead. +func referenceDigests(hashes *[]cdx.Hash) []Digest { + if hashes == nil { + return nil + } + var out []Digest + for _, hash := range *hashes { + if digest, ok := ingestedDigest(string(hash.Algorithm), hash.Value); ok { + out = append(out, digest) + } + } + return out +} + // componentFromCycloneDX projects one CycloneDX component onto the neutral // model. Both decode paths (the component inventory and the metadata-only // fallback) share it so they cannot drift apart. @@ -367,6 +392,16 @@ func componentFromCycloneDX(comp cdx.Component) Component { if comp.Supplier != nil && comp.Supplier.Name != "" { component.Supplier = comp.Supplier.Name component.SupplierType = "Organization" + if comp.Supplier.Contact != nil { + for _, c := range *comp.Supplier.Contact { + if c.Name == "" && c.Email == "" && c.Phone == "" { + continue + } + component.SupplierContacts = append(component.SupplierContacts, Contact{ + Name: c.Name, Email: c.Email, Phone: c.Phone, + }) + } + } if comp.Supplier.URL != nil { for _, u := range *comp.Supplier.URL { if isPublishableReferenceURL(u) { @@ -403,6 +438,7 @@ func componentFromCycloneDX(comp cdx.Component) Component { Type: string(ref.Type), URL: ref.URL, Comment: ref.Comment, + Digests: referenceDigests(ref.Hashes), }) case cdx.ERTypeVCS: // Untrusted input: gate and normalize it exactly like a @@ -431,6 +467,7 @@ func componentFromCycloneDX(comp cdx.Component) Component { Type: string(ref.Type), URL: ref.URL, Comment: ref.Comment, + Digests: referenceDigests(ref.Hashes), }) } } @@ -473,11 +510,15 @@ func cycloneDXComponentReferences(component Component) []cdx.ExternalReference { } for _, ref := range component.ExternalRefs { - refs = append(refs, cdx.ExternalReference{ + emitted := cdx.ExternalReference{ Type: externalReferenceType(ref.Type), URL: ref.URL, Comment: ref.Comment, - }) + } + if hashes := cycloneDXHashes(ref.Digests); len(hashes) > 0 { + emitted.Hashes = &hashes + } + refs = append(refs, emitted) } if len(refs) == 0 { diff --git a/internal/sbom/distribution_test.go b/internal/sbom/distribution_test.go index a508d975..afa64b81 100644 --- a/internal/sbom/distribution_test.go +++ b/internal/sbom/distribution_test.go @@ -424,12 +424,16 @@ func TestBlake3DigestSurvivesRoundTrip(t *testing.T) { } pkg := spdxPackageFor(t, g, BuildOptions{}) - if len(pkg.PackageChecksums) != 1 || pkg.PackageChecksums[0].Algorithm != common.BLAKE3 { - t.Fatalf("spdx blake3 checksum = %+v, want it preserved", pkg.PackageChecksums) + if len(pkg.PackageChecksums) != 1 || + pkg.PackageChecksums[0].Algorithm != common.BLAKE3 || + pkg.PackageChecksums[0].Value != value { + t.Fatalf("spdx blake3 checksum = %+v, want it preserved with its value", pkg.PackageChecksums) } comp := cycloneDXComponentFor(t, g, BuildOptions{}) - if comp.Hashes == nil || len(*comp.Hashes) != 1 || (*comp.Hashes)[0].Algorithm != cdx.HashAlgoBlake3 { - t.Fatalf("cyclonedx blake3 hash = %+v, want it preserved", comp.Hashes) + if comp.Hashes == nil || len(*comp.Hashes) != 1 || + (*comp.Hashes)[0].Algorithm != cdx.HashAlgoBlake3 || + (*comp.Hashes)[0].Value != value { + t.Fatalf("cyclonedx blake3 hash = %+v, want it preserved with its value", comp.Hashes) } } @@ -470,12 +474,16 @@ func TestEveryAcceptedDigestAlgorithmRoundTrips(t *testing.T) { // malformed value reaches the output unchecked. func TestNormalizedAlgorithmsHaveLengthEntries(t *testing.T) { for _, algorithm := range []string{ - "md5", "sha1", "sha256", "sha384", "sha512", + "md5", "md2", "md4", "md6", "adler32", + "sha1", "sha224", "sha256", "sha384", "sha512", "sha3-256", "sha3-384", "sha3-512", "blake2b-256", "blake2b-384", "blake2b-512", "blake3", "streebog-256", "streebog-512", } { canonical := normalizeDigestAlgorithm(algorithm) + if _, variable := variableLengthDigests[canonical]; variable { + continue + } if _, ok := digestHexSizes[canonical]; !ok { t.Fatalf("%q is emitted by the encoders but has no length entry, so ingest cannot validate it", algorithm) } diff --git a/internal/sbom/graph.go b/internal/sbom/graph.go index 2dd63ed0..e0203b35 100644 --- a/internal/sbom/graph.go +++ b/internal/sbom/graph.go @@ -137,11 +137,24 @@ func mergeIngestedNode(existing, incoming *sdk.Dependency) { existing.CPEs = unionStrings(existing.CPEs, incoming.CPEs) existing.Digests = unionDigests(existing.Digests, incoming.Digests) - if len(sdk.DetectionLicenses(existing)) == 0 { - if licenses := sdk.DetectionLicenses(incoming); len(licenses) > 0 { - sdk.SetDetectionLicenses(existing, licenses) + // Licenses are set-valued here too: two components collapsing onto one + // PURL may each declare a different choice. + if incomingLicenses := sdk.DetectionLicenses(incoming); len(incomingLicenses) > 0 { + merged := sdk.DetectionLicenses(existing) + seen := make(map[sdk.PackageLicense]struct{}, len(merged)) + for _, license := range merged { + seen[license] = struct{}{} } + for _, license := range incomingLicenses { + if _, ok := seen[license]; ok { + continue + } + seen[license] = struct{}{} + merged = append(merged, license) + } + sdk.SetDetectionLicenses(existing, merged) } + mergeLocatorPairs(existing, incoming) for key, value := range incoming.Metadata { if existing.Metadata == nil { existing.Metadata = make(map[string]any, len(incoming.Metadata)) @@ -149,6 +162,15 @@ func mergeIngestedNode(existing, incoming *sdk.Dependency) { // External references are a set carried under one key, so a key-level // fill-gaps merge would drop the duplicate's whole list whenever the // first component had any. Union them instead. + if _, paired := locatorCommentKeys[key]; paired { + // Handled atomically below so a URL never picks up another + // locator's comment. + continue + } + if key == metadataKeySupplierPeople { + existing.Metadata[key] = unionAnyRecords(existing.Metadata[key], value, "name", "email", "phone") + continue + } if key == metadataKeySupplierURLs { existing.Metadata[key] = unionAnyStrings(existing.Metadata[key], value) continue @@ -163,6 +185,43 @@ func mergeIngestedNode(existing, incoming *sdk.Dependency) { } } +// locatorCommentKeys pairs each classified locator with its comment. The two +// must merge together: filling them independently can attach one locator's +// comment to a different locator's URL. +var locatorCommentKeys = map[string]string{ + metadataKeyArtifactURL: metadataKeyArtifactNote, + metadataKeyVCSURL: metadataKeyVCSNote, + metadataKeyRegistryURL: metadataKeyRegistryNote, + metadataKeyArtifactNote: metadataKeyArtifactURL, + metadataKeyVCSNote: metadataKeyVCSURL, + metadataKeyRegistryNote: metadataKeyRegistryURL, +} + +// mergeLocatorPairs fills each empty locator slot from incoming, moving the +// URL and its comment as one unit. +func mergeLocatorPairs(existing, incoming *sdk.Dependency) { + for _, pair := range []struct{ url, note string }{ + {metadataKeyArtifactURL, metadataKeyArtifactNote}, + {metadataKeyVCSURL, metadataKeyVCSNote}, + {metadataKeyRegistryURL, metadataKeyRegistryNote}, + } { + if _, present := existing.Metadata[pair.url]; present { + continue + } + value, ok := incoming.Metadata[pair.url] + if !ok { + continue + } + if existing.Metadata == nil { + existing.Metadata = make(map[string]any) + } + existing.Metadata[pair.url] = value + if note, ok := incoming.Metadata[pair.note]; ok { + existing.Metadata[pair.note] = note + } + } +} + // unionAnyStrings merges two serialized string lists, preserving order and // dropping duplicates. func unionAnyStrings(base, extra any) any { @@ -194,6 +253,52 @@ func unionAnyStrings(base, extra any) any { return baseList } +// unionAnyRecords merges two serialized lists of maps, deduplicating on the +// named fields. Contacts are maps rather than strings, so the string union +// would silently discard every one of them. +func unionAnyRecords(base, extra any, keyFields ...string) any { + baseList, _ := base.([]any) + extraList, _ := extra.([]any) + if len(extraList) == 0 { + return base + } + if len(baseList) == 0 { + return extra + } + + identity := func(entry any) (string, bool) { + fields, ok := entry.(map[string]any) + if !ok { + return "", false + } + parts := make([]string, 0, len(keyFields)) + for _, name := range keyFields { + value, _ := fields[name].(string) + parts = append(parts, value) + } + return strings.Join(parts, "\x00"), true + } + + seen := make(map[string]struct{}, len(baseList)) + for _, entry := range baseList { + if id, ok := identity(entry); ok { + seen[id] = struct{}{} + } + } + for _, entry := range extraList { + id, ok := identity(entry) + if !ok { + continue + } + if _, present := seen[id]; present { + continue + } + seen[id] = struct{}{} + baseList = append(baseList, entry) + } + return baseList +} + // unionExternalRefValues merges two serialized external-reference lists, // keyed by type and URL. Entries that are not the expected shape are kept // as-is on the base side and skipped on the incoming side. diff --git a/internal/sbom/ingest_metadata.go b/internal/sbom/ingest_metadata.go index fc811827..8703c7b9 100644 --- a/internal/sbom/ingest_metadata.go +++ b/internal/sbom/ingest_metadata.go @@ -17,6 +17,7 @@ const ( metadataKeySupplierType = "bomly.sbom.supplier_type" metadataKeySupplierURLs = "bomly.sbom.supplier_urls" metadataKeyNoDownload = "bomly.sbom.no_download_location" + metadataKeySupplierPeople = "bomly.sbom.supplier_contacts" metadataKeyOriginator = "bomly.sbom.originator" metadataKeyOriginatorType = "bomly.sbom.originator_type" metadataKeyDescription = "bomly.sbom.description" @@ -72,6 +73,16 @@ func setIngestedMetadata(dep *sdk.Dependency, component Component) { } dep.Metadata[metadataKeySupplierURLs] = urls } + if len(component.SupplierContacts) > 0 { + people := make([]any, 0, len(component.SupplierContacts)) + for _, c := range component.SupplierContacts { + people = append(people, map[string]any{"name": c.Name, "email": c.Email, "phone": c.Phone}) + } + if dep.Metadata == nil { + dep.Metadata = make(map[string]any) + } + dep.Metadata[metadataKeySupplierPeople] = people + } if component.NoDownloadLocation { if dep.Metadata == nil { dep.Metadata = make(map[string]any) @@ -84,11 +95,21 @@ func setIngestedMetadata(dep *sdk.Dependency, component Component) { } refs := make([]any, 0, len(component.ExternalRefs)) for _, ref := range component.ExternalRefs { - refs = append(refs, map[string]any{ + entry := map[string]any{ "type": ref.Type, "url": ref.URL, "comment": ref.Comment, - }) + } + if len(ref.Digests) > 0 { + digests := make([]any, 0, len(ref.Digests)) + for _, digest := range ref.Digests { + digests = append(digests, map[string]any{ + "algorithm": digest.Algorithm, "value": digest.Value, + }) + } + entry["digests"] = digests + } + refs = append(refs, entry) } if dep.Metadata == nil { dep.Metadata = make(map[string]any) @@ -147,6 +168,22 @@ func applyIngestedMetadata(component *Component, metadata map[string]any) { } } } + if people, ok := metadata[metadataKeySupplierPeople].([]any); ok { + for _, entry := range people { + fields, ok := entry.(map[string]any) + if !ok { + continue + } + name, _ := fields["name"].(string) + email, _ := fields["email"].(string) + phone, _ := fields["phone"].(string) + if name == "" && email == "" && phone == "" { + continue + } + component.SupplierContacts = unionContacts(component.SupplierContacts, + []Contact{{Name: name, Email: email, Phone: phone}}) + } + } if none, ok := metadata[metadataKeyNoDownload].(bool); ok && none { component.NoDownloadLocation = true } @@ -167,7 +204,21 @@ func applyIngestedMetadata(component *Component, metadata map[string]any) { if refType == "" || refURL == "" { continue } - refs = append(refs, ExternalRef{Type: refType, URL: refURL, Comment: comment}) + restored := ExternalRef{Type: refType, URL: refURL, Comment: comment} + if rawDigests, ok := fields["digests"].([]any); ok { + for _, entry := range rawDigests { + digestFields, ok := entry.(map[string]any) + if !ok { + continue + } + algorithm, _ := digestFields["algorithm"].(string) + value, _ := digestFields["value"].(string) + if digest, ok := ingestedDigest(algorithm, value); ok { + restored.Digests = append(restored.Digests, digest) + } + } + } + refs = append(refs, restored) } if len(refs) > 0 { component.ExternalRefs = refs diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index a37ef3cd..962b0eaa 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -74,10 +74,15 @@ func pinLocator(locator Locator, revision string) Locator { // credentialQueryKeys are query parameter names that carry a secret. A // download URL using one of them must not be published. var credentialQueryKeys = []string{ - "token", "access_token", "auth", "authorization", "apikey", "api_key", - "key", "secret", "password", "passwd", "pwd", "credential", "sig", - "signature", "x-amz-signature", "x-amz-credential", "x-amz-security-token", - "x-goog-signature", "se", "sp", "sv", "sr", // Azure SAS parameters + "token", "access_token", "refresh_token", "id_token", "auth", + "authorization", "apikey", "api_key", "api-key", "key", "secret", + "client_secret", "client_id", "password", "passwd", "pwd", "pass", + "credential", "credentials", "session", "sessionid", "session_id", + "sig", "signature", "hmac", "nonce", + "x-amz-signature", "x-amz-credential", "x-amz-security-token", + "x-goog-signature", "goog-signature", + "se", "sp", "sv", "sr", "sig_", // Azure SAS parameters + "private_token", "personal_token", "auth_token", "authtoken", } // hasCredentialQuery reports whether a URL's query carries something that @@ -150,8 +155,16 @@ func classifyURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosyste // and `path`, pipenv `path`, pub `path`, npm link entries), and they do // not consistently carry DependencySourceFile, so the check must be on // the value rather than on source. + // Detector-derived values stay HTTP-only: nothing asserts what they are, + // so the narrowest gate is right. A value the source document declared a + // download location may also use the network transports the external + // reference gate accepts. switch strings.ToLower(parsed.Scheme) { case "http", "https": + case "ftp", "ftps": + if !allowBenignQuery { + return Locator{} + } default: return Locator{} } @@ -560,12 +573,12 @@ func isPublishableReferenceURL(raw string) bool { return false } switch strings.ToLower(parsed.Scheme) { - case "http", "https": - return parsed.Host != "" && parsed.RawQuery == "" && parsed.Fragment == "" - case "git", "ftp", "ftps": - // Network transports CycloneDX legitimately uses for distribution and - // version-control references. - return parsed.Host != "" && parsed.RawQuery == "" && parsed.Fragment == "" + case "http", "https", "git", "ftp", "ftps": + // These are source-declared references, so a benign query such as + // "?version=1" is part of the assertion and only credential-shaped + // parameters disqualify it. Fragments stay rejected: they carry no + // meaning for a reference and are a common place to hide a secret. + return parsed.Host != "" && parsed.Fragment == "" && !hasCredentialQuery(parsed) case "mailto", "urn": // Opaque identifiers with no host and no filesystem reach. A `bom` // reference to "urn:uuid:..." is the common CycloneDX case. diff --git a/internal/sbom/model.go b/internal/sbom/model.go index 846556d5..363256bb 100644 --- a/internal/sbom/model.go +++ b/internal/sbom/model.go @@ -166,12 +166,16 @@ type Component struct { // Assertions preserved verbatim from an ingested SBOM. Bomly never // invents these; they are present only when the source document, or a // matcher, actually asserted them. - Supplier string - SupplierType string - SupplierURLs []string - Originator string - OriginatorType string - Description string + Supplier string + SupplierType string + SupplierURLs []string + // SupplierContacts are the supplier's own published contact entries. + // Preserving them republishes nothing the source document did not + // already publish, and CycloneDX represents them directly. + SupplierContacts []Contact + Originator string + OriginatorType string + Description string // Summary is SPDX's short-form description. SPDX 2.3 represents it and // PackageDescription as distinct fields; CycloneDX has only one, so the // summary is used there just as a fallback. @@ -216,9 +220,8 @@ func mergeComponentAssertions(dst *Component, src Component) { dst.Digests = unionComponentDigests(dst.Digests, src.Digests) dst.Licenses = unionLicenses(dst.Licenses, src.Licenses) dst.SupplierURLs = unionStrings(dst.SupplierURLs, src.SupplierURLs) - if len(dst.Vulnerabilities) == 0 { - dst.Vulnerabilities = src.Vulnerabilities - } + dst.SupplierContacts = unionContacts(dst.SupplierContacts, src.SupplierContacts) + dst.Vulnerabilities = unionVulnerabilities(dst.Vulnerabilities, src.Vulnerabilities) if !dst.NoDownloadLocation { dst.NoDownloadLocation = src.NoDownloadLocation } @@ -228,6 +231,47 @@ func mergeComponentAssertions(dst *Component, src Component) { dst.ExternalRefs = unionExternalRefs(dst.ExternalRefs, src.ExternalRefs) } +// unionContacts appends contacts from extra that base does not carry. +func unionContacts(base, extra []Contact) []Contact { + if len(extra) == 0 { + return base + } + seen := make(map[Contact]struct{}, len(base)) + for _, contact := range base { + seen[contact] = struct{}{} + } + for _, contact := range extra { + if _, ok := seen[contact]; ok { + continue + } + seen[contact] = struct{}{} + base = append(base, contact) + } + return base +} + +// unionVulnerabilities appends advisories from extra that base does not carry, +// keyed by the source and identifier pair that identifies one advisory. +func unionVulnerabilities(base, extra []Vulnerability) []Vulnerability { + if len(extra) == 0 { + return base + } + type key struct{ source, id string } + seen := make(map[key]struct{}, len(base)) + for _, vuln := range base { + seen[key{vuln.Source, vuln.ID}] = struct{}{} + } + for _, vuln := range extra { + k := key{vuln.Source, vuln.ID} + if _, ok := seen[k]; ok { + continue + } + seen[k] = struct{}{} + base = append(base, vuln) + } + return base +} + // unionLicenses appends licenses from extra that base does not carry, keyed // by the normalized expression. func unionLicenses(base, extra []License) []License { @@ -289,12 +333,23 @@ func unionExternalRefs(base, extra []ExternalRef) []ExternalRef { return base } +// Contact is one organizational contact entry on a supplier. +type Contact struct { + Name string + Email string + Phone string +} + // ExternalRef is an external reference carried through from an ingested SBOM // so that a format conversion does not silently discard it. type ExternalRef struct { Type string URL string Comment string + // Digests are the reference's own integrity assertion, distinct from the + // component's hashes: a distribution URL may carry the checksum of the + // archive it points at. + Digests []Digest } // Dependency describes one package relationship list in the intermediate SBOM model. diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 565ca9f3..6451981b 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -1327,3 +1327,167 @@ func TestRealWorldCPEsAccepted(t *testing.T) { } } } + +// TestSupplierContactsAndReferenceHashesSurvive covers two assertions the +// producer already published: the supplier's contact entries and the integrity +// hash attached to an external reference. Preserving them republishes nothing +// new, and CycloneDX represents both directly. +func TestSupplierContactsAndReferenceHashesSurvive(t *testing.T) { + hash := strings.Repeat("e", 64) + in := `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", "type": "library", "name": "a", "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "supplier": { + "name": "Acme", + "contact": [{"name": "Security Team", "email": "security@acme.example"}] + }, + "externalReferences": [ + {"type": "documentation", "url": "https://docs.example/page", + "hashes": [{"alg": "SHA-256", "content": "` + hash + `"}]} + ] + }] + }` + + out := string(ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON)) + for name, want := range map[string]string{ + "contact name": "Security Team", + "contact email": "security@acme.example", + "reference hash": hash, + } { + if !strings.Contains(out, want) { + t.Fatalf("%s was dropped:\n%s", name, out) + } + } +} + +// TestBenignQueryOnGeneralReferenceSurvives covers references other than +// distribution. They are source-declared too, so a benign query is part of the +// assertion while a credential-shaped one is not. +func TestBenignQueryOnGeneralReferenceSurvives(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", "type": "library", "name": "a", "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "externalReferences": [ + {"type": "documentation", "url": "https://docs.example/page?version=1"}, + {"type": "website", "url": "https://site.example/x?client_secret=s3cret"} + ] + }] + }` + + out := string(ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON)) + if !strings.Contains(out, "https://docs.example/page?version=1") { + t.Fatalf("a benign query on a general reference was dropped:\n%s", out) + } + if strings.Contains(out, "s3cret") { + t.Fatalf("a client_secret query was republished:\n%s", out) + } +} + +// TestNoneDownloadLocationOutranksRecoveredRepository pins the precedence: a +// package can record a source repository and still declare it is not +// downloadable, and the explicit NONE must not be replaced by the repository. +func TestNoneDownloadLocationOutranksRecoveredRepository(t *testing.T) { + const in = `{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [{ + "name": "a", "SPDXID": "SPDXRef-a", "versionInfo": "1.0.0", + "downloadLocation": "NONE", "filesAnalyzed": false, + "sourceInfo": "Source repository: git+https://github.com/org/repo@deadbeef" + }] + }` + + out := ingestAndReexport(t, []byte(in), TargetSPDX23JSON) + var doc v23.Document + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, p := range doc.Packages { + if p == nil || p.PackageName != "a" { + continue + } + if p.PackageDownloadLocation != "NONE" { + t.Fatalf("downloadLocation = %q, want the explicit NONE preserved", p.PackageDownloadLocation) + } + return + } + t.Fatal("component missing from output") +} + +// TestDuplicatePURLVulnerabilitiesAndLicensesAreUnioned closes the last two +// set-valued fields at the graph level. +func TestDuplicatePURLVulnerabilitiesAndLicensesAreUnioned(t *testing.T) { + doc := &Document{ + Components: []Component{ + { + ID: "first", Name: "a", Version: "1.0.0", PURL: "pkg:npm/a@1.0.0", + Licenses: []License{{Value: "MIT", SPDXExpression: "MIT"}}, + }, + { + ID: "second", Name: "a", Version: "1.0.0", PURL: "pkg:npm/a@1.0.0", + Licenses: []License{{Value: "Apache-2.0", SPDXExpression: "Apache-2.0"}}, + }, + }, + Dependencies: []Dependency{{Ref: "first", DependsOn: []string{"second"}}}, + } + + graph, err := ToGraph(doc) + if err != nil { + t.Fatalf("to graph: %v", err) + } + out, err := MarshalDepGraphJSON(graph, TargetCycloneDX17JSON, BuildOptions{}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + for _, want := range []string{"MIT", "Apache-2.0"} { + if !strings.Contains(string(out), want) { + t.Fatalf("license %q was discarded by the graph merge:\n%s", want, out) + } + } +} + +// TestDuplicatePURLSupplierContactsAreUnioned exercises the graph-level merge +// for contacts specifically. Contacts are serialized as maps, so the string +// union used for supplier URLs would silently discard every one of them. +func TestDuplicatePURLSupplierContactsAreUnioned(t *testing.T) { + doc := &Document{ + Components: []Component{ + { + ID: "first", Name: "a", Version: "1.0.0", PURL: "pkg:npm/a@1.0.0", + Supplier: "Acme", + SupplierContacts: []Contact{{Name: "First Contact", Email: "first@acme.example"}}, + }, + { + ID: "second", Name: "a", Version: "1.0.0", PURL: "pkg:npm/a@1.0.0", + Supplier: "Acme", + SupplierContacts: []Contact{{Name: "Second Contact", Email: "second@acme.example"}}, + }, + }, + Dependencies: []Dependency{{Ref: "first", DependsOn: []string{"second"}}}, + } + + graph, err := ToGraph(doc) + if err != nil { + t.Fatalf("to graph: %v", err) + } + out, err := MarshalDepGraphJSON(graph, TargetCycloneDX17JSON, BuildOptions{}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + for _, want := range []string{"First Contact", "Second Contact"} { + if !strings.Contains(string(out), want) { + t.Fatalf("contact %q was discarded by the graph merge:\n%s", want, out) + } + } +} diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 33913e80..12042e6e 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -412,6 +412,14 @@ func spdxChecksumAlgorithm(algorithm string) common.ChecksumAlgorithm { return common.SHA384 case "sha512", "sha-512": return common.SHA512 + case "adler32": + return common.ADLER32 + case "md2": + return common.MD2 + case "md4": + return common.MD4 + case "md6": + return common.MD6 case "blake3": return common.BLAKE3 case "blake2b-256": @@ -525,15 +533,17 @@ const spdxSourceInfoPrefix = "Source repository: " // would read it as the artifact's origin, which is false. NOASSERTION is the // honest answer when only a registry root is known. func spdxDownloadLocation(component Component) string { - if location := firstNonEmpty(component.ArtifactURL, component.VCSURL); location != "" { - return location - } // SPDX separates "there is no download location" from "no claim was - // made". Collapsing NONE to NOASSERTION would weaken an explicit - // assertion the source document made. + // made". The explicit NONE assertion outranks a recovered locator: a + // package can record a source repository in PackageSourceInfo while + // still declaring it is not downloadable, and letting that repository + // win would silently replace the stronger claim. if component.NoDownloadLocation { return "NONE" } + if location := firstNonEmpty(component.ArtifactURL, component.VCSURL); location != "" { + return location + } return "NOASSERTION" } diff --git a/internal/sbom/transform.go b/internal/sbom/transform.go index 09a9fbd4..fe73d718 100644 --- a/internal/sbom/transform.go +++ b/internal/sbom/transform.go @@ -235,6 +235,9 @@ var digestHexSizes = map[string]int{ // Every algorithm the encoders accept needs an entry here, or an ingested // value of the wrong length passes validation unchecked. "blake3": 32, + "adler32": 4, + "md2": 16, + "md4": 16, "streebog-256": 32, "streebog-512": 64, "blake2b-256": 32, @@ -259,6 +262,14 @@ func normalizeDigestValue(algorithm, value string) string { return value } +// variableLengthDigests are algorithms with no single digest width, so a +// length check is not available for them. They are still hex-validated; the +// set is explicit so the encoder-coverage invariant can tell "deliberately +// unmeasurable" apart from "forgotten". +var variableLengthDigests = map[string]struct{}{ + "md6": {}, +} + // ingestedDigest normalizes and validates a digest taken from an untrusted // SBOM, returning ok=false when the value cannot be a digest of the stated // algorithm. From 679223fba4f602fe541c1b5e4f00e9e9ef87ee38 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 13:04:32 -0700 Subject: [PATCH 16/23] fix(sbom): parse URLs before splitting revisions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two P1 credential paths, both introduced by my own earlier fixes, both the same root cause: "@" in a URL is ambiguous between userinfo and a revision suffix, and I resolved it inconsistently in two places. validatedVCSLocator split on "@" before parsing. For "git+https://ghp_secret@github.com" that reads the secret as the host and "github.com" as a revision, so url.Parse sees no userinfo, the revision passes isSafeRevision, and the original credential is rebuilt and republished. Splitting before parsing was exactly the wrong order. normalizeVCS validated only revisions recovered from a query or fragment. An already-rendered "@" sits in the URL path, where url.Parse leaves it alone, so a token survived on the two paths that reach normalizeVCS without going through validatedVCSLocator — a CycloneDX vcs reference and an SPDX downloadLocation. Both now use one splitVCSRevision helper that parses first, rejects userinfo, and only then separates a trailing revision. Each fix was mutation-checked by restoring the old ordering. Also: - swiftpm records its resolved commit under "revision", not "source_revision", so a reproducible pin exported as a moving repository. Both keys are read now. - Reference digests were carried on the distribution and default branches but not on extra vcs references. Three test defects, all mine, all the same shape — a test proving less than its name claims: - The reference-hash assertion matched the value as a substring, which would still pass if the algorithm were relabelled. It now decodes and asserts the algorithm too. - TestDuplicatePURLVulnerabilitiesAndLicensesAreUnioned set no vulnerabilities. Adding them would not have helped: ToGraph does not carry vulnerabilities, so they cannot survive that path whatever the merge does. The test is renamed to what it actually covers, and the vulnerability union is tested directly against mergeComponentAssertions along with every other set. - The trailing-backslash CPE case turned out to be genuinely covered already; I verified by mutation rather than assuming either way. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 1 + internal/sbom/distribution_test.go | 27 ++++++ internal/sbom/locator.go | 68 ++++++++++---- internal/sbom/roundtrip_test.go | 139 +++++++++++++++++++++++++++-- 4 files changed, 210 insertions(+), 25 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 99e40382..79647cd5 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -458,6 +458,7 @@ func componentFromCycloneDX(comp cdx.Component) Component { Type: string(ref.Type), URL: vcs, Comment: ref.Comment, + Digests: referenceDigests(ref.Hashes), }) default: if !isPublishableReferenceURL(ref.URL) { diff --git a/internal/sbom/distribution_test.go b/internal/sbom/distribution_test.go index afa64b81..3a6d1f72 100644 --- a/internal/sbom/distribution_test.go +++ b/internal/sbom/distribution_test.go @@ -489,3 +489,30 @@ func TestNormalizedAlgorithmsHaveLengthEntries(t *testing.T) { } } } + +// TestSwiftPMRevisionKeyIsRead covers the detector that records its resolved +// commit under "revision" rather than "source_revision". Reading only the +// latter exported a reproducible SwiftPM pin as a moving repository. +func TestSwiftPMRevisionKeyIsRead(t *testing.T) { + g := sdk.New() + node := sdk.NewDependencyWithID("pkg@1.0.0", sdk.Dependency{ + Coordinates: sdk.Coordinates{ + Name: "swift-nio", Version: "2.0.0", + PURL: "pkg:swift/github.com/apple/swift-nio@2.0.0", Ecosystem: sdk.EcosystemSwift, + }, + Source: sdk.DependencySourceGit, + ResolvedURL: "https://github.com/apple/swift-nio", + Metadata: map[string]any{"revision": "9f8e7d6c5b4a"}, + }) + if err := g.AddNode(node); err != nil { + t.Fatalf("add node: %v", err) + } + + out, err := MarshalDepGraphJSON(g, TargetSPDX23JSON, BuildOptions{}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(out), "git+https://github.com/apple/swift-nio@9f8e7d6c5b4a") { + t.Fatalf("swiftpm resolved revision was not pinned:\n%s", out) + } +} diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index 962b0eaa..c7c96223 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -46,16 +46,20 @@ var artifactExtensions = []string{ // metadataKeySourceRevision is the Dependency.Metadata key several detectors // (ruby, pub, and the python family) use to record the commit a git // dependency resolved to, separately from the repository URL. -const metadataKeySourceRevision = "source_revision" +// Detectors do not agree on one key: ruby, pub, and the python family write +// "source_revision", while swiftpm writes "revision". +var metadataRevisionKeys = []string{"source_revision", "revision"} // sourceRevisionFrom returns a detector-recorded resolved commit, or "". func sourceRevisionFrom(metadata map[string]any) string { - revision, _ := metadata[metadataKeySourceRevision].(string) - revision = strings.TrimSpace(revision) - if !isSafeRevision(revision) { - return "" + for _, key := range metadataRevisionKeys { + revision, _ := metadata[key].(string) + revision = strings.TrimSpace(revision) + if isSafeRevision(revision) { + return revision + } } - return revision + return "" } // pinLocator attaches a detector-recorded revision to a VCS locator that does @@ -302,10 +306,22 @@ func normalizeVCS(parsed *url.URL) Locator { // token such as "#ghp_abcd1234" passes isSafeRevision and would then be // republished after the "@". A query value is held to the looser rule // because its key names it a revision. + // An already-rendered "@" lives in the path, where url.Parse + // leaves it untouched. Splitting it off here is what stops a token from + // riding through on a value that was normalized once already: this + // function is reached from paths that never call validatedVCSLocator. + parsed, pathRevision, ok := splitVCSRevision(parsed.String()) + if !ok { + return Locator{} + } + revision := strings.TrimSpace(parsed.Fragment) if !isCommitFragment(revision) { revision = "" } + if revision == "" { + revision = pathRevision + } if revision == "" { for _, key := range []string{"rev", "tag", "branch"} { if value := strings.TrimSpace(parsed.Query().Get(key)); value != "" { @@ -513,6 +529,31 @@ func classifyAssertedDownloadLocation(raw string) Locator { return locator } +// splitVCSRevision parses a version-control URL and separates any trailing +// "@" that is already rendered into its path. +// +// Parsing has to happen first. "@" before the host is userinfo — a credential +// — while "@" after the path is a revision, and the two are only +// distinguishable once the URL is parsed. Splitting on "@" beforehand reads +// "https://ghp_secret@github.com" as host "ghp_secret" with revision +// "github.com", which passes every later check and reconstructs the original +// credential. +// +// ok is false when the URL is unsafe to publish at all. +func splitVCSRevision(raw string) (parsed *url.URL, revision string, ok bool) { + parsed, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || parsed.User != nil || parsed.Host == "" { + return nil, "", false + } + if idx := strings.LastIndex(parsed.Path, "@"); idx >= 0 { + revision = parsed.Path[idx+1:] + clean := *parsed + clean.Path = parsed.Path[:idx] + parsed = &clean + } + return parsed, revision, true +} + // validatedVCSLocator checks an already-rendered "git+://…" // locator and returns it unchanged when it is safe to republish, or "". // @@ -527,17 +568,8 @@ func validatedVCSLocator(locator string) string { if !strings.HasPrefix(locator, "git+") { return "" } - base := strings.TrimPrefix(locator, "git+") - - revision := "" - if scheme, rest, ok := strings.Cut(base, "://"); ok { - if path, suffix, found := strings.Cut(rest, "@"); found { - base, revision = scheme+"://"+path, suffix - } - } - - parsed, err := url.Parse(base) - if err != nil || parsed.User != nil || parsed.Host == "" { + parsed, revision, ok := splitVCSRevision(strings.TrimPrefix(locator, "git+")) + if !ok { return "" } switch strings.ToLower(parsed.Scheme) { @@ -549,7 +581,7 @@ func validatedVCSLocator(locator string) string { return "" } - out := "git+" + base + out := "git+" + strings.TrimSuffix(parsed.String(), "/") if isSafeRevision(revision) { out += "@" + revision } diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 6451981b..a4e643e7 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -1267,6 +1267,12 @@ func TestVCSLocatorRevisionIsValidated(t *testing.T) { {"git+https://github.com/org/repo@ghp_abcd1234", "git+https://github.com/org/repo"}, {"git+https://github.com/org/repo@glpat-Abc123", "git+https://github.com/org/repo"}, {"git+https://tok:s3cret@github.com/org/repo", ""}, + // Userinfo with no path: splitting on "@" before parsing read + // "ghp_secret" as the host and "github.com" as a revision, then + // rebuilt the original credential. + {"git+https://ghp_secret@github.com", ""}, + {"git+https://ghp_secret@github.com/org/repo", ""}, + {"git+https://user@github.com/org/repo", ""}, {"git+file:///Users/victim/repo", ""}, {"https://github.com/org/repo", ""}, {"", ""}, @@ -1352,16 +1358,41 @@ func TestSupplierContactsAndReferenceHashesSurvive(t *testing.T) { }] }` - out := string(ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON)) + raw := ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON) for name, want := range map[string]string{ - "contact name": "Security Team", - "contact email": "security@acme.example", - "reference hash": hash, + "contact name": "Security Team", + "contact email": "security@acme.example", } { - if !strings.Contains(out, want) { - t.Fatalf("%s was dropped:\n%s", name, out) + if !strings.Contains(string(raw), want) { + t.Fatalf("%s was dropped:\n%s", name, raw) } } + + // Assert the reference hash structurally. A substring match on the value + // would still pass if the algorithm were relabelled underneath it. + var bom cdx.BOM + if err := json.Unmarshal(raw, &bom); err != nil { + t.Fatalf("unmarshal: %v", err) + } + comp := (*bom.Components)[0] + if comp.ExternalReferences == nil { + t.Fatalf("no external references in output:\n%s", raw) + } + found := false + for _, ref := range *comp.ExternalReferences { + if ref.Type != cdx.ERTypeDocumentation || ref.Hashes == nil { + continue + } + for _, h := range *ref.Hashes { + if h.Algorithm != cdx.HashAlgoSHA256 || h.Value != hash { + t.Fatalf("reference hash = %+v, want SHA-256 with the asserted value", h) + } + found = true + } + } + if !found { + t.Fatalf("the reference integrity assertion was dropped:\n%s", raw) + } } // TestBenignQueryOnGeneralReferenceSurvives covers references other than @@ -1427,7 +1458,11 @@ func TestNoneDownloadLocationOutranksRecoveredRepository(t *testing.T) { // TestDuplicatePURLVulnerabilitiesAndLicensesAreUnioned closes the last two // set-valued fields at the graph level. -func TestDuplicatePURLVulnerabilitiesAndLicensesAreUnioned(t *testing.T) { +// Only licenses are exercised here. Vulnerabilities are not carried by +// ToGraph, so they cannot survive this path whatever the merge does; +// TestMergeComponentAssertionsUnionsEverySet covers them at the helper level +// instead of putting fixture data here that could never round-trip. +func TestDuplicatePURLLicensesAreUnioned(t *testing.T) { doc := &Document{ Components: []Component{ { @@ -1491,3 +1526,93 @@ func TestDuplicatePURLSupplierContactsAreUnioned(t *testing.T) { } } } + +// TestRenderedVCSRevisionIsValidatedOnEveryPath covers the paths that reach +// normalizeVCS without going through validatedVCSLocator: a CycloneDX vcs +// reference and an SPDX downloadLocation. An already-rendered "@" +// sits in the URL path, where url.Parse leaves it untouched, so it has to be +// split off and checked there too. +func TestRenderedVCSRevisionIsValidatedOnEveryPath(t *testing.T) { + cdxIn := `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", "type": "library", "name": "a", "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "externalReferences": [{"type": "vcs", "url": "git+https://github.com/org/repo@ghp_abcd1234"}] + }] + }` + + spdxIn := `{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [{ + "name": "a", "SPDXID": "SPDXRef-a", "versionInfo": "1.0.0", + "downloadLocation": "git+https://github.com/org/repo@ghp_abcd1234", + "filesAnalyzed": false + }] + }` + + for name, in := range map[string]string{"cyclonedx-vcs-ref": cdxIn, "spdx-download-location": spdxIn} { + for _, target := range []Target{TargetSPDX23JSON, TargetCycloneDX17JSON} { + out := string(ingestAndReexport(t, []byte(in), target)) + if strings.Contains(out, "ghp_abcd1234") { + t.Fatalf("%s -> %s republished a token from a rendered revision:\n%s", name, target, out) + } + if !strings.Contains(out, "github.com/org/repo") { + t.Fatalf("%s -> %s dropped the repository along with the token:\n%s", name, target, out) + } + } + } +} + +// TestMergeComponentAssertionsUnionsEverySet covers the merge helper directly, +// including vulnerabilities, which no ingest path currently carries — the +// union is part of the helper's contract even where an end-to-end fixture +// cannot reach it. +func TestMergeComponentAssertionsUnionsEverySet(t *testing.T) { + dst := Component{ + Licenses: []License{{Value: "MIT", SPDXExpression: "MIT"}}, + CPEs: []string{"cpe:2.3:a:v:p:1.0:*:*:*:*:*:*:*"}, + Digests: []Digest{{Algorithm: "sha256", Value: strings.Repeat("a", 64)}}, + Vulnerabilities: []Vulnerability{{ID: "CVE-2024-0001", Source: "osv"}}, + SupplierURLs: []string{"https://first.example"}, + SupplierContacts: []Contact{{Name: "First"}}, + ExternalRefs: []ExternalRef{{Type: "website", URL: "https://site.example"}}, + } + src := Component{ + Licenses: []License{{Value: "Apache-2.0", SPDXExpression: "Apache-2.0"}}, + CPEs: []string{"cpe:2.3:a:v:p:2.0:*:*:*:*:*:*:*"}, + Digests: []Digest{{Algorithm: "sha256", Value: strings.Repeat("b", 64)}}, + Vulnerabilities: []Vulnerability{{ID: "CVE-2024-0002", Source: "osv"}}, + SupplierURLs: []string{"https://second.example"}, + SupplierContacts: []Contact{{Name: "Second"}}, + ExternalRefs: []ExternalRef{{Type: "documentation", URL: "https://docs.example"}}, + } + + mergeComponentAssertions(&dst, src) + + for name, got := range map[string]int{ + "licenses": len(dst.Licenses), + "cpes": len(dst.CPEs), + "digests": len(dst.Digests), + "vulnerabilities": len(dst.Vulnerabilities), + "supplier urls": len(dst.SupplierURLs), + "contacts": len(dst.SupplierContacts), + "external refs": len(dst.ExternalRefs), + } { + if got != 2 { + t.Fatalf("%s: got %d entries after merge, want both retained", name, got) + } + } + + // Merging the same values again must not duplicate them. + mergeComponentAssertions(&dst, src) + if len(dst.Vulnerabilities) != 2 || len(dst.ExternalRefs) != 2 || len(dst.SupplierContacts) != 2 { + t.Fatalf("re-merging duplicated entries: %+v", dst) + } +} From bd5d5398056d9e3f5b7bf15e7261b328f74559b4 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 13:39:46 -0700 Subject: [PATCH 17/23] fix(sbom): close remaining credential shapes and preservation gaps Ten findings across two reviews. Two P1 credential paths, and the security-boundary decision record had gone stale. Bare token query. "?ghp_abcd1234" parses as a nameless key with an empty value, and hasCredentialQuery checked names only against a fixed list while applying the shape check to values. The shape check now runs on names too. mailto targets. The opaque body of a mailto reference is inspected by none of the userinfo, query, or revision gates, so "mailto:ghp_abcd1234" was republished verbatim. A mailto reference now has to be a plausible address. Documentation. dev-docs/ARCHITECTURE.md still said every non-VCS query is rejected and that ingested URLs use the same gate as detector-derived values. Both stopped being true when the asserted-download path gained its credential-aware exception, and a stale security-boundary record is worse than none: it invites a future change to remove the exception or, worse, extend it to detector data. The record now states the three gates separately, says plainly that the distinction is the boundary, and records the parse-before-splitting rule that reintroduced a credential leak twice. Over-strict validation dropping real data: - CPE 2.2 URI bindings were checked with the 2.3 formatted-string rules, which reject empty components and the packed edition's tildes. A genuine identifier such as "cpe:/a:hp:insight_diagnostics:7.4.0.1570::~~online~win2003~x64~" was discarded. The URI binding now has its own grammar. - svn+, hg+, and bzr+ version-control references were rejected because the prefix trim assumed Git. - A CycloneDX supplier identifying itself only by URL or contact was dropped entirely: the name is optional on an organizational entity, and the whole block was gated on it. Under-strict validation preserving bad data: - An escaped rune in a CPE component skipped the printable-ASCII check, so a backslash followed by a control byte passed. Preservation gaps: - Integrity assertions on classified distribution and vcs references were lost, because those land in scalar locator fields rather than in ExternalRefs where digests were carried. - unionExternalRefs treated a matching type and URL as a complete duplicate, discarding the later record's digests and comment. - Reference URLs were stored untrimmed although the gate validated a trimmed copy. The three security fixes were mutation-checked with the mutation asserted to have applied. Co-Authored-By: Claude Opus 5 --- dev-docs/ARCHITECTURE.md | 14 ++- internal/sbom/cyclonedx.go | 45 ++++++--- internal/sbom/graph.go | 13 +++ internal/sbom/ingest_metadata.go | 48 +++++++++ internal/sbom/locator.go | 76 ++++++++++++-- internal/sbom/model.go | 25 ++++- internal/sbom/roundtrip_test.go | 168 +++++++++++++++++++++++++++++++ internal/sbom/spdx23.go | 32 +++++- internal/sbom/transform.go | 7 +- 9 files changed, 396 insertions(+), 32 deletions(-) diff --git a/dev-docs/ARCHITECTURE.md b/dev-docs/ARCHITECTURE.md index 3f46e839..5b41ec26 100644 --- a/dev-docs/ARCHITECTURE.md +++ b/dev-docs/ARCHITECTURE.md @@ -172,12 +172,20 @@ Reachability data lives on `sdk.Vulnerability.Reachability` rather than on `Find `internal/sbom/locator.go` therefore classifies each value into artifact / VCS / registry-root / nothing before it can reach an SBOM, rather than mapping the field straight onto `PackageDownloadLocation`. Two rules are load-bearing: -1. **An http(s) scheme is required, and credentials are rejected.** This is what keeps build-machine directory layout and private-registry credentials out of a published document. Credentials travel in two places: `user:password@host` userinfo, and query or fragment values on signed and private-registry links (`?token=`, `?X-Amz-Signature=`). A benign query parameter cannot be told apart from a credential, so any query or fragment disqualifies a non-VCS locator; VCS locators are exempt only because `normalizeVCS` discards both and keeps a character-checked revision. The check is on the value, never on `Source` — uv's `path`/`editable` values arrive under non-`file` sources. +1. **A network scheme is required, and credentials are rejected.** This is what keeps build-machine directory layout and private-registry credentials out of a published document. Credentials travel in several places: `user:password@host` userinfo, query parameter names *and* values on signed and private-registry links (`?token=`, `?X-Amz-Signature=`, or a bare `?ghp_…` that parses as a nameless key), fragments, `mailto:` bodies, and the `@` suffix on a version-control locator. The check is on the value, never on `Source` — uv's `path`/`editable` values arrive under non-`file` sources. - The same gate applies to URLs read out of an **ingested** SBOM. Those are untrusted input that gets re-emitted verbatim, so a hostile or careless document could otherwise launder a `file://` path or a credential into output Bomly publishes. + **The gate is deliberately not uniform, and the distinction is the security boundary — do not collapse it.** + + - A **detector-derived** value has nothing asserting what it is, so it gets the narrowest gate: `http(s)` only, and *any* query or fragment disqualifies it. Omitting costs nothing there, and a benign parameter cannot be told apart from a credential. + - A value the **source document itself declared** to be a download location or a reference — an SPDX `downloadLocation`, a CycloneDX `distribution`, or any other external reference — is a real assertion, and discarding it wholesale loses data the producer published. Those paths accept `ftp`/`ftps` alongside `http(s)`, and reject a query only when a parameter name or value is credential-shaped (`classifyAssertedDownloadLocation`, `isPublishableReferenceURL`, both via `hasCredentialQuery`). Fragments stay rejected. + - **VCS locators** are exempt from the query gate only because `normalizeVCS` discards query and fragment outright and keeps a revision that is separately validated: a fragment must be commit-shaped hex, and a query-named or metadata-supplied revision must not carry a known credential prefix. + + Ingested URLs are untrusted input that gets re-emitted, so they are still gated — a hostile or careless document must not be able to launder a `file://` path or a credential into output Bomly publishes. The relaxation above is about *which* gate applies, never about skipping one. + + One subtlety worth stating, because getting it backwards reintroduced a credential leak twice: **parse before splitting a revision.** In a URL, `@` before the host is userinfo and `@` after the path is a revision, and the two are only distinguishable once parsed. Splitting `https://ghp_secret@github.com` on `@` first reads the secret as the host and `github.com` as the revision, which passes every later check and rebuilds the credential. `splitVCSRevision` is the single place that ordering is encoded. 2. **A registry root never becomes a download location.** `https://rubygems.org/` is schema-valid and both validators accept it, so the failure would be silent and plausible: every consumer would read it as the artifact's origin. `NOASSERTION` is the honest answer. -Unrecognized shapes degrade toward the weaker claim (registry root, then nothing) rather than toward the stronger one. `FuzzClassifyResolvedURL` asserts the safety property directly: a classified value is either empty or an absolute http(s)/`git+http(s)` URL with no userinfo. +Unrecognized shapes degrade toward the weaker claim (registry root, then nothing) rather than toward the stronger one. `FuzzClassifyResolvedURL` asserts the safety property directly: a classified value is either empty or an absolute network URL with no userinfo. ### Decision: ingested SBOM assertions ride `Dependency.Metadata` diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 79647cd5..30fb955e 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -47,11 +47,13 @@ func (c cycloneDXCodec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, e if props := cycloneDXEOLProperties(comp.EOL); len(props) > 0 { component.Properties = &props } - // CycloneDX has no person-valued supplier: the field is an - // organizational entity. An SPDX document that explicitly said - // "Person: Alice" would be recast as an organization, changing a - // compliance-relevant assertion, so that case is omitted instead. - if comp.Supplier != "" && !strings.EqualFold(comp.SupplierType, "Person") { + // Emit whenever any part of the entity survives, not only when it has + // a name — the name is optional on a CycloneDX organizational entity. + // A Person-typed supplier is still omitted: CycloneDX has no + // person-valued supplier, so an SPDX "Person: Alice" would be recast + // as an organization, changing a compliance-relevant assertion. + hasSupplier := comp.Supplier != "" || len(comp.SupplierURLs) > 0 || len(comp.SupplierContacts) > 0 + if hasSupplier && !strings.EqualFold(comp.SupplierType, "Person") { supplier := &cdx.OrganizationalEntity{Name: comp.Supplier} if len(comp.SupplierURLs) > 0 { urls := append([]string(nil), comp.SupplierURLs...) @@ -389,9 +391,14 @@ func componentFromCycloneDX(comp cdx.Component) Component { if isValidCPE(comp.CPE) { component.CPEs = []string{comp.CPE} } - if comp.Supplier != nil && comp.Supplier.Name != "" { - component.Supplier = comp.Supplier.Name - component.SupplierType = "Organization" + // The name is optional on a CycloneDX organizational entity. Gating the + // whole block on it discarded the URLs and contacts of a supplier that + // identifies itself only that way. + if comp.Supplier != nil { + if comp.Supplier.Name != "" { + component.Supplier = comp.Supplier.Name + component.SupplierType = "Organization" + } if comp.Supplier.Contact != nil { for _, c := range *comp.Supplier.Contact { if c.Name == "" && c.Email == "" && c.Phone == "" { @@ -426,7 +433,8 @@ func componentFromCycloneDX(comp cdx.Component) Component { // classified value takes it and the rest are preserved // verbatim rather than overwriting the earlier assertion. if component.ArtifactURL == "" && component.RegistryURL == "" { - applyLocatorComment(&component, classifyAssertedDownloadLocation(ref.URL), ref.Comment) + applyLocatorComment(&component, classifyAssertedDownloadLocation(ref.URL), ref.Comment, + referenceDigests(ref.Hashes)...) if component.ArtifactURL != "" || component.RegistryURL != "" { continue } @@ -436,7 +444,7 @@ func componentFromCycloneDX(comp cdx.Component) Component { } component.ExternalRefs = append(component.ExternalRefs, ExternalRef{ Type: string(ref.Type), - URL: ref.URL, + URL: strings.TrimSpace(ref.URL), Comment: ref.Comment, Digests: referenceDigests(ref.Hashes), }) @@ -452,6 +460,7 @@ func componentFromCycloneDX(comp cdx.Component) Component { } if component.VCSURL == "" { component.VCSURL, component.VCSComment = vcs, ref.Comment + component.VCSDigests = referenceDigests(ref.Hashes) continue } component.ExternalRefs = append(component.ExternalRefs, ExternalRef{ @@ -466,7 +475,7 @@ func componentFromCycloneDX(comp cdx.Component) Component { } component.ExternalRefs = append(component.ExternalRefs, ExternalRef{ Type: string(ref.Type), - URL: ref.URL, + URL: strings.TrimSpace(ref.URL), Comment: ref.Comment, Digests: referenceDigests(ref.Hashes), }) @@ -488,11 +497,15 @@ func cycloneDXComponentReferences(component Component) []cdx.ExternalReference { switch { case component.ArtifactURL != "": - refs = append(refs, cdx.ExternalReference{ + artifact := cdx.ExternalReference{ Type: cdx.ERTypeDistribution, URL: component.ArtifactURL, Comment: component.ArtifactComment, - }) + } + if hashes := cycloneDXHashes(component.ArtifactDigests); len(hashes) > 0 { + artifact.Hashes = &hashes + } + refs = append(refs, artifact) case component.RegistryURL != "": // Only explain the value when the producer said nothing: replacing a // source document's own comment could contradict what it asserted. @@ -507,7 +520,11 @@ func cycloneDXComponentReferences(component Component) []cdx.ExternalReference { // scorecard repository. Emitting both would assert the same repository // twice from two sources. if vcs := firstNonEmpty(component.VCSURL, component.Repository); vcs != "" { - refs = append(refs, cdx.ExternalReference{Type: cdx.ERTypeVCS, URL: vcs, Comment: component.VCSComment}) + emitted := cdx.ExternalReference{Type: cdx.ERTypeVCS, URL: vcs, Comment: component.VCSComment} + if hashes := cycloneDXHashes(component.VCSDigests); len(hashes) > 0 { + emitted.Hashes = &hashes + } + refs = append(refs, emitted) } for _, ref := range component.ExternalRefs { diff --git a/internal/sbom/graph.go b/internal/sbom/graph.go index e0203b35..27cbbda6 100644 --- a/internal/sbom/graph.go +++ b/internal/sbom/graph.go @@ -162,6 +162,10 @@ func mergeIngestedNode(existing, incoming *sdk.Dependency) { // External references are a set carried under one key, so a key-level // fill-gaps merge would drop the duplicate's whole list whenever the // first component had any. Union them instead. + if key == metadataKeyLocatorDigests { + existing.Metadata[key] = firstPresent(existing.Metadata[key], value) + continue + } if _, paired := locatorCommentKeys[key]; paired { // Handled atomically below so a URL never picks up another // locator's comment. @@ -253,6 +257,15 @@ func unionAnyStrings(base, extra any) any { return baseList } +// firstPresent keeps an existing value when there is one, matching the +// fill-gaps rule the locator slots themselves follow. +func firstPresent(existing, incoming any) any { + if existing != nil { + return existing + } + return incoming +} + // unionAnyRecords merges two serialized lists of maps, deduplicating on the // named fields. Contacts are maps rather than strings, so the string union // would silently discard every one of them. diff --git a/internal/sbom/ingest_metadata.go b/internal/sbom/ingest_metadata.go index 8703c7b9..a59b1809 100644 --- a/internal/sbom/ingest_metadata.go +++ b/internal/sbom/ingest_metadata.go @@ -18,6 +18,7 @@ const ( metadataKeySupplierURLs = "bomly.sbom.supplier_urls" metadataKeyNoDownload = "bomly.sbom.no_download_location" metadataKeySupplierPeople = "bomly.sbom.supplier_contacts" + metadataKeyLocatorDigests = "bomly.sbom.locator_digests" metadataKeyOriginator = "bomly.sbom.originator" metadataKeyOriginatorType = "bomly.sbom.originator_type" metadataKeyDescription = "bomly.sbom.description" @@ -83,6 +84,29 @@ func setIngestedMetadata(dep *sdk.Dependency, component Component) { } dep.Metadata[metadataKeySupplierPeople] = people } + if locators := map[string][]Digest{ + "artifact": component.ArtifactDigests, + "vcs": component.VCSDigests, + "registry": component.RegistryDigests, + }; len(locators["artifact"])+len(locators["vcs"])+len(locators["registry"]) > 0 { + encoded := map[string]any{} + for slot, digests := range locators { + if len(digests) == 0 { + continue + } + entries := make([]any, 0, len(digests)) + for _, digest := range digests { + entries = append(entries, map[string]any{ + "algorithm": digest.Algorithm, "value": digest.Value, + }) + } + encoded[slot] = entries + } + if dep.Metadata == nil { + dep.Metadata = make(map[string]any) + } + dep.Metadata[metadataKeyLocatorDigests] = encoded + } if component.NoDownloadLocation { if dep.Metadata == nil { dep.Metadata = make(map[string]any) @@ -184,6 +208,30 @@ func applyIngestedMetadata(component *Component, metadata map[string]any) { []Contact{{Name: name, Email: email, Phone: phone}}) } } + if encoded, ok := metadata[metadataKeyLocatorDigests].(map[string]any); ok { + targets := map[string]*[]Digest{ + "artifact": &component.ArtifactDigests, + "vcs": &component.VCSDigests, + "registry": &component.RegistryDigests, + } + for slot, target := range targets { + entries, ok := encoded[slot].([]any) + if !ok { + continue + } + for _, entry := range entries { + fields, ok := entry.(map[string]any) + if !ok { + continue + } + algorithm, _ := fields["algorithm"].(string) + value, _ := fields["value"].(string) + if digest, ok := ingestedDigest(algorithm, value); ok { + *target = unionComponentDigests(*target, []Digest{digest}) + } + } + } + } if none, ok := metadata[metadataKeyNoDownload].(bool); ok && none { component.NoDownloadLocation = true } diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index c7c96223..bb245eb6 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -107,6 +107,11 @@ func hasCredentialQuery(parsed *url.URL) bool { return true } } + // A bare "?ghp_abcd1234" parses as a key with an empty value, so the + // shape check has to run on names as well as values. + if looksLikeCredential(key) { + return true + } for _, value := range vals { if looksLikeCredential(value) { return true @@ -492,7 +497,11 @@ func classifyIngestedVCS(raw string) string { if raw == "" { return "" } - raw = strings.TrimPrefix(raw, "git+") + tool, rest, hadPrefix := splitVCSToolPrefix(raw) + if !hadPrefix { + tool = "git+" + } + raw = rest parsed, err := url.Parse(raw) if err != nil || parsed.User != nil || parsed.Host == "" { return "" @@ -506,7 +515,11 @@ func classifyIngestedVCS(raw string) string { default: return "" } - return normalizeVCS(parsed).URL + locator := normalizeVCS(parsed).URL + if locator == "" || tool == "git+" { + return locator + } + return tool + strings.TrimPrefix(locator, "git+") } // classifyAssertedDownloadLocation classifies a value that its source document @@ -565,10 +578,11 @@ func splitVCSRevision(raw string) (parsed *url.URL, revision string, ok bool) { // normalizeVCS does. func validatedVCSLocator(locator string) string { locator = strings.TrimSpace(locator) - if !strings.HasPrefix(locator, "git+") { + tool, rest, hadPrefix := splitVCSToolPrefix(locator) + if !hadPrefix { return "" } - parsed, revision, ok := splitVCSRevision(strings.TrimPrefix(locator, "git+")) + parsed, revision, ok := splitVCSRevision(rest) if !ok { return "" } @@ -581,7 +595,7 @@ func validatedVCSLocator(locator string) string { return "" } - out := "git+" + strings.TrimSuffix(parsed.String(), "/") + out := tool + strings.TrimSuffix(parsed.String(), "/") if isSafeRevision(revision) { out += "@" + revision } @@ -611,10 +625,16 @@ func isPublishableReferenceURL(raw string) bool { // parameters disqualify it. Fragments stay rejected: they carry no // meaning for a reference and are a common place to hide a secret. return parsed.Host != "" && parsed.Fragment == "" && !hasCredentialQuery(parsed) - case "mailto", "urn": + case "mailto": + // The opaque body is never inspected by the userinfo, query, or + // revision gates, so "mailto:ghp_abcd1234" would otherwise be + // republished verbatim. Require an actual address. + return parsed.RawQuery == "" && parsed.Fragment == "" && isEmailAddress(parsed.Opaque) + case "urn": // Opaque identifiers with no host and no filesystem reach. A `bom` // reference to "urn:uuid:..." is the common CycloneDX case. - return parsed.Opaque != "" && parsed.RawQuery == "" && parsed.Fragment == "" + return parsed.Opaque != "" && parsed.RawQuery == "" && parsed.Fragment == "" && + !looksLikeCredential(parsed.Opaque) default: // Anything else — notably file:, data:, and javascript: — stays // rejected. Denying unknown schemes is the safe default: these values @@ -624,6 +644,48 @@ func isPublishableReferenceURL(raw string) bool { } } +// isEmailAddress reports whether value is a plausible "local@domain" address. +// +// This is a safety check rather than RFC validation: its purpose is to stop an +// arbitrary opaque string — a token, a path — from being published as a +// mailto reference. +func isEmailAddress(value string) bool { + value = strings.TrimSpace(value) + if value == "" || len(value) > 320 || looksLikeCredential(value) { + return false + } + local, domain, ok := strings.Cut(value, "@") + if !ok || local == "" || !isHostname(domain) { + return false + } + for _, r := range local { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '.', r == '_', r == '-', r == '+': + default: + return false + } + } + return true +} + +// vcsToolPrefixes are the version-control tool prefixes SPDX and CycloneDX +// use ahead of a transport. Assuming every repository reference is Git would +// drop the others outright. +var vcsToolPrefixes = []string{"git+", "svn+", "hg+", "bzr+"} + +// splitVCSToolPrefix separates a recognized tool prefix from a locator, +// returning the prefix (with its "+") and the remainder. +func splitVCSToolPrefix(locator string) (prefix, rest string, ok bool) { + lowered := strings.ToLower(locator) + for _, candidate := range vcsToolPrefixes { + if strings.HasPrefix(lowered, candidate) { + return candidate, locator[len(candidate):], true + } + } + return "", locator, false +} + // isHostname reports whether value looks like a dotted DNS hostname. func isHostname(value string) bool { if !strings.Contains(value, ".") || strings.HasPrefix(value, ".") || strings.HasSuffix(value, ".") { diff --git a/internal/sbom/model.go b/internal/sbom/model.go index 363256bb..a110bec4 100644 --- a/internal/sbom/model.go +++ b/internal/sbom/model.go @@ -159,6 +159,13 @@ type Component struct { VCSComment string RegistryComment string + // Digests asserted on the reference that supplied each locator. These + // belong to the referenced artifact, not to the component, so they are + // kept apart from Component.Digests. + ArtifactDigests []Digest + VCSDigests []Digest + RegistryDigests []Digest + // Repository is a canonical "github.com/owner/repo" source repository // supplied by the OpenSSF Scorecard matcher during enrichment. Repository string @@ -218,6 +225,9 @@ func mergeComponentAssertions(dst *Component, src Component) { // discard the second set whenever the first had any. dst.CPEs = unionStrings(dst.CPEs, src.CPEs) dst.Digests = unionComponentDigests(dst.Digests, src.Digests) + dst.ArtifactDigests = unionComponentDigests(dst.ArtifactDigests, src.ArtifactDigests) + dst.VCSDigests = unionComponentDigests(dst.VCSDigests, src.VCSDigests) + dst.RegistryDigests = unionComponentDigests(dst.RegistryDigests, src.RegistryDigests) dst.Licenses = unionLicenses(dst.Licenses, src.Licenses) dst.SupplierURLs = unionStrings(dst.SupplierURLs, src.SupplierURLs) dst.SupplierContacts = unionContacts(dst.SupplierContacts, src.SupplierContacts) @@ -318,16 +328,21 @@ func unionExternalRefs(base, extra []ExternalRef) []ExternalRef { return base } type key struct{ refType, url string } - seen := make(map[key]struct{}, len(base)) - for _, ref := range base { - seen[key{ref.Type, ref.URL}] = struct{}{} + index := make(map[key]int, len(base)) + for i, ref := range base { + index[key{ref.Type, ref.URL}] = i } for _, ref := range extra { k := key{ref.Type, ref.URL} - if _, ok := seen[k]; ok { + if i, ok := index[k]; ok { + // Same reference, possibly different assertions about it. + base[i].Digests = unionComponentDigests(base[i].Digests, ref.Digests) + if base[i].Comment == "" { + base[i].Comment = ref.Comment + } continue } - seen[k] = struct{}{} + index[k] = len(base) base = append(base, ref) } return base diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index a4e643e7..6ad45de6 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -1616,3 +1616,171 @@ func TestMergeComponentAssertionsUnionsEverySet(t *testing.T) { t.Fatalf("re-merging duplicated entries: %+v", dst) } } + +// TestBareTokenQueryIsRejected covers a query with no "=": url.ParseQuery +// turns "?ghp_abcd1234" into a nameless key, so the credential-shape check has +// to run on parameter names as well as values. +func TestBareTokenQueryIsRejected(t *testing.T) { + for _, raw := range []string{ + "https://repo.example/download?ghp_abcd1234", + "https://repo.example/download?glpat-Abc123", + } { + if got := classifyAssertedDownloadLocation(raw); got.Kind != LocatorNone { + t.Fatalf("classifyAssertedDownloadLocation(%q) = %+v, want it rejected", raw, got) + } + if isPublishableReferenceURL(raw) { + t.Fatalf("isPublishableReferenceURL(%q) = true, want it rejected", raw) + } + } + // A genuinely benign nameless query is still fine. + if got := classifyAssertedDownloadLocation("https://repo.example/download?raw"); got.Kind == LocatorNone { + t.Fatal("a benign nameless query was rejected") + } +} + +// TestMailtoTargetIsValidated covers the opaque body of a mailto reference, +// which none of the userinfo, query, or revision gates inspect. +func TestMailtoTargetIsValidated(t *testing.T) { + valid := []string{"mailto:security@example.com", "mailto:first.last+tag@sub.example.org"} + for _, value := range valid { + if !isPublishableReferenceURL(value) { + t.Fatalf("isPublishableReferenceURL(%q) = false, want a real address accepted", value) + } + } + invalid := []string{ + "mailto:ghp_abcd1234", "mailto:notanaddress", "mailto:", "mailto:@example.com", + "mailto:user@", "mailto:user@nodot", + } + for _, value := range invalid { + if isPublishableReferenceURL(value) { + t.Fatalf("isPublishableReferenceURL(%q) = true, want it rejected", value) + } + } +} + +// TestNonGitVCSReferencesSurvive covers the other version-control tools. The +// "git+" trim did not apply to them, so their scheme was rejected outright. +func TestNonGitVCSReferencesSurvive(t *testing.T) { + for _, locator := range []string{ + "svn+https://svn.example.org/project", + "hg+https://hg.example.org/project", + "bzr+https://bzr.example.org/project", + } { + if got := validatedVCSLocator(locator); got == "" { + t.Fatalf("validatedVCSLocator(%q) = \"\", want the repository preserved", locator) + } + } + // The safety gate still applies to them. + if got := validatedVCSLocator("svn+https://tok:s3cret@svn.example.org/p"); got != "" { + t.Fatalf("credential-bearing svn locator accepted: %q", got) + } +} + +// TestNamelessSupplierEntitySurvives covers a CycloneDX organizational entity +// that identifies itself only by URL or contact. The name is optional there, +// so gating the whole entity on it discarded compliance-relevant assertions. +func TestNamelessSupplierEntitySurvives(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", "type": "library", "name": "a", "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "supplier": { + "url": ["https://supplier.example.com"], + "contact": [{"name": "Security Team", "email": "security@supplier.example.com"}] + } + }] + }` + + out := string(ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON)) + for _, want := range []string{"https://supplier.example.com", "Security Team", "security@supplier.example.com"} { + if !strings.Contains(out, want) { + t.Fatalf("nameless supplier lost %q:\n%s", want, out) + } + } +} + +// TestURIBoundCPEUsesItsOwnGrammar covers CPE 2.2, whose components may be +// empty and whose edition field packs five values with "~" separators. +// Applying the 2.3 formatted-string rules rejected genuine identifiers. +func TestURIBoundCPEUsesItsOwnGrammar(t *testing.T) { + valid := []string{ + "cpe:/a:hp:insight_diagnostics:7.4.0.1570::~~online~win2003~x64~", + "cpe:/a:apache:log4j:2.14.1", + "cpe:/o:linux:linux_kernel", + "cpe:/a:vendor:product::update", + } + for _, value := range valid { + if !isValidCPE(value) { + t.Fatalf("isValidCPE(%q) = false, want a valid URI-bound CPE accepted", value) + } + } + for _, value := range []string{"cpe:/a:vendor with space:product", "cpe:/z:vendor:product"} { + if isValidCPE(value) { + t.Fatalf("isValidCPE(%q) = true, want it rejected", value) + } + } +} + +// TestEscapedControlInCPEIsRejected covers the escape branch, which skipped +// the printable-ASCII check for whatever followed a backslash. +func TestEscapedControlInCPEIsRejected(t *testing.T) { + for _, value := range []string{ + "cpe:2.3:a:vendor:pro\\\x00duct:1.0:*:*:*:*:*:*:*", + "cpe:2.3:a:vendor:pro\\ duct:1.0:*:*:*:*:*:*:*", + "cpe:2.3:a:vendor:pro\\é:1.0:*:*:*:*:*:*:*", + } { + if isValidCPE(value) { + t.Fatalf("isValidCPE(%q) = true, want an escaped non-printable rejected", value) + } + } + // A legitimately escaped delimiter still works. + if !isValidCPE(`cpe:2.3:a:ven\:dor:product:1.0:*:*:*:*:*:*:*`) { + t.Fatal("a legitimately escaped colon was rejected") + } +} + +// TestClassifiedLocatorHashesSurvive covers integrity assertions attached to a +// distribution or vcs reference. Those land in scalar locator fields, so they +// bypassed the path where reference digests were preserved. +func TestClassifiedLocatorHashesSurvive(t *testing.T) { + hash := strings.Repeat("f", 64) + in := `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", "type": "library", "name": "a", "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "externalReferences": [ + {"type": "distribution", "url": "https://reg.example/a-1.0.0.tgz", + "hashes": [{"alg": "SHA-256", "content": "` + hash + `"}]} + ] + }] + }` + + raw := ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON) + var bom cdx.BOM + if err := json.Unmarshal(raw, &bom); err != nil { + t.Fatalf("unmarshal: %v", err) + } + comp := (*bom.Components)[0] + if comp.ExternalReferences == nil { + t.Fatalf("no external references:\n%s", raw) + } + for _, ref := range *comp.ExternalReferences { + if ref.Type != cdx.ERTypeDistribution { + continue + } + if ref.Hashes == nil || len(*ref.Hashes) != 1 { + t.Fatalf("distribution reference lost its integrity assertion: %+v", ref) + } + if h := (*ref.Hashes)[0]; h.Algorithm != cdx.HashAlgoSHA256 || h.Value != hash { + t.Fatalf("distribution hash = %+v, want SHA-256 with the asserted value", h) + } + return + } + t.Fatalf("distribution reference missing:\n%s", raw) +} diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 12042e6e..2e18658b 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -704,8 +704,12 @@ func isValidCPE(value string) bool { if !isCPEPart(parts[0]) { return false } + // The URI binding has its own grammar: components may be empty, and + // the edition field packs five values with "~" separators + // ("~~online~win2003~x64~"). Applying the formatted-string rules here + // rejected genuine identifiers. for _, part := range parts[1:] { - if !isCPEComponent(part) { + if !isCPEURIComponent(part) { return false } } @@ -729,6 +733,26 @@ func isCPEPart(part string) bool { } } +// isCPEURIComponent reports whether a component of a CPE 2.2 URI binding is +// well formed. +// +// Unlike the formatted string, an empty component is legal here — it means +// "unspecified" — values are percent-encoded rather than backslash-escaped, +// and "~" is meaningful as the packed-edition separator. The check is +// therefore a character gate: printable ASCII, no whitespace, and none of the +// delimiters that would change the parse. +func isCPEURIComponent(value string) bool { + for _, r := range value { + switch { + case r < '!' || r > '~': + return false + case r == ':' || r == '/' || r == '?' || r == '#' || r == '[' || r == ']' || r == '@': + return false + } + } + return true +} + // isCPEComponent reports whether a CPE attribute value is well formed. // // The logical values "*" and "-" stand alone. Otherwise the value is a quoted @@ -751,6 +775,12 @@ func isCPEComponent(value string) bool { for _, r := range value { switch { case escaped: + // An escape must introduce a printable ASCII character. Without + // this, "\" followed by a control byte or a non-ASCII rune + // skipped the check below entirely. + if r < '!' || r > '~' { + return false + } escaped = false case r == '\\': escaped = true diff --git a/internal/sbom/transform.go b/internal/sbom/transform.go index fe73d718..e022a917 100644 --- a/internal/sbom/transform.go +++ b/internal/sbom/transform.go @@ -394,15 +394,18 @@ func applyLocator(component *Component, locator Locator) { } // applyLocatorComment is applyLocator for an ingested reference, carrying the -// producer's own comment alongside the classified URL. -func applyLocatorComment(component *Component, locator Locator, comment string) { +// producer's own comment and integrity assertion alongside the classified URL. +func applyLocatorComment(component *Component, locator Locator, comment string, digests ...Digest) { switch locator.Kind { case LocatorArtifact: component.ArtifactURL, component.ArtifactComment = locator.URL, comment + component.ArtifactDigests = digests case LocatorVCS: component.VCSURL, component.VCSComment = locator.URL, comment + component.VCSDigests = digests case LocatorRegistryRoot: component.RegistryURL, component.RegistryComment = locator.URL, comment + component.RegistryDigests = digests } } From 0b2830ef4278b980ef78d92ea2faedb64c6c89ec Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 14:31:46 -0700 Subject: [PATCH 18/23] fix(sbom): stop promoting marked registry roots on re-ingest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worst of this batch is a regression against the PR's own core guarantee, reached through Bomly's own output. Bomly exports a registry-backed dependency as a CycloneDX distribution reference marked "Registry root; not the exact artifact location". Re-ingesting that document promoted every registry-shaped URL to an artifact, so an SPDX export published https://rubygems.org/ as the exact downloadLocation for 130 packages — precisely the failure the classifier exists to prevent. CycloneDX defines `distribution` as where the artifact can be obtained, so promotion is faithful to the spec and the marked case is the exception. The marker is now a shared constant that ingest believes over the URL's path shape. Verified end to end: the ruby round trip returns 136 NOASSERTION with the marker preserved, while npm keeps its 136 real download locations. Second P1: a URN's payload is segmented, so "urn:uuid:ghp_abcd1234" has opaque "uuid:ghp_abcd1234" and a prefix check on the whole string never saw the token. Each segment is inspected now. Merges that could fabricate an assertion: - Locator URL, comment, and digests were merged as independent fields, so a surviving URL could take another URL's comment and both URLs' hashes — a false integrity assertion. Each locator now merges as one record, and the losing locator is preserved as an external reference rather than dropped. - The three locator digest sets ride under one metadata key, and the whole map was kept whenever any slot existed, so a duplicate's hashes were lost for slots the first component never filled. Merged per slot. Also: URI-bound CPEs now require well-formed percent escapes; the SPDX source-info recovery branch recognizes every VCS tool prefix rather than only git+; an ingested VCS assertion outranks the Scorecard repository, matching the documented precedence; and a host-only Scorecard URL is rejected, since it names no repository. The registry-root marker and the atomic locator merge were mutation-checked. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 20 +++++- internal/sbom/graph.go | 30 ++++++-- internal/sbom/locator.go | 38 +++++++++- internal/sbom/model.go | 52 ++++++++++---- internal/sbom/roundtrip_test.go | 122 ++++++++++++++++++++++++++++++++ internal/sbom/spdx23.go | 28 +++++++- 6 files changed, 263 insertions(+), 27 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 30fb955e..26294745 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -375,6 +375,14 @@ func referenceDigests(hashes *[]cdx.Hash) []Digest { // componentFromCycloneDX projects one CycloneDX component onto the neutral // model. Both decode paths (the component inventory and the metadata-only // fallback) share it so they cannot drift apart. +// registryRootMarker is the comment Bomly attaches to a distribution +// reference that names a registry rather than an exact artifact. Recognizing +// it on ingest is what keeps Bomly's own round trip honest: CycloneDX defines +// `distribution` as where the artifact can be obtained, so an unmarked +// reference is promoted to a download location, and without the marker a +// re-ingested "https://rubygems.org/" would be republished as one. +const registryRootMarker = "Registry root; not the exact artifact location" + func componentFromCycloneDX(comp cdx.Component) Component { component := Component{ ID: comp.BOMRef, @@ -433,8 +441,14 @@ func componentFromCycloneDX(comp cdx.Component) Component { // classified value takes it and the rest are preserved // verbatim rather than overwriting the earlier assertion. if component.ArtifactURL == "" && component.RegistryURL == "" { - applyLocatorComment(&component, classifyAssertedDownloadLocation(ref.URL), ref.Comment, - referenceDigests(ref.Hashes)...) + locator := classifyAssertedDownloadLocation(ref.URL) + if strings.EqualFold(strings.TrimSpace(ref.Comment), registryRootMarker) && + locator.Kind == LocatorArtifact { + // The producer said this is a registry, not the exact + // artifact. Believe it rather than the path shape. + locator.Kind = LocatorRegistryRoot + } + applyLocatorComment(&component, locator, ref.Comment, referenceDigests(ref.Hashes)...) if component.ArtifactURL != "" || component.RegistryURL != "" { continue } @@ -512,7 +526,7 @@ func cycloneDXComponentReferences(component Component) []cdx.ExternalReference { refs = append(refs, cdx.ExternalReference{ Type: cdx.ERTypeDistribution, URL: component.RegistryURL, - Comment: firstNonEmpty(component.RegistryComment, "Registry root; not the exact artifact location"), + Comment: firstNonEmpty(component.RegistryComment, registryRootMarker), }) } diff --git a/internal/sbom/graph.go b/internal/sbom/graph.go index 27cbbda6..58b338de 100644 --- a/internal/sbom/graph.go +++ b/internal/sbom/graph.go @@ -163,7 +163,7 @@ func mergeIngestedNode(existing, incoming *sdk.Dependency) { // fill-gaps merge would drop the duplicate's whole list whenever the // first component had any. Union them instead. if key == metadataKeyLocatorDigests { - existing.Metadata[key] = firstPresent(existing.Metadata[key], value) + existing.Metadata[key] = mergeLocatorDigestMaps(existing.Metadata[key], value) continue } if _, paired := locatorCommentKeys[key]; paired { @@ -257,13 +257,31 @@ func unionAnyStrings(base, extra any) any { return baseList } -// firstPresent keeps an existing value when there is one, matching the -// fill-gaps rule the locator slots themselves follow. -func firstPresent(existing, incoming any) any { - if existing != nil { +// mergeLocatorDigestMaps merges the per-slot locator digest maps. +// +// The three slots ride under one key, so keeping the whole existing map +// discarded a duplicate's hashes for a slot the first component never filled +// — and mergeLocatorPairs can still adopt that duplicate's URL, which would +// then be re-emitted without its asserted hash. +func mergeLocatorDigestMaps(existing, incoming any) any { + incomingMap, ok := incoming.(map[string]any) + if !ok { return existing } - return incoming + existingMap, ok := existing.(map[string]any) + if !ok { + return incoming + } + merged := make(map[string]any, len(existingMap)+len(incomingMap)) + for slot, value := range existingMap { + merged[slot] = value + } + for slot, value := range incomingMap { + if _, present := merged[slot]; !present { + merged[slot] = value + } + } + return merged } // unionAnyRecords merges two serialized lists of maps, deduplicating on the diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index bb245eb6..c111ba89 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -460,6 +460,12 @@ func normalizeRepositoryURL(repo string) string { } switch strings.ToLower(parsed.Scheme) { case "http", "https": + // The scheme-less branch below requires an owner/repo path; + // an absolute URL has to clear the same bar or it names no + // repository at all. + if strings.Trim(parsed.Path, "/") == "" { + return "" + } return parsed.String() default: return "" @@ -535,13 +541,29 @@ func classifyIngestedVCS(raw string) string { // safety gate — scheme, host, userinfo, fragment — applies unchanged, so local // paths and secrets are still dropped. func classifyAssertedDownloadLocation(raw string) Locator { - locator := classifyURL(raw, "", "", true) + locator := classifyAssertedReference(raw) if locator.Kind == LocatorRegistryRoot { locator.Kind = LocatorArtifact } return locator } +// classifyAssertedReference classifies a URL a source document declared to be +// a reference, without claiming it is an exact download location. +// +// The distinction matters because the two formats assert different things. +// SPDX PackageDownloadLocation says "this is where the package came from", so +// promoting a registry-shaped URL there is faithful. A CycloneDX +// `distribution` external reference says only "a distribution point", which a +// registry root legitimately is — promoting it would republish +// "https://rubygems.org/" as an exact download location, which is the failure +// this whole classifier exists to prevent. Bomly's own export marks such +// references as registry roots, so promoting on ingest would corrupt its own +// round trip. +func classifyAssertedReference(raw string) Locator { + return classifyURL(raw, "", "", true) +} + // splitVCSRevision parses a version-control URL and separates any trailing // "@" that is already rendered into its path. // @@ -633,8 +655,18 @@ func isPublishableReferenceURL(raw string) bool { case "urn": // Opaque identifiers with no host and no filesystem reach. A `bom` // reference to "urn:uuid:..." is the common CycloneDX case. - return parsed.Opaque != "" && parsed.RawQuery == "" && parsed.Fragment == "" && - !looksLikeCredential(parsed.Opaque) + if parsed.Opaque == "" || parsed.RawQuery != "" || parsed.Fragment != "" { + return false + } + // "urn:uuid:ghp_abcd1234" has opaque "uuid:ghp_abcd1234", so a prefix + // check on the whole string never sees the token. Inspect each + // namespace segment. + for _, segment := range strings.Split(parsed.Opaque, ":") { + if looksLikeCredential(segment) { + return false + } + } + return true default: // Anything else — notably file:, data:, and javascript: — stays // rejected. Denying unknown schemes is the safe default: these values diff --git a/internal/sbom/model.go b/internal/sbom/model.go index a110bec4..7de86cad 100644 --- a/internal/sbom/model.go +++ b/internal/sbom/model.go @@ -146,15 +146,15 @@ type Component struct { VCSURL string RegistryURL string - // Comments a source document attached to the references above. Kept - // separately so an ingested comment is preserved rather than replaced by - // Bomly's own, which could contradict what the producer asserted. // NoDownloadLocation records SPDX's "NONE" marker, which asserts the // package is not downloadable. That is a different claim from // NOASSERTION, which says the producer made none, so the two cannot // collapse into an empty locator. NoDownloadLocation bool + // Comments a source document attached to the references above. Kept + // separately so an ingested comment is preserved rather than replaced by + // Bomly's own, which could contradict what the producer asserted. ArtifactComment string VCSComment string RegistryComment string @@ -208,12 +208,6 @@ func mergeComponentAssertions(dst *Component, src Component) { {&dst.Description, src.Description}, {&dst.Summary, src.Summary}, {&dst.Repository, src.Repository}, - {&dst.ArtifactURL, src.ArtifactURL}, - {&dst.VCSURL, src.VCSURL}, - {&dst.RegistryURL, src.RegistryURL}, - {&dst.ArtifactComment, src.ArtifactComment}, - {&dst.VCSComment, src.VCSComment}, - {&dst.RegistryComment, src.RegistryComment}, {&dst.Copyright, src.Copyright}, } { if *field.target == "" { @@ -225,9 +219,15 @@ func mergeComponentAssertions(dst *Component, src Component) { // discard the second set whenever the first had any. dst.CPEs = unionStrings(dst.CPEs, src.CPEs) dst.Digests = unionComponentDigests(dst.Digests, src.Digests) - dst.ArtifactDigests = unionComponentDigests(dst.ArtifactDigests, src.ArtifactDigests) - dst.VCSDigests = unionComponentDigests(dst.VCSDigests, src.VCSDigests) - dst.RegistryDigests = unionComponentDigests(dst.RegistryDigests, src.RegistryDigests) + // Each locator is one record: URL, its comment, and its digests. Merging + // those fields independently could attach one URL's comment or, worse, + // another URL's hashes to a different URL — a false integrity assertion. + mergeLocatorSlot(&dst.ArtifactURL, &dst.ArtifactComment, &dst.ArtifactDigests, + src.ArtifactURL, src.ArtifactComment, src.ArtifactDigests, "distribution", &dst.ExternalRefs) + mergeLocatorSlot(&dst.VCSURL, &dst.VCSComment, &dst.VCSDigests, + src.VCSURL, src.VCSComment, src.VCSDigests, "vcs", &dst.ExternalRefs) + mergeLocatorSlot(&dst.RegistryURL, &dst.RegistryComment, &dst.RegistryDigests, + src.RegistryURL, src.RegistryComment, src.RegistryDigests, "distribution", &dst.ExternalRefs) dst.Licenses = unionLicenses(dst.Licenses, src.Licenses) dst.SupplierURLs = unionStrings(dst.SupplierURLs, src.SupplierURLs) dst.SupplierContacts = unionContacts(dst.SupplierContacts, src.SupplierContacts) @@ -282,6 +282,34 @@ func unionVulnerabilities(base, extra []Vulnerability) []Vulnerability { return base } +// mergeLocatorSlot folds one classified locator into another as a unit. +// +// An empty slot takes the incoming record whole. A slot that already holds a +// different URL keeps its own and preserves the incoming one as an external +// reference, so neither assertion is lost and neither borrows the other's +// comment or hashes. The same URL merges its digests. +func mergeLocatorSlot( + url, comment *string, digests *[]Digest, + srcURL, srcComment string, srcDigests []Digest, + refType string, refs *[]ExternalRef, +) { + switch { + case srcURL == "": + return + case *url == "": + *url, *comment, *digests = srcURL, srcComment, srcDigests + case *url == srcURL: + *digests = unionComponentDigests(*digests, srcDigests) + if *comment == "" { + *comment = srcComment + } + default: + *refs = unionExternalRefs(*refs, []ExternalRef{{ + Type: refType, URL: srcURL, Comment: srcComment, Digests: srcDigests, + }}) + } +} + // unionLicenses appends licenses from extra that base does not carry, keyed // by the normalized expression. func unionLicenses(base, extra []License) []License { diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 6ad45de6..2627bb9f 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -1784,3 +1784,125 @@ func TestClassifiedLocatorHashesSurvive(t *testing.T) { } t.Fatalf("distribution reference missing:\n%s", raw) } + +// TestRegistryRootSurvivesBomlyRoundTrip is the regression test for the +// sharpest failure this classifier exists to prevent, reached through Bomly's +// own output: a registry root republished as an exact download location. +// +// CycloneDX defines `distribution` as where the artifact can be obtained, so +// an unmarked reference is promoted on ingest. Bomly marks its own +// registry-root references, and that marker has to be believed over the URL's +// path shape. +func TestRegistryRootSurvivesBomlyRoundTrip(t *testing.T) { + in := `{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "version": 1, + "components": [ + {"bom-ref": "pkg:gem/a@1.0.0", "type": "library", "name": "a", "version": "1.0.0", + "purl": "pkg:gem/a@1.0.0", + "externalReferences": [ + {"type": "distribution", "url": "https://rubygems.org/", "comment": "` + registryRootMarker + `"}]}, + {"bom-ref": "pkg:npm/b@1.0.0", "type": "library", "name": "b", "version": "1.0.0", + "purl": "pkg:npm/b@1.0.0", + "externalReferences": [ + {"type": "distribution", "url": "https://registry.npmjs.org/b/-/b-1.0.0.tgz"}]} + ] + }` + + out := ingestAndReexport(t, []byte(in), TargetSPDX23JSON) + var doc v23.Document + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, p := range doc.Packages { + if p == nil { + continue + } + switch p.PackageName { + case "a": + if p.PackageDownloadLocation != "NOASSERTION" { + t.Fatalf("a marked registry root became a download location: %q", p.PackageDownloadLocation) + } + case "b": + if p.PackageDownloadLocation != "https://registry.npmjs.org/b/-/b-1.0.0.tgz" { + t.Fatalf("an unmarked exact artifact was not promoted: %q", p.PackageDownloadLocation) + } + } + } +} + +// TestLocatorSlotsMergeAtomically covers the merge of two copies of one +// component that each assert a different distribution URL with its own hash. +// Merging URL, comment, and digests independently would attach one URL's hash +// to the other — a false integrity assertion. +func TestLocatorSlotsMergeAtomically(t *testing.T) { + first := strings.Repeat("a", 64) + second := strings.Repeat("b", 64) + + dst := Component{ + ArtifactURL: "https://primary.example/a-1.0.0.tgz", + ArtifactDigests: []Digest{{Algorithm: "sha256", Value: first}}, + } + src := Component{ + ArtifactURL: "https://mirror.example/a-1.0.0.tgz", + ArtifactComment: "mirror", + ArtifactDigests: []Digest{{Algorithm: "sha256", Value: second}}, + } + + mergeComponentAssertions(&dst, src) + + if len(dst.ArtifactDigests) != 1 || dst.ArtifactDigests[0].Value != first { + t.Fatalf("the surviving URL picked up the other URL's hashes: %+v", dst.ArtifactDigests) + } + if dst.ArtifactComment != "" { + t.Fatalf("the surviving URL picked up the other URL's comment: %q", dst.ArtifactComment) + } + // The losing locator is preserved rather than dropped. + var kept *ExternalRef + for i := range dst.ExternalRefs { + if dst.ExternalRefs[i].URL == "https://mirror.example/a-1.0.0.tgz" { + kept = &dst.ExternalRefs[i] + } + } + if kept == nil { + t.Fatalf("the losing locator was discarded: %+v", dst.ExternalRefs) + } + if kept.Comment != "mirror" || len(kept.Digests) != 1 || kept.Digests[0].Value != second { + t.Fatalf("the preserved locator lost its own assertions: %+v", kept) + } +} + +// TestURIBoundCPEPercentEscapes covers the URI binding's encoding rules. +func TestURIBoundCPEPercentEscapes(t *testing.T) { + if !isValidCPE("cpe:/a:vendor%3Aname:product") { + t.Fatal("a valid percent escape was rejected") + } + for _, value := range []string{"cpe:/a:vendor%ZZ:product", "cpe:/a:vendor%:product", "cpe:/a:vendor%4:product"} { + if isValidCPE(value) { + t.Fatalf("isValidCPE(%q) = true, want a malformed percent escape rejected", value) + } + } +} + +// TestNonGitSourceInfoRoundTrips covers the SPDX recovery branch, which only +// recognized "git+" while the writer can emit any tool prefix. +func TestNonGitSourceInfoRoundTrips(t *testing.T) { + repo, vcs := parseSPDXSourceInfo("Source repository: svn+https://svn.example.org/project") + if vcs != "svn+https://svn.example.org/project" || repo != "" { + t.Fatalf("parseSPDXSourceInfo dropped a non-Git locator: repo=%q vcs=%q", repo, vcs) + } +} + +// TestIngestedVCSOutranksScorecardInSourceInfo pins the precedence when an +// artifact owns the download location and both sources name a repository. +func TestIngestedVCSOutranksScorecardInSourceInfo(t *testing.T) { + got := spdxSourceInfo(Component{ + ArtifactURL: "https://reg.example/a-1.0.0.tgz", + VCSURL: "git+https://github.com/ingested/repo", + Repository: "https://github.com/scorecard/repo", + }) + if !strings.Contains(got, "ingested/repo") { + t.Fatalf("sourceInfo = %q, want the ingested assertion to outrank the matcher", got) + } +} diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 2e18658b..69ee8fca 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -513,7 +513,7 @@ func parseSPDXSourceInfo(value string) (repository, vcs string) { return "", "" } locator := strings.TrimSpace(strings.TrimPrefix(value, spdxSourceInfoPrefix)) - if strings.HasPrefix(locator, "git+") { + if _, _, isVCS := splitVCSToolPrefix(locator); isVCS { // spdxSourceInfo writes the version-control form when the artifact // owns downloadLocation, so it has to be recognized here or the // repository is lost on the way back. @@ -564,7 +564,15 @@ func spdxSourceInfo(component Component) string { if component.VCSURL != "" && component.ArtifactURL == "" { return "" } - repo := strings.TrimSpace(component.Repository) + // The ingested VCS assertion outranks the matcher-derived Scorecard + // repository, matching the ingest-before-enrichment precedence. + repo := "" + if component.ArtifactURL != "" { + repo = strings.TrimSpace(component.VCSURL) + } + if repo == "" { + repo = strings.TrimSpace(component.Repository) + } if repo == "" && component.ArtifactURL != "" { // Keep the version-control form intact. Stripping "git+" from // "git+https://host/org/repo@deadbeef" turns the revision into part @@ -742,17 +750,31 @@ func isCPEPart(part string) bool { // therefore a character gate: printable ASCII, no whitespace, and none of the // delimiters that would change the parse. func isCPEURIComponent(value string) bool { - for _, r := range value { + runes := []rune(value) + for i := 0; i < len(runes); i++ { + r := runes[i] switch { case r < '!' || r > '~': return false case r == ':' || r == '/' || r == '?' || r == '#' || r == '[' || r == ']' || r == '@': return false + case r == '%': + // The URI binding percent-encodes its specials, so a "%" that is + // not followed by two hex digits is malformed rather than literal. + if i+2 >= len(runes) || !isHexRune(runes[i+1]) || !isHexRune(runes[i+2]) { + return false + } + i += 2 } } return true } +// isHexRune reports whether r is a hexadecimal digit. +func isHexRune(r rune) bool { + return (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') +} + // isCPEComponent reports whether a CPE attribute value is well formed. // // The logical values "*" and "-" stand alone. Otherwise the value is a quoted From 4bc1fc7958998bfdac1883e4f592f8cc2339f2cc Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 14:51:43 -0700 Subject: [PATCH 19/23] fix(sbom): reject credentials in URL paths, sweep the graph merge layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1: a token in the URL path was never inspected. looksLikeCredential is applied to query names and values, revisions, mail addresses, and URN segments, but "https://repo.example/download/ghp_abcd1234/pkg.tgz" has no userinfo, no query, and no fragment, so every gate passed it through verbatim. Path segments are now decoded and checked at all four entry points. The rest of this batch is a layer sweep rather than another round of instance fixes. Three findings were the graph-level twins of merge defects I fixed at the model level last round: a conflicting locator discarded instead of preserved as a mirror, a matching locator's digests dropped, and a duplicate external reference skipped rather than merged. Fixing each report where it landed is what let the same bug persist one layer down, so the graph helpers now mirror the model helpers directly. Two more of the same shape: - The shared classifier stripped only "git+", so an SPDX downloadLocation of "svn+https://…" failed the transport gate and re-exported as NOASSERTION. It uses the shared VCS-prefix parser now, like the two callers that already did. - The registry-root encoder branch recreated its reference without hashes, though the artifact and vcs branches had been fixed. Also: SPDX sentinel comparison is case-sensitive again. The literals are NONE and NOASSERTION, so case-folding silently deleted ordinary free text such as a description of "None" from the newly preserved fields. The credential-path gate, the graph mirror preservation, and the registry-root hashes were each mutation-checked. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 8 +- internal/sbom/graph.go | 104 ++++++++++++++++++---- internal/sbom/locator.go | 37 ++++++-- internal/sbom/roundtrip_test.go | 149 ++++++++++++++++++++++++++++++++ internal/sbom/spdx23.go | 5 +- 5 files changed, 276 insertions(+), 27 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 26294745..9ab1c7b7 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -523,11 +523,15 @@ func cycloneDXComponentReferences(component Component) []cdx.ExternalReference { case component.RegistryURL != "": // Only explain the value when the producer said nothing: replacing a // source document's own comment could contradict what it asserted. - refs = append(refs, cdx.ExternalReference{ + registry := cdx.ExternalReference{ Type: cdx.ERTypeDistribution, URL: component.RegistryURL, Comment: firstNonEmpty(component.RegistryComment, registryRootMarker), - }) + } + if hashes := cycloneDXHashes(component.RegistryDigests); len(hashes) > 0 { + registry.Hashes = &hashes + } + refs = append(refs, registry) } // VCSURL is detector-supplied and version-exact, so it wins over the diff --git a/internal/sbom/graph.go b/internal/sbom/graph.go index 58b338de..9adedf74 100644 --- a/internal/sbom/graph.go +++ b/internal/sbom/graph.go @@ -204,28 +204,81 @@ var locatorCommentKeys = map[string]string{ // mergeLocatorPairs fills each empty locator slot from incoming, moving the // URL and its comment as one unit. func mergeLocatorPairs(existing, incoming *sdk.Dependency) { - for _, pair := range []struct{ url, note string }{ - {metadataKeyArtifactURL, metadataKeyArtifactNote}, - {metadataKeyVCSURL, metadataKeyVCSNote}, - {metadataKeyRegistryURL, metadataKeyRegistryNote}, + for _, pair := range []struct{ url, note, slot, refType string }{ + {metadataKeyArtifactURL, metadataKeyArtifactNote, "artifact", "distribution"}, + {metadataKeyVCSURL, metadataKeyVCSNote, "vcs", "vcs"}, + {metadataKeyRegistryURL, metadataKeyRegistryNote, "registry", "distribution"}, } { - if _, present := existing.Metadata[pair.url]; present { - continue - } - value, ok := incoming.Metadata[pair.url] - if !ok { + incomingURL, ok := incoming.Metadata[pair.url].(string) + if !ok || incomingURL == "" { continue } if existing.Metadata == nil { existing.Metadata = make(map[string]any) } - existing.Metadata[pair.url] = value - if note, ok := incoming.Metadata[pair.note]; ok { - existing.Metadata[pair.note] = note + + existingURL, present := existing.Metadata[pair.url].(string) + switch { + case !present || existingURL == "": + existing.Metadata[pair.url] = incomingURL + if note, ok := incoming.Metadata[pair.note]; ok { + existing.Metadata[pair.note] = note + } + existing.Metadata[metadataKeyLocatorDigests] = mergeLocatorDigestSlot( + existing.Metadata[metadataKeyLocatorDigests], incoming.Metadata[metadataKeyLocatorDigests], pair.slot) + case existingURL == incomingURL: + // Same locator described twice: union its integrity assertions + // rather than keeping only the first list. + existing.Metadata[metadataKeyLocatorDigests] = mergeLocatorDigestSlot( + existing.Metadata[metadataKeyLocatorDigests], incoming.Metadata[metadataKeyLocatorDigests], pair.slot) + if note, ok := existing.Metadata[pair.note].(string); !ok || note == "" { + if incomingNote, ok := incoming.Metadata[pair.note]; ok { + existing.Metadata[pair.note] = incomingNote + } + } + default: + // A different locator for the same slot is a mirror, not a + // replacement. Keep it as an external reference so the assertion + // survives without displacing the first. + note, _ := incoming.Metadata[pair.note].(string) + ref := map[string]any{"type": pair.refType, "url": incomingURL, "comment": note} + if digests := locatorDigestSlot(incoming.Metadata[metadataKeyLocatorDigests], pair.slot); digests != nil { + ref["digests"] = digests + } + existing.Metadata[metadataKeyExternalRefs] = unionExternalRefValues( + existing.Metadata[metadataKeyExternalRefs], []any{ref}) } } } +// locatorDigestSlot returns the serialized digest list for one locator slot. +func locatorDigestSlot(encoded any, slot string) any { + slots, ok := encoded.(map[string]any) + if !ok { + return nil + } + return slots[slot] +} + +// mergeLocatorDigestSlot folds one slot's digests from incoming into existing, +// leaving the other slots untouched. +func mergeLocatorDigestSlot(existing, incoming any, slot string) any { + incomingDigests := locatorDigestSlot(incoming, slot) + if incomingDigests == nil { + return existing + } + slots, ok := existing.(map[string]any) + if !ok { + slots = map[string]any{} + } + merged := make(map[string]any, len(slots)+1) + for k, v := range slots { + merged[k] = v + } + merged[slot] = unionAnyRecords(merged[slot], incomingDigests, "algorithm", "value") + return merged +} + // unionAnyStrings merges two serialized string lists, preserving order and // dropping duplicates. func unionAnyStrings(base, extra any) any { @@ -354,10 +407,10 @@ func unionExternalRefValues(base, extra any) any { return key{refType, url}, true } - seen := make(map[key]struct{}, len(baseList)) - for _, entry := range baseList { + index := make(map[key]int, len(baseList)) + for i, entry := range baseList { if k, ok := refKey(entry); ok { - seen[k] = struct{}{} + index[k] = i } } for _, entry := range extraList { @@ -365,11 +418,26 @@ func unionExternalRefValues(base, extra any) any { if !ok { continue } - if _, present := seen[k]; present { + i, present := index[k] + if !present { + index[k] = len(baseList) + baseList = append(baseList, entry) continue } - seen[k] = struct{}{} - baseList = append(baseList, entry) + // Same reference, possibly different assertions about it: mirror the + // model-level merge rather than treating it as a plain duplicate. + existing, okExisting := baseList[i].(map[string]any) + fields, okIncoming := entry.(map[string]any) + if !okExisting || !okIncoming { + continue + } + existing["digests"] = unionAnyRecords(existing["digests"], fields["digests"], "algorithm", "value") + if comment, _ := existing["comment"].(string); comment == "" { + if incoming, ok := fields["comment"].(string); ok && incoming != "" { + existing["comment"] = incoming + } + } + baseList[i] = existing } return baseList } diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index c111ba89..3ba57f85 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -89,6 +89,25 @@ var credentialQueryKeys = []string{ "private_token", "personal_token", "auth_token", "authtoken", } +// hasCredentialPath reports whether any path segment looks like a secret. +// +// looksLikeCredential is applied to query names and values, revisions, mail +// addresses, and URN segments, but a token can just as easily sit in the path: +// "https://repo.example/download/ghp_abcd1234/pkg.tgz" has no userinfo, no +// query, and no fragment, so every other gate passes it. +func hasCredentialPath(parsed *url.URL) bool { + path := parsed.Path + if decoded, err := url.PathUnescape(path); err == nil { + path = decoded + } + for _, segment := range strings.Split(path, "/") { + if looksLikeCredential(segment) { + return true + } + } + return false +} + // hasCredentialQuery reports whether a URL's query carries something that // looks like a secret, by parameter name or by value shape. func hasCredentialQuery(parsed *url.URL) bool { @@ -150,8 +169,13 @@ func classifyURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosyste raw, hint = strings.TrimPrefix(raw, "registry+"), LocatorRegistryRoot case strings.HasPrefix(raw, "sparse+"): raw, hint = strings.TrimPrefix(raw, "sparse+"), LocatorRegistryRoot - case strings.HasPrefix(raw, "git+"): - raw, hint = strings.TrimPrefix(raw, "git+"), LocatorVCS + default: + // Any recognized version-control tool prefix, not just git+: an SPDX + // downloadLocation of "svn+https://…" is a valid VCS location, and + // leaving it prefixed makes the transport gate reject it outright. + if _, rest, isVCS := splitVCSToolPrefix(raw); isVCS { + raw, hint = rest, LocatorVCS + } } parsed, err := url.Parse(raw) @@ -184,7 +208,7 @@ func classifyURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosyste // A lockfile pointing at a private registry can embed a token. Publishing // it in an SBOM would leak a live credential, so drop the value entirely // rather than try to strip the userinfo and emit the rest. - if parsed.User != nil { + if parsed.User != nil || hasCredentialPath(parsed) { return Locator{} } @@ -509,7 +533,7 @@ func classifyIngestedVCS(raw string) string { } raw = rest parsed, err := url.Parse(raw) - if err != nil || parsed.User != nil || parsed.Host == "" { + if err != nil || parsed.User != nil || parsed.Host == "" || hasCredentialPath(parsed) { return "" } // git:// is a legitimate transport for a version-control reference, and @@ -577,7 +601,7 @@ func classifyAssertedReference(raw string) Locator { // ok is false when the URL is unsafe to publish at all. func splitVCSRevision(raw string) (parsed *url.URL, revision string, ok bool) { parsed, err := url.Parse(strings.TrimSpace(raw)) - if err != nil || parsed.User != nil || parsed.Host == "" { + if err != nil || parsed.User != nil || parsed.Host == "" || hasCredentialPath(parsed) { return nil, "", false } if idx := strings.LastIndex(parsed.Path, "@"); idx >= 0 { @@ -646,7 +670,8 @@ func isPublishableReferenceURL(raw string) bool { // "?version=1" is part of the assertion and only credential-shaped // parameters disqualify it. Fragments stay rejected: they carry no // meaning for a reference and are a common place to hide a secret. - return parsed.Host != "" && parsed.Fragment == "" && !hasCredentialQuery(parsed) + return parsed.Host != "" && parsed.Fragment == "" && + !hasCredentialQuery(parsed) && !hasCredentialPath(parsed) case "mailto": // The opaque body is never inspected by the userinfo, query, or // revision gates, so "mailto:ghp_abcd1234" would otherwise be diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 2627bb9f..61051a44 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -1906,3 +1906,152 @@ func TestIngestedVCSOutranksScorecardInSourceInfo(t *testing.T) { t.Fatalf("sourceInfo = %q, want the ingested assertion to outrank the matcher", got) } } + +// TestCredentialInPathIsRejected covers a token sitting in the URL path. +// looksLikeCredential was applied to query names and values, revisions, mail +// addresses, and URN segments, but never to path segments — where a value has +// no userinfo, no query, and no fragment, so every other gate passes it. +func TestCredentialInPathIsRejected(t *testing.T) { + tokenPaths := []string{ + "https://repo.example/download/ghp_abcd1234/pkg.tgz", + "https://repo.example/glpat-Abc123/pkg.tgz", + "https://repo.example/download/%67hp_abcd1234/pkg.tgz", + } + for _, raw := range tokenPaths { + if got := classifyResolvedURL(raw, "", ""); got.Kind != LocatorNone { + t.Fatalf("classifyResolvedURL(%q) = %+v, want it rejected", raw, got) + } + if got := classifyAssertedDownloadLocation(raw); got.Kind != LocatorNone { + t.Fatalf("classifyAssertedDownloadLocation(%q) = %+v, want it rejected", raw, got) + } + if isPublishableReferenceURL(raw) { + t.Fatalf("isPublishableReferenceURL(%q) = true, want it rejected", raw) + } + } + // An ordinary package path is unaffected. + if got := classifyResolvedURL("https://registry.npmjs.org/a/-/a-1.0.0.tgz", "", ""); got.Kind == LocatorNone { + t.Fatal("an ordinary package path was rejected") + } +} + +// TestNonGitVCSDownloadLocationRoundTrips covers an SPDX downloadLocation that +// uses a non-Git tool prefix. The shared classifier recognized only "git+", so +// the remaining compound scheme failed the transport gate. +func TestNonGitVCSDownloadLocationRoundTrips(t *testing.T) { + for _, locator := range []string{ + "svn+https://svn.example.org/project", + "hg+https://hg.example.org/project", + } { + in := `{ + "spdxVersion": "SPDX-2.3", "SPDXID": "SPDXRef-DOCUMENT", "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [{"name": "a", "SPDXID": "SPDXRef-a", "versionInfo": "1.0.0", + "downloadLocation": "` + locator + `", "filesAnalyzed": false}] + }` + out := string(ingestAndReexport(t, []byte(in), TargetSPDX23JSON)) + if !strings.Contains(out, "example.org/project") { + t.Fatalf("%s was dropped on re-export:\n%s", locator, out) + } + } +} + +// TestSPDXSentinelsAreCaseSensitive covers ordinary free text that happens to +// spell a reserved marker in mixed case. +func TestSPDXSentinelsAreCaseSensitive(t *testing.T) { + for _, value := range []string{"None", "NoAssertion", "none"} { + if got := parseSPDXEntity(value); got != value { + t.Fatalf("parseSPDXEntity(%q) = %q, want the free text preserved", value, got) + } + } + for _, value := range []string{"NONE", "NOASSERTION", ""} { + if got := parseSPDXEntity(value); got != "" { + t.Fatalf("parseSPDXEntity(%q) = %q, want the sentinel treated as absent", value, got) + } + } +} + +// TestRegistryRootReferenceKeepsItsHashes covers the encoder branch that +// recreated a marked registry-root reference without its integrity assertion. +func TestRegistryRootReferenceKeepsItsHashes(t *testing.T) { + hash := strings.Repeat("e", 64) + in := `{ + "bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, + "components": [{ + "bom-ref": "pkg:gem/a@1.0.0", "type": "library", "name": "a", "version": "1.0.0", + "purl": "pkg:gem/a@1.0.0", + "externalReferences": [{ + "type": "distribution", "url": "https://rubygems.org/", + "comment": "` + registryRootMarker + `", + "hashes": [{"alg": "SHA-256", "content": "` + hash + `"}]}] + }] + }` + + raw := ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON) + var bom cdx.BOM + if err := json.Unmarshal(raw, &bom); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, ref := range *(*bom.Components)[0].ExternalReferences { + if ref.URL != "https://rubygems.org/" { + continue + } + if ref.Hashes == nil || len(*ref.Hashes) != 1 { + t.Fatalf("registry-root reference lost its hashes: %+v", ref) + } + if h := (*ref.Hashes)[0]; h.Algorithm != cdx.HashAlgoSHA256 || h.Value != hash { + t.Fatalf("registry-root hash = %+v, want SHA-256 with the asserted value", h) + } + return + } + t.Fatalf("registry-root reference missing:\n%s", raw) +} + +// TestGraphLocatorMergeMirrorsModel covers the graph layer, which had the same +// three merge defects the model layer did: a conflicting locator discarded, a +// matching locator's digests dropped, and a duplicate external reference +// skipped instead of merged. +func TestGraphLocatorMergeMirrorsModel(t *testing.T) { + first := strings.Repeat("a", 64) + second := strings.Repeat("b", 64) + + doc := &Document{ + Components: []Component{ + { + ID: "first", Name: "a", Version: "1.0.0", PURL: "pkg:npm/a@1.0.0", + ArtifactURL: "https://primary.example/a-1.0.0.tgz", + ArtifactDigests: []Digest{{Algorithm: "sha256", Value: first}}, + ExternalRefs: []ExternalRef{{Type: "documentation", URL: "https://docs.example"}}, + }, + { + ID: "second", Name: "a", Version: "1.0.0", PURL: "pkg:npm/a@1.0.0", + ArtifactURL: "https://mirror.example/a-1.0.0.tgz", + ArtifactDigests: []Digest{{Algorithm: "sha256", Value: second}}, + ExternalRefs: []ExternalRef{{ + Type: "documentation", URL: "https://docs.example", Comment: "from the second copy", + }}, + }, + }, + Dependencies: []Dependency{{Ref: "first", DependsOn: []string{"second"}}}, + } + + graph, err := ToGraph(doc) + if err != nil { + t.Fatalf("to graph: %v", err) + } + out, err := MarshalDepGraphJSON(graph, TargetCycloneDX17JSON, BuildOptions{}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + rendered := string(out) + for name, want := range map[string]string{ + "primary URL": "https://primary.example/a-1.0.0.tgz", + "mirror URL": "https://mirror.example/a-1.0.0.tgz", + "primary hash": first, + "duplicate's comment": "from the second copy", + } { + if !strings.Contains(rendered, want) { + t.Fatalf("%s was discarded by the graph merge:\n%s", name, rendered) + } + } +} diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 69ee8fca..71ca542a 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -491,7 +491,10 @@ func spdxCopyrightValue(value string) string { // NOASSERTION and NONE markers as absent rather than as literal values. func parseSPDXEntity(value string) string { value = strings.TrimSpace(value) - switch strings.ToUpper(value) { + // SPDX sentinels are the exact uppercase literals, so ordinary free text + // such as "None" or "NoAssertion" is a real description — case-folding + // here silently deleted it from the newly preserved text fields. + switch value { case "", "NOASSERTION", "NONE": return "" } From bd394c308b63a89599591243d7f41ef18123e63d Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 15:21:45 -0700 Subject: [PATCH 20/23] fix(sbom): validate rendered VCS revisions and every CPE component P1: normalizeRepositoryURL was a fifth entry point into the path gate and did not use it. An SPDX "Source repository: https://github.com/ghp_abcd1234/repo" checked userinfo, query, and fragment but never the path, so the token was stored as Repository and republished in both formats. I added the gate at four call sites last commit and missed this one. normalizeVCS rebuilt every locator with "git+", so an asserted svn+/hg+/bzr+ download location came back as Git. That changes which version-control system the document asserts rather than normalizing it. The detected tool is carried through now. Two more, both caught only because the reviewer read the tests rather than trusting them: - validatedVCSLocator accepted "git+https://github.com", which names no repository. The other normalization paths already required a path. - A package declaring downloadLocation NONE alongside a source repository lost the repository entirely: NONE is emitted as the download location, so source info was suppressed for a duplicate that was never written. Suppression now applies only when the VCS locator is the value actually emitted. Both of my earlier tests were too weak to catch their own bugs. The non-Git round trip asserted only that the host and path survived, which "git+" rewriting preserves; it now asserts the whole locator. The NONE test asserted only that NONE survived; the repository assertion it was silently dropping is now covered. All four fixes were mutation-checked, with the mutation asserted to have applied. Co-Authored-By: Claude Opus 5 --- internal/sbom/locator.go | 33 +++++++----- internal/sbom/roundtrip_test.go | 93 ++++++++++++++++++++++++++++++++- internal/sbom/spdx23.go | 10 ++-- 3 files changed, 118 insertions(+), 18 deletions(-) diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index 3ba57f85..cf0c3a23 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -164,6 +164,7 @@ func classifyURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosyste // Cargo records the raw Cargo.lock `source` string, which carries its own // scheme prefix. The prefix is a stronger signal than anything else here. hint := LocatorNone + vcsTool := "" switch { case strings.HasPrefix(raw, "registry+"): raw, hint = strings.TrimPrefix(raw, "registry+"), LocatorRegistryRoot @@ -173,8 +174,10 @@ func classifyURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosyste // Any recognized version-control tool prefix, not just git+: an SPDX // downloadLocation of "svn+https://…" is a valid VCS location, and // leaving it prefixed makes the transport gate reject it outright. - if _, rest, isVCS := splitVCSToolPrefix(raw); isVCS { - raw, hint = rest, LocatorVCS + // The tool is carried through so normalization does not silently + // rewrite the asserted version-control system as Git. + if prefix, rest, isVCS := splitVCSToolPrefix(raw); isVCS { + raw, hint, vcsTool = rest, LocatorVCS, prefix } } @@ -224,17 +227,17 @@ func classifyURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosyste // ("https://host/repo.git?rev="). Gating first would silently drop // those pins. if hint == LocatorVCS { - return normalizeVCS(parsed) + return normalizeVCS(parsed, vcsTool) } if hint != LocatorRegistryRoot { // Swift package identity is the repository URL, so swiftpm records a // repo even for `kind: registry` pins. Without this the extension // check below would demote them to registry roots. if ecosystem == sdk.EcosystemSwift { - return normalizeVCS(parsed) + return normalizeVCS(parsed, vcsTool) } if source == sdk.DependencySourceGit || strings.HasSuffix(parsed.Path, ".git") { - return normalizeVCS(parsed) + return normalizeVCS(parsed, vcsTool) } } @@ -324,7 +327,7 @@ func isConcreteArtifactPath(path string) bool { // That grammar has no query component, so a cargo value such as // "git+https://host/a/b?rev=abc" cannot be passed through as-is: the revision // is moved to the "@" suffix and the query is dropped. -func normalizeVCS(parsed *url.URL) Locator { +func normalizeVCS(parsed *url.URL, tool string) Locator { // The fragment carries the resolved commit and the query carries what was // requested, so "?branch=main#abc123" locked abc123. Preferring the // fragment records the immutable commit rather than a moving branch, and @@ -373,7 +376,10 @@ func normalizeVCS(parsed *url.URL) Locator { } base := strings.TrimSuffix(clean.String(), "/") - out := "git+" + base + if tool == "" { + tool = "git+" + } + out := tool + base if isSafeRevision(revision) { out += "@" + revision } @@ -487,7 +493,7 @@ func normalizeRepositoryURL(repo string) string { // The scheme-less branch below requires an owner/repo path; // an absolute URL has to clear the same bar or it names no // repository at all. - if strings.Trim(parsed.Path, "/") == "" { + if strings.Trim(parsed.Path, "/") == "" || hasCredentialPath(parsed) { return "" } return parsed.String() @@ -545,11 +551,7 @@ func classifyIngestedVCS(raw string) string { default: return "" } - locator := normalizeVCS(parsed).URL - if locator == "" || tool == "git+" { - return locator - } - return tool + strings.TrimPrefix(locator, "git+") + return normalizeVCS(parsed, tool).URL } // classifyAssertedDownloadLocation classifies a value that its source document @@ -641,6 +643,11 @@ func validatedVCSLocator(locator string) string { return "" } + // A repository needs a path: "git+https://github.com" names none, and the + // other normalization paths already reject that shape. + if strings.Trim(parsed.Path, "/") == "" { + return "" + } out := tool + strings.TrimSuffix(parsed.String(), "/") if isSafeRevision(revision) { out += "@" + revision diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 61051a44..36e3117d 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -1950,8 +1950,11 @@ func TestNonGitVCSDownloadLocationRoundTrips(t *testing.T) { "downloadLocation": "` + locator + `", "filesAnalyzed": false}] }` out := string(ingestAndReexport(t, []byte(in), TargetSPDX23JSON)) - if !strings.Contains(out, "example.org/project") { - t.Fatalf("%s was dropped on re-export:\n%s", locator, out) + // Assert the whole locator, not just the host and path: rebuilding it + // with "git+" would keep those while silently changing which + // version-control system the document asserts. + if !strings.Contains(out, locator) { + t.Fatalf("%s was not preserved verbatim on re-export:\n%s", locator, out) } } } @@ -2055,3 +2058,89 @@ func TestGraphLocatorMergeMirrorsModel(t *testing.T) { } } } + +// TestCredentialPathInSourceInfoIsRejected covers normalizeRepositoryURL, the +// fifth entry point into the path gate. The other four were covered; this one +// checked userinfo, query, and fragment but not path segments. +func TestCredentialPathInSourceInfoIsRejected(t *testing.T) { + repo, vcs := parseSPDXSourceInfo("Source repository: https://github.com/ghp_abcd1234/repo") + if repo != "" || vcs != "" { + t.Fatalf("a credential-shaped path was accepted: repo=%q vcs=%q", repo, vcs) + } + if got := normalizeRepositoryURL("https://github.com/ghp_abcd1234/repo"); got != "" { + t.Fatalf("normalizeRepositoryURL kept a credential path: %q", got) + } + // A legitimate repository is unaffected. + if got := normalizeRepositoryURL("https://github.com/owner/repo"); got == "" { + t.Fatal("an ordinary repository URL was rejected") + } +} + +// TestHostOnlyVCSLocatorIsRejected covers a locator that names no repository. +func TestHostOnlyVCSLocatorIsRejected(t *testing.T) { + for _, locator := range []string{"git+https://github.com", "git+https://github.com/", "svn+https://svn.example.org"} { + if got := validatedVCSLocator(locator); got != "" { + t.Fatalf("validatedVCSLocator(%q) = %q, want a host-only locator rejected", locator, got) + } + } + if got := validatedVCSLocator("git+https://github.com/org/repo"); got == "" { + t.Fatal("a real repository locator was rejected") + } +} + +// TestNoneDownloadLocationKeepsRepositoryAssertion covers a package that both +// declares NONE and records a repository. NONE is emitted as the download +// location, so the repository is not duplicated there — suppressing source +// info as well would lose it outright. +func TestNoneDownloadLocationKeepsRepositoryAssertion(t *testing.T) { + const in = `{ + "spdxVersion": "SPDX-2.3", "SPDXID": "SPDXRef-DOCUMENT", "name": "doc", + "documentNamespace": "https://example.com/doc", + "creationInfo": {"created": "2024-01-01T00:00:00Z", "creators": ["Tool: t"]}, + "packages": [{ + "name": "a", "SPDXID": "SPDXRef-a", "versionInfo": "1.0.0", + "downloadLocation": "NONE", + "sourceInfo": "Source repository: git+https://github.com/org/repo@deadbeef", + "filesAnalyzed": false + }] + }` + + out := ingestAndReexport(t, []byte(in), TargetSPDX23JSON) + var doc v23.Document + if err := json.Unmarshal(out, &doc); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for _, p := range doc.Packages { + if p == nil || p.PackageName != "a" { + continue + } + if p.PackageDownloadLocation != "NONE" { + t.Fatalf("downloadLocation = %q, want the NONE assertion kept", p.PackageDownloadLocation) + } + if !strings.Contains(p.PackageSourceInfo, "git+https://github.com/org/repo@deadbeef") { + t.Fatalf("sourceInfo = %q, want the repository assertion kept alongside NONE", p.PackageSourceInfo) + } + return + } + t.Fatal("package missing from output") +} + +// TestVCSToolIsNotRewrittenToGit covers the classifier itself: stripping a +// tool prefix and rebuilding with "git+" changes which version-control system +// the document asserts, rather than merely normalizing the locator. +func TestVCSToolIsNotRewrittenToGit(t *testing.T) { + cases := map[string]string{ + "svn+https://svn.example.org/project": "svn+https://svn.example.org/project", + "hg+https://hg.example.org/project": "hg+https://hg.example.org/project", + "bzr+https://bzr.example.org/project": "bzr+https://bzr.example.org/project", + "git+https://github.com/org/repo": "git+https://github.com/org/repo", + } + for in, want := range cases { + if got := classifyAssertedDownloadLocation(in); got.URL != want { + t.Fatalf("classifyAssertedDownloadLocation(%q).URL = %q, want %q", in, got.URL, want) + } + if got := classifyIngestedVCS(in); got != want { + t.Fatalf("classifyIngestedVCS(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 71ca542a..c285d4df 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -564,19 +564,23 @@ func spdxSourceInfo(component Component) string { // distribution and a vcs reference; there the artifact owns // downloadLocation, and dropping the repository as well would lose it // entirely even though PackageSourceInfo can represent it. - if component.VCSURL != "" && component.ArtifactURL == "" { + // Suppress only when this VCS locator is the value spdxDownloadLocation + // actually emits. With NONE asserted, or with an artifact owning the + // field, the repository is not duplicated there — so dropping it here + // would lose the assertion outright. + if component.VCSURL != "" && component.ArtifactURL == "" && !component.NoDownloadLocation { return "" } // The ingested VCS assertion outranks the matcher-derived Scorecard // repository, matching the ingest-before-enrichment precedence. repo := "" - if component.ArtifactURL != "" { + if component.ArtifactURL != "" || component.NoDownloadLocation { repo = strings.TrimSpace(component.VCSURL) } if repo == "" { repo = strings.TrimSpace(component.Repository) } - if repo == "" && component.ArtifactURL != "" { + if repo == "" && (component.ArtifactURL != "" || component.NoDownloadLocation) { // Keep the version-control form intact. Stripping "git+" from // "git+https://host/org/repo@deadbeef" turns the revision into part // of the URL path, producing a repository address that does not From c1c3beffcb04c49dfd765a0d260d10ce641dc69d Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Fri, 14 Aug 2026 16:27:13 -0700 Subject: [PATCH 21/23] fix(sbom): add the missing CPE fuzz target and close credential paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The most important item here is procedural: this change introduced a hand-written CPE parser over untrusted SPDX and CycloneDX input — two grammars, backslash escapes, percent encoding, a packed edition — and registered no fuzz target for it. CLAUDE.md requires one for exactly this, and I registered targets only for the locator helpers. Added FuzzIsValidCPE with valid, malformed, and truncated seeds, bounded by testkit.MaxFuzzInputSize, registered in scripts/run-fuzz.sh. It found a real bug within seconds. isValidCPE trims internally, but the CycloneDX path stored the untrusted original, so a padded CPE passed validation and was re-emitted with its whitespace — the same gate-trims-but-caller-stores-raw defect already fixed for external references and supplier URLs. All three sites are consistent now, and a test covers the class rather than the instance. Three credential paths: - normalizeRepositoryURL has two branches and only the absolute-URL one had the path gate. A scheme-less "github.com/ghp_abcd1234/repo" from SPDX source info kept the token. Fixing this also showed the prefix check I added alongside it was dead — hasCredentialPath already covers the case, segment-wise and more correctly — so it is removed rather than left as decoration. - Supplier contact emails reached the output without the validation a mailto reference gets, so "ghp_abcd1234@example.com" was republished. An unusable address is dropped and the contact kept. - URN payloads were checked for credential prefixes but not syntax, so "urn:foo bar" or a bad percent escape could break the document's IRI-reference constraint. Every fix was mutation-checked. One mutation reported zero failures, which is what surfaced the dead check above. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 21 ++++-- internal/sbom/locator.go | 32 ++++++++- internal/sbom/locator_fuzz_test.go | 57 ++++++++++++++++ internal/sbom/roundtrip_test.go | 105 +++++++++++++++++++++++++++++ scripts/run-fuzz.sh | 1 + 5 files changed, 210 insertions(+), 6 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 9ab1c7b7..80e1cefc 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -396,8 +396,10 @@ func componentFromCycloneDX(comp cdx.Component) Component { Description: comp.Description, Originator: comp.Publisher, } - if isValidCPE(comp.CPE) { - component.CPEs = []string{comp.CPE} + // Store the trimmed value isValidCPE actually validated: a padded CPE + // would otherwise pass the check and be re-emitted with its whitespace. + if cpe := strings.TrimSpace(comp.CPE); isValidCPE(cpe) { + component.CPEs = []string{cpe} } // The name is optional on a CycloneDX organizational entity. Gating the // whole block on it discarded the URLs and contacts of a supplier that @@ -412,15 +414,26 @@ func componentFromCycloneDX(comp cdx.Component) Component { if c.Name == "" && c.Email == "" && c.Phone == "" { continue } + // A contact email is republished verbatim, so it needs the + // same validation a mailto reference gets — otherwise a + // credential-shaped address passes straight through. + email := c.Email + if email != "" && !isEmailAddress(email) { + email = "" + } + if c.Name == "" && email == "" && c.Phone == "" { + continue + } component.SupplierContacts = append(component.SupplierContacts, Contact{ - Name: c.Name, Email: c.Email, Phone: c.Phone, + Name: c.Name, Email: email, Phone: c.Phone, }) } } if comp.Supplier.URL != nil { for _, u := range *comp.Supplier.URL { if isPublishableReferenceURL(u) { - component.SupplierURLs = append(component.SupplierURLs, u) + // Store the trimmed value the gate actually validated. + component.SupplierURLs = append(component.SupplierURLs, strings.TrimSpace(u)) } } } diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index cf0c3a23..6ff03667 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -514,7 +514,7 @@ func normalizeRepositoryURL(repo string) string { // something url.Parse rejects, such as an invalid percent-escape. candidate := "https://" + repo parsed, err := url.Parse(candidate) - if err != nil || parsed.Host != host || parsed.User != nil { + if err != nil || parsed.Host != host || parsed.User != nil || hasCredentialPath(parsed) { return "" } return candidate @@ -698,7 +698,10 @@ func isPublishableReferenceURL(raw string) bool { return false } } - return true + // The raw value is re-emitted, so malformed opaque text such as + // "urn:foo bar" or a bad percent escape would break the document's + // IRI-reference constraint. + return isURNPayload(parsed.Opaque) default: // Anything else — notably file:, data:, and javascript: — stays // rejected. Denying unknown schemes is the safe default: these values @@ -750,6 +753,31 @@ func splitVCSToolPrefix(locator string) (prefix, rest string, ok bool) { return "", locator, false } +// isURNPayload reports whether a URN's opaque part is syntactically usable. +// +// This is a character and escape gate rather than full RFC 8141 parsing: its +// job is to keep a value that cannot be a valid IRI reference from being +// re-emitted into a document that must satisfy that constraint. +func isURNPayload(value string) bool { + runes := []rune(value) + for i := 0; i < len(runes); i++ { + r := runes[i] + switch { + case r < '!' || r > '~': + // Spaces, control characters, and non-ASCII. + return false + case r == '%': + if i+2 >= len(runes) || !isHexRune(runes[i+1]) || !isHexRune(runes[i+2]) { + return false + } + i += 2 + case r == '<' || r == '>' || r == '"' || r == '\\' || r == '^' || r == '`' || r == '{' || r == '}' || r == '|': + return false + } + } + return true +} + // isHostname reports whether value looks like a dotted DNS hostname. func isHostname(value string) bool { if !strings.Contains(value, ".") || strings.HasPrefix(value, ".") || strings.HasSuffix(value, ".") { diff --git a/internal/sbom/locator_fuzz_test.go b/internal/sbom/locator_fuzz_test.go index b317a3c9..7d445b03 100644 --- a/internal/sbom/locator_fuzz_test.go +++ b/internal/sbom/locator_fuzz_test.go @@ -1,6 +1,7 @@ package sbom import ( + "strings" "testing" "github.com/bomly-dev/bomly-sdk" @@ -117,3 +118,59 @@ func FuzzNormalizeRepositoryURL(f *testing.F) { } }) } + +// FuzzIsValidCPE exercises the CPE validator, a hand-written parser over +// values taken from untrusted SPDX and CycloneDX documents. It carries two +// separate grammars — the 2.3 formatted string with backslash escapes, and the +// 2.2 URI binding with percent encoding and a packed edition — so it meets the +// repository's rule that every new parser of untrusted data gets a target. +// +// The invariant is the one that matters for output: anything accepted is +// re-emitted as a package identity assertion, so an accepted value must be +// printable ASCII with no delimiter that would change how a consumer parses it. +func FuzzIsValidCPE(f *testing.F) { + for _, seed := range []string{ + "cpe:2.3:a:example:left-pad:1.3.0:*:*:*:*:*:*:*", + "cpe:2.3:o:vendor:os:1.0:-:*:*:*:*:*:*", + `cpe:2.3:a:ven\:dor:prod:1.0:*:*:*:*:*:*:*`, + "cpe:/a:hp:insight_diagnostics:7.4.0.1570::~~online~win2003~x64~", + "cpe:/a:apache:log4j:2.14.1", + "cpe:/a:vendor%3Aname:product", + "cpe:/a:vendor%ZZ:product", + "cpe:2.3:a:vendor:product:1.0:*:*:*:*:*:*:*:extra", + "cpe:2.3:z:vendor:product:1.0:*:*:*:*:*:*:*", + "cpe:2.3:a:vendor with space:product:1.0:*:*:*:*:*:*:*", + `cpe:2.3:a:vendor:pro\`, + "cpe:2.3:", "cpe:/", "cpe:", "not-a-cpe", "", " ", + "cpe:2.3:a:v:p:1.0:*:*:*:*:*:*:\x00", + "cpe:/a:v:\x7f", + "cpe:2.3:a:v:p:1.0:*:*:*:*:*:*:é", + } { + f.Add(seed) + } + + f.Fuzz(func(t *testing.T, raw string) { + if len(raw) > testkit.MaxFuzzInputSize { + return + } + got := isValidCPE(raw) + if again := isValidCPE(raw); again != got { + t.Fatalf("nondeterministic validation of %q: %v vs %v", raw, got, again) + } + if !got { + return + } + // isValidCPE trims, so the value callers must store is the trimmed + // one. Every caller is responsible for storing that form; asserting + // on it here is what pins the contract. + trimmed := strings.TrimSpace(raw) + for _, r := range trimmed { + if r < '!' || r > '~' { + t.Fatalf("accepted %q containing a non-printable or non-ASCII rune %q", trimmed, r) + } + } + if !strings.HasPrefix(trimmed, "cpe:2.3:") && !strings.HasPrefix(trimmed, "cpe:/") { + t.Fatalf("accepted %q, which is neither CPE binding", trimmed) + } + }) +} diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index 36e3117d..e19bc043 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -2144,3 +2144,108 @@ func TestVCSToolIsNotRewrittenToGit(t *testing.T) { } } } + +// TestSchemeLessRepositoryCredentialIsRejected covers the second branch of +// normalizeRepositoryURL. The absolute-URL branch got the path gate first; +// its scheme-less sibling in the same function did not. +func TestSchemeLessRepositoryCredentialIsRejected(t *testing.T) { + for _, value := range []string{ + "github.com/ghp_abcd1234/repo", + "github.com/owner/glpat-Abc123", + } { + if got := normalizeRepositoryURL(value); got != "" { + t.Fatalf("normalizeRepositoryURL(%q) = %q, want a credential path rejected", value, got) + } + } + repo, _ := parseSPDXSourceInfo("Source repository: github.com/ghp_abcd1234/repo") + if repo != "" { + t.Fatalf("source info kept a credential path: %q", repo) + } + if got := normalizeRepositoryURL("github.com/owner/repo"); got != "https://github.com/owner/repo" { + t.Fatalf("an ordinary scheme-less repository was rejected: %q", got) + } +} + +// TestSupplierContactEmailIsValidated covers organizational contacts, which +// reach the output without passing through the mailto gate. +func TestSupplierContactEmailIsValidated(t *testing.T) { + const in = `{ + "bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", "type": "library", "name": "a", "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "supplier": {"name": "Acme", "contact": [ + {"name": "Bot", "email": "ghp_abcd1234@example.com"}, + {"name": "Security Team", "email": "security@example.com"} + ]} + }] + }` + + out := string(ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON)) + if strings.Contains(out, "ghp_abcd1234") { + t.Fatalf("a credential-shaped contact email was republished:\n%s", out) + } + // The contact itself survives without its unusable address, and a real + // address is untouched. + for _, want := range []string{"Bot", "security@example.com"} { + if !strings.Contains(out, want) { + t.Fatalf("%q was dropped alongside the rejected email:\n%s", want, out) + } + } +} + +// TestMalformedURNIsRejected covers the URN payload, which is re-emitted +// verbatim and so must be a usable IRI reference. +func TestMalformedURNIsRejected(t *testing.T) { + valid := []string{ + "urn:uuid:3f2504e0-4f89-41d3-9a0c-0305e82c3301", + "urn:cdx:serial/1", + "urn:example:a%3Ab", + } + for _, value := range valid { + if !isPublishableReferenceURL(value) { + t.Fatalf("isPublishableReferenceURL(%q) = false, want a valid URN accepted", value) + } + } + invalid := []string{ + "urn:foo bar", "urn:uuid:%ZZ", "urn:uuid:%4", "urn:a\x00b", "urn:ac", "urn:", + } + for _, value := range invalid { + if isPublishableReferenceURL(value) { + t.Fatalf("isPublishableReferenceURL(%q) = true, want a malformed URN rejected", value) + } + } +} + +// TestPaddedValuesAreStoredTrimmed covers the whole class: a gate that trims +// its local copy while the caller stores the original. +func TestPaddedValuesAreStoredTrimmed(t *testing.T) { + in := `{ + "bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, + "components": [{ + "bom-ref": "pkg:npm/a@1.0.0", "type": "library", "name": "a", "version": "1.0.0", + "purl": "pkg:npm/a@1.0.0", + "cpe": " cpe:2.3:a:v:p:1.0:*:*:*:*:*:*:* ", + "supplier": {"name": "Acme", "url": [" https://supplier.example.com "]}, + "externalReferences": [{"type": "documentation", "url": " https://docs.example "}] + }] + }` + + raw := ingestAndReexport(t, []byte(in), TargetCycloneDX17JSON) + var bom cdx.BOM + if err := json.Unmarshal(raw, &bom); err != nil { + t.Fatalf("unmarshal: %v", err) + } + comp := (*bom.Components)[0] + if comp.CPE != "cpe:2.3:a:v:p:1.0:*:*:*:*:*:*:*" { + t.Fatalf("cpe = %q, want it stored trimmed", comp.CPE) + } + if comp.Supplier == nil || comp.Supplier.URL == nil || (*comp.Supplier.URL)[0] != "https://supplier.example.com" { + t.Fatalf("supplier url was not stored trimmed: %+v", comp.Supplier) + } + for _, ref := range *comp.ExternalReferences { + if ref.Type == cdx.ERTypeDocumentation && ref.URL != "https://docs.example" { + t.Fatalf("reference url = %q, want it stored trimmed", ref.URL) + } + } +} diff --git a/scripts/run-fuzz.sh b/scripts/run-fuzz.sh index 36476e5b..eb2795e2 100755 --- a/scripts/run-fuzz.sh +++ b/scripts/run-fuzz.sh @@ -33,6 +33,7 @@ targets=( "github.com/bomly-dev/bomly-cli/internal/sbom FuzzNormalizeSPDXLicenseExpression" "github.com/bomly-dev/bomly-cli/internal/sbom FuzzClassifyResolvedURL" "github.com/bomly-dev/bomly-cli/internal/sbom FuzzNormalizeRepositoryURL" + "github.com/bomly-dev/bomly-cli/internal/sbom FuzzIsValidCPE" "github.com/bomly-dev/bomly-cli/internal/baseline FuzzLoad" "github.com/bomly-dev/bomly-cli/internal/engine FuzzConsolidateVulnerabilities" "github.com/bomly-dev/bomly-cli/internal/plugin FuzzPluginPathSanitizers" From 56aa7b02d80f28569a7b9f926b6c408729cb7cd4 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Mon, 17 Aug 2026 22:17:41 -0400 Subject: [PATCH 22/23] fix(sbom): scan for credentials instead of splitting on delimiters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The credential check was segment-based: split a URL component on a set of delimiters, then prefix-match each segment. Every delimiter missing from that set was a silent gap, and the new reference-URL fuzz target found them one after another — a token after "&", then after ".", then after ":" — each fix revealing the next. That is the same instance-by-instance pattern this review has hit repeatedly, so the splitting is gone. containsCredential scans a whole component for an issuer prefix at a token boundary, which has no delimiter set to get wrong and is the same check wherever it is applied. A prefix must also be followed by at least eight token characters: a bare "ghp_" is a prefix, not a secret, and treating it as one rejected ordinary URLs. The boundary requirement is what keeps it quiet on real text — "task-runner" no longer trips the "sk-" prefix. Two P1 gaps closed with it: - Hostnames were never inspected. "https://ghp_abcd1234.repo.example/…" has nil userinfo, a clean path, and no query, so every gate passed it. - isPublishableReferenceURL had no fuzz target, though it parses untrusted URLs across several schemes, mail addresses, URN payloads, and percent escapes. AGENTS.md requires one for every such parser; the CPE validator needed the same and got it last commit. Applying the host check by blanket string replacement produced "!a || b" where "!(a || b)" was meant, inverting the reference gate so it accepted credential hostnames. The fuzz target caught that on its first run, which is the argument for the target existing. Also from review: escaped CPE runes are validated against the formatted-string escape vocabulary rather than all printable ASCII; the conflicting-locator digest merge no longer attaches the incoming URL's hashes to the surviving URL; and an exact artifact now wins the download slot over a registry root regardless of array order. Test fixtures used toy tokens like "glpat-Abc123" whose bodies are shorter than any real credential, which is what let the body-length rule look wrong at first. They carry realistic lengths now. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 49 +++++--- internal/sbom/graph.go | 25 ++-- internal/sbom/locator.go | 110 +++++++++++++++--- internal/sbom/locator_fuzz_test.go | 69 +++++++++++ internal/sbom/locator_test.go | 14 +-- internal/sbom/roundtrip_test.go | 8 +- internal/sbom/spdx23.go | 12 +- .../8725790abef9b9ce | 2 + scripts/run-fuzz.sh | 1 + 9 files changed, 232 insertions(+), 58 deletions(-) create mode 100644 internal/sbom/testdata/fuzz/FuzzIsPublishableReferenceURL/8725790abef9b9ce diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 80e1cefc..96829c4c 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -449,23 +449,40 @@ func componentFromCycloneDX(comp cdx.Component) Component { for _, ref := range *comp.ExternalReferences { switch ref.Type { case cdx.ERTypeDistribution: - // A component may list several distribution URLs (mirrors). - // The neutral model has one artifact slot, so the first - // classified value takes it and the rest are preserved - // verbatim rather than overwriting the earlier assertion. - if component.ArtifactURL == "" && component.RegistryURL == "" { - locator := classifyAssertedDownloadLocation(ref.URL) - if strings.EqualFold(strings.TrimSpace(ref.Comment), registryRootMarker) && - locator.Kind == LocatorArtifact { - // The producer said this is a registry, not the exact - // artifact. Believe it rather than the path shape. - locator.Kind = LocatorRegistryRoot - } - applyLocatorComment(&component, locator, ref.Comment, referenceDigests(ref.Hashes)...) - if component.ArtifactURL != "" || component.RegistryURL != "" { - continue + // Every distribution reference is classified, not just the + // first: an exact artifact must win the download-location slot + // no matter where it sits in the array. Listing a registry + // root ahead of the archive previously left the archive as a + // plain reference, which spdxDownloadLocation never reads. + locator := classifyAssertedDownloadLocation(ref.URL) + if strings.EqualFold(strings.TrimSpace(ref.Comment), registryRootMarker) && + locator.Kind == LocatorArtifact { + // The producer said this is a registry, not the exact + // artifact. Believe it rather than the path shape. + locator.Kind = LocatorRegistryRoot + } + digests := referenceDigests(ref.Hashes) + + switch { + case locator.Kind == LocatorArtifact && component.ArtifactURL == "": + // An artifact displaces a registry root already in the + // weaker slot; that root is kept as an extra reference. + if component.RegistryURL != "" { + component.ExternalRefs = append(component.ExternalRefs, ExternalRef{ + Type: string(ref.Type), + URL: component.RegistryURL, + Comment: component.RegistryComment, + Digests: component.RegistryDigests, + }) + component.RegistryURL, component.RegistryComment, component.RegistryDigests = "", "", nil } + applyLocatorComment(&component, locator, ref.Comment, digests...) + continue + case locator.Kind == LocatorRegistryRoot && component.RegistryURL == "" && component.ArtifactURL == "": + applyLocatorComment(&component, locator, ref.Comment, digests...) + continue } + if !isPublishableReferenceURL(ref.URL) { continue } @@ -473,7 +490,7 @@ func componentFromCycloneDX(comp cdx.Component) Component { Type: string(ref.Type), URL: strings.TrimSpace(ref.URL), Comment: ref.Comment, - Digests: referenceDigests(ref.Hashes), + Digests: digests, }) case cdx.ERTypeVCS: // Untrusted input: gate and normalize it exactly like a diff --git a/internal/sbom/graph.go b/internal/sbom/graph.go index 9adedf74..a2e28741 100644 --- a/internal/sbom/graph.go +++ b/internal/sbom/graph.go @@ -154,7 +154,7 @@ func mergeIngestedNode(existing, incoming *sdk.Dependency) { } sdk.SetDetectionLicenses(existing, merged) } - mergeLocatorPairs(existing, incoming) + conflictedSlots := mergeLocatorPairs(existing, incoming) for key, value := range incoming.Metadata { if existing.Metadata == nil { existing.Metadata = make(map[string]any, len(incoming.Metadata)) @@ -163,7 +163,7 @@ func mergeIngestedNode(existing, incoming *sdk.Dependency) { // fill-gaps merge would drop the duplicate's whole list whenever the // first component had any. Union them instead. if key == metadataKeyLocatorDigests { - existing.Metadata[key] = mergeLocatorDigestMaps(existing.Metadata[key], value) + existing.Metadata[key] = mergeLocatorDigestMaps(existing.Metadata[key], value, conflictedSlots) continue } if _, paired := locatorCommentKeys[key]; paired { @@ -203,7 +203,8 @@ var locatorCommentKeys = map[string]string{ // mergeLocatorPairs fills each empty locator slot from incoming, moving the // URL and its comment as one unit. -func mergeLocatorPairs(existing, incoming *sdk.Dependency) { +func mergeLocatorPairs(existing, incoming *sdk.Dependency) (conflicted map[string]struct{}) { + conflicted = map[string]struct{}{} for _, pair := range []struct{ url, note, slot, refType string }{ {metadataKeyArtifactURL, metadataKeyArtifactNote, "artifact", "distribution"}, {metadataKeyVCSURL, metadataKeyVCSNote, "vcs", "vcs"}, @@ -247,8 +248,14 @@ func mergeLocatorPairs(existing, incoming *sdk.Dependency) { } existing.Metadata[metadataKeyExternalRefs] = unionExternalRefValues( existing.Metadata[metadataKeyExternalRefs], []any{ref}) + // The incoming digests belong to the reference just preserved, not + // to the URL that kept the slot. Marking the slot conflicted stops + // the generic map merge below from attaching them to the survivor + // and fabricating an integrity assertion. + conflicted[pair.slot] = struct{}{} } } + return conflicted } // locatorDigestSlot returns the serialized digest list for one locator slot. @@ -316,20 +323,22 @@ func unionAnyStrings(base, extra any) any { // discarded a duplicate's hashes for a slot the first component never filled // — and mergeLocatorPairs can still adopt that duplicate's URL, which would // then be re-emitted without its asserted hash. -func mergeLocatorDigestMaps(existing, incoming any) any { +func mergeLocatorDigestMaps(existing, incoming any, conflicted map[string]struct{}) any { incomingMap, ok := incoming.(map[string]any) if !ok { return existing } - existingMap, ok := existing.(map[string]any) - if !ok { - return incoming - } + existingMap, _ := existing.(map[string]any) merged := make(map[string]any, len(existingMap)+len(incomingMap)) for slot, value := range existingMap { merged[slot] = value } for slot, value := range incomingMap { + // A conflicted slot kept a different URL, and mergeLocatorPairs has + // already moved this slot's digests onto the preserved reference. + if _, isConflicted := conflicted[slot]; isConflicted { + continue + } if _, present := merged[slot]; !present { merged[slot] = value } diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index 6ff03667..d47deddc 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -100,12 +100,16 @@ func hasCredentialPath(parsed *url.URL) bool { if decoded, err := url.PathUnescape(path); err == nil { path = decoded } - for _, segment := range strings.Split(path, "/") { - if looksLikeCredential(segment) { - return true - } - } - return false + return containsCredential(path) +} + +// hasCredentialHost reports whether any hostname label looks like a secret. +// +// Userinfo, path, query, fragment, and revision positions are all gated, but +// a token can sit in the host too: "https://ghp_abcd1234.repo.example/a.tgz" +// has a nil User, a clean path, and no query, so every other check passes it. +func hasCredentialHost(parsed *url.URL) bool { + return containsCredential(parsed.Hostname()) } // hasCredentialQuery reports whether a URL's query carries something that @@ -211,7 +215,7 @@ func classifyURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosyste // A lockfile pointing at a private registry can embed a token. Publishing // it in an SBOM would leak a live credential, so drop the value entirely // rather than try to strip the userinfo and emit the rest. - if parsed.User != nil || hasCredentialPath(parsed) { + if parsed.User != nil || hasCredentialHost(parsed) { return Locator{} } @@ -241,6 +245,14 @@ func classifyURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosyste } } + // The path is checked here rather than above because every VCS branch has + // already returned: those go through normalizeVCS, which separates the + // "@" suffix and then checks what remains, so a bad revision + // costs only the revision instead of the whole repository. + if hasCredentialPath(parsed) { + return Locator{} + } + // Credentials also travel outside userinfo: signed and private-registry // URLs carry them as query parameters or fragments // ("...?token=", "...?X-Amz-Signature=..."). @@ -424,6 +436,62 @@ var credentialPrefixes = []string{ "shpat_", "shpss_", // Shopify } +// credentialBodyMinimum is how many token characters must follow an issuer +// prefix before the value is treated as a real secret. A bare "ghp_" is a +// prefix, not a credential, and rejecting it would discard ordinary URLs. +const credentialBodyMinimum = 8 + +// containsCredential reports whether text contains a recognizable access token +// at a token boundary. +// +// This scans rather than splitting on delimiters. Splitting needs the exact +// delimiter set for every position a token might sit in, and each missing +// separator is a silent gap; a boundary-aware scan has no such gaps and is the +// same check wherever it is applied — path, host, or query. +// +// The boundary requirement is what keeps it from firing on ordinary text: the +// prefix must start the value or follow a non-token character, so "task-runner" +// does not match the "sk-" prefix. +func containsCredential(text string) bool { + lowered := strings.ToLower(text) + for _, prefix := range credentialPrefixes { + for offset := 0; ; { + idx := strings.Index(lowered[offset:], prefix) + if idx < 0 { + break + } + at := offset + idx + offset = at + 1 + + if at > 0 && isTokenRune(rune(lowered[at-1])) { + continue // mid-word, not a token boundary + } + body := 0 + for _, r := range lowered[at+len(prefix):] { + if !isTokenRune(r) { + break + } + body++ + } + if body >= credentialBodyMinimum { + return true + } + } + } + return false +} + +// isTokenRune reports whether a rune can appear inside an access token. +func isTokenRune(r rune) bool { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + return true + case r == '_', r == '-': + return true + } + return false +} + // looksLikeCredential reports whether a value carries a recognizable // access-token prefix. // @@ -434,13 +502,7 @@ var credentialPrefixes = []string{ // narrow — a bespoke or unrecognized secret format would still pass. The // stronger guarantee lives on the fragment path, which requires bare hex. func looksLikeCredential(value string) bool { - lowered := strings.ToLower(strings.TrimSpace(value)) - for _, prefix := range credentialPrefixes { - if strings.HasPrefix(lowered, prefix) { - return true - } - } - return false + return containsCredential(strings.TrimSpace(value)) } // isSafeRevision reports whether revision is a plausible git revision that can @@ -493,7 +555,7 @@ func normalizeRepositoryURL(repo string) string { // The scheme-less branch below requires an owner/repo path; // an absolute URL has to clear the same bar or it names no // repository at all. - if strings.Trim(parsed.Path, "/") == "" || hasCredentialPath(parsed) { + if strings.Trim(parsed.Path, "/") == "" || hasCredentialPath(parsed) || hasCredentialHost(parsed) { return "" } return parsed.String() @@ -514,7 +576,7 @@ func normalizeRepositoryURL(repo string) string { // something url.Parse rejects, such as an invalid percent-escape. candidate := "https://" + repo parsed, err := url.Parse(candidate) - if err != nil || parsed.Host != host || parsed.User != nil || hasCredentialPath(parsed) { + if err != nil || parsed.Host != host || parsed.User != nil || hasCredentialPath(parsed) || hasCredentialHost(parsed) { return "" } return candidate @@ -539,7 +601,10 @@ func classifyIngestedVCS(raw string) string { } raw = rest parsed, err := url.Parse(raw) - if err != nil || parsed.User != nil || parsed.Host == "" || hasCredentialPath(parsed) { + // The path is deliberately not checked here: normalizeVCS separates the + // "@" suffix first and checks what remains, so a bad revision + // costs the revision rather than the whole repository. + if err != nil || parsed.User != nil || parsed.Host == "" || hasCredentialHost(parsed) { return "" } // git:// is a legitimate transport for a version-control reference, and @@ -603,15 +668,22 @@ func classifyAssertedReference(raw string) Locator { // ok is false when the URL is unsafe to publish at all. func splitVCSRevision(raw string) (parsed *url.URL, revision string, ok bool) { parsed, err := url.Parse(strings.TrimSpace(raw)) - if err != nil || parsed.User != nil || parsed.Host == "" || hasCredentialPath(parsed) { + if err != nil || parsed.User != nil || parsed.Host == "" || hasCredentialHost(parsed) { return nil, "", false } + // Separate the revision before checking the path. "@" is a path delimiter + // for the credential scan, so leaving a "@" suffix in place + // would reject the whole locator for a bad revision instead of dropping + // just that revision and keeping the repository. if idx := strings.LastIndex(parsed.Path, "@"); idx >= 0 { revision = parsed.Path[idx+1:] clean := *parsed clean.Path = parsed.Path[:idx] parsed = &clean } + if hasCredentialPath(parsed) { + return nil, "", false + } return parsed, revision, true } @@ -678,7 +750,7 @@ func isPublishableReferenceURL(raw string) bool { // parameters disqualify it. Fragments stay rejected: they carry no // meaning for a reference and are a common place to hide a secret. return parsed.Host != "" && parsed.Fragment == "" && - !hasCredentialQuery(parsed) && !hasCredentialPath(parsed) + !hasCredentialQuery(parsed) && !hasCredentialPath(parsed) && !hasCredentialHost(parsed) case "mailto": // The opaque body is never inspected by the userinfo, query, or // revision gates, so "mailto:ghp_abcd1234" would otherwise be diff --git a/internal/sbom/locator_fuzz_test.go b/internal/sbom/locator_fuzz_test.go index 7d445b03..8be34678 100644 --- a/internal/sbom/locator_fuzz_test.go +++ b/internal/sbom/locator_fuzz_test.go @@ -1,6 +1,7 @@ package sbom import ( + "net/url" "strings" "testing" @@ -174,3 +175,71 @@ func FuzzIsValidCPE(f *testing.F) { } }) } + +// FuzzIsPublishableReferenceURL exercises the gate every ingested external +// reference passes through. It spans several schemes, HTTP queries and paths, +// mail addresses, URN payloads, percent escapes, and credential detection, and +// a value it accepts is re-emitted verbatim into a published document. +// +// The invariant is what that re-emission requires: an accepted value carries +// no credential in any position and is a usable reference target. +func FuzzIsPublishableReferenceURL(f *testing.F) { + for _, seed := range []string{ + "https://example.com/docs", + "http://example.com/a?version=1", + "ftp://files.example.com/pkg.tgz", + "git://github.com/org/repo", + "mailto:security@example.com", + "urn:uuid:3f2504e0-4f89-41d3-9a0c-0305e82c3301", + "urn:example:a%3Ab", + "https://ghp_abcd1234.repo.example/a.tgz", + "https://repo.example/download/ghp_abcd1234/pkg.tgz", + "https://repo.example/download?token=s3cret", + "https://repo.example/download?ghp_abcd1234", + "https://tok:s3cret@example.com/a", + "mailto:ghp_abcd1234", + "mailto:ghp_abcd1234@example.com", + "urn:foo bar", "urn:uuid:%ZZ", + "file:///Users/victim/x", "javascript:alert(1)", "data:text/html,x", + "/Users/victim/x", "https://", "", " ", "\x00", "%%%", + } { + f.Add(seed) + } + + f.Fuzz(func(t *testing.T, raw string) { + if len(raw) > testkit.MaxFuzzInputSize { + return + } + got := isPublishableReferenceURL(raw) + if again := isPublishableReferenceURL(raw); again != got { + t.Fatalf("nondeterministic result for %q: %v vs %v", raw, got, again) + } + if !got { + return + } + + trimmed := strings.TrimSpace(raw) + parsed, err := url.Parse(trimmed) + if err != nil { + t.Fatalf("accepted an unparseable reference %q", trimmed) + } + if parsed.User != nil { + t.Fatalf("accepted a reference carrying userinfo: %q", trimmed) + } + switch strings.ToLower(parsed.Scheme) { + case "http", "https", "git", "ftp", "ftps", "mailto", "urn": + default: + t.Fatalf("accepted an out-of-vocabulary scheme %q in %q", parsed.Scheme, trimmed) + } + // No credential may survive anywhere in the emitted value. Scanning + // the whole string keeps this independent of how the implementation + // happens to split components. + decoded := trimmed + if unescaped, err := url.PathUnescape(trimmed); err == nil { + decoded = unescaped + } + if containsCredential(decoded) { + t.Fatalf("accepted %q, which contains a recognizable credential", trimmed) + } + }) +} diff --git a/internal/sbom/locator_test.go b/internal/sbom/locator_test.go index 6772d121..ac04603e 100644 --- a/internal/sbom/locator_test.go +++ b/internal/sbom/locator_test.go @@ -78,13 +78,13 @@ func TestClassifyResolvedURL(t *testing.T) { // A fragment must be commit-shaped, not merely character-safe: an // access token passes isSafeRevision and would be republished. {"pat fragment is not a revision", "https://github.com/org/repo#ghp_abcd1234", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, - {"fine-grained pat fragment is not a revision", "https://github.com/org/repo#github_pat_11ABCDE", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, + {"fine-grained pat fragment is not a revision", "https://github.com/org/repo#github_pat_11ABCDEFGHIJKLMNOP", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, {"non-hex fragment is not a revision", "https://github.com/org/repo#release-candidate", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, {"named query revision still allows tags", "https://github.com/org/repo?tag=v1.2.3", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo@v1.2.3"}, // A query key names its value a revision, so tags and branches are // legitimate there — but a recognizable token is not. {"token in rev query is rejected", "https://github.com/org/repo?rev=ghp_abcd1234", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, - {"token in branch query is rejected", "https://github.com/org/repo?branch=glpat-Abc123", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, + {"token in branch query is rejected", "https://github.com/org/repo?branch=glpat-Abc123XYZ789def", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo"}, {"underscored branch still allowed", "https://github.com/org/repo?branch=release_candidate", sdk.DependencySourceGit, sdk.EcosystemNPM, LocatorVCS, "git+https://github.com/org/repo@release_candidate"}, {"requested rev used when no commit resolved", "https://github.com/a/b?rev=v1.2.3", sdk.DependencySourceGit, sdk.EcosystemPython, LocatorVCS, "git+https://github.com/a/b@v1.2.3"}, @@ -237,11 +237,11 @@ func TestClassifyAssertedDownloadLocation(t *testing.T) { func TestIsSafeRevisionRejectsCredentialShapes(t *testing.T) { rejected := []string{ - "ghp_abcd1234", "github_pat_11ABCDE_xyz", "gho_abc", "ghs_abc", - "glpat-Abc123", "npm_abcdef", "pypi-AgEIcHlwaS5vcmc", - "xoxb-123-456-abc", "sk-abcdef123456", "sk_live_abc", - "AKIAIOSFODNN7EXAMPLE", "AIzaSyA-abc123", "hf_abcDEF", - "dop_v1_abc", "shpat_abc123", + "ghp_abcd1234", "github_pat_11ABCDEFGHIJKLMNOP_xyz", "gho_abcdefghijklmnop", "ghs_abcdefghijklmnop", + "glpat-Abc123XYZ789def", "npm_abcdefghijklmnop", "pypi-AgEIcHlwaS5vcmc", + "xoxb-123456789-abcdefghijk", "sk-abcdef123456ghijk", "sk_live_abcdefghijkl", + "AKIAIOSFODNN7EXAMPLE", "AIzaSyA-abc123defghijklmnop", "hf_abcDEFghijklmnop", + "dop_v1_abcdefghijklmnop", "shpat_abc123defghijk", } for _, value := range rejected { if isSafeRevision(value) { diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index e19bc043..e5be13b3 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -1265,7 +1265,7 @@ func TestVCSLocatorRevisionIsValidated(t *testing.T) { {"git+https://github.com/org/repo@v1.2.3", "git+https://github.com/org/repo@v1.2.3"}, // Token-shaped revision: the repository survives, the secret does not. {"git+https://github.com/org/repo@ghp_abcd1234", "git+https://github.com/org/repo"}, - {"git+https://github.com/org/repo@glpat-Abc123", "git+https://github.com/org/repo"}, + {"git+https://github.com/org/repo@glpat-Abc123XYZ789def", "git+https://github.com/org/repo"}, {"git+https://tok:s3cret@github.com/org/repo", ""}, // Userinfo with no path: splitting on "@" before parsing read // "ghp_secret" as the host and "github.com" as a revision, then @@ -1623,7 +1623,7 @@ func TestMergeComponentAssertionsUnionsEverySet(t *testing.T) { func TestBareTokenQueryIsRejected(t *testing.T) { for _, raw := range []string{ "https://repo.example/download?ghp_abcd1234", - "https://repo.example/download?glpat-Abc123", + "https://repo.example/download?glpat-Abc123XYZ789def", } { if got := classifyAssertedDownloadLocation(raw); got.Kind != LocatorNone { t.Fatalf("classifyAssertedDownloadLocation(%q) = %+v, want it rejected", raw, got) @@ -1914,7 +1914,7 @@ func TestIngestedVCSOutranksScorecardInSourceInfo(t *testing.T) { func TestCredentialInPathIsRejected(t *testing.T) { tokenPaths := []string{ "https://repo.example/download/ghp_abcd1234/pkg.tgz", - "https://repo.example/glpat-Abc123/pkg.tgz", + "https://repo.example/glpat-Abc123XYZ789def/pkg.tgz", "https://repo.example/download/%67hp_abcd1234/pkg.tgz", } for _, raw := range tokenPaths { @@ -2151,7 +2151,7 @@ func TestVCSToolIsNotRewrittenToGit(t *testing.T) { func TestSchemeLessRepositoryCredentialIsRejected(t *testing.T) { for _, value := range []string{ "github.com/ghp_abcd1234/repo", - "github.com/owner/glpat-Abc123", + "github.com/owner/glpat-Abc123XYZ789def", } { if got := normalizeRepositoryURL(value); got != "" { t.Fatalf("normalizeRepositoryURL(%q) = %q, want a credential path rejected", value, got) diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index c285d4df..97777c6f 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -782,6 +782,10 @@ func isHexRune(r rune) bool { return (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') } +// cpeEscapable is the punctuation a CPE 2.3 formatted string may quote with +// a backslash. Anything else after a backslash is malformed, not literal. +const cpeEscapable = `!"#$%&'()*+,-./:;<=>?@[\]^` + "`" + `{|}~_` + // isCPEComponent reports whether a CPE attribute value is well formed. // // The logical values "*" and "-" stand alone. Otherwise the value is a quoted @@ -804,10 +808,10 @@ func isCPEComponent(value string) bool { for _, r := range value { switch { case escaped: - // An escape must introduce a printable ASCII character. Without - // this, "\" followed by a control byte or a non-ASCII rune - // skipped the check below entirely. - if r < '!' || r > '~' { + // The formatted-string binding quotes only its own punctuation. + // Printable ASCII is broader than that vocabulary, so "\q" was + // accepted as a valid identity when it is malformed. + if !strings.ContainsRune(cpeEscapable, r) { return false } escaped = false diff --git a/internal/sbom/testdata/fuzz/FuzzIsPublishableReferenceURL/8725790abef9b9ce b/internal/sbom/testdata/fuzz/FuzzIsPublishableReferenceURL/8725790abef9b9ce new file mode 100644 index 00000000..32f86be4 --- /dev/null +++ b/internal/sbom/testdata/fuzz/FuzzIsPublishableReferenceURL/8725790abef9b9ce @@ -0,0 +1,2 @@ +go test fuzz v1 +string("http://0/:ghp_") diff --git a/scripts/run-fuzz.sh b/scripts/run-fuzz.sh index eb2795e2..7e57966f 100755 --- a/scripts/run-fuzz.sh +++ b/scripts/run-fuzz.sh @@ -34,6 +34,7 @@ targets=( "github.com/bomly-dev/bomly-cli/internal/sbom FuzzClassifyResolvedURL" "github.com/bomly-dev/bomly-cli/internal/sbom FuzzNormalizeRepositoryURL" "github.com/bomly-dev/bomly-cli/internal/sbom FuzzIsValidCPE" + "github.com/bomly-dev/bomly-cli/internal/sbom FuzzIsPublishableReferenceURL" "github.com/bomly-dev/bomly-cli/internal/baseline FuzzLoad" "github.com/bomly-dev/bomly-cli/internal/engine FuzzConsolidateVulnerabilities" "github.com/bomly-dev/bomly-cli/internal/plugin FuzzPluginPathSanitizers" From 172dd33e0c180d0136d61eaa0cabc98ba23b8b21 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Mon, 17 Aug 2026 22:35:25 -0400 Subject: [PATCH 23/23] fix(sbom): re-validate restored locators, merge digests by algorithm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dependency.Metadata is not private to the SBOM detector. Any detector, including an external plugin, can set bomly.sbom.* keys, and applyIngestedMetadata trusted them: a plugin returning "bomly.sbom.artifact_url: file:///home/runner/secret" overwrote a locator that had just passed the classifier and reached the published document. Restoring is an ingest path, so restored values now clear the assertion-level gate, the same one they passed on the way in. My own fuzz helper had drifted from the classifier. assertPublishableLocator still accepted only git+http(s), while the classifier deliberately keeps svn+, hg+, and bzr+ — so `make fuzz` and the scheduled workflow would have failed on correct behavior. It now accepts every prefix in vcsToolPrefixes. Conflicting integrity assertions are resolved rather than accumulated. An ingested SHA-256 and an enrichment SHA-256 with different values were both emitted, leaving two incompatible claims about one component; digests merge by algorithm now, so ingest wins a conflict and enrichment adds only algorithms the document lacked. The test asserted both values survived, which is the contradiction rather than the fix, so it now pins ingest-wins and additive-on-new-algorithm. Also: - The checksum-fragment exception exists for Yarn's detector-derived values; applying it to source-declared references silently changed another producer's asserted target. It is detector-only now. - The NONE marker merged independently of locator state, so duplicate PURLs could emit NONE in SPDX while CycloneDX still emitted the artifact. It follows the locator's first-component-wins policy. - URI-bound CPEs rejected raw backslashes and the other characters the binding requires to be percent-encoded. - Supplier contact emails are stored trimmed, as supplier URLs already were. Not changed: ssh VCS locators. Accepting "git+ssh://git@host/repo" means carving an exception into the userinfo rule, which has been the source of two P1 credential leaks in this PR. That trade deserves its own change rather than round nineteen of this one; noted for the follow-up. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 2 +- internal/sbom/distribution_test.go | 22 +++-- internal/sbom/graph.go | 16 ++++ internal/sbom/ingest_metadata.go | 36 +++++++- internal/sbom/locator.go | 6 +- internal/sbom/locator_test.go | 25 +++++- internal/sbom/roundtrip_test.go | 131 +++++++++++++++++++++++++++++ internal/sbom/spdx23.go | 3 + internal/sbom/transform.go | 27 +++++- 9 files changed, 253 insertions(+), 15 deletions(-) diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 96829c4c..bdea0f6a 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -417,7 +417,7 @@ func componentFromCycloneDX(comp cdx.Component) Component { // A contact email is republished verbatim, so it needs the // same validation a mailto reference gets — otherwise a // credential-shaped address passes straight through. - email := c.Email + email := strings.TrimSpace(c.Email) if email != "" && !isEmailAddress(email) { email = "" } diff --git a/internal/sbom/distribution_test.go b/internal/sbom/distribution_test.go index 3a6d1f72..89a1dbe4 100644 --- a/internal/sbom/distribution_test.go +++ b/internal/sbom/distribution_test.go @@ -376,6 +376,7 @@ func TestEnrichmentDoesNotClobberIngestedSets(t *testing.T) { const matcherCPE = "cpe:2.3:a:matcher:a:1.0.0:*:*:*:*:*:*:*" ingestedHash := strings.Repeat("a", 64) matcherHash := strings.Repeat("b", 64) + matcherSHA1 := strings.Repeat("c", 40) g := sdk.New() node := sdk.NewDependencyWithID(purl, sdk.Dependency{ @@ -391,22 +392,33 @@ func TestEnrichmentDoesNotClobberIngestedSets(t *testing.T) { pkg := registry.Ensure(purl) pkg.Name, pkg.Version = "a", "1.0.0" pkg.CPEs = []string{matcherCPE} - pkg.Digests = []sdk.Digest{{Algorithm: sdk.DigestAlgorithmSHA256, Value: matcherHash}} + // Same algorithm as the ingested digest: a contradictory assertion, so the + // ingested value wins. The SHA-1 is a new algorithm and is additive. + pkg.Digests = []sdk.Digest{ + {Algorithm: sdk.DigestAlgorithmSHA256, Value: matcherHash}, + {Algorithm: sdk.DigestAlgorithmSHA1, Value: matcherSHA1}, + } out, err := MarshalDepGraphJSON(g, TargetSPDX23JSON, BuildOptions{Registry: registry}, EncodeOptions{}) if err != nil { t.Fatalf("marshal: %v", err) } for name, want := range map[string]string{ - "ingested CPE": ingestedCPE, - "matcher CPE": matcherCPE, - "ingested hash": ingestedHash, - "matcher hash": matcherHash, + "ingested CPE": ingestedCPE, + "matcher CPE": matcherCPE, + "ingested hash": ingestedHash, + "matcher's new algorithm": matcherSHA1, } { if !strings.Contains(string(out), want) { t.Fatalf("%s was discarded by enrichment:\n%s", name, out) } } + // Two different SHA-256 values for one component are contradictory rather + // than complementary, so the enrichment value must not appear beside the + // ingested one. + if strings.Contains(string(out), matcherHash) { + t.Fatalf("enrichment overrode the ingested SHA-256 with a conflicting value:\n%s", out) + } } // TestBlake3DigestSurvivesRoundTrip pairs with the BLAKE3 encoder mappings: diff --git a/internal/sbom/graph.go b/internal/sbom/graph.go index a2e28741..d7693522 100644 --- a/internal/sbom/graph.go +++ b/internal/sbom/graph.go @@ -162,6 +162,22 @@ func mergeIngestedNode(existing, incoming *sdk.Dependency) { // External references are a set carried under one key, so a key-level // fill-gaps merge would drop the duplicate's whole list whenever the // first component had any. Union them instead. + // NONE is part of the locator assertion, not an independent fact: a + // first component asserting an exact URL and a second asserting NONE + // are contradictory, and merging the marker alone would make SPDX emit + // NONE while CycloneDX still emitted the artifact. First component + // wins, matching the locator policy. + if key == metadataKeyNoDownload { + if _, hasLocator := existing.Metadata[metadataKeyArtifactURL]; hasLocator { + continue + } + if _, hasLocator := existing.Metadata[metadataKeyVCSURL]; hasLocator { + continue + } + if _, hasLocator := existing.Metadata[metadataKeyRegistryURL]; hasLocator { + continue + } + } if key == metadataKeyLocatorDigests { existing.Metadata[key] = mergeLocatorDigestMaps(existing.Metadata[key], value, conflictedSlots) continue diff --git a/internal/sbom/ingest_metadata.go b/internal/sbom/ingest_metadata.go index a59b1809..a5665081 100644 --- a/internal/sbom/ingest_metadata.go +++ b/internal/sbom/ingest_metadata.go @@ -1,6 +1,8 @@ package sbom import ( + "strings" + "github.com/bomly-dev/bomly-sdk" ) @@ -141,6 +143,26 @@ func setIngestedMetadata(dep *sdk.Dependency, component Component) { dep.Metadata[metadataKeyExternalRefs] = refs } +// restoredLocator re-validates a locator recovered from Dependency.Metadata, +// returning "" when it is not publishable as the kind it claims to be. +func restoredLocator(value any, kind LocatorKind) string { + raw, _ := value.(string) + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + if kind == LocatorVCS { + return validatedVCSLocator(raw) + } + locator := classifyAssertedReference(raw) + if locator.Kind == LocatorNone { + return "" + } + // An artifact slot may hold a value the classifier reads as a registry + // root and vice versa; what matters is that it is publishable at all. + return locator.URL +} + // applyIngestedMetadata restores assertions stashed by setIngestedMetadata. // // The ingested values win over anything Bomly derived: they are the source @@ -178,9 +200,17 @@ func applyIngestedMetadata(component *Component, metadata map[string]any) { // The three distribution fields move as a set: a partial overwrite could // leave a component claiming both an ingested artifact URL and a stale // derived registry root. - artifact, _ := metadata[metadataKeyArtifactURL].(string) - vcs, _ := metadata[metadataKeyVCSURL].(string) - registry, _ := metadata[metadataKeyRegistryURL].(string) + // + // Every value is re-validated rather than trusted. Dependency.Metadata is + // not private to the SBOM detector — any detector, including an external + // plugin, can set these keys — so an unchecked assignment here would let + // "file:///home/runner/secret" or a credential-bearing URL overwrite a + // locator that just passed the classifier and reach the published + // document. Restoring is an ingest path, so the assertion-level gate + // applies, matching how these values were validated on the way in. + artifact := restoredLocator(metadata[metadataKeyArtifactURL], LocatorArtifact) + vcs := restoredLocator(metadata[metadataKeyVCSURL], LocatorVCS) + registry := restoredLocator(metadata[metadataKeyRegistryURL], LocatorRegistryRoot) if artifact != "" || vcs != "" || registry != "" { component.ArtifactURL, component.VCSURL, component.RegistryURL = artifact, vcs, registry } diff --git a/internal/sbom/locator.go b/internal/sbom/locator.go index d47deddc..61cf79b0 100644 --- a/internal/sbom/locator.go +++ b/internal/sbom/locator.go @@ -277,7 +277,11 @@ func classifyURL(raw string, source sdk.DependencySource, ecosystem sdk.Ecosyste // package in a Yarn lockfile. Strip it and keep the URL; anything that // is not digest-shaped is still treated as a secret. if parsed.Fragment != "" { - if !isChecksumFragment(parsed.Fragment) { + // The checksum exception exists for Yarn's detector-derived `resolved` + // values. A source-declared reference rejects fragments outright: + // stripping one there would silently change the target another + // producer asserted. + if allowBenignQuery || !isChecksumFragment(parsed.Fragment) { return Locator{} } clean := *parsed diff --git a/internal/sbom/locator_test.go b/internal/sbom/locator_test.go index ac04603e..d0393e71 100644 --- a/internal/sbom/locator_test.go +++ b/internal/sbom/locator_test.go @@ -144,15 +144,32 @@ func assertPublishableLocator(t *testing.T, got Locator, input string) { t.Fatalf("kind %v carried an empty url for input %q", got.Kind, input) } switch { - case strings.HasPrefix(got.URL, "https://"), strings.HasPrefix(got.URL, "http://"), - strings.HasPrefix(got.URL, "git+https://"), strings.HasPrefix(got.URL, "git+http://"): + case strings.HasPrefix(got.URL, "https://"), strings.HasPrefix(got.URL, "http://"): default: - t.Fatalf("emitted non-http(s) locator %q for input %q", got.URL, input) + // The classifier deliberately keeps the asserted version-control tool + // rather than rewriting everything to git+, so accept every prefix it + // can produce. Hard-coding git+ here would fail `make fuzz` on valid + // behavior. + matched := false + for _, prefix := range vcsToolPrefixes { + if strings.HasPrefix(got.URL, prefix+"https://") || strings.HasPrefix(got.URL, prefix+"http://") || + strings.HasPrefix(got.URL, prefix+"git://") { + matched = true + break + } + } + if !matched { + t.Fatalf("emitted an out-of-vocabulary locator %q for input %q", got.URL, input) + } } // Check userinfo structurally rather than by looking for "@": scoped npm // packages ("/@babel/core/") and VCS revisions ("@deadbeef") both contain // one legitimately. - parsed, err := url.Parse(strings.TrimPrefix(got.URL, "git+")) + bare := got.URL + for _, prefix := range vcsToolPrefixes { + bare = strings.TrimPrefix(bare, prefix) + } + parsed, err := url.Parse(bare) if err != nil { t.Fatalf("emitted unparseable locator %q for input %q", got.URL, input) } diff --git a/internal/sbom/roundtrip_test.go b/internal/sbom/roundtrip_test.go index e5be13b3..25cfc047 100644 --- a/internal/sbom/roundtrip_test.go +++ b/internal/sbom/roundtrip_test.go @@ -2249,3 +2249,134 @@ func TestPaddedValuesAreStoredTrimmed(t *testing.T) { } } } + +// TestRestoredLocatorMetadataIsRevalidated covers the trust boundary on +// Dependency.Metadata. Those keys are not private to the SBOM detector — any +// detector, including an external plugin, can set them — so a value restored +// from metadata must clear the same gate as one the classifier just produced. +func TestRestoredLocatorMetadataIsRevalidated(t *testing.T) { + hostile := map[string]string{ + "local path": "file:///home/runner/secret", + "credential userinfo": "https://tok:s3cret@nexus.corp/a-1.0.tgz", + "credential path": "https://repo.example/ghp_abcd1234abcd/a-1.0.tgz", + "not a url": "/home/runner/secret", + } + for name, value := range hostile { + g := sdk.New() + node := sdk.NewDependencyWithID("pkg@1.0.0", sdk.Dependency{ + Coordinates: sdk.Coordinates{ + Name: "pkg", Version: "1.0.0", + PURL: "pkg:npm/pkg@1.0.0", Ecosystem: sdk.EcosystemNPM, + }, + // A plugin-supplied node claiming an already-classified locator. + Metadata: map[string]any{"bomly.sbom.artifact_url": value}, + }) + if err := g.AddNode(node); err != nil { + t.Fatalf("add node: %v", err) + } + + for _, target := range []Target{TargetSPDX23JSON, TargetCycloneDX17JSON} { + out, err := MarshalDepGraphJSON(g, target, BuildOptions{}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal %s: %v", target, err) + } + needle := value + if idx := strings.Index(needle, "://"); idx >= 0 { + needle = needle[idx+3:] + } + if strings.Contains(string(out), needle) { + t.Fatalf("%s: restored metadata bypassed the gate for %s:\n%s", target, name, out) + } + } + } + + // A legitimate value still round-trips. + g := sdk.New() + node := sdk.NewDependencyWithID("pkg@1.0.0", sdk.Dependency{ + Coordinates: sdk.Coordinates{ + Name: "pkg", Version: "1.0.0", + PURL: "pkg:npm/pkg@1.0.0", Ecosystem: sdk.EcosystemNPM, + }, + Metadata: map[string]any{"bomly.sbom.artifact_url": "https://reg.example/a-1.0.0.tgz"}, + }) + if err := g.AddNode(node); err != nil { + t.Fatalf("add node: %v", err) + } + out, err := MarshalDepGraphJSON(g, TargetSPDX23JSON, BuildOptions{}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(out), "https://reg.example/a-1.0.0.tgz") { + t.Fatalf("a valid restored locator was dropped:\n%s", out) + } +} + +// TestAssertedReferenceKeepsItsFragment covers the checksum-fragment +// exception, which exists for Yarn's detector-derived values. Stripping it from +// a source-declared reference would silently change the asserted target. +func TestAssertedReferenceKeepsItsFragment(t *testing.T) { + withDigestFragment := "https://reg.example/a-1.0.0.tgz#" + strings.Repeat("a", 40) + + // Detector-derived: the Yarn shape, stripped and kept. + if got := classifyResolvedURL(withDigestFragment, "", ""); got.URL != "https://reg.example/a-1.0.0.tgz" { + t.Fatalf("detector value = %q, want the checksum fragment stripped", got.URL) + } + // Source-declared: rejected rather than silently rewritten. + if got := classifyAssertedDownloadLocation(withDigestFragment); got.Kind != LocatorNone { + t.Fatalf("asserted value = %+v, want a fragment rejected rather than stripped", got) + } +} + +// TestURIBoundCPERejectsRawReservedCharacters covers characters the URI +// binding requires to be percent-encoded. +func TestURIBoundCPERejectsRawReservedCharacters(t *testing.T) { + for _, value := range []string{ + `cpe:/a:ven\dor:product`, + `cpe:/a:ven"dor:product`, + "cpe:/a:ven' || r == '{' || r == '}' || r == '|' || r == '^' || r == '`': + // The URI binding percent-encodes these; raw, they are malformed. + return false case r == '%': // The URI binding percent-encodes its specials, so a "%" that is // not followed by two hex digits is malformed rather than literal. diff --git a/internal/sbom/transform.go b/internal/sbom/transform.go index e022a917..6dda0862 100644 --- a/internal/sbom/transform.go +++ b/internal/sbom/transform.go @@ -329,7 +329,11 @@ func enrichComponentFromRegistry(component *Component, registry *sdk.PackageRegi // Union rather than replace: an ingested document's identifiers are its // own assertions, and enrichment fills gaps rather than overwriting. component.CPEs = unionStrings(component.CPEs, pkg.CPEs) - component.Digests = unionComponentDigests(component.Digests, componentDigests(pkg.Digests)) + // Merge by algorithm, not by whole record: two different SHA-256 values + // for one component are contradictory integrity assertions, and the + // ingested one wins under ingest-before-enrichment. Enrichment may still + // add an algorithm the document did not carry. + component.Digests = mergeDigestsByAlgorithm(component.Digests, componentDigests(pkg.Digests)) if len(pkg.Vulnerabilities) > 0 { component.Vulnerabilities = vulnerabilitiesFromPackage(pkg.EcosystemName(), pkg.Vulnerabilities) } @@ -355,6 +359,27 @@ func enrichComponentFromRegistry(component *Component, registry *sdk.PackageRegi } } +// mergeDigestsByAlgorithm keeps base's value for any algorithm it already +// asserts and appends only algorithms base lacks. +func mergeDigestsByAlgorithm(base, extra []Digest) []Digest { + if len(extra) == 0 { + return base + } + seen := make(map[string]struct{}, len(base)) + for _, digest := range base { + seen[normalizeDigestAlgorithm(digest.Algorithm)] = struct{}{} + } + for _, digest := range extra { + algorithm := normalizeDigestAlgorithm(digest.Algorithm) + if _, present := seen[algorithm]; present { + continue + } + seen[algorithm] = struct{}{} + base = append(base, digest) + } + return base +} + // normalizeDigestAlgorithm renders a decoded checksum algorithm in the form // both encoders recognize. //