From b1868c7b9284761078d944fc9369a92df5e26fe0 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 00:54:47 -0400 Subject: [PATCH 01/17] feat(detectors): add detector-asserted package origin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Detectors know what their lockfile fields mean: npm's `resolved` is a tarball, cargo's `git+...#sha` is a pinned repository, uv's `editable` is a local path. Recovering that from the URL string alone, downstream, cannot be done reliably — every shape has an ecosystem-specific counterexample. Add the carrier and its single invariant so each detector can assert where a package came from, and so SBOM export can publish it without re-deciding anything: - `bomly.origin.*` metadata keys hold an exact artifact URL, or a repository URL plus the resolved revision, or nothing. - `NormalizeOriginURL` is the one rule every published origin satisfies: absolute http(s), host present, no userinfo, re-serialized from the parse. Local paths, file://, ssh, scp-style remotes, and credentialed URLs cannot reach an SBOM. It runs on the way in and again on the way out, so a plugin-supplied graph is held to the same rule as a built-in detector. - Command output filters the shared key prefix: origin is a transport between detection and export, and the SBOM is where users read it. No detector emits yet, and nothing reads the keys yet. Co-Authored-By: Claude Opus 5 --- internal/detectors/origin.go | 194 +++++++++++++++++++++++ internal/detectors/origin_fuzz_test.go | 112 ++++++++++++++ internal/detectors/origin_test.go | 198 ++++++++++++++++++++++++ internal/output/origin_metadata_test.go | 79 ++++++++++ internal/output/types.go | 12 ++ 5 files changed, 595 insertions(+) create mode 100644 internal/detectors/origin.go create mode 100644 internal/detectors/origin_fuzz_test.go create mode 100644 internal/detectors/origin_test.go create mode 100644 internal/output/origin_metadata_test.go diff --git a/internal/detectors/origin.go b/internal/detectors/origin.go new file mode 100644 index 00000000..ac61e63e --- /dev/null +++ b/internal/detectors/origin.go @@ -0,0 +1,194 @@ +package detectors + +import ( + "net/url" + "strings" + + "github.com/bomly-dev/bomly-sdk" +) + +// Origin metadata keys. Detectors record where a package came from under these +// keys on sdk.Dependency.Metadata; SBOM export reads them back. The values are +// a transport detail between detection and export, so command output filters +// the shared prefix out rather than publishing it. +const ( + // MetadataKeyOriginPrefix is the common prefix of every origin key. + MetadataKeyOriginPrefix = "bomly.origin." + // MetadataKeyOriginArtifactURL holds the exact artifact a package was + // resolved from (a tarball, wheel, gem, crate, ...). + MetadataKeyOriginArtifactURL = MetadataKeyOriginPrefix + "artifact_url" + // MetadataKeyOriginVCSURL holds the source repository a package was + // resolved from. + MetadataKeyOriginVCSURL = MetadataKeyOriginPrefix + "vcs_url" + // MetadataKeyOriginVCSRevision holds the resolved revision (commit, tag) + // pinned alongside MetadataKeyOriginVCSURL. + MetadataKeyOriginVCSRevision = MetadataKeyOriginPrefix + "vcs_revision" +) + +// maxOriginRevisionLength bounds a recorded revision. Real commit hashes and +// tags are far shorter; anything longer is not a revision. +const maxOriginRevisionLength = 128 + +// Origin is where a package came from, as asserted by the detector that +// resolved it. At most one location is set: a package is either downloaded as +// an artifact or checked out from a repository. An empty Origin means the +// detector had nothing publishable to say, which is the normal case for +// registry-resolved packages whose lockfile records only an index root. +type Origin struct { + // ArtifactURL is the exact file the package was downloaded from. + ArtifactURL string + // VCSURL is the source repository the package was resolved from. + VCSURL string + // VCSRevision is the revision pinned in VCSURL, when the lockfile + // recorded one. Never set without VCSURL. + VCSRevision string +} + +// Empty reports whether no location is set. +func (o Origin) Empty() bool { + return o.ArtifactURL == "" && o.VCSURL == "" +} + +// NormalizeOriginURL is the single invariant every published origin URL must +// satisfy. It is applied when a detector records a URL and again when export +// reads one back, so a plugin-supplied or hand-built graph is held to the same +// rule as a built-in detector. +// +// A value passes only when it is an absolute http or https URL with a host and +// no embedded credentials; the result is always re-serialized from the parse, +// never the caller's raw string. Everything else — local paths, file://, +// git@host:org/repo, ssh://, git+ssh://, and URLs carrying userinfo — is +// rejected, so filesystem layout and credentials cannot reach an SBOM. +// +// The vcs argument selects the repository form: query and fragment are dropped +// (they carry the requested ref, not the resolved one, which callers pass +// separately) and a non-empty path is required, since a bare host names no +// repository. The artifact form instead drops the fragment (a checksum or +// anchor, never part of the location) and rejects a value carrying a query, +// which marks a signed or tokenized link rather than a stable location. +func NormalizeOriginURL(raw string, vcs bool) (string, bool) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return "", false + } + parsed, err := url.Parse(trimmed) + if err != nil { + return "", false + } + switch strings.ToLower(parsed.Scheme) { + case "http", "https": + default: + return "", false + } + // Hostname also rejects a malformed host such as "https://:8080/pkg". + if parsed.Hostname() == "" || parsed.User != nil { + return "", false + } + parsed.Scheme = strings.ToLower(parsed.Scheme) + parsed.Fragment = "" + parsed.RawFragment = "" + if vcs { + parsed.RawQuery = "" + parsed.ForceQuery = false + if strings.Trim(parsed.Path, "/") == "" { + return "", false + } + } else if parsed.RawQuery != "" || parsed.ForceQuery { + return "", false + } + normalized := parsed.String() + if normalized == "" { + return "", false + } + return normalized, true +} + +// SetOriginArtifact records the exact artifact dep was resolved from. Callers +// pass the lockfile field verbatim; values that are not publishable URLs are +// dropped silently, since a missing origin is correct output and a wrong one +// is not. No-op when dep is nil. +func SetOriginArtifact(dep *sdk.Dependency, rawURL string) { + if dep == nil { + return + } + normalized, ok := NormalizeOriginURL(rawURL, false) + if !ok { + return + } + setOriginValue(dep, MetadataKeyOriginArtifactURL, normalized) +} + +// SetOriginVCS records the source repository dep was resolved from, plus the +// revision the lockfile pinned. An unpublishable URL drops the whole origin; an +// unusable revision drops only the revision, keeping the repository. No-op when +// dep is nil. +func SetOriginVCS(dep *sdk.Dependency, rawURL, revision string) { + if dep == nil { + return + } + normalized, ok := NormalizeOriginURL(rawURL, true) + if !ok { + return + } + setOriginValue(dep, MetadataKeyOriginVCSURL, normalized) + if pinned := strings.TrimSpace(revision); isValidOriginRevision(pinned) { + setOriginValue(dep, MetadataKeyOriginVCSRevision, pinned) + } +} + +// OriginFrom reads the origin a detector recorded on metadata, re-validating +// every value. Anything that fails the invariant is dropped, so export cannot +// publish a location no detector could legitimately have produced. An artifact +// wins over a repository in the case — which the setters never produce — where +// metadata carries both. +func OriginFrom(metadata map[string]any) Origin { + if len(metadata) == 0 { + return Origin{} + } + if artifact, ok := NormalizeOriginURL(originString(metadata, MetadataKeyOriginArtifactURL), false); ok { + return Origin{ArtifactURL: artifact} + } + repository, ok := NormalizeOriginURL(originString(metadata, MetadataKeyOriginVCSURL), true) + if !ok { + return Origin{} + } + origin := Origin{VCSURL: repository} + if pinned := strings.TrimSpace(originString(metadata, MetadataKeyOriginVCSRevision)); isValidOriginRevision(pinned) { + origin.VCSRevision = pinned + } + return origin +} + +// isValidOriginRevision reports whether revision is safe to publish beside a +// repository URL. The charset keeps commit hashes, tags, and branch-style refs +// while excluding whitespace, "@", and percent escapes, which would break the +// SPDX "git+@" locator grammar. +func isValidOriginRevision(revision string) bool { + if revision == "" || len(revision) > maxOriginRevisionLength { + return false + } + for _, r := range revision { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '.', r == '_', r == '-', r == '+', r == '/': + default: + return false + } + } + return true +} + +// setOriginValue stores one origin fact, allocating the metadata map on demand. +func setOriginValue(dep *sdk.Dependency, key, value string) { + if dep.Metadata == nil { + dep.Metadata = make(map[string]any, 1) + } + dep.Metadata[key] = value +} + +// originString reads a string-valued metadata entry, tolerating a map built by +// something other than the setters. +func originString(metadata map[string]any, key string) string { + value, _ := metadata[key].(string) + return value +} diff --git a/internal/detectors/origin_fuzz_test.go b/internal/detectors/origin_fuzz_test.go new file mode 100644 index 00000000..ae3e7658 --- /dev/null +++ b/internal/detectors/origin_fuzz_test.go @@ -0,0 +1,112 @@ +package detectors_test + +import ( + "net/url" + "testing" + + "github.com/bomly-dev/bomly-cli/internal/detectors" + "github.com/bomly-dev/bomly-sdk" + testutil "github.com/bomly-dev/bomly-sdk/testkit" +) + +// FuzzSetOrigin drives the origin invariant with arbitrary lockfile-derived +// strings. Detectors pass raw lockfile fields straight through, so whatever a +// repository can put in a lockfile can reach these setters. +func FuzzSetOrigin(f *testing.F) { + f.Add("https://registry.npmjs.org/react/-/react-18.2.0.tgz", "") + f.Add("https://github.com/owner/repo.git", "9f8e7d6c5b4a3928176554433221100ffeeddcc0") + f.Add("https://github.com/example/helper?rev=main#abc123", "v1.2.3") + f.Add("https://user:s3cret@nexus.corp/repo/pkg.tgz", "main") + f.Add("git+ssh://git@github.com/owner/repo.git#9f8e7d6", "9f8e7d6") + f.Add("file:///home/someone/wheels/pkg.whl", "") + f.Add("/Users/someone/src/project", "") + f.Add("http://0#0", "0") + f.Add("http://0/0#\x02", "\x02") + f.Add("%./0", "%") + f.Add("https://", "") + f.Add("https://:8080/pkg.tgz", "") + f.Add("https://例え.テスト/パッケージ.tgz", "リビジョン") + + f.Fuzz(func(t *testing.T, rawURL, revision string) { + if len(rawURL)+len(revision) > testutil.MaxFuzzInputSize { + return + } + + artifact := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginArtifact(artifact, rawURL) + repository := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginVCS(repository, rawURL, revision) + + // Whatever was stored must satisfy the invariant: export publishes + // these values into an SBOM without re-deciding anything. + assertPublishable(t, detectors.OriginFrom(artifact.Metadata)) + assertPublishable(t, detectors.OriginFrom(repository.Metadata)) + + // A second pass over the same input must reach the same conclusion, + // and reading back what was written must be a fixed point. + again := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginVCS(again, rawURL, revision) + first, second := detectors.OriginFrom(repository.Metadata), detectors.OriginFrom(again.Metadata) + if first != second { + t.Fatalf("nondeterministic origin: %+v then %+v", first, second) + } + if reread := detectors.OriginFrom(map[string]any{ + detectors.MetadataKeyOriginVCSURL: first.VCSURL, + detectors.MetadataKeyOriginVCSRevision: first.VCSRevision, + }); reread != first { + t.Fatalf("stored origin did not survive a re-read: %+v became %+v", first, reread) + } + }) +} + +// assertPublishable fails when an origin carries anything an SBOM must never +// show: a non-web location, a host-less URL, embedded credentials, or a +// revision that would break the SPDX "git+@" grammar. +func assertPublishable(t *testing.T, origin detectors.Origin) { + t.Helper() + + if origin.ArtifactURL != "" && origin.VCSURL != "" { + t.Fatalf("origin claims two locations at once: %+v", origin) + } + if origin.VCSRevision != "" && origin.VCSURL == "" { + t.Fatalf("revision %q recorded without a repository", origin.VCSRevision) + } + for _, raw := range []string{origin.ArtifactURL, origin.VCSURL} { + if raw == "" { + continue + } + parsed, err := url.Parse(raw) + if err != nil { + t.Fatalf("published URL %q does not parse: %v", raw, err) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + t.Fatalf("published URL %q is not a web location", raw) + } + if parsed.Hostname() == "" { + t.Fatalf("published URL %q has no host", raw) + } + if parsed.User != nil { + t.Fatalf("published URL %q carries credentials", raw) + } + if parsed.Fragment != "" { + t.Fatalf("published URL %q carries a fragment", raw) + } + } + if origin.VCSURL != "" { + parsed, err := url.Parse(origin.VCSURL) + if err != nil { + t.Fatalf("repository URL %q does not parse: %v", origin.VCSURL, err) + } + if parsed.RawQuery != "" || parsed.ForceQuery { + t.Fatalf("repository URL %q carries a query", origin.VCSURL) + } + } + for _, r := range origin.VCSRevision { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + case r == '.', r == '_', r == '-', r == '+', r == '/': + default: + t.Fatalf("revision %q carries %q, which breaks the SPDX locator grammar", origin.VCSRevision, r) + } + } +} diff --git a/internal/detectors/origin_test.go b/internal/detectors/origin_test.go new file mode 100644 index 00000000..834bc71e --- /dev/null +++ b/internal/detectors/origin_test.go @@ -0,0 +1,198 @@ +package detectors_test + +import ( + "strings" + "testing" + + "github.com/bomly-dev/bomly-cli/internal/detectors" + "github.com/bomly-dev/bomly-sdk" +) + +func TestSetOriginArtifact(t *testing.T) { + cases := []struct { + name string + raw string + want string + }{ + {name: "registry tarball", raw: "https://registry.npmjs.org/react/-/react-18.2.0.tgz", want: "https://registry.npmjs.org/react/-/react-18.2.0.tgz"}, + {name: "yarn digest fragment is stripped", raw: "https://registry.npmjs.org/react/-/react-18.2.0.tgz#ceeba773e3e9d2b6f1a2b6b9f4f1cb2f9c2e1a55", want: "https://registry.npmjs.org/react/-/react-18.2.0.tgz"}, + {name: "codeload tarball", raw: "https://codeload.github.com/owner/repo/tar.gz/9f8e7d6c5b4a3928176554433221100ffeeddcc", want: "https://codeload.github.com/owner/repo/tar.gz/9f8e7d6c5b4a3928176554433221100ffeeddcc"}, + {name: "uppercase scheme is normalized", raw: "HTTPS://files.pythonhosted.org/packages/x/django-5.0.tar.gz", want: "https://files.pythonhosted.org/packages/x/django-5.0.tar.gz"}, + {name: "signed link carrying a query is dropped", raw: "https://nexus.corp/repo/pkg.tgz?token=abc123", want: ""}, + {name: "userinfo is dropped", raw: "https://user:s3cret@nexus.corp/repo/pkg.tgz", want: ""}, + {name: "npm link directory is dropped", raw: "packages/lib", want: ""}, + {name: "absolute local path is dropped", raw: "/Users/someone/src/project", want: ""}, + {name: "file url is dropped", raw: "file:///home/someone/wheels/pkg.whl", want: ""}, + {name: "git+ssh is dropped", raw: "git+ssh://git@github.com/owner/repo.git#9f8e7d6", want: ""}, + {name: "scp-style remote is dropped", raw: "git@github.com:owner/repo.git", want: ""}, + {name: "git+https prefix is not a plain URL", raw: "git+https://github.com/owner/repo.git", want: ""}, + {name: "non-web scheme is dropped", raw: "ftp://files.example.com/pkg.tgz", want: ""}, + {name: "windows path is dropped", raw: `C:\src\project`, want: ""}, + {name: "malformed host is dropped", raw: "https://:8080/pkg.tgz", want: ""}, + {name: "scheme without host is dropped", raw: "https://", want: ""}, + {name: "empty is dropped", raw: " ", want: ""}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dep := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginArtifact(dep, tc.raw) + + origin := detectors.OriginFrom(dep.Metadata) + if origin.ArtifactURL != tc.want { + t.Fatalf("artifact URL = %q, want %q", origin.ArtifactURL, tc.want) + } + if origin.VCSURL != "" || origin.VCSRevision != "" { + t.Fatalf("artifact origin leaked repository data: %+v", origin) + } + if tc.want == "" && len(dep.Metadata) != 0 { + t.Fatalf("rejected value still recorded metadata: %v", dep.Metadata) + } + }) + } +} + +func TestSetOriginVCS(t *testing.T) { + cases := []struct { + name string + raw string + revision string + wantURL string + wantRevision string + }{ + { + name: "repository with resolved commit", + raw: "https://github.com/owner/repo.git", + revision: "9f8e7d6c5b4a3928176554433221100ffeeddcc0", + wantURL: "https://github.com/owner/repo.git", + wantRevision: "9f8e7d6c5b4a3928176554433221100ffeeddcc0", + }, + { + name: "requested ref in query and fragment is dropped for the resolved one", + raw: "https://github.com/example/helper?rev=main#abc123", + revision: "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567", + wantURL: "https://github.com/example/helper", + wantRevision: "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567", + }, + {name: "tag pin", raw: "https://github.com/owner/repo", revision: "v1.2.3", wantURL: "https://github.com/owner/repo", wantRevision: "v1.2.3"}, + {name: "branch-style ref", raw: "https://github.com/owner/repo", revision: "release/2026-08", wantURL: "https://github.com/owner/repo", wantRevision: "release/2026-08"}, + {name: "unpinned repository", raw: "https://github.com/owner/repo", revision: "", wantURL: "https://github.com/owner/repo"}, + {name: "revision breaking the SPDX locator keeps the repository", raw: "https://github.com/owner/repo", revision: "feature@login", wantURL: "https://github.com/owner/repo"}, + {name: "whitespace revision keeps the repository", raw: "https://github.com/owner/repo", revision: "not a revision", wantURL: "https://github.com/owner/repo"}, + {name: "overlong revision keeps the repository", raw: "https://github.com/owner/repo", revision: strings.Repeat("a", 129), wantURL: "https://github.com/owner/repo"}, + {name: "bare host names no repository", raw: "https://github.com", revision: "9f8e7d6", wantURL: ""}, + {name: "root path names no repository", raw: "https://github.com/", revision: "9f8e7d6", wantURL: ""}, + {name: "userinfo is dropped", raw: "https://oauth2:glpat-xxxxxxxxxxxxxxxxxxxx@gitlab.corp/team/repo.git", revision: "9f8e7d6", wantURL: ""}, + {name: "local checkout is dropped", raw: "/Users/someone/src/repo", revision: "9f8e7d6", wantURL: ""}, + {name: "ssh remote is dropped", raw: "ssh://git@github.com/owner/repo.git", revision: "9f8e7d6", wantURL: ""}, + {name: "ssh remote without userinfo is dropped", raw: "ssh://github.com/owner/repo.git", revision: "9f8e7d6", wantURL: ""}, + {name: "git+https prefix must be stripped by the detector", raw: "git+https://github.com/owner/repo.git", revision: "9f8e7d6", wantURL: ""}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dep := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginVCS(dep, tc.raw, tc.revision) + + origin := detectors.OriginFrom(dep.Metadata) + if origin.VCSURL != tc.wantURL { + t.Fatalf("VCS URL = %q, want %q", origin.VCSURL, tc.wantURL) + } + if origin.VCSRevision != tc.wantRevision { + t.Fatalf("VCS revision = %q, want %q", origin.VCSRevision, tc.wantRevision) + } + if origin.ArtifactURL != "" { + t.Fatalf("repository origin leaked an artifact URL: %q", origin.ArtifactURL) + } + if tc.wantURL == "" && len(dep.Metadata) != 0 { + t.Fatalf("rejected value still recorded metadata: %v", dep.Metadata) + } + }) + } +} + +func TestSetOriginAllocatesMetadataAndPreservesOtherKeys(t *testing.T) { + dep := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginArtifact(dep, "https://registry.npmjs.org/react/-/react-18.2.0.tgz") + if dep.Metadata == nil { + t.Fatal("metadata map was not allocated") + } + + dep.Metadata["unrelated"] = "keep me" + detectors.SetOriginArtifact(dep, "https://registry.npmjs.org/react/-/react-18.3.0.tgz") + if got := dep.Metadata["unrelated"]; got != "keep me" { + t.Fatalf("unrelated metadata = %v, want %q", got, "keep me") + } + if got := detectors.OriginFrom(dep.Metadata).ArtifactURL; got != "https://registry.npmjs.org/react/-/react-18.3.0.tgz" { + t.Fatalf("artifact URL = %q, want the most recent value", got) + } +} + +func TestSetOriginNilDependency(t *testing.T) { + // Must not panic: detectors call these on nodes that may not exist. + detectors.SetOriginArtifact(nil, "https://registry.npmjs.org/react/-/react-18.2.0.tgz") + detectors.SetOriginVCS(nil, "https://github.com/owner/repo", "9f8e7d6") +} + +func TestOriginFromRevalidatesHandBuiltMetadata(t *testing.T) { + cases := []struct { + name string + metadata map[string]any + want detectors.Origin + }{ + {name: "nil metadata", metadata: nil}, + {name: "unrelated keys only", metadata: map[string]any{"npm": struct{}{}}}, + { + name: "credentialed artifact is dropped", + metadata: map[string]any{detectors.MetadataKeyOriginArtifactURL: "https://user:s3cret@nexus.corp/pkg.tgz"}, + }, + { + name: "local path is dropped", + metadata: map[string]any{detectors.MetadataKeyOriginVCSURL: "file:///home/someone/repo"}, + }, + { + name: "non-string value is dropped", + metadata: map[string]any{detectors.MetadataKeyOriginArtifactURL: 42}, + }, + { + name: "revision without a repository is dropped", + metadata: map[string]any{detectors.MetadataKeyOriginVCSRevision: "9f8e7d6"}, + }, + { + name: "artifact wins over repository", + metadata: map[string]any{ + detectors.MetadataKeyOriginArtifactURL: "https://registry.npmjs.org/react/-/react-18.2.0.tgz", + detectors.MetadataKeyOriginVCSURL: "https://github.com/facebook/react", + }, + want: detectors.Origin{ArtifactURL: "https://registry.npmjs.org/react/-/react-18.2.0.tgz"}, + }, + { + name: "query and fragment are stripped from a hand-built repository", + metadata: map[string]any{ + detectors.MetadataKeyOriginVCSURL: "https://github.com/owner/repo?rev=main#abc", + detectors.MetadataKeyOriginVCSRevision: "9f8e7d6", + }, + want: detectors.Origin{VCSURL: "https://github.com/owner/repo", VCSRevision: "9f8e7d6"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := detectors.OriginFrom(tc.metadata); got != tc.want { + t.Fatalf("OriginFrom() = %+v, want %+v", got, tc.want) + } + }) + } +} + +func TestOriginEmpty(t *testing.T) { + if !(detectors.Origin{}).Empty() { + t.Fatal("zero Origin should be empty") + } + if (detectors.Origin{ArtifactURL: "https://example.com/pkg.tgz"}).Empty() { + t.Fatal("artifact origin should not be empty") + } + if (detectors.Origin{VCSURL: "https://example.com/repo"}).Empty() { + t.Fatal("repository origin should not be empty") + } +} diff --git a/internal/output/origin_metadata_test.go b/internal/output/origin_metadata_test.go new file mode 100644 index 00000000..7d4144a5 --- /dev/null +++ b/internal/output/origin_metadata_test.go @@ -0,0 +1,79 @@ +package output + +import ( + "testing" + + "github.com/bomly-dev/bomly-cli/internal/detectors" + "github.com/bomly-dev/bomly-sdk" +) + +// Origin metadata is a transport between detection and SBOM export. It must not +// surface in scan/diff/explain payloads, where it would be noise in every +// package entry and churn every golden. +func TestPackageRefOmitsOriginMetadata(t *testing.T) { + dep := sdk.NewDependencyWithID("npm:react", sdk.Dependency{ + Coordinates: sdk.Coordinates{ + PURL: "pkg:npm/react@18.2.0", + Ecosystem: sdk.EcosystemNPM, + Name: "react", + Version: "18.2.0", + }, + }) + detectors.SetOriginArtifact(dep, "https://registry.npmjs.org/react/-/react-18.2.0.tgz") + dep.Metadata["npm"] = &sdk.NPMPackageMetadata{Bundled: true} + + ref := PackageFromDependencyAndRegistry(dep, nil) + + if _, found := ref.Metadata[detectors.MetadataKeyOriginArtifactURL]; found { + t.Fatalf("origin metadata reached command output: %v", ref.Metadata) + } + if _, found := ref.Metadata["npm"]; !found { + t.Fatalf("unrelated metadata was dropped: %v", ref.Metadata) + } +} + +// A dependency whose only metadata is origin must render no metadata object at +// all, or `omitempty` stops firing and every such package grows an empty block. +func TestPackageRefMetadataAbsentWhenOnlyOrigin(t *testing.T) { + dep := sdk.NewDependencyWithID("npm:react", sdk.Dependency{ + Coordinates: sdk.Coordinates{ + PURL: "pkg:npm/react@18.2.0", + Ecosystem: sdk.EcosystemNPM, + Name: "react", + Version: "18.2.0", + }, + }) + detectors.SetOriginVCS(dep, "https://github.com/facebook/react", "9f8e7d6c5b4a3928176554433221100ffeeddcc0") + + if ref := PackageFromDependencyAndRegistry(dep, nil); ref.Metadata != nil { + t.Fatalf("metadata = %v, want nil so the field is omitted", ref.Metadata) + } +} + +// The same filter guards the registry-sourced package listing. +func TestScanPackageEntriesOmitOriginMetadata(t *testing.T) { + registry := sdk.NewPackageRegistry() + registry.Add(&sdk.Package{ + Coordinates: sdk.Coordinates{ + PURL: "pkg:npm/react@18.2.0", + Ecosystem: sdk.EcosystemNPM, + Name: "react", + Version: "18.2.0", + }, + Metadata: map[string]any{ + detectors.MetadataKeyOriginArtifactURL: "https://registry.npmjs.org/react/-/react-18.2.0.tgz", + "npm": &sdk.NPMPackageMetadata{Bundled: true}, + }, + }) + + entries := PackagesFromRegistry(registry) + if len(entries) != 1 { + t.Fatalf("got %d package entries, want 1", len(entries)) + } + if _, found := entries[0].Metadata[detectors.MetadataKeyOriginArtifactURL]; found { + t.Fatalf("origin metadata reached the package listing: %v", entries[0].Metadata) + } + if _, found := entries[0].Metadata["npm"]; !found { + t.Fatalf("unrelated metadata was dropped: %v", entries[0].Metadata) + } +} diff --git a/internal/output/types.go b/internal/output/types.go index 2dbfcd06..354d4bbd 100644 --- a/internal/output/types.go +++ b/internal/output/types.go @@ -4,6 +4,7 @@ import ( "sort" "strings" + "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) @@ -262,14 +263,25 @@ func cloneAffectedSymbols(src []sdk.AffectedSymbol) []sdk.AffectedSymbol { return out } +// cloneRefMetadata copies package metadata for command output, dropping keys +// that carry data between pipeline stages rather than facts about the package. +// Origin keys are such a transport: detectors record where a package came from +// so SBOM export can publish it, and the SBOM is where users read it. func cloneRefMetadata(src map[string]any) map[string]any { if len(src) == 0 { return nil } clone := make(map[string]any, len(src)) for key, value := range src { + if strings.HasPrefix(key, detectors.MetadataKeyOriginPrefix) { + continue + } clone[key] = value } + // Metadata is omitempty; an emptied map must read as absent, not as {}. + if len(clone) == 0 { + return nil + } return clone } From 130f38daea3e5e3826de2f4312aa1ab9efcc585e Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 03:03:10 -0400 Subject: [PATCH 02/17] feat(detectors): emit package origin from lockfile source fields Each detector now says where a package came from, using the field its own lockfile records it in: - npm, pnpm, yarn, and bun assert the registry tarball they fetched. Yarn Classic's checksum fragment is dropped, pnpm v9 entries carrying only an integrity hash assert nothing, and npm workspace members keep asserting nothing because their "resolved" is a local directory. - uv, poetry, pipenv, and pip read their explicit source types: a repository plus the commit that was locked, a direct archive URL, or nothing for index installs, editable projects, and local paths. - cargo unwraps "git+", taking the resolved commit from the URL fragment and falling back to the requested rev/tag/branch; index sources assert nothing. - Bundler emits for GIT sections, SwiftPM for source-control pins, and pub for git packages -- not for gem servers, registry pins, or local checkouts. Registry and index roots are deliberately absent everywhere: they say where an ecosystem fetches from, not where this package came from, and a private server URL with a path is indistinguishable from a repository once it is out of context. ResolvedURL keeps its existing value at every site, so repository resolution in the scorecard matcher is unchanged. Co-Authored-By: Claude Opus 5 --- internal/detectors/cargo/detector.go | 4 +- internal/detectors/cargo/origin.go | 42 ++++ internal/detectors/cargo/origin_test.go | 92 ++++++++ internal/detectors/cargo/workspace.go | 4 +- .../detectors/node/bun/bun_lockfile_parser.go | 4 + .../detectors/node/npm/npm_lockfile_parser.go | 5 + internal/detectors/node/npm/origin_test.go | 64 ++++++ .../detectors/node/origin_integration_test.go | 115 ++++++++++ .../node/pnpm/pnpm_lockfile_parser.go | 5 + .../node/yarn/yarn_lockfile_parser.go | 5 + internal/detectors/pub/detector.go | 5 + internal/detectors/pub/origin_test.go | 82 +++++++ internal/detectors/python/common.go | 1 + internal/detectors/python/origin.go | 72 ++++++ internal/detectors/python/origin_test.go | 214 ++++++++++++++++++ internal/detectors/python/pipenv.go | 1 + internal/detectors/python/poetrylock.go | 1 + internal/detectors/python/uvlock.go | 1 + internal/detectors/ruby/detector.go | 8 +- internal/detectors/ruby/origin_test.go | 70 ++++++ internal/detectors/swiftpm/detector.go | 7 + internal/detectors/swiftpm/origin_test.go | 64 ++++++ 22 files changed, 863 insertions(+), 3 deletions(-) create mode 100644 internal/detectors/cargo/origin.go create mode 100644 internal/detectors/cargo/origin_test.go create mode 100644 internal/detectors/node/npm/origin_test.go create mode 100644 internal/detectors/node/origin_integration_test.go create mode 100644 internal/detectors/pub/origin_test.go create mode 100644 internal/detectors/python/origin.go create mode 100644 internal/detectors/python/origin_test.go create mode 100644 internal/detectors/ruby/origin_test.go create mode 100644 internal/detectors/swiftpm/origin_test.go diff --git a/internal/detectors/cargo/detector.go b/internal/detectors/cargo/detector.go index 83d2c503..19c4fe69 100644 --- a/internal/detectors/cargo/detector.go +++ b/internal/detectors/cargo/detector.go @@ -382,7 +382,7 @@ func packageNode(pkg metadataPackage, id string, workspace map[string]struct{}) source = sdk.DependencySourceWorkspace } } - return sdk.NewDependency(sdk.Dependency{Coordinates: sdk.Coordinates{Ecosystem: sdk.EcosystemRust, + node := sdk.NewDependency(sdk.Dependency{Coordinates: sdk.Coordinates{Ecosystem: sdk.EcosystemRust, Name: pkg.Name, Version: pkg.Version, PackageManager: sdk.PackageManagerCargo, @@ -390,6 +390,8 @@ func packageNode(pkg metadataPackage, id string, workspace map[string]struct{}) Language: "rust", PURL: sdk.BuildPackageURL("cargo", "", pkg.Name, pkg.Version)}, Source: source, ResolvedURL: pkg.Source, }) + setCargoOrigin(node, pkg.Source) + return node } diff --git a/internal/detectors/cargo/origin.go b/internal/detectors/cargo/origin.go new file mode 100644 index 00000000..6ae26d78 --- /dev/null +++ b/internal/detectors/cargo/origin.go @@ -0,0 +1,42 @@ +package cargo + +import ( + "net/url" + "strings" + + "github.com/bomly-dev/bomly-cli/internal/detectors" + "github.com/bomly-dev/bomly-sdk" +) + +// setCargoOrigin records the repository a git-sourced crate was resolved from. +// Cargo writes one source string per package: "registry+"/"sparse+" name an +// index root rather than this crate's location, path and workspace members +// carry no source at all, and only "git+" identifies where the code came from. +func setCargoOrigin(node *sdk.Dependency, source string) { + trimmed := strings.TrimSpace(source) + if !strings.HasPrefix(trimmed, "git+") { + return + } + repository := strings.TrimPrefix(trimmed, "git+") + detectors.SetOriginVCS(node, repository, cargoSourceRevision(repository)) +} + +// cargoSourceRevision returns the revision cargo locked. The URL fragment holds +// the resolved commit; the "rev", "tag", and "branch" query parameters hold +// what the manifest asked for, which is the weaker answer. +func cargoSourceRevision(repository string) string { + parsed, err := url.Parse(strings.TrimSpace(repository)) + if err != nil { + return "" + } + if parsed.Fragment != "" { + return parsed.Fragment + } + query := parsed.Query() + for _, key := range []string{"rev", "tag", "branch"} { + if value := strings.TrimSpace(query.Get(key)); value != "" { + return value + } + } + return "" +} diff --git a/internal/detectors/cargo/origin_test.go b/internal/detectors/cargo/origin_test.go new file mode 100644 index 00000000..56ecfb55 --- /dev/null +++ b/internal/detectors/cargo/origin_test.go @@ -0,0 +1,92 @@ +package cargo + +import ( + "testing" + + "github.com/bomly-dev/bomly-cli/internal/detectors" + "github.com/bomly-dev/bomly-sdk" +) + +// Cargo.lock records one source string per package. Only "git+" names where the +// code came from; the index prefixes name a registry, and path or workspace +// members carry no source at all. +func TestSetCargoOriginBySourcePrefix(t *testing.T) { + cases := []struct { + name string + source string + want detectors.Origin + }{ + { + name: "git dependency pins the resolved commit in the fragment", + source: "git+https://github.com/example/helper?rev=main#3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f", + want: detectors.Origin{VCSURL: "https://github.com/example/helper", VCSRevision: "3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f"}, + }, + { + name: "requested tag is used when no commit was recorded", + source: "git+https://github.com/example/helper?tag=v1.2.3", + want: detectors.Origin{VCSURL: "https://github.com/example/helper", VCSRevision: "v1.2.3"}, + }, + { + name: "branch dependency without a pin keeps the repository", + source: "git+https://github.com/example/helper", + want: detectors.Origin{VCSURL: "https://github.com/example/helper"}, + }, + {name: "crates.io index root", source: "registry+https://github.com/rust-lang/crates.io-index"}, + {name: "sparse index root", source: "sparse+https://index.crates.io/"}, + {name: "path or workspace member", source: ""}, + {name: "credentialed private git remote", source: "git+https://token:s3cret-value-here@git.corp/team/helper#4d5e6f70"}, + {name: "ssh git remote", source: "git+ssh://git@github.com/example/helper#5e6f7081"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + node := sdk.NewDependency(sdk.Dependency{Coordinates: sdk.Coordinates{Name: "helper", Version: "1.0.0"}}) + setCargoOrigin(node, tc.source) + if got := detectors.OriginFrom(node.Metadata); got != tc.want { + t.Fatalf("origin = %+v, want %+v", got, tc.want) + } + }) + } +} + +// The lockfile path builds nodes through the same helper. +func TestCargoLockGraphCarriesOrigin(t *testing.T) { + lock := []byte(` +[[package]] +name = "demo" +version = "0.1.0" +dependencies = ["helper", "serde"] + +[[package]] +name = "helper" +version = "1.0.0" +source = "git+https://github.com/example/helper?rev=main#6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192" + +[[package]] +name = "serde" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +`) + manifest := []byte("[package]\nname = \"demo\"\nversion = \"0.1.0\"\n[dependencies]\nhelper = { git = \"https://github.com/example/helper\" }\nserde = \"1\"\n") + + graph, err := depGraphFromLock(lock, manifest) + if err != nil { + t.Fatalf("depGraphFromLock() error = %v", err) + } + + helper, ok := graph.Node("helper@1.0.0") + if !ok { + t.Fatal("expected helper in graph") + } + want := detectors.Origin{VCSURL: "https://github.com/example/helper", VCSRevision: "6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192"} + if got := detectors.OriginFrom(helper.Metadata); got != want { + t.Fatalf("helper origin = %+v, want %+v", got, want) + } + serde, ok := graph.Node("serde@1.0.0") + if !ok { + t.Fatal("expected serde in graph") + } + if got := detectors.OriginFrom(serde.Metadata); !got.Empty() { + t.Fatalf("registry crate asserted an origin: %+v", got) + } +} diff --git a/internal/detectors/cargo/workspace.go b/internal/detectors/cargo/workspace.go index eebdcf13..1e600b29 100644 --- a/internal/detectors/cargo/workspace.go +++ b/internal/detectors/cargo/workspace.go @@ -181,7 +181,7 @@ func depGraphFromLockWorkspace(lockRaw []byte, rootManifest cargoManifest, membe pkgType = "application" source = sdk.DependencySourceWorkspace } - return sdk.NewDependency(sdk.Dependency{Coordinates: sdk.Coordinates{Ecosystem: sdk.EcosystemRust, + node := sdk.NewDependency(sdk.Dependency{Coordinates: sdk.Coordinates{Ecosystem: sdk.EcosystemRust, Name: pkg.Name, Version: pkg.Version, PackageManager: sdk.PackageManagerCargo, @@ -189,6 +189,8 @@ func depGraphFromLockWorkspace(lockRaw []byte, rootManifest cargoManifest, membe Language: "rust", PURL: sdk.BuildPackageURL("cargo", "", pkg.Name, pkg.Version)}, Source: source, ResolvedURL: pkg.Source, }) + setCargoOrigin(node, pkg.Source) + return node } lockPackageFor := func(manifest cargoManifest) lockPackage { if pkg, ok := byName[manifest.Name]; ok && pkg.Version != "" { diff --git a/internal/detectors/node/bun/bun_lockfile_parser.go b/internal/detectors/node/bun/bun_lockfile_parser.go index 153eff8f..7fcf960d 100644 --- a/internal/detectors/node/bun/bun_lockfile_parser.go +++ b/internal/detectors/node/bun/bun_lockfile_parser.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/Masterminds/semver/v3" + "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-cli/internal/detectors/node" "github.com/bomly-dev/bomly-sdk" "github.com/bomly-dev/bomly-sdk/system" @@ -141,6 +142,9 @@ func depGraphFromBunLockfile(projectPath string) (bunLockfileGraphs, error) { if _, exists := graph.Node(pkgNode.ID); exists { pkgNode = sdk.NewDependencyWithID("bun-package:"+key, dep) } + // Bun's tuple carries the registry tarball it fetched. Workspace + // members and git specs resolve to values the invariant rejects. + detectors.SetOriginArtifact(pkgNode, entry.resolved) if err := node.AddNodeIfMissing(graph, pkgNode); err != nil { return bunLockfileGraphs{}, err } diff --git a/internal/detectors/node/npm/npm_lockfile_parser.go b/internal/detectors/node/npm/npm_lockfile_parser.go index 7828f1f5..7be67391 100644 --- a/internal/detectors/node/npm/npm_lockfile_parser.go +++ b/internal/detectors/node/npm/npm_lockfile_parser.go @@ -9,6 +9,7 @@ import ( "sort" "strings" + "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-cli/internal/detectors/node" "github.com/bomly-dev/bomly-sdk" "github.com/bomly-dev/bomly-sdk/system" @@ -247,6 +248,10 @@ func depGraphFromNPMLockfile(projectPath string) (npmLockfileGraphs, error) { pkg.Metadata = map[string]any{sdk.MetadataKeyNPM: meta} } pkgNode := sdk.NewDependency(pkg) + // npm records the registry tarball a package was installed from. + // Workspace members cleared ResolvedURL above (it names a local + // directory), and git or file specs are rejected by the invariant. + detectors.SetOriginArtifact(pkgNode, pkg.ResolvedURL) if entry.License != "" { sdk.SetDetectionLicenses(pkgNode, []sdk.PackageLicense{{Value: entry.License, Type: "declared"}}) } diff --git a/internal/detectors/node/npm/origin_test.go b/internal/detectors/node/npm/origin_test.go new file mode 100644 index 00000000..f79b7a8f --- /dev/null +++ b/internal/detectors/node/npm/origin_test.go @@ -0,0 +1,64 @@ +package npm + +import ( + "os" + "path/filepath" + "testing" + + "github.com/bomly-dev/bomly-cli/internal/detectors" +) + +// npm writes whatever it installed from into "resolved": a registry tarball, +// but also a git remote or a local path. Only the first is a location an SBOM +// can publish. +func TestNPMOriginByResolvedShape(t *testing.T) { + projectDir := t.TempDir() + lockfile := `{ + "name": "demo", + "version": "1.0.0", + "lockfileVersion": 3, + "packages": { + "": {"name": "demo", "version": "1.0.0", "dependencies": {"from-registry": "1.0.0", "from-git": "1.0.0", "from-file": "1.0.0", "from-private": "1.0.0"}}, + "node_modules/from-registry": {"version": "1.0.0", "resolved": "https://registry.npmjs.org/from-registry/-/from-registry-1.0.0.tgz"}, + "node_modules/from-git": {"version": "1.0.0", "resolved": "git+ssh://git@github.com/owner/repo.git#9f8e7d6c5b4a3928176554433221100ffeeddcc0"}, + "node_modules/from-file": {"version": "1.0.0", "resolved": "file:../vendor/from-file"}, + "node_modules/from-private": {"version": "1.0.0", "resolved": "https://build:s3cret-token-value@nexus.corp/repo/from-private-1.0.0.tgz"} + } + }` + if err := os.WriteFile(filepath.Join(projectDir, "package-lock.json"), []byte(lockfile), 0o644); err != nil { + t.Fatal(err) + } + + graphs, err := depGraphFromNPMLockfile(projectDir) + if err != nil { + t.Fatalf("depGraphFromNPMLockfile() error = %v", err) + } + + cases := []struct { + id string + want string + }{ + {id: "from-registry@1.0.0", want: "https://registry.npmjs.org/from-registry/-/from-registry-1.0.0.tgz"}, + {id: "from-git@1.0.0"}, // a git remote npm can reach, not a published location + {id: "from-file@1.0.0"}, // a path on the machine that ran npm install + {id: "from-private@1.0.0"}, // carries a credential + } + for _, tc := range cases { + node, ok := graphs.graph.Node(tc.id) + if !ok { + t.Fatalf("expected %s in graph", tc.id) + } + origin := detectors.OriginFrom(node.Metadata) + if origin.ArtifactURL != tc.want { + t.Errorf("%s artifact origin = %q, want %q", tc.id, origin.ArtifactURL, tc.want) + } + if origin.VCSURL != "" { + t.Errorf("%s asserted a repository %q", tc.id, origin.VCSURL) + } + // ResolvedURL is a separate contract (the scorecard matcher resolves + // repositories from it) and must keep carrying the raw lockfile value. + if node.ResolvedURL == "" { + t.Errorf("%s lost its ResolvedURL", tc.id) + } + } +} diff --git a/internal/detectors/node/origin_integration_test.go b/internal/detectors/node/origin_integration_test.go new file mode 100644 index 00000000..d58c8955 --- /dev/null +++ b/internal/detectors/node/origin_integration_test.go @@ -0,0 +1,115 @@ +package node_test + +import ( + "context" + "testing" + + "github.com/bomly-dev/bomly-cli/internal/detectors" + "github.com/bomly-dev/bomly-cli/internal/detectors/node/bun" + "github.com/bomly-dev/bomly-cli/internal/detectors/node/npm" + "github.com/bomly-dev/bomly-cli/internal/detectors/node/pnpm" + "github.com/bomly-dev/bomly-cli/internal/detectors/node/yarn" + "github.com/bomly-dev/bomly-sdk" +) + +// requireArtifactOrigin asserts a package asserts exactly the given artifact. +func requireArtifactOrigin(t *testing.T, g *sdk.Graph, name, version, want string) { + t.Helper() + origin := detectors.OriginFrom(requirePackage(t, g, name, version).Metadata) + if origin.ArtifactURL != want { + t.Errorf("%s@%s artifact origin = %q, want %q", name, version, origin.ArtifactURL, want) + } + if origin.VCSURL != "" { + t.Errorf("%s@%s also asserted a repository %q", name, version, origin.VCSURL) + } +} + +// requireNoOrigin asserts a package publishes no location at all. +func requireNoOrigin(t *testing.T, g *sdk.Graph, name, version string) { + t.Helper() + if origin := detectors.OriginFrom(requirePackage(t, g, name, version).Metadata); !origin.Empty() { + t.Errorf("%s@%s asserted an origin it should not have: %+v", name, version, origin) + } +} + +func TestNPMLockfileOriginIsTheRegistryTarball(t *testing.T) { + g, err := resolveLockfileGraph(t, npm.LockfileDetector{}, fixture("npm-v3")) + if err != nil { + t.Fatal(err) + } + requireArtifactOrigin(t, g, "lodash", "4.17.21", "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz") + requireArtifactOrigin(t, g, "jest", "29.7.0", "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz") +} + +// Workspace members are local directories. npm records that directory as the +// member's "resolved" value, which must never reach an SBOM. +func TestNPMWorkspaceMembersAssertNoOrigin(t *testing.T) { + result, err := (npm.LockfileDetector{}).ResolveGraph(context.Background(), sdk.DetectionRequest{ProjectPath: fixture("npm-v3-workspaces")}) + if err != nil { + t.Fatal(err) + } + g, err := result.Graphs.ConsolidatedGraph() + if err != nil { + t.Fatal(err) + } + requireArtifactOrigin(t, g, "lodash", "4.17.21", "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz") + requireNoOrigin(t, g, "web", "0.2.0") + requireNoOrigin(t, g, "lib", "1.0.0") +} + +func TestPNPMLockfileOriginIsTheResolutionTarball(t *testing.T) { + g, err := resolveLockfileGraph(t, pnpm.LockfileDetector{}, fixture("pnpm-v5")) + if err != nil { + t.Fatal(err) + } + requireArtifactOrigin(t, g, "react", "18.2.0", "https://registry.npmjs.org/react/-/react-18.2.0.tgz") +} + +// pnpm v9 records only an integrity hash for registry packages. A hash is not a +// location, and the registry root is not this package's origin, so there is +// nothing to assert. +func TestPNPMIntegrityOnlyEntriesAssertNoOrigin(t *testing.T) { + result, err := (pnpm.LockfileDetector{}).ResolveGraph(context.Background(), sdk.DetectionRequest{ProjectPath: fixture("pnpm-v9-workspaces")}) + if err != nil { + t.Fatal(err) + } + g, err := result.Graphs.ConsolidatedGraph() + if err != nil { + t.Fatal(err) + } + requireNoOrigin(t, g, "lodash", "4.17.21") + requireNoOrigin(t, g, "shared-transitive", "2.0.0") +} + +// Yarn Classic appends the package checksum to the tarball URL as a fragment. +// It identifies the file's contents, not a location, so it is dropped. +func TestYarnClassicOriginDropsTheChecksumFragment(t *testing.T) { + g, err := resolveLockfileGraph(t, yarn.LockfileDetector{}, fixture("yarn-v1")) + if err != nil { + t.Fatal(err) + } + requireArtifactOrigin(t, g, "react", "18.2.0", "https://registry.npmjs.org/react/-/react-18.2.0.tgz") + requireArtifactOrigin(t, g, "js-tokens", "4.0.0", "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz") +} + +// Berry lockfiles record a resolution identity rather than a fetched URL. +func TestYarnBerryAssertsNoOrigin(t *testing.T) { + g, err := resolveLockfileGraph(t, yarn.LockfileDetector{}, fixture("yarn-berry")) + if err != nil { + t.Fatal(err) + } + requireNoOrigin(t, g, "react", "18.2.0") +} + +func TestBunLockfileOriginIsTheRegistryTarball(t *testing.T) { + result, err := (bun.LockfileDetector{}).ResolveGraph(context.Background(), sdk.DetectionRequest{ProjectPath: fixture("bun-v1-workspaces")}) + if err != nil { + t.Fatal(err) + } + g, err := result.Graphs.ConsolidatedGraph() + if err != nil { + t.Fatal(err) + } + requireArtifactOrigin(t, g, "is-number", "7.0.0", "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz") + requireNoOrigin(t, g, "workspace:packages/lib", "") +} diff --git a/internal/detectors/node/pnpm/pnpm_lockfile_parser.go b/internal/detectors/node/pnpm/pnpm_lockfile_parser.go index 19100f8e..b99a74c7 100644 --- a/internal/detectors/node/pnpm/pnpm_lockfile_parser.go +++ b/internal/detectors/node/pnpm/pnpm_lockfile_parser.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-cli/internal/detectors/node" "github.com/bomly-dev/bomly-sdk" "github.com/bomly-dev/bomly-sdk/system" @@ -138,6 +139,10 @@ func depGraphFromPNPMLockfile(projectPath string) (pnpmLockfileGraphs, error) { if existing, ok := depsGraph.Node(pkgNode.ID); ok && existing.Type == sdk.PackageTypeApplication { pkgNode = sdk.NewDependencyWithID("pnpm-package:"+key, pkg) } + // pnpm records a tarball only when it resolved one; v9 lockfiles + // often carry just an integrity hash, and git or directory + // resolutions are not parsed, so those packages assert no origin. + detectors.SetOriginArtifact(pkgNode, entry.Resolution.Tarball) if entry.License != "" { sdk.SetDetectionLicenses(pkgNode, []sdk.PackageLicense{{Value: entry.License, Type: "declared"}}) } diff --git a/internal/detectors/node/yarn/yarn_lockfile_parser.go b/internal/detectors/node/yarn/yarn_lockfile_parser.go index c69024cc..cd552219 100644 --- a/internal/detectors/node/yarn/yarn_lockfile_parser.go +++ b/internal/detectors/node/yarn/yarn_lockfile_parser.go @@ -10,6 +10,7 @@ import ( "unicode" "github.com/Masterminds/semver/v3" + "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-cli/internal/detectors/node" "github.com/bomly-dev/bomly-sdk" "github.com/bomly-dev/bomly-sdk/system" @@ -81,6 +82,10 @@ func depGraphFromYarnLockfile(projectPath string) (*sdk.Graph, error) { if existing, ok := depsGraph.Node(pkgNode.ID); ok && existing.Type == sdk.PackageTypeApplication { pkgNode = sdk.NewDependencyWithID(fmt.Sprintf("yarn-package:%d", idx), pkg) } + // Yarn Classic records the tarball it fetched, with the package + // checksum as a URL fragment the invariant strips. Berry entries + // carry no resolved location, and git specs are rejected. + detectors.SetOriginArtifact(pkgNode, entry.Resolved) if err := node.AddNodeIfMissing(depsGraph, pkgNode); err != nil { return "", err } diff --git a/internal/detectors/pub/detector.go b/internal/detectors/pub/detector.go index 5cf03c5d..e442963f 100644 --- a/internal/detectors/pub/detector.go +++ b/internal/detectors/pub/detector.go @@ -198,6 +198,11 @@ func packageNode(name string, pkg pubLockPackage) *sdk.Dependency { if resolved := resolvedURL(pkg.Description); resolved != "" { node.ResolvedURL = resolved } + if pubDependencySource(pkg.Source) == sdk.DependencySourceGit { + // A git package names its repository and the commit pub resolved. + // A hosted package's "url" is the pub server, and path is local. + detectors.SetOriginVCS(node, descriptionString(pkg.Description, "url"), descriptionString(pkg.Description, "resolved-ref")) + } return node } diff --git a/internal/detectors/pub/origin_test.go b/internal/detectors/pub/origin_test.go new file mode 100644 index 00000000..20ac4413 --- /dev/null +++ b/internal/detectors/pub/origin_test.go @@ -0,0 +1,82 @@ +package pub + +import ( + "testing" + + "github.com/bomly-dev/bomly-cli/internal/detectors" +) + +// A pubspec.lock hosted package's description URL is the pub server, shared by +// every hosted package, and a path package is local. Only a git package names +// where its own code came from. +func TestPubOriginBySourceType(t *testing.T) { + lock := []byte(`packages: + collection: + dependency: transitive + description: + name: collection + sha256: abc + url: "https://pub.dev" + source: hosted + version: "1.18.0" + corp_widgets: + dependency: transitive + description: + name: corp_widgets + sha256: def + url: "https://dart.corp/internal/feed" + source: hosted + version: "3.1.0" + helper: + dependency: "direct main" + description: + url: "https://github.com/example/helper.git" + ref: main + resolved-ref: a3b4c5d6e7f8091a2b3c4d5e6f70819213243546 + path: "." + source: git + version: "2.0.0" + local_tools: + dependency: "direct dev" + description: + path: "../local_tools" + relative: true + source: path + version: "0.1.0" +`) + manifest := pubspec{ + Name: "demo", + Version: "1.0.0", + Dependencies: map[string]any{"helper": "any"}, + DevDependencies: map[string]any{"local_tools": "any"}, + } + + graph, err := depGraphFromLock(lock, manifest) + if err != nil { + t.Fatalf("depGraphFromLock() error = %v", err) + } + + cases := []struct { + id string + want detectors.Origin + }{ + {id: "collection@1.18.0"}, + // A self-hosted pub server's URL has a path, so nothing but the + // source kind distinguishes it from a repository URL. + {id: "corp_widgets@3.1.0"}, + {id: "helper@2.0.0", want: detectors.Origin{ + VCSURL: "https://github.com/example/helper.git", + VCSRevision: "a3b4c5d6e7f8091a2b3c4d5e6f70819213243546", + }}, + {id: "local_tools@0.1.0"}, + } + for _, tc := range cases { + node, ok := graph.Node(tc.id) + if !ok { + t.Fatalf("expected %s in graph", tc.id) + } + if got := detectors.OriginFrom(node.Metadata); got != tc.want { + t.Errorf("%s origin = %+v, want %+v", tc.id, got, tc.want) + } + } +} diff --git a/internal/detectors/python/common.go b/internal/detectors/python/common.go index cb13691e..71456702 100644 --- a/internal/detectors/python/common.go +++ b/internal/detectors/python/common.go @@ -217,6 +217,7 @@ func depGraphFromPipInspect(raw []byte, rootNode *sdk.Dependency, declared map[s Name: normalizePythonName(pkg.Metadata.Name), Version: pkg.Metadata.Version}, Source: pipInspectDependencySource(pkg.DirectURL), ResolvedURL: pipInspectResolvedURL(pkg.DirectURL), Metadata: sourceRevisionMetadata(pipInspectRevision(pkg.DirectURL)), }) + setPipInspectOrigin(node, pkg.DirectURL) if _, exists := nodesByName[node.Name]; !exists { nodesByName[node.Name] = node diff --git a/internal/detectors/python/origin.go b/internal/detectors/python/origin.go new file mode 100644 index 00000000..29587738 --- /dev/null +++ b/internal/detectors/python/origin.go @@ -0,0 +1,72 @@ +package python + +import ( + "strings" + + "github.com/bomly-dev/bomly-cli/internal/detectors" + "github.com/bomly-dev/bomly-sdk" +) + +// Python lockfiles name a package's source explicitly, so each resolver can say +// what it resolved rather than leaving the shape of a URL to be guessed at. +// Index roots (PyPI, a private mirror) and local checkouts assert nothing: they +// describe how the environment was built, not where this package came from. + +// setUVOrigin records the origin uv resolved for a package. +func setUVOrigin(node *sdk.Dependency, source uvLockSource) { + switch { + case strings.TrimSpace(source.Git) != "": + // uv writes the resolved commit as the URL fragment and the + // requested ref as a query parameter; uvSourceRevision prefers the + // former, and the invariant drops both from the repository URL. + detectors.SetOriginVCS(node, source.Git, uvSourceRevision(source)) + case strings.TrimSpace(source.URL) != "": + detectors.SetOriginArtifact(node, source.URL) + } +} + +// setPoetryOrigin records the origin poetry resolved for a package. +func setPoetryOrigin(node *sdk.Dependency, pkg *poetryLockPackage) { + switch strings.ToLower(strings.TrimSpace(pkg.Source.Type)) { + case "git": + // ResolvedReference is the commit poetry locked; Reference is the + // branch or tag that was asked for. + detectors.SetOriginVCS(node, pkg.Source.URL, firstNonEmpty(pkg.Source.ResolvedReference, pkg.Source.Reference)) + case "url": + detectors.SetOriginArtifact(node, pkg.Source.URL) + } +} + +// setPipenvOrigin records the origin pipenv resolved for a package. +func setPipenvOrigin(node *sdk.Dependency, pkg pipfileLockPackage) { + switch { + case strings.TrimSpace(pkg.Git) != "": + detectors.SetOriginVCS(node, pkg.Git, pkg.Ref) + case strings.TrimSpace(pkg.File) != "": + // "file" holds a remote archive for URL requirements and a local + // path for file:// ones; the invariant keeps only the former. + detectors.SetOriginArtifact(node, pkg.File) + } +} + +// setPipInspectOrigin records the origin recorded in an installed package's +// PEP 610 direct_url.json, which pip writes only for packages installed from a +// repository, an archive URL, or a local directory. +func setPipInspectOrigin(node *sdk.Dependency, directURL map[string]any) { + resolved := pipInspectResolvedURL(directURL) + if resolved == "" { + return + } + if vcsInfo, ok := directURL["vcs_info"].(map[string]any); ok { + vcs, _ := vcsInfo["vcs"].(string) + if strings.EqualFold(strings.TrimSpace(vcs), "git") { + detectors.SetOriginVCS(node, resolved, pipInspectRevision(directURL)) + } + // Mercurial, Subversion, and Bazaar have no locator form here. + return + } + if _, ok := directURL["archive_info"]; ok { + detectors.SetOriginArtifact(node, resolved) + } + // dir_info marks a local directory install. +} diff --git a/internal/detectors/python/origin_test.go b/internal/detectors/python/origin_test.go new file mode 100644 index 00000000..1c094d4e --- /dev/null +++ b/internal/detectors/python/origin_test.go @@ -0,0 +1,214 @@ +package python + +import ( + "os" + "path/filepath" + "testing" + + "github.com/bomly-dev/bomly-cli/internal/detectors" + "github.com/bomly-dev/bomly-sdk" +) + +// requireOrigin asserts the exact origin a named package asserts. +func requireOrigin(t *testing.T, graph *sdk.Graph, id string, want detectors.Origin) { + t.Helper() + node, ok := graph.Node(id) + if !ok { + t.Fatalf("expected %s in graph", id) + } + if got := detectors.OriginFrom(node.Metadata); got != want { + t.Errorf("%s origin = %+v, want %+v", id, got, want) + } +} + +func TestUVLockOriginBySourceType(t *testing.T) { + dir := t.TempDir() + lock := ` +version = 1 + +[[package]] +name = "project" +version = "0.1.0" +source = { editable = "." } + +[[package]] +name = "from-git" +version = "1.0.0" +source = { git = "https://github.com/example/from-git?rev=main#9f8e7d6c5b4a3928176554433221100ffeeddcc0" } + +[[package]] +name = "from-url" +version = "2.0.0" +source = { url = "https://files.pythonhosted.org/packages/ab/from_url-2.0.0-py3-none-any.whl" } + +[[package]] +name = "from-registry" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "from-path" +version = "4.0.0" +source = { path = "../vendor/from-path" } +` + path := filepath.Join(dir, "uv.lock") + if err := os.WriteFile(path, []byte(lock), 0o644); err != nil { + t.Fatal(err) + } + + graph, err := depGraphFromUVLock(path) + if err != nil { + t.Fatalf("depGraphFromUVLock() error = %v", err) + } + + // The fragment carries the commit uv resolved; the "rev" query carries + // what the manifest asked for. + requireOrigin(t, graph, "from-git@1.0.0", detectors.Origin{ + VCSURL: "https://github.com/example/from-git", + VCSRevision: "9f8e7d6c5b4a3928176554433221100ffeeddcc0", + }) + requireOrigin(t, graph, "from-url@2.0.0", detectors.Origin{ + ArtifactURL: "https://files.pythonhosted.org/packages/ab/from_url-2.0.0-py3-none-any.whl", + }) + // An index root is not this package's origin, and a path is local. + requireOrigin(t, graph, "from-registry@3.0.0", detectors.Origin{}) + requireOrigin(t, graph, "from-path@4.0.0", detectors.Origin{}) +} + +func TestPoetryLockOriginBySourceType(t *testing.T) { + dir := t.TempDir() + lock := ` +[[package]] +name = "from-pypi" +version = "1.0.0" + +[[package]] +name = "from-git" +version = "2.0.0" +[package.source] +type = "git" +url = "https://github.com/example/from-git.git" +reference = "main" +resolved_reference = "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567" + +[[package]] +name = "from-url" +version = "3.0.0" +[package.source] +type = "url" +url = "https://files.pythonhosted.org/packages/cd/from_url-3.0.0.tar.gz" + +[[package]] +name = "from-private-index" +version = "4.0.0" +[package.source] +type = "legacy" +url = "https://pypi.example.test/simple" +reference = "internal" + +[[package]] +name = "from-directory" +version = "5.0.0" +[package.source] +type = "directory" +url = "../vendor/from-directory" +` + lockPath := filepath.Join(dir, "poetry.lock") + if err := os.WriteFile(lockPath, []byte(lock), 0o644); err != nil { + t.Fatal(err) + } + + graph, err := depGraphFromPoetryLock(lockPath, dir) + if err != nil { + t.Fatalf("depGraphFromPoetryLock() error = %v", err) + } + + requireOrigin(t, graph, "from-pypi@1.0.0", detectors.Origin{}) + // resolved_reference is the commit poetry locked; reference is the branch. + requireOrigin(t, graph, "from-git@2.0.0", detectors.Origin{ + VCSURL: "https://github.com/example/from-git.git", + VCSRevision: "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567", + }) + requireOrigin(t, graph, "from-url@3.0.0", detectors.Origin{ + ArtifactURL: "https://files.pythonhosted.org/packages/cd/from_url-3.0.0.tar.gz", + }) + requireOrigin(t, graph, "from-private-index@4.0.0", detectors.Origin{}) + requireOrigin(t, graph, "from-directory@5.0.0", detectors.Origin{}) +} + +func TestPipfileLockOriginBySourceType(t *testing.T) { + dir := t.TempDir() + lock := `{ + "default": { + "from-pypi": {"version": "==1.0.0", "index": "pypi"}, + "from-git": {"git": "https://github.com/example/from-git.git", "ref": "1f2e3d4c5b6a79880912a3b4c5d6e7f809172635"}, + "from-archive": {"file": "https://files.pythonhosted.org/packages/ef/from_archive-2.0.0.tar.gz"}, + "from-local": {"file": "file:///workspace/wheels/from_local-3.0.0.whl"}, + "from-path": {"path": "../vendor/from-path"} + }, + "develop": {} + }` + path := filepath.Join(dir, "Pipfile.lock") + if err := os.WriteFile(path, []byte(lock), 0o644); err != nil { + t.Fatal(err) + } + + graph, err := depGraphFromPipfileLock(path, "demo") + if err != nil { + t.Fatalf("depGraphFromPipfileLock() error = %v", err) + } + + requireOrigin(t, graph, "from-pypi@1.0.0", detectors.Origin{}) + requireOrigin(t, graph, "from-git", detectors.Origin{ + VCSURL: "https://github.com/example/from-git.git", + VCSRevision: "1f2e3d4c5b6a79880912a3b4c5d6e7f809172635", + }) + requireOrigin(t, graph, "from-archive", detectors.Origin{ + ArtifactURL: "https://files.pythonhosted.org/packages/ef/from_archive-2.0.0.tar.gz", + }) + requireOrigin(t, graph, "from-local", detectors.Origin{}) + requireOrigin(t, graph, "from-path", detectors.Origin{}) +} + +// pip records a PEP 610 direct_url.json for anything not installed from an +// index, distinguishing repositories, archives, and local directories. +func TestPipInspectOriginByDirectURLShape(t *testing.T) { + cases := []struct { + name string + directURL map[string]any + want detectors.Origin + }{ + {name: "index install", directURL: nil}, + { + name: "git checkout", + directURL: map[string]any{ + "url": "https://github.com/example/pkg.git", + "vcs_info": map[string]any{"vcs": "git", "commit_id": "2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e", "requested_revision": "main"}, + }, + want: detectors.Origin{VCSURL: "https://github.com/example/pkg.git", VCSRevision: "2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e"}, + }, + { + name: "archive URL", + directURL: map[string]any{"url": "https://example.test/pkg-1.0.0-py3-none-any.whl", "archive_info": map[string]any{}}, + want: detectors.Origin{ArtifactURL: "https://example.test/pkg-1.0.0-py3-none-any.whl"}, + }, + { + name: "local directory", + directURL: map[string]any{"url": "file:///workspace/pkg", "dir_info": map[string]any{}}, + }, + { + name: "mercurial checkout", + directURL: map[string]any{"url": "https://hg.example.test/pkg", "vcs_info": map[string]any{"vcs": "hg", "commit_id": "9f8e7d6"}}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + node := sdk.NewDependency(sdk.Dependency{Coordinates: sdk.Coordinates{Name: "pkg", Version: "1.0.0"}}) + setPipInspectOrigin(node, tc.directURL) + if got := detectors.OriginFrom(node.Metadata); got != tc.want { + t.Fatalf("origin = %+v, want %+v", got, tc.want) + } + }) + } +} diff --git a/internal/detectors/python/pipenv.go b/internal/detectors/python/pipenv.go index 004928d5..a102f532 100644 --- a/internal/detectors/python/pipenv.go +++ b/internal/detectors/python/pipenv.go @@ -242,6 +242,7 @@ func addPipfileLockPackages(depsGraph *sdk.Graph, root *sdk.Dependency, packages Name: normalizedName, Version: strings.TrimPrefix(pkg.Version, "==")}, Source: pipfileDependencySource(pkg), ResolvedURL: pipfileResolvedURL(pkg), Metadata: sourceRevisionMetadata(pkg.Ref), Scopes: sdk.ScopesOf(scope), }) + setPipenvOrigin(node, pkg) if _, exists := depsGraph.Node(node.ID); !exists { if err := depsGraph.AddNode(node); err != nil { diff --git a/internal/detectors/python/poetrylock.go b/internal/detectors/python/poetrylock.go index f9480b77..a586f62d 100644 --- a/internal/detectors/python/poetrylock.go +++ b/internal/detectors/python/poetrylock.go @@ -87,6 +87,7 @@ func depGraphFromPoetryLock(lockPath, projectPath string) (*sdk.Graph, error) { Type: sdk.PackageTypePackage, PURL: sdk.BuildPackageURL("pypi", "", pkg.Name, pkg.Version)}, Source: poetryDependencySource(pkg.Source.Type), ResolvedURL: strings.TrimSpace(pkg.Source.URL), Metadata: sourceRevisionMetadata(firstNonEmpty(pkg.Source.ResolvedReference, pkg.Source.Reference)), }) + setPoetryOrigin(node, pkg) for _, group := range pkg.Groups { if group == "main" { diff --git a/internal/detectors/python/uvlock.go b/internal/detectors/python/uvlock.go index fa57b835..0a47f350 100644 --- a/internal/detectors/python/uvlock.go +++ b/internal/detectors/python/uvlock.go @@ -71,6 +71,7 @@ func depGraphFromUVLock(uvLockPath string) (*sdk.Graph, error) { Name: normalizePythonName(pkg.Name), Version: pkg.Version}, Source: uvDependencySource(pkg.Source), ResolvedURL: uvResolvedURL(pkg.Source), Metadata: sourceRevisionMetadata(uvSourceRevision(pkg.Source)), }) + setUVOrigin(node, pkg.Source) nodesByName[normalizePythonName(pkg.Name)] = node } diff --git a/internal/detectors/ruby/detector.go b/internal/detectors/ruby/detector.go index e8cee75a..6369aa54 100644 --- a/internal/detectors/ruby/detector.go +++ b/internal/detectors/ruby/detector.go @@ -488,13 +488,19 @@ func gemNode(spec lockSpec) *sdk.Dependency { if revision := strings.TrimSpace(spec.Revision); revision != "" { metadata = map[string]any{"source_revision": revision} } - return sdk.NewDependency(sdk.Dependency{Coordinates: sdk.Coordinates{Ecosystem: sdk.EcosystemRuby, + node := sdk.NewDependency(sdk.Dependency{Coordinates: sdk.Coordinates{Ecosystem: sdk.EcosystemRuby, Name: strings.TrimSpace(spec.Name), Version: strings.TrimSpace(spec.Version), PackageManager: sdk.PackageManagerBundler, Type: "gem", Language: "ruby"}, Source: spec.Source, ResolvedURL: strings.TrimSpace(spec.ResolvedURL), Metadata: metadata, }) + if spec.Source == sdk.DependencySourceGit { + // A GIT section names the repository and the commit Bundler locked. + // A GEM section's remote is the gem server, and PATH is local. + detectors.SetOriginVCS(node, spec.ResolvedURL, spec.Revision) + } + return node } diff --git a/internal/detectors/ruby/origin_test.go b/internal/detectors/ruby/origin_test.go new file mode 100644 index 00000000..99dd3577 --- /dev/null +++ b/internal/detectors/ruby/origin_test.go @@ -0,0 +1,70 @@ +package ruby + +import ( + "testing" + + "github.com/bomly-dev/bomly-cli/internal/detectors" +) + +// A Gemfile.lock names its sources by section. GEM's remote is the gem server +// every gem in that section came from, not a per-gem location; PATH is a +// directory on the machine that ran bundle install. Only GIT identifies where +// a specific gem's code came from. +func TestBundlerOriginBySection(t *testing.T) { + raw := []byte(`GEM + remote: https://rubygems.org/ + specs: + rack (3.1.8) + +GEM + remote: https://gems.corp/private/feed/ + specs: + corp-auth (2.4.0) + +GIT + remote: https://github.com/example/helper.git + revision: 708192a3b4c5d6e7f8091a2b3c4d5e6f70819213 + specs: + helper (1.0.0) + +PATH + remote: ../local-gem + specs: + local-gem (0.1.0) + +DEPENDENCIES + corp-auth + helper! + local-gem! + rack +`) + + graph, err := depGraphFromLock(raw, nil) + if err != nil { + t.Fatalf("depGraphFromLock() error = %v", err) + } + + cases := []struct { + id string + want detectors.Origin + }{ + {id: "rack@3.1.8"}, + // A private gem server's remote has a path, so nothing but the + // section kind distinguishes it from a repository URL. + {id: "corp-auth@2.4.0"}, + {id: "helper@1.0.0", want: detectors.Origin{ + VCSURL: "https://github.com/example/helper.git", + VCSRevision: "708192a3b4c5d6e7f8091a2b3c4d5e6f70819213", + }}, + {id: "local-gem@0.1.0"}, + } + for _, tc := range cases { + node, ok := graph.Node(tc.id) + if !ok { + t.Fatalf("expected %s in graph", tc.id) + } + if got := detectors.OriginFrom(node.Metadata); got != tc.want { + t.Errorf("%s origin = %+v, want %+v", tc.id, got, tc.want) + } + } +} diff --git a/internal/detectors/swiftpm/detector.go b/internal/detectors/swiftpm/detector.go index acb00c2d..fd21fb1d 100644 --- a/internal/detectors/swiftpm/detector.go +++ b/internal/detectors/swiftpm/detector.go @@ -275,6 +275,13 @@ func packageNode(pkg swiftPackage) *sdk.Dependency { Metadata: metadata, }) + if swiftDependencySource(pkg.SourceKind, pkg.Repository) == sdk.DependencySourceGit { + // Source-control pins name the repository and the commit SwiftPM + // resolved. Registry pins are identity-only, and local packages + // point at a checkout on this machine. + detectors.SetOriginVCS(node, pkg.Repository, pkg.Revision) + } + // SwiftPM does not distinguish dev scope; all packages are runtime. node.AddScope(sdk.ScopeRuntime) return node diff --git a/internal/detectors/swiftpm/origin_test.go b/internal/detectors/swiftpm/origin_test.go new file mode 100644 index 00000000..4fda4cb5 --- /dev/null +++ b/internal/detectors/swiftpm/origin_test.go @@ -0,0 +1,64 @@ +package swiftpm + +import ( + "testing" + + "github.com/bomly-dev/bomly-cli/internal/detectors" +) + +// A Package.resolved pin says how SwiftPM obtained a package. Source-control +// pins name a repository and the commit that was resolved; registry pins are +// identity-only; local pins point at a checkout on this machine. +func TestSwiftPMOriginByPinKind(t *testing.T) { + resolved := []byte(`{ + "pins": [ + { + "identity": "swift-argument-parser", + "kind": "remoteSourceControl", + "location": "https://github.com/apple/swift-argument-parser.git", + "state": {"revision": "8192a3b4c5d6e7f8091a2b3c4d5e6f7081921324", "version": "1.3.0"} + }, + { + "identity": "internal-tools", + "kind": "registry", + "state": {"version": "2.0.0"} + }, + { + "identity": "local-helper", + "kind": "localSourceControl", + "location": "/Users/someone/src/local-helper", + "state": {"revision": "92a3b4c5d6e7f8091a2b3c4d5e6f708192132435", "version": "0.1.0"} + } + ], + "version": 2 + }`) + + graph, err := depGraphFromSwiftPM(resolved, nil) + if err != nil { + t.Fatalf("depGraphFromSwiftPM() error = %v", err) + } + + var checked int + for _, node := range graph.Nodes() { + origin := detectors.OriginFrom(node.Metadata) + switch node.Name { + case "swift-argument-parser": + checked++ + want := detectors.Origin{ + VCSURL: "https://github.com/apple/swift-argument-parser.git", + VCSRevision: "8192a3b4c5d6e7f8091a2b3c4d5e6f7081921324", + } + if origin != want { + t.Errorf("%s origin = %+v, want %+v", node.Name, origin, want) + } + case "internal-tools", "local-helper": + checked++ + if !origin.Empty() { + t.Errorf("%s asserted an origin it should not have: %+v", node.Name, origin) + } + } + } + if checked != 3 { + t.Fatalf("checked %d pins, want 3", checked) + } +} From 406e4496908ce4c941a8750d404a5a29d6d81e30 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 03:09:07 -0400 Subject: [PATCH 03/17] feat(sbom): publish detector-asserted origin as download and repository locations SPDX packages now carry a real download location instead of a constant NOASSERTION: the artifact URL a detector resolved, or the repository in SPDX 2.3's version-control form, "git+@". CycloneDX components gain a distribution or vcs external reference, the latter as a plain URL since the format has no revision slot on references. Export decides nothing. It reads the origin detection recorded, re-validates it against the same invariant that admitted it, and projects the result; a package whose detector asserted nothing keeps NOASSERTION rather than a guess. The re-validation is what makes this safe for graphs Bomly did not build itself, such as a plugin's. The scorecard matcher's canonical repository fills the gap for packages whose lockfile named no repository, and never overrides one a detector asserted. Verified against the official SPDX validator (spdxlib.ValidateDocument) and the CycloneDX 1.4/1.5/1.6 JSON schemas, on real npm and cargo scans. Co-Authored-By: Claude Opus 5 --- internal/sbom/cyclonedx.go | 19 +++ internal/sbom/model.go | 8 ++ internal/sbom/origin_test.go | 270 +++++++++++++++++++++++++++++++++++ internal/sbom/spdx23.go | 31 +++- internal/sbom/transform.go | 34 +++++ scripts/run-fuzz.sh | 1 + 6 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 internal/sbom/origin_test.go diff --git a/internal/sbom/cyclonedx.go b/internal/sbom/cyclonedx.go index 27fbe393..c33f7f03 100644 --- a/internal/sbom/cyclonedx.go +++ b/internal/sbom/cyclonedx.go @@ -47,6 +47,9 @@ func (c cycloneDXCodec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, e if props := cycloneDXEOLProperties(comp.EOL); len(props) > 0 { component.Properties = &props } + if refs := cycloneDXComponentReferences(comp); len(refs) > 0 { + component.ExternalReferences = &refs + } components = append(components, component) } bom.Components = &components @@ -248,6 +251,22 @@ func cycloneDXTools(names []string, primaryTool, toolVersion string) *cdx.ToolsC // cycloneDXSecurityReferences maps provenance contact fields onto external // references attached to the primary component. +// cycloneDXComponentReferences renders where a package came from: the exact +// file it was fetched from as a distribution reference, or the repository it +// was resolved from as a vcs reference. The repository is rendered as a plain +// URL -- CycloneDX external references carry no revision, so the commit a +// detector resolved is only expressible in the SPDX locator form. +func cycloneDXComponentReferences(component Component) []cdx.ExternalReference { + refs := make([]cdx.ExternalReference, 0, 2) + if artifact := strings.TrimSpace(component.ArtifactURL); artifact != "" { + refs = append(refs, cdx.ExternalReference{Type: cdx.ERTypeDistribution, URL: artifact}) + } + if repository := strings.TrimSpace(component.VCSURL); repository != "" { + refs = append(refs, cdx.ExternalReference{Type: cdx.ERTypeVCS, URL: repository}) + } + return refs +} + func cycloneDXSecurityReferences(p Provenance) []cdx.ExternalReference { refs := make([]cdx.ExternalReference, 0, 2) if contact := strings.TrimSpace(p.SecurityContact); contact != "" { diff --git a/internal/sbom/model.go b/internal/sbom/model.go index 31607005..363c724f 100644 --- a/internal/sbom/model.go +++ b/internal/sbom/model.go @@ -135,6 +135,14 @@ type Component struct { Digests []Digest Vulnerabilities []Vulnerability EOL *EOL + + // Where the package came from, as asserted by the detector that resolved + // it. At most one of ArtifactURL and VCSURL is set, and VCSRevision only + // accompanies VCSURL. Both are plain absolute http(s) URLs; composing them + // into a format's locator grammar is the encoder's job. + ArtifactURL string + VCSURL string + VCSRevision string } // Dependency describes one package relationship list in the intermediate SBOM model. diff --git a/internal/sbom/origin_test.go b/internal/sbom/origin_test.go new file mode 100644 index 00000000..317bf733 --- /dev/null +++ b/internal/sbom/origin_test.go @@ -0,0 +1,270 @@ +package sbom + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/bomly-dev/bomly-cli/internal/detectors" + "github.com/bomly-dev/bomly-sdk" +) + +// originGraph builds a two-package graph whose nodes carry whatever origin +// metadata a case wants to exercise. +func originGraph(t *testing.T, mutate func(app, pkg *sdk.Dependency)) *sdk.Graph { + t.Helper() + + g := sdk.New() + app := sdk.NewDependencyRef("app", "1.0.0") + pkg := sdk.NewDependencyRef("react", "18.2.0") + mutate(app, pkg) + for _, n := range []*sdk.Dependency{app, pkg} { + if err := g.AddNode(n); err != nil { + t.Fatalf("add package %s: %v", n.ID, err) + } + } + if err := g.AddEdge(app.ID, pkg.ID); err != nil { + t.Fatalf("add edge: %v", err) + } + return g +} + +// spdxPackageByName decodes an SPDX document and returns one package object. +func spdxPackageByName(t *testing.T, raw []byte, name string) map[string]any { + t.Helper() + var doc struct { + Packages []map[string]any `json:"packages"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatalf("decode SPDX: %v", err) + } + for _, pkg := range doc.Packages { + if pkg["name"] == name { + return pkg + } + } + t.Fatalf("package %q not found in SPDX document", name) + return nil +} + +// cycloneDXReferences decodes a CycloneDX document and returns one component's +// external references as type→URL pairs. +func cycloneDXReferences(t *testing.T, raw []byte, name string) map[string]string { + t.Helper() + var doc struct { + Components []struct { + Name string `json:"name"` + ExternalReferences []struct { + Type string `json:"type"` + URL string `json:"url"` + } `json:"externalReferences"` + } `json:"components"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatalf("decode CycloneDX: %v", err) + } + for _, component := range doc.Components { + if component.Name != name { + continue + } + refs := make(map[string]string, len(component.ExternalReferences)) + for _, ref := range component.ExternalReferences { + refs[ref.Type] = ref.URL + } + return refs + } + t.Fatalf("component %q not found in CycloneDX document", name) + return nil +} + +func marshalBoth(t *testing.T, g *sdk.Graph) (spdxRaw, cdxRaw []byte) { + t.Helper() + opts := BuildOptions{DocumentName: "origin-test", ToolVersion: "test"} + spdxRaw, err := MarshalDepGraphJSON(g, TargetSPDX23JSON, opts, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal SPDX: %v", err) + } + cdxRaw, err = MarshalDepGraphJSON(g, TargetCycloneDX17JSON, opts, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal CycloneDX: %v", err) + } + return spdxRaw, cdxRaw +} + +func TestArtifactOriginIsPublishedInBothFormats(t *testing.T) { + const artifact = "https://registry.npmjs.org/react/-/react-18.2.0.tgz" + g := originGraph(t, func(_, pkg *sdk.Dependency) { + detectors.SetOriginArtifact(pkg, artifact) + }) + + spdxRaw, cdxRaw := marshalBoth(t, g) + + if got := spdxPackageByName(t, spdxRaw, "react")["downloadLocation"]; got != artifact { + t.Errorf("SPDX downloadLocation = %v, want %q", got, artifact) + } + if got := cycloneDXReferences(t, cdxRaw, "react")["distribution"]; got != artifact { + t.Errorf("CycloneDX distribution ref = %q, want %q", got, artifact) + } +} + +func TestRepositoryOriginIsPublishedInBothFormats(t *testing.T) { + const ( + repository = "https://github.com/facebook/react" + revision = "b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7" + ) + g := originGraph(t, func(_, pkg *sdk.Dependency) { + detectors.SetOriginVCS(pkg, repository, revision) + }) + + spdxRaw, cdxRaw := marshalBoth(t, g) + + // SPDX 2.3's version-control form carries the revision after "@". + want := "git+" + repository + "@" + revision + if got := spdxPackageByName(t, spdxRaw, "react")["downloadLocation"]; got != want { + t.Errorf("SPDX downloadLocation = %v, want %q", got, want) + } + // CycloneDX external references have no revision slot, so the reference is + // the plain repository URL. + if got := cycloneDXReferences(t, cdxRaw, "react")["vcs"]; got != repository { + t.Errorf("CycloneDX vcs ref = %q, want %q", got, repository) + } + if got := cycloneDXReferences(t, cdxRaw, "react")["distribution"]; got != "" { + t.Errorf("repository origin also emitted a distribution ref: %q", got) + } +} + +func TestUnpinnedRepositoryOriginOmitsTheRevisionSuffix(t *testing.T) { + const repository = "https://github.com/facebook/react" + g := originGraph(t, func(_, pkg *sdk.Dependency) { + detectors.SetOriginVCS(pkg, repository, "") + }) + + spdxRaw, _ := marshalBoth(t, g) + + if got := spdxPackageByName(t, spdxRaw, "react")["downloadLocation"]; got != "git+"+repository { + t.Errorf("SPDX downloadLocation = %v, want %q", got, "git+"+repository) + } +} + +// A package whose detector asserted nothing must say so, not guess. +func TestPackageWithoutOriginKeepsNOASSERTION(t *testing.T) { + g := originGraph(t, func(_, _ *sdk.Dependency) {}) + + spdxRaw, cdxRaw := marshalBoth(t, g) + + if got := spdxPackageByName(t, spdxRaw, "react")["downloadLocation"]; got != "NOASSERTION" { + t.Errorf("SPDX downloadLocation = %v, want NOASSERTION", got) + } + if refs := cycloneDXReferences(t, cdxRaw, "react"); len(refs) != 0 { + t.Errorf("CycloneDX emitted references for a package with no origin: %v", refs) + } +} + +// Origin metadata can reach export from a plugin or a hand-built graph that +// never went through the setters. Export re-validates rather than trusting it, +// so a bad value is dropped instead of published. +func TestExportRevalidatesOriginMetadata(t *testing.T) { + hostile := []struct { + name string + metadata map[string]any + }{ + {name: "credentialed artifact", metadata: map[string]any{ + detectors.MetadataKeyOriginArtifactURL: "https://build:s3cret-token-value@nexus.corp/repo/react-18.2.0.tgz", + }}, + {name: "local path", metadata: map[string]any{ + detectors.MetadataKeyOriginArtifactURL: "/Users/someone/src/project/react.tgz", + }}, + {name: "file url", metadata: map[string]any{ + detectors.MetadataKeyOriginVCSURL: "file:///Users/someone/src/react", + }}, + {name: "revision breaking the locator grammar", metadata: map[string]any{ + detectors.MetadataKeyOriginVCSURL: "https://github.com/facebook/react", + detectors.MetadataKeyOriginVCSRevision: "main@evil.test/x", + }}, + {name: "non-string value", metadata: map[string]any{ + detectors.MetadataKeyOriginArtifactURL: 42, + }}, + } + + for _, tc := range hostile { + t.Run(tc.name, func(t *testing.T) { + g := originGraph(t, func(_, pkg *sdk.Dependency) { + pkg.Metadata = tc.metadata + }) + + spdxRaw, cdxRaw := marshalBoth(t, g) + + download, _ := spdxPackageByName(t, spdxRaw, "react")["downloadLocation"].(string) + refs := cycloneDXReferences(t, cdxRaw, "react") + published := append([]string{download}, refs["distribution"], refs["vcs"]) + for _, value := range published { + for _, forbidden := range []string{"s3cret", "@nexus.corp", "/Users/", "file://", "evil.test"} { + if strings.Contains(value, forbidden) { + t.Fatalf("published %q, which contains %q", value, forbidden) + } + } + } + // The revision case keeps a valid repository; the rest publish nothing. + if tc.name == "revision breaking the locator grammar" { + if download != "git+https://github.com/facebook/react" { + t.Fatalf("downloadLocation = %q, want the repository without a revision", download) + } + return + } + if download != "NOASSERTION" { + t.Fatalf("downloadLocation = %q, want NOASSERTION", download) + } + }) + } +} + +// The scorecard matcher resolves a canonical source repository during +// enrichment. It fills the gap when a lockfile named no repository, and never +// overrides one the detector asserted. +func TestScorecardRepositoryFillsTheOriginGap(t *testing.T) { + const purl = "pkg:npm/react@18.2.0" + + build := func(t *testing.T, detectorRepository string) []byte { + t.Helper() + g := sdk.New() + react := sdk.NewDependencyWithID("react@18.2.0", sdk.Dependency{Coordinates: sdk.Coordinates{ + Name: "react", Version: "18.2.0", PURL: purl, Ecosystem: "npm"}}) + if detectorRepository != "" { + detectors.SetOriginVCS(react, detectorRepository, "") + } + if err := g.AddNode(react); err != nil { + t.Fatalf("add node: %v", err) + } + registry := sdk.NewPackageRegistry() + pkg := registry.Ensure(purl) + pkg.Name, pkg.Version, pkg.Matched = "react", "18.2.0", true + pkg.Scorecard = &sdk.PackageScorecard{Source: "api.scorecard.dev", Repository: "github.com/facebook/react"} + + raw, err := MarshalDepGraphJSON(g, TargetCycloneDX17JSON, BuildOptions{Registry: registry}, EncodeOptions{}) + if err != nil { + t.Fatalf("marshal CycloneDX: %v", err) + } + return raw + } + + t.Run("fills the gap", func(t *testing.T) { + if got := cycloneDXReferences(t, build(t, ""), "react")["vcs"]; got != "https://github.com/facebook/react" { + t.Errorf("vcs ref = %q, want the scorecard repository as an https URL", got) + } + }) + + t.Run("never overrides the detector", func(t *testing.T) { + const asserted = "https://github.com/facebook/react-fork" + if got := cycloneDXReferences(t, build(t, asserted), "react")["vcs"]; got != asserted { + t.Errorf("vcs ref = %q, want the detector-asserted repository %q", got, asserted) + } + }) + + t.Run("absent without enrichment", func(t *testing.T) { + g := originGraph(t, func(_, _ *sdk.Dependency) {}) + _, cdxRaw := marshalBoth(t, g) + if refs := cycloneDXReferences(t, cdxRaw, "react"); len(refs) != 0 { + t.Errorf("unenriched export emitted references: %v", refs) + } + }) +} diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index cff750e4..1bdda8a4 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -44,7 +44,7 @@ func (spdx23Codec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, error) PackageName: c.NameOrID(), PackageSPDXIdentifier: spdxID, PackageVersion: c.Version, - PackageDownloadLocation: "NOASSERTION", + PackageDownloadLocation: spdxDownloadLocation(c), FilesAnalyzed: false, PackageComment: spdxPackageComment(c), PackageLicenseDeclared: spdxLicenseValue(c.Licenses), @@ -428,6 +428,35 @@ func spdxCopyrightValue(value string) string { return value } +// spdxDownloadLocation renders where a package came from. SPDX requires the +// field, so a package whose detector asserted nothing keeps NOASSERTION rather +// than a guess. +func spdxDownloadLocation(component Component) string { + if artifact := strings.TrimSpace(component.ArtifactURL); artifact != "" { + return artifact + } + if locator := spdxVCSLocator(component); locator != "" { + return locator + } + return "NOASSERTION" +} + +// spdxVCSLocator renders a repository in SPDX 2.3's version-control form, +// "+://[@]". The grammar has no query +// component and no room for anything but the revision after "@", which is why +// the origin invariant strips both before a URL gets here. +func spdxVCSLocator(component Component) string { + repository := strings.TrimSpace(component.VCSURL) + if repository == "" { + return "" + } + locator := "git+" + repository + if revision := strings.TrimSpace(component.VCSRevision); revision != "" { + locator += "@" + revision + } + return locator +} + func spdxExternalReferences(component Component) []*v23.PackageExternalReference { refs := make([]*v23.PackageExternalReference, 0, 1+len(component.CPEs)+len(component.Vulnerabilities)) if purl := strings.TrimSpace(component.PURL); purl != "" { diff --git a/internal/sbom/transform.go b/internal/sbom/transform.go index 891d4f4a..500248ee 100644 --- a/internal/sbom/transform.go +++ b/internal/sbom/transform.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) @@ -47,6 +48,7 @@ func FromDepGraph(g *sdk.Graph, opts BuildOptions) (*Document, error) { Licenses: componentLicenses(sdk.DetectionLicenses(pkg)), Digests: componentDigests(pkg.Digests), } + applyOrigin(&component, detectors.OriginFrom(pkg.Metadata)) enrichComponentFromRegistry(&component, opts.Registry, pkg.PURL) components = append(components, component) depsByRef[pkg.ID] = nil @@ -263,6 +265,32 @@ func uniqueToolNames(values []string) []string { // enrichComponentFromRegistry folds matching-stage data resolved by PURL onto a // component: registry-learned licenses (preferred over detection-time when // present), CPEs, digests, vulnerabilities, and EOL. registry may be nil. +// scorecardRepositoryURL renders a scorecard repository, which is a canonical +// host/owner/name identifier with no scheme, as a URL. It is held to the same +// invariant as detector-asserted origins. +func scorecardRepositoryURL(scorecard *sdk.PackageScorecard) (string, bool) { + if scorecard == nil { + return "", false + } + repository := strings.TrimSpace(scorecard.Repository) + if repository == "" { + return "", false + } + if !strings.Contains(repository, "://") { + repository = "https://" + repository + } + return detectors.NormalizeOriginURL(repository, true) +} + +// applyOrigin projects the origin a detector asserted onto a component. The +// values were validated when they were read, so there is nothing to decide +// here: export publishes what detection resolved, or nothing. +func applyOrigin(component *Component, origin detectors.Origin) { + component.ArtifactURL = origin.ArtifactURL + component.VCSURL = origin.VCSURL + component.VCSRevision = origin.VCSRevision +} + func enrichComponentFromRegistry(component *Component, registry *sdk.PackageRegistry, purl string) { if component == nil || registry == nil || purl == "" { return @@ -283,6 +311,12 @@ func enrichComponentFromRegistry(component *Component, registry *sdk.PackageRegi if len(pkg.Vulnerabilities) > 0 { component.Vulnerabilities = vulnerabilitiesFromPackage(pkg.EcosystemName(), pkg.Vulnerabilities) } + if repository, ok := scorecardRepositoryURL(pkg.Scorecard); ok && component.VCSURL == "" { + // The scorecard matcher resolved a canonical source repository for + // this package. A detector-asserted repository is the stronger claim + // (it came from the lockfile), so this only fills a gap. + component.VCSURL = repository + } if pkg.EOL != nil { component.EOL = &EOL{ EOL: pkg.EOL.EOL, diff --git a/scripts/run-fuzz.sh b/scripts/run-fuzz.sh index 36bf6147..4271a274 100755 --- a/scripts/run-fuzz.sh +++ b/scripts/run-fuzz.sh @@ -31,6 +31,7 @@ targets=( "github.com/bomly-dev/bomly-cli/internal/detectors/swiftpm FuzzDepGraphFromSwiftResolved" "github.com/bomly-dev/bomly-cli/internal/sbom FuzzUnmarshalAutoJSON" "github.com/bomly-dev/bomly-cli/internal/sbom FuzzNormalizeSPDXLicenseExpression" + "github.com/bomly-dev/bomly-cli/internal/detectors FuzzSetOrigin" "github.com/bomly-dev/bomly-cli/internal/baseline FuzzLoad" "github.com/bomly-dev/bomly-cli/internal/engine FuzzConsolidateVulnerabilities" "github.com/bomly-dev/bomly-cli/internal/plugin FuzzPluginPathSanitizers" From a37737d74303d091e16257774ac313c6ffc0e237 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 03:17:02 -0400 Subject: [PATCH 04/17] docs(sbom): document detector-asserted package origin; cover it in smoke docs/SBOM.md gains a "Where a package came from" section: what each detector reports, how the two shapes map onto each format, and the four kinds of value that are never published -- registry roots, local paths, non-web remotes, and credentialed URLs -- with the reasoning for each. dev-docs records the decision and, more usefully, why the export-side classifier it replaces could not work: ResolvedURL is not one kind of value, so recovering its meaning downstream is guesswork with a per-ecosystem counterexample for every rule. The new smoke case scans a real npm repository and asserts on the exported bytes rather than a golden -- SBOM documents carry a namespace, serial number, timestamp, and tool version that change every run. It checks that real lockfiles produce real download locations, and that nothing about the scanning machine reaches the output. Both slice matrices gain the test and the node toolchain it needs, so it cannot silently skip. Co-Authored-By: Claude Opus 5 --- .github/workflows/smoke.yml | 3 +- .github/workflows/update-smoke-goldens.yml | 3 +- dev-docs/ARCHITECTURE.md | 14 +++ docs/SBOM.md | 60 +++++++++++- test/smoke/smoke_test.go | 104 +++++++++++++++++++++ 5 files changed, 180 insertions(+), 4 deletions(-) diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 4ab67556..b2141605 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -97,7 +97,8 @@ jobs: - name: ruby run: 'TestScan$/scan-bundler' - name: sbom - run: 'TestScan$/scan-sbom|TestDiff/(diff-sbom$|diff-sbom-detail-change$)|TestLiteScan/lite-scan-sbom|TestScanSBOMSyftJSONRejected$' + run: 'TestScan$/scan-sbom|TestDiff/(diff-sbom$|diff-sbom-detail-change$)|TestLiteScan/lite-scan-sbom|TestScanSBOMSyftJSONRejected$|TestScanSBOMExportOrigin$' + node: true - name: dotnet run: 'TestScan$/scan-nuget' dotnet: true diff --git a/.github/workflows/update-smoke-goldens.yml b/.github/workflows/update-smoke-goldens.yml index ade2b7d8..25b7d307 100644 --- a/.github/workflows/update-smoke-goldens.yml +++ b/.github/workflows/update-smoke-goldens.yml @@ -121,7 +121,8 @@ jobs: - name: cpp run: 'TestScan$/scan-cpp-conan' - name: sbom - run: 'TestScan$/scan-sbom|TestDiff/(diff-sbom$|diff-sbom-detail-change$)|TestLiteScan/lite-scan-sbom|TestScanSBOMSyftJSONRejected$' + run: 'TestScan$/scan-sbom|TestDiff/(diff-sbom$|diff-sbom-detail-change$)|TestLiteScan/lite-scan-sbom|TestScanSBOMSyftJSONRejected$|TestScanSBOMExportOrigin$' + node: true - name: plugin run: 'TestPluginWorkflows' - name: container diff --git a/dev-docs/ARCHITECTURE.md b/dev-docs/ARCHITECTURE.md index b556d462..3dabdab0 100644 --- a/dev-docs/ARCHITECTURE.md +++ b/dev-docs/ARCHITECTURE.md @@ -594,6 +594,20 @@ Document identity is shared across formats: one generated UUIDv4 becomes both th Further identity and claim rules follow the same only-say-what-we-know principle. The project version comes from `--ref` or `git describe` and is stamped onto the primary component and first-party (main-module) components only — third-party versions are never touched, and no version is emitted when Git has nothing to say. The CycloneDX composition declaration is `complete` only for an unfiltered scan with no detector warnings; a `--scope` filter downgrades it to `incomplete` and degraded resolution to `unknown`. Vulnerability `recommendation` text is rendered only from enrichment-known fixed versions. Deprecated SPDX license identifiers are normalized to their current names token-wise inside expressions (`GPL-2.0` → `GPL-2.0-only`), leaving free-text license values untouched. Every SPDX package carries a `PrimaryPackagePurpose`; decode still prefers the `bomly:type=` comment so round-trips keep the richer domain types (workflow, action) that SPDX's vocabulary lacks. +### Decision: package origin is detector-asserted; SBOM export only projects it + +An SBOM should say where each package came from — SPDX `downloadLocation`, CycloneDX `distribution`/`vcs` references. The obvious place to derive that is export, which already sees every package: classify `Dependency.ResolvedURL` by shape and map it onto the format's fields. That was built and abandoned. It does not work, and the reason generalizes. + +`ResolvedURL` is not one kind of value. npm writes a registry tarball there, but also a local directory for link entries and a git remote for git specs. uv writes a repository, an archive, an index root, or an editable path, depending on the source stanza. Cargo writes a prefixed source string, Bundler writes the section's `remote:` (a gem server, a repository, or a directory), pub writes the pub server for hosted packages and a repository for git ones. Recovering the meaning downstream means guessing from the string, and every guess has an ecosystem-specific counterexample: an archive-extension check misclassifies real repositories whose names end in `.zip` or `.conda`; a fragment is a resolved commit in uv and cargo but a content checksum in Yarn Classic; a private registry root with a path is indistinguishable from a repository; and distinguishing an opaque token from a content hash is not decidable at all, because they have the same shape. Roughly twenty review rounds of layered special cases did not converge. + +Origin is therefore asserted where the meaning is known. Each detector reads its own lockfile's structured source fields and records at most one of an artifact URL or a repository URL plus resolved revision, on `Dependency.Metadata` under `bomly.origin.*` keys — the same well-known-key transport `bomly.detection.licenses` uses, so no SDK change was needed. Registry and index roots are deliberately not representable: they describe an ecosystem's fetch configuration, not a package's provenance. + +One rule governs every published value, `detectors.NormalizeOriginURL`: absolute `http`/`https`, non-empty host, no userinfo, output re-serialized from the parse rather than copied from input. The repository form additionally strips query and fragment (they carry the *requested* ref; the *resolved* one arrives separately from the detector's own field) and requires a non-empty path, because SPDX's `git+@` grammar has no query component and an empty path would make the `@` suffix re-parse as userinfo. This one function replaces the entire classifier: no archive-extension table, no credential-prefix list, no secret-shape heuristic. Local paths, `file:`, and ssh-style remotes fail the scheme or host check rather than a bespoke rule, and a credentialed URL fails the userinfo check. + +The invariant runs twice — when a detector records a value and again when export reads it. The second pass is not redundant: graphs also arrive from plugins and from hand-built callers, and export must not publish a location no built-in detector could have produced. Composition into a format's locator grammar stays in the encoders, so `Component.VCSURL` remains a plain URL and only SPDX builds the `git+…@…` form; CycloneDX external references have no revision slot, so a resolved commit survives an SPDX round trip and not a CycloneDX one. + +Two consequences worth stating. Origin keys are filtered out of `scan`/`diff`/`explain` payloads by prefix in `output.cloneRefMetadata` — they are transport between two pipeline stages, and the SBOM is where users read them; the filter returns nil for an emptied map so `omitempty` still fires. And consolidation's first-wins node dedup can drop the origin of a duplicate occurrence, which matches the existing behavior of `ResolvedURL` itself. + ## Build Modes Syft and Grype each support two build modes: diff --git a/docs/SBOM.md b/docs/SBOM.md index 7afca480..62dc0f43 100644 --- a/docs/SBOM.md +++ b/docs/SBOM.md @@ -99,6 +99,58 @@ Both formats carry: knows fixed versions, each vulnerability carries a `recommendation` ("Upgrade to "). No guidance is invented when no fix is known. SPDX 2.3 has no equivalent field. +- Where each package came from, when its lockfile says: an exact download + location or a source repository. See "Where a package came from" below. + +### Where a package came from + +Each detector reports the origin of the packages it resolves, reading the field +its own lockfile records it in. Bomly does not infer origin from the shape of a +URL, because the same string means different things in different ecosystems. + +A detector reports one of two things, or nothing at all: + +| What the lockfile records | SPDX 2.3 | CycloneDX | +|---|---|---| +| The exact file the package was fetched from | `downloadLocation` | `distribution` external reference | +| The repository it was resolved from, plus the commit | `downloadLocation` as `git+@` | `vcs` external reference (URL only) | +| Neither | `NOASSERTION` | no reference | + +CycloneDX external references have no field for a revision, so the commit a +detector resolved appears only in the SPDX form. + +What each ecosystem yields: + +- **npm, pnpm, yarn, bun** — the registry tarball recorded in the lockfile. + Yarn Classic appends the package checksum to that URL; it identifies + contents rather than a location, so it is dropped. pnpm v9 entries that + record only an integrity hash report nothing. +- **uv, poetry, pipenv, pip** — a repository plus the commit that was locked, + or a direct archive URL, depending on the recorded source type. +- **cargo, Bundler, SwiftPM, pub** — the repository and resolved commit for + git dependencies and source-control pins. +- **Go modules, Maven, Gradle, NuGet, and the other detectors** — nothing yet; + their manifests do not record a per-package location. +- Packages found by Syft, and packages read from an ingested SBOM, carry no + origin. + +Four kinds of value are never published, in any ecosystem: + +- **Registry and index roots** (`https://rubygems.org/`, `https://pub.dev`, the + crates.io index). They say where an ecosystem fetches from, not where a + package came from — and once out of context, a private server URL is + indistinguishable from a repository. +- **Local paths** — workspace members, editable installs, `file:` and `path:` + dependencies. These describe the machine that ran the scan. +- **Non-web locations** — `ssh://`, `git@host:org/repo`, and similar remotes + that name a transport rather than a fetchable address. +- **URLs carrying credentials.** A lockfile pointing at a private registry can + embed a token; publishing it in an SBOM would leak a live secret. + +Every published location is an absolute `http`/`https` URL with a host and no +embedded credentials. Values are re-serialized from a parse rather than copied +from the lockfile, and the same check runs again at export, so origin supplied +by a plugin is held to the same rule as origin from a built-in detector. ### Document identity @@ -171,8 +223,9 @@ Reachability annotations and other Bomly-specific metadata are emitted in the JS ### Preservation and conversion limits Bomly preserves component identity (including PURL), dependency edges, roots, -scope, package type, licenses, digests, CPEs, and the enrichment fields described -above when the destination format has an equivalent representation. Encoding is +scope, package type, licenses, digests, CPEs, package origin, and the enrichment +fields described above when the destination format has an equivalent +representation. Encoding is deterministic when the scan timestamp and document identifiers are fixed. Some information necessarily becomes less specific during conversion: @@ -184,6 +237,9 @@ Some information necessarily becomes less specific during conversion: round trip. - Development scope maps to CycloneDX `excluded`; runtime scope maps to `required`. SPDX stores Bomly's normalized scope in the package comment. +- A resolved commit survives an SPDX round trip (it is part of the + `git+@` download location) but not a CycloneDX one, where an + external reference carries only the repository URL. - Bomly relationship confidence (`direct`, `transitive`, or `unknown`), source provenance, reachability analysis, policy findings, and run diagnostics are report data rather than portable SBOM fields. Use JSON when those distinctions diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index bcd0d6ab..7becafdb 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -12,11 +12,13 @@ package smoke import ( + "bytes" "encoding/json" "fmt" "os" "os/exec" "path/filepath" + "regexp" "runtime" "strings" "testing" @@ -546,6 +548,108 @@ func TestScanSBOMSyftJSONRejected(t *testing.T) { } } +// TestScanSBOMExportOrigin drives a real npm project end to end and asserts on +// the exported SBOM bytes rather than a golden: SPDX and CycloneDX documents +// carry a namespace, serial number, timestamp, and tool version that change on +// every run, and normalizeJSON targets Bomly's scan document, not SBOM output. +// +// The properties asserted here are the ones that matter and that unit tests +// cannot reach: a real lockfile from a real repository produces real download +// locations, and nothing about the machine that ran the scan leaks into them. +func TestScanSBOMExportOrigin(t *testing.T) { + t.Parallel() + requireTool(t, "npm") + + outputDir := t.TempDir() + spdxPath := filepath.Join(outputDir, "out.spdx.json") + cdxPath := filepath.Join(outputDir, "out.cdx.json") + + _, stderr, code := runBomly(t, + "scan", "--url", "https://github.com/bomly-dev/example-javascript-npm", "--ref", "v1.0.0", + "--detectors", "npm", "--format", "json", + "-o", "spdx="+spdxPath, "-o", "cyclonedx="+cdxPath, + ) + if code != 0 { + t.Fatalf("bomly exited %d\nstderr:\n%s", code, stderr) + } + + spdxRaw, err := os.ReadFile(spdxPath) + if err != nil { + t.Fatalf("read SPDX output: %v", err) + } + cdxRaw, err := os.ReadFile(cdxPath) + if err != nil { + t.Fatalf("read CycloneDX output: %v", err) + } + + var spdxDoc struct { + Packages []struct { + Name string `json:"name"` + DownloadLocation string `json:"downloadLocation"` + } `json:"packages"` + } + if err := json.Unmarshal(spdxRaw, &spdxDoc); err != nil { + t.Fatalf("decode SPDX output: %v", err) + } + var cdxDoc struct { + Components []struct { + Name string `json:"name"` + ExternalReferences []struct { + Type string `json:"type"` + URL string `json:"url"` + } `json:"externalReferences"` + } `json:"components"` + } + if err := json.Unmarshal(cdxRaw, &cdxDoc); err != nil { + t.Fatalf("decode CycloneDX output: %v", err) + } + + // npm records a registry tarball for every installed package, so a real + // scan must produce real download locations, not a document of NOASSERTION. + downloads := 0 + for _, pkg := range spdxDoc.Packages { + if strings.HasPrefix(pkg.DownloadLocation, "https://") { + downloads++ + } + } + if downloads == 0 { + t.Fatalf("no SPDX package carried a download location; got %d packages", len(spdxDoc.Packages)) + } + distributions := 0 + for _, component := range cdxDoc.Components { + for _, ref := range component.ExternalReferences { + if ref.Type == "distribution" && strings.HasPrefix(ref.URL, "https://") { + distributions++ + } + } + } + if distributions == 0 { + t.Fatalf("no CycloneDX component carried a distribution reference; got %d components", len(cdxDoc.Components)) + } + + // Nothing about the machine that ran the scan may reach the documents. + // The clone directory is the specific hazard: it is a real path that a + // detector could pass through verbatim. + cloneMarkers := []string{"file://", `:"/home/`, `:"/Users/`, `:"/var/folders/`, `:"/private/`} + for _, output := range []struct { + name string + raw []byte + }{{"SPDX", spdxRaw}, {"CycloneDX", cdxRaw}} { + for _, marker := range cloneMarkers { + if bytes.Contains(output.raw, []byte(marker)) { + t.Errorf("%s output contains %q, which points at the scanning machine", output.name, marker) + } + } + // A URL of the form scheme://userinfo@host would carry a credential. + if credentialedURL.Match(output.raw) { + t.Errorf("%s output contains a URL with embedded credentials", output.name) + } + } +} + +// credentialedURL matches an http(s) URL carrying userinfo before its host. +var credentialedURL = regexp.MustCompile(`https?://[^"/\s]*@`) + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- From 4bb79cf7b6cb7bba148b30406b1e267e77772dfc Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 21:17:59 -0400 Subject: [PATCH 05/17] =?UTF-8?q?fix(sbom):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20origin=20replacement,=20host=20roots,=20npm=20v1,=20round-tr?= =?UTF-8?q?ip=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Setters replace rather than merge. A second assertion on the same node no longer leaves both origin forms behind, and an unpinned repository no longer inherits the previous one's revision -- which would have named a commit that repository may not contain. A rejected value still leaves an earlier origin intact. - A host root is rejected for artifacts too, not just repositories. It names a server, not a package, so "https://registry.example/" was exactly the registry-root case this feature excludes. - npm v1 lockfiles now publish origin. They have no packages map, so they resolve through the flat dependencies tree, whose node type dropped the "resolved" field those lockfiles do record. This also picks up `npm ls --json` output, which carries the same field. - Correct the SPDX round-trip claim: origin is written on export and not read back on ingest, so re-exporting an ingested document says NOASSERTION. The docs said the opposite; a test now pins the real behavior. - Correct the Component comment: registry enrichment can fill VCSURL beside an artifact URL, so "at most one" held only for detector-asserted origin. - Clarify in docs that a repository may be unpinned, and that the http(s) rule governs detector origin while SPDX composes it into git+@. Co-Authored-By: Claude Opus 5 --- docs/SBOM.md | 25 ++++---- internal/detectors/node/common.go | 10 +++- .../detectors/node/origin_integration_test.go | 11 ++++ internal/detectors/origin.go | 55 +++++++++++++----- internal/detectors/origin_test.go | 58 +++++++++++++++++++ internal/sbom/model.go | 10 ++-- internal/sbom/origin_test.go | 35 +++++++++++ 7 files changed, 171 insertions(+), 33 deletions(-) diff --git a/docs/SBOM.md b/docs/SBOM.md index 62dc0f43..edbd22b3 100644 --- a/docs/SBOM.md +++ b/docs/SBOM.md @@ -113,7 +113,7 @@ A detector reports one of two things, or nothing at all: | What the lockfile records | SPDX 2.3 | CycloneDX | |---|---|---| | The exact file the package was fetched from | `downloadLocation` | `distribution` external reference | -| The repository it was resolved from, plus the commit | `downloadLocation` as `git+@` | `vcs` external reference (URL only) | +| The repository it was resolved from, and the commit when the lockfile pinned one | `downloadLocation` as `git+`, with `@` when pinned | `vcs` external reference (URL only) | | Neither | `NOASSERTION` | no reference | CycloneDX external references have no field for a revision, so the commit a @@ -147,10 +147,13 @@ Four kinds of value are never published, in any ecosystem: - **URLs carrying credentials.** A lockfile pointing at a private registry can embed a token; publishing it in an SBOM would leak a live secret. -Every published location is an absolute `http`/`https` URL with a host and no -embedded credentials. Values are re-serialized from a parse rather than copied -from the lockfile, and the same check runs again at export, so origin supplied -by a plugin is held to the same rule as origin from a built-in detector. +Every origin a detector reports is an absolute `http`/`https` URL with a host, a +non-empty path, and no embedded credentials. Values are re-serialized from a +parse rather than copied from the lockfile, and the same check runs again at +export, so origin supplied by a plugin is held to the same rule as origin from a +built-in detector. SPDX then composes the validated repository URL into its +`git+@` locator form, which is the only place a revision +appears. ### Document identity @@ -223,9 +226,8 @@ Reachability annotations and other Bomly-specific metadata are emitted in the JS ### Preservation and conversion limits Bomly preserves component identity (including PURL), dependency edges, roots, -scope, package type, licenses, digests, CPEs, package origin, and the enrichment -fields described above when the destination format has an equivalent -representation. Encoding is +scope, package type, licenses, digests, CPEs, and the enrichment fields +described above when the destination format has an equivalent representation. Encoding is deterministic when the scan timestamp and document identifiers are fixed. Some information necessarily becomes less specific during conversion: @@ -237,9 +239,10 @@ Some information necessarily becomes less specific during conversion: round trip. - Development scope maps to CycloneDX `excluded`; runtime scope maps to `required`. SPDX stores Bomly's normalized scope in the package comment. -- A resolved commit survives an SPDX round trip (it is part of the - `git+@` download location) but not a CycloneDX one, where an - external reference carries only the repository URL. +- Package origin is written on export but not read back on ingest: scanning an + SBOM produces packages with no origin, so re-exporting that graph emits + `NOASSERTION` and no distribution or vcs reference. Origin comes from a + lockfile, and an ingested document is not one. - Bomly relationship confidence (`direct`, `transitive`, or `unknown`), source provenance, reachability analysis, policy findings, and run diagnostics are report data rather than portable SBOM fields. Use JSON when those distinctions diff --git a/internal/detectors/node/common.go b/internal/detectors/node/common.go index 1aa645a8..274014bb 100644 --- a/internal/detectors/node/common.go +++ b/internal/detectors/node/common.go @@ -11,6 +11,7 @@ import ( "path/filepath" "time" + "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-cli/internal/logging" "github.com/bomly-dev/bomly-sdk" logkit "github.com/bomly-dev/bomly-sdk/logkit" @@ -26,8 +27,12 @@ type BaseDetector struct { // NPMListNode is the npm list JSON node shape used by npm CLI and v1 package-lock parsing. type NPMListNode struct { - Name string `json:"name"` - Version string `json:"version"` + Name string `json:"name"` + Version string `json:"version"` + // Resolved is the tarball a package was installed from. Both sources of + // this shape record it: `npm ls --json` for registry packages, and the + // flat "dependencies" map of a v1 package-lock.json. + Resolved string `json:"resolved"` Dependencies map[string]*NPMListNode `json:"dependencies"` } @@ -175,6 +180,7 @@ func DepGraphFromNPMNode(root *NPMListNode) (*sdk.Graph, error) { Name: name, Version: depNode.Version}, }) + detectors.SetOriginArtifact(node, depNode.Resolved) if err := AddNodeIfMissing(depsGraph, node); err != nil { return nil, err diff --git a/internal/detectors/node/origin_integration_test.go b/internal/detectors/node/origin_integration_test.go index d58c8955..7bf8e1df 100644 --- a/internal/detectors/node/origin_integration_test.go +++ b/internal/detectors/node/origin_integration_test.go @@ -41,6 +41,17 @@ func TestNPMLockfileOriginIsTheRegistryTarball(t *testing.T) { requireArtifactOrigin(t, g, "jest", "29.7.0", "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz") } +// A v1 lockfile has no packages map, so it resolves through the flat +// dependencies tree. It records the same tarball URLs, and must publish them. +func TestNPMLockfileV1OriginIsTheRegistryTarball(t *testing.T) { + g, err := resolveLockfileGraph(t, npm.LockfileDetector{}, fixture("npm-v1")) + if err != nil { + t.Fatal(err) + } + requireArtifactOrigin(t, g, "react", "18.2.0", "https://registry.npmjs.org/react/-/react-18.2.0.tgz") + requireArtifactOrigin(t, g, "loose-envify", "1.4.0", "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz") +} + // Workspace members are local directories. npm records that directory as the // member's "resolved" value, which must never reach an SBOM. func TestNPMWorkspaceMembersAssertNoOrigin(t *testing.T) { diff --git a/internal/detectors/origin.go b/internal/detectors/origin.go index ac61e63e..dcd16445 100644 --- a/internal/detectors/origin.go +++ b/internal/detectors/origin.go @@ -60,12 +60,15 @@ func (o Origin) Empty() bool { // git@host:org/repo, ssh://, git+ssh://, and URLs carrying userinfo — is // rejected, so filesystem layout and credentials cannot reach an SBOM. // -// The vcs argument selects the repository form: query and fragment are dropped -// (they carry the requested ref, not the resolved one, which callers pass -// separately) and a non-empty path is required, since a bare host names no -// repository. The artifact form instead drops the fragment (a checksum or -// anchor, never part of the location) and rejects a value carrying a query, -// which marks a signed or tokenized link rather than a stable location. +// Both forms require a non-empty path, so a bare host -- a registry or index +// root -- is never published. +// +// The vcs argument selects the repository form: query and fragment are dropped, +// because they carry the requested ref rather than the resolved one, which +// callers pass separately. The artifact form instead drops the fragment (a +// checksum or anchor, never part of the location) and rejects a value carrying +// a query, which marks a signed or tokenized link rather than a stable +// location. func NormalizeOriginURL(raw string, vcs bool) (string, bool) { trimmed := strings.TrimSpace(raw) if trimmed == "" { @@ -87,12 +90,16 @@ func NormalizeOriginURL(raw string, vcs bool) (string, bool) { parsed.Scheme = strings.ToLower(parsed.Scheme) parsed.Fragment = "" parsed.RawFragment = "" + // A host root names a server, not a package: it is a registry or index + // root on the artifact side and no repository at all on the VCS side. + // An empty path would also make the SPDX "@" suffix re-parse + // as userinfo. + if strings.Trim(parsed.Path, "/") == "" { + return "", false + } if vcs { parsed.RawQuery = "" parsed.ForceQuery = false - if strings.Trim(parsed.Path, "/") == "" { - return "", false - } } else if parsed.RawQuery != "" || parsed.ForceQuery { return "", false } @@ -103,10 +110,10 @@ func NormalizeOriginURL(raw string, vcs bool) (string, bool) { return normalized, true } -// SetOriginArtifact records the exact artifact dep was resolved from. Callers -// pass the lockfile field verbatim; values that are not publishable URLs are -// dropped silently, since a missing origin is correct output and a wrong one -// is not. No-op when dep is nil. +// SetOriginArtifact records the exact artifact dep was resolved from, replacing +// any origin already recorded. Callers pass the lockfile field verbatim; values +// that are not publishable URLs are dropped silently, since a missing origin is +// correct output and a wrong one is not. No-op when dep is nil. func SetOriginArtifact(dep *sdk.Dependency, rawURL string) { if dep == nil { return @@ -115,13 +122,14 @@ func SetOriginArtifact(dep *sdk.Dependency, rawURL string) { if !ok { return } + clearOrigin(dep) setOriginValue(dep, MetadataKeyOriginArtifactURL, normalized) } // SetOriginVCS records the source repository dep was resolved from, plus the -// revision the lockfile pinned. An unpublishable URL drops the whole origin; an -// unusable revision drops only the revision, keeping the repository. No-op when -// dep is nil. +// revision the lockfile pinned, replacing any origin already recorded. An +// unpublishable URL drops the whole origin; an unusable revision drops only the +// revision, keeping the repository. No-op when dep is nil. func SetOriginVCS(dep *sdk.Dependency, rawURL, revision string) { if dep == nil { return @@ -130,6 +138,7 @@ func SetOriginVCS(dep *sdk.Dependency, rawURL, revision string) { if !ok { return } + clearOrigin(dep) setOriginValue(dep, MetadataKeyOriginVCSURL, normalized) if pinned := strings.TrimSpace(revision); isValidOriginRevision(pinned) { setOriginValue(dep, MetadataKeyOriginVCSRevision, pinned) @@ -178,6 +187,20 @@ func isValidOriginRevision(revision string) bool { return true } +// clearOrigin drops any origin already recorded on dep, so a later assertion +// replaces an earlier one rather than merging with it: a package has one +// origin, and a stale revision left beside a new repository would name a commit +// that repository may not contain. Only a value that passed the invariant +// clears the previous one, so a rejected input leaves an earlier origin intact. +func clearOrigin(dep *sdk.Dependency) { + if dep.Metadata == nil { + return + } + delete(dep.Metadata, MetadataKeyOriginArtifactURL) + delete(dep.Metadata, MetadataKeyOriginVCSURL) + delete(dep.Metadata, MetadataKeyOriginVCSRevision) +} + // setOriginValue stores one origin fact, allocating the metadata map on demand. func setOriginValue(dep *sdk.Dependency, key, value string) { if dep.Metadata == nil { diff --git a/internal/detectors/origin_test.go b/internal/detectors/origin_test.go index 834bc71e..346fa872 100644 --- a/internal/detectors/origin_test.go +++ b/internal/detectors/origin_test.go @@ -30,6 +30,7 @@ func TestSetOriginArtifact(t *testing.T) { {name: "windows path is dropped", raw: `C:\src\project`, want: ""}, {name: "malformed host is dropped", raw: "https://:8080/pkg.tgz", want: ""}, {name: "scheme without host is dropped", raw: "https://", want: ""}, + {name: "registry root names no artifact", raw: "https://registry.example.test/", want: ""}, {name: "empty is dropped", raw: " ", want: ""}, } @@ -81,6 +82,7 @@ func TestSetOriginVCS(t *testing.T) { {name: "whitespace revision keeps the repository", raw: "https://github.com/owner/repo", revision: "not a revision", wantURL: "https://github.com/owner/repo"}, {name: "overlong revision keeps the repository", raw: "https://github.com/owner/repo", revision: strings.Repeat("a", 129), wantURL: "https://github.com/owner/repo"}, {name: "bare host names no repository", raw: "https://github.com", revision: "9f8e7d6", wantURL: ""}, + {name: "index root names no repository", raw: "https://index.crates.io/", revision: "9f8e7d6", wantURL: ""}, {name: "root path names no repository", raw: "https://github.com/", revision: "9f8e7d6", wantURL: ""}, {name: "userinfo is dropped", raw: "https://oauth2:glpat-xxxxxxxxxxxxxxxxxxxx@gitlab.corp/team/repo.git", revision: "9f8e7d6", wantURL: ""}, {name: "local checkout is dropped", raw: "/Users/someone/src/repo", revision: "9f8e7d6", wantURL: ""}, @@ -111,6 +113,62 @@ func TestSetOriginVCS(t *testing.T) { } } +// A package has one origin. A later assertion replaces an earlier one rather +// than merging with it, so metadata never names two locations or pairs a +// repository with a revision that belongs to a different one. +func TestSetOriginReplacesRatherThanMerges(t *testing.T) { + const ( + artifact = "https://registry.npmjs.org/react/-/react-18.2.0.tgz" + repository = "https://github.com/facebook/react" + fork = "https://github.com/facebook/react-fork" + revision = "c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8" + ) + + t.Run("repository replaces artifact", func(t *testing.T) { + dep := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginArtifact(dep, artifact) + detectors.SetOriginVCS(dep, repository, revision) + + want := detectors.Origin{VCSURL: repository, VCSRevision: revision} + if got := detectors.OriginFrom(dep.Metadata); got != want { + t.Fatalf("origin = %+v, want %+v", got, want) + } + }) + + t.Run("artifact replaces repository", func(t *testing.T) { + dep := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginVCS(dep, repository, revision) + detectors.SetOriginArtifact(dep, artifact) + + want := detectors.Origin{ArtifactURL: artifact} + if got := detectors.OriginFrom(dep.Metadata); got != want { + t.Fatalf("origin = %+v, want %+v", got, want) + } + }) + + t.Run("an unpinned repository drops the earlier revision", func(t *testing.T) { + dep := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginVCS(dep, repository, revision) + detectors.SetOriginVCS(dep, fork, "") + + want := detectors.Origin{VCSURL: fork} + if got := detectors.OriginFrom(dep.Metadata); got != want { + t.Fatalf("origin = %+v, want %+v; a revision must not follow a repository it did not come from", got, want) + } + }) + + t.Run("a rejected value leaves the earlier origin intact", func(t *testing.T) { + dep := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginArtifact(dep, artifact) + detectors.SetOriginVCS(dep, "/Users/someone/src/react", revision) + + want := detectors.Origin{ArtifactURL: artifact} + if got := detectors.OriginFrom(dep.Metadata); got != want { + t.Fatalf("origin = %+v, want %+v", got, want) + } + }) +} + func TestSetOriginAllocatesMetadataAndPreservesOtherKeys(t *testing.T) { dep := &sdk.Dependency{ID: "pkg"} detectors.SetOriginArtifact(dep, "https://registry.npmjs.org/react/-/react-18.2.0.tgz") diff --git a/internal/sbom/model.go b/internal/sbom/model.go index 363c724f..c3f71a6b 100644 --- a/internal/sbom/model.go +++ b/internal/sbom/model.go @@ -136,10 +136,12 @@ type Component struct { Vulnerabilities []Vulnerability EOL *EOL - // Where the package came from, as asserted by the detector that resolved - // it. At most one of ArtifactURL and VCSURL is set, and VCSRevision only - // accompanies VCSURL. Both are plain absolute http(s) URLs; composing them - // into a format's locator grammar is the encoder's job. + // Where the package came from. A detector asserts at most one of + // ArtifactURL and VCSURL; registry enrichment may then fill VCSURL from a + // resolved source repository when it is empty, so a component enriched + // under --enrich can carry both. VCSRevision only accompanies VCSURL. + // Both are plain absolute http(s) URLs; composing them into a format's + // locator grammar is the encoder's job. ArtifactURL string VCSURL string VCSRevision string diff --git a/internal/sbom/origin_test.go b/internal/sbom/origin_test.go index 317bf733..998180cf 100644 --- a/internal/sbom/origin_test.go +++ b/internal/sbom/origin_test.go @@ -268,3 +268,38 @@ func TestScorecardRepositoryFillsTheOriginGap(t *testing.T) { } }) } + +// Origin is written on export and not read back on ingest: it describes what a +// lockfile said, and an ingested document is not a lockfile. Scanning an SBOM +// therefore yields packages with no origin, and re-exporting that graph says +// NOASSERTION rather than repeating a claim Bomly did not resolve itself. +// +// This pins the documented limitation; preserving third-party origin across +// ingest is tracked separately. +func TestOriginIsNotReadBackFromAnIngestedDocument(t *testing.T) { + g := originGraph(t, func(_, pkg *sdk.Dependency) { + detectors.SetOriginVCS(pkg, "https://github.com/facebook/react", "d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f809") + }) + + exported, _ := marshalBoth(t, g) + if got := spdxPackageByName(t, exported, "react")["downloadLocation"]; got == "NOASSERTION" { + t.Fatal("precondition failed: the first export carried no origin") + } + + ingested, _, err := UnmarshalAutoJSON(exported) + if err != nil { + t.Fatalf("ingest SPDX: %v", err) + } + reingestedGraph, err := ToGraph(ingested) + if err != nil { + t.Fatalf("to graph: %v", err) + } + reexported, err := MarshalDepGraphJSON(reingestedGraph, TargetSPDX23JSON, BuildOptions{DocumentName: "origin-test", ToolVersion: "test"}, EncodeOptions{}) + if err != nil { + t.Fatalf("re-export SPDX: %v", err) + } + + if got := spdxPackageByName(t, reexported, "react")["downloadLocation"]; got != "NOASSERTION" { + t.Fatalf("re-exported downloadLocation = %v, want NOASSERTION; if origin now survives ingest, docs/SBOM.md must say so", got) + } +} From 20ad5041f72706e0c91aea00b35ba0452fff59aa Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 21:27:03 -0400 Subject: [PATCH 06/17] fix(detectors): reconcile origin when one package appears more than once A lockfile can record the same package at several places in a tree, and the copies can disagree -- one nested under a package pinned to a private mirror, one at the top level from the public registry. They share a name and version, so they become one graph node. The flat npm path made that worse: it walked a map, so which copy won varied between runs of the same lockfile, and an SBOM that changes run to run is not reproducible. Every other node lockfile path already sorted its keys; this one now does too. Sorting alone would only make the arbitrary winner stable, so occurrences are now reconciled where a duplicate folds into an existing node. Absence is not a disagreement: an occurrence asserting nothing leaves an origin standing, and one asserting something fills a gap. Two different assertions cancel -- one node is one package, and omitting a location is honest where taking a side of a contradiction is not. Applied in the shared node helper, so npm, pnpm, yarn, and bun all get it. Co-Authored-By: Claude Opus 5 --- dev-docs/ARCHITECTURE.md | 4 +- internal/detectors/node/common.go | 13 +++- internal/detectors/node/npm/origin_test.go | 61 +++++++++++++++ internal/detectors/origin.go | 41 +++++++++++ internal/detectors/origin_test.go | 86 ++++++++++++++++++++++ 5 files changed, 203 insertions(+), 2 deletions(-) diff --git a/dev-docs/ARCHITECTURE.md b/dev-docs/ARCHITECTURE.md index 3dabdab0..c4fe45fa 100644 --- a/dev-docs/ARCHITECTURE.md +++ b/dev-docs/ARCHITECTURE.md @@ -606,7 +606,9 @@ One rule governs every published value, `detectors.NormalizeOriginURL`: absolute The invariant runs twice — when a detector records a value and again when export reads it. The second pass is not redundant: graphs also arrive from plugins and from hand-built callers, and export must not publish a location no built-in detector could have produced. Composition into a format's locator grammar stays in the encoders, so `Component.VCSURL` remains a plain URL and only SPDX builds the `git+…@…` form; CycloneDX external references have no revision slot, so a resolved commit survives an SPDX round trip and not a CycloneDX one. -Two consequences worth stating. Origin keys are filtered out of `scan`/`diff`/`explain` payloads by prefix in `output.cloneRefMetadata` — they are transport between two pipeline stages, and the SBOM is where users read them; the filter returns nil for an emptied map so `omitempty` still fires. And consolidation's first-wins node dedup can drop the origin of a duplicate occurrence, which matches the existing behavior of `ResolvedURL` itself. +Where one package appears several times in a lockfile, its occurrences must agree before anything is published. `detectors.MergeOrigin`, applied wherever a detector folds a duplicate into an existing node, treats absence as compatible (an occurrence asserting nothing leaves an origin standing; one asserting something fills a gap) and treats two different assertions as cancelling. Picking a winner would make the output depend on traversal order rather than on the lockfile, and an SBOM that omits a location is honest where one that takes a side of a contradiction is not. Detector graph builds walk their inputs in sorted order for the same reason. + +Two consequences worth stating. Origin keys are filtered out of `scan`/`diff`/`explain` payloads by prefix in `output.cloneRefMetadata` — they are transport between two pipeline stages, and the SBOM is where users read them; the filter returns nil for an emptied map so `omitempty` still fires. And consolidation's first-wins node dedup, which runs later and across detectors, can still drop the origin of a duplicate occurrence, matching the existing behavior of `ResolvedURL` and detection licenses. ## Build Modes diff --git a/internal/detectors/node/common.go b/internal/detectors/node/common.go index 274014bb..cbd2ec79 100644 --- a/internal/detectors/node/common.go +++ b/internal/detectors/node/common.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "sort" "time" "github.com/bomly-dev/bomly-cli/internal/detectors" @@ -168,7 +169,16 @@ func DepGraphFromNPMNode(root *NPMListNode) (*sdk.Graph, error) { current := stack[len(stack)-1] stack = stack[:len(stack)-1] - for depName, depNode := range current.deps { + depNames := make([]string, 0, len(current.deps)) + for depName := range current.deps { + depNames = append(depNames, depName) + } + // Walk in a fixed order: a map's iteration order would let two + // occurrences of one package decide the graph differently per run. + sort.Strings(depNames) + + for _, depName := range depNames { + depNode := current.deps[depName] if depNode == nil { continue } @@ -330,6 +340,7 @@ func splitYarnTreeName(value string) (string, string, error) { func AddNodeIfMissing(depsGraph *sdk.Graph, node *sdk.Dependency) error { if existing, ok := depsGraph.Node(node.ID); ok { existing.AddScope(node.PrimaryScope()) + detectors.MergeOrigin(existing, node) return nil } if err := depsGraph.AddNode(node); err != nil { diff --git a/internal/detectors/node/npm/origin_test.go b/internal/detectors/node/npm/origin_test.go index f79b7a8f..7b057c5b 100644 --- a/internal/detectors/node/npm/origin_test.go +++ b/internal/detectors/node/npm/origin_test.go @@ -62,3 +62,64 @@ func TestNPMOriginByResolvedShape(t *testing.T) { } } } + +// A v1 lockfile repeats a package at every place it is installed, and the +// copies can disagree: one nested under a package pinned to a private mirror, +// one at the top level from the public registry. They share a name and version, +// so they become one graph node, and the node must not report whichever copy +// the traversal reached first. +func TestNPMv1DuplicateEntriesReconcileOrigin(t *testing.T) { + projectDir := t.TempDir() + lockfile := `{ + "name": "demo", + "version": "1.0.0", + "lockfileVersion": 1, + "dependencies": { + "a-first": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/a-first/-/a-first-1.0.0.tgz", + "dependencies": { + "disputed": {"version": "2.0.0", "resolved": "https://npm.corp/mirror/disputed/-/disputed-2.0.0.tgz"}, + "agreed": {"version": "3.0.0", "resolved": "https://registry.npmjs.org/agreed/-/agreed-3.0.0.tgz"} + } + }, + "z-last": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/z-last/-/z-last-1.0.0.tgz", + "dependencies": { + "disputed": {"version": "2.0.0", "resolved": "https://registry.npmjs.org/disputed/-/disputed-2.0.0.tgz"}, + "agreed": {"version": "3.0.0", "resolved": "https://registry.npmjs.org/agreed/-/agreed-3.0.0.tgz"} + } + } + } + }` + if err := os.WriteFile(filepath.Join(projectDir, "package-lock.json"), []byte(lockfile), 0o644); err != nil { + t.Fatal(err) + } + + // Repeat: the traversal walks maps, so an order-dependent result would + // show up as a different answer between runs rather than a stable wrong one. + for range 25 { + graphs, err := depGraphFromNPMLockfile(projectDir) + if err != nil { + t.Fatalf("depGraphFromNPMLockfile() error = %v", err) + } + + disputed, ok := graphs.graph.Node("disputed@2.0.0") + if !ok { + t.Fatal("expected disputed@2.0.0 in graph") + } + if origin := detectors.OriginFrom(disputed.Metadata); !origin.Empty() { + t.Fatalf("disputed origin = %+v, want none: its two copies name different locations", origin) + } + + agreed, ok := graphs.graph.Node("agreed@3.0.0") + if !ok { + t.Fatal("expected agreed@3.0.0 in graph") + } + const want = "https://registry.npmjs.org/agreed/-/agreed-3.0.0.tgz" + if got := detectors.OriginFrom(agreed.Metadata).ArtifactURL; got != want { + t.Fatalf("agreed origin = %q, want %q: its copies agree", got, want) + } + } +} diff --git a/internal/detectors/origin.go b/internal/detectors/origin.go index dcd16445..a857935e 100644 --- a/internal/detectors/origin.go +++ b/internal/detectors/origin.go @@ -168,6 +168,47 @@ func OriginFrom(metadata map[string]any) Origin { return origin } +// MergeOrigin reconciles the origin of two nodes a detector resolved to the +// same package, which happens when a lockfile records one package at several +// places in a tree. +// +// Absence is not a disagreement: an occurrence that asserts nothing leaves an +// existing origin standing, and an occurrence that asserts one fills a gap. +// Two occurrences asserting *different* origins cancel. One graph node is one +// package, so publishing whichever occurrence happened to be visited first +// would make the output depend on traversal order rather than on the lockfile +// -- and an SBOM that omits a location is honest, while one that picks a side +// of a contradiction is not. +func MergeOrigin(existing, duplicate *sdk.Dependency) { + if existing == nil || duplicate == nil { + return + } + incoming := OriginFrom(duplicate.Metadata) + if incoming.Empty() { + return + } + switch current := OriginFrom(existing.Metadata); { + case current.Empty(): + storeOrigin(existing, incoming) + case current != incoming: + clearOrigin(existing) + } +} + +// storeOrigin writes an already-validated origin onto dep. +func storeOrigin(dep *sdk.Dependency, origin Origin) { + clearOrigin(dep) + switch { + case origin.ArtifactURL != "": + setOriginValue(dep, MetadataKeyOriginArtifactURL, origin.ArtifactURL) + case origin.VCSURL != "": + setOriginValue(dep, MetadataKeyOriginVCSURL, origin.VCSURL) + if origin.VCSRevision != "" { + setOriginValue(dep, MetadataKeyOriginVCSRevision, origin.VCSRevision) + } + } +} + // isValidOriginRevision reports whether revision is safe to publish beside a // repository URL. The charset keeps commit hashes, tags, and branch-style refs // while excluding whitespace, "@", and percent escapes, which would break the diff --git a/internal/detectors/origin_test.go b/internal/detectors/origin_test.go index 346fa872..dc4aa42e 100644 --- a/internal/detectors/origin_test.go +++ b/internal/detectors/origin_test.go @@ -254,3 +254,89 @@ func TestOriginEmpty(t *testing.T) { t.Fatal("repository origin should not be empty") } } + +// One package can appear at several places in a dependency tree. Its +// occurrences must agree before an origin is published. +func TestMergeOrigin(t *testing.T) { + const ( + artifact = "https://registry.npmjs.org/react/-/react-18.2.0.tgz" + mirror = "https://npm.corp/mirror/react/-/react-18.2.0.tgz" + repo = "https://github.com/facebook/react" + ) + withArtifact := func(url string) *sdk.Dependency { + dep := &sdk.Dependency{ID: "react@18.2.0"} + detectors.SetOriginArtifact(dep, url) + return dep + } + + cases := []struct { + name string + existing *sdk.Dependency + duplicate *sdk.Dependency + want detectors.Origin + }{ + { + name: "occurrences agree", + existing: withArtifact(artifact), + duplicate: withArtifact(artifact), + want: detectors.Origin{ArtifactURL: artifact}, + }, + { + name: "occurrences disagree, so the graph cannot say", + existing: withArtifact(artifact), + duplicate: withArtifact(mirror), + }, + { + name: "a different kind of origin is also a disagreement", + existing: withArtifact(artifact), + duplicate: func() *sdk.Dependency { + d := &sdk.Dependency{ID: "react@18.2.0"} + detectors.SetOriginVCS(d, repo, "") + return d + }(), + }, + { + name: "an occurrence asserting nothing is not a disagreement", + existing: withArtifact(artifact), + duplicate: &sdk.Dependency{ID: "react@18.2.0"}, + want: detectors.Origin{ArtifactURL: artifact}, + }, + { + name: "an occurrence fills a gap the first one left", + existing: &sdk.Dependency{ID: "react@18.2.0"}, + duplicate: withArtifact(artifact), + want: detectors.Origin{ArtifactURL: artifact}, + }, + { + name: "neither asserts anything", + existing: &sdk.Dependency{ID: "react@18.2.0"}, + duplicate: &sdk.Dependency{ID: "react@18.2.0"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + detectors.MergeOrigin(tc.existing, tc.duplicate) + if got := detectors.OriginFrom(tc.existing.Metadata); got != tc.want { + t.Fatalf("merged origin = %+v, want %+v", got, tc.want) + } + }) + } + + t.Run("a pinned repository merging with an unpinned one disagrees", func(t *testing.T) { + existing := &sdk.Dependency{ID: "react@18.2.0"} + detectors.SetOriginVCS(existing, repo, "e7f8091a2b3c4d5e6f708192a3b4c5d6e7f80912") + duplicate := &sdk.Dependency{ID: "react@18.2.0"} + detectors.SetOriginVCS(duplicate, repo, "") + + detectors.MergeOrigin(existing, duplicate) + if got := detectors.OriginFrom(existing.Metadata); !got.Empty() { + t.Fatalf("merged origin = %+v, want none: the occurrences pin different commits", got) + } + }) + + t.Run("nil is a no-op", func(t *testing.T) { + detectors.MergeOrigin(nil, withArtifact(artifact)) + detectors.MergeOrigin(withArtifact(artifact), nil) + }) +} From d2e81505cd3e2093bdeef9cda213f814b0c49383 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 21:46:04 -0400 Subject: [PATCH 07/17] fix(detectors): keep origin conflicts sticky; pin native SwiftPM and pub runs Three findings from review, all real: - A disagreement between occurrences did not stay a disagreement. With copies claiming A, B, then A, the B conflict cleared the origin and the third copy stored A again -- publishing one side of a contradiction, which is what the rule added last commit was meant to prevent. The disagreement is now recorded under a metadata key that no later merge lifts. A detector setting an origin outright still supersedes it: that is an assertion about what was resolved, not a fold of two occurrences. - SwiftPM and pub have build-tool-backed primaries, and neither tool reports what this feature needs: `swift package show-dependencies` prints no revision, and `dart pub deps --json` prints no source description. So on a machine with swift installed, repositories exported unpinned; with dart installed, git packages exported nothing at all -- while the committed-file fallback exported both correctly. Each native path now reads its committed file back and joins the origins onto the graph, best effort. What Bomly reports no longer depends on which resolver ran. The join lives inside the function the detector calls rather than beside it, so deleting it fails a test instead of silently narrowing coverage -- the first version of this fix was tested at the helper and left the wiring uncovered. Co-Authored-By: Claude Opus 5 --- docs/SBOM.md | 5 +- internal/detectors/origin.go | 55 ++++++++-- internal/detectors/origin_test.go | 48 +++++++++ internal/detectors/pub/origin_test.go | 105 ++++++++++++++++++ internal/detectors/pub/pub_native.go | 50 ++++++++- internal/detectors/swiftpm/origin_test.go | 107 +++++++++++++++++++ internal/detectors/swiftpm/swiftpm_native.go | 69 +++++++++++- 7 files changed, 430 insertions(+), 9 deletions(-) diff --git a/docs/SBOM.md b/docs/SBOM.md index edbd22b3..9d71a68b 100644 --- a/docs/SBOM.md +++ b/docs/SBOM.md @@ -128,7 +128,10 @@ What each ecosystem yields: - **uv, poetry, pipenv, pip** — a repository plus the commit that was locked, or a direct archive URL, depending on the recorded source type. - **cargo, Bundler, SwiftPM, pub** — the repository and resolved commit for - git dependencies and source-control pins. + git dependencies and source-control pins. SwiftPM and pub report the same + origin whether the build tool ran or Bomly read the committed file: the tools + do not print a commit, so it is read back from `Package.resolved` and + `pubspec.lock`. - **Go modules, Maven, Gradle, NuGet, and the other detectors** — nothing yet; their manifests do not record a per-package location. - Packages found by Syft, and packages read from an ingested SBOM, carry no diff --git a/internal/detectors/origin.go b/internal/detectors/origin.go index a857935e..640b2bfa 100644 --- a/internal/detectors/origin.go +++ b/internal/detectors/origin.go @@ -23,6 +23,12 @@ const ( // MetadataKeyOriginVCSRevision holds the resolved revision (commit, tag) // pinned alongside MetadataKeyOriginVCSURL. MetadataKeyOriginVCSRevision = MetadataKeyOriginPrefix + "vcs_revision" + // MetadataKeyOriginConflict marks a package whose occurrences disagreed + // about where it came from. The mark outlives the occurrence that caused + // it, so a later occurrence repeating one of the disputed values cannot + // revive it: with three occurrences claiming A, B, then A, the package + // still has no agreed origin. + MetadataKeyOriginConflict = MetadataKeyOriginPrefix + "conflict" ) // maxOriginRevisionLength bounds a recorded revision. Real commit hashes and @@ -154,6 +160,9 @@ func OriginFrom(metadata map[string]any) Origin { if len(metadata) == 0 { return Origin{} } + if conflicted, _ := metadata[MetadataKeyOriginConflict].(bool); conflicted { + return Origin{} + } if artifact, ok := NormalizeOriginURL(originString(metadata, MetadataKeyOriginArtifactURL), false); ok { return Origin{ArtifactURL: artifact} } @@ -174,15 +183,26 @@ func OriginFrom(metadata map[string]any) Origin { // // Absence is not a disagreement: an occurrence that asserts nothing leaves an // existing origin standing, and an occurrence that asserts one fills a gap. -// Two occurrences asserting *different* origins cancel. One graph node is one -// package, so publishing whichever occurrence happened to be visited first -// would make the output depend on traversal order rather than on the lockfile -// -- and an SBOM that omits a location is honest, while one that picks a side -// of a contradiction is not. +// Two occurrences asserting *different* origins cancel, and stay cancelled: the +// disagreement is recorded so a third occurrence repeating one of the disputed +// values cannot revive it. One graph node is one package, so publishing +// whichever occurrence happened to be visited first would make the output +// depend on traversal order rather than on the lockfile -- and an SBOM that +// omits a location is honest, while one that picks a side of a contradiction +// is not. func MergeOrigin(existing, duplicate *sdk.Dependency) { if existing == nil || duplicate == nil { return } + if originConflicted(existing) { + // Already cancelled. Nothing a later occurrence says can settle a + // disagreement that happened, so the mark is not lifted here. + return + } + if originConflicted(duplicate) { + markOriginConflict(existing) + return + } incoming := OriginFrom(duplicate.Metadata) if incoming.Empty() { return @@ -191,8 +211,27 @@ func MergeOrigin(existing, duplicate *sdk.Dependency) { case current.Empty(): storeOrigin(existing, incoming) case current != incoming: - clearOrigin(existing) + markOriginConflict(existing) + } +} + +// originConflicted reports whether dep's occurrences already disagreed. +func originConflicted(dep *sdk.Dependency) bool { + if dep == nil || dep.Metadata == nil { + return false + } + conflicted, _ := dep.Metadata[MetadataKeyOriginConflict].(bool) + return conflicted +} + +// markOriginConflict drops dep's origin and records that its occurrences +// disagreed, so no later merge can restore one of the disputed values. +func markOriginConflict(dep *sdk.Dependency) { + clearOrigin(dep) + if dep.Metadata == nil { + dep.Metadata = make(map[string]any, 1) } + dep.Metadata[MetadataKeyOriginConflict] = true } // storeOrigin writes an already-validated origin onto dep. @@ -240,6 +279,10 @@ func clearOrigin(dep *sdk.Dependency) { delete(dep.Metadata, MetadataKeyOriginArtifactURL) delete(dep.Metadata, MetadataKeyOriginVCSURL) delete(dep.Metadata, MetadataKeyOriginVCSRevision) + // A detector setting an origin outright is asserting what it resolved, + // which supersedes a disagreement between earlier occurrences. Only + // merging leaves the mark in place. + delete(dep.Metadata, MetadataKeyOriginConflict) } // setOriginValue stores one origin fact, allocating the metadata map on demand. diff --git a/internal/detectors/origin_test.go b/internal/detectors/origin_test.go index dc4aa42e..7466e2fe 100644 --- a/internal/detectors/origin_test.go +++ b/internal/detectors/origin_test.go @@ -335,6 +335,54 @@ func TestMergeOrigin(t *testing.T) { } }) + // A disagreement is a fact about the package, not about the pair of + // occurrences that exposed it: a third copy repeating one of the disputed + // values must not settle it. + t.Run("a disagreement stays cancelled", func(t *testing.T) { + existing := withArtifact(artifact) + detectors.MergeOrigin(existing, withArtifact(mirror)) + detectors.MergeOrigin(existing, withArtifact(artifact)) + + if got := detectors.OriginFrom(existing.Metadata); !got.Empty() { + t.Fatalf("origin = %+v, want none: occurrences A, B, A never agreed", got) + } + }) + + t.Run("a cancelled origin does not spread by absence", func(t *testing.T) { + existing := withArtifact(artifact) + detectors.MergeOrigin(existing, withArtifact(mirror)) + detectors.MergeOrigin(existing, &sdk.Dependency{ID: "react@18.2.0"}) + + if got := detectors.OriginFrom(existing.Metadata); !got.Empty() { + t.Fatalf("origin = %+v, want none", got) + } + }) + + t.Run("a conflicted duplicate cancels an agreed origin", func(t *testing.T) { + conflicted := withArtifact(artifact) + detectors.MergeOrigin(conflicted, withArtifact(mirror)) + + existing := withArtifact(artifact) + detectors.MergeOrigin(existing, conflicted) + + if got := detectors.OriginFrom(existing.Metadata); !got.Empty() { + t.Fatalf("origin = %+v, want none: the duplicate had already disagreed with itself", got) + } + }) + + // Merging folds occurrences together; a detector setting an origin is + // asserting what it resolved, which is authoritative. + t.Run("a detector assertion supersedes a recorded disagreement", func(t *testing.T) { + dep := withArtifact(artifact) + detectors.MergeOrigin(dep, withArtifact(mirror)) + detectors.SetOriginArtifact(dep, mirror) + + want := detectors.Origin{ArtifactURL: mirror} + if got := detectors.OriginFrom(dep.Metadata); got != want { + t.Fatalf("origin = %+v, want %+v", got, want) + } + }) + t.Run("nil is a no-op", func(t *testing.T) { detectors.MergeOrigin(nil, withArtifact(artifact)) detectors.MergeOrigin(withArtifact(artifact), nil) diff --git a/internal/detectors/pub/origin_test.go b/internal/detectors/pub/origin_test.go index 20ac4413..56fd8c34 100644 --- a/internal/detectors/pub/origin_test.go +++ b/internal/detectors/pub/origin_test.go @@ -1,9 +1,13 @@ package pub import ( + "os" + "path/filepath" "testing" "github.com/bomly-dev/bomly-cli/internal/detectors" + "github.com/bomly-dev/bomly-sdk" + "go.uber.org/zap" ) // A pubspec.lock hosted package's description URL is the pub server, shared by @@ -80,3 +84,104 @@ func TestPubOriginBySourceType(t *testing.T) { } } } + +// `dart pub deps --json` reports a name, version, and kind but not a package's +// source description, so the native path alone would export no origin for git +// dependencies. The descriptions are read back from pubspec.lock. +func TestPubNativeOriginIsReadFromPubspecLock(t *testing.T) { + workingDir := t.TempDir() + lock := `packages: + helper: + dependency: "direct main" + description: + url: "https://github.com/example/helper.git" + ref: main + resolved-ref: 1a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d + path: "." + source: git + version: "2.0.0" + collection: + dependency: transitive + description: + name: collection + sha256: abc + url: "https://pub.dev" + source: hosted + version: "1.18.0" + local_tools: + dependency: "direct dev" + description: + path: "../local_tools" + relative: true + source: path + version: "0.1.0" +` + if err := os.WriteFile(filepath.Join(workingDir, "pubspec.lock"), []byte(lock), 0o644); err != nil { + t.Fatal(err) + } + + // `dart pub deps --json` reports a name, version, kind, and source per + // package, and no source description at all. + depsJSON := []byte(`{ + "root": "demo", + "packages": [ + {"name": "demo", "version": "1.0.0", "kind": "root", "source": "root", "dependencies": ["helper", "collection", "local_tools"]}, + {"name": "helper", "version": "2.0.0", "kind": "direct", "source": "git", "dependencies": []}, + {"name": "collection", "version": "1.18.0", "kind": "transitive", "source": "hosted", "dependencies": []}, + {"name": "local_tools", "version": "0.1.0", "kind": "dev", "source": "path", "dependencies": []} + ] + }`) + + g, err := nativeGraph(depsJSON, workingDir, zap.NewNop()) + if err != nil { + t.Fatalf("nativeGraph() error = %v", err) + } + + want := detectors.Origin{ + VCSURL: "https://github.com/example/helper.git", + VCSRevision: "1a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d", + } + var checked int + g.WalkNodes(func(dep *sdk.Dependency) bool { + origin := detectors.OriginFrom(dep.Metadata) + switch dep.Name { + case "helper": + checked++ + if origin != want { + t.Errorf("helper origin = %+v, want %+v", origin, want) + } + case "collection", "local_tools": + checked++ + if !origin.Empty() { + t.Errorf("%s asserted an origin: %+v", dep.Name, origin) + } + } + return true + }) + if checked != 3 { + t.Fatalf("checked %d packages, want 3", checked) + } +} + +// A project with no pubspec.lock keeps the graph as it is. +func TestPubNativeOriginSurvivesMissingLock(t *testing.T) { + depsJSON := []byte(`{ + "root": "demo", + "packages": [ + {"name": "demo", "version": "1.0.0", "kind": "root", "source": "root", "dependencies": ["helper"]}, + {"name": "helper", "version": "2.0.0", "kind": "direct", "source": "git", "dependencies": []} + ] + }`) + + g, err := nativeGraph(depsJSON, t.TempDir(), zap.NewNop()) + if err != nil { + t.Fatalf("nativeGraph() error = %v", err) + } + + g.WalkNodes(func(dep *sdk.Dependency) bool { + if got := detectors.OriginFrom(dep.Metadata); !got.Empty() { + t.Fatalf("%s origin = %+v, want none", dep.Name, got) + } + return true + }) +} diff --git a/internal/detectors/pub/pub_native.go b/internal/detectors/pub/pub_native.go index 160c18cc..8010a862 100644 --- a/internal/detectors/pub/pub_native.go +++ b/internal/detectors/pub/pub_native.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "path/filepath" "time" "github.com/bomly-dev/bomly-cli/internal/detectors" @@ -14,6 +15,7 @@ import ( logkit "github.com/bomly-dev/bomly-sdk/logkit" "github.com/bomly-dev/bomly-sdk/system" "go.uber.org/zap" + "gopkg.in/yaml.v3" ) // NativeDetector resolves Dart pub dependency graphs by running `dart pub deps --json`. @@ -75,7 +77,7 @@ func (d NativeDetector) ResolveGraph(_ context.Context, req sdk.DetectionRequest return sdk.DetectionResult{}, fmt.Errorf("dart pub deps: %w", err) } - g, err := depGraphFromPubDepsJSON(out.Bytes()) + g, err := nativeGraph(out.Bytes(), workingDir, logger) if err != nil { return sdk.DetectionResult{}, fmt.Errorf("parse dart pub deps output: %w", err) } @@ -85,6 +87,52 @@ func (d NativeDetector) ResolveGraph(_ context.Context, req sdk.DetectionRequest }, nil } +// nativeGraph builds the dependency graph for a native pub run: the tool's own +// output for structure, and the committed pubspec.lock for the package sources +// that output omits. +func nativeGraph(raw []byte, workingDir string, logger *zap.Logger) (*sdk.Graph, error) { + g, err := depGraphFromPubDepsJSON(raw) + if err != nil { + return nil, err + } + applyLockOrigins(g, workingDir, logger) + return g, nil +} + +// applyLockOrigins records where a native graph's git packages came from. +// `dart pub deps --json` reports a name, version, and kind but not a package's +// source description, so without this the default path would export no origin +// for git dependencies while the committed-file fallback exports the +// repository and the commit pub resolved. +// +// Best effort: a project with no readable pubspec.lock keeps the graph as it is. +func applyLockOrigins(g *sdk.Graph, workingDir string, logger *zap.Logger) { + raw, err := system.ReadRepositoryFile(filepath.Join(workingDir, "pubspec.lock")) + if err != nil { + return + } + var lock pubLock + if err := yaml.Unmarshal(raw, &lock); err != nil { + logger.Debug("pub: could not read pubspec.lock for origin", zap.Error(err)) + return + } + if len(lock.Packages) == 0 { + return + } + + recorded := 0 + g.WalkNodes(func(dep *sdk.Dependency) bool { + pkg, ok := lock.Packages[dep.Name] + if !ok || pubDependencySource(pkg.Source) != sdk.DependencySourceGit { + return true + } + detectors.SetOriginVCS(dep, descriptionString(pkg.Description, "url"), descriptionString(pkg.Description, "resolved-ref")) + recorded++ + return true + }) + logger.Debug(fmt.Sprintf("pub: recorded %d package origins from pubspec.lock", recorded)) +} + // FallbackDetector returns the configured fallback detector. func (d NativeDetector) FallbackDetector() sdk.Detector { return d.Fallback diff --git a/internal/detectors/swiftpm/origin_test.go b/internal/detectors/swiftpm/origin_test.go index 4fda4cb5..e96a1a40 100644 --- a/internal/detectors/swiftpm/origin_test.go +++ b/internal/detectors/swiftpm/origin_test.go @@ -1,9 +1,13 @@ package swiftpm import ( + "os" + "path/filepath" "testing" "github.com/bomly-dev/bomly-cli/internal/detectors" + "github.com/bomly-dev/bomly-sdk" + "go.uber.org/zap" ) // A Package.resolved pin says how SwiftPM obtained a package. Source-control @@ -62,3 +66,106 @@ func TestSwiftPMOriginByPinKind(t *testing.T) { t.Fatalf("checked %d pins, want 3", checked) } } + +// `swift package show-dependencies` reports a repository and a version but no +// revision, so the native path alone would export unpinned repositories while +// the committed-file fallback exports pinned ones. The pins are read back from +// Package.resolved and joined onto the graph. +func TestSwiftPMNativeOriginIsPinnedFromPackageResolved(t *testing.T) { + workingDir := t.TempDir() + resolved := `{ + "pins": [ + { + "identity": "swift-argument-parser", + "kind": "remoteSourceControl", + "location": "https://github.com/apple/swift-argument-parser.git", + "state": {"revision": "f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b", "version": "1.3.0"} + }, + { + "identity": "local-helper", + "kind": "localSourceControl", + "location": "/Users/someone/src/local-helper", + "state": {"revision": "091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c", "version": "0.1.0"} + } + ], + "version": 2 + }` + if err := os.WriteFile(filepath.Join(workingDir, "Package.resolved"), []byte(resolved), 0o644); err != nil { + t.Fatal(err) + } + + // `swift package show-dependencies` reports a URL and a version per node, + // and no revision anywhere. + showDependencies := []byte(`{ + "name": "demo", + "url": "/workspace/demo", + "version": "unspecified", + "dependencies": [ + {"name": "swift-argument-parser", "url": "https://github.com/apple/swift-argument-parser.git", "version": "1.3.0", "dependencies": []}, + {"name": "local-helper", "url": "/Users/someone/src/local-helper", "version": "0.1.0", "dependencies": []} + ] + }`) + + g, err := nativeGraph(showDependencies, workingDir, zap.NewNop()) + if err != nil { + t.Fatalf("nativeGraph() error = %v", err) + } + + want := detectors.Origin{ + VCSURL: "https://github.com/apple/swift-argument-parser.git", + VCSRevision: "f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b", + } + var checked int + g.WalkNodes(func(dep *sdk.Dependency) bool { + origin := detectors.OriginFrom(dep.Metadata) + switch dep.Name { + case "swift-argument-parser": + checked++ + if origin != want { + t.Errorf("remote origin = %+v, want %+v", origin, want) + } + case "local-helper": + checked++ + // A local checkout is a path on this machine, pinned or not. + if !origin.Empty() { + t.Errorf("local package asserted an origin: %+v", origin) + } + } + return true + }) + if checked != 2 { + t.Fatalf("checked %d packages, want 2", checked) + } +} + +// A project with no Package.resolved keeps whatever the native graph carried. +func TestSwiftPMNativeOriginSurvivesMissingPackageResolved(t *testing.T) { + showDependencies := []byte(`{ + "name": "demo", + "url": "/workspace/demo", + "version": "unspecified", + "dependencies": [ + {"name": "swift-argument-parser", "url": "https://github.com/apple/swift-argument-parser.git", "version": "1.3.0", "dependencies": []} + ] + }`) + + g, err := nativeGraph(showDependencies, t.TempDir(), zap.NewNop()) + if err != nil { + t.Fatalf("nativeGraph() error = %v", err) + } + + want := detectors.Origin{VCSURL: "https://github.com/apple/swift-argument-parser.git"} + var checked int + g.WalkNodes(func(dep *sdk.Dependency) bool { + if dep.Name == "swift-argument-parser" { + checked++ + if got := detectors.OriginFrom(dep.Metadata); got != want { + t.Fatalf("origin = %+v, want the unpinned repository %+v", got, want) + } + } + return true + }) + if checked != 1 { + t.Fatal("expected the remote package in the graph") + } +} diff --git a/internal/detectors/swiftpm/swiftpm_native.go b/internal/detectors/swiftpm/swiftpm_native.go index eeefcdd6..b74ec0ef 100644 --- a/internal/detectors/swiftpm/swiftpm_native.go +++ b/internal/detectors/swiftpm/swiftpm_native.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "strings" "time" "github.com/bomly-dev/bomly-cli/internal/detectors" @@ -75,7 +76,7 @@ func (d NativeDetector) ResolveGraph(_ context.Context, req sdk.DetectionRequest return sdk.DetectionResult{}, fmt.Errorf("swift package show-dependencies: %w", err) } - g, err := depGraphFromSwiftShowDeps(out.Bytes()) + g, err := nativeGraph(out.Bytes(), workingDir, logger) if err != nil { return sdk.DetectionResult{}, fmt.Errorf("parse swift show-dependencies output: %w", err) } @@ -85,6 +86,72 @@ func (d NativeDetector) ResolveGraph(_ context.Context, req sdk.DetectionRequest }, nil } +// nativeGraph builds the dependency graph for a native SwiftPM run: the tool's +// own output for structure, and the committed Package.resolved for the commits +// that output omits. +func nativeGraph(raw []byte, workingDir string, logger *zap.Logger) (*sdk.Graph, error) { + g, err := depGraphFromSwiftShowDeps(raw) + if err != nil { + return nil, err + } + applyResolvedOrigins(g, workingDir, logger) + return g, nil +} + +// applyResolvedOrigins pins the repositories in a native graph to the commits +// Package.resolved recorded. `swift package show-dependencies` reports a URL +// and a version but no revision, so without this the default path would export +// unpinned repositories while the committed-file fallback exports pinned ones. +// +// Best effort: a project with no readable Package.resolved keeps the origins +// the graph already carries. +func applyResolvedOrigins(g *sdk.Graph, workingDir string, logger *zap.Logger) { + raw, path, err := readFirstExisting(workingDir, []string{"Package.resolved", ".package.resolved", "project.xcworkspace/xcshareddata/swiftpm/Package.resolved"}) + if err != nil || len(raw) == 0 { + return + } + pins, err := parseResolved(raw) + if err != nil { + logger.Debug("swiftpm: could not read pins for origin", zap.String("path", path), zap.Error(err)) + return + } + if len(pins) == 0 { + return + } + + byRepository := make(map[string]swiftPackage, len(pins)) + for _, pin := range pins { + if key := repositoryKey(pin.Repository); key != "" { + byRepository[key] = pin + } + } + + pinned := 0 + g.WalkNodes(func(dep *sdk.Dependency) bool { + pin, ok := byRepository[repositoryKey(dep.ResolvedURL)] + if !ok { + if pin, ok = pins[dep.Name]; !ok { + return true + } + } + if pin.Revision == "" || swiftDependencySource(pin.SourceKind, pin.Repository) != sdk.DependencySourceGit { + return true + } + detectors.SetOriginVCS(dep, pin.Repository, pin.Revision) + pinned++ + return true + }) + logger.Debug(fmt.Sprintf("swiftpm: pinned %d package origins from %s", pinned, path)) +} + +// repositoryKey normalizes a repository URL for matching a pin to a graph +// node: SwiftPM reports the same repository with and without a ".git" suffix +// and in either case. +func repositoryKey(repository string) string { + key := strings.TrimSuffix(strings.TrimSuffix(strings.ToLower(strings.TrimSpace(repository)), "/"), ".git") + return key +} + // FallbackDetector returns the configured fallback detector. func (d NativeDetector) FallbackDetector() sdk.Detector { return d.Fallback From 8bfcc031811d0128f950ae7b31b21e6bb6240cea Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 21:59:06 -0400 Subject: [PATCH 08/17] fix(detectors): never credit a local checkout to the pin it replaced `swift package edit --path ...` swaps a dependency for a local checkout while Package.resolved keeps the pin that checkout replaced. The pin lookup missed on the local path and fell through to matching by identity, so the SBOM claimed the edited local code came from the remote repository at that commit -- a false provenance claim, which is the failure this feature exists to avoid. Reproduced, then fixed by requiring the graph node's own source to be git: what the build resolved is the truth, and the committed file only supplies the commit the tool omitted. pub can reach the same state through dependency_overrides, so it gets the same guard. Also document that `--enrich` can attach a repository no lockfile claimed: the Scorecard matcher resolves one from package identity, which fills the vcs reference for packages whose detector reported nothing -- including ecosystems the docs list as yielding nothing, and Syft-detected packages. It is a network lookup rather than a manifest claim, carries no revision, and always loses to a detector-asserted repository. Co-Authored-By: Claude Opus 5 --- docs/SBOM.md | 16 ++++++- internal/detectors/pub/origin_test.go | 48 +++++++++++++++++++ internal/detectors/pub/pub_native.go | 6 +++ internal/detectors/swiftpm/origin_test.go | 50 ++++++++++++++++++++ internal/detectors/swiftpm/swiftpm_native.go | 7 +++ 5 files changed, 126 insertions(+), 1 deletion(-) diff --git a/docs/SBOM.md b/docs/SBOM.md index 9d71a68b..f2471ae1 100644 --- a/docs/SBOM.md +++ b/docs/SBOM.md @@ -135,7 +135,18 @@ What each ecosystem yields: - **Go modules, Maven, Gradle, NuGet, and the other detectors** — nothing yet; their manifests do not record a per-package location. - Packages found by Syft, and packages read from an ingested SBOM, carry no - origin. + detector origin. + +With `--enrich`, a package can also get a repository it has no lockfile claim +to. The OpenSSF Scorecard matcher resolves a canonical source repository from a +package's identity, and that repository fills the `vcs` reference for any +package whose own detector reported nothing — including the ecosystems listed +above as yielding nothing, and Syft-detected packages. It is a network lookup +keyed on package identity rather than a claim any manifest made, so it is +weaker evidence: a detector-asserted repository always wins, and no revision is +attached, since a Scorecard repository names a project rather than a resolved +commit. Without `--enrich`, or without the scorecard matcher selected, nothing +of this kind appears. Four kinds of value are never published, in any ecosystem: @@ -223,6 +234,9 @@ registry (keyed by PURL): - Vulnerabilities — CycloneDX as a first-class `vulnerabilities` array (ratings, CWEs, advisories, `affects`); SPDX as `SECURITY`/`advisory` external references. - End-of-life status (CycloneDX `bomly:eol*` properties, SPDX package comment). +- A source repository resolved by the OpenSSF Scorecard matcher, used only when + the detector reported no repository of its own (see "Where a package came + from" above). Reachability annotations and other Bomly-specific metadata are emitted in the JSON output (`--json` or `--format json`), not in the standard SBOM formats. See [Output formats](OUTPUT_FORMATS.md). diff --git a/internal/detectors/pub/origin_test.go b/internal/detectors/pub/origin_test.go index 56fd8c34..311a1d67 100644 --- a/internal/detectors/pub/origin_test.go +++ b/internal/detectors/pub/origin_test.go @@ -163,6 +163,54 @@ func TestPubNativeOriginIsReadFromPubspecLock(t *testing.T) { } } +// An override can point a package at a local path while pubspec.lock still +// describes the git dependency it replaced. The built code is local, so it must +// not be credited to that repository. +func TestPubOverriddenPackageIsNotCreditedToTheLockedRepository(t *testing.T) { + workingDir := t.TempDir() + lock := `packages: + helper: + dependency: "direct main" + description: + url: "https://github.com/example/helper.git" + ref: main + resolved-ref: 3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f + path: "." + source: git + version: "2.0.0" +` + if err := os.WriteFile(filepath.Join(workingDir, "pubspec.lock"), []byte(lock), 0o644); err != nil { + t.Fatal(err) + } + depsJSON := []byte(`{ + "root": "demo", + "packages": [ + {"name": "demo", "version": "1.0.0", "kind": "root", "source": "root", "dependencies": ["helper"]}, + {"name": "helper", "version": "2.0.0", "kind": "direct", "source": "path", "dependencies": []} + ] + }`) + + g, err := nativeGraph(depsJSON, workingDir, zap.NewNop()) + if err != nil { + t.Fatalf("nativeGraph() error = %v", err) + } + + var checked int + g.WalkNodes(func(dep *sdk.Dependency) bool { + if dep.Name != "helper" { + return true + } + checked++ + if got := detectors.OriginFrom(dep.Metadata); !got.Empty() { + t.Fatalf("overridden package origin = %+v, want none", got) + } + return true + }) + if checked != 1 { + t.Fatal("expected the overridden package in the graph") + } +} + // A project with no pubspec.lock keeps the graph as it is. func TestPubNativeOriginSurvivesMissingLock(t *testing.T) { depsJSON := []byte(`{ diff --git a/internal/detectors/pub/pub_native.go b/internal/detectors/pub/pub_native.go index 8010a862..a7447a4c 100644 --- a/internal/detectors/pub/pub_native.go +++ b/internal/detectors/pub/pub_native.go @@ -122,6 +122,12 @@ func applyLockOrigins(g *sdk.Graph, workingDir string, logger *zap.Logger) { recorded := 0 g.WalkNodes(func(dep *sdk.Dependency) bool { + if dep.Source != sdk.DependencySourceGit { + // An override can point a package at a local path while the lock + // still describes the git dependency it replaced. What pub + // resolved for this build is the truth. + return true + } pkg, ok := lock.Packages[dep.Name] if !ok || pubDependencySource(pkg.Source) != sdk.DependencySourceGit { return true diff --git a/internal/detectors/swiftpm/origin_test.go b/internal/detectors/swiftpm/origin_test.go index e96a1a40..62782b94 100644 --- a/internal/detectors/swiftpm/origin_test.go +++ b/internal/detectors/swiftpm/origin_test.go @@ -138,6 +138,56 @@ func TestSwiftPMNativeOriginIsPinnedFromPackageResolved(t *testing.T) { } } +// `swift package edit --path ...` points a dependency at a local +// checkout, and Package.resolved keeps the pin that checkout replaced. The +// package being built is the local code, so it must not be credited to the +// remote repository and commit that pin still names. +func TestSwiftPMEditedPackageIsNotCreditedToItsFormerPin(t *testing.T) { + workingDir := t.TempDir() + resolved := `{ + "pins": [ + { + "identity": "helper", + "kind": "remoteSourceControl", + "location": "https://github.com/example/helper.git", + "state": {"revision": "2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e", "version": "1.0.0"} + } + ], + "version": 2 + }` + if err := os.WriteFile(filepath.Join(workingDir, "Package.resolved"), []byte(resolved), 0o644); err != nil { + t.Fatal(err) + } + showDependencies := []byte(`{ + "name": "demo", + "url": "/workspace/demo", + "version": "unspecified", + "dependencies": [ + {"name": "helper", "url": "/workspace/helper", "version": "unspecified", "dependencies": []} + ] + }`) + + g, err := nativeGraph(showDependencies, workingDir, zap.NewNop()) + if err != nil { + t.Fatalf("nativeGraph() error = %v", err) + } + + var checked int + g.WalkNodes(func(dep *sdk.Dependency) bool { + if dep.Name != "helper" { + return true + } + checked++ + if got := detectors.OriginFrom(dep.Metadata); !got.Empty() { + t.Fatalf("edited package origin = %+v, want none: it is built from a local checkout", got) + } + return true + }) + if checked != 1 { + t.Fatal("expected the edited package in the graph") + } +} + // A project with no Package.resolved keeps whatever the native graph carried. func TestSwiftPMNativeOriginSurvivesMissingPackageResolved(t *testing.T) { showDependencies := []byte(`{ diff --git a/internal/detectors/swiftpm/swiftpm_native.go b/internal/detectors/swiftpm/swiftpm_native.go index b74ec0ef..e389cffa 100644 --- a/internal/detectors/swiftpm/swiftpm_native.go +++ b/internal/detectors/swiftpm/swiftpm_native.go @@ -128,6 +128,13 @@ func applyResolvedOrigins(g *sdk.Graph, workingDir string, logger *zap.Logger) { pinned := 0 g.WalkNodes(func(dep *sdk.Dependency) bool { + if dep.Source != sdk.DependencySourceGit { + // `swift package edit` replaces a dependency with a local + // checkout while Package.resolved keeps the pin it replaced. + // What the build resolved is the truth, so a local node is left + // alone rather than credited to the repository it stands in for. + return true + } pin, ok := byRepository[repositoryKey(dep.ResolvedURL)] if !ok { if pin, ok = pins[dep.Name]; !ok { From 993b3159f48acc044f62ee283edd4712b5f67c91 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 22:15:16 -0400 Subject: [PATCH 09/17] fix(engine): settle package origin across manifests before graphs merge Each subproject is resolved on its own, so a package two of them share arrives as two nodes. The SDK's graph merge keeps whichever it meets first and discards the rest, so a recursive or multi-manifest scan would publish one subproject's answer for a package the scan saw resolved two different ways -- a monorepo where one workspace pulls from a private mirror and another from the public registry is enough to trigger it. ConsolidateGraphs now settles origin across the selected entries while both occurrences are still visible, and writes the verdict onto every one of them so the surviving node carries it whichever the merge keeps. The merge itself lives in the pinned SDK and is not changed here. A recorded disagreement is part of that verdict, which is why it is a metadata key rather than an absent value: absence would let a later fold refill it. The test proves that property directly rather than only observing an empty origin, after a mutation check showed the two were indistinguishable. This supersedes the earlier decision to leave cross-detector dedup alone: the rule is now the same at every level that folds occurrences together. Co-Authored-By: Claude Opus 5 --- dev-docs/ARCHITECTURE.md | 4 +- internal/detectors/origin.go | 29 ++++ internal/detectors/origin_test.go | 31 ++++ .../engine/consolidation/consolidation.go | 26 ++++ internal/engine/consolidation/origin_test.go | 134 ++++++++++++++++++ 5 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 internal/engine/consolidation/origin_test.go diff --git a/dev-docs/ARCHITECTURE.md b/dev-docs/ARCHITECTURE.md index c4fe45fa..9949a247 100644 --- a/dev-docs/ARCHITECTURE.md +++ b/dev-docs/ARCHITECTURE.md @@ -608,7 +608,9 @@ The invariant runs twice — when a detector records a value and again when expo Where one package appears several times in a lockfile, its occurrences must agree before anything is published. `detectors.MergeOrigin`, applied wherever a detector folds a duplicate into an existing node, treats absence as compatible (an occurrence asserting nothing leaves an origin standing; one asserting something fills a gap) and treats two different assertions as cancelling. Picking a winner would make the output depend on traversal order rather than on the lockfile, and an SBOM that omits a location is honest where one that takes a side of a contradiction is not. Detector graph builds walk their inputs in sorted order for the same reason. -Two consequences worth stating. Origin keys are filtered out of `scan`/`diff`/`explain` payloads by prefix in `output.cloneRefMetadata` — they are transport between two pipeline stages, and the SBOM is where users read them; the filter returns nil for an emptied map so `omitempty` still fires. And consolidation's first-wins node dedup, which runs later and across detectors, can still drop the origin of a duplicate occurrence, matching the existing behavior of `ResolvedURL` and detection licenses. +The same rule holds across manifests. Each subproject is resolved on its own, so a package two of them share arrives as two nodes, and the SDK's graph merge keeps whichever it meets first while discarding the rest — which would publish one subproject's answer for a package the scan saw resolved two different ways. `ConsolidateGraphs` therefore settles origin across the selected entries before they are merged, writing the verdict onto every occurrence so the surviving node carries it whichever one that turns out to be. A recorded disagreement is part of that verdict, which is why it is a metadata key rather than simply an absent value: absence would let a later fold refill it. + +One consequence worth stating: origin keys are filtered out of `scan`/`diff`/`explain` payloads by prefix in `output.cloneRefMetadata` — they are transport between two pipeline stages, and the SBOM is where users read them; the filter returns nil for an emptied map so `omitempty` still fires. ## Build Modes diff --git a/internal/detectors/origin.go b/internal/detectors/origin.go index 640b2bfa..0084d202 100644 --- a/internal/detectors/origin.go +++ b/internal/detectors/origin.go @@ -215,6 +215,35 @@ func MergeOrigin(existing, duplicate *sdk.Dependency) { } } +// ReconcileOrigins settles the origin of several nodes that describe one +// package, leaving every one of them carrying the same answer. +// +// Detectors resolve one manifest at a time, so a package used by two +// subprojects arrives as two nodes, each with its own origin. They are merged +// into a single node later, by a merge that keeps whichever it encounters +// first and discards the rest -- which would publish one subproject's answer +// for a package the scan saw resolved two different ways. Settling the +// disagreement here means the surviving node carries the reconciled verdict +// whichever one that turns out to be. +func ReconcileOrigins(occurrences []*sdk.Dependency) { + if len(occurrences) < 2 { + return + } + verdict := occurrences[0] + for _, occurrence := range occurrences[1:] { + MergeOrigin(verdict, occurrence) + } + + settled, conflicted := OriginFrom(verdict.Metadata), originConflicted(verdict) + for _, occurrence := range occurrences[1:] { + if conflicted { + markOriginConflict(occurrence) + continue + } + storeOrigin(occurrence, settled) + } +} + // originConflicted reports whether dep's occurrences already disagreed. func originConflicted(dep *sdk.Dependency) bool { if dep == nil || dep.Metadata == nil { diff --git a/internal/detectors/origin_test.go b/internal/detectors/origin_test.go index 7466e2fe..92d438cb 100644 --- a/internal/detectors/origin_test.go +++ b/internal/detectors/origin_test.go @@ -383,6 +383,37 @@ func TestMergeOrigin(t *testing.T) { } }) + // Reconciling several occurrences leaves every one of them carrying the + // verdict, including the record of a disagreement -- otherwise a later + // fold against a node that still asserts something would revive it. + t.Run("every occurrence carries the settled verdict", func(t *testing.T) { + occurrences := []*sdk.Dependency{withArtifact(artifact), withArtifact(mirror), withArtifact(artifact)} + detectors.ReconcileOrigins(occurrences) + + for i, occurrence := range occurrences { + if got := detectors.OriginFrom(occurrence.Metadata); !got.Empty() { + t.Fatalf("occurrence %d = %+v, want none", i, got) + } + // Whichever occurrence a later merge keeps must stay settled. + detectors.MergeOrigin(occurrence, withArtifact(artifact)) + if got := detectors.OriginFrom(occurrence.Metadata); !got.Empty() { + t.Fatalf("occurrence %d revived %+v after the disagreement was settled", i, got) + } + } + }) + + t.Run("agreement is broadcast to every occurrence", func(t *testing.T) { + occurrences := []*sdk.Dependency{&sdk.Dependency{ID: "react@18.2.0"}, withArtifact(artifact), &sdk.Dependency{ID: "react@18.2.0"}} + detectors.ReconcileOrigins(occurrences) + + want := detectors.Origin{ArtifactURL: artifact} + for i, occurrence := range occurrences { + if got := detectors.OriginFrom(occurrence.Metadata); got != want { + t.Fatalf("occurrence %d = %+v, want %+v", i, got, want) + } + } + }) + t.Run("nil is a no-op", func(t *testing.T) { detectors.MergeOrigin(nil, withArtifact(artifact)) detectors.MergeOrigin(withArtifact(artifact), nil) diff --git a/internal/engine/consolidation/consolidation.go b/internal/engine/consolidation/consolidation.go index 45cde7b1..2c4de133 100644 --- a/internal/engine/consolidation/consolidation.go +++ b/internal/engine/consolidation/consolidation.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) @@ -39,9 +40,34 @@ func ConsolidateGraphs(results []sdk.DetectionResult) (sdk.ConsolidatedGraph, er } consolidated.Subprojects[idx].RootManifestIDs = append(consolidated.Subprojects[idx].RootManifestIDs, selected.RootManifestID) } + reconcileEntryOrigins(consolidated.Graphs.Entries) return consolidated, nil } +// reconcileEntryOrigins settles package origin across the selected manifests +// before their graphs are merged into one. Each manifest is resolved on its +// own, so one package used by two subprojects arrives as two nodes; merging +// keeps the first and drops the rest, which would publish one subproject's +// answer for a package the scan saw resolved two different ways. +func reconcileEntryOrigins(entries []sdk.GraphEntry) { + if len(entries) < 2 { + return + } + occurrences := make(map[string][]*sdk.Dependency) + for _, entry := range entries { + if entry.Graph == nil { + continue + } + entry.Graph.WalkNodes(func(node *sdk.Dependency) bool { + occurrences[node.ID] = append(occurrences[node.ID], node) + return true + }) + } + for _, nodes := range occurrences { + detectors.ReconcileOrigins(nodes) + } +} + type consolidatedEntryCandidate struct { entry sdk.GraphEntry subproject sdk.Subproject diff --git a/internal/engine/consolidation/origin_test.go b/internal/engine/consolidation/origin_test.go new file mode 100644 index 00000000..62c5b44a --- /dev/null +++ b/internal/engine/consolidation/origin_test.go @@ -0,0 +1,134 @@ +package consolidation + +import ( + "testing" + + "github.com/bomly-dev/bomly-cli/internal/detectors" + "github.com/bomly-dev/bomly-sdk" +) + +// subprojectResult builds one manifest's detection result carrying a single +// package whose origin the caller chooses. +func subprojectResult(t *testing.T, relativePath, manifest, artifactURL string) sdk.DetectionResult { + t.Helper() + + g := sdk.New() + pkg := sdk.NewDependencyWithID("lodash@4.17.21", sdk.Dependency{Coordinates: sdk.Coordinates{ + Name: "lodash", Version: "4.17.21", Ecosystem: sdk.EcosystemNPM, PURL: "pkg:npm/lodash@4.17.21"}}) + if artifactURL != "" { + detectors.SetOriginArtifact(pkg, artifactURL) + } + if err := g.AddNode(pkg); err != nil { + t.Fatal(err) + } + return sdk.DetectionResult{ + SubprojectInfo: sdk.Subproject{ + ExecutionTarget: sdk.ExecutionTarget{Kind: sdk.ExecutionTargetWorkingDirectory, Location: "/repo"}, + RelativePath: relativePath, + PrimaryDetector: "npm-detector", + DetectedPackageManagers: []sdk.PackageManager{sdk.PackageManagerNPM}, + Ecosystem: sdk.EcosystemNPM, + }, + DetectorName: "npm-detector", + Graphs: sdk.SingleGraphContainer(g, sdk.ManifestMetadata{Path: manifest, Kind: "package-lock.json"}), + } +} + +// graphIDs lists node ids for failure messages. +func graphIDs(g *sdk.Graph) []string { + var ids []string + g.WalkNodes(func(dep *sdk.Dependency) bool { + ids = append(ids, dep.ID) + return true + }) + return ids +} + +// Each manifest is resolved on its own, so a package two subprojects share +// arrives as two nodes. Merging keeps one and drops the other, so the +// disagreement has to be settled while both are still visible. +func TestConsolidateGraphsSettlesOriginAcrossManifests(t *testing.T) { + const ( + public = "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" + private = "https://npm.corp/mirror/lodash/-/lodash-4.17.21.tgz" + ) + + cases := []struct { + name string + left string + right string + want detectors.Origin + }{ + { + name: "subprojects agree", + left: public, + right: public, + want: detectors.Origin{ArtifactURL: public}, + }, + { + name: "one subproject resolved a private mirror", + left: public, + right: private, + }, + { + name: "one subproject recorded nothing", + left: public, + right: "", + want: detectors.Origin{ArtifactURL: public}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + consolidated, err := ConsolidateGraphs([]sdk.DetectionResult{ + subprojectResult(t, "apps/web", "apps/web/package-lock.json", tc.left), + subprojectResult(t, "services/api", "services/api/package-lock.json", tc.right), + }) + if err != nil { + t.Fatalf("ConsolidateGraphs() error = %v", err) + } + + merged, err := consolidated.Graphs.ConsolidatedGraph() + if err != nil { + t.Fatalf("ConsolidatedGraph() error = %v", err) + } + var node *sdk.Dependency + merged.WalkNodes(func(dep *sdk.Dependency) bool { + if dep.Name == "lodash" { + node = dep + } + return true + }) + if node == nil { + t.Fatalf("expected lodash in the merged graph; ids present: %v", graphIDs(merged)) + } + if got := detectors.OriginFrom(node.Metadata); got != tc.want { + t.Fatalf("merged origin = %+v, want %+v", got, tc.want) + } + }) + } +} + +// The surviving node is whichever the merge happens to keep, so every +// occurrence has to carry the settled answer, not just the first. +func TestConsolidateGraphsSettlesEveryOccurrence(t *testing.T) { + consolidated, err := ConsolidateGraphs([]sdk.DetectionResult{ + subprojectResult(t, "apps/web", "apps/web/package-lock.json", "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"), + subprojectResult(t, "services/api", "services/api/package-lock.json", "https://npm.corp/mirror/lodash/-/lodash-4.17.21.tgz"), + }) + if err != nil { + t.Fatalf("ConsolidateGraphs() error = %v", err) + } + + for _, entry := range consolidated.Graphs.Entries { + if entry.Graph == nil { + continue + } + entry.Graph.WalkNodes(func(node *sdk.Dependency) bool { + if got := detectors.OriginFrom(node.Metadata); !got.Empty() { + t.Errorf("%s still claims %+v after the subprojects disagreed", node.ID, got) + } + return true + }) + } +} From 73e6e926826d9aa455a4a9ab97767c8a0fd2d91c Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 22:31:48 -0400 Subject: [PATCH 10/17] fix(cargo): cancel origin when one crate resolves from two sources Cargo can resolve one crate name and version from two sources -- the same crate pulled from two git remotes. They share a PURL, so they are one graph node, and the node kept whichever source was walked first. The walk was over a map, so the answer varied between runs of the same project: 40 runs of one fixture produced repository A 33 times and repository B 7 times. The package map is now walked in a fixed order, and cargo's node dedup reconciles origin the way the node detectors do, so disagreeing sources cancel instead of racing. Also correct what the docs say about enrichment. The Scorecard repository fills in whenever the detector reported no repository -- including for packages that already have a download location, which the text implied it skipped. An artifact and a repository answer different questions, so a package can carry both; SPDX now records the repository as source info in that case, where before it reached CycloneDX and vanished from SPDX entirely. Co-Authored-By: Claude Opus 5 --- docs/SBOM.md | 25 +++++---- internal/detectors/cargo/detector.go | 22 +++++++- internal/detectors/cargo/origin_test.go | 68 +++++++++++++++++++++++++ internal/sbom/origin_test.go | 57 +++++++++++++++++++++ internal/sbom/spdx23.go | 18 +++++++ 5 files changed, 179 insertions(+), 11 deletions(-) diff --git a/docs/SBOM.md b/docs/SBOM.md index f2471ae1..3be9e9b6 100644 --- a/docs/SBOM.md +++ b/docs/SBOM.md @@ -113,7 +113,7 @@ A detector reports one of two things, or nothing at all: | What the lockfile records | SPDX 2.3 | CycloneDX | |---|---|---| | The exact file the package was fetched from | `downloadLocation` | `distribution` external reference | -| The repository it was resolved from, and the commit when the lockfile pinned one | `downloadLocation` as `git+`, with `@` when pinned | `vcs` external reference (URL only) | +| The repository it was resolved from, and the commit when the lockfile pinned one | `downloadLocation` as `git+`, with `@` when pinned; `sourceInfo` when a download location is already taken by an artifact | `vcs` external reference (URL only) | | Neither | `NOASSERTION` | no reference | CycloneDX external references have no field for a revision, so the commit a @@ -139,14 +139,21 @@ What each ecosystem yields: With `--enrich`, a package can also get a repository it has no lockfile claim to. The OpenSSF Scorecard matcher resolves a canonical source repository from a -package's identity, and that repository fills the `vcs` reference for any -package whose own detector reported nothing — including the ecosystems listed -above as yielding nothing, and Syft-detected packages. It is a network lookup -keyed on package identity rather than a claim any manifest made, so it is -weaker evidence: a detector-asserted repository always wins, and no revision is -attached, since a Scorecard repository names a project rather than a resolved -commit. Without `--enrich`, or without the scorecard matcher selected, nothing -of this kind appears. +package's identity, and that repository is used whenever the detector reported +no repository of its own — including for the ecosystems listed above as +yielding nothing, for Syft-detected packages, and **for packages that already +have a download location**. An artifact and a repository answer different +questions (which file was fetched, where the source lives), so a package can +carry both: the artifact stays the SPDX `downloadLocation` and the CycloneDX +`distribution` reference, while the repository becomes the `vcs` reference and, +in SPDX, the package's source info. Only a detector-asserted repository +displaces it. + +It is a network lookup keyed on package identity rather than a claim any +manifest made, so it is weaker evidence, and no revision is attached: a +Scorecard repository names a project, not a resolved commit. Without +`--enrich`, or without the scorecard matcher selected, nothing of this kind +appears. Four kinds of value are never published, in any ecosystem: diff --git a/internal/detectors/cargo/detector.go b/internal/detectors/cargo/detector.go index 19c4fe69..f4402b0f 100644 --- a/internal/detectors/cargo/detector.go +++ b/internal/detectors/cargo/detector.go @@ -304,7 +304,8 @@ func metadataGraphWithMembers(raw []byte, scopeFilter sdk.Scope) (*sdk.Graph, [] return nil, nil, fmt.Errorf("add root node: %w", err) } } - for id, pkg := range packagesByID { + for _, id := range sortedPackageIDs(packagesByID) { + pkg := packagesByID[id] node := packageNode(pkg, id, workspace) if err := addNodeIfMissing(g, node); err != nil { return nil, nil, err @@ -428,8 +429,25 @@ func sortedWorkspaceMembers(workspace map[string]struct{}) []string { return values } +// sortedPackageIDs orders cargo's package map so one lockfile always builds +// the same graph: map iteration would let two records of one crate decide the +// node differently per run. +func sortedPackageIDs(packages map[string]metadataPackage) []string { + ids := make([]string, 0, len(packages)) + for id := range packages { + ids = append(ids, id) + } + sort.Strings(ids) + return ids +} + func addNodeIfMissing(g *sdk.Graph, node *sdk.Dependency) error { - if _, ok := g.Node(node.ID); ok { + if existing, ok := g.Node(node.ID); ok { + // Cargo can resolve one crate name and version from two sources -- the + // same crate pulled from two git remotes, say. They share a PURL, so + // they are one node, and the node must not claim whichever source was + // visited first. + detectors.MergeOrigin(existing, node) return nil } if err := g.AddNode(node); err != nil { diff --git a/internal/detectors/cargo/origin_test.go b/internal/detectors/cargo/origin_test.go index 56ecfb55..08941259 100644 --- a/internal/detectors/cargo/origin_test.go +++ b/internal/detectors/cargo/origin_test.go @@ -49,6 +49,74 @@ func TestSetCargoOriginBySourcePrefix(t *testing.T) { } } +// Cargo can resolve one crate name and version from two sources -- the same +// crate pulled from two git remotes. They share a PURL, so they become one +// node, and that node must not claim whichever source was walked first. +func TestCargoDuplicateCrateSourcesCancelOrigin(t *testing.T) { + metadata := []byte(`{ + "packages": [ + {"id": "demo 0.1.0 (path+file:///w)", "name": "demo", "version": "0.1.0", "source": null, "dependencies": []}, + {"id": "helper 1.0.0 (git+https://github.com/a/helper#aaaa)", "name": "helper", "version": "1.0.0", "source": "git+https://github.com/a/helper#aaaabbbbccccddddeeeeffff0000111122223333", "dependencies": []}, + {"id": "helper 1.0.0 (git+https://github.com/b/helper#bbbb)", "name": "helper", "version": "1.0.0", "source": "git+https://github.com/b/helper#bbbbccccddddeeeeffff00001111222233334444", "dependencies": []} + ], + "workspace_members": ["demo 0.1.0 (path+file:///w)"], + "resolve": {"nodes": [], "root": "demo 0.1.0 (path+file:///w)"} + }`) + + // Repeat: the packages arrive in a map, so an order-dependent answer shows + // up as a different result between runs rather than a stable wrong one. + for range 25 { + graph, err := depGraphFromMetadata(metadata) + if err != nil { + t.Fatalf("depGraphFromMetadata() error = %v", err) + } + var checked int + graph.WalkNodes(func(dep *sdk.Dependency) bool { + if dep.Name != "helper" { + return true + } + checked++ + if got := detectors.OriginFrom(dep.Metadata); !got.Empty() { + t.Fatalf("origin = %+v, want none: the crate resolved from two repositories", got) + } + return true + }) + if checked != 1 { + t.Fatalf("found %d helper nodes, want 1", checked) + } + } +} + +// Two records naming the same source still publish it. +func TestCargoDuplicateCrateSameSourceKeepsOrigin(t *testing.T) { + metadata := []byte(`{ + "packages": [ + {"id": "demo 0.1.0 (path+file:///w)", "name": "demo", "version": "0.1.0", "source": null, "dependencies": []}, + {"id": "helper 1.0.0 (git+https://github.com/a/helper#aaaa)", "name": "helper", "version": "1.0.0", "source": "git+https://github.com/a/helper#aaaabbbbccccddddeeeeffff0000111122223333", "dependencies": []}, + {"id": "helper 1.0.0 (git+https://github.com/a/helper#aaaa2)", "name": "helper", "version": "1.0.0", "source": "git+https://github.com/a/helper#aaaabbbbccccddddeeeeffff0000111122223333", "dependencies": []} + ], + "workspace_members": ["demo 0.1.0 (path+file:///w)"], + "resolve": {"nodes": [], "root": "demo 0.1.0 (path+file:///w)"} + }`) + + graph, err := depGraphFromMetadata(metadata) + if err != nil { + t.Fatalf("depGraphFromMetadata() error = %v", err) + } + want := detectors.Origin{ + VCSURL: "https://github.com/a/helper", + VCSRevision: "aaaabbbbccccddddeeeeffff0000111122223333", + } + graph.WalkNodes(func(dep *sdk.Dependency) bool { + if dep.Name == "helper" { + if got := detectors.OriginFrom(dep.Metadata); got != want { + t.Fatalf("origin = %+v, want %+v", got, want) + } + } + return true + }) +} + // The lockfile path builds nodes through the same helper. func TestCargoLockGraphCarriesOrigin(t *testing.T) { lock := []byte(` diff --git a/internal/sbom/origin_test.go b/internal/sbom/origin_test.go index 998180cf..e13d1825 100644 --- a/internal/sbom/origin_test.go +++ b/internal/sbom/origin_test.go @@ -260,6 +260,63 @@ func TestScorecardRepositoryFillsTheOriginGap(t *testing.T) { } }) + // An artifact says where the file was fetched; a repository says where the + // source lives. They are complementary, so an enriched package can carry + // both -- the repository as a vcs reference in CycloneDX, and as source + // info in SPDX, whose single download location the artifact holds. + t.Run("a repository accompanies an artifact", func(t *testing.T) { + g := sdk.New() + react := sdk.NewDependencyWithID("react@18.2.0", sdk.Dependency{Coordinates: sdk.Coordinates{ + Name: "react", Version: "18.2.0", PURL: purl, Ecosystem: "npm"}}) + detectors.SetOriginArtifact(react, "https://registry.npmjs.org/react/-/react-18.2.0.tgz") + if err := g.AddNode(react); err != nil { + t.Fatal(err) + } + registry := sdk.NewPackageRegistry() + pkg := registry.Ensure(purl) + pkg.Name, pkg.Version, pkg.Matched = "react", "18.2.0", true + pkg.Scorecard = &sdk.PackageScorecard{Source: "api.scorecard.dev", Repository: "github.com/facebook/react"} + + opts := BuildOptions{Registry: registry} + cdxRaw, err := MarshalDepGraphJSON(g, TargetCycloneDX17JSON, opts, EncodeOptions{}) + if err != nil { + t.Fatal(err) + } + spdxRaw, err := MarshalDepGraphJSON(g, TargetSPDX23JSON, opts, EncodeOptions{}) + if err != nil { + t.Fatal(err) + } + + refs := cycloneDXReferences(t, cdxRaw, "react") + if refs["distribution"] != "https://registry.npmjs.org/react/-/react-18.2.0.tgz" { + t.Errorf("distribution ref = %q, want the detector artifact", refs["distribution"]) + } + if refs["vcs"] != "https://github.com/facebook/react" { + t.Errorf("vcs ref = %q, want the scorecard repository", refs["vcs"]) + } + + spdxPkg := spdxPackageByName(t, spdxRaw, "react") + if spdxPkg["downloadLocation"] != "https://registry.npmjs.org/react/-/react-18.2.0.tgz" { + t.Errorf("downloadLocation = %v, want the artifact", spdxPkg["downloadLocation"]) + } + if got, _ := spdxPkg["sourceInfo"].(string); got != "Source repository: git+https://github.com/facebook/react" { + t.Errorf("sourceInfo = %q, want the repository", got) + } + }) + + // With no artifact, the repository is the download location, so repeating + // it as source info would say the same thing twice. + t.Run("a repository alone is not repeated as source info", func(t *testing.T) { + g := originGraph(t, func(_, pkg *sdk.Dependency) { + detectors.SetOriginVCS(pkg, "https://github.com/facebook/react", "") + }) + spdxRaw, _ := marshalBoth(t, g) + spdxPkg := spdxPackageByName(t, spdxRaw, "react") + if got, found := spdxPkg["sourceInfo"]; found && got != "" { + t.Errorf("sourceInfo = %v, want none: the repository is the download location", got) + } + }) + t.Run("absent without enrichment", func(t *testing.T) { g := originGraph(t, func(_, _ *sdk.Dependency) {}) _, cdxRaw := marshalBoth(t, g) diff --git a/internal/sbom/spdx23.go b/internal/sbom/spdx23.go index 1bdda8a4..5196d388 100644 --- a/internal/sbom/spdx23.go +++ b/internal/sbom/spdx23.go @@ -51,6 +51,7 @@ func (spdx23Codec) encodeJSON(doc *Document, opts EncodeOptions) ([]byte, error) PackageLicenseConcluded: spdxLicenseValue(c.Licenses), PackageCopyrightText: spdxCopyrightValue(c.Copyright), PackageChecksums: spdxChecksums(c.Digests), + PackageSourceInfo: spdxSourceInfo(c), PackageExternalReferences: spdxExternalReferences(c), PrimaryPackagePurpose: spdxPrimaryPackagePurpose(c.Type), } @@ -428,6 +429,10 @@ func spdxCopyrightValue(value string) string { return value } +// spdxSourceInfoPrefix labels the repository recorded in PackageSourceInfo, +// which SPDX defines as free text. +const spdxSourceInfoPrefix = "Source repository: " + // spdxDownloadLocation renders where a package came from. SPDX requires the // field, so a package whose detector asserted nothing keeps NOASSERTION rather // than a guess. @@ -457,6 +462,19 @@ func spdxVCSLocator(component Component) string { return locator } +// spdxSourceInfo records the source repository when it is not already the +// download location. A package downloaded as an artifact still has a +// repository worth naming, and SPDX has one download location per package, so +// the repository goes here rather than being dropped. +func spdxSourceInfo(component Component) string { + repository := strings.TrimSpace(component.VCSURL) + if repository == "" || strings.TrimSpace(component.ArtifactURL) == "" { + // With no artifact, the repository is the download location already. + return "" + } + return spdxSourceInfoPrefix + spdxVCSLocator(component) +} + func spdxExternalReferences(component Component) []*v23.PackageExternalReference { refs := make([]*v23.PackageExternalReference, 0, 1+len(component.CPEs)+len(component.Vulnerabilities)) if purl := strings.TrimSpace(component.PURL); purl != "" { From 2bf2745b21251cd2f110e6da6cc650583f3e9fe3 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 22:40:32 -0400 Subject: [PATCH 11/17] fix(detectors): nil-safe reconciliation and best-effort logging Three review points, all cheap and all real: - ReconcileOrigins is exported, so a caller can hand it a slice containing nil. It would have dereferenced through clearOrigin and panicked. Nil occurrences are now skipped, and a nil first element returns rather than reading its metadata. - The two committed-file joins take a logger and log at debug when a file will not parse, which is exactly where a nil logger bites. Both now fall back to zap.NewNop(), which is the convention this repo states. - Two tests walked a graph asserting on a named node without requiring it to be there, so they would have passed if construction dropped it. Both now count what they checked -- verified by removing the node and watching them fail. Co-Authored-By: Claude Opus 5 --- internal/detectors/cargo/origin_test.go | 5 ++++ internal/detectors/origin.go | 6 +++++ internal/detectors/origin_test.go | 13 ++++++++++ internal/detectors/pub/origin_test.go | 26 ++++++++++++++++++++ internal/detectors/pub/pub_native.go | 3 +++ internal/detectors/swiftpm/origin_test.go | 18 ++++++++++++++ internal/detectors/swiftpm/swiftpm_native.go | 3 +++ 7 files changed, 74 insertions(+) diff --git a/internal/detectors/cargo/origin_test.go b/internal/detectors/cargo/origin_test.go index 08941259..4856c47e 100644 --- a/internal/detectors/cargo/origin_test.go +++ b/internal/detectors/cargo/origin_test.go @@ -107,14 +107,19 @@ func TestCargoDuplicateCrateSameSourceKeepsOrigin(t *testing.T) { VCSURL: "https://github.com/a/helper", VCSRevision: "aaaabbbbccccddddeeeeffff0000111122223333", } + var checked int graph.WalkNodes(func(dep *sdk.Dependency) bool { if dep.Name == "helper" { + checked++ if got := detectors.OriginFrom(dep.Metadata); got != want { t.Fatalf("origin = %+v, want %+v", got, want) } } return true }) + if checked != 1 { + t.Fatalf("found %d helper nodes, want 1", checked) + } } // The lockfile path builds nodes through the same helper. diff --git a/internal/detectors/origin.go b/internal/detectors/origin.go index 0084d202..7af5cb11 100644 --- a/internal/detectors/origin.go +++ b/internal/detectors/origin.go @@ -230,12 +230,18 @@ func ReconcileOrigins(occurrences []*sdk.Dependency) { return } verdict := occurrences[0] + if verdict == nil { + return + } for _, occurrence := range occurrences[1:] { MergeOrigin(verdict, occurrence) } settled, conflicted := OriginFrom(verdict.Metadata), originConflicted(verdict) for _, occurrence := range occurrences[1:] { + if occurrence == nil { + continue + } if conflicted { markOriginConflict(occurrence) continue diff --git a/internal/detectors/origin_test.go b/internal/detectors/origin_test.go index 92d438cb..94010d5f 100644 --- a/internal/detectors/origin_test.go +++ b/internal/detectors/origin_test.go @@ -414,6 +414,19 @@ func TestMergeOrigin(t *testing.T) { } }) + // Exported, so a caller can hand it anything. + t.Run("nil occurrences are skipped", func(t *testing.T) { + detectors.ReconcileOrigins([]*sdk.Dependency{nil, withArtifact(artifact)}) + detectors.ReconcileOrigins([]*sdk.Dependency{withArtifact(artifact), nil}) + detectors.ReconcileOrigins([]*sdk.Dependency{nil, nil}) + + occurrences := []*sdk.Dependency{withArtifact(artifact), nil, withArtifact(mirror)} + detectors.ReconcileOrigins(occurrences) + if got := detectors.OriginFrom(occurrences[0].Metadata); !got.Empty() { + t.Fatalf("origin = %+v, want none: the non-nil occurrences disagreed", got) + } + }) + t.Run("nil is a no-op", func(t *testing.T) { detectors.MergeOrigin(nil, withArtifact(artifact)) detectors.MergeOrigin(withArtifact(artifact), nil) diff --git a/internal/detectors/pub/origin_test.go b/internal/detectors/pub/origin_test.go index 311a1d67..f285cdf4 100644 --- a/internal/detectors/pub/origin_test.go +++ b/internal/detectors/pub/origin_test.go @@ -226,10 +226,36 @@ func TestPubNativeOriginSurvivesMissingLock(t *testing.T) { t.Fatalf("nativeGraph() error = %v", err) } + var checked int g.WalkNodes(func(dep *sdk.Dependency) bool { + if dep.Name == "helper" { + checked++ + } if got := detectors.OriginFrom(dep.Metadata); !got.Empty() { t.Fatalf("%s origin = %+v, want none", dep.Name, got) } return true }) + if checked != 1 { + t.Fatalf("found %d helper nodes, want 1", checked) + } +} + +// Loggers may be nil; a best-effort join must not be the thing that panics. +func TestPubNativeOriginToleratesNilLogger(t *testing.T) { + workingDir := t.TempDir() + depsJSON := []byte(`{"root":"demo","packages":[{"name":"demo","version":"1.0.0","kind":"root","source":"root","dependencies":[]}]}`) + if err := os.WriteFile(filepath.Join(workingDir, "pubspec.lock"), []byte("packages:\n helper:\n source: git\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := nativeGraph(depsJSON, workingDir, nil); err != nil { + t.Fatalf("nativeGraph() error = %v", err) + } + // An unparseable lock reaches the debug log, which is where a nil logger bites. + if err := os.WriteFile(filepath.Join(workingDir, "pubspec.lock"), []byte("packages: [oops"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := nativeGraph(depsJSON, workingDir, nil); err != nil { + t.Fatalf("nativeGraph() error = %v", err) + } } diff --git a/internal/detectors/pub/pub_native.go b/internal/detectors/pub/pub_native.go index a7447a4c..42fb0b8a 100644 --- a/internal/detectors/pub/pub_native.go +++ b/internal/detectors/pub/pub_native.go @@ -107,6 +107,9 @@ func nativeGraph(raw []byte, workingDir string, logger *zap.Logger) (*sdk.Graph, // // Best effort: a project with no readable pubspec.lock keeps the graph as it is. func applyLockOrigins(g *sdk.Graph, workingDir string, logger *zap.Logger) { + if logger == nil { + logger = zap.NewNop() + } raw, err := system.ReadRepositoryFile(filepath.Join(workingDir, "pubspec.lock")) if err != nil { return diff --git a/internal/detectors/swiftpm/origin_test.go b/internal/detectors/swiftpm/origin_test.go index 62782b94..1db9e76a 100644 --- a/internal/detectors/swiftpm/origin_test.go +++ b/internal/detectors/swiftpm/origin_test.go @@ -219,3 +219,21 @@ func TestSwiftPMNativeOriginSurvivesMissingPackageResolved(t *testing.T) { t.Fatal("expected the remote package in the graph") } } + +// Loggers may be nil; a best-effort join must not be the thing that panics. +func TestSwiftPMNativeOriginToleratesNilLogger(t *testing.T) { + workingDir := t.TempDir() + if err := os.WriteFile(filepath.Join(workingDir, "Package.resolved"), []byte(`{"pins":[],"version":2}`), 0o644); err != nil { + t.Fatal(err) + } + if _, err := nativeGraph([]byte(`{"name":"demo","url":"/w","version":"unspecified","dependencies":[]}`), workingDir, nil); err != nil { + t.Fatalf("nativeGraph() error = %v", err) + } + // An unparseable file reaches the debug log, which is where a nil logger bites. + if err := os.WriteFile(filepath.Join(workingDir, "Package.resolved"), []byte("{not json"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := nativeGraph([]byte(`{"name":"demo","url":"/w","version":"unspecified","dependencies":[]}`), workingDir, nil); err != nil { + t.Fatalf("nativeGraph() error = %v", err) + } +} diff --git a/internal/detectors/swiftpm/swiftpm_native.go b/internal/detectors/swiftpm/swiftpm_native.go index e389cffa..be0e5ef3 100644 --- a/internal/detectors/swiftpm/swiftpm_native.go +++ b/internal/detectors/swiftpm/swiftpm_native.go @@ -106,6 +106,9 @@ func nativeGraph(raw []byte, workingDir string, logger *zap.Logger) (*sdk.Graph, // Best effort: a project with no readable Package.resolved keeps the origins // the graph already carries. func applyResolvedOrigins(g *sdk.Graph, workingDir string, logger *zap.Logger) { + if logger == nil { + logger = zap.NewNop() + } raw, path, err := readFirstExisting(workingDir, []string{"Package.resolved", ".package.resolved", "project.xcworkspace/xcshareddata/swiftpm/Package.resolved"}) if err != nil || len(raw) == 0 { return From 744c367176e6158ed4708abc6ecc933c5eee6ce3 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 22:50:38 -0400 Subject: [PATCH 12/17] fix(detectors): treat host casing as the same location Hosts are case-insensitive, so two lockfiles writing one host differently name the same place. Comparing the URLs as strings made reconciliation read a disagreement and drop a perfectly good origin over formatting alone -- the merge rules added here turned a cosmetic difference into lost data. The host is now lowercased when a URL is normalized. The path is deliberately left alone, and a test asserts that two paths differing only in case still reconcile to a disagreement, so the fix does not over-reach. Found by review on the SDK port (bomly-dev/bomly-sdk#1), where the same rule lives; fixed in both. Co-Authored-By: Claude Opus 5 --- internal/detectors/origin.go | 5 +++++ internal/detectors/origin_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/internal/detectors/origin.go b/internal/detectors/origin.go index 7af5cb11..deba85b9 100644 --- a/internal/detectors/origin.go +++ b/internal/detectors/origin.go @@ -94,6 +94,11 @@ func NormalizeOriginURL(raw string, vcs bool) (string, bool) { return "", false } parsed.Scheme = strings.ToLower(parsed.Scheme) + // Hosts are case-insensitive, so two lockfiles writing one host + // differently name the same location. Without this they compare unequal + // and reconcile to a disagreement, losing an origin to formatting alone. + // The path is left alone: it is case-sensitive. + parsed.Host = strings.ToLower(parsed.Host) parsed.Fragment = "" parsed.RawFragment = "" // A host root names a server, not a package: it is a registry or index diff --git a/internal/detectors/origin_test.go b/internal/detectors/origin_test.go index 94010d5f..aa58132c 100644 --- a/internal/detectors/origin_test.go +++ b/internal/detectors/origin_test.go @@ -186,6 +186,34 @@ func TestSetOriginAllocatesMetadataAndPreservesOtherKeys(t *testing.T) { } } +// Hosts are case-insensitive. Two lockfiles spelling one host differently name +// the same place and must not reconcile to a disagreement. +func TestOriginHostCaseIsCanonical(t *testing.T) { + upper := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginVCS(upper, "https://GitHub.com/Owner/Repo", "aaaabbbbccccddddeeeeffff0000111122223333") + lower := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginVCS(lower, "https://github.com/Owner/Repo", "aaaabbbbccccddddeeeeffff0000111122223333") + + if got := detectors.OriginFrom(upper.Metadata).VCSURL; got != "https://github.com/Owner/Repo" { + t.Fatalf("repository = %q, want a lowercased host and an untouched path", got) + } + + detectors.MergeOrigin(upper, lower) + if got := detectors.OriginFrom(upper.Metadata); got.Empty() { + t.Fatal("host casing alone must not read as a disagreement") + } + + // The path is case-sensitive, so these really are different locations. + left := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginArtifact(left, "https://example.test/Pkg-1.0.0.tgz") + right := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginArtifact(right, "https://example.test/pkg-1.0.0.tgz") + detectors.MergeOrigin(left, right) + if got := detectors.OriginFrom(left.Metadata); !got.Empty() { + t.Fatalf("origin = %+v, want a disagreement: the paths differ", got) + } +} + func TestSetOriginNilDependency(t *testing.T) { // Must not panic: detectors call these on nodes that may not exist. detectors.SetOriginArtifact(nil, "https://registry.npmjs.org/react/-/react-18.2.0.tgz") From 77b55f5afddf569b7191fbe85969d8dc563b8ed9 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 22:59:26 -0400 Subject: [PATCH 13/17] fix(detectors): treat a default port as the same location "https://host:443/pkg" and "https://host/pkg" name one place, but comparing the URLs as strings made reconciliation read a disagreement and drop a good origin over formatting -- the same class as the host-casing fix, found by review on the SDK port (bomly-dev/bomly-sdk#1) and fixed in both. IPv6 literals keep their brackets, and a non-default port stays part of the location. Co-Authored-By: Claude Opus 5 --- internal/detectors/origin.go | 10 ++++++++++ internal/detectors/origin_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/internal/detectors/origin.go b/internal/detectors/origin.go index deba85b9..e12949c9 100644 --- a/internal/detectors/origin.go +++ b/internal/detectors/origin.go @@ -99,6 +99,16 @@ func NormalizeOriginURL(raw string, vcs bool) (string, bool) { // and reconcile to a disagreement, losing an origin to formatting alone. // The path is left alone: it is case-sensitive. parsed.Host = strings.ToLower(parsed.Host) + // An explicit default port names the same origin as no port at all, so + // dropping it keeps two spellings of one location from reading as a + // disagreement. + if port := parsed.Port(); (parsed.Scheme == "https" && port == "443") || (parsed.Scheme == "http" && port == "80") { + host := parsed.Hostname() + if strings.Contains(host, ":") { + host = "[" + host + "]" // an IPv6 literal keeps its brackets + } + parsed.Host = host + } parsed.Fragment = "" parsed.RawFragment = "" // A host root names a server, not a package: it is a registry or index diff --git a/internal/detectors/origin_test.go b/internal/detectors/origin_test.go index aa58132c..b2801d44 100644 --- a/internal/detectors/origin_test.go +++ b/internal/detectors/origin_test.go @@ -214,6 +214,34 @@ func TestOriginHostCaseIsCanonical(t *testing.T) { } } +// An explicit default port names the same origin as no port at all. +func TestOriginDefaultPortIsCanonical(t *testing.T) { + cases := []struct{ name, raw, want string }{ + {name: "https default port", raw: "https://example.test:443/pkg-1.0.0.tgz", want: "https://example.test/pkg-1.0.0.tgz"}, + {name: "http default port", raw: "http://example.test:80/pkg-1.0.0.tgz", want: "http://example.test/pkg-1.0.0.tgz"}, + {name: "a non-default port is part of the location", raw: "https://example.test:8443/pkg-1.0.0.tgz", want: "https://example.test:8443/pkg-1.0.0.tgz"}, + {name: "an IPv6 literal keeps its brackets", raw: "https://[2001:db8::1]:443/pkg-1.0.0.tgz", want: "https://[2001:db8::1]/pkg-1.0.0.tgz"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dep := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginArtifact(dep, tc.raw) + if got := detectors.OriginFrom(dep.Metadata).ArtifactURL; got != tc.want { + t.Fatalf("artifact = %q, want %q", got, tc.want) + } + }) + } + + withPort := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginArtifact(withPort, "https://example.test:443/pkg-1.0.0.tgz") + without := &sdk.Dependency{ID: "pkg"} + detectors.SetOriginArtifact(without, "https://example.test/pkg-1.0.0.tgz") + detectors.MergeOrigin(withPort, without) + if detectors.OriginFrom(withPort.Metadata).Empty() { + t.Fatal("a default port alone must not read as a disagreement") + } +} + func TestSetOriginNilDependency(t *testing.T) { // Must not panic: detectors call these on nodes that may not exist. detectors.SetOriginArtifact(nil, "https://registry.npmjs.org/react/-/react-18.2.0.tgz") From 47b63ce0d31c5ffcae55577522e171b35b31d179 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 23:27:57 -0400 Subject: [PATCH 14/17] fix(detectors): reject ports nothing can connect to url.Parse only checks that a port is numeric, so "https://host:99999/pkg" was accepted and would have been published as a location no client can reach. Ports outside 1-65535 are now rejected. Found by review on the SDK port (bomly-dev/bomly-sdk#1); fixed in both. Co-Authored-By: Claude Opus 5 --- internal/detectors/origin.go | 9 +++++++++ internal/detectors/origin_test.go | 3 +++ 2 files changed, 12 insertions(+) diff --git a/internal/detectors/origin.go b/internal/detectors/origin.go index e12949c9..b34493b8 100644 --- a/internal/detectors/origin.go +++ b/internal/detectors/origin.go @@ -2,6 +2,7 @@ package detectors import ( "net/url" + "strconv" "strings" "github.com/bomly-dev/bomly-sdk" @@ -102,6 +103,14 @@ func NormalizeOriginURL(raw string, vcs bool) (string, bool) { // An explicit default port names the same origin as no port at all, so // dropping it keeps two spellings of one location from reading as a // disagreement. + if port := parsed.Port(); port != "" { + // url.Parse only checks that a port is numeric, so a value no client + // could connect to still reaches here. + number, err := strconv.Atoi(port) + if err != nil || number < 1 || number > 65535 { + return "", false + } + } if port := parsed.Port(); (parsed.Scheme == "https" && port == "443") || (parsed.Scheme == "http" && port == "80") { host := parsed.Hostname() if strings.Contains(host, ":") { diff --git a/internal/detectors/origin_test.go b/internal/detectors/origin_test.go index b2801d44..c6f60d35 100644 --- a/internal/detectors/origin_test.go +++ b/internal/detectors/origin_test.go @@ -220,6 +220,9 @@ func TestOriginDefaultPortIsCanonical(t *testing.T) { {name: "https default port", raw: "https://example.test:443/pkg-1.0.0.tgz", want: "https://example.test/pkg-1.0.0.tgz"}, {name: "http default port", raw: "http://example.test:80/pkg-1.0.0.tgz", want: "http://example.test/pkg-1.0.0.tgz"}, {name: "a non-default port is part of the location", raw: "https://example.test:8443/pkg-1.0.0.tgz", want: "https://example.test:8443/pkg-1.0.0.tgz"}, + {name: "the highest usable port", raw: "https://example.test:65535/pkg-1.0.0.tgz", want: "https://example.test:65535/pkg-1.0.0.tgz"}, + {name: "a port nothing can connect to", raw: "https://example.test:99999/pkg-1.0.0.tgz", want: ""}, + {name: "port zero", raw: "https://example.test:0/pkg-1.0.0.tgz", want: ""}, {name: "an IPv6 literal keeps its brackets", raw: "https://[2001:db8::1]:443/pkg-1.0.0.tgz", want: "https://[2001:db8::1]/pkg-1.0.0.tgz"}, } for _, tc := range cases { From 721103435af1362eae25e81521fc6fb5ff2dfac9 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 23:32:00 -0400 Subject: [PATCH 15/17] fix: settle origin when two nodes collapse to one identity Two findings, both producing false provenance: - A single manifest can record one package twice with different locations -- a Bun lockfile listing one name and version from two mirrors. Identity normalization collapses both nodes onto one canonical PURL and keeps the first, so the SBOM published one mirror as authoritative. Cross-manifest reconciliation could not help: it runs later, and returns early for a single-manifest scan. The collapse now reconciles origin while both occurrences are still there. - SwiftPM repository matching lowercased the whole URL, so on a case-sensitive host "/Team/Helper" and "/team/helper" shared a lookup key and a package could take the pin belonging to a different repository. Only the scheme and host are case-insensitive; the path keeps its case. Co-Authored-By: Claude Opus 5 --- internal/detectors/swiftpm/origin_test.go | 55 ++++++++++++++ internal/detectors/swiftpm/swiftpm_native.go | 18 ++++- internal/engine/consolidation/enrichment.go | 9 ++- internal/engine/consolidation/origin_test.go | 79 ++++++++++++++++++++ 4 files changed, 158 insertions(+), 3 deletions(-) diff --git a/internal/detectors/swiftpm/origin_test.go b/internal/detectors/swiftpm/origin_test.go index 1db9e76a..cb560c94 100644 --- a/internal/detectors/swiftpm/origin_test.go +++ b/internal/detectors/swiftpm/origin_test.go @@ -237,3 +237,58 @@ func TestSwiftPMNativeOriginToleratesNilLogger(t *testing.T) { t.Fatalf("nativeGraph() error = %v", err) } } + +// Repository paths are case-sensitive on a self-hosted host, so two packages +// differing only in path case are different repositories. Folding their keys +// together would attach one repository's pin to the other's package. +func TestSwiftPMRepositoryKeyPreservesPathCase(t *testing.T) { + if repositoryKey("https://git.corp/Team/Helper.git") == repositoryKey("https://git.corp/team/helper.git") { + t.Fatal("two repositories differing only in path case share a lookup key") + } + // Scheme and host are case-insensitive, so those spellings are one key. + if repositoryKey("HTTPS://Git.Corp/Team/Helper.git") != repositoryKey("https://git.corp/Team/Helper.git") { + t.Fatal("host casing should not change the key") + } + // The suffix and trailing slash still normalize away. + if repositoryKey("https://git.corp/Team/Helper/") != repositoryKey("https://git.corp/Team/Helper.git") { + t.Fatal("a trailing slash or .git suffix should not change the key") + } +} + +// A pin for a differently-cased path must not be attached to this package. +func TestSwiftPMNativeOriginDoesNotMatchAcrossPathCase(t *testing.T) { + workingDir := t.TempDir() + resolved := `{ + "pins": [ + { + "identity": "other-helper", + "kind": "remoteSourceControl", + "location": "https://git.corp/team/helper.git", + "state": {"revision": "5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f7081", "version": "1.0.0"} + } + ], + "version": 2 + }` + if err := os.WriteFile(filepath.Join(workingDir, "Package.resolved"), []byte(resolved), 0o644); err != nil { + t.Fatal(err) + } + showDependencies := []byte(`{ + "name": "demo", + "url": "/workspace/demo", + "version": "unspecified", + "dependencies": [ + {"name": "Helper", "url": "https://git.corp/Team/Helper.git", "version": "2.0.0", "dependencies": []} + ] + }`) + + g, err := nativeGraph(showDependencies, workingDir, zap.NewNop()) + if err != nil { + t.Fatalf("nativeGraph() error = %v", err) + } + g.WalkNodes(func(dep *sdk.Dependency) bool { + if origin := detectors.OriginFrom(dep.Metadata); origin.VCSRevision == "5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f7081" { + t.Fatalf("%s took a pin belonging to a differently-cased repository: %+v", dep.Name, origin) + } + return true + }) +} diff --git a/internal/detectors/swiftpm/swiftpm_native.go b/internal/detectors/swiftpm/swiftpm_native.go index be0e5ef3..cd7eb023 100644 --- a/internal/detectors/swiftpm/swiftpm_native.go +++ b/internal/detectors/swiftpm/swiftpm_native.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "net/url" "strings" "time" @@ -158,8 +159,21 @@ func applyResolvedOrigins(g *sdk.Graph, workingDir string, logger *zap.Logger) { // node: SwiftPM reports the same repository with and without a ".git" suffix // and in either case. func repositoryKey(repository string) string { - key := strings.TrimSuffix(strings.TrimSuffix(strings.ToLower(strings.TrimSpace(repository)), "/"), ".git") - return key + trimmed := strings.TrimSpace(repository) + if trimmed == "" { + return "" + } + parsed, err := url.Parse(trimmed) + if err != nil || parsed.Host == "" { + return strings.TrimSuffix(strings.TrimSuffix(trimmed, "/"), ".git") + } + parsed.Scheme = strings.ToLower(parsed.Scheme) + parsed.Host = strings.ToLower(parsed.Host) + // The path keeps its case: on a case-sensitive host "/Team/Helper" and + // "/team/helper" are different repositories, and folding them together + // would attach one repository's pin to the other's package. + parsed.Path = strings.TrimSuffix(strings.TrimSuffix(parsed.Path, "/"), ".git") + return parsed.String() } // FallbackDetector returns the configured fallback detector. diff --git a/internal/engine/consolidation/enrichment.go b/internal/engine/consolidation/enrichment.go index fa572b4d..6b917dea 100644 --- a/internal/engine/consolidation/enrichment.go +++ b/internal/engine/consolidation/enrichment.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) @@ -33,10 +34,16 @@ func normalizeGraphPackageIdentity(src *sdk.Graph) (*sdk.Graph, error) { if clone.ID == "" { return nil, fmt.Errorf("dependency %q has no canonical identity", node.QualifiedName()) } - if _, exists := normalized.Node(clone.ID); !exists { + if existing, exists := normalized.Node(clone.ID); !exists { if err := normalized.AddNode(clone); err != nil { return nil, fmt.Errorf("add normalized dependency %q: %w", clone.ID, err) } + } else { + // Two nodes normalizing to one identity are one package. Only one + // survives, so the discarded occurrence still gets a say about + // where the package came from -- otherwise a lockfile recording + // one package from two mirrors publishes whichever came first. + detectors.MergeOrigin(existing, clone) } idMapping[node.ID] = clone.ID } diff --git a/internal/engine/consolidation/origin_test.go b/internal/engine/consolidation/origin_test.go index 62c5b44a..82671c52 100644 --- a/internal/engine/consolidation/origin_test.go +++ b/internal/engine/consolidation/origin_test.go @@ -1,6 +1,7 @@ package consolidation import ( + "fmt" "testing" "github.com/bomly-dev/bomly-cli/internal/detectors" @@ -132,3 +133,81 @@ func TestConsolidateGraphsSettlesEveryOccurrence(t *testing.T) { }) } } + +// One manifest can record a package twice with different locations -- a Bun +// lockfile listing one name and version from two mirrors. Both nodes normalize +// to one canonical identity and only one survives, so the disagreement has to +// be settled while both are still there. This is the single-manifest case, +// which never reaches cross-entry reconciliation. +func TestConsolidateGraphsSettlesOriginWithinOneManifest(t *testing.T) { + const ( + public = "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" + private = "https://npm.corp/mirror/lodash/-/lodash-4.17.21.tgz" + ) + + cases := []struct { + name string + left string + right string + want detectors.Origin + }{ + {name: "entries agree", left: public, right: public, want: detectors.Origin{ArtifactURL: public}}, + {name: "entries disagree", left: public, right: private}, + {name: "one entry says nothing", left: public, right: "", want: detectors.Origin{ArtifactURL: public}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Distinct node IDs, one canonical PURL: what a lockfile parser + // produces when it disambiguates a duplicate package key. + g := sdk.New() + for i, artifactURL := range []string{tc.left, tc.right} { + pkg := sdk.NewDependencyWithID( + fmt.Sprintf("bun-package:lodash@4.17.21#%d", i), + sdk.Dependency{Coordinates: sdk.Coordinates{ + Name: "lodash", Version: "4.17.21", Ecosystem: sdk.EcosystemNPM, PURL: "pkg:npm/lodash@4.17.21"}}, + ) + if artifactURL != "" { + detectors.SetOriginArtifact(pkg, artifactURL) + } + if err := g.AddNode(pkg); err != nil { + t.Fatal(err) + } + } + + consolidated, err := ConsolidateGraphs([]sdk.DetectionResult{{ + SubprojectInfo: sdk.Subproject{ + ExecutionTarget: sdk.ExecutionTarget{Kind: sdk.ExecutionTargetWorkingDirectory, Location: "/repo"}, + RelativePath: ".", + PrimaryDetector: "bun-detector", + DetectedPackageManagers: []sdk.PackageManager{sdk.PackageManagerBun}, + Ecosystem: sdk.EcosystemNPM, + }, + DetectorName: "bun-detector", + Graphs: sdk.SingleGraphContainer(g, sdk.ManifestMetadata{Path: "bun.lock", Kind: "bun.lock"}), + }}) + if err != nil { + t.Fatalf("ConsolidateGraphs() error = %v", err) + } + + merged, err := consolidated.Graphs.ConsolidatedGraph() + if err != nil { + t.Fatalf("ConsolidatedGraph() error = %v", err) + } + var checked int + merged.WalkNodes(func(dep *sdk.Dependency) bool { + if dep.Name != "lodash" { + return true + } + checked++ + if got := detectors.OriginFrom(dep.Metadata); got != tc.want { + t.Fatalf("origin = %+v, want %+v", got, tc.want) + } + return true + }) + if checked != 1 { + t.Fatalf("found %d lodash nodes, want 1", checked) + } + }) + } +} From 78d2b314099a145f9d5721b9187769f22297b866 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 23:48:33 -0400 Subject: [PATCH 16/17] refactor: adopt the SDK's package origin (bomly-sdk v0.4.0) Origin began as CLI-internal metadata keys because promoting it needed an SDK release. bomly-sdk v0.4.0 has it as a typed field, so the CLI now uses that and deletes its own copy. - Detectors assign `dep.Origin` from `sdk.ArtifactOrigin` / `sdk.RepositoryOrigin` instead of calling internal setters. External plugins can now do the same, which was the point: the rule they need was unreachable inside `internal/`. - Export reads `pkg.Origin.Normalized()`, which applies the same validation the CLI used to perform on read. - `internal/detectors/origin.go` and its tests and fuzz target are gone; the SDK carries the rule and its own fuzzing, and it validates at the JSON boundary too, which the CLI's version never did. - The origin filter in `output.cloneRefMetadata` is gone. It existed to keep metadata keys out of command payloads; a typed field never reached them, because those documents are built from explicit projections. - Cross-manifest reconciliation in `ConsolidateGraphs` is gone: the SDK's graph merge now reconciles. The tests that covered the behavior stay and pass unchanged, which is what makes the deletion safe. Reconciliation the SDK does not own -- node and cargo node dedup, and identity collapse in `normalizeGraphPackageIdentity` -- stays here, now calling `sdk.ReconcileOrigin`. Generated schemas pick up `Digest.Subject` from the SDK; it is omitempty, so scan output is unchanged for every package that does not set it. Co-Authored-By: Claude Opus 5 --- dev-docs/ARCHITECTURE.md | 10 +- docs/schemas/diff.md | 1 + docs/schemas/diff.schema.json | 3 + docs/schemas/scan.md | 1 + docs/schemas/scan.schema.json | 3 + go.mod | 2 +- go.sum | 2 + internal/detectors/cargo/detector.go | 2 +- internal/detectors/cargo/origin.go | 3 +- internal/detectors/cargo/origin_test.go | 39 +- .../detectors/node/bun/bun_lockfile_parser.go | 3 +- internal/detectors/node/common.go | 5 +- .../detectors/node/npm/npm_lockfile_parser.go | 3 +- internal/detectors/node/npm/origin_test.go | 25 +- .../detectors/node/origin_integration_test.go | 21 +- .../node/pnpm/pnpm_lockfile_parser.go | 3 +- .../node/yarn/yarn_lockfile_parser.go | 3 +- internal/detectors/origin.go | 360 ------------- internal/detectors/origin_fuzz_test.go | 112 ---- internal/detectors/origin_test.go | 493 ------------------ internal/detectors/pub/detector.go | 2 +- internal/detectors/pub/origin_test.go | 35 +- internal/detectors/pub/pub_native.go | 2 +- internal/detectors/python/origin.go | 17 +- internal/detectors/python/origin_test.go | 65 ++- internal/detectors/ruby/detector.go | 2 +- internal/detectors/ruby/origin_test.go | 25 +- internal/detectors/swiftpm/detector.go | 2 +- internal/detectors/swiftpm/origin_test.go | 37 +- internal/detectors/swiftpm/swiftpm_native.go | 2 +- .../engine/consolidation/consolidation.go | 26 - internal/engine/consolidation/enrichment.go | 3 +- internal/engine/consolidation/origin_test.go | 57 +- internal/output/origin_metadata_test.go | 79 --- internal/output/types.go | 13 +- internal/sbom/origin_test.go | 51 +- internal/sbom/transform.go | 19 +- scripts/run-fuzz.sh | 1 - 38 files changed, 256 insertions(+), 1276 deletions(-) delete mode 100644 internal/detectors/origin.go delete mode 100644 internal/detectors/origin_fuzz_test.go delete mode 100644 internal/detectors/origin_test.go delete mode 100644 internal/output/origin_metadata_test.go diff --git a/dev-docs/ARCHITECTURE.md b/dev-docs/ARCHITECTURE.md index 9949a247..f1923d32 100644 --- a/dev-docs/ARCHITECTURE.md +++ b/dev-docs/ARCHITECTURE.md @@ -600,17 +600,17 @@ An SBOM should say where each package came from — SPDX `downloadLocation`, Cyc `ResolvedURL` is not one kind of value. npm writes a registry tarball there, but also a local directory for link entries and a git remote for git specs. uv writes a repository, an archive, an index root, or an editable path, depending on the source stanza. Cargo writes a prefixed source string, Bundler writes the section's `remote:` (a gem server, a repository, or a directory), pub writes the pub server for hosted packages and a repository for git ones. Recovering the meaning downstream means guessing from the string, and every guess has an ecosystem-specific counterexample: an archive-extension check misclassifies real repositories whose names end in `.zip` or `.conda`; a fragment is a resolved commit in uv and cargo but a content checksum in Yarn Classic; a private registry root with a path is indistinguishable from a repository; and distinguishing an opaque token from a content hash is not decidable at all, because they have the same shape. Roughly twenty review rounds of layered special cases did not converge. -Origin is therefore asserted where the meaning is known. Each detector reads its own lockfile's structured source fields and records at most one of an artifact URL or a repository URL plus resolved revision, on `Dependency.Metadata` under `bomly.origin.*` keys — the same well-known-key transport `bomly.detection.licenses` uses, so no SDK change was needed. Registry and index roots are deliberately not representable: they describe an ecosystem's fetch configuration, not a package's provenance. +Origin is therefore asserted where the meaning is known. Each detector reads its own lockfile's structured source fields and records at most one of an artifact URL or a repository URL plus resolved revision, on `sdk.Dependency.Origin` — a typed field on the shared contract, so an external plugin in its own module can assert origin exactly as a built-in detector does. (It began as CLI-internal metadata keys, which no plugin could reach; `bomly-sdk` v0.4.0 promoted it.) Registry and index roots are deliberately not representable: they describe an ecosystem's fetch configuration, not a package's provenance. -One rule governs every published value, `detectors.NormalizeOriginURL`: absolute `http`/`https`, non-empty host, no userinfo, output re-serialized from the parse rather than copied from input. The repository form additionally strips query and fragment (they carry the *requested* ref; the *resolved* one arrives separately from the detector's own field) and requires a non-empty path, because SPDX's `git+@` grammar has no query component and an empty path would make the `@` suffix re-parse as userinfo. This one function replaces the entire classifier: no archive-extension table, no credential-prefix list, no secret-shape heuristic. Local paths, `file:`, and ssh-style remotes fail the scheme or host check rather than a bespoke rule, and a credentialed URL fails the userinfo check. +One rule governs every published value, `sdk.NormalizeOriginURL`: absolute `http`/`https`, non-empty host, non-empty path, a usable port, no userinfo, host case and default ports canonicalized, output re-serialized from the parse rather than copied from input. The repository form additionally strips query and fragment (they carry the *requested* ref; the *resolved* one arrives separately from the detector's own field) and requires a non-empty path, because SPDX's `git+@` grammar has no query component and an empty path would make the `@` suffix re-parse as userinfo. This one function replaces the entire classifier: no archive-extension table, no credential-prefix list, no secret-shape heuristic. Local paths, `file:`, and ssh-style remotes fail the scheme or host check rather than a bespoke rule, and a credentialed URL fails the userinfo check. -The invariant runs twice — when a detector records a value and again when export reads it. The second pass is not redundant: graphs also arrive from plugins and from hand-built callers, and export must not publish a location no built-in detector could have produced. Composition into a format's locator grammar stays in the encoders, so `Component.VCSURL` remains a plain URL and only SPDX builds the `git+…@…` form; CycloneDX external references have no revision slot, so a resolved commit survives an SPDX round trip and not a CycloneDX one. +The invariant runs when a detector records a value, again at the JSON boundary in both directions, and again when export reads it through `Origin.Normalized()`. The second pass is not redundant: graphs also arrive from plugins and from hand-built callers, and export must not publish a location no built-in detector could have produced. Composition into a format's locator grammar stays in the encoders, so `Component.VCSURL` remains a plain URL and only SPDX builds the `git+…@…` form; CycloneDX external references have no revision slot, so a resolved commit survives an SPDX round trip and not a CycloneDX one. Where one package appears several times in a lockfile, its occurrences must agree before anything is published. `detectors.MergeOrigin`, applied wherever a detector folds a duplicate into an existing node, treats absence as compatible (an occurrence asserting nothing leaves an origin standing; one asserting something fills a gap) and treats two different assertions as cancelling. Picking a winner would make the output depend on traversal order rather than on the lockfile, and an SBOM that omits a location is honest where one that takes a side of a contradiction is not. Detector graph builds walk their inputs in sorted order for the same reason. -The same rule holds across manifests. Each subproject is resolved on its own, so a package two of them share arrives as two nodes, and the SDK's graph merge keeps whichever it meets first while discarding the rest — which would publish one subproject's answer for a package the scan saw resolved two different ways. `ConsolidateGraphs` therefore settles origin across the selected entries before they are merged, writing the verdict onto every occurrence so the surviving node carries it whichever one that turns out to be. A recorded disagreement is part of that verdict, which is why it is a metadata key rather than simply an absent value: absence would let a later fold refill it. +The same rule holds wherever occurrences fold together, and the fold happens at several depths. Within one detector's graph the CLI reconciles: `node.AddNodeIfMissing`, cargo's node dedup, and `normalizeGraphPackageIdentity`, where two nodes collapse onto one canonical PURL. Across manifests the SDK's own graph merge reconciles, so the CLI no longer needs a pass of its own. A disagreement is recorded on the origin as `Disputed` rather than left as an absent value, because absence would let a later fold refill a location the project never agreed on. -One consequence worth stating: origin keys are filtered out of `scan`/`diff`/`explain` payloads by prefix in `output.cloneRefMetadata` — they are transport between two pipeline stages, and the SBOM is where users read them; the filter returns nil for an emptied map so `omitempty` still fires. +One consequence worth stating: origin does not appear in `scan`/`diff`/`explain` payloads, because those documents are built from explicit projections rather than from the SDK types. It is provenance for the SBOM, which is where users read it. (While origin rode on metadata, keeping it out took an explicit prefix filter in `output.cloneRefMetadata`; the typed field made that unnecessary.) ## Build Modes diff --git a/docs/schemas/diff.md b/docs/schemas/diff.md index 7aded733..2aa2bb82 100644 --- a/docs/schemas/diff.md +++ b/docs/schemas/diff.md @@ -245,6 +245,7 @@ Complete reference for the `bomly diff` JSON output. |-------|------|-------------| | `algorithm` | `string` | | | `value` | `string` | | +| `subject` | `string` | | ### `EPSSScore` diff --git a/docs/schemas/diff.schema.json b/docs/schemas/diff.schema.json index c15348c9..77380653 100644 --- a/docs/schemas/diff.schema.json +++ b/docs/schemas/diff.schema.json @@ -348,6 +348,9 @@ "algorithm": { "type": "string" }, + "subject": { + "type": "string" + }, "value": { "type": "string" } diff --git a/docs/schemas/scan.md b/docs/schemas/scan.md index 5c53ea0a..0d84caaf 100644 --- a/docs/schemas/scan.md +++ b/docs/schemas/scan.md @@ -107,6 +107,7 @@ Complete reference for the `bomly scan` JSON output. |-------|------|-------------| | `algorithm` | `string` | | | `value` | `string` | | +| `subject` | `string` | | ### `EPSSScore` diff --git a/docs/schemas/scan.schema.json b/docs/schemas/scan.schema.json index 9437ae7b..c23e06c0 100644 --- a/docs/schemas/scan.schema.json +++ b/docs/schemas/scan.schema.json @@ -330,6 +330,9 @@ "algorithm": { "type": "string" }, + "subject": { + "type": "string" + }, "value": { "type": "string" } diff --git a/go.mod b/go.mod index c537345c..0d230dc0 100644 --- a/go.mod +++ b/go.mod @@ -114,7 +114,7 @@ require ( github.com/bodgit/plumbing v1.3.0 // indirect github.com/bodgit/sevenzip v1.6.1 // indirect github.com/bodgit/windows v1.0.1 // indirect - github.com/bomly-dev/bomly-sdk v0.3.0 + github.com/bomly-dev/bomly-sdk v0.4.0 github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect github.com/charmbracelet/harmonica v0.2.0 // indirect diff --git a/go.sum b/go.sum index 48b533e4..4b7ac967 100644 --- a/go.sum +++ b/go.sum @@ -259,6 +259,8 @@ github.com/bomly-dev/bomly-plugin-syft-detector v0.1.0 h1:izjWbFILBnjSrRyCwBQS2d github.com/bomly-dev/bomly-plugin-syft-detector v0.1.0/go.mod h1:uLBBHXzEJaNczZAyTIZjxfTSDoU+yLVC36qBE+vIAyk= github.com/bomly-dev/bomly-sdk v0.3.0 h1:JtC7qZ9yq3r4fUYyq7e/Os4f9wGMa4Qot8U0l9MepFA= github.com/bomly-dev/bomly-sdk v0.3.0/go.mod h1:yn1LBkoHG9gDBXKyRj0UNJo0BlXl8Bj9Ymb3WKLIh78= +github.com/bomly-dev/bomly-sdk v0.4.0 h1:C+3gOVF6KFZ8KOq4jV94snP+qn8s1QzxG8PhE30yDaI= +github.com/bomly-dev/bomly-sdk v0.4.0/go.mod h1:yn1LBkoHG9gDBXKyRj0UNJo0BlXl8Bj9Ymb3WKLIh78= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= diff --git a/internal/detectors/cargo/detector.go b/internal/detectors/cargo/detector.go index f4402b0f..c2652704 100644 --- a/internal/detectors/cargo/detector.go +++ b/internal/detectors/cargo/detector.go @@ -447,7 +447,7 @@ func addNodeIfMissing(g *sdk.Graph, node *sdk.Dependency) error { // same crate pulled from two git remotes, say. They share a PURL, so // they are one node, and the node must not claim whichever source was // visited first. - detectors.MergeOrigin(existing, node) + existing.Origin = sdk.ReconcileOrigin(existing.Origin, node.Origin) return nil } if err := g.AddNode(node); err != nil { diff --git a/internal/detectors/cargo/origin.go b/internal/detectors/cargo/origin.go index 6ae26d78..6076a825 100644 --- a/internal/detectors/cargo/origin.go +++ b/internal/detectors/cargo/origin.go @@ -4,7 +4,6 @@ import ( "net/url" "strings" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) @@ -18,7 +17,7 @@ func setCargoOrigin(node *sdk.Dependency, source string) { return } repository := strings.TrimPrefix(trimmed, "git+") - detectors.SetOriginVCS(node, repository, cargoSourceRevision(repository)) + node.Origin = sdk.RepositoryOrigin(repository, cargoSourceRevision(repository)) } // cargoSourceRevision returns the revision cargo locked. The URL fragment holds diff --git a/internal/detectors/cargo/origin_test.go b/internal/detectors/cargo/origin_test.go index 4856c47e..a19dfd2e 100644 --- a/internal/detectors/cargo/origin_test.go +++ b/internal/detectors/cargo/origin_test.go @@ -3,10 +3,21 @@ package cargo import ( "testing" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) +// originOf returns the origin a node publishes, or the zero value when it has +// none, so cases can compare plain structs. +func originOf(dep *sdk.Dependency) sdk.PackageOrigin { + if dep == nil { + return sdk.PackageOrigin{} + } + if origin := dep.Origin.Normalized(); origin != nil { + return *origin + } + return sdk.PackageOrigin{} +} + // Cargo.lock records one source string per package. Only "git+" names where the // code came from; the index prefixes name a registry, and path or workspace // members carry no source at all. @@ -14,22 +25,22 @@ func TestSetCargoOriginBySourcePrefix(t *testing.T) { cases := []struct { name string source string - want detectors.Origin + want sdk.PackageOrigin }{ { name: "git dependency pins the resolved commit in the fragment", source: "git+https://github.com/example/helper?rev=main#3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f", - want: detectors.Origin{VCSURL: "https://github.com/example/helper", VCSRevision: "3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f"}, + want: sdk.PackageOrigin{Repository: "https://github.com/example/helper", Revision: "3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f"}, }, { name: "requested tag is used when no commit was recorded", source: "git+https://github.com/example/helper?tag=v1.2.3", - want: detectors.Origin{VCSURL: "https://github.com/example/helper", VCSRevision: "v1.2.3"}, + want: sdk.PackageOrigin{Repository: "https://github.com/example/helper", Revision: "v1.2.3"}, }, { name: "branch dependency without a pin keeps the repository", source: "git+https://github.com/example/helper", - want: detectors.Origin{VCSURL: "https://github.com/example/helper"}, + want: sdk.PackageOrigin{Repository: "https://github.com/example/helper"}, }, {name: "crates.io index root", source: "registry+https://github.com/rust-lang/crates.io-index"}, {name: "sparse index root", source: "sparse+https://index.crates.io/"}, @@ -42,7 +53,7 @@ func TestSetCargoOriginBySourcePrefix(t *testing.T) { t.Run(tc.name, func(t *testing.T) { node := sdk.NewDependency(sdk.Dependency{Coordinates: sdk.Coordinates{Name: "helper", Version: "1.0.0"}}) setCargoOrigin(node, tc.source) - if got := detectors.OriginFrom(node.Metadata); got != tc.want { + if got := originOf(node); got != tc.want { t.Fatalf("origin = %+v, want %+v", got, tc.want) } }) @@ -76,7 +87,7 @@ func TestCargoDuplicateCrateSourcesCancelOrigin(t *testing.T) { return true } checked++ - if got := detectors.OriginFrom(dep.Metadata); !got.Empty() { + if got := originOf(dep); !got.Empty() { t.Fatalf("origin = %+v, want none: the crate resolved from two repositories", got) } return true @@ -103,15 +114,15 @@ func TestCargoDuplicateCrateSameSourceKeepsOrigin(t *testing.T) { if err != nil { t.Fatalf("depGraphFromMetadata() error = %v", err) } - want := detectors.Origin{ - VCSURL: "https://github.com/a/helper", - VCSRevision: "aaaabbbbccccddddeeeeffff0000111122223333", + want := sdk.PackageOrigin{ + Repository: "https://github.com/a/helper", + Revision: "aaaabbbbccccddddeeeeffff0000111122223333", } var checked int graph.WalkNodes(func(dep *sdk.Dependency) bool { if dep.Name == "helper" { checked++ - if got := detectors.OriginFrom(dep.Metadata); got != want { + if got := originOf(dep); got != want { t.Fatalf("origin = %+v, want %+v", got, want) } } @@ -151,15 +162,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" if !ok { t.Fatal("expected helper in graph") } - want := detectors.Origin{VCSURL: "https://github.com/example/helper", VCSRevision: "6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192"} - if got := detectors.OriginFrom(helper.Metadata); got != want { + want := sdk.PackageOrigin{Repository: "https://github.com/example/helper", Revision: "6f708192a3b4c5d6e7f8091a2b3c4d5e6f708192"} + if got := originOf(helper); got != want { t.Fatalf("helper origin = %+v, want %+v", got, want) } serde, ok := graph.Node("serde@1.0.0") if !ok { t.Fatal("expected serde in graph") } - if got := detectors.OriginFrom(serde.Metadata); !got.Empty() { + if got := originOf(serde); !got.Empty() { t.Fatalf("registry crate asserted an origin: %+v", got) } } diff --git a/internal/detectors/node/bun/bun_lockfile_parser.go b/internal/detectors/node/bun/bun_lockfile_parser.go index 7fcf960d..6e76fb20 100644 --- a/internal/detectors/node/bun/bun_lockfile_parser.go +++ b/internal/detectors/node/bun/bun_lockfile_parser.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/Masterminds/semver/v3" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-cli/internal/detectors/node" "github.com/bomly-dev/bomly-sdk" "github.com/bomly-dev/bomly-sdk/system" @@ -144,7 +143,7 @@ func depGraphFromBunLockfile(projectPath string) (bunLockfileGraphs, error) { } // Bun's tuple carries the registry tarball it fetched. Workspace // members and git specs resolve to values the invariant rejects. - detectors.SetOriginArtifact(pkgNode, entry.resolved) + pkgNode.Origin = sdk.ArtifactOrigin(entry.resolved) if err := node.AddNodeIfMissing(graph, pkgNode); err != nil { return bunLockfileGraphs{}, err } diff --git a/internal/detectors/node/common.go b/internal/detectors/node/common.go index cbd2ec79..ad0ea390 100644 --- a/internal/detectors/node/common.go +++ b/internal/detectors/node/common.go @@ -12,7 +12,6 @@ import ( "sort" "time" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-cli/internal/logging" "github.com/bomly-dev/bomly-sdk" logkit "github.com/bomly-dev/bomly-sdk/logkit" @@ -190,7 +189,7 @@ func DepGraphFromNPMNode(root *NPMListNode) (*sdk.Graph, error) { Name: name, Version: depNode.Version}, }) - detectors.SetOriginArtifact(node, depNode.Resolved) + node.Origin = sdk.ArtifactOrigin(depNode.Resolved) if err := AddNodeIfMissing(depsGraph, node); err != nil { return nil, err @@ -340,7 +339,7 @@ func splitYarnTreeName(value string) (string, string, error) { func AddNodeIfMissing(depsGraph *sdk.Graph, node *sdk.Dependency) error { if existing, ok := depsGraph.Node(node.ID); ok { existing.AddScope(node.PrimaryScope()) - detectors.MergeOrigin(existing, node) + existing.Origin = sdk.ReconcileOrigin(existing.Origin, node.Origin) return nil } if err := depsGraph.AddNode(node); err != nil { diff --git a/internal/detectors/node/npm/npm_lockfile_parser.go b/internal/detectors/node/npm/npm_lockfile_parser.go index 7be67391..5a80b7ed 100644 --- a/internal/detectors/node/npm/npm_lockfile_parser.go +++ b/internal/detectors/node/npm/npm_lockfile_parser.go @@ -9,7 +9,6 @@ import ( "sort" "strings" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-cli/internal/detectors/node" "github.com/bomly-dev/bomly-sdk" "github.com/bomly-dev/bomly-sdk/system" @@ -251,7 +250,7 @@ func depGraphFromNPMLockfile(projectPath string) (npmLockfileGraphs, error) { // npm records the registry tarball a package was installed from. // Workspace members cleared ResolvedURL above (it names a local // directory), and git or file specs are rejected by the invariant. - detectors.SetOriginArtifact(pkgNode, pkg.ResolvedURL) + pkgNode.Origin = sdk.ArtifactOrigin(pkg.ResolvedURL) if entry.License != "" { sdk.SetDetectionLicenses(pkgNode, []sdk.PackageLicense{{Value: entry.License, Type: "declared"}}) } diff --git a/internal/detectors/node/npm/origin_test.go b/internal/detectors/node/npm/origin_test.go index 7b057c5b..9d06487c 100644 --- a/internal/detectors/node/npm/origin_test.go +++ b/internal/detectors/node/npm/origin_test.go @@ -1,13 +1,24 @@ package npm import ( + "github.com/bomly-dev/bomly-sdk" "os" "path/filepath" "testing" - - "github.com/bomly-dev/bomly-cli/internal/detectors" ) +// originOf returns the origin a node publishes, or the zero value when it has +// none, so cases can compare plain structs. +func originOf(dep *sdk.Dependency) sdk.PackageOrigin { + if dep == nil { + return sdk.PackageOrigin{} + } + if origin := dep.Origin.Normalized(); origin != nil { + return *origin + } + return sdk.PackageOrigin{} +} + // npm writes whatever it installed from into "resolved": a registry tarball, // but also a git remote or a local path. Only the first is a location an SBOM // can publish. @@ -48,12 +59,12 @@ func TestNPMOriginByResolvedShape(t *testing.T) { if !ok { t.Fatalf("expected %s in graph", tc.id) } - origin := detectors.OriginFrom(node.Metadata) + origin := originOf(node) if origin.ArtifactURL != tc.want { t.Errorf("%s artifact origin = %q, want %q", tc.id, origin.ArtifactURL, tc.want) } - if origin.VCSURL != "" { - t.Errorf("%s asserted a repository %q", tc.id, origin.VCSURL) + if origin.Repository != "" { + t.Errorf("%s asserted a repository %q", tc.id, origin.Repository) } // ResolvedURL is a separate contract (the scorecard matcher resolves // repositories from it) and must keep carrying the raw lockfile value. @@ -109,7 +120,7 @@ func TestNPMv1DuplicateEntriesReconcileOrigin(t *testing.T) { if !ok { t.Fatal("expected disputed@2.0.0 in graph") } - if origin := detectors.OriginFrom(disputed.Metadata); !origin.Empty() { + if origin := originOf(disputed); !origin.Empty() { t.Fatalf("disputed origin = %+v, want none: its two copies name different locations", origin) } @@ -118,7 +129,7 @@ func TestNPMv1DuplicateEntriesReconcileOrigin(t *testing.T) { t.Fatal("expected agreed@3.0.0 in graph") } const want = "https://registry.npmjs.org/agreed/-/agreed-3.0.0.tgz" - if got := detectors.OriginFrom(agreed.Metadata).ArtifactURL; got != want { + if got := originOf(agreed).ArtifactURL; got != want { t.Fatalf("agreed origin = %q, want %q: its copies agree", got, want) } } diff --git a/internal/detectors/node/origin_integration_test.go b/internal/detectors/node/origin_integration_test.go index 7bf8e1df..d95a72be 100644 --- a/internal/detectors/node/origin_integration_test.go +++ b/internal/detectors/node/origin_integration_test.go @@ -4,7 +4,6 @@ import ( "context" "testing" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-cli/internal/detectors/node/bun" "github.com/bomly-dev/bomly-cli/internal/detectors/node/npm" "github.com/bomly-dev/bomly-cli/internal/detectors/node/pnpm" @@ -12,22 +11,34 @@ import ( "github.com/bomly-dev/bomly-sdk" ) +// originOf returns the origin a node publishes, or the zero value when it has +// none, so cases can compare plain structs. +func originOf(dep *sdk.Dependency) sdk.PackageOrigin { + if dep == nil { + return sdk.PackageOrigin{} + } + if origin := dep.Origin.Normalized(); origin != nil { + return *origin + } + return sdk.PackageOrigin{} +} + // requireArtifactOrigin asserts a package asserts exactly the given artifact. func requireArtifactOrigin(t *testing.T, g *sdk.Graph, name, version, want string) { t.Helper() - origin := detectors.OriginFrom(requirePackage(t, g, name, version).Metadata) + origin := originOf(requirePackage(t, g, name, version)) if origin.ArtifactURL != want { t.Errorf("%s@%s artifact origin = %q, want %q", name, version, origin.ArtifactURL, want) } - if origin.VCSURL != "" { - t.Errorf("%s@%s also asserted a repository %q", name, version, origin.VCSURL) + if origin.Repository != "" { + t.Errorf("%s@%s also asserted a repository %q", name, version, origin.Repository) } } // requireNoOrigin asserts a package publishes no location at all. func requireNoOrigin(t *testing.T, g *sdk.Graph, name, version string) { t.Helper() - if origin := detectors.OriginFrom(requirePackage(t, g, name, version).Metadata); !origin.Empty() { + if origin := originOf(requirePackage(t, g, name, version)); !origin.Empty() { t.Errorf("%s@%s asserted an origin it should not have: %+v", name, version, origin) } } diff --git a/internal/detectors/node/pnpm/pnpm_lockfile_parser.go b/internal/detectors/node/pnpm/pnpm_lockfile_parser.go index b99a74c7..7a30d0a4 100644 --- a/internal/detectors/node/pnpm/pnpm_lockfile_parser.go +++ b/internal/detectors/node/pnpm/pnpm_lockfile_parser.go @@ -9,7 +9,6 @@ import ( "strconv" "strings" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-cli/internal/detectors/node" "github.com/bomly-dev/bomly-sdk" "github.com/bomly-dev/bomly-sdk/system" @@ -142,7 +141,7 @@ func depGraphFromPNPMLockfile(projectPath string) (pnpmLockfileGraphs, error) { // pnpm records a tarball only when it resolved one; v9 lockfiles // often carry just an integrity hash, and git or directory // resolutions are not parsed, so those packages assert no origin. - detectors.SetOriginArtifact(pkgNode, entry.Resolution.Tarball) + pkgNode.Origin = sdk.ArtifactOrigin(entry.Resolution.Tarball) if entry.License != "" { sdk.SetDetectionLicenses(pkgNode, []sdk.PackageLicense{{Value: entry.License, Type: "declared"}}) } diff --git a/internal/detectors/node/yarn/yarn_lockfile_parser.go b/internal/detectors/node/yarn/yarn_lockfile_parser.go index cd552219..70a33329 100644 --- a/internal/detectors/node/yarn/yarn_lockfile_parser.go +++ b/internal/detectors/node/yarn/yarn_lockfile_parser.go @@ -10,7 +10,6 @@ import ( "unicode" "github.com/Masterminds/semver/v3" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-cli/internal/detectors/node" "github.com/bomly-dev/bomly-sdk" "github.com/bomly-dev/bomly-sdk/system" @@ -85,7 +84,7 @@ func depGraphFromYarnLockfile(projectPath string) (*sdk.Graph, error) { // Yarn Classic records the tarball it fetched, with the package // checksum as a URL fragment the invariant strips. Berry entries // carry no resolved location, and git specs are rejected. - detectors.SetOriginArtifact(pkgNode, entry.Resolved) + pkgNode.Origin = sdk.ArtifactOrigin(entry.Resolved) if err := node.AddNodeIfMissing(depsGraph, pkgNode); err != nil { return "", err } diff --git a/internal/detectors/origin.go b/internal/detectors/origin.go deleted file mode 100644 index b34493b8..00000000 --- a/internal/detectors/origin.go +++ /dev/null @@ -1,360 +0,0 @@ -package detectors - -import ( - "net/url" - "strconv" - "strings" - - "github.com/bomly-dev/bomly-sdk" -) - -// Origin metadata keys. Detectors record where a package came from under these -// keys on sdk.Dependency.Metadata; SBOM export reads them back. The values are -// a transport detail between detection and export, so command output filters -// the shared prefix out rather than publishing it. -const ( - // MetadataKeyOriginPrefix is the common prefix of every origin key. - MetadataKeyOriginPrefix = "bomly.origin." - // MetadataKeyOriginArtifactURL holds the exact artifact a package was - // resolved from (a tarball, wheel, gem, crate, ...). - MetadataKeyOriginArtifactURL = MetadataKeyOriginPrefix + "artifact_url" - // MetadataKeyOriginVCSURL holds the source repository a package was - // resolved from. - MetadataKeyOriginVCSURL = MetadataKeyOriginPrefix + "vcs_url" - // MetadataKeyOriginVCSRevision holds the resolved revision (commit, tag) - // pinned alongside MetadataKeyOriginVCSURL. - MetadataKeyOriginVCSRevision = MetadataKeyOriginPrefix + "vcs_revision" - // MetadataKeyOriginConflict marks a package whose occurrences disagreed - // about where it came from. The mark outlives the occurrence that caused - // it, so a later occurrence repeating one of the disputed values cannot - // revive it: with three occurrences claiming A, B, then A, the package - // still has no agreed origin. - MetadataKeyOriginConflict = MetadataKeyOriginPrefix + "conflict" -) - -// maxOriginRevisionLength bounds a recorded revision. Real commit hashes and -// tags are far shorter; anything longer is not a revision. -const maxOriginRevisionLength = 128 - -// Origin is where a package came from, as asserted by the detector that -// resolved it. At most one location is set: a package is either downloaded as -// an artifact or checked out from a repository. An empty Origin means the -// detector had nothing publishable to say, which is the normal case for -// registry-resolved packages whose lockfile records only an index root. -type Origin struct { - // ArtifactURL is the exact file the package was downloaded from. - ArtifactURL string - // VCSURL is the source repository the package was resolved from. - VCSURL string - // VCSRevision is the revision pinned in VCSURL, when the lockfile - // recorded one. Never set without VCSURL. - VCSRevision string -} - -// Empty reports whether no location is set. -func (o Origin) Empty() bool { - return o.ArtifactURL == "" && o.VCSURL == "" -} - -// NormalizeOriginURL is the single invariant every published origin URL must -// satisfy. It is applied when a detector records a URL and again when export -// reads one back, so a plugin-supplied or hand-built graph is held to the same -// rule as a built-in detector. -// -// A value passes only when it is an absolute http or https URL with a host and -// no embedded credentials; the result is always re-serialized from the parse, -// never the caller's raw string. Everything else — local paths, file://, -// git@host:org/repo, ssh://, git+ssh://, and URLs carrying userinfo — is -// rejected, so filesystem layout and credentials cannot reach an SBOM. -// -// Both forms require a non-empty path, so a bare host -- a registry or index -// root -- is never published. -// -// The vcs argument selects the repository form: query and fragment are dropped, -// because they carry the requested ref rather than the resolved one, which -// callers pass separately. The artifact form instead drops the fragment (a -// checksum or anchor, never part of the location) and rejects a value carrying -// a query, which marks a signed or tokenized link rather than a stable -// location. -func NormalizeOriginURL(raw string, vcs bool) (string, bool) { - trimmed := strings.TrimSpace(raw) - if trimmed == "" { - return "", false - } - parsed, err := url.Parse(trimmed) - if err != nil { - return "", false - } - switch strings.ToLower(parsed.Scheme) { - case "http", "https": - default: - return "", false - } - // Hostname also rejects a malformed host such as "https://:8080/pkg". - if parsed.Hostname() == "" || parsed.User != nil { - return "", false - } - parsed.Scheme = strings.ToLower(parsed.Scheme) - // Hosts are case-insensitive, so two lockfiles writing one host - // differently name the same location. Without this they compare unequal - // and reconcile to a disagreement, losing an origin to formatting alone. - // The path is left alone: it is case-sensitive. - parsed.Host = strings.ToLower(parsed.Host) - // An explicit default port names the same origin as no port at all, so - // dropping it keeps two spellings of one location from reading as a - // disagreement. - if port := parsed.Port(); port != "" { - // url.Parse only checks that a port is numeric, so a value no client - // could connect to still reaches here. - number, err := strconv.Atoi(port) - if err != nil || number < 1 || number > 65535 { - return "", false - } - } - if port := parsed.Port(); (parsed.Scheme == "https" && port == "443") || (parsed.Scheme == "http" && port == "80") { - host := parsed.Hostname() - if strings.Contains(host, ":") { - host = "[" + host + "]" // an IPv6 literal keeps its brackets - } - parsed.Host = host - } - parsed.Fragment = "" - parsed.RawFragment = "" - // A host root names a server, not a package: it is a registry or index - // root on the artifact side and no repository at all on the VCS side. - // An empty path would also make the SPDX "@" suffix re-parse - // as userinfo. - if strings.Trim(parsed.Path, "/") == "" { - return "", false - } - if vcs { - parsed.RawQuery = "" - parsed.ForceQuery = false - } else if parsed.RawQuery != "" || parsed.ForceQuery { - return "", false - } - normalized := parsed.String() - if normalized == "" { - return "", false - } - return normalized, true -} - -// SetOriginArtifact records the exact artifact dep was resolved from, replacing -// any origin already recorded. Callers pass the lockfile field verbatim; values -// that are not publishable URLs are dropped silently, since a missing origin is -// correct output and a wrong one is not. No-op when dep is nil. -func SetOriginArtifact(dep *sdk.Dependency, rawURL string) { - if dep == nil { - return - } - normalized, ok := NormalizeOriginURL(rawURL, false) - if !ok { - return - } - clearOrigin(dep) - setOriginValue(dep, MetadataKeyOriginArtifactURL, normalized) -} - -// SetOriginVCS records the source repository dep was resolved from, plus the -// revision the lockfile pinned, replacing any origin already recorded. An -// unpublishable URL drops the whole origin; an unusable revision drops only the -// revision, keeping the repository. No-op when dep is nil. -func SetOriginVCS(dep *sdk.Dependency, rawURL, revision string) { - if dep == nil { - return - } - normalized, ok := NormalizeOriginURL(rawURL, true) - if !ok { - return - } - clearOrigin(dep) - setOriginValue(dep, MetadataKeyOriginVCSURL, normalized) - if pinned := strings.TrimSpace(revision); isValidOriginRevision(pinned) { - setOriginValue(dep, MetadataKeyOriginVCSRevision, pinned) - } -} - -// OriginFrom reads the origin a detector recorded on metadata, re-validating -// every value. Anything that fails the invariant is dropped, so export cannot -// publish a location no detector could legitimately have produced. An artifact -// wins over a repository in the case — which the setters never produce — where -// metadata carries both. -func OriginFrom(metadata map[string]any) Origin { - if len(metadata) == 0 { - return Origin{} - } - if conflicted, _ := metadata[MetadataKeyOriginConflict].(bool); conflicted { - return Origin{} - } - if artifact, ok := NormalizeOriginURL(originString(metadata, MetadataKeyOriginArtifactURL), false); ok { - return Origin{ArtifactURL: artifact} - } - repository, ok := NormalizeOriginURL(originString(metadata, MetadataKeyOriginVCSURL), true) - if !ok { - return Origin{} - } - origin := Origin{VCSURL: repository} - if pinned := strings.TrimSpace(originString(metadata, MetadataKeyOriginVCSRevision)); isValidOriginRevision(pinned) { - origin.VCSRevision = pinned - } - return origin -} - -// MergeOrigin reconciles the origin of two nodes a detector resolved to the -// same package, which happens when a lockfile records one package at several -// places in a tree. -// -// Absence is not a disagreement: an occurrence that asserts nothing leaves an -// existing origin standing, and an occurrence that asserts one fills a gap. -// Two occurrences asserting *different* origins cancel, and stay cancelled: the -// disagreement is recorded so a third occurrence repeating one of the disputed -// values cannot revive it. One graph node is one package, so publishing -// whichever occurrence happened to be visited first would make the output -// depend on traversal order rather than on the lockfile -- and an SBOM that -// omits a location is honest, while one that picks a side of a contradiction -// is not. -func MergeOrigin(existing, duplicate *sdk.Dependency) { - if existing == nil || duplicate == nil { - return - } - if originConflicted(existing) { - // Already cancelled. Nothing a later occurrence says can settle a - // disagreement that happened, so the mark is not lifted here. - return - } - if originConflicted(duplicate) { - markOriginConflict(existing) - return - } - incoming := OriginFrom(duplicate.Metadata) - if incoming.Empty() { - return - } - switch current := OriginFrom(existing.Metadata); { - case current.Empty(): - storeOrigin(existing, incoming) - case current != incoming: - markOriginConflict(existing) - } -} - -// ReconcileOrigins settles the origin of several nodes that describe one -// package, leaving every one of them carrying the same answer. -// -// Detectors resolve one manifest at a time, so a package used by two -// subprojects arrives as two nodes, each with its own origin. They are merged -// into a single node later, by a merge that keeps whichever it encounters -// first and discards the rest -- which would publish one subproject's answer -// for a package the scan saw resolved two different ways. Settling the -// disagreement here means the surviving node carries the reconciled verdict -// whichever one that turns out to be. -func ReconcileOrigins(occurrences []*sdk.Dependency) { - if len(occurrences) < 2 { - return - } - verdict := occurrences[0] - if verdict == nil { - return - } - for _, occurrence := range occurrences[1:] { - MergeOrigin(verdict, occurrence) - } - - settled, conflicted := OriginFrom(verdict.Metadata), originConflicted(verdict) - for _, occurrence := range occurrences[1:] { - if occurrence == nil { - continue - } - if conflicted { - markOriginConflict(occurrence) - continue - } - storeOrigin(occurrence, settled) - } -} - -// originConflicted reports whether dep's occurrences already disagreed. -func originConflicted(dep *sdk.Dependency) bool { - if dep == nil || dep.Metadata == nil { - return false - } - conflicted, _ := dep.Metadata[MetadataKeyOriginConflict].(bool) - return conflicted -} - -// markOriginConflict drops dep's origin and records that its occurrences -// disagreed, so no later merge can restore one of the disputed values. -func markOriginConflict(dep *sdk.Dependency) { - clearOrigin(dep) - if dep.Metadata == nil { - dep.Metadata = make(map[string]any, 1) - } - dep.Metadata[MetadataKeyOriginConflict] = true -} - -// storeOrigin writes an already-validated origin onto dep. -func storeOrigin(dep *sdk.Dependency, origin Origin) { - clearOrigin(dep) - switch { - case origin.ArtifactURL != "": - setOriginValue(dep, MetadataKeyOriginArtifactURL, origin.ArtifactURL) - case origin.VCSURL != "": - setOriginValue(dep, MetadataKeyOriginVCSURL, origin.VCSURL) - if origin.VCSRevision != "" { - setOriginValue(dep, MetadataKeyOriginVCSRevision, origin.VCSRevision) - } - } -} - -// isValidOriginRevision reports whether revision is safe to publish beside a -// repository URL. The charset keeps commit hashes, tags, and branch-style refs -// while excluding whitespace, "@", and percent escapes, which would break the -// SPDX "git+@" locator grammar. -func isValidOriginRevision(revision string) bool { - if revision == "" || len(revision) > maxOriginRevisionLength { - return false - } - for _, r := range revision { - switch { - case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': - case r == '.', r == '_', r == '-', r == '+', r == '/': - default: - return false - } - } - return true -} - -// clearOrigin drops any origin already recorded on dep, so a later assertion -// replaces an earlier one rather than merging with it: a package has one -// origin, and a stale revision left beside a new repository would name a commit -// that repository may not contain. Only a value that passed the invariant -// clears the previous one, so a rejected input leaves an earlier origin intact. -func clearOrigin(dep *sdk.Dependency) { - if dep.Metadata == nil { - return - } - delete(dep.Metadata, MetadataKeyOriginArtifactURL) - delete(dep.Metadata, MetadataKeyOriginVCSURL) - delete(dep.Metadata, MetadataKeyOriginVCSRevision) - // A detector setting an origin outright is asserting what it resolved, - // which supersedes a disagreement between earlier occurrences. Only - // merging leaves the mark in place. - delete(dep.Metadata, MetadataKeyOriginConflict) -} - -// setOriginValue stores one origin fact, allocating the metadata map on demand. -func setOriginValue(dep *sdk.Dependency, key, value string) { - if dep.Metadata == nil { - dep.Metadata = make(map[string]any, 1) - } - dep.Metadata[key] = value -} - -// originString reads a string-valued metadata entry, tolerating a map built by -// something other than the setters. -func originString(metadata map[string]any, key string) string { - value, _ := metadata[key].(string) - return value -} diff --git a/internal/detectors/origin_fuzz_test.go b/internal/detectors/origin_fuzz_test.go deleted file mode 100644 index ae3e7658..00000000 --- a/internal/detectors/origin_fuzz_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package detectors_test - -import ( - "net/url" - "testing" - - "github.com/bomly-dev/bomly-cli/internal/detectors" - "github.com/bomly-dev/bomly-sdk" - testutil "github.com/bomly-dev/bomly-sdk/testkit" -) - -// FuzzSetOrigin drives the origin invariant with arbitrary lockfile-derived -// strings. Detectors pass raw lockfile fields straight through, so whatever a -// repository can put in a lockfile can reach these setters. -func FuzzSetOrigin(f *testing.F) { - f.Add("https://registry.npmjs.org/react/-/react-18.2.0.tgz", "") - f.Add("https://github.com/owner/repo.git", "9f8e7d6c5b4a3928176554433221100ffeeddcc0") - f.Add("https://github.com/example/helper?rev=main#abc123", "v1.2.3") - f.Add("https://user:s3cret@nexus.corp/repo/pkg.tgz", "main") - f.Add("git+ssh://git@github.com/owner/repo.git#9f8e7d6", "9f8e7d6") - f.Add("file:///home/someone/wheels/pkg.whl", "") - f.Add("/Users/someone/src/project", "") - f.Add("http://0#0", "0") - f.Add("http://0/0#\x02", "\x02") - f.Add("%./0", "%") - f.Add("https://", "") - f.Add("https://:8080/pkg.tgz", "") - f.Add("https://例え.テスト/パッケージ.tgz", "リビジョン") - - f.Fuzz(func(t *testing.T, rawURL, revision string) { - if len(rawURL)+len(revision) > testutil.MaxFuzzInputSize { - return - } - - artifact := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginArtifact(artifact, rawURL) - repository := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginVCS(repository, rawURL, revision) - - // Whatever was stored must satisfy the invariant: export publishes - // these values into an SBOM without re-deciding anything. - assertPublishable(t, detectors.OriginFrom(artifact.Metadata)) - assertPublishable(t, detectors.OriginFrom(repository.Metadata)) - - // A second pass over the same input must reach the same conclusion, - // and reading back what was written must be a fixed point. - again := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginVCS(again, rawURL, revision) - first, second := detectors.OriginFrom(repository.Metadata), detectors.OriginFrom(again.Metadata) - if first != second { - t.Fatalf("nondeterministic origin: %+v then %+v", first, second) - } - if reread := detectors.OriginFrom(map[string]any{ - detectors.MetadataKeyOriginVCSURL: first.VCSURL, - detectors.MetadataKeyOriginVCSRevision: first.VCSRevision, - }); reread != first { - t.Fatalf("stored origin did not survive a re-read: %+v became %+v", first, reread) - } - }) -} - -// assertPublishable fails when an origin carries anything an SBOM must never -// show: a non-web location, a host-less URL, embedded credentials, or a -// revision that would break the SPDX "git+@" grammar. -func assertPublishable(t *testing.T, origin detectors.Origin) { - t.Helper() - - if origin.ArtifactURL != "" && origin.VCSURL != "" { - t.Fatalf("origin claims two locations at once: %+v", origin) - } - if origin.VCSRevision != "" && origin.VCSURL == "" { - t.Fatalf("revision %q recorded without a repository", origin.VCSRevision) - } - for _, raw := range []string{origin.ArtifactURL, origin.VCSURL} { - if raw == "" { - continue - } - parsed, err := url.Parse(raw) - if err != nil { - t.Fatalf("published URL %q does not parse: %v", raw, err) - } - if parsed.Scheme != "http" && parsed.Scheme != "https" { - t.Fatalf("published URL %q is not a web location", raw) - } - if parsed.Hostname() == "" { - t.Fatalf("published URL %q has no host", raw) - } - if parsed.User != nil { - t.Fatalf("published URL %q carries credentials", raw) - } - if parsed.Fragment != "" { - t.Fatalf("published URL %q carries a fragment", raw) - } - } - if origin.VCSURL != "" { - parsed, err := url.Parse(origin.VCSURL) - if err != nil { - t.Fatalf("repository URL %q does not parse: %v", origin.VCSURL, err) - } - if parsed.RawQuery != "" || parsed.ForceQuery { - t.Fatalf("repository URL %q carries a query", origin.VCSURL) - } - } - for _, r := range origin.VCSRevision { - switch { - case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': - case r == '.', r == '_', r == '-', r == '+', r == '/': - default: - t.Fatalf("revision %q carries %q, which breaks the SPDX locator grammar", origin.VCSRevision, r) - } - } -} diff --git a/internal/detectors/origin_test.go b/internal/detectors/origin_test.go deleted file mode 100644 index c6f60d35..00000000 --- a/internal/detectors/origin_test.go +++ /dev/null @@ -1,493 +0,0 @@ -package detectors_test - -import ( - "strings" - "testing" - - "github.com/bomly-dev/bomly-cli/internal/detectors" - "github.com/bomly-dev/bomly-sdk" -) - -func TestSetOriginArtifact(t *testing.T) { - cases := []struct { - name string - raw string - want string - }{ - {name: "registry tarball", raw: "https://registry.npmjs.org/react/-/react-18.2.0.tgz", want: "https://registry.npmjs.org/react/-/react-18.2.0.tgz"}, - {name: "yarn digest fragment is stripped", raw: "https://registry.npmjs.org/react/-/react-18.2.0.tgz#ceeba773e3e9d2b6f1a2b6b9f4f1cb2f9c2e1a55", want: "https://registry.npmjs.org/react/-/react-18.2.0.tgz"}, - {name: "codeload tarball", raw: "https://codeload.github.com/owner/repo/tar.gz/9f8e7d6c5b4a3928176554433221100ffeeddcc", want: "https://codeload.github.com/owner/repo/tar.gz/9f8e7d6c5b4a3928176554433221100ffeeddcc"}, - {name: "uppercase scheme is normalized", raw: "HTTPS://files.pythonhosted.org/packages/x/django-5.0.tar.gz", want: "https://files.pythonhosted.org/packages/x/django-5.0.tar.gz"}, - {name: "signed link carrying a query is dropped", raw: "https://nexus.corp/repo/pkg.tgz?token=abc123", want: ""}, - {name: "userinfo is dropped", raw: "https://user:s3cret@nexus.corp/repo/pkg.tgz", want: ""}, - {name: "npm link directory is dropped", raw: "packages/lib", want: ""}, - {name: "absolute local path is dropped", raw: "/Users/someone/src/project", want: ""}, - {name: "file url is dropped", raw: "file:///home/someone/wheels/pkg.whl", want: ""}, - {name: "git+ssh is dropped", raw: "git+ssh://git@github.com/owner/repo.git#9f8e7d6", want: ""}, - {name: "scp-style remote is dropped", raw: "git@github.com:owner/repo.git", want: ""}, - {name: "git+https prefix is not a plain URL", raw: "git+https://github.com/owner/repo.git", want: ""}, - {name: "non-web scheme is dropped", raw: "ftp://files.example.com/pkg.tgz", want: ""}, - {name: "windows path is dropped", raw: `C:\src\project`, want: ""}, - {name: "malformed host is dropped", raw: "https://:8080/pkg.tgz", want: ""}, - {name: "scheme without host is dropped", raw: "https://", want: ""}, - {name: "registry root names no artifact", raw: "https://registry.example.test/", want: ""}, - {name: "empty is dropped", raw: " ", want: ""}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - dep := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginArtifact(dep, tc.raw) - - origin := detectors.OriginFrom(dep.Metadata) - if origin.ArtifactURL != tc.want { - t.Fatalf("artifact URL = %q, want %q", origin.ArtifactURL, tc.want) - } - if origin.VCSURL != "" || origin.VCSRevision != "" { - t.Fatalf("artifact origin leaked repository data: %+v", origin) - } - if tc.want == "" && len(dep.Metadata) != 0 { - t.Fatalf("rejected value still recorded metadata: %v", dep.Metadata) - } - }) - } -} - -func TestSetOriginVCS(t *testing.T) { - cases := []struct { - name string - raw string - revision string - wantURL string - wantRevision string - }{ - { - name: "repository with resolved commit", - raw: "https://github.com/owner/repo.git", - revision: "9f8e7d6c5b4a3928176554433221100ffeeddcc0", - wantURL: "https://github.com/owner/repo.git", - wantRevision: "9f8e7d6c5b4a3928176554433221100ffeeddcc0", - }, - { - name: "requested ref in query and fragment is dropped for the resolved one", - raw: "https://github.com/example/helper?rev=main#abc123", - revision: "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567", - wantURL: "https://github.com/example/helper", - wantRevision: "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567", - }, - {name: "tag pin", raw: "https://github.com/owner/repo", revision: "v1.2.3", wantURL: "https://github.com/owner/repo", wantRevision: "v1.2.3"}, - {name: "branch-style ref", raw: "https://github.com/owner/repo", revision: "release/2026-08", wantURL: "https://github.com/owner/repo", wantRevision: "release/2026-08"}, - {name: "unpinned repository", raw: "https://github.com/owner/repo", revision: "", wantURL: "https://github.com/owner/repo"}, - {name: "revision breaking the SPDX locator keeps the repository", raw: "https://github.com/owner/repo", revision: "feature@login", wantURL: "https://github.com/owner/repo"}, - {name: "whitespace revision keeps the repository", raw: "https://github.com/owner/repo", revision: "not a revision", wantURL: "https://github.com/owner/repo"}, - {name: "overlong revision keeps the repository", raw: "https://github.com/owner/repo", revision: strings.Repeat("a", 129), wantURL: "https://github.com/owner/repo"}, - {name: "bare host names no repository", raw: "https://github.com", revision: "9f8e7d6", wantURL: ""}, - {name: "index root names no repository", raw: "https://index.crates.io/", revision: "9f8e7d6", wantURL: ""}, - {name: "root path names no repository", raw: "https://github.com/", revision: "9f8e7d6", wantURL: ""}, - {name: "userinfo is dropped", raw: "https://oauth2:glpat-xxxxxxxxxxxxxxxxxxxx@gitlab.corp/team/repo.git", revision: "9f8e7d6", wantURL: ""}, - {name: "local checkout is dropped", raw: "/Users/someone/src/repo", revision: "9f8e7d6", wantURL: ""}, - {name: "ssh remote is dropped", raw: "ssh://git@github.com/owner/repo.git", revision: "9f8e7d6", wantURL: ""}, - {name: "ssh remote without userinfo is dropped", raw: "ssh://github.com/owner/repo.git", revision: "9f8e7d6", wantURL: ""}, - {name: "git+https prefix must be stripped by the detector", raw: "git+https://github.com/owner/repo.git", revision: "9f8e7d6", wantURL: ""}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - dep := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginVCS(dep, tc.raw, tc.revision) - - origin := detectors.OriginFrom(dep.Metadata) - if origin.VCSURL != tc.wantURL { - t.Fatalf("VCS URL = %q, want %q", origin.VCSURL, tc.wantURL) - } - if origin.VCSRevision != tc.wantRevision { - t.Fatalf("VCS revision = %q, want %q", origin.VCSRevision, tc.wantRevision) - } - if origin.ArtifactURL != "" { - t.Fatalf("repository origin leaked an artifact URL: %q", origin.ArtifactURL) - } - if tc.wantURL == "" && len(dep.Metadata) != 0 { - t.Fatalf("rejected value still recorded metadata: %v", dep.Metadata) - } - }) - } -} - -// A package has one origin. A later assertion replaces an earlier one rather -// than merging with it, so metadata never names two locations or pairs a -// repository with a revision that belongs to a different one. -func TestSetOriginReplacesRatherThanMerges(t *testing.T) { - const ( - artifact = "https://registry.npmjs.org/react/-/react-18.2.0.tgz" - repository = "https://github.com/facebook/react" - fork = "https://github.com/facebook/react-fork" - revision = "c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f8" - ) - - t.Run("repository replaces artifact", func(t *testing.T) { - dep := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginArtifact(dep, artifact) - detectors.SetOriginVCS(dep, repository, revision) - - want := detectors.Origin{VCSURL: repository, VCSRevision: revision} - if got := detectors.OriginFrom(dep.Metadata); got != want { - t.Fatalf("origin = %+v, want %+v", got, want) - } - }) - - t.Run("artifact replaces repository", func(t *testing.T) { - dep := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginVCS(dep, repository, revision) - detectors.SetOriginArtifact(dep, artifact) - - want := detectors.Origin{ArtifactURL: artifact} - if got := detectors.OriginFrom(dep.Metadata); got != want { - t.Fatalf("origin = %+v, want %+v", got, want) - } - }) - - t.Run("an unpinned repository drops the earlier revision", func(t *testing.T) { - dep := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginVCS(dep, repository, revision) - detectors.SetOriginVCS(dep, fork, "") - - want := detectors.Origin{VCSURL: fork} - if got := detectors.OriginFrom(dep.Metadata); got != want { - t.Fatalf("origin = %+v, want %+v; a revision must not follow a repository it did not come from", got, want) - } - }) - - t.Run("a rejected value leaves the earlier origin intact", func(t *testing.T) { - dep := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginArtifact(dep, artifact) - detectors.SetOriginVCS(dep, "/Users/someone/src/react", revision) - - want := detectors.Origin{ArtifactURL: artifact} - if got := detectors.OriginFrom(dep.Metadata); got != want { - t.Fatalf("origin = %+v, want %+v", got, want) - } - }) -} - -func TestSetOriginAllocatesMetadataAndPreservesOtherKeys(t *testing.T) { - dep := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginArtifact(dep, "https://registry.npmjs.org/react/-/react-18.2.0.tgz") - if dep.Metadata == nil { - t.Fatal("metadata map was not allocated") - } - - dep.Metadata["unrelated"] = "keep me" - detectors.SetOriginArtifact(dep, "https://registry.npmjs.org/react/-/react-18.3.0.tgz") - if got := dep.Metadata["unrelated"]; got != "keep me" { - t.Fatalf("unrelated metadata = %v, want %q", got, "keep me") - } - if got := detectors.OriginFrom(dep.Metadata).ArtifactURL; got != "https://registry.npmjs.org/react/-/react-18.3.0.tgz" { - t.Fatalf("artifact URL = %q, want the most recent value", got) - } -} - -// Hosts are case-insensitive. Two lockfiles spelling one host differently name -// the same place and must not reconcile to a disagreement. -func TestOriginHostCaseIsCanonical(t *testing.T) { - upper := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginVCS(upper, "https://GitHub.com/Owner/Repo", "aaaabbbbccccddddeeeeffff0000111122223333") - lower := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginVCS(lower, "https://github.com/Owner/Repo", "aaaabbbbccccddddeeeeffff0000111122223333") - - if got := detectors.OriginFrom(upper.Metadata).VCSURL; got != "https://github.com/Owner/Repo" { - t.Fatalf("repository = %q, want a lowercased host and an untouched path", got) - } - - detectors.MergeOrigin(upper, lower) - if got := detectors.OriginFrom(upper.Metadata); got.Empty() { - t.Fatal("host casing alone must not read as a disagreement") - } - - // The path is case-sensitive, so these really are different locations. - left := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginArtifact(left, "https://example.test/Pkg-1.0.0.tgz") - right := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginArtifact(right, "https://example.test/pkg-1.0.0.tgz") - detectors.MergeOrigin(left, right) - if got := detectors.OriginFrom(left.Metadata); !got.Empty() { - t.Fatalf("origin = %+v, want a disagreement: the paths differ", got) - } -} - -// An explicit default port names the same origin as no port at all. -func TestOriginDefaultPortIsCanonical(t *testing.T) { - cases := []struct{ name, raw, want string }{ - {name: "https default port", raw: "https://example.test:443/pkg-1.0.0.tgz", want: "https://example.test/pkg-1.0.0.tgz"}, - {name: "http default port", raw: "http://example.test:80/pkg-1.0.0.tgz", want: "http://example.test/pkg-1.0.0.tgz"}, - {name: "a non-default port is part of the location", raw: "https://example.test:8443/pkg-1.0.0.tgz", want: "https://example.test:8443/pkg-1.0.0.tgz"}, - {name: "the highest usable port", raw: "https://example.test:65535/pkg-1.0.0.tgz", want: "https://example.test:65535/pkg-1.0.0.tgz"}, - {name: "a port nothing can connect to", raw: "https://example.test:99999/pkg-1.0.0.tgz", want: ""}, - {name: "port zero", raw: "https://example.test:0/pkg-1.0.0.tgz", want: ""}, - {name: "an IPv6 literal keeps its brackets", raw: "https://[2001:db8::1]:443/pkg-1.0.0.tgz", want: "https://[2001:db8::1]/pkg-1.0.0.tgz"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - dep := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginArtifact(dep, tc.raw) - if got := detectors.OriginFrom(dep.Metadata).ArtifactURL; got != tc.want { - t.Fatalf("artifact = %q, want %q", got, tc.want) - } - }) - } - - withPort := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginArtifact(withPort, "https://example.test:443/pkg-1.0.0.tgz") - without := &sdk.Dependency{ID: "pkg"} - detectors.SetOriginArtifact(without, "https://example.test/pkg-1.0.0.tgz") - detectors.MergeOrigin(withPort, without) - if detectors.OriginFrom(withPort.Metadata).Empty() { - t.Fatal("a default port alone must not read as a disagreement") - } -} - -func TestSetOriginNilDependency(t *testing.T) { - // Must not panic: detectors call these on nodes that may not exist. - detectors.SetOriginArtifact(nil, "https://registry.npmjs.org/react/-/react-18.2.0.tgz") - detectors.SetOriginVCS(nil, "https://github.com/owner/repo", "9f8e7d6") -} - -func TestOriginFromRevalidatesHandBuiltMetadata(t *testing.T) { - cases := []struct { - name string - metadata map[string]any - want detectors.Origin - }{ - {name: "nil metadata", metadata: nil}, - {name: "unrelated keys only", metadata: map[string]any{"npm": struct{}{}}}, - { - name: "credentialed artifact is dropped", - metadata: map[string]any{detectors.MetadataKeyOriginArtifactURL: "https://user:s3cret@nexus.corp/pkg.tgz"}, - }, - { - name: "local path is dropped", - metadata: map[string]any{detectors.MetadataKeyOriginVCSURL: "file:///home/someone/repo"}, - }, - { - name: "non-string value is dropped", - metadata: map[string]any{detectors.MetadataKeyOriginArtifactURL: 42}, - }, - { - name: "revision without a repository is dropped", - metadata: map[string]any{detectors.MetadataKeyOriginVCSRevision: "9f8e7d6"}, - }, - { - name: "artifact wins over repository", - metadata: map[string]any{ - detectors.MetadataKeyOriginArtifactURL: "https://registry.npmjs.org/react/-/react-18.2.0.tgz", - detectors.MetadataKeyOriginVCSURL: "https://github.com/facebook/react", - }, - want: detectors.Origin{ArtifactURL: "https://registry.npmjs.org/react/-/react-18.2.0.tgz"}, - }, - { - name: "query and fragment are stripped from a hand-built repository", - metadata: map[string]any{ - detectors.MetadataKeyOriginVCSURL: "https://github.com/owner/repo?rev=main#abc", - detectors.MetadataKeyOriginVCSRevision: "9f8e7d6", - }, - want: detectors.Origin{VCSURL: "https://github.com/owner/repo", VCSRevision: "9f8e7d6"}, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := detectors.OriginFrom(tc.metadata); got != tc.want { - t.Fatalf("OriginFrom() = %+v, want %+v", got, tc.want) - } - }) - } -} - -func TestOriginEmpty(t *testing.T) { - if !(detectors.Origin{}).Empty() { - t.Fatal("zero Origin should be empty") - } - if (detectors.Origin{ArtifactURL: "https://example.com/pkg.tgz"}).Empty() { - t.Fatal("artifact origin should not be empty") - } - if (detectors.Origin{VCSURL: "https://example.com/repo"}).Empty() { - t.Fatal("repository origin should not be empty") - } -} - -// One package can appear at several places in a dependency tree. Its -// occurrences must agree before an origin is published. -func TestMergeOrigin(t *testing.T) { - const ( - artifact = "https://registry.npmjs.org/react/-/react-18.2.0.tgz" - mirror = "https://npm.corp/mirror/react/-/react-18.2.0.tgz" - repo = "https://github.com/facebook/react" - ) - withArtifact := func(url string) *sdk.Dependency { - dep := &sdk.Dependency{ID: "react@18.2.0"} - detectors.SetOriginArtifact(dep, url) - return dep - } - - cases := []struct { - name string - existing *sdk.Dependency - duplicate *sdk.Dependency - want detectors.Origin - }{ - { - name: "occurrences agree", - existing: withArtifact(artifact), - duplicate: withArtifact(artifact), - want: detectors.Origin{ArtifactURL: artifact}, - }, - { - name: "occurrences disagree, so the graph cannot say", - existing: withArtifact(artifact), - duplicate: withArtifact(mirror), - }, - { - name: "a different kind of origin is also a disagreement", - existing: withArtifact(artifact), - duplicate: func() *sdk.Dependency { - d := &sdk.Dependency{ID: "react@18.2.0"} - detectors.SetOriginVCS(d, repo, "") - return d - }(), - }, - { - name: "an occurrence asserting nothing is not a disagreement", - existing: withArtifact(artifact), - duplicate: &sdk.Dependency{ID: "react@18.2.0"}, - want: detectors.Origin{ArtifactURL: artifact}, - }, - { - name: "an occurrence fills a gap the first one left", - existing: &sdk.Dependency{ID: "react@18.2.0"}, - duplicate: withArtifact(artifact), - want: detectors.Origin{ArtifactURL: artifact}, - }, - { - name: "neither asserts anything", - existing: &sdk.Dependency{ID: "react@18.2.0"}, - duplicate: &sdk.Dependency{ID: "react@18.2.0"}, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - detectors.MergeOrigin(tc.existing, tc.duplicate) - if got := detectors.OriginFrom(tc.existing.Metadata); got != tc.want { - t.Fatalf("merged origin = %+v, want %+v", got, tc.want) - } - }) - } - - t.Run("a pinned repository merging with an unpinned one disagrees", func(t *testing.T) { - existing := &sdk.Dependency{ID: "react@18.2.0"} - detectors.SetOriginVCS(existing, repo, "e7f8091a2b3c4d5e6f708192a3b4c5d6e7f80912") - duplicate := &sdk.Dependency{ID: "react@18.2.0"} - detectors.SetOriginVCS(duplicate, repo, "") - - detectors.MergeOrigin(existing, duplicate) - if got := detectors.OriginFrom(existing.Metadata); !got.Empty() { - t.Fatalf("merged origin = %+v, want none: the occurrences pin different commits", got) - } - }) - - // A disagreement is a fact about the package, not about the pair of - // occurrences that exposed it: a third copy repeating one of the disputed - // values must not settle it. - t.Run("a disagreement stays cancelled", func(t *testing.T) { - existing := withArtifact(artifact) - detectors.MergeOrigin(existing, withArtifact(mirror)) - detectors.MergeOrigin(existing, withArtifact(artifact)) - - if got := detectors.OriginFrom(existing.Metadata); !got.Empty() { - t.Fatalf("origin = %+v, want none: occurrences A, B, A never agreed", got) - } - }) - - t.Run("a cancelled origin does not spread by absence", func(t *testing.T) { - existing := withArtifact(artifact) - detectors.MergeOrigin(existing, withArtifact(mirror)) - detectors.MergeOrigin(existing, &sdk.Dependency{ID: "react@18.2.0"}) - - if got := detectors.OriginFrom(existing.Metadata); !got.Empty() { - t.Fatalf("origin = %+v, want none", got) - } - }) - - t.Run("a conflicted duplicate cancels an agreed origin", func(t *testing.T) { - conflicted := withArtifact(artifact) - detectors.MergeOrigin(conflicted, withArtifact(mirror)) - - existing := withArtifact(artifact) - detectors.MergeOrigin(existing, conflicted) - - if got := detectors.OriginFrom(existing.Metadata); !got.Empty() { - t.Fatalf("origin = %+v, want none: the duplicate had already disagreed with itself", got) - } - }) - - // Merging folds occurrences together; a detector setting an origin is - // asserting what it resolved, which is authoritative. - t.Run("a detector assertion supersedes a recorded disagreement", func(t *testing.T) { - dep := withArtifact(artifact) - detectors.MergeOrigin(dep, withArtifact(mirror)) - detectors.SetOriginArtifact(dep, mirror) - - want := detectors.Origin{ArtifactURL: mirror} - if got := detectors.OriginFrom(dep.Metadata); got != want { - t.Fatalf("origin = %+v, want %+v", got, want) - } - }) - - // Reconciling several occurrences leaves every one of them carrying the - // verdict, including the record of a disagreement -- otherwise a later - // fold against a node that still asserts something would revive it. - t.Run("every occurrence carries the settled verdict", func(t *testing.T) { - occurrences := []*sdk.Dependency{withArtifact(artifact), withArtifact(mirror), withArtifact(artifact)} - detectors.ReconcileOrigins(occurrences) - - for i, occurrence := range occurrences { - if got := detectors.OriginFrom(occurrence.Metadata); !got.Empty() { - t.Fatalf("occurrence %d = %+v, want none", i, got) - } - // Whichever occurrence a later merge keeps must stay settled. - detectors.MergeOrigin(occurrence, withArtifact(artifact)) - if got := detectors.OriginFrom(occurrence.Metadata); !got.Empty() { - t.Fatalf("occurrence %d revived %+v after the disagreement was settled", i, got) - } - } - }) - - t.Run("agreement is broadcast to every occurrence", func(t *testing.T) { - occurrences := []*sdk.Dependency{&sdk.Dependency{ID: "react@18.2.0"}, withArtifact(artifact), &sdk.Dependency{ID: "react@18.2.0"}} - detectors.ReconcileOrigins(occurrences) - - want := detectors.Origin{ArtifactURL: artifact} - for i, occurrence := range occurrences { - if got := detectors.OriginFrom(occurrence.Metadata); got != want { - t.Fatalf("occurrence %d = %+v, want %+v", i, got, want) - } - } - }) - - // Exported, so a caller can hand it anything. - t.Run("nil occurrences are skipped", func(t *testing.T) { - detectors.ReconcileOrigins([]*sdk.Dependency{nil, withArtifact(artifact)}) - detectors.ReconcileOrigins([]*sdk.Dependency{withArtifact(artifact), nil}) - detectors.ReconcileOrigins([]*sdk.Dependency{nil, nil}) - - occurrences := []*sdk.Dependency{withArtifact(artifact), nil, withArtifact(mirror)} - detectors.ReconcileOrigins(occurrences) - if got := detectors.OriginFrom(occurrences[0].Metadata); !got.Empty() { - t.Fatalf("origin = %+v, want none: the non-nil occurrences disagreed", got) - } - }) - - t.Run("nil is a no-op", func(t *testing.T) { - detectors.MergeOrigin(nil, withArtifact(artifact)) - detectors.MergeOrigin(withArtifact(artifact), nil) - }) -} diff --git a/internal/detectors/pub/detector.go b/internal/detectors/pub/detector.go index e442963f..6fa62e27 100644 --- a/internal/detectors/pub/detector.go +++ b/internal/detectors/pub/detector.go @@ -201,7 +201,7 @@ func packageNode(name string, pkg pubLockPackage) *sdk.Dependency { if pubDependencySource(pkg.Source) == sdk.DependencySourceGit { // A git package names its repository and the commit pub resolved. // A hosted package's "url" is the pub server, and path is local. - detectors.SetOriginVCS(node, descriptionString(pkg.Description, "url"), descriptionString(pkg.Description, "resolved-ref")) + node.Origin = sdk.RepositoryOrigin(descriptionString(pkg.Description, "url"), descriptionString(pkg.Description, "resolved-ref")) } return node } diff --git a/internal/detectors/pub/origin_test.go b/internal/detectors/pub/origin_test.go index f285cdf4..0176c470 100644 --- a/internal/detectors/pub/origin_test.go +++ b/internal/detectors/pub/origin_test.go @@ -5,11 +5,22 @@ import ( "path/filepath" "testing" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" "go.uber.org/zap" ) +// originOf returns the origin a node publishes, or the zero value when it has +// none, so cases can compare plain structs. +func originOf(dep *sdk.Dependency) sdk.PackageOrigin { + if dep == nil { + return sdk.PackageOrigin{} + } + if origin := dep.Origin.Normalized(); origin != nil { + return *origin + } + return sdk.PackageOrigin{} +} + // A pubspec.lock hosted package's description URL is the pub server, shared by // every hosted package, and a path package is local. Only a git package names // where its own code came from. @@ -62,15 +73,15 @@ func TestPubOriginBySourceType(t *testing.T) { cases := []struct { id string - want detectors.Origin + want sdk.PackageOrigin }{ {id: "collection@1.18.0"}, // A self-hosted pub server's URL has a path, so nothing but the // source kind distinguishes it from a repository URL. {id: "corp_widgets@3.1.0"}, - {id: "helper@2.0.0", want: detectors.Origin{ - VCSURL: "https://github.com/example/helper.git", - VCSRevision: "a3b4c5d6e7f8091a2b3c4d5e6f70819213243546", + {id: "helper@2.0.0", want: sdk.PackageOrigin{ + Repository: "https://github.com/example/helper.git", + Revision: "a3b4c5d6e7f8091a2b3c4d5e6f70819213243546", }}, {id: "local_tools@0.1.0"}, } @@ -79,7 +90,7 @@ func TestPubOriginBySourceType(t *testing.T) { if !ok { t.Fatalf("expected %s in graph", tc.id) } - if got := detectors.OriginFrom(node.Metadata); got != tc.want { + if got := originOf(node); got != tc.want { t.Errorf("%s origin = %+v, want %+v", tc.id, got, tc.want) } } @@ -137,13 +148,13 @@ func TestPubNativeOriginIsReadFromPubspecLock(t *testing.T) { t.Fatalf("nativeGraph() error = %v", err) } - want := detectors.Origin{ - VCSURL: "https://github.com/example/helper.git", - VCSRevision: "1a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d", + want := sdk.PackageOrigin{ + Repository: "https://github.com/example/helper.git", + Revision: "1a2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d", } var checked int g.WalkNodes(func(dep *sdk.Dependency) bool { - origin := detectors.OriginFrom(dep.Metadata) + origin := originOf(dep) switch dep.Name { case "helper": checked++ @@ -201,7 +212,7 @@ func TestPubOverriddenPackageIsNotCreditedToTheLockedRepository(t *testing.T) { return true } checked++ - if got := detectors.OriginFrom(dep.Metadata); !got.Empty() { + if got := originOf(dep); !got.Empty() { t.Fatalf("overridden package origin = %+v, want none", got) } return true @@ -231,7 +242,7 @@ func TestPubNativeOriginSurvivesMissingLock(t *testing.T) { if dep.Name == "helper" { checked++ } - if got := detectors.OriginFrom(dep.Metadata); !got.Empty() { + if got := originOf(dep); !got.Empty() { t.Fatalf("%s origin = %+v, want none", dep.Name, got) } return true diff --git a/internal/detectors/pub/pub_native.go b/internal/detectors/pub/pub_native.go index 42fb0b8a..9d555d67 100644 --- a/internal/detectors/pub/pub_native.go +++ b/internal/detectors/pub/pub_native.go @@ -135,7 +135,7 @@ func applyLockOrigins(g *sdk.Graph, workingDir string, logger *zap.Logger) { if !ok || pubDependencySource(pkg.Source) != sdk.DependencySourceGit { return true } - detectors.SetOriginVCS(dep, descriptionString(pkg.Description, "url"), descriptionString(pkg.Description, "resolved-ref")) + dep.Origin = sdk.RepositoryOrigin(descriptionString(pkg.Description, "url"), descriptionString(pkg.Description, "resolved-ref")) recorded++ return true }) diff --git a/internal/detectors/python/origin.go b/internal/detectors/python/origin.go index 29587738..55f362d2 100644 --- a/internal/detectors/python/origin.go +++ b/internal/detectors/python/origin.go @@ -3,7 +3,6 @@ package python import ( "strings" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) @@ -19,9 +18,9 @@ func setUVOrigin(node *sdk.Dependency, source uvLockSource) { // uv writes the resolved commit as the URL fragment and the // requested ref as a query parameter; uvSourceRevision prefers the // former, and the invariant drops both from the repository URL. - detectors.SetOriginVCS(node, source.Git, uvSourceRevision(source)) + node.Origin = sdk.RepositoryOrigin(source.Git, uvSourceRevision(source)) case strings.TrimSpace(source.URL) != "": - detectors.SetOriginArtifact(node, source.URL) + node.Origin = sdk.ArtifactOrigin(source.URL) } } @@ -31,9 +30,9 @@ func setPoetryOrigin(node *sdk.Dependency, pkg *poetryLockPackage) { case "git": // ResolvedReference is the commit poetry locked; Reference is the // branch or tag that was asked for. - detectors.SetOriginVCS(node, pkg.Source.URL, firstNonEmpty(pkg.Source.ResolvedReference, pkg.Source.Reference)) + node.Origin = sdk.RepositoryOrigin(pkg.Source.URL, firstNonEmpty(pkg.Source.ResolvedReference, pkg.Source.Reference)) case "url": - detectors.SetOriginArtifact(node, pkg.Source.URL) + node.Origin = sdk.ArtifactOrigin(pkg.Source.URL) } } @@ -41,11 +40,11 @@ func setPoetryOrigin(node *sdk.Dependency, pkg *poetryLockPackage) { func setPipenvOrigin(node *sdk.Dependency, pkg pipfileLockPackage) { switch { case strings.TrimSpace(pkg.Git) != "": - detectors.SetOriginVCS(node, pkg.Git, pkg.Ref) + node.Origin = sdk.RepositoryOrigin(pkg.Git, pkg.Ref) case strings.TrimSpace(pkg.File) != "": // "file" holds a remote archive for URL requirements and a local // path for file:// ones; the invariant keeps only the former. - detectors.SetOriginArtifact(node, pkg.File) + node.Origin = sdk.ArtifactOrigin(pkg.File) } } @@ -60,13 +59,13 @@ func setPipInspectOrigin(node *sdk.Dependency, directURL map[string]any) { if vcsInfo, ok := directURL["vcs_info"].(map[string]any); ok { vcs, _ := vcsInfo["vcs"].(string) if strings.EqualFold(strings.TrimSpace(vcs), "git") { - detectors.SetOriginVCS(node, resolved, pipInspectRevision(directURL)) + node.Origin = sdk.RepositoryOrigin(resolved, pipInspectRevision(directURL)) } // Mercurial, Subversion, and Bazaar have no locator form here. return } if _, ok := directURL["archive_info"]; ok { - detectors.SetOriginArtifact(node, resolved) + node.Origin = sdk.ArtifactOrigin(resolved) } // dir_info marks a local directory install. } diff --git a/internal/detectors/python/origin_test.go b/internal/detectors/python/origin_test.go index 1c094d4e..689dc347 100644 --- a/internal/detectors/python/origin_test.go +++ b/internal/detectors/python/origin_test.go @@ -5,18 +5,29 @@ import ( "path/filepath" "testing" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) +// originOf returns the origin a node publishes, or the zero value when it has +// none, so cases can compare plain structs. +func originOf(dep *sdk.Dependency) sdk.PackageOrigin { + if dep == nil { + return sdk.PackageOrigin{} + } + if origin := dep.Origin.Normalized(); origin != nil { + return *origin + } + return sdk.PackageOrigin{} +} + // requireOrigin asserts the exact origin a named package asserts. -func requireOrigin(t *testing.T, graph *sdk.Graph, id string, want detectors.Origin) { +func requireOrigin(t *testing.T, graph *sdk.Graph, id string, want sdk.PackageOrigin) { t.Helper() node, ok := graph.Node(id) if !ok { t.Fatalf("expected %s in graph", id) } - if got := detectors.OriginFrom(node.Metadata); got != want { + if got := originOf(node); got != want { t.Errorf("%s origin = %+v, want %+v", id, got, want) } } @@ -63,16 +74,16 @@ source = { path = "../vendor/from-path" } // The fragment carries the commit uv resolved; the "rev" query carries // what the manifest asked for. - requireOrigin(t, graph, "from-git@1.0.0", detectors.Origin{ - VCSURL: "https://github.com/example/from-git", - VCSRevision: "9f8e7d6c5b4a3928176554433221100ffeeddcc0", + requireOrigin(t, graph, "from-git@1.0.0", sdk.PackageOrigin{ + Repository: "https://github.com/example/from-git", + Revision: "9f8e7d6c5b4a3928176554433221100ffeeddcc0", }) - requireOrigin(t, graph, "from-url@2.0.0", detectors.Origin{ + requireOrigin(t, graph, "from-url@2.0.0", sdk.PackageOrigin{ ArtifactURL: "https://files.pythonhosted.org/packages/ab/from_url-2.0.0-py3-none-any.whl", }) // An index root is not this package's origin, and a path is local. - requireOrigin(t, graph, "from-registry@3.0.0", detectors.Origin{}) - requireOrigin(t, graph, "from-path@4.0.0", detectors.Origin{}) + requireOrigin(t, graph, "from-registry@3.0.0", sdk.PackageOrigin{}) + requireOrigin(t, graph, "from-path@4.0.0", sdk.PackageOrigin{}) } func TestPoetryLockOriginBySourceType(t *testing.T) { @@ -123,17 +134,17 @@ url = "../vendor/from-directory" t.Fatalf("depGraphFromPoetryLock() error = %v", err) } - requireOrigin(t, graph, "from-pypi@1.0.0", detectors.Origin{}) + requireOrigin(t, graph, "from-pypi@1.0.0", sdk.PackageOrigin{}) // resolved_reference is the commit poetry locked; reference is the branch. - requireOrigin(t, graph, "from-git@2.0.0", detectors.Origin{ - VCSURL: "https://github.com/example/from-git.git", - VCSRevision: "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567", + requireOrigin(t, graph, "from-git@2.0.0", sdk.PackageOrigin{ + Repository: "https://github.com/example/from-git.git", + Revision: "0a1b2c3d4e5f60718293a4b5c6d7e8f901234567", }) - requireOrigin(t, graph, "from-url@3.0.0", detectors.Origin{ + requireOrigin(t, graph, "from-url@3.0.0", sdk.PackageOrigin{ ArtifactURL: "https://files.pythonhosted.org/packages/cd/from_url-3.0.0.tar.gz", }) - requireOrigin(t, graph, "from-private-index@4.0.0", detectors.Origin{}) - requireOrigin(t, graph, "from-directory@5.0.0", detectors.Origin{}) + requireOrigin(t, graph, "from-private-index@4.0.0", sdk.PackageOrigin{}) + requireOrigin(t, graph, "from-directory@5.0.0", sdk.PackageOrigin{}) } func TestPipfileLockOriginBySourceType(t *testing.T) { @@ -158,16 +169,16 @@ func TestPipfileLockOriginBySourceType(t *testing.T) { t.Fatalf("depGraphFromPipfileLock() error = %v", err) } - requireOrigin(t, graph, "from-pypi@1.0.0", detectors.Origin{}) - requireOrigin(t, graph, "from-git", detectors.Origin{ - VCSURL: "https://github.com/example/from-git.git", - VCSRevision: "1f2e3d4c5b6a79880912a3b4c5d6e7f809172635", + requireOrigin(t, graph, "from-pypi@1.0.0", sdk.PackageOrigin{}) + requireOrigin(t, graph, "from-git", sdk.PackageOrigin{ + Repository: "https://github.com/example/from-git.git", + Revision: "1f2e3d4c5b6a79880912a3b4c5d6e7f809172635", }) - requireOrigin(t, graph, "from-archive", detectors.Origin{ + requireOrigin(t, graph, "from-archive", sdk.PackageOrigin{ ArtifactURL: "https://files.pythonhosted.org/packages/ef/from_archive-2.0.0.tar.gz", }) - requireOrigin(t, graph, "from-local", detectors.Origin{}) - requireOrigin(t, graph, "from-path", detectors.Origin{}) + requireOrigin(t, graph, "from-local", sdk.PackageOrigin{}) + requireOrigin(t, graph, "from-path", sdk.PackageOrigin{}) } // pip records a PEP 610 direct_url.json for anything not installed from an @@ -176,7 +187,7 @@ func TestPipInspectOriginByDirectURLShape(t *testing.T) { cases := []struct { name string directURL map[string]any - want detectors.Origin + want sdk.PackageOrigin }{ {name: "index install", directURL: nil}, { @@ -185,12 +196,12 @@ func TestPipInspectOriginByDirectURLShape(t *testing.T) { "url": "https://github.com/example/pkg.git", "vcs_info": map[string]any{"vcs": "git", "commit_id": "2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e", "requested_revision": "main"}, }, - want: detectors.Origin{VCSURL: "https://github.com/example/pkg.git", VCSRevision: "2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e"}, + want: sdk.PackageOrigin{Repository: "https://github.com/example/pkg.git", Revision: "2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c4d5e"}, }, { name: "archive URL", directURL: map[string]any{"url": "https://example.test/pkg-1.0.0-py3-none-any.whl", "archive_info": map[string]any{}}, - want: detectors.Origin{ArtifactURL: "https://example.test/pkg-1.0.0-py3-none-any.whl"}, + want: sdk.PackageOrigin{ArtifactURL: "https://example.test/pkg-1.0.0-py3-none-any.whl"}, }, { name: "local directory", @@ -206,7 +217,7 @@ func TestPipInspectOriginByDirectURLShape(t *testing.T) { t.Run(tc.name, func(t *testing.T) { node := sdk.NewDependency(sdk.Dependency{Coordinates: sdk.Coordinates{Name: "pkg", Version: "1.0.0"}}) setPipInspectOrigin(node, tc.directURL) - if got := detectors.OriginFrom(node.Metadata); got != tc.want { + if got := originOf(node); got != tc.want { t.Fatalf("origin = %+v, want %+v", got, tc.want) } }) diff --git a/internal/detectors/ruby/detector.go b/internal/detectors/ruby/detector.go index 6369aa54..560a2b80 100644 --- a/internal/detectors/ruby/detector.go +++ b/internal/detectors/ruby/detector.go @@ -498,7 +498,7 @@ func gemNode(spec lockSpec) *sdk.Dependency { if spec.Source == sdk.DependencySourceGit { // A GIT section names the repository and the commit Bundler locked. // A GEM section's remote is the gem server, and PATH is local. - detectors.SetOriginVCS(node, spec.ResolvedURL, spec.Revision) + node.Origin = sdk.RepositoryOrigin(spec.ResolvedURL, spec.Revision) } return node diff --git a/internal/detectors/ruby/origin_test.go b/internal/detectors/ruby/origin_test.go index 99dd3577..5e550524 100644 --- a/internal/detectors/ruby/origin_test.go +++ b/internal/detectors/ruby/origin_test.go @@ -1,11 +1,22 @@ package ruby import ( + "github.com/bomly-dev/bomly-sdk" "testing" - - "github.com/bomly-dev/bomly-cli/internal/detectors" ) +// originOf returns the origin a node publishes, or the zero value when it has +// none, so cases can compare plain structs. +func originOf(dep *sdk.Dependency) sdk.PackageOrigin { + if dep == nil { + return sdk.PackageOrigin{} + } + if origin := dep.Origin.Normalized(); origin != nil { + return *origin + } + return sdk.PackageOrigin{} +} + // A Gemfile.lock names its sources by section. GEM's remote is the gem server // every gem in that section came from, not a per-gem location; PATH is a // directory on the machine that ran bundle install. Only GIT identifies where @@ -46,15 +57,15 @@ DEPENDENCIES cases := []struct { id string - want detectors.Origin + want sdk.PackageOrigin }{ {id: "rack@3.1.8"}, // A private gem server's remote has a path, so nothing but the // section kind distinguishes it from a repository URL. {id: "corp-auth@2.4.0"}, - {id: "helper@1.0.0", want: detectors.Origin{ - VCSURL: "https://github.com/example/helper.git", - VCSRevision: "708192a3b4c5d6e7f8091a2b3c4d5e6f70819213", + {id: "helper@1.0.0", want: sdk.PackageOrigin{ + Repository: "https://github.com/example/helper.git", + Revision: "708192a3b4c5d6e7f8091a2b3c4d5e6f70819213", }}, {id: "local-gem@0.1.0"}, } @@ -63,7 +74,7 @@ DEPENDENCIES if !ok { t.Fatalf("expected %s in graph", tc.id) } - if got := detectors.OriginFrom(node.Metadata); got != tc.want { + if got := originOf(node); got != tc.want { t.Errorf("%s origin = %+v, want %+v", tc.id, got, tc.want) } } diff --git a/internal/detectors/swiftpm/detector.go b/internal/detectors/swiftpm/detector.go index fd21fb1d..77f1a520 100644 --- a/internal/detectors/swiftpm/detector.go +++ b/internal/detectors/swiftpm/detector.go @@ -279,7 +279,7 @@ func packageNode(pkg swiftPackage) *sdk.Dependency { // Source-control pins name the repository and the commit SwiftPM // resolved. Registry pins are identity-only, and local packages // point at a checkout on this machine. - detectors.SetOriginVCS(node, pkg.Repository, pkg.Revision) + node.Origin = sdk.RepositoryOrigin(pkg.Repository, pkg.Revision) } // SwiftPM does not distinguish dev scope; all packages are runtime. diff --git a/internal/detectors/swiftpm/origin_test.go b/internal/detectors/swiftpm/origin_test.go index cb560c94..e34fb970 100644 --- a/internal/detectors/swiftpm/origin_test.go +++ b/internal/detectors/swiftpm/origin_test.go @@ -5,11 +5,22 @@ import ( "path/filepath" "testing" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" "go.uber.org/zap" ) +// originOf returns the origin a node publishes, or the zero value when it has +// none, so cases can compare plain structs. +func originOf(dep *sdk.Dependency) sdk.PackageOrigin { + if dep == nil { + return sdk.PackageOrigin{} + } + if origin := dep.Origin.Normalized(); origin != nil { + return *origin + } + return sdk.PackageOrigin{} +} + // A Package.resolved pin says how SwiftPM obtained a package. Source-control // pins name a repository and the commit that was resolved; registry pins are // identity-only; local pins point at a checkout on this machine. @@ -44,13 +55,13 @@ func TestSwiftPMOriginByPinKind(t *testing.T) { var checked int for _, node := range graph.Nodes() { - origin := detectors.OriginFrom(node.Metadata) + origin := originOf(node) switch node.Name { case "swift-argument-parser": checked++ - want := detectors.Origin{ - VCSURL: "https://github.com/apple/swift-argument-parser.git", - VCSRevision: "8192a3b4c5d6e7f8091a2b3c4d5e6f7081921324", + want := sdk.PackageOrigin{ + Repository: "https://github.com/apple/swift-argument-parser.git", + Revision: "8192a3b4c5d6e7f8091a2b3c4d5e6f7081921324", } if origin != want { t.Errorf("%s origin = %+v, want %+v", node.Name, origin, want) @@ -111,13 +122,13 @@ func TestSwiftPMNativeOriginIsPinnedFromPackageResolved(t *testing.T) { t.Fatalf("nativeGraph() error = %v", err) } - want := detectors.Origin{ - VCSURL: "https://github.com/apple/swift-argument-parser.git", - VCSRevision: "f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b", + want := sdk.PackageOrigin{ + Repository: "https://github.com/apple/swift-argument-parser.git", + Revision: "f8091a2b3c4d5e6f708192a3b4c5d6e7f8091a2b", } var checked int g.WalkNodes(func(dep *sdk.Dependency) bool { - origin := detectors.OriginFrom(dep.Metadata) + origin := originOf(dep) switch dep.Name { case "swift-argument-parser": checked++ @@ -178,7 +189,7 @@ func TestSwiftPMEditedPackageIsNotCreditedToItsFormerPin(t *testing.T) { return true } checked++ - if got := detectors.OriginFrom(dep.Metadata); !got.Empty() { + if got := originOf(dep); !got.Empty() { t.Fatalf("edited package origin = %+v, want none: it is built from a local checkout", got) } return true @@ -204,12 +215,12 @@ func TestSwiftPMNativeOriginSurvivesMissingPackageResolved(t *testing.T) { t.Fatalf("nativeGraph() error = %v", err) } - want := detectors.Origin{VCSURL: "https://github.com/apple/swift-argument-parser.git"} + want := sdk.PackageOrigin{Repository: "https://github.com/apple/swift-argument-parser.git"} var checked int g.WalkNodes(func(dep *sdk.Dependency) bool { if dep.Name == "swift-argument-parser" { checked++ - if got := detectors.OriginFrom(dep.Metadata); got != want { + if got := originOf(dep); got != want { t.Fatalf("origin = %+v, want the unpinned repository %+v", got, want) } } @@ -286,7 +297,7 @@ func TestSwiftPMNativeOriginDoesNotMatchAcrossPathCase(t *testing.T) { t.Fatalf("nativeGraph() error = %v", err) } g.WalkNodes(func(dep *sdk.Dependency) bool { - if origin := detectors.OriginFrom(dep.Metadata); origin.VCSRevision == "5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f7081" { + if origin := originOf(dep); origin.Revision == "5e6f708192a3b4c5d6e7f8091a2b3c4d5e6f7081" { t.Fatalf("%s took a pin belonging to a differently-cased repository: %+v", dep.Name, origin) } return true diff --git a/internal/detectors/swiftpm/swiftpm_native.go b/internal/detectors/swiftpm/swiftpm_native.go index cd7eb023..2427bc84 100644 --- a/internal/detectors/swiftpm/swiftpm_native.go +++ b/internal/detectors/swiftpm/swiftpm_native.go @@ -148,7 +148,7 @@ func applyResolvedOrigins(g *sdk.Graph, workingDir string, logger *zap.Logger) { if pin.Revision == "" || swiftDependencySource(pin.SourceKind, pin.Repository) != sdk.DependencySourceGit { return true } - detectors.SetOriginVCS(dep, pin.Repository, pin.Revision) + dep.Origin = sdk.RepositoryOrigin(pin.Repository, pin.Revision) pinned++ return true }) diff --git a/internal/engine/consolidation/consolidation.go b/internal/engine/consolidation/consolidation.go index 2c4de133..45cde7b1 100644 --- a/internal/engine/consolidation/consolidation.go +++ b/internal/engine/consolidation/consolidation.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) @@ -40,34 +39,9 @@ func ConsolidateGraphs(results []sdk.DetectionResult) (sdk.ConsolidatedGraph, er } consolidated.Subprojects[idx].RootManifestIDs = append(consolidated.Subprojects[idx].RootManifestIDs, selected.RootManifestID) } - reconcileEntryOrigins(consolidated.Graphs.Entries) return consolidated, nil } -// reconcileEntryOrigins settles package origin across the selected manifests -// before their graphs are merged into one. Each manifest is resolved on its -// own, so one package used by two subprojects arrives as two nodes; merging -// keeps the first and drops the rest, which would publish one subproject's -// answer for a package the scan saw resolved two different ways. -func reconcileEntryOrigins(entries []sdk.GraphEntry) { - if len(entries) < 2 { - return - } - occurrences := make(map[string][]*sdk.Dependency) - for _, entry := range entries { - if entry.Graph == nil { - continue - } - entry.Graph.WalkNodes(func(node *sdk.Dependency) bool { - occurrences[node.ID] = append(occurrences[node.ID], node) - return true - }) - } - for _, nodes := range occurrences { - detectors.ReconcileOrigins(nodes) - } -} - type consolidatedEntryCandidate struct { entry sdk.GraphEntry subproject sdk.Subproject diff --git a/internal/engine/consolidation/enrichment.go b/internal/engine/consolidation/enrichment.go index 6b917dea..09c3c4fb 100644 --- a/internal/engine/consolidation/enrichment.go +++ b/internal/engine/consolidation/enrichment.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) @@ -43,7 +42,7 @@ func normalizeGraphPackageIdentity(src *sdk.Graph) (*sdk.Graph, error) { // survives, so the discarded occurrence still gets a say about // where the package came from -- otherwise a lockfile recording // one package from two mirrors publishes whichever came first. - detectors.MergeOrigin(existing, clone) + existing.Origin = sdk.ReconcileOrigin(existing.Origin, clone.Origin) } idMapping[node.ID] = clone.ID } diff --git a/internal/engine/consolidation/origin_test.go b/internal/engine/consolidation/origin_test.go index 82671c52..070d64f5 100644 --- a/internal/engine/consolidation/origin_test.go +++ b/internal/engine/consolidation/origin_test.go @@ -4,10 +4,21 @@ import ( "fmt" "testing" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) +// originOf returns the origin a node publishes, or the zero value when it has +// none, so cases can compare plain structs. +func originOf(dep *sdk.Dependency) sdk.PackageOrigin { + if dep == nil { + return sdk.PackageOrigin{} + } + if origin := dep.Origin.Normalized(); origin != nil { + return *origin + } + return sdk.PackageOrigin{} +} + // subprojectResult builds one manifest's detection result carrying a single // package whose origin the caller chooses. func subprojectResult(t *testing.T, relativePath, manifest, artifactURL string) sdk.DetectionResult { @@ -17,7 +28,7 @@ func subprojectResult(t *testing.T, relativePath, manifest, artifactURL string) pkg := sdk.NewDependencyWithID("lodash@4.17.21", sdk.Dependency{Coordinates: sdk.Coordinates{ Name: "lodash", Version: "4.17.21", Ecosystem: sdk.EcosystemNPM, PURL: "pkg:npm/lodash@4.17.21"}}) if artifactURL != "" { - detectors.SetOriginArtifact(pkg, artifactURL) + pkg.Origin = sdk.ArtifactOrigin(artifactURL) } if err := g.AddNode(pkg); err != nil { t.Fatal(err) @@ -58,13 +69,13 @@ func TestConsolidateGraphsSettlesOriginAcrossManifests(t *testing.T) { name string left string right string - want detectors.Origin + want sdk.PackageOrigin }{ { name: "subprojects agree", left: public, right: public, - want: detectors.Origin{ArtifactURL: public}, + want: sdk.PackageOrigin{ArtifactURL: public}, }, { name: "one subproject resolved a private mirror", @@ -75,7 +86,7 @@ func TestConsolidateGraphsSettlesOriginAcrossManifests(t *testing.T) { name: "one subproject recorded nothing", left: public, right: "", - want: detectors.Origin{ArtifactURL: public}, + want: sdk.PackageOrigin{ArtifactURL: public}, }, } @@ -103,37 +114,13 @@ func TestConsolidateGraphsSettlesOriginAcrossManifests(t *testing.T) { if node == nil { t.Fatalf("expected lodash in the merged graph; ids present: %v", graphIDs(merged)) } - if got := detectors.OriginFrom(node.Metadata); got != tc.want { + if got := originOf(node); got != tc.want { t.Fatalf("merged origin = %+v, want %+v", got, tc.want) } }) } } -// The surviving node is whichever the merge happens to keep, so every -// occurrence has to carry the settled answer, not just the first. -func TestConsolidateGraphsSettlesEveryOccurrence(t *testing.T) { - consolidated, err := ConsolidateGraphs([]sdk.DetectionResult{ - subprojectResult(t, "apps/web", "apps/web/package-lock.json", "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"), - subprojectResult(t, "services/api", "services/api/package-lock.json", "https://npm.corp/mirror/lodash/-/lodash-4.17.21.tgz"), - }) - if err != nil { - t.Fatalf("ConsolidateGraphs() error = %v", err) - } - - for _, entry := range consolidated.Graphs.Entries { - if entry.Graph == nil { - continue - } - entry.Graph.WalkNodes(func(node *sdk.Dependency) bool { - if got := detectors.OriginFrom(node.Metadata); !got.Empty() { - t.Errorf("%s still claims %+v after the subprojects disagreed", node.ID, got) - } - return true - }) - } -} - // One manifest can record a package twice with different locations -- a Bun // lockfile listing one name and version from two mirrors. Both nodes normalize // to one canonical identity and only one survives, so the disagreement has to @@ -149,11 +136,11 @@ func TestConsolidateGraphsSettlesOriginWithinOneManifest(t *testing.T) { name string left string right string - want detectors.Origin + want sdk.PackageOrigin }{ - {name: "entries agree", left: public, right: public, want: detectors.Origin{ArtifactURL: public}}, + {name: "entries agree", left: public, right: public, want: sdk.PackageOrigin{ArtifactURL: public}}, {name: "entries disagree", left: public, right: private}, - {name: "one entry says nothing", left: public, right: "", want: detectors.Origin{ArtifactURL: public}}, + {name: "one entry says nothing", left: public, right: "", want: sdk.PackageOrigin{ArtifactURL: public}}, } for _, tc := range cases { @@ -168,7 +155,7 @@ func TestConsolidateGraphsSettlesOriginWithinOneManifest(t *testing.T) { Name: "lodash", Version: "4.17.21", Ecosystem: sdk.EcosystemNPM, PURL: "pkg:npm/lodash@4.17.21"}}, ) if artifactURL != "" { - detectors.SetOriginArtifact(pkg, artifactURL) + pkg.Origin = sdk.ArtifactOrigin(artifactURL) } if err := g.AddNode(pkg); err != nil { t.Fatal(err) @@ -200,7 +187,7 @@ func TestConsolidateGraphsSettlesOriginWithinOneManifest(t *testing.T) { return true } checked++ - if got := detectors.OriginFrom(dep.Metadata); got != tc.want { + if got := originOf(dep); got != tc.want { t.Fatalf("origin = %+v, want %+v", got, tc.want) } return true diff --git a/internal/output/origin_metadata_test.go b/internal/output/origin_metadata_test.go deleted file mode 100644 index 7d4144a5..00000000 --- a/internal/output/origin_metadata_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package output - -import ( - "testing" - - "github.com/bomly-dev/bomly-cli/internal/detectors" - "github.com/bomly-dev/bomly-sdk" -) - -// Origin metadata is a transport between detection and SBOM export. It must not -// surface in scan/diff/explain payloads, where it would be noise in every -// package entry and churn every golden. -func TestPackageRefOmitsOriginMetadata(t *testing.T) { - dep := sdk.NewDependencyWithID("npm:react", sdk.Dependency{ - Coordinates: sdk.Coordinates{ - PURL: "pkg:npm/react@18.2.0", - Ecosystem: sdk.EcosystemNPM, - Name: "react", - Version: "18.2.0", - }, - }) - detectors.SetOriginArtifact(dep, "https://registry.npmjs.org/react/-/react-18.2.0.tgz") - dep.Metadata["npm"] = &sdk.NPMPackageMetadata{Bundled: true} - - ref := PackageFromDependencyAndRegistry(dep, nil) - - if _, found := ref.Metadata[detectors.MetadataKeyOriginArtifactURL]; found { - t.Fatalf("origin metadata reached command output: %v", ref.Metadata) - } - if _, found := ref.Metadata["npm"]; !found { - t.Fatalf("unrelated metadata was dropped: %v", ref.Metadata) - } -} - -// A dependency whose only metadata is origin must render no metadata object at -// all, or `omitempty` stops firing and every such package grows an empty block. -func TestPackageRefMetadataAbsentWhenOnlyOrigin(t *testing.T) { - dep := sdk.NewDependencyWithID("npm:react", sdk.Dependency{ - Coordinates: sdk.Coordinates{ - PURL: "pkg:npm/react@18.2.0", - Ecosystem: sdk.EcosystemNPM, - Name: "react", - Version: "18.2.0", - }, - }) - detectors.SetOriginVCS(dep, "https://github.com/facebook/react", "9f8e7d6c5b4a3928176554433221100ffeeddcc0") - - if ref := PackageFromDependencyAndRegistry(dep, nil); ref.Metadata != nil { - t.Fatalf("metadata = %v, want nil so the field is omitted", ref.Metadata) - } -} - -// The same filter guards the registry-sourced package listing. -func TestScanPackageEntriesOmitOriginMetadata(t *testing.T) { - registry := sdk.NewPackageRegistry() - registry.Add(&sdk.Package{ - Coordinates: sdk.Coordinates{ - PURL: "pkg:npm/react@18.2.0", - Ecosystem: sdk.EcosystemNPM, - Name: "react", - Version: "18.2.0", - }, - Metadata: map[string]any{ - detectors.MetadataKeyOriginArtifactURL: "https://registry.npmjs.org/react/-/react-18.2.0.tgz", - "npm": &sdk.NPMPackageMetadata{Bundled: true}, - }, - }) - - entries := PackagesFromRegistry(registry) - if len(entries) != 1 { - t.Fatalf("got %d package entries, want 1", len(entries)) - } - if _, found := entries[0].Metadata[detectors.MetadataKeyOriginArtifactURL]; found { - t.Fatalf("origin metadata reached the package listing: %v", entries[0].Metadata) - } - if _, found := entries[0].Metadata["npm"]; !found { - t.Fatalf("unrelated metadata was dropped: %v", entries[0].Metadata) - } -} diff --git a/internal/output/types.go b/internal/output/types.go index 354d4bbd..fce71f41 100644 --- a/internal/output/types.go +++ b/internal/output/types.go @@ -4,7 +4,6 @@ import ( "sort" "strings" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) @@ -263,25 +262,15 @@ func cloneAffectedSymbols(src []sdk.AffectedSymbol) []sdk.AffectedSymbol { return out } -// cloneRefMetadata copies package metadata for command output, dropping keys -// that carry data between pipeline stages rather than facts about the package. -// Origin keys are such a transport: detectors record where a package came from -// so SBOM export can publish it, and the SBOM is where users read it. +// cloneRefMetadata copies package metadata for command output. func cloneRefMetadata(src map[string]any) map[string]any { if len(src) == 0 { return nil } clone := make(map[string]any, len(src)) for key, value := range src { - if strings.HasPrefix(key, detectors.MetadataKeyOriginPrefix) { - continue - } clone[key] = value } - // Metadata is omitempty; an emptied map must read as absent, not as {}. - if len(clone) == 0 { - return nil - } return clone } diff --git a/internal/sbom/origin_test.go b/internal/sbom/origin_test.go index e13d1825..7084331f 100644 --- a/internal/sbom/origin_test.go +++ b/internal/sbom/origin_test.go @@ -5,7 +5,6 @@ import ( "strings" "testing" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) @@ -94,7 +93,7 @@ func marshalBoth(t *testing.T, g *sdk.Graph) (spdxRaw, cdxRaw []byte) { func TestArtifactOriginIsPublishedInBothFormats(t *testing.T) { const artifact = "https://registry.npmjs.org/react/-/react-18.2.0.tgz" g := originGraph(t, func(_, pkg *sdk.Dependency) { - detectors.SetOriginArtifact(pkg, artifact) + pkg.Origin = sdk.ArtifactOrigin(artifact) }) spdxRaw, cdxRaw := marshalBoth(t, g) @@ -113,7 +112,7 @@ func TestRepositoryOriginIsPublishedInBothFormats(t *testing.T) { revision = "b4c5d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7" ) g := originGraph(t, func(_, pkg *sdk.Dependency) { - detectors.SetOriginVCS(pkg, repository, revision) + pkg.Origin = sdk.RepositoryOrigin(repository, revision) }) spdxRaw, cdxRaw := marshalBoth(t, g) @@ -136,7 +135,7 @@ func TestRepositoryOriginIsPublishedInBothFormats(t *testing.T) { func TestUnpinnedRepositoryOriginOmitsTheRevisionSuffix(t *testing.T) { const repository = "https://github.com/facebook/react" g := originGraph(t, func(_, pkg *sdk.Dependency) { - detectors.SetOriginVCS(pkg, repository, "") + pkg.Origin = sdk.RepositoryOrigin(repository, "") }) spdxRaw, _ := marshalBoth(t, g) @@ -160,43 +159,37 @@ func TestPackageWithoutOriginKeepsNOASSERTION(t *testing.T) { } } -// Origin metadata can reach export from a plugin or a hand-built graph that -// never went through the setters. Export re-validates rather than trusting it, +// A hand-built origin can reach export from a plugin or a caller that never +// went through the constructors. Export re-validates rather than trusting it, // so a bad value is dropped instead of published. -func TestExportRevalidatesOriginMetadata(t *testing.T) { +func TestExportRevalidatesOrigin(t *testing.T) { hostile := []struct { - name string - metadata map[string]any + name string + origin *sdk.PackageOrigin }{ - {name: "credentialed artifact", metadata: map[string]any{ - detectors.MetadataKeyOriginArtifactURL: "https://build:s3cret-token-value@nexus.corp/repo/react-18.2.0.tgz", + {name: "credentialed artifact", origin: &sdk.PackageOrigin{ArtifactURL: "https://build:s3cret-token-value@nexus.corp/repo/react-18.2.0.tgz"}}, + {name: "local path", origin: &sdk.PackageOrigin{ArtifactURL: "/Users/someone/src/project/react.tgz"}}, + {name: "file url", origin: &sdk.PackageOrigin{Repository: "file:///Users/someone/src/react"}}, + {name: "registry root", origin: &sdk.PackageOrigin{ArtifactURL: "https://registry.npmjs.org/"}}, + {name: "revision breaking the locator grammar", origin: &sdk.PackageOrigin{ + Repository: "https://github.com/facebook/react", Revision: "main@evil.test/x", }}, - {name: "local path", metadata: map[string]any{ - detectors.MetadataKeyOriginArtifactURL: "/Users/someone/src/project/react.tgz", - }}, - {name: "file url", metadata: map[string]any{ - detectors.MetadataKeyOriginVCSURL: "file:///Users/someone/src/react", - }}, - {name: "revision breaking the locator grammar", metadata: map[string]any{ - detectors.MetadataKeyOriginVCSURL: "https://github.com/facebook/react", - detectors.MetadataKeyOriginVCSRevision: "main@evil.test/x", - }}, - {name: "non-string value", metadata: map[string]any{ - detectors.MetadataKeyOriginArtifactURL: 42, + {name: "a recorded disagreement publishes nothing", origin: &sdk.PackageOrigin{ + Disputed: true, ArtifactURL: "https://registry.npmjs.org/react/-/react-18.2.0.tgz", }}, } for _, tc := range hostile { t.Run(tc.name, func(t *testing.T) { g := originGraph(t, func(_, pkg *sdk.Dependency) { - pkg.Metadata = tc.metadata + pkg.Origin = tc.origin }) spdxRaw, cdxRaw := marshalBoth(t, g) download, _ := spdxPackageByName(t, spdxRaw, "react")["downloadLocation"].(string) refs := cycloneDXReferences(t, cdxRaw, "react") - published := append([]string{download}, refs["distribution"], refs["vcs"]) + published := []string{download, refs["distribution"], refs["vcs"]} for _, value := range published { for _, forbidden := range []string{"s3cret", "@nexus.corp", "/Users/", "file://", "evil.test"} { if strings.Contains(value, forbidden) { @@ -230,7 +223,7 @@ func TestScorecardRepositoryFillsTheOriginGap(t *testing.T) { react := sdk.NewDependencyWithID("react@18.2.0", sdk.Dependency{Coordinates: sdk.Coordinates{ Name: "react", Version: "18.2.0", PURL: purl, Ecosystem: "npm"}}) if detectorRepository != "" { - detectors.SetOriginVCS(react, detectorRepository, "") + react.Origin = sdk.RepositoryOrigin(detectorRepository, "") } if err := g.AddNode(react); err != nil { t.Fatalf("add node: %v", err) @@ -268,7 +261,7 @@ func TestScorecardRepositoryFillsTheOriginGap(t *testing.T) { g := sdk.New() react := sdk.NewDependencyWithID("react@18.2.0", sdk.Dependency{Coordinates: sdk.Coordinates{ Name: "react", Version: "18.2.0", PURL: purl, Ecosystem: "npm"}}) - detectors.SetOriginArtifact(react, "https://registry.npmjs.org/react/-/react-18.2.0.tgz") + react.Origin = sdk.ArtifactOrigin("https://registry.npmjs.org/react/-/react-18.2.0.tgz") if err := g.AddNode(react); err != nil { t.Fatal(err) } @@ -308,7 +301,7 @@ func TestScorecardRepositoryFillsTheOriginGap(t *testing.T) { // it as source info would say the same thing twice. t.Run("a repository alone is not repeated as source info", func(t *testing.T) { g := originGraph(t, func(_, pkg *sdk.Dependency) { - detectors.SetOriginVCS(pkg, "https://github.com/facebook/react", "") + pkg.Origin = sdk.RepositoryOrigin("https://github.com/facebook/react", "") }) spdxRaw, _ := marshalBoth(t, g) spdxPkg := spdxPackageByName(t, spdxRaw, "react") @@ -335,7 +328,7 @@ func TestScorecardRepositoryFillsTheOriginGap(t *testing.T) { // ingest is tracked separately. func TestOriginIsNotReadBackFromAnIngestedDocument(t *testing.T) { g := originGraph(t, func(_, pkg *sdk.Dependency) { - detectors.SetOriginVCS(pkg, "https://github.com/facebook/react", "d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f809") + pkg.Origin = sdk.RepositoryOrigin("https://github.com/facebook/react", "d6e7f8091a2b3c4d5e6f708192a3b4c5d6e7f809") }) exported, _ := marshalBoth(t, g) diff --git a/internal/sbom/transform.go b/internal/sbom/transform.go index 500248ee..103681f1 100644 --- a/internal/sbom/transform.go +++ b/internal/sbom/transform.go @@ -12,7 +12,6 @@ import ( "strings" "time" - "github.com/bomly-dev/bomly-cli/internal/detectors" "github.com/bomly-dev/bomly-sdk" ) @@ -48,7 +47,7 @@ func FromDepGraph(g *sdk.Graph, opts BuildOptions) (*Document, error) { Licenses: componentLicenses(sdk.DetectionLicenses(pkg)), Digests: componentDigests(pkg.Digests), } - applyOrigin(&component, detectors.OriginFrom(pkg.Metadata)) + applyOrigin(&component, pkg.Origin.Normalized()) enrichComponentFromRegistry(&component, opts.Registry, pkg.PURL) components = append(components, component) depsByRef[pkg.ID] = nil @@ -279,16 +278,20 @@ func scorecardRepositoryURL(scorecard *sdk.PackageScorecard) (string, bool) { if !strings.Contains(repository, "://") { repository = "https://" + repository } - return detectors.NormalizeOriginURL(repository, true) + return sdk.NormalizeOriginURL(repository, true) } // applyOrigin projects the origin a detector asserted onto a component. The -// values were validated when they were read, so there is nothing to decide -// here: export publishes what detection resolved, or nothing. -func applyOrigin(component *Component, origin detectors.Origin) { +// value arrives already validated -- Normalized applies the SDK's rule and +// returns nothing when a location does not survive it -- so there is nothing to +// decide here: export publishes what detection resolved, or nothing. +func applyOrigin(component *Component, origin *sdk.PackageOrigin) { + if origin == nil { + return + } component.ArtifactURL = origin.ArtifactURL - component.VCSURL = origin.VCSURL - component.VCSRevision = origin.VCSRevision + component.VCSURL = origin.Repository + component.VCSRevision = origin.Revision } func enrichComponentFromRegistry(component *Component, registry *sdk.PackageRegistry, purl string) { diff --git a/scripts/run-fuzz.sh b/scripts/run-fuzz.sh index 4271a274..36bf6147 100755 --- a/scripts/run-fuzz.sh +++ b/scripts/run-fuzz.sh @@ -31,7 +31,6 @@ targets=( "github.com/bomly-dev/bomly-cli/internal/detectors/swiftpm FuzzDepGraphFromSwiftResolved" "github.com/bomly-dev/bomly-cli/internal/sbom FuzzUnmarshalAutoJSON" "github.com/bomly-dev/bomly-cli/internal/sbom FuzzNormalizeSPDXLicenseExpression" - "github.com/bomly-dev/bomly-cli/internal/detectors FuzzSetOrigin" "github.com/bomly-dev/bomly-cli/internal/baseline FuzzLoad" "github.com/bomly-dev/bomly-cli/internal/engine FuzzConsolidateVulnerabilities" "github.com/bomly-dev/bomly-cli/internal/plugin FuzzPluginPathSanitizers" From 6d00e96b32865e334299a27dd949a2f6ab8dfe99 Mon Sep 17 00:00:00 2001 From: Ahmed ElMallah Date: Tue, 18 Aug 2026 23:53:01 -0400 Subject: [PATCH 17/17] fix(swiftpm): do not credit a package to a pin for another repository A package can be built from a mirror while Package.resolved still pins the upstream host. The names match, so the identity fallback attached the pinned repository and its commit to a package that was never fetched from there -- reproduced: a node resolved from mirror.corp came out claiming git.corp at a specific revision. Matching by identity is now only used when the graph offers nothing better. A node that names a repository and did not match one has a repository the pins do not describe, and a same-named pin is a guess rather than evidence. Co-Authored-By: Claude Opus 5 --- internal/detectors/swiftpm/origin_test.go | 53 ++++++++++++++++++++ internal/detectors/swiftpm/swiftpm_native.go | 8 +++ 2 files changed, 61 insertions(+) diff --git a/internal/detectors/swiftpm/origin_test.go b/internal/detectors/swiftpm/origin_test.go index e34fb970..031b3ea5 100644 --- a/internal/detectors/swiftpm/origin_test.go +++ b/internal/detectors/swiftpm/origin_test.go @@ -199,6 +199,59 @@ func TestSwiftPMEditedPackageIsNotCreditedToItsFormerPin(t *testing.T) { } } +// A package can be built from a mirror while Package.resolved still pins the +// upstream host. The names match, so an identity fallback would credit the +// build to a repository it never fetched from. +func TestSwiftPMPinIsNotAttachedToADifferentRepository(t *testing.T) { + workingDir := t.TempDir() + resolved := `{ + "pins": [ + { + "identity": "helper", + "kind": "remoteSourceControl", + "location": "https://git.corp/team/helper.git", + "state": {"revision": "aaaabbbbccccddddeeeeffff0000111122223333", "version": "1.0.0"} + } + ], + "version": 2 + }` + if err := os.WriteFile(filepath.Join(workingDir, "Package.resolved"), []byte(resolved), 0o644); err != nil { + t.Fatal(err) + } + showDependencies := []byte(`{ + "name": "demo", + "url": "/workspace/demo", + "version": "unspecified", + "dependencies": [ + {"name": "helper", "url": "https://mirror.corp/team/helper.git", "version": "2.0.0", "dependencies": []} + ] + }`) + + g, err := nativeGraph(showDependencies, workingDir, zap.NewNop()) + if err != nil { + t.Fatalf("nativeGraph() error = %v", err) + } + + var checked int + g.WalkNodes(func(dep *sdk.Dependency) bool { + if dep.ResolvedURL == "" { + return true + } + checked++ + origin := originOf(dep) + if origin.Repository == "https://git.corp/team/helper.git" { + t.Fatalf("a package built from a mirror was credited to %q", origin.Repository) + } + if origin.Revision != "" { + t.Fatalf("origin = %+v, want no pin: the pinned repository is not this one", origin) + } + return true + }) + if checked != 1 { + t.Fatalf("checked %d packages, want 1", checked) + } +} + // A project with no Package.resolved keeps whatever the native graph carried. func TestSwiftPMNativeOriginSurvivesMissingPackageResolved(t *testing.T) { showDependencies := []byte(`{ diff --git a/internal/detectors/swiftpm/swiftpm_native.go b/internal/detectors/swiftpm/swiftpm_native.go index 2427bc84..ae44b6a8 100644 --- a/internal/detectors/swiftpm/swiftpm_native.go +++ b/internal/detectors/swiftpm/swiftpm_native.go @@ -141,6 +141,14 @@ func applyResolvedOrigins(g *sdk.Graph, workingDir string, logger *zap.Logger) { } pin, ok := byRepository[repositoryKey(dep.ResolvedURL)] if !ok { + // Matching by identity is only safe when the graph offers nothing + // better. A node that names a repository and did not match one has + // a repository the pins do not describe -- a mirror, say -- and + // crediting it to a same-named pin would name a location the build + // did not use. + if strings.TrimSpace(dep.ResolvedURL) != "" { + return true + } if pin, ok = pins[dep.Name]; !ok { return true }