From dc49b44bf0643ad0aab412ec30dfa4ecf733974f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Prpi=C4=8D?= Date: Wed, 4 Nov 2026 14:15:44 -0500 Subject: [PATCH 1/4] feat(pkgmetadata): detect declared license with SPDX ambiguity info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `declared_license_from_raw_metadata()`, which picks the license a package declares in its core metadata and reports whether it maps to an SPDX expression without guessing. Sources are tried in the order agreed in #1378: `License-Expression`, license classifiers that name exactly one license, then free text that is a valid SPDX expression or a curated mapping. When nothing maps, it returns the raw declaration with `spdx=None` so the SBOM can record it as a `LicenseRef`. SPDX expressions are validated and canonicalized with `packaging.licenses`, the check build backends apply to `License-Expression`. Shorthands such as `GPL` are rejected instead of resolved to `GPL-1.0-or-later`, `WITH` misuse is rejected, and deprecated ids are kept. It reads `RawMetadata` and never raises, so bad upstream metadata cannot fail a build. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Martin Prpič --- src/fromager/pkgmetadata/__init__.py | 11 +- src/fromager/pkgmetadata/pep639.py | 136 +++++++++++- tests/test_pkgmetadata.py | 305 +++++++++++++++++++++++++++ 3 files changed, 449 insertions(+), 3 deletions(-) diff --git a/src/fromager/pkgmetadata/__init__.py b/src/fromager/pkgmetadata/__init__.py index 33ea5c89e..df36679ee 100644 --- a/src/fromager/pkgmetadata/__init__.py +++ b/src/fromager/pkgmetadata/__init__.py @@ -6,10 +6,19 @@ """ from .pep376 import dist_info_name, verbatim_dist_name -from .pep639 import license_from_metadata, license_from_metadata_values +from .pep639 import ( + DeclaredLicense, + LicenseSource, + declared_license_from_raw_metadata, + license_from_metadata, + license_from_metadata_values, +) from .pep753 import normalize_project_urls, project_urls_from_metadata __all__ = ( + "DeclaredLicense", + "LicenseSource", + "declared_license_from_raw_metadata", "dist_info_name", "license_from_metadata", "license_from_metadata_values", diff --git a/src/fromager/pkgmetadata/pep639.py b/src/fromager/pkgmetadata/pep639.py index 2817f4a11..091692ba0 100644 --- a/src/fromager/pkgmetadata/pep639.py +++ b/src/fromager/pkgmetadata/pep639.py @@ -6,9 +6,13 @@ - https://spdx.org/licenses/ """ +import dataclasses +import enum +import inspect + from license_expression import ExpressionError, LicenseExpression, get_spdx_licensing from packaging.licenses import InvalidLicenseExpression, canonicalize_license_expression -from packaging.metadata import Metadata +from packaging.metadata import Metadata, RawMetadata def license_from_metadata(metadata: Metadata) -> LicenseExpression: @@ -32,7 +36,9 @@ def license_from_metadata_values( 2. Fall back to *license_text*. Perform some unambiguous translatons (e.g. ``Apache 2`` to ``Apache-2.0``) and attempt to parse the string as SPDX license expression. - 3. Finally fall back to trove classifiers. + 3. Finally fall back to trove classifiers. Several mapped classifiers + are joined with ``AND``; `declared_license_from_raw_metadata()` + treats them as ambiguous instead. Raises an exception if license is missing, ambiguous, or not a valid SPDX license expression. @@ -59,6 +65,108 @@ def license_from_metadata_values( raise ExceptionGroup("unable to detect license", errors) +class LicenseSource(enum.StrEnum): + """Core metadata field a declared license was taken from""" + + LICENSE_EXPRESSION = "License-Expression" + CLASSIFIERS = "License classifiers" + LICENSE = "License" + + +@dataclasses.dataclass(frozen=True) +class DeclaredLicense: + """License as declared in a package's core metadata + + ``raw`` keeps the declaration (license classifiers joined by newlines) + so it can be quoted when ``spdx`` is ``None``, which means it has no + unambiguous SPDX mapping. + """ + + source: LicenseSource + raw: str + spdx: str | None + + +def declared_license_from_raw_metadata( + metadata: RawMetadata, +) -> DeclaredLicense | None: + """Detect the declared license for SBOMs without guessing + + Unlike `license_from_metadata_values()`, this never raises and reports + an ambiguous declaration instead of guessing an SPDX id, so the caller + can record it verbatim. Sources are tried in the order agreed in + issue #1378: ``License-Expression``, unambiguous license classifiers, + then free text that is an exact SPDX expression or a curated mapping. + Without a clean mapping, the first declaration present is returned + with ``spdx=None``, preferring free text over ambiguous classifiers + because it is usually more specific. Classifiers that name more than + one license are ambiguous here, while `license_from_metadata_values()` + joins them with ``AND``. + + Returns ``None`` when the metadata declares no license at all. + """ + expression = _declared_value(metadata.get("license_expression", "").strip()) + text = _declared_value(inspect.cleandoc(metadata.get("license", "")).strip()) + classifiers = [ + c for c in metadata.get("classifiers", []) if c.startswith("License ::") + ] + joined_classifiers = "\n".join(classifiers) + + canonical_expression = _canonical_spdx(expression) if expression else None + if expression and canonical_expression: + return DeclaredLicense( + LicenseSource.LICENSE_EXPRESSION, expression, canonical_expression + ) + classifier_spdx = _single_classifier_license(classifiers) + if classifier_spdx: + return DeclaredLicense( + LicenseSource.CLASSIFIERS, joined_classifiers, classifier_spdx + ) + if text: + text_spdx = _strict_text_to_spdx(text) + if text_spdx is not None: + return DeclaredLicense(LicenseSource.LICENSE, text, text_spdx) + + if expression: + return DeclaredLicense(LicenseSource.LICENSE_EXPRESSION, expression, None) + if text: + return DeclaredLicense(LicenseSource.LICENSE, text, None) + if classifiers: + return DeclaredLicense(LicenseSource.CLASSIFIERS, joined_classifiers, None) + return None + + +# license classifiers that do not name a license +_NON_LICENSE_CLASSIFIERS = frozenset( + {"License :: DFSG approved", "License :: OSI Approved"} +) + + +def _single_classifier_license(classifiers: list[str]) -> str | None: + """SPDX id of the license classifiers if they name exactly one license + + Several different licenses may mean a choice (OR) or a combination + (AND); classifiers cannot say which, so that is not guessed. + """ + mapped: set[str] = set() + for classifier in classifiers: + if classifier in _NON_LICENSE_CLASSIFIERS: + continue + # ambiguous, or a license classifier missing from the table + spdx = _TROVE_SPDX.get(classifier) + if spdx is None: + return None + mapped.add(spdx) + return mapped.pop() if len(mapped) == 1 else None + + +def _declared_value(value: str) -> str | None: + """Treat empty values and the legacy setuptools ``UNKNOWN`` as absent""" + if not value or value.casefold() == "unknown": + return None + return value + + _SPDX = get_spdx_licensing() @@ -106,6 +214,30 @@ def _license_text_to_spdx(text: str) -> LicenseExpression: raise ValueError(text[:100]) +def _canonical_spdx(text: str) -> str | None: + """Canonical form of a valid SPDX expression, else ``None`` + + Uses the same validation build backends apply to ``License-Expression`` + (PEP 639). Unlike the license library, it rejects shorthands such as + ``GPL`` and ``WITH`` misuse, and keeps deprecated SPDX ids as written. + """ + try: + return canonicalize_license_expression(text) + except InvalidLicenseExpression: + return None + + +def _strict_text_to_spdx(text: str) -> str | None: + """Map free text only when the mapping is certain, else ``None`` + + Free text must be a curated mapping or a valid SPDX expression; + anything else would overstate what upstream declared. + """ + if text in _LICENSE_STRING_TO_SPDX: + return _canonical_spdx(_LICENSE_STRING_TO_SPDX[text]) + return _canonical_spdx(text) + + # unambiguous text to SPDX # The keys are common cases seen in the wild on PyPI.org _LICENSE_STRING_TO_SPDX: dict[str, str] = { diff --git a/tests/test_pkgmetadata.py b/tests/test_pkgmetadata.py index ffc9384c2..855821bb0 100644 --- a/tests/test_pkgmetadata.py +++ b/tests/test_pkgmetadata.py @@ -3,6 +3,9 @@ from packaging.metadata import Metadata, RawMetadata from fromager.pkgmetadata import ( + DeclaredLicense, + LicenseSource, + declared_license_from_raw_metadata, license_from_metadata, license_from_metadata_values, normalize_project_urls, @@ -167,3 +170,305 @@ def test_license_from_metadata_canonicalizes_licenseref_prefix() -> None: # PEP 639 matches the LicenseRef- prefix case-insensitively expr = license_from_metadata_values(license_expression="licenseref-foo") assert str(expr) == "LicenseRef-foo" + + +BSD = "License :: OSI Approved :: BSD License" +APACHE = "License :: OSI Approved :: Apache Software License" +MIT_C = "License :: OSI Approved :: MIT License" +MPL_C = "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)" +ISC_C = "License :: OSI Approved :: ISC License (ISCL)" +LGPL3_C = "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)" +DOCUTILS_TEXT = "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)" +EXPR = LicenseSource.LICENSE_EXPRESSION +CLS = LicenseSource.CLASSIFIERS +TEXT = LicenseSource.LICENSE + + +@pytest.mark.parametrize( + ["license_expression", "license", "classifiers", "expected"], + [ + # rows named after a package mirror its real wheel METADATA + pytest.param( + "Apache-2.0 OR BSD-2-Clause", + None, + None, + DeclaredLicense( + EXPR, "Apache-2.0 OR BSD-2-Clause", "Apache-2.0 OR BSD-2-Clause" + ), + id="packaging-26.3", + ), + pytest.param( + "MPL-2.0 AND (Apache-2.0 OR MIT)", + None, + [APACHE, MIT_C], + DeclaredLicense( + EXPR, + "MPL-2.0 AND (Apache-2.0 OR MIT)", + "MPL-2.0 AND (Apache-2.0 OR MIT)", + ), + id="orjson-3.13.0", + ), + pytest.param( + "LGPL-3.0", + None, + None, + DeclaredLicense(EXPR, "LGPL-3.0", "LGPL-3.0"), + id="expr-deprecated", + ), + pytest.param( + "MIT OR LicenseRef-Proprietary", + None, + None, + DeclaredLicense( + EXPR, + "MIT OR LicenseRef-Proprietary", + "MIT OR LicenseRef-Proprietary", + ), + id="expr-licenseref", + ), + # expressions are canonicalized the way build backends do it + pytest.param( + "Apache-2.0 or mit", + None, + None, + DeclaredLicense(EXPR, "Apache-2.0 or mit", "Apache-2.0 OR MIT"), + id="expr-canonicalized", + ), + pytest.param( + "GPL", + None, + None, + DeclaredLicense(EXPR, "GPL", None), + id="expr-gpl-alias", + ), + pytest.param( + "Apache 2", + None, + [MIT_C], + DeclaredLicense(CLS, MIT_C, "MIT"), + id="invalid-expr-falls-through", + ), + pytest.param( + "Apache 2", + None, + None, + DeclaredLicense(EXPR, "Apache 2", None), + id="invalid-expr-only", + ), + pytest.param( + "MIT WITH Apache-2.0", + None, + None, + DeclaredLicense(EXPR, "MIT WITH Apache-2.0", None), + id="with-misuse", + ), + pytest.param( + " ", + None, + [MIT_C], + DeclaredLicense(CLS, MIT_C, "MIT"), + id="empty-expression", + ), + pytest.param( + None, + "MPL-2.0 AND MIT", + [MIT_C, MPL_C], + # two classifiers may mean a choice, so the free text decides + DeclaredLicense(TEXT, "MPL-2.0 AND MIT", "MPL-2.0 AND MIT"), + id="tqdm-4.66.1", + ), + pytest.param( + None, + "UNKNOWN", + [ISC_C], + DeclaredLicense(CLS, ISC_C, "ISC"), + id="ptyprocess-0.7.0", + ), + pytest.param( + None, + "Apache 2.0", + [APACHE], + DeclaredLicense(TEXT, "Apache 2.0", "Apache-2.0"), + id="requests-2.31.0", + ), + pytest.param( + None, + "BSD-3-Clause", + [BSD], + DeclaredLicense(TEXT, "BSD-3-Clause", "BSD-3-Clause"), + id="click-8.1.7", + ), + pytest.param( + None, + "mit", + None, + DeclaredLicense(TEXT, "mit", "MIT"), + id="text-lowercase-id", + ), + pytest.param( + None, + "GPL", + None, + DeclaredLicense(TEXT, "GPL", None), + id="text-gpl-alias", + ), + pytest.param( + None, + "LGPL-3.0", + [LGPL3_C], + DeclaredLicense(TEXT, "LGPL-3.0", "LGPL-3.0"), + id="easydict-1.13", + ), + pytest.param( + None, + "NVIDIA Proprietary Software", + None, + DeclaredLicense( + TEXT, + "NVIDIA Proprietary Software", + "LicenseRef-NVIDIA-SOFTWARE-LICENSE", + ), + id="text-curated-ref", + ), + pytest.param( + None, + "Copyright (c) 2015\n line two\n \n line four\n", + None, + DeclaredLicense(TEXT, "Copyright (c) 2015\nline two\n\nline four", None), + id="text-folded", + ), + pytest.param( + None, + None, + [BSD], + DeclaredLicense(CLS, BSD, None), + id="idna-3.6", + ), + pytest.param( + None, + None, + [APACHE, BSD], + DeclaredLicense(CLS, f"{APACHE}\n{BSD}", None), + id="packaging-25.0", + ), + pytest.param( + None, + "Apache-2.0 with LLVM exception", + [APACHE], + DeclaredLicense(TEXT, "Apache-2.0 with LLVM exception", None), + id="lit-18.1.8", + ), + pytest.param( + None, + DOCUTILS_TEXT, + [ + BSD, + "License :: OSI Approved :: GNU General Public License (GPL)", + "License :: OSI Approved :: Python Software Foundation License", + "License :: Public Domain", + ], + DeclaredLicense(TEXT, DOCUTILS_TEXT, None), + id="docutils-0.20.1", + ), + pytest.param( + None, + None, + ["License :: OSI Approved"], + DeclaredLicense(CLS, "License :: OSI Approved", None), + id="bare-osi-approved", + ), + # malformed expressions are rejected, not raised + pytest.param( + "()", + None, + None, + DeclaredLicense(EXPR, "()", None), + id="expr-parser-crash", + ), + pytest.param( + None, + "( AND LLVM-exception", + None, + DeclaredLicense(TEXT, "( AND LLVM-exception", None), + id="text-parser-crash", + ), + pytest.param( + None, + None, + [MIT_C, "License :: OSI Approved :: Python Software Foundation License"], + DeclaredLicense( + CLS, + f"{MIT_C}\nLicense :: OSI Approved :: Python Software Foundation License", + None, + ), + id="two-mapped-classifiers", + ), + pytest.param( + None, + None, + [ + "License :: OSI Approved :: Nokia Open Source License", + "License :: Nokia Open Source License (NOKOS)", + ], + DeclaredLicense( + CLS, + "License :: OSI Approved :: Nokia Open Source License\n" + "License :: Nokia Open Source License (NOKOS)", + "Nokia", + ), + id="classifiers-same-license", + ), + pytest.param( + None, + "MIT License \n", + None, + DeclaredLicense(TEXT, "MIT License", "MIT"), + id="text-trailing-whitespace", + ), + # license classifiers missing from the table are ambiguous + pytest.param( + None, + None, + [MIT_C, "License :: OSI Approved :: Intel Open Source License"], + DeclaredLicense( + CLS, + f"{MIT_C}\nLicense :: OSI Approved :: Intel Open Source License", + None, + ), + id="unknown-license-classifier", + ), + # classifiers that do not name a license are skipped + pytest.param( + None, + None, + [MIT_C, "License :: OSI Approved"], + DeclaredLicense(CLS, f"{MIT_C}\nLicense :: OSI Approved", "MIT"), + id="non-license-classifier", + ), + pytest.param(None, "unknown", None, None, id="unknown-only"), + pytest.param( + None, + None, + ["Programming Language :: Python :: 3"], + None, + id="ipadic-1.0.0", + ), + ], +) +def test_declared_license_from_raw_metadata( + license_expression: str | None, + license: str | None, + classifiers: list[str] | None, + expected: DeclaredLicense | None, +) -> None: + raw = RawMetadata(metadata_version="2.4", name="test", version="1.0") + if license_expression is not None: + raw["license_expression"] = license_expression + if license is not None: + raw["license"] = license + if classifiers is not None: + raw["classifiers"] = classifiers + + result = declared_license_from_raw_metadata(raw) + + assert result == expected From f51b9c36b136d787c064b275119147cd9286589e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Prpi=C4=8D?= Date: Wed, 4 Nov 2026 14:18:17 -0500 Subject: [PATCH 2/4] feat(sbom): populate license fields from wheel metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The embedded SPDX SBOM had no license fields. Both package entries now carry `licenseDeclared` from the license the built wheel declares in its own METADATA, `licenseConcluded: NOASSERTION` because fromager does not audit licenses, and a `licenseComments` note saying the value is inherited from upstream and not verified. A declaration without an unambiguous SPDX id becomes a deterministic `LicenseRef--` with a matching `hasExtractedLicensingInfos` entry quoting it, as agreed in #1378 (Option A). LicenseRefs that come from upstream get an entry too, so the document passes SPDX validation; the metadata has no license text for them, so their text is `NOASSERTION` and a comment says where they were declared. METADATA is parsed raw so invalid upstream values cannot fail the build. Unreadable license fields are logged and named in the comment. A built wheel without METADATA is invalid and now fails when SBOMs are enabled, instead of getting an SBOM that claims no license exists. Closes: #1378 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Martin Prpič --- src/fromager/sbom.py | 117 +++++++++++++++++++++- src/fromager/wheels.py | 30 ++++++ tests/test_sbom.py | 221 ++++++++++++++++++++++++++++++++++++++++- tests/test_wheels.py | 96 ++++++++++++++++++ 4 files changed, 461 insertions(+), 3 deletions(-) diff --git a/src/fromager/sbom.py b/src/fromager/sbom.py index 027d86795..b92fb8c8b 100644 --- a/src/fromager/sbom.py +++ b/src/fromager/sbom.py @@ -6,6 +6,7 @@ from __future__ import annotations +import hashlib import importlib.metadata import json import logging @@ -13,14 +14,18 @@ import re import typing import uuid +from collections.abc import Iterable from datetime import UTC, datetime from urllib.parse import quote from packageurl import PackageURL +from packaging.metadata import RawMetadata from packaging.requirements import Requirement from packaging.utils import NormalizedName, canonicalize_name from packaging.version import Version +from .pkgmetadata import DeclaredLicense, declared_license_from_raw_metadata + if typing.TYPE_CHECKING: from . import context from .packagesettings import PackageBuildInfo, SbomSettings @@ -29,6 +34,99 @@ SBOM_FILENAME = "fromager.spdx.json" +_LICENSE_COMMENT = ( + "License inherited from the upstream project's declared Python package " + "metadata ({source}) and not independently verified by fromager; it may " + "be imprecise or incomplete." +) +_UNMAPPED_COMMENT = " No unambiguous SPDX mapping for: {raw}." +_NO_LICENSE_COMMENT = ( + "No license information found in the upstream project's declared Python " + "package metadata." +) +_UNREADABLE_LICENSE_COMMENT = ( + "License information could not be read from the upstream project's " + "declared Python package metadata." +) +_UNPARSED_COMMENT = " Unparseable fields: {names}." +# Full license texts can be tens of KB; the comment only quotes the start. +_COMMENT_RAW_MAX = 200 +_UPSTREAM_REF_COMMENT = ( + "Declared in the upstream project's {source}: {raw}. The license text is " + "not available in the package metadata." +) +# characters allowed in SPDX ids (SPDX 2.3, idstring) +_SPDX_IDSTRING_CHARS = r"A-Za-z0-9.\-" +_LICENSE_REF_RE = re.compile(rf"LicenseRef-[{_SPDX_IDSTRING_CHARS}]+") + + +def _license_ref_id(raw: str) -> str: + """Build a stable LicenseRef id for a declaration without an SPDX id. + + The slug keeps the id readable and the hash keeps it unique, so + rebuilds produce the same id and SBOMs stay diff-clean. + """ + slug = re.sub(r"[^a-z0-9]+", "-", raw.lower()).strip("-")[:40].rstrip("-") + digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:8] + return f"LicenseRef-{slug}-{digest}" if slug else f"LicenseRef-{digest}" + + +def _quote_for_comment(raw: str) -> str: + """Shorten a declaration to one line for ``licenseComments``.""" + flat = raw.replace("\n", "; ") + if len(flat) > _COMMENT_RAW_MAX: + return flat[:_COMMENT_RAW_MAX] + "..." + return flat + + +def _license_fields( + declared: DeclaredLicense | None, + unparsed_fields: Iterable[str], +) -> tuple[dict[str, str], list[dict[str, str]]]: + """Build SPDX package license fields and extracted licensing infos. + + fromager does not audit licenses, so ``licenseConcluded`` is always + ``NOASSERTION`` and the comment says the value is inherited from + upstream. Every ``LicenseRef`` used must be declared in the document. + A ``LicenseRef`` fromager generates quotes the raw declaration as its + text. One that upstream named has no license text in the metadata, so + its text is ``NOASSERTION`` and a comment says where it was declared. + """ + extracted: list[dict[str, str]] = [] + names = ", ".join(unparsed_fields) + if declared is None: + license_declared = "NOASSERTION" + # unreadable fields may hold a license, so do not claim there is none + comment = _UNREADABLE_LICENSE_COMMENT if names else _NO_LICENSE_COMMENT + else: + comment = _LICENSE_COMMENT.format(source=declared.source) + if declared.spdx is None: + license_declared = _license_ref_id(declared.raw) + comment += _UNMAPPED_COMMENT.format(raw=_quote_for_comment(declared.raw)) + extracted = [{"licenseId": license_declared, "extractedText": declared.raw}] + else: + license_declared = declared.spdx + ref_comment = _UPSTREAM_REF_COMMENT.format( + source=declared.source, raw=_quote_for_comment(declared.raw) + ) + extracted = [ + { + "licenseId": ref, + "name": ref.removeprefix("LicenseRef-"), + "extractedText": "NOASSERTION", + "comment": ref_comment, + } + for ref in dict.fromkeys(_LICENSE_REF_RE.findall(declared.spdx)) + ] + if names: + comment += _UNPARSED_COMMENT.format(names=names) + fields = { + "licenseDeclared": license_declared, + "licenseConcluded": "NOASSERTION", + "licenseComments": comment, + } + return fields, extracted + def _spdx_id(kind: str, name: str, version: str, doc_suffix: str) -> str: """Build a package SPDX id unique to this package and document. @@ -40,7 +138,7 @@ def _spdx_id(kind: str, name: str, version: str, doc_suffix: str) -> str: data. SPDX ids only allow letters, digits, ``.`` and ``-``. """ spdx_id = f"SPDXRef-{kind}-{name}-{version}-{doc_suffix}" - return re.sub(r"[^A-Za-z0-9.\-]+", "-", spdx_id) + return re.sub(rf"[^{_SPDX_IDSTRING_CHARS}]+", "-", spdx_id) def _build_downstream_purl( @@ -105,7 +203,9 @@ def generate_sbom( ctx: context.WorkContext, req: Requirement, version: Version, + metadata: RawMetadata, wheel_version: Version | None = None, + unparsed_fields: Iterable[str] = (), ) -> dict[str, typing.Any]: """Generate a minimal SPDX 2.3 JSON document for a wheel. @@ -113,6 +213,10 @@ def generate_sbom( the upstream source as a second package, and DESCRIBES / GENERATED_FROM relationships. + Both packages carry the license declared in the wheel's core + *metadata*. *unparsed_fields* names license fields that could not be + read, so the SBOM does not claim they are absent. + The document name and namespace are derived from the downstream purl (including any ``purl`` field overrides), so the document identity always matches the wheel package entry it describes. @@ -127,6 +231,9 @@ def generate_sbom( Package SPDX ids include a per-document suffix and are not predictable, so consumers should find the wheel package through the document's DESCRIBES relationship rather than by a fixed id. + + .. versionchanged:: 0.104.0 + ``metadata`` is required, and both packages carry license fields. """ sbom_settings = ctx.settings.sbom_settings if sbom_settings is None: @@ -172,12 +279,17 @@ def generate_sbom( wheel_id = _spdx_id("wheel", doc_name, doc_version, doc_suffix) upstream_id = _spdx_id("upstream", upstream.name, upstream_version, doc_suffix) + license_fields, extracted_licenses = _license_fields( + declared_license_from_raw_metadata(metadata), unparsed_fields + ) + wheel_entry: dict[str, typing.Any] = { "SPDXID": wheel_id, "name": doc_name, "versionInfo": doc_version, "downloadLocation": "NOASSERTION", "supplier": sbom_settings.supplier, + **license_fields, "externalRefs": [ { "referenceCategory": "PACKAGE-MANAGER", @@ -193,6 +305,7 @@ def generate_sbom( "versionInfo": upstream_version, "downloadLocation": "NOASSERTION", "supplier": "NOASSERTION", + **license_fields, "externalRefs": [ { "referenceCategory": "PACKAGE-MANAGER", @@ -226,6 +339,8 @@ def generate_sbom( }, ], } + if extracted_licenses: + doc["hasExtractedLicensingInfos"] = extracted_licenses return doc diff --git a/src/fromager/wheels.py b/src/fromager/wheels.py index 98fb3a71d..e442ffed0 100644 --- a/src/fromager/wheels.py +++ b/src/fromager/wheels.py @@ -13,6 +13,7 @@ import elfdeps import tomlkit +from packaging.metadata import parse_email from packaging.requirements import Requirement from packaging.tags import Tag from packaging.utils import ( @@ -48,6 +49,13 @@ FROMAGER_ELF_PROVIDES = "fromager-elf-provides.txt" FROMAGER_ELF_REQUIRES = "fromager-elf-requires.txt" FROMAGER_BUILD_REQ_PREFIX = "fromager" +# core metadata fields the SBOM license is derived from, keyed by the +# lowercased names parse_email() reports, mapped to their header spelling +_LICENSE_METADATA_FIELDS = { + "license": "License", + "license-expression": "License-Expression", + "classifier": "Classifier", +} def _validate_build_tag_segments(segments: list[str]) -> None: @@ -318,10 +326,32 @@ def add_extra_metadata_to_wheels( sbom_settings = ctx.settings.sbom_settings if sbom_settings is not None: + metadata_file = dist_info_dir / "METADATA" + if not metadata_file.is_file(): + raise FileNotFoundError( + f"{wheel_file.name} has no {dist_info_dir.name}/METADATA" + ) + # Raw parsing: validated `Metadata` access raises on invalid + # upstream values, which must not fail the build. + raw_metadata, unparsed = parse_email(metadata_file.read_bytes()) + unparsed_license_fields = [ + _LICENSE_METADATA_FIELDS[name] + for name in unparsed + if name in _LICENSE_METADATA_FIELDS + ] + if unparsed_license_fields: + logger.warning( + "%s: cannot parse license metadata field(s) %s; " + "SBOM license information will be incomplete", + req.name, + ", ".join(unparsed_license_fields), + ) sbom_doc = sbom.generate_sbom( ctx=ctx, req=req, version=version, + metadata=raw_metadata, + unparsed_fields=unparsed_license_fields, # dist_version comes from the wheel filename and includes # any local version added by a patch, so the SBOM # identifies the wheel that actually contains it diff --git a/tests/test_sbom.py b/tests/test_sbom.py index 85f951e3c..2022588d9 100644 --- a/tests/test_sbom.py +++ b/tests/test_sbom.py @@ -6,6 +6,7 @@ from unittest.mock import patch from conftest import make_sbom_ctx +from packaging.metadata import RawMetadata from packaging.requirements import Requirement from packaging.version import Version from spdx_tools.spdx.parser.jsonlikedict.json_like_dict_parser import ( @@ -24,6 +25,13 @@ def _validate_spdx(doc: dict[str, typing.Any]) -> None: assert not errors, "\n".join(e.validation_message for e in errors) +def _metadata(**fields: typing.Any) -> RawMetadata: + """Build core metadata for the package an SBOM describes.""" + raw = RawMetadata(metadata_version="2.4", name="test", version="1.0") + raw.update(fields) # type: ignore[typeddict-item] + return raw + + def test_generate_sbom_structure(tmp_path: pathlib.Path) -> None: """Verify the generated SBOM has the required SPDX 2.3 fields.""" ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) @@ -31,6 +39,7 @@ def test_generate_sbom_structure(tmp_path: pathlib.Path) -> None: ctx=ctx, req=Requirement("example-pkg==1.2.3"), version=Version("1.2.3"), + metadata=_metadata(), ) assert doc["spdxVersion"] == "SPDX-2.3" @@ -51,6 +60,7 @@ def test_generate_sbom_default_purls(tmp_path: pathlib.Path) -> None: ctx=ctx, req=Requirement("my-package==2.0.0"), version=Version("2.0.0"), + metadata=_metadata(), ) wheel = doc["packages"][0] @@ -72,6 +82,7 @@ def test_generate_sbom_repository_url_qualifier(tmp_path: pathlib.Path) -> None: ctx=ctx, req=Requirement("numpy==1.26.0"), version=Version("1.26.0"), + metadata=_metadata(), ) wheel = doc["packages"][0] @@ -97,6 +108,7 @@ def test_generate_sbom_custom_settings(tmp_path: pathlib.Path) -> None: ctx=ctx, req=Requirement("my-package==2.0.0"), version=Version("2.0.0"), + metadata=_metadata(), ) wheel = doc["packages"][0] @@ -133,6 +145,7 @@ def test_generate_sbom_document_identity_follows_purl_version_override( ctx=ctx, req=Requirement("test-pkg==0.5.5"), version=Version("0.5.5"), + metadata=_metadata(), ) assert doc["name"] == "test-pkg-0.5.5+corp.1" @@ -161,6 +174,7 @@ def test_generate_sbom_upstream_uses_source_version_without_upstream_override( ctx=ctx, req=Requirement("test-pkg==0.5.5"), version=Version("0.5.5"), + metadata=_metadata(), ) wheel, upstream = doc["packages"] @@ -189,6 +203,7 @@ def test_generate_sbom_wheel_version_used_without_purl_override( req=Requirement("test-pkg==0.5.5"), version=Version("0.5.5"), wheel_version=Version("0.5.5+local.1"), + metadata=_metadata(), ) assert doc["name"] == "test-pkg-0.5.5+local.1" @@ -219,6 +234,7 @@ def test_generate_sbom_purl_version_override_wins_over_wheel_version( req=Requirement("test-pkg==0.5.5"), version=Version("0.5.5"), wheel_version=Version("0.5.5+local.1"), + metadata=_metadata(), ) wheel = doc["packages"][0] @@ -236,6 +252,7 @@ def test_generate_sbom_namespace_is_unique_per_document( "ctx": ctx, "req": Requirement("test-pkg==0.5.5"), "version": Version("0.5.5"), + "metadata": _metadata(), } doc1 = sbom.generate_sbom(**kwargs) @@ -265,6 +282,7 @@ def test_generate_sbom_namespace_percent_encodes_purl_overrides( ctx=ctx, req=Requirement("test-pkg==1.0"), version=Version("1.0"), + metadata=_metadata(), ) # Document and package names keep the raw override values @@ -291,6 +309,7 @@ def test_generate_sbom_purl_field_overrides(tmp_path: pathlib.Path) -> None: ctx=ctx, req=Requirement("test-pkg==1.0.0"), version=Version("1.0.0"), + metadata=_metadata(), ) wheel = doc["packages"][0] @@ -322,6 +341,7 @@ def test_generate_sbom_package_repository_url_override(tmp_path: pathlib.Path) - ctx=ctx, req=Requirement("test-pkg==1.0.0"), version=Version("1.0.0"), + metadata=_metadata(), ) wheel = doc["packages"][0] @@ -349,6 +369,7 @@ def test_generate_sbom_upstream_purl_override(tmp_path: pathlib.Path) -> None: ctx=ctx, req=Requirement("test-pkg==0.2.0"), version=Version("0.2.0"), + metadata=_metadata(), ) wheel = doc["packages"][0] @@ -374,6 +395,7 @@ def test_generate_sbom_canonicalizes_name(tmp_path: pathlib.Path) -> None: ctx=ctx, req=Requirement("My_Package==1.0.0"), version=Version("1.0.0"), + metadata=_metadata(), ) wheel = doc["packages"][0] @@ -390,6 +412,7 @@ def test_generate_sbom_relationships(tmp_path: pathlib.Path) -> None: ctx=ctx, req=Requirement("test==0.1.0"), version=Version("0.1.0"), + metadata=_metadata(), ) wheel_id, upstream_id = (p["SPDXID"] for p in doc["packages"]) @@ -415,6 +438,7 @@ def test_generate_sbom_spdx_ids_name_package_and_document( ctx=ctx, req=Requirement("test==0.1.0"), version=Version("0.1.0"), + metadata=_metadata(), ) assert [p["SPDXID"] for p in doc["packages"]] == [ @@ -437,7 +461,12 @@ def test_generate_sbom_spdx_ids_differ_between_packages( fixed_uuid = uuid.UUID("12345678-1234-5678-1234-567812345678") with patch("fromager.sbom.uuid.uuid4", return_value=fixed_uuid): docs = [ - sbom.generate_sbom(ctx=ctx, req=Requirement(req), version=Version(ver)) + sbom.generate_sbom( + ctx=ctx, + req=Requirement(req), + version=Version(ver), + metadata=_metadata(), + ) for req, ver in [("idna==3.6", "3.6"), ("packaging==26.3", "26.3")] ] @@ -456,7 +485,10 @@ def test_generate_sbom_spdx_ids_differ_between_documents( ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) docs = [ sbom.generate_sbom( - ctx=ctx, req=Requirement("idna==3.6"), version=Version("3.6") + ctx=ctx, + req=Requirement("idna==3.6"), + version=Version("3.6"), + metadata=_metadata(), ) for _ in range(2) ] @@ -476,6 +508,7 @@ def test_generate_sbom_spdx_ids_are_sanitized(tmp_path: pathlib.Path) -> None: ctx=ctx, req=Requirement("test-pkg==0.5.5"), version=Version("0.5.5"), + metadata=_metadata(), ) wheel_id, upstream_id = (p["SPDXID"] for p in doc["packages"]) @@ -494,6 +527,7 @@ def test_generate_sbom_upstream_supplier(tmp_path: pathlib.Path) -> None: ctx=ctx, req=Requirement("numpy==1.26.0"), version=Version("1.26.0"), + metadata=_metadata(), ) wheel = doc["packages"][0] @@ -513,6 +547,7 @@ def test_write_sbom_creates_file(tmp_path: pathlib.Path) -> None: ctx=ctx, req=Requirement("pkg==1.0"), version=Version("1.0"), + metadata=_metadata(), ) result = sbom.write_sbom(sbom=doc, dist_info_dir=dist_info_dir) @@ -537,6 +572,7 @@ def test_write_sbom_preserves_existing_files(tmp_path: pathlib.Path) -> None: ctx=ctx, req=Requirement("pkg==1.0"), version=Version("1.0"), + metadata=_metadata(), ) sbom.write_sbom(sbom=doc, dist_info_dir=dist_info_dir) @@ -545,3 +581,184 @@ def test_write_sbom_preserves_existing_files(tmp_path: pathlib.Path) -> None: assert json.loads(existing.read_text())["bomFormat"] == "CycloneDX" # New file should also exist assert (sboms_dir / "fromager.spdx.json").exists() + + +def _license_sbom( + tmp_path: pathlib.Path, **fields: typing.Any +) -> dict[str, typing.Any]: + """Generate an SBOM for a package with the given license metadata.""" + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + return sbom.generate_sbom( + ctx=ctx, + req=Requirement("test==1.0"), + version=Version("1.0"), + metadata=_metadata(**fields), + ) + + +def test_license_ref_id_matches_issue_example() -> None: + """Verify the LicenseRef id reproduces the example agreed in #1378.""" + result = sbom._license_ref_id("License :: OSI Approved :: BSD License") + + assert result == "LicenseRef-license-osi-approved-bsd-license-a9d12a80" + + +def test_license_ref_id_without_ascii_slug() -> None: + """Verify ids stay valid when the text has no leading ASCII words.""" + assert re.fullmatch(r"LicenseRef-[0-9a-f]{8}", sbom._license_ref_id("版权所有")) + assert sbom._license_ref_id("=====\nThe Kiwi licensing terms").startswith( + "LicenseRef-the-kiwi-licensing-terms-" + ) + + +def test_generate_sbom_license_from_expression(tmp_path: pathlib.Path) -> None: + """Verify a valid License-Expression is used verbatim on both packages.""" + doc = _license_sbom(tmp_path, license_expression="Apache-2.0 OR BSD-2-Clause") + + for package in doc["packages"]: + assert package["licenseDeclared"] == "Apache-2.0 OR BSD-2-Clause" + assert package["licenseConcluded"] == "NOASSERTION" + assert package["licenseComments"] == sbom._LICENSE_COMMENT.format( + source="License-Expression" + ) + assert "hasExtractedLicensingInfos" not in doc + _validate_spdx(doc) + + +def test_generate_sbom_license_ambiguous_classifier(tmp_path: pathlib.Path) -> None: + """Verify an ambiguous classifier yields the Option A output from #1378.""" + doc = _license_sbom( + tmp_path, classifiers=["License :: OSI Approved :: BSD License"] + ) + + expected = { + "licenseDeclared": "LicenseRef-license-osi-approved-bsd-license-a9d12a80", + "licenseConcluded": "NOASSERTION", + "licenseComments": ( + "License inherited from the upstream project's declared Python " + "package metadata (License classifiers) and not independently " + "verified by fromager; it may be imprecise or incomplete. No " + "unambiguous SPDX mapping for: License :: OSI Approved :: BSD License." + ), + } + for package in doc["packages"]: + assert {key: package[key] for key in expected} == expected + assert doc["hasExtractedLicensingInfos"] == [ + { + "licenseId": "LicenseRef-license-osi-approved-bsd-license-a9d12a80", + "extractedText": "License :: OSI Approved :: BSD License", + } + ] + _validate_spdx(doc) + + +def test_generate_sbom_no_license_metadata(tmp_path: pathlib.Path) -> None: + """Verify missing license data is reported as NOASSERTION.""" + doc = _license_sbom(tmp_path) + + for package in doc["packages"]: + assert package["licenseDeclared"] == "NOASSERTION" + assert package["licenseConcluded"] == "NOASSERTION" + assert package["licenseComments"] == sbom._NO_LICENSE_COMMENT + assert "hasExtractedLicensingInfos" not in doc + _validate_spdx(doc) + + +def test_generate_sbom_license_ref_in_expression(tmp_path: pathlib.Path) -> None: + """Verify an upstream LicenseRef gets the extracted info SPDX requires.""" + doc = _license_sbom(tmp_path, license_expression="MIT OR LicenseRef-Proprietary") + + assert doc["packages"][0]["licenseDeclared"] == "MIT OR LicenseRef-Proprietary" + assert doc["hasExtractedLicensingInfos"] == [ + { + "licenseId": "LicenseRef-Proprietary", + "name": "Proprietary", + "extractedText": "NOASSERTION", + "comment": ( + "Declared in the upstream project's License-Expression: " + "MIT OR LicenseRef-Proprietary. The license text is not " + "available in the package metadata." + ), + } + ] + _validate_spdx(doc) + + +def test_generate_sbom_license_refs_listed_once_in_order( + tmp_path: pathlib.Path, +) -> None: + """Verify each distinct upstream LicenseRef gets exactly one entry.""" + expression = "LicenseRef-B OR (MIT AND LicenseRef-A) OR LicenseRef-B" + doc = _license_sbom(tmp_path, license_expression=expression) + + assert doc["packages"][0]["licenseDeclared"] == expression + assert [e["licenseId"] for e in doc["hasExtractedLicensingInfos"]] == [ + "LicenseRef-B", + "LicenseRef-A", + ] + _validate_spdx(doc) + + +def test_generate_sbom_curated_license_ref(tmp_path: pathlib.Path) -> None: + """Verify a curated mapping to a LicenseRef also gets extracted info.""" + doc = _license_sbom(tmp_path, license="NVIDIA Proprietary Software") + + assert doc["packages"][0]["licenseDeclared"] == "LicenseRef-NVIDIA-SOFTWARE-LICENSE" + assert doc["hasExtractedLicensingInfos"] == [ + { + "licenseId": "LicenseRef-NVIDIA-SOFTWARE-LICENSE", + "name": "NVIDIA-SOFTWARE-LICENSE", + "extractedText": "NOASSERTION", + "comment": ( + "Declared in the upstream project's License: NVIDIA Proprietary " + "Software. The license text is not available in the package " + "metadata." + ), + } + ] + _validate_spdx(doc) + + +def test_generate_sbom_long_license_text(tmp_path: pathlib.Path) -> None: + """Verify folded full license text keeps ids and comments short.""" + text = "Copyright (c) 2005-2023, NumPy Developers.\n" + "\n".join( + f" line {i} of the license text" for i in range(2000) + ) + + doc = _license_sbom(tmp_path, license=text) + + package = doc["packages"][0] + (extracted,) = doc["hasExtractedLicensingInfos"] + assert extracted["licenseId"] == package["licenseDeclared"] + assert not any( + line.startswith(" ") for line in extracted["extractedText"].splitlines() + ) + slug = package["licenseDeclared"].removeprefix("LicenseRef-").rsplit("-", 1)[0] + assert len(slug) <= 40 + assert len(package["licenseComments"]) <= ( + len(sbom._LICENSE_COMMENT) + + len(sbom._UNMAPPED_COMMENT) + + sbom._COMMENT_RAW_MAX + + 10 + ) + _validate_spdx(doc) + + +def test_generate_sbom_license_with_unparsed_field(tmp_path: pathlib.Path) -> None: + """Verify a found license keeps its comment and names unreadable fields.""" + ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + doc = sbom.generate_sbom( + ctx=ctx, + req=Requirement("test==1.0"), + version=Version("1.0"), + metadata=_metadata(license_expression="MIT"), + unparsed_fields=["Classifier"], + ) + + package = doc["packages"][0] + assert package["licenseDeclared"] == "MIT" + assert package["licenseComments"] == ( + sbom._LICENSE_COMMENT.format(source="License-Expression") + + " Unparseable fields: Classifier." + ) + _validate_spdx(doc) diff --git a/tests/test_wheels.py b/tests/test_wheels.py index 5d98a74be..f9fd22f0f 100644 --- a/tests/test_wheels.py +++ b/tests/test_wheels.py @@ -1,4 +1,5 @@ import json +import logging import pathlib import typing import zipfile @@ -823,3 +824,98 @@ def hook(**kwargs: object) -> list[str]: ) assert result == () assert calls == [] + + +def _sbom_of_built_wheel( + tmp_path: pathlib.Path, metadata: bytes | None +) -> dict[str, typing.Any]: + """Run `add_extra_metadata_to_wheels()` on a fake wheel, return its SBOM. + + *metadata* is the wheel's METADATA file, or ``None`` to omit it. + """ + sbom_ctx = make_sbom_ctx(tmp_path, sbom_settings=SbomSettings()) + sbom_ctx.setup() + wheel_dir = tmp_path / "wheel_build" + wheel_dir.mkdir() + wheel_file = wheel_dir / "test_pkg-1.0.0-py3-none-any.whl" + with zipfile.ZipFile(wheel_file, "w") as zf: + zf.writestr("test_pkg/__init__.py", "") + if metadata is not None: + zf.writestr("test_pkg-1.0.0.dist-info/METADATA", metadata) + zf.writestr( + "test_pkg-1.0.0.dist-info/WHEEL", + "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + sdist_dir = tmp_path / "sdist" + sdist_dir.mkdir() + captured: dict[str, typing.Any] = {} + + def fake_run(cmd: list[str], **kwargs: object) -> str: + # wheel pack receives the unpacked wheel directory + sbom_file = ( + pathlib.Path(cmd[2]) / "test_pkg-1.0.0.dist-info/sboms/fromager.spdx.json" + ) + captured.update(json.loads(sbom_file.read_text())) + (wheel_dir / "test_pkg-1.0.0-0-py3-none-any.whl").touch() + return "" + + with patch("fromager.external_commands.run", side_effect=fake_run): + wheels.add_extra_metadata_to_wheels( + ctx=sbom_ctx, + req=Requirement("test_pkg==1.0.0"), + version=Version("1.0.0"), + extra_environ={}, + sdist_root_dir=sdist_dir, + wheel_file=wheel_file, + ) + return captured + + +def test_add_extra_metadata_sbom_includes_license(tmp_path: pathlib.Path) -> None: + """Verify the SBOM license comes from the wheel's own METADATA. + + The expression is invalid and the metadata version too old for it, so + validating `Metadata` access would fail; classifiers must be used. + """ + doc = _sbom_of_built_wheel( + tmp_path, + b"Metadata-Version: 2.1\nName: test_pkg\nVersion: 1.0.0\n" + b"License-Expression: Apache 2\n" + b"Classifier: License :: OSI Approved :: MIT License\n", + ) + + for package in doc["packages"]: + assert package["licenseDeclared"] == "MIT" + assert "(License classifiers)" in doc["packages"][0]["licenseComments"] + + +def test_add_extra_metadata_sbom_unparsed_license_field( + tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture +) -> None: + """Verify unreadable license fields are not reported as absent.""" + with caplog.at_level(logging.WARNING): + doc = _sbom_of_built_wheel( + tmp_path, + b"Metadata-Version: 2.4\nName: test_pkg\nVersion: 1.0.0\n" + b"License-Expression: MIT\nLicense-Expression: BSD-3-Clause\n", + ) + + package = doc["packages"][0] + assert package["licenseDeclared"] == "NOASSERTION" + assert package["licenseComments"] == ( + "License information could not be read from the upstream project's " + "declared Python package metadata. Unparseable fields: License-Expression." + ) + assert any( + "test_pkg" in r.getMessage() and "License-Expression" in r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING + ) + + +def test_add_extra_metadata_sbom_missing_metadata_fails( + tmp_path: pathlib.Path, +) -> None: + """Verify a wheel without METADATA fails instead of a misleading SBOM.""" + with pytest.raises(FileNotFoundError, match="METADATA"): + _sbom_of_built_wheel(tmp_path, None) From 0a4167a32076b484e8fed7a2eba26a76e097c346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Prpi=C4=8D?= Date: Wed, 4 Nov 2026 14:18:48 -0500 Subject: [PATCH 3/4] docs(sbom): describe SBOM license fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Explain where the SBOM license fields come from, why `licenseConcluded` is always `NOASSERTION`, and that a built wheel without METADATA fails when SBOMs are enabled. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Martin Prpič --- src/fromager/packagesettings/_models.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/fromager/packagesettings/_models.py b/src/fromager/packagesettings/_models.py index 5ff69fc82..948d8dfd5 100644 --- a/src/fromager/packagesettings/_models.py +++ b/src/fromager/packagesettings/_models.py @@ -92,6 +92,19 @@ class SbomSettings(pydantic.BaseModel): repository_url: "https://example.com/simple" creators: - "Organization: ExampleCo" + + Both package entries in the SBOM carry ``licenseDeclared`` from the + license the built wheel declares in its ``METADATA``. Sources are tried + in order: ``License-Expression``, unambiguous license classifiers, then + a ``License`` value that is an exact SPDX expression or a known common + spelling. Anything else is recorded as a ``LicenseRef-`` id with the raw + declaration in ``hasExtractedLicensingInfos``. ``licenseConcluded`` is + always ``NOASSERTION`` because fromager does not audit licenses, and + ``licenseComments`` says the value is inherited from upstream. A built + wheel without ``METADATA`` fails when SBOMs are enabled. + + .. versionchanged:: 0.104.0 + Populate SBOM license fields from wheel metadata. """ model_config = MODEL_CONFIG From 0876e82b7e4ed9361d8d2f2668d7dd93b02ab543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Prpi=C4=8D?= Date: Wed, 4 Nov 2026 16:33:10 -0500 Subject: [PATCH 4/4] test(e2e): check SBOM license fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bootstrap idna 3.6 and tomli 2.0.1 with SBOM generation enabled and check the license fields in each embedded SBOM. idna only has the ambiguous "BSD License" classifier and must get the `LicenseRef` with extracted licensing info agreed in #1378; tomli maps cleanly to `MIT`. Every built wheel, including build dependencies, must also have matching license fields on both packages, `licenseConcluded: NOASSERTION`, and extracted info for each `LicenseRef` it uses. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Martin Prpič --- e2e/ci_config_suite.sh | 1 + e2e/sbom_settings.yaml | 3 ++ e2e/test_sbom_licenses.sh | 82 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 e2e/sbom_settings.yaml create mode 100755 e2e/test_sbom_licenses.sh diff --git a/e2e/ci_config_suite.sh b/e2e/ci_config_suite.sh index c880d9133..62f6dfc7b 100755 --- a/e2e/ci_config_suite.sh +++ b/e2e/ci_config_suite.sh @@ -15,6 +15,7 @@ test_section "configuration tests" run_test "build_settings" run_test "override" run_test "extra_metadata" +run_test "sbom_licenses" run_test "lint_requirements" finish_suite diff --git a/e2e/sbom_settings.yaml b/e2e/sbom_settings.yaml new file mode 100644 index 000000000..d025f44eb --- /dev/null +++ b/e2e/sbom_settings.yaml @@ -0,0 +1,3 @@ +sbom: + supplier: "Organization: ExampleCo" + namespace: "https://sbom.test" diff --git a/e2e/test_sbom_licenses.sh b/e2e/test_sbom_licenses.sh new file mode 100755 index 000000000..3412d916a --- /dev/null +++ b/e2e/test_sbom_licenses.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test that the SBOM embedded in each built wheel carries the license the +# wheel declares in its METADATA (issue #1378). +# +# idna 3.6 only has the ambiguous "BSD License" classifier, so it must get +# a LicenseRef with extracted licensing info. tomli 2.0.1 has the "MIT +# License" classifier, which maps cleanly to MIT. + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +fromager \ + --log-file="$OUTDIR/bootstrap.log" \ + --error-log-file="$OUTDIR/fromager-errors.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + --settings-file="$SCRIPTDIR/sbom_settings.yaml" \ + bootstrap 'idna==3.6' 'tomli==2.0.1' + +find "$OUTDIR/wheels-repo/downloads/" -name '*.whl' + +python3 - "$OUTDIR/wheels-repo/downloads" <<'EOF' +import json +import pathlib +import re +import sys +import zipfile + +downloads = pathlib.Path(sys.argv[1]) +failures = [] + + +def sbom_of(wheel): + with zipfile.ZipFile(wheel) as zf: + (name,) = [n for n in zf.namelist() if n.endswith(".dist-info/sboms/fromager.spdx.json")] + return json.loads(zf.read(name)) + + +def check(condition, message): + if not condition: + failures.append(message) + + +# invariants for every built wheel, including build dependencies +wheels = sorted(downloads.glob("*.whl")) +check(wheels, f"no wheels found in {downloads}") +for wheel in wheels: + doc = sbom_of(wheel) + wheel_pkg, upstream_pkg = doc["packages"] + for key in ("licenseDeclared", "licenseConcluded", "licenseComments"): + check(key in wheel_pkg and key in upstream_pkg, f"{wheel.name}: {key} missing from a package") + check(wheel_pkg.get(key) == upstream_pkg.get(key), f"{wheel.name}: {key} differs between packages") + check(wheel_pkg.get("licenseConcluded") == "NOASSERTION", f"{wheel.name}: licenseConcluded is not NOASSERTION") + declared_ids = {e["licenseId"] for e in doc.get("hasExtractedLicensingInfos", [])} + for ref in re.findall(r"LicenseRef-[A-Za-z0-9.\-]+", wheel_pkg.get("licenseDeclared", "")): + check(ref in declared_ids, f"{wheel.name}: {ref} has no extracted licensing info") + +# ambiguous classifier: LicenseRef plus extracted licensing info +idna = sbom_of(next(downloads.glob("idna-3.6-*.whl"))) +idna_ref = "LicenseRef-license-osi-approved-bsd-license-a9d12a80" +idna_pkg = idna["packages"][0] +check(idna_pkg.get("licenseDeclared") == idna_ref, f"idna: licenseDeclared is {idna_pkg.get('licenseDeclared')!r}") +check( + idna.get("hasExtractedLicensingInfos") == [{"licenseId": idna_ref, "extractedText": "License :: OSI Approved :: BSD License"}], + f"idna: unexpected hasExtractedLicensingInfos {idna.get('hasExtractedLicensingInfos')!r}", +) +check("(License classifiers)" in idna_pkg.get("licenseComments", ""), "idna: comment does not name the source") + +# unambiguous classifier: plain SPDX id, no extracted licensing info +tomli = sbom_of(next(downloads.glob("tomli-2.0.1-*.whl"))) +tomli_pkg = tomli["packages"][0] +check(tomli_pkg.get("licenseDeclared") == "MIT", f"tomli: licenseDeclared is {tomli_pkg.get('licenseDeclared')!r}") +check("hasExtractedLicensingInfos" not in tomli, "tomli: unexpected hasExtractedLicensingInfos") + +for failure in failures: + print(f"FAIL: {failure}", file=sys.stderr) +print(f"checked {len(wheels)} wheels, {len(failures)} failure(s)") +sys.exit(1 if failures else 0) +EOF