diff --git a/README.md b/README.md index 7addf99..584811e 100644 --- a/README.md +++ b/README.md @@ -147,7 +147,9 @@ uv run licensecheck --only-licenses mit apache --show-only-failing -g dev usage: licensecheck [-h] [--license LICENSE] [--format FORMAT] [--requirements-paths REQUIREMENTS_PATHS [REQUIREMENTS_PATHS ...]] [--groups GROUPS [GROUPS ...]] [--extras EXTRAS [EXTRAS ...]] [--file FILE] [--ignore-packages IGNORE_PACKAGES [IGNORE_PACKAGES ...]] [--fail-packages FAIL_PACKAGES [FAIL_PACKAGES ...]] - [--ignore-licenses IGNORE_LICENSES [IGNORE_LICENSES ...]] [--fail-licenses FAIL_LICENSES [FAIL_LICENSES ...]] + [--ignore-licenses IGNORE_LICENSES [IGNORE_LICENSES ...]] + [--allowed-license-references ALLOWED_LICENSE_REFERENCES [ALLOWED_LICENSE_REFERENCES ...]] + [--fail-licenses FAIL_LICENSES [FAIL_LICENSES ...]] [--only-licenses ONLY_LICENSES [ONLY_LICENSES ...]] [--skip-dependencies SKIP_DEPENDENCIES [SKIP_DEPENDENCIES ...]] [--hide-output-parameters HIDE_OUTPUT_PARAMETERS [HIDE_OUTPUT_PARAMETERS ...]] [--show-only-failing] @@ -169,11 +171,13 @@ options: Select extras from supported files --file FILE, -o FILE Filename to write output to (omit this for stdout) --ignore-packages IGNORE_PACKAGES [IGNORE_PACKAGES ...] - List of packages/dependencies to ignore (compat=True), globs are supported + List of packages/dependencies to ignore (compat=True); names, name==version, and globs are supported --fail-packages FAIL_PACKAGES [FAIL_PACKAGES ...] - List of packages/dependencies to fail (compat=False), globs are supported + List of packages/dependencies to fail (compat=False); names, name==version, and globs are supported --ignore-licenses IGNORE_LICENSES [IGNORE_LICENSES ...] List of licenses to ignore (skipped, compat may still be False) + --allowed-license-references ALLOWED_LICENSE_REFERENCES [ALLOWED_LICENSE_REFERENCES ...] + List of exact LicenseRef-* identifiers to accept --fail-licenses FAIL_LICENSES [FAIL_LICENSES ...] List of licenses to fail (compat=False) --only-licenses ONLY_LICENSES [ONLY_LICENSES ...] @@ -210,9 +214,11 @@ requirements_paths = [] # List of filenames to read from groups = [] # List of selected groups extras = [] # List of selected extras file = "" # Output file (leave empty for stdout) -ignore_packages = [] # Packages/dependencies to ignore -fail_packages = [] # Packages/dependencies that cause failure +ignore_packages = [] # Names, name==version entries, or globs to ignore +license_overrides = { "sample==1.2.3" = "BSD-3-Clause" } # Reviewed licenses for exact versions +fail_packages = [] # Names, name==version entries, or globs that fail ignore_licenses = [] # Licenses to ignore +allowed_license_references = [] # Exact LicenseRef identifiers to accept fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) skip_dependencies = [] # Dependencies to skip (compatibility = True) @@ -237,7 +243,9 @@ zero = false # Return non-zero exit code for incompatible licen "groups": [], "hide_output_parameters": [], "ignore_licenses": [], + "allowed_license_references": [], "ignore_packages": [], + "license_overrides": {"sample==1.2.3": "BSD-3-Clause"}, "license": "mit", "only_licenses": [], "pypi_api": "https://pypi.org", diff --git a/documentation/user/README.md b/documentation/user/README.md index 128b3e4..54fa614 100644 --- a/documentation/user/README.md +++ b/documentation/user/README.md @@ -91,9 +91,11 @@ uv run licensecheck --only-licenses mit apache --show-only-failing -g dev ## Supported tools/ standards Licensecheck supports a broad range of different tools and workflows. Though please note that -for some of these tools, behaviour may differ from what is expected. We use `uv` for the dependency -resolution due to the good performance across projects, with a fallback to a native parser in case of -an error, which will be logged +for some of these tools, behaviour may differ from what is expected. For a `pyproject.toml` with an +adjacent `uv.lock`, Licensecheck exports the locked dependency graph. Otherwise it uses `uv` to +resolve dependencies. Resolution errors for a `pyproject.toml` are reported directly instead of +falling back to a less accurate dependency graph. Editable local projects emitted by `uv` are +audited using the project metadata from their own `pyproject.toml` files. Note that `uv` supports requirements.in files. If a pyproject.toml, setup.py, or setup.cfg file is provided, `uv` will extract the requirements for the relevant project. In testing this seems to have @@ -158,19 +160,20 @@ classifiers = [ Previous versions of the licensecheck tool implemented a custom resolver to discover packages. Current versions look to move away from this for a number of reasons, such as correctness and reducing the maintenance burden. Over time many contributors helped out with the custom -resolver which is very much appreciated. Now, we use `uv` to attempt to resolve deps before -falling back to the legacy approach, which is needed in certain cases where uv fails +resolver which is very much appreciated. Now, we use `uv` to export an adjacent lockfile or resolve +dependencies from the supplied project file. The legacy parser remains for formats that `uv pip +compile` does not accept directly. Q: Why doesn't this use packages from my lockfile? -A: The answer to this somewhat depends on what resolver licensecheck ends up using to -find all of the packages in use by your project. Ideally, `uv` is used which has pretty -good support for pyproject.toml and some other standard requirements formats and will discover -packages. The legacy resolver is deprecated and may result in funky output in some cases +A: When a `uv.lock` is adjacent to a supplied `pyproject.toml`, Licensecheck uses `uv export +--locked` and audits those exact versions. Without an adjacent lockfile it resolves the supplied +requirements with `uv`. Q: The license for my dep has changed in v >1.0, so I'm using v < 1.0, why doesn't licensecheck report the correct license version? -A: In some cases it will, for example if licensecheck can find the dep metadata via importlib. Otherwise we reach out to pypi.org for this metadata. There are no plans at present to resolve this - -[note to me: I 'might' look at this as we should have a copy of the package version ] +A: Licensecheck requests the exact resolved version from the PyPI JSON API. If the package is not +available from public PyPI or that response lacks usable license metadata, it asks `uv` to fetch +the exact wheel from the indexes configured for the current project and reads the wheel metadata. +Index credentials and source selection remain `uv`'s responsibility. ## License lookup format diff --git a/licensecheck/checker.py b/licensecheck/checker.py index 5f8f3c9..d575ded 100644 --- a/licensecheck/checker.py +++ b/licensecheck/checker.py @@ -4,6 +4,11 @@ from fnmatch import fnmatch +from loguru import logger +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name + from licensecheck import license_matrix from licensecheck.models.constants import JOINS from licensecheck.models.license import License @@ -11,23 +16,84 @@ from licensecheck.packageinforesolver import PackageInfoManager +def _package_matches(package: PackageInfo, patterns: set[str]) -> bool: + package_names = {package.name.upper()} + if package.version: + package_names.add(f"{package.name}=={package.version}".upper()) + return any( + fnmatch(package_name, pattern.upper()) + for package_name in package_names + for pattern in patterns + ) + + +def _matches_allowed_license_reference( + allowed_license_references: set[str], dependency_license: str +) -> bool: + dependency_license = dependency_license.strip().casefold() + return dependency_license.startswith("licenseref-") and dependency_license in { + license_ref.strip().casefold() for license_ref in allowed_license_references + } + + +def _parse_license_overrides( + license_overrides: dict[str, str], +) -> list[tuple[str, SpecifierSet, str]]: + """Parse the configured overrides once, rather than once per package.""" + parsed: list[tuple[str, SpecifierSet, str]] = [] + for package_requirement, license_value in license_overrides.items(): + requirement = Requirement(package_requirement) + if not requirement.specifier: + # An unversioned override would silently apply to every version of the package. + logger.warning( + f"Ignoring license override '{package_requirement}': " + f"an exact name==version is required" + ) + continue + parsed.append( + (canonicalize_name(requirement.name), requirement.specifier, license_value.strip()) + ) + return parsed + + +def _license_override( + package: PackageInfo, license_overrides: list[tuple[str, SpecifierSet, str]] +) -> str | None: + if package.version is None: + return None + package_name = canonicalize_name(package.name) + for name, specifier, license_value in license_overrides: + if name == package_name and specifier.contains(package.version, prereleases=True): + return license_value + return None + + def check( requirements_paths: set[str], groups: set[str], extras: set[str], this_license: License, package_info_manager: PackageInfoManager, + *, + this_license_text: str | None = None, ignore_packages: set[str] | None = None, + license_overrides: dict[str, str] | None = None, fail_packages: set[str] | None = None, ignore_licenses: set[str] | None = None, + allowed_license_references: set[str] | None = None, fail_licenses: set[str] | None = None, only_licenses: set[str] | None = None, skip_dependencies: set[str] | None = None, ) -> tuple[bool, set[PackageInfo]]: # Def values ignore_packages = ignore_packages or set() + parsed_license_overrides = _parse_license_overrides(license_overrides or {}) fail_packages = fail_packages or set() ignore_licenses = ignore_licenses or set() + # The project's own license reference is always an accepted reference + allowed_license_references = (allowed_license_references or set()) | ( + {this_license_text} if this_license_text else set() + ) fail_licenses = fail_licenses or set() only_licenses = only_licenses or set() skip_dependencies = skip_dependencies or set() @@ -51,13 +117,19 @@ def check( # Check it is compatible with packages and add a note packages = package_info_manager.getPackages() for package in packages: + if override := _license_override(package, parsed_license_overrides): + package.license = override + package.licenseSource = "configured override" # Deal with --ignore-packages and --fail-packages package.licenseCompat = False - packageName = package.name.upper() - if any(fnmatch(packageName, pattern.upper()) for pattern in ignore_packages): + if _package_matches(package, ignore_packages): package.licenseCompat = True - elif any(fnmatch(packageName, pattern.upper()) for pattern in fail_packages): + elif _package_matches(package, fail_packages): pass # package.licenseCompat = False + elif license_matrix.licenseType(str(package.license), ignore_licenses) & failLicensesType: + pass + elif _matches_allowed_license_reference(allowed_license_references, str(package.license)): + package.licenseCompat = True # Else get compat with myLice else: package.licenseCompat = license_matrix.depCompatWMyLice( diff --git a/licensecheck/io/cli.py b/licensecheck/io/cli.py index e8a4467..6a30060 100644 --- a/licensecheck/io/cli.py +++ b/licensecheck/io/cli.py @@ -66,12 +66,18 @@ def cli() -> None: # pragma: no cover ) parser.add_argument( "--ignore-packages", - help="set of packages/dependencies to ignore (compat=True), globs are supported", + help=( + "set of packages/dependencies to ignore (compat=True); names, " + "name==version, and globs are supported" + ), nargs="+", ) parser.add_argument( "--fail-packages", - help="set of packages/dependencies to fail (compat=False), globs are supported", + help=( + "set of packages/dependencies to fail (compat=False); names, " + "name==version, and globs are supported" + ), nargs="+", ) parser.add_argument( @@ -79,6 +85,11 @@ def cli() -> None: # pragma: no cover help="set of licenses to ignore (skipped, compat may still be False)", nargs="+", ) + parser.add_argument( + "--allowed-license-references", + help="set of exact LicenseRef-* identifiers to accept", + nargs="+", + ) parser.add_argument( "--fail-licenses", help="set of licenses to fail (compat=False)", @@ -164,10 +175,13 @@ def main(licensecheckConf: LC_Config) -> ExitCode: groups=licensecheckConf.groups, extras=licensecheckConf.extras, this_license=this_license, + this_license_text=this_license_text, package_info_manager=package_info_manager, ignore_packages=licensecheckConf.ignore_packages, + license_overrides=licensecheckConf.license_overrides, fail_packages=licensecheckConf.fail_packages, ignore_licenses=licensecheckConf.ignore_licenses, + allowed_license_references=licensecheckConf.allowed_license_references, fail_licenses=licensecheckConf.fail_licenses, only_licenses=licensecheckConf.only_licenses, skip_dependencies=licensecheckConf.skip_dependencies, diff --git a/licensecheck/io/fmt.py b/licensecheck/io/fmt.py index 0d71b2d..bdb21cc 100644 --- a/licensecheck/io/fmt.py +++ b/licensecheck/io/fmt.py @@ -163,6 +163,9 @@ def ansi( table.add_column("Package", style="magenta") if license_bool := "license" in packages[0]: table.add_column("License(s)", style="magenta") + license_source_bool = any("licenseSource" in package for package in packages) + if license_source_bool: + table.add_column("License Source", style="magenta") licenseCompat = ( "[red]✖[/]", "[green]✔[/]", @@ -173,6 +176,7 @@ def ansi( ([licenseCompat[x.get("licenseCompat", 0)]] if licensecompat_bool else []) + ([x.get("name")] if name_bool else []) + ([x.get("license")] if license_bool else []) + + ([x.get("licenseSource", "")] if license_source_bool else []) ) ) for x in packages @@ -227,6 +231,7 @@ def markdown( "homePage": "HomePage", "author": "Author", "license": "License", + "licenseSource": "License Source", "licenseCompat": "Compatible", "size": "Size", } @@ -295,7 +300,8 @@ def rawCsv( _ = myLice string = StringIO() - writer = csv.DictWriter(string, fieldnames=list(packages[0]), lineterminator="\n") + fieldnames = list(dict.fromkeys(key for package in packages for key in package)) + writer = csv.DictWriter(string, fieldnames=fieldnames, lineterminator="\n") writer.writeheader() writer.writerows(packages) return string.getvalue() diff --git a/licensecheck/license_matrix/__init__.py b/licensecheck/license_matrix/__init__.py index c957595..40a48fa 100644 --- a/licensecheck/license_matrix/__init__.py +++ b/licensecheck/license_matrix/__init__.py @@ -52,6 +52,12 @@ with Path(THISDIR / "matrix.csv").open(mode="r", newline="", encoding="utf-8") as csv_file: LICENSE_MATRIX: list[list[str]] = list[list[str]](csv.reader(csv_file)) +# Look the matrix up by license name because the csv column/row order does not match. +LICENSE_MATRIX_ROWS: dict[str, list[str]] = {row[0]: row for row in LICENSE_MATRIX[1:]} +LICENSE_MATRIX_COLUMNS: dict[str, int] = { + name: index for index, name in enumerate(LICENSE_MATRIX[0]) +} + termToLicenseData = { "UNKNOWN": L.UNKNOWN, @@ -183,6 +189,9 @@ def depCompatWMyLice( ignoreLicenses = ignoreLicenses or set() onlyLicenses = onlyLicenses or set() + if depLice & failLicenses: + return False + return any( liceCompat( myLicense, @@ -218,11 +227,11 @@ def liceCompat( return True if len(onlyLicenses) > 0 and (lice not in onlyLicenses): return False - licenses = list(L) - row, col = licenses.index(myLicense) + 1, licenses.index(lice) + 1 + if myLicense in {L.UNKNOWN, L.NO_LICENSE} or lice in {L.UNKNOWN, L.NO_LICENSE}: + return False try: - return LICENSE_MATRIX[row][col] == "1" - except KeyError: + return LICENSE_MATRIX_ROWS[myLicense.name][LICENSE_MATRIX_COLUMNS[lice.name]] == "1" + except (IndexError, KeyError): logger.warning(f"Licenses {myLicense} and {lice} cannot be compared") return False diff --git a/licensecheck/models/config.py b/licensecheck/models/config.py index 79a859d..ab46c9c 100644 --- a/licensecheck/models/config.py +++ b/licensecheck/models/config.py @@ -4,6 +4,7 @@ from typing import Any, Literal from depgather.models.defaultonnone import DefaultOnNoneModel +from packaging.requirements import InvalidRequirement, Requirement from pydantic import field_validator from licensecheck.io.fmt import FMT @@ -23,8 +24,10 @@ class LC_Config(DefaultOnNoneModel): groups: set[str] = field(default_factory=set) extras: set[str] = field(default_factory=set) ignore_packages: set[str] = field(default_factory=set) + license_overrides: dict[str, str] = field(default_factory=dict) fail_packages: set[str] = field(default_factory=set) ignore_licenses: set[str] = field(default_factory=set) + allowed_license_references: set[str] = field(default_factory=set) fail_licenses: set[str] = field(default_factory=set) only_licenses: set[str] = field(default_factory=set) skip_dependencies: set[str] = field(default_factory=set) @@ -36,3 +39,29 @@ def normalize_format(cls, value: Any) -> Any | Literal[FMT.simple]: if value not in FMT: return FMT.simple return value + + @field_validator("license_overrides") + @classmethod + def validate_license_overrides(cls, value: dict[str, str]) -> dict[str, str]: + for package, license_value in value.items(): + try: + requirement = Requirement(package) + except InvalidRequirement as exc: + message = f"Invalid license override package: {package}" + raise ValueError(message) from exc + + specifiers = list(requirement.specifier) + if ( + requirement.url + or requirement.extras + or requirement.marker + or len(specifiers) != 1 + or specifiers[0].operator != "==" + or specifiers[0].version.endswith(".*") + ): + message = f"License override packages must use an exact name==version: {package}" + raise ValueError(message) + if not license_value.strip(): + message = f"License override must not be empty: {package}" + raise ValueError(message) + return value diff --git a/licensecheck/models/packageinfo.py b/licensecheck/models/packageinfo.py index 7d70380..8b07cb1 100644 --- a/licensecheck/models/packageinfo.py +++ b/licensecheck/models/packageinfo.py @@ -16,6 +16,7 @@ class PackageInfo: homePage: str | None = None author: str | None = None license: str | None = None + licenseSource: str | None = field(default=None, compare=False, hash=False) licenseCompat: bool = False errorCode: int = 0 @@ -35,4 +36,5 @@ def get_filtered_dict(self, hide_output_parameters: set[str]) -> dict: k: (v if v is not None else UNKNOWN) for k, v in self.__dict__.items() if k.upper() not in hide_output_parameters_upper + and not (k == "licenseSource" and v is None) } diff --git a/licensecheck/packageinforesolver.py b/licensecheck/packageinforesolver.py index 37c4f39..804d1be 100644 --- a/licensecheck/packageinforesolver.py +++ b/licensecheck/packageinforesolver.py @@ -4,23 +4,33 @@ import configparser import contextlib +import functools +import os import re +import subprocess +import tempfile from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, replace from email.message import Message from importlib import metadata from importlib.metadata._meta import PackageMetadata from pathlib import Path from typing import Any +from urllib.parse import urlparse +from urllib.request import url2pathname import license_expression import requests +import requirements import tomli from boolean.boolean import Expression from depgather.models.pypijson import ProjectResponse from depgather.parse import gather from license_expression import Licensing -from packaging.requirements import Requirement +from loguru import logger +from packaging.requirements import InvalidRequirement, Requirement from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, Version from licensecheck.models.constants import JOINS, UNKNOWN from licensecheck.models.packageinfo import PackageInfo @@ -29,6 +39,536 @@ RAW_JOINS = " AND " HTTP_OK = 200 +EXPLICIT_LICENSE_ALIASES = { + "Apache 2.0": "Apache-2.0", +} + +# uv can block indefinitely (for example prompting for index credentials), so cap each call +UV_RESOLVE_TIMEOUT_SECONDS = 900 +UV_ARTIFACT_TIMEOUT_SECONDS = 600 + + +class UvUnavailableError(RuntimeError): + """Raised when the optional uv executable is unavailable.""" + + +@dataclass(frozen=True) +class _UvIndex: + name: str | None + url: str + format: str | None = None + + +@dataclass(frozen=True) +class _UvResolutionContext: + directory: Path + base_index_url: str + source_url: str | None = None + index_args: tuple[str, ...] = () + index_environment: tuple[tuple[str, str], ...] = () + prefer_artifact: bool = False + remote_matches_source: bool = False + + +def _has_usable_license(license_value: str | None) -> bool: + return bool( + license_value + and license_value.strip() + and license_value.strip().upper() not in {UNKNOWN, "NONE"} + ) + + +@functools.lru_cache(maxsize=1) +def _spdx_licensing() -> Licensing: + """ + Build the SPDX licensing index once. + + ``license_expression.get_spdx_licensing`` re-reads and re-parses a large vendored JSON + index on every call, so it must not be called per package. + """ + return license_expression.get_spdx_licensing() + + +def _recognizable_explicit_license(license_value: str | None) -> str | None: + if not _has_usable_license(license_value): + return None + + value = str(license_value).strip() + if re.fullmatch(r"LicenseRef-[A-Za-z0-9.-]+", value, flags=re.IGNORECASE): + return value + if value in EXPLICIT_LICENSE_ALIASES: + return EXPLICIT_LICENSE_ALIASES[value] + + with contextlib.suppress(license_expression.ExpressionError): + _spdx_licensing().parse(value, validate=True) + return value + return None + + +def _license_from_metadata( + license_expression_value: str | None, + classifiers: list[str] | None, + legacy_license: str | None, +) -> str | None: + return ( + license_expression_value + or _recognizable_explicit_license(legacy_license) + or from_classifiers(classifiers) + or legacy_license + ) + + +def _versions_match(expected: str | None, actual: str | None) -> bool: + if expected is None: + return True + if actual is None: + return False + + try: + return Version(expected) == Version(actual) + except InvalidVersion: + return expected == actual + + +def _exact_requirement_version(requirement: Requirement | PackageInfo) -> str | None: + if not isinstance(requirement, Requirement): + return requirement.version + versions = { + item.version + for item in requirement.specifier + if item.operator in {"==", "==="} and "*" not in item.version + } + return next(iter(versions)) if len(versions) == 1 else None + + +def _requirement_key( + requirement: Requirement | PackageInfo, +) -> tuple[str, str | None, str | None]: + return ( + canonicalize_name(requirement.name), + _exact_requirement_version(requirement), + getattr(requirement, "url", None), + ) + + +@functools.cache +def _normalized_index_url(url: str) -> str: + parsed = urlparse(url) + if not parsed.scheme and Path(url).is_absolute(): + return Path(url).resolve().as_uri().rstrip("/") + if parsed.scheme == "file": + return Path(url2pathname(parsed.path)).resolve().as_uri().rstrip("/") + normalized = url.rstrip("/").removesuffix("/simple") + parsed = urlparse(normalized) + if parsed.hostname: + host = parsed.hostname.lower() + if parsed.port: + host = f"{host}:{parsed.port}" + normalized = parsed._replace(netloc=host, query="", fragment="").geturl() + return normalized + + +def _is_public_pypi(url: str) -> bool: + parsed = urlparse(_normalized_index_url(url)) + return parsed.hostname in {"pypi.org", "www.pypi.org"} and parsed.path in {"", "/"} + + +def _same_index(left: str, right: str) -> bool: + return _normalized_index_url(left) == _normalized_index_url(right) + + +def _read_uv_configuration(directory: Path) -> tuple[dict[str, Any], Path]: + for candidate_directory in (directory, *directory.parents): + uv_toml = candidate_directory / "uv.toml" + if uv_toml.is_file(): + return tomli.loads(uv_toml.read_text(encoding="utf-8")), candidate_directory + + pyproject_path = candidate_directory / "pyproject.toml" + if pyproject_path.is_file(): + pyproject = tomli.loads(pyproject_path.read_text(encoding="utf-8")) + uv_config = pyproject.get("tool", {}).get("uv") + if isinstance(uv_config, dict): + return uv_config, candidate_directory + + return {}, directory + + +def _resolved_uv_index_url(raw_url: object, config_directory: Path) -> str: + url = str(raw_url) + if Path(url).is_absolute() or not urlparse(url).scheme: + return (config_directory / url).resolve().as_uri() + return url + + +def _legacy_uv_indexes( + config: dict[str, Any], + config_directory: Path, +) -> list[_UvIndex]: + indexes: list[_UvIndex] = [] + default_index = config.get("index-url") + if default_index: + indexes.append( + _UvIndex( + name=None, + url=_resolved_uv_index_url(default_index, config_directory), + ) + ) + extra_indexes = config.get("extra-index-url", []) + if isinstance(extra_indexes, str): + extra_indexes = [extra_indexes] + indexes.extend( + _UvIndex(name=None, url=_resolved_uv_index_url(url, config_directory)) + for url in extra_indexes + ) + find_links = config.get("find-links", []) + if isinstance(find_links, str): + find_links = [find_links] + indexes.extend( + _UvIndex( + name=None, + url=_resolved_uv_index_url(url, config_directory), + format="flat", + ) + for url in find_links + ) + return indexes + + +def _configured_uv_indexes( + uv_config: dict[str, Any], + config_directory: Path, +) -> list[_UvIndex]: + raw_indexes = uv_config.get("index", []) + if isinstance(raw_indexes, dict): + raw_indexes = [raw_indexes] + + indexes: list[_UvIndex] = [] + for index in raw_indexes: + if not isinstance(index, dict) or not index.get("url"): + continue + indexes.append( + _UvIndex( + name=index.get("name"), + url=_resolved_uv_index_url(index["url"], config_directory), + format=index.get("format"), + ) + ) + + indexes.extend(_legacy_uv_indexes(uv_config, config_directory)) + pip_config = uv_config.get("pip", {}) + if isinstance(pip_config, dict): + indexes.extend(_legacy_uv_indexes(pip_config, config_directory)) + + return indexes + + +def _environment_uv_indexes() -> list[_UvIndex]: + indexes: list[_UvIndex] = [] + for variable in ("UV_INDEX", "UV_EXTRA_INDEX_URL"): + for value in os.environ.get(variable, "").split(): + if urlparse(value).scheme: + indexes.append(_UvIndex(name=None, url=value)) + continue + name, separator, url = value.partition("=") + indexes.append(_UvIndex(name=name if separator else None, url=url or name)) + indexes.extend( + _UvIndex(name=None, url=value) + for variable in ("UV_DEFAULT_INDEX", "UV_INDEX_URL") + if (value := os.environ.get(variable)) + ) + indexes.extend( + _UvIndex(name=None, url=value, format="flat") + for value in os.environ.get("UV_FIND_LINKS", "").split() + ) + return indexes + + +def _index_invocation_for_source( + source_url: str, + indexes: list[_UvIndex], +) -> tuple[tuple[str, ...], tuple[tuple[str, str], ...]]: + matching_index = next( + (index for index in indexes if _same_index(index.url, source_url)), + None, + ) + if matching_index is not None and matching_index.format == "flat": + parsed_index = urlparse(matching_index.url) + if parsed_index.username or parsed_index.password or parsed_index.query: + return ("--no-index",), (("UV_FIND_LINKS", matching_index.url),) + return ("--no-index", "--find-links", matching_index.url), () + + if matching_index is not None and matching_index.name: + parsed_index = urlparse(matching_index.url) + if parsed_index.username or parsed_index.password or parsed_index.query: + index_value = f"{matching_index.name}={matching_index.url}" + return (), (("UV_INDEX", index_value),) + return ( + ("--index", f"{matching_index.name}={source_url}"), + (), + ) + + if matching_index is not None and ( + urlparse(matching_index.url).username + or urlparse(matching_index.url).password + or urlparse(matching_index.url).query + ): + return (), (("UV_INDEX", matching_index.url),) + if urlparse(source_url).username or urlparse(source_url).password or urlparse(source_url).query: + return (), (("UV_INDEX", source_url),) + + return ("--index", source_url), () + + +def _annotated_requirement_sources( + raw_requirements: str, +) -> dict[tuple[str, str | None, str | None], str]: + sources: dict[tuple[str, str | None, str | None], str] = {} + current_requirement: Requirement | None = None + for raw_line in raw_requirements.splitlines(): + line = raw_line.strip() + if line.startswith("# from ") and current_requirement is not None: + sources[_requirement_key(current_requirement)] = line.removeprefix("# from ").strip() + continue + if not line or line.startswith("#"): + continue + if line.startswith(("-e ", "--editable ", "--")): + current_requirement = None + continue + try: + current_requirement = Requirement(line) + except InvalidRequirement: + current_requirement = None + return sources + + +def _locked_requirement_sources( + lock_path: Path, +) -> dict[tuple[str, str | None, str | None], str]: + if not lock_path.is_file(): + return {} + + lock = tomli.loads(lock_path.read_text(encoding="utf-8")) + sources: dict[tuple[str, str | None, str | None], str] = {} + for package in lock.get("package", []): + if not isinstance(package, dict): + continue + name = package.get("name") + version = package.get("version") + source = package.get("source", {}) + registry = source.get("registry") if isinstance(source, dict) else None + if name and registry: + sources[(canonicalize_name(name), version, None)] = str(registry) + return sources + + +def _editable_project_path(line: str, base_path: Path) -> Path | None: + stripped_line = line.strip() + for prefix in ("-e ", "--editable "): + if stripped_line.startswith(prefix): + target = stripped_line.removeprefix(prefix).strip() + break + else: + return None + + parsed_url = urlparse(target) + if parsed_url.scheme and parsed_url.scheme != "file": + return None + + if parsed_url.scheme == "file": + path = Path(url2pathname(parsed_url.path)) + else: + # A bare path is not a URL; only strip a trailing "#egg=" style fragment. + path = Path(target.split("#", 1)[0]) + if not path.is_absolute(): + path = base_path / path + return path.resolve() + + +def _requirement_project_path(requirement: Requirement, base_path: Path) -> Path | None: + if not requirement.url: + return None + parsed_url = urlparse(requirement.url) + if parsed_url.scheme != "file": + return None + + path = Path(url2pathname(parsed_url.path)) + if not path.is_absolute(): + path = base_path / path + return path.resolve() + + +def _parse_uv_requirements( + raw_requirements: str, + skip_dependencies: set[str], + base_path: Path, +) -> tuple[ + set[Requirement], + set[Path], + dict[tuple[str, str | None, str | None], str], +]: + skip_names = {canonicalize_name(name) for name in skip_dependencies} + parsed_requirements: set[Requirement] = set() + editable_paths: set[Path] = set() + + for parsed in requirements.parse(raw_requirements): + if parsed.editable: + if editable_path := _editable_project_path(parsed.line, base_path): + editable_paths.add(editable_path) + continue + if not parsed.name or canonicalize_name(parsed.name) in skip_names: + continue + + requirement = Requirement(parsed.line) + requirement.name = canonicalize_name(requirement.name) + parsed_requirements.add(requirement) + + return parsed_requirements, editable_paths, _annotated_requirement_sources(raw_requirements) + + +def _uv_requirement_command( + requirements_path: Path, + *, + use_lock: bool, + base_index_url: str, +) -> list[str]: + if use_lock: + return [ + "uv", + "export", + "--project", + requirements_path.parent.as_posix(), + "--format", + "requirements.txt", + "--locked", + "--no-hashes", + "--no-header", + "--no-default-groups", + "--no-emit-project", + "--color", + "never", + ] + + command = [ + "uv", + "pip", + "compile", + "--color", + "never", + "--emit-index-annotation", + ] + if not _is_public_pypi(base_index_url): + command.extend(["--default-index", base_index_url]) + command.append(requirements_path.as_posix()) + return command + + +def _gather_uv_requirements( + requirements_path: Path, + groups: set[str], + extras: set[str], + skip_dependencies: set[str], + base_index_url: str, +) -> tuple[ + set[Requirement], + set[Path], + dict[tuple[str, str | None, str | None], str], +]: + requirements_path = requirements_path.resolve() + lock_path = requirements_path.with_name("uv.lock") + use_lock = requirements_path.name == "pyproject.toml" and lock_path.is_file() + command = _uv_requirement_command( + requirements_path, + use_lock=use_lock, + base_index_url=base_index_url, + ) + for group in groups: + command.extend(["--group", group]) + for extra in extras: + command.extend(["--extra", extra]) + + if not use_lock and requirements_path.name == "pyproject.toml": + pyproject = tomli.loads(requirements_path.read_text(encoding="utf-8")) + prerelease = pyproject.get("tool", {}).get("uv", {}).get("prerelease") + if prerelease: + command.extend(["--prerelease", prerelease]) + + try: + result = subprocess.run( # noqa: S603 + command, + capture_output=True, + text=True, + check=False, + cwd=requirements_path.parent, + stdin=subprocess.DEVNULL, + timeout=UV_RESOLVE_TIMEOUT_SECONDS, + ) + except FileNotFoundError as error: + raise UvUnavailableError from error + except subprocess.TimeoutExpired as error: + message = f"Timed out after {UV_RESOLVE_TIMEOUT_SECONDS}s running: {' '.join(command)}" + raise RuntimeError(message) from error + except OSError as error: + raise RuntimeError from error + + if result.returncode != 0: + message = f"Non-zero returncode: {result.stderr}, {result.stdout}" + raise RuntimeError(message) + + parsed_requirements, editable_paths, sources = _parse_uv_requirements( + result.stdout, + skip_dependencies, + requirements_path.parent.resolve(), + ) + if use_lock: + sources.update(_locked_requirement_sources(lock_path)) + return parsed_requirements, editable_paths, sources + + +def _project_indexes(directory: Path) -> list[_UvIndex]: + """ + Collect the uv indexes that apply to ``directory``. + + This reads configuration files and the environment, so it is resolved once per + requirements file rather than once per requirement. + """ + uv_config, config_directory = _read_uv_configuration(directory) + return [ + *_configured_uv_indexes(uv_config, config_directory), + *_environment_uv_indexes(), + ] + + +def _resolution_context( + directory: Path, + indexes: list[_UvIndex], + requirement: Requirement, + source_url: str | None, + base_index_url: str, +) -> _UvResolutionContext: + index_args, index_environment = ( + _index_invocation_for_source(source_url, indexes) if source_url else ((), ()) + ) + has_custom_index = any(not _is_public_pypi(index.url) for index in indexes) + prefer_artifact = bool( + requirement.url + or (source_url and not _is_public_pypi(source_url)) + or (source_url is None and has_custom_index) + or not _is_public_pypi(base_index_url) + ) + remote_matches_source = bool( + (source_url and _same_index(source_url, base_index_url)) + or (source_url is None and not requirement.url and not _is_public_pypi(base_index_url)) + ) + return _UvResolutionContext( + directory=directory, + base_index_url=base_index_url, + source_url=source_url, + index_args=index_args, + index_environment=index_environment, + prefer_artifact=prefer_artifact, + remote_matches_source=remote_matches_source, + ) + class PackageInfoManager: """Manages retrieval of local and remote package information.""" @@ -42,6 +582,10 @@ def __init__(self, base_pypi_url: str = "https://pypi.org") -> None: """ self.base_pypi_url = base_pypi_url self.reqs: set[Requirement] = set() + self.local_projects: dict[str, PackageInfo] = {} + self.resolution_contexts: dict[ + tuple[str, str | None, str | None], _UvResolutionContext + ] = {} def resolve_requirements( self, @@ -51,15 +595,151 @@ def resolve_requirements( skip_dependencies: set[str], ) -> None: for requirements_path in requirements_paths: - self.reqs.update( - gather( + requirements_path_obj = Path(requirements_path).resolve() + try: + resolved_requirements, editable_paths, source_urls = _gather_uv_requirements( + requirements_path=requirements_path_obj, + groups=groups, + extras=extras, + skip_dependencies=skip_dependencies, + base_index_url=self.base_pypi_url, + ) + except RuntimeError as error: + # UvUnavailableError is a RuntimeError; only a genuine resolution failure + # for a pyproject.toml is fatal. + if not isinstance(error, UvUnavailableError) and ( + requirements_path_obj.name == "pyproject.toml" + ): + raise + logger.warning( + f"Falling back to the legacy resolver for {requirements_path_obj}: {error}" + ) + resolved_requirements = gather( skipDependencies=skip_dependencies, groups=groups, extras=extras, - requirementsPath=Path(requirements_path), + requirementsPath=requirements_path_obj, base_index_url=self.base_pypi_url, ) + editable_paths = set() + source_urls = {} + + self._register_editable_projects( + resolved_requirements, + editable_paths, + skip_dependencies, + ) + self._register_direct_local_projects( + resolved_requirements, + requirements_path_obj.parent, + ) + directory = requirements_path_obj.parent.resolve() + indexes = _project_indexes(directory) + for requirement in resolved_requirements: + key = _requirement_key(requirement) + self.resolution_contexts[key] = _resolution_context( + directory, + indexes, + requirement, + source_urls.get(key), + self.base_pypi_url, + ) + self.reqs.update(resolved_requirements) + + def _register_editable_projects( + self, + resolved_requirements: set[Requirement], + editable_paths: set[Path], + skip_dependencies: set[str], + ) -> None: + skip_names = {canonicalize_name(name) for name in skip_dependencies} + for editable_path in editable_paths: + pyproject_path = ( + editable_path + if editable_path.name == "pyproject.toml" + else editable_path / "pyproject.toml" + ) + package = self._read_project_package(pyproject_path) + if package is None: + continue + + self.local_projects[package.name] = package + if package.name in skip_names: + continue + + requirement = package.name + if package.version: + requirement = f"{requirement}=={package.version}" + resolved_requirements.add(Requirement(requirement)) + + def _register_direct_local_projects( + self, + resolved_requirements: set[Requirement], + base_path: Path, + ) -> None: + for requirement in resolved_requirements: + project_path = _requirement_project_path(requirement, base_path) + if project_path is None: + continue + pyproject_path = ( + project_path + if project_path.name == "pyproject.toml" + else project_path / "pyproject.toml" ) + package = self._read_project_package(pyproject_path) + if package is not None and package.name == canonicalize_name(requirement.name): + self.local_projects[package.name] = package + + @staticmethod + def _read_project_package(pyproject_path: Path) -> PackageInfo | None: + if not pyproject_path.is_file(): + return None + + pyproject = tomli.loads(pyproject_path.read_text(encoding="utf-8")) + tool = pyproject.get("tool", {}) + project = ( + pyproject.get("project") + or tool.get("poetry") + or tool.get("flit", {}).get("metadata", {}) + ) + if not isinstance(project, dict): + return None + + name = project.get("name") or project.get("dist-name") or project.get("module") + if not name: + return None + + license_value = project.get("license", UNKNOWN) + if isinstance(license_value, dict): + license_value = license_value.get("text", UNKNOWN) + if not _has_usable_license(str(license_value)): + license_value = from_classifiers(project.get("classifiers")) or UNKNOWN + license_value = normalize_license(str(license_value)) + + authors = project.get("authors", project.get("author", [])) + if isinstance(authors, str): + authors = [authors] + author_names = [ + author.get("name", "") if isinstance(author, dict) else str(author) + for author in authors + ] + project_urls = project.get("urls", {}) + if not isinstance(project_urls, dict): + project_urls = {} + + return PackageInfo( + name=canonicalize_name(name), + version=project.get("version"), + homePage=( + project_urls.get("Homepage") + or project_urls.get("homepage") + or project.get("homepage") + or project.get("home-page") + ), + author=", ".join(filter(None, author_names)), + license=license_value, + errorCode=0, + ) def getPackages(self) -> set[PackageInfo]: """ @@ -78,32 +758,100 @@ def _get_package_info(self, package: Requirement) -> PackageInfo: :param Requirement package: package info to unpack :return PackageInfo: Information about the package. """ - versions: set[str | None] = {None} + context = self.resolution_contexts.get(_requirement_key(package)) package.name = canonicalize_name(package.name) - specifier = getattr(package, "specifier", None) - if specifier is not None: - parsed_versions = {item.version for item in specifier} - if parsed_versions: - versions = parsed_versions - - package.name = canonicalize_name(package.name) + if local_project := self.local_projects.get(package.name): + local_package = replace(local_project) + if local_package.license: + local_package.license = normalize_license(local_package.license) + return local_package base_pkg_info: PackageInfo = PackageInfo( - name=package.name, version=versions.pop(), errorCode=1 + name=package.name, + version=_exact_requirement_version(package), + errorCode=1, ) lpi = LocalPackageInfo(package=base_pkg_info) + resolved_source = context is not None and ( + context.source_url is not None or context.prefer_artifact + ) + local_matches = not resolved_source and _versions_match( + base_pkg_info.version, + lpi.get_version(), + ) + local_name = lpi.get_name() if local_matches else None + local_license = lpi.get_license() if local_matches else None + + preferred_index: IndexPackageInfo | None = None + if context is not None and context.prefer_artifact: + preferred_index = IndexPackageInfo( + package=base_pkg_info, + requirement=package, + context=context, + ) + if preferred_index.get_name(): + pkg_info = PackageInfo( + name=package.name, + version=base_pkg_info.version or preferred_index.get_version(), + homePage=preferred_index.get_homePage(), + author=preferred_index.get_author(), + license=preferred_index.get_license(), + errorCode=0, + ) + if pkg_info.license: + pkg_info.license = normalize_license(pkg_info.license) + return pkg_info + if not context.remote_matches_source: + return PackageInfo( + name=package.name, + version=base_pkg_info.version, + errorCode=1, + ) + rpi = RemotePackageInfo(pypi_api=self.base_pypi_url, package=base_pkg_info) + rpi.lazy_fetch() + remote_license = rpi.get_license() + + needs_index = (not local_name and rpi.http_code != HTTP_OK) or not any( + _has_usable_license(value) for value in (local_license, remote_license) + ) + # Reuse the artifact fetch already attempted above rather than re-running `uv pip install` + ipi = preferred_index + if ipi is None and needs_index: + ipi = IndexPackageInfo( + package=base_pkg_info, + requirement=package, + context=context, + ) + index_name = ipi.get_name() if ipi is not None else None + index_license = ipi.get_license() if ipi is not None else None + license_candidates = (local_license, index_license, remote_license) + license_value = next( + (value for value in license_candidates if _has_usable_license(value)), + next((value for value in license_candidates if value), None), + ) pkg_info = PackageInfo( name=package.name, - version=lpi.get_version() or rpi.get_version(), - size=lpi.get_size() or rpi.get_size(), - homePage=lpi.get_homePage() or rpi.get_homePage(), - author=lpi.get_author() or rpi.get_author(), - license=str(lpi.get_license() or rpi.get_license()), - errorCode=rpi.http_code if rpi.http_code != HTTP_OK else 0, + version=base_pkg_info.version + or (lpi.get_version() if local_matches else None) + or (ipi.get_version() if ipi is not None else None) + or rpi.get_version(), + size=(lpi.get_size() if local_matches else None) + or (ipi.get_size() if ipi is not None else None) + or rpi.get_size(), + homePage=(lpi.get_homePage() if local_matches else None) + or (ipi.get_homePage() if ipi is not None else None) + or rpi.get_homePage(), + author=(lpi.get_author() if local_matches else None) + or (ipi.get_author() if ipi is not None else None) + or rpi.get_author(), + license=license_value, + errorCode=( + 0 if rpi.http_code == HTTP_OK or local_name or index_name else rpi.http_code + ), ) # normailzing the license @@ -132,15 +880,15 @@ def __init__(self, package: PackageInfo) -> None: self.package: PackageInfo = package # email message appears to mostly conform to the protocol # https://packaging.python.org/en/latest/specifications/core-metadata/#core-metadata - self.meta: PackageMetadata = Message() + self.meta: Message[str, str] | PackageMetadata = Message() with contextlib.suppress(metadata.PackageNotFoundError): self.meta = metadata.metadata(package.name) def get_license(self) -> str | None: - return ( - self.meta.get("License-Expression") - or from_classifiers(self.meta.get_all("Classifier")) - or self.meta.get("License") + return _license_from_metadata( + self.meta.get("License-Expression"), + self.meta.get_all("Classifier"), + self.meta.get("License"), ) def get_name(self) -> str | None: @@ -169,6 +917,109 @@ def get_size(self) -> int | None: return None # Package not found +class IndexPackageInfo: + """Handles package metadata from indexes configured for uv.""" + + def __init__( + self, + package: PackageInfo, + requirement: Requirement | PackageInfo | None = None, + context: _UvResolutionContext | None = None, + ) -> None: + self.package = package + self.requirement = requirement + self.context = context + self.meta: Message[str, str] | PackageMetadata = Message() + self.fetched = False + + def lazy_fetch(self) -> None: + if self.fetched: + return + self.fetched = True + + requirement_url = getattr(self.requirement, "url", None) + requirement = str(self.requirement) if requirement_url else self.package.name + if self.package.version and not requirement_url: + requirement = f"{requirement}=={self.package.version}" + + with tempfile.TemporaryDirectory(prefix="licensecheck-") as target: + command = [ + "uv", + "pip", + "install", + "--color", + "never", + "--no-progress", + "--no-deps", + "--only-binary", + ":all:", + "--target", + target, + ] + if self.context is not None: + if not _is_public_pypi(self.context.base_index_url): + command.extend(["--default-index", self.context.base_index_url]) + command.extend(self.context.index_args) + command.append(requirement) + run_environment = None + if self.context is not None and self.context.index_environment: + run_environment = os.environ.copy() + run_environment.update(dict(self.context.index_environment)) + try: + result = subprocess.run( # noqa: S603 + command, + capture_output=True, + text=True, + check=False, + cwd=self.context.directory if self.context is not None else None, + env=run_environment, + stdin=subprocess.DEVNULL, + timeout=UV_ARTIFACT_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired) as error: + logger.warning(f"Could not fetch the artifact for {self.package.name}: {error}") + return + + if result.returncode != 0: + logger.warning( + f"Could not fetch the artifact for {self.package.name}: {result.stderr}" + ) + return + + for distribution in metadata.distributions(path=[target]): + name = distribution.metadata.get("Name") + if name and canonicalize_name(name) == self.package.name: + self.meta = distribution.metadata + return + + def get_license(self) -> str | None: + self.lazy_fetch() + return _license_from_metadata( + self.meta.get("License-Expression"), + self.meta.get_all("Classifier"), + self.meta.get("License"), + ) + + def get_name(self) -> str | None: + self.lazy_fetch() + return self.meta.get("Name") + + def get_version(self) -> str | None: + self.lazy_fetch() + return self.meta.get("Version") + + def get_homePage(self) -> str | None: + self.lazy_fetch() + return self.meta.get("Home-page") + + def get_author(self) -> str | None: + self.lazy_fetch() + return self.meta.get("Author") + + def get_size(self) -> None: + return None + + class RemotePackageInfo: """Handles retrieval of package info from PyPI.""" @@ -177,21 +1028,27 @@ def __init__(self, pypi_api: str, package: PackageInfo) -> None: self.pypi_api_integrity = pypi_api + "/integrity" self.package = package self.http_code: int = 0 - self.resp: ProjectResponse = None + self.resp: ProjectResponse | None = None def lazy_fetch(self) -> None: if self.resp is None: - # Attempt to get versioned info first - rc, raw_resp = self.make_req( - url=f"{self.pypi_api_pypi}{self.package.name}/{self.package.version}/json" - ) - # Otherwise just get the latest - if rc != HTTP_OK: + if self.package.version: + rc, raw_resp = self.make_req( + url=f"{self.pypi_api_pypi}/{self.package.name}/{self.package.version}/json" + ) + else: rc, raw_resp = self.make_req(url=f"{self.pypi_api_pypi}/{self.package.name}/json") self.http_code = rc self.resp = ProjectResponse.model_validate(raw_resp) + def _response(self) -> ProjectResponse: + self.lazy_fetch() + if self.resp is None: + message = "Package metadata response was not initialized" + raise RuntimeError(message) + return self.resp + def make_req( self, url: str, headers: dict[str, str] | None = None ) -> tuple[int, dict[str, Any]]: @@ -206,33 +1063,29 @@ def make_req( return -2, {} def get_name(self) -> str: - self.lazy_fetch() - return self.resp.info.name + return self._response().info.name def get_version(self) -> str: - self.lazy_fetch() - return self.resp.info.version + return self._response().info.version def get_homePage(self) -> str: - self.lazy_fetch() - return self.resp.info.home_page + return self._response().info.home_page def get_author(self) -> str: - self.lazy_fetch() - author_email = self.resp.info.author_email or "" - return self.resp.info.author or author_email.split("<")[0].strip() + response = self._response() + author_email = response.info.author_email or "" + return response.info.author or author_email.split("<")[0].strip() def get_license(self) -> str: - self.lazy_fetch() - return ( - self.resp.info.license_expression - or from_classifiers(self.resp.info.classifiers) - or self.resp.info.license + response = self._response() + return _license_from_metadata( + response.info.license_expression, + response.info.classifiers, + response.info.license, ) def get_size(self) -> int | None: - self.lazy_fetch() - urls = self.resp.urls + urls = self._response().urls return urls[-1].size if len(urls) > 0 else None diff --git a/pyproject.toml b/pyproject.toml index 61efbe7..6e5396e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "pydantic>=2.13.4", "requests>=2.32.5", "requests-cache>=1.3.2", + "requirements-parser>=0.13.1", "rich>=15.0.0", "tomli>=2.2.1", ] @@ -105,8 +106,10 @@ groups = [] # List of selected groups extras = [] # List of selected extras file = "" # Output file (leave empty for stdout) ignore_packages = [] # Packages/dependencies to ignore +license_overrides = {} # Reviewed licenses for exact package versions fail_packages = [] # Packages/dependencies that cause failure ignore_licenses = [] # Licenses to ignore +allowed_license_references = [] # Exact LicenseRef identifiers to accept fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) skip_dependencies = [] # Dependencies to skip (compatibility = True) diff --git a/tests/io/test_cli_main.py b/tests/io/test_cli_main.py index f70f8b5..0226d8b 100644 --- a/tests/io/test_cli_main.py +++ b/tests/io/test_cli_main.py @@ -209,6 +209,7 @@ def fake_check(**kwargs: dict[str, Any]) -> tuple[Literal[False], list[Any]]: assert called["groups"] == config.groups assert called["extras"] == config.extras assert called["skip_dependencies"] == config.skip_dependencies + assert called["license_overrides"] == config.license_overrides def test_main_closes_output_file( diff --git a/tests/io/test_formatter.py b/tests/io/test_formatter.py index 4fffea2..692f759 100644 --- a/tests/io/test_formatter.py +++ b/tests/io/test_formatter.py @@ -5,6 +5,7 @@ import pytest from licensecheck.io import fmt +from licensecheck.io.fmt import FMT from licensecheck.models.license import License from licensecheck.models.packageinfo import PackageInfo @@ -88,3 +89,15 @@ def assert_eq(actual_input: str, expected_output: str) -> bool: return False return [x.strip() for x in actual] == [x.strip() for x in expected] + + +def test_override_source_is_visible() -> None: + package = PackageInfo( + name="example", + version="1.0.0", + license="BSD-3-Clause", + licenseSource="configured override", + licenseCompat=True, + ) + + assert "configured override" in fmt.fmt(FMT.simple, myLice, [package]) diff --git a/tests/models/test_config.py b/tests/models/test_config.py index 40219fa..b550f78 100644 --- a/tests/models/test_config.py +++ b/tests/models/test_config.py @@ -1,3 +1,4 @@ +import pytest import tomli from licensecheck.io.fmt import FMT @@ -19,8 +20,10 @@ def test_basic_config() -> None: extras = [] # List of selected extras file = "" # Output file (leave empty for stdout) ignore_packages = [] # Packages/dependencies to ignore +license_overrides = { "sample==1.2.3" = "MIT" } # Reviewed exact-version licenses fail_packages = [] # Packages/dependencies that cause failure ignore_licenses = [] # Licenses to ignore +allowed_license_references = [] # Exact LicenseRef identifiers to accept fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) skip_dependencies = [] # Dependencies to skip (compatibility = True) @@ -34,6 +37,7 @@ def test_basic_config() -> None: conf = LC_Config.model_validate(raw_conf["tool"]["licensecheck"]) assert conf.format == FMT.simple assert conf.pypi_api == "https://pypi.org" + assert conf.license_overrides == {"sample==1.2.3": "MIT"} def test_basic_config2() -> None: @@ -44,6 +48,7 @@ def test_basic_config2() -> None: ignore_packages = [] # Packages/dependencies to ignore fail_packages = [] # Packages/dependencies that cause failure ignore_licenses = [] # Licenses to ignore +allowed_license_references = [] # Exact LicenseRef identifiers to accept fail_licenses = [] # Licenses that cause failure only_licenses = [] # Allowed licenses (all others will fail) skip_dependencies = [] # Dependencies to skip (compatibility = True) @@ -56,3 +61,22 @@ def test_basic_config2() -> None: conf = LC_Config.model_validate(raw_conf["tool"]["licensecheck"]) assert conf.format == FMT.simple assert conf.pypi_api == "" + + +@pytest.mark.parametrize( + "package", + [ + "sample", + "sample>=1.2.3", + "sample==1.*", + "sample==1.2.3; python_version > '3.11'", + ], +) +def test_license_overrides_require_exact_versions(package: str) -> None: + with pytest.raises(ValueError, match="exact name==version"): + LC_Config.model_validate({"license_overrides": {package: "MIT"}}) + + +def test_license_overrides_require_nonempty_licenses() -> None: + with pytest.raises(ValueError, match="must not be empty"): + LC_Config.model_validate({"license_overrides": {"sample==1.2.3": " "}}) diff --git a/tests/test_checker.py b/tests/test_checker.py index 736c2bc..fde6019 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -62,3 +62,188 @@ def test_check( ) assert incompatible == expected_incompatible, packages + + +@pytest.mark.parametrize( + ("ignore_packages", "expected_incompatible"), + [ + ({"private-package==1.2.3"}, False), + ({"private-package==1.2.4"}, True), + ({"private-package==1.*"}, False), + ], +) +def test_ignore_packages_can_match_versions( + mock_package_info_manager: PackageInfoManager, + ignore_packages: set[str], + *, + expected_incompatible: bool, +) -> None: + mock_package_info_manager.getPackages.return_value = { + PackageInfo( + name="private-package", + version="1.2.3", + license="PROPRIETARY", + ) + } + + incompatible, packages = check( + requirements_paths={"requirements.txt"}, + groups=set(), + extras=set(), + this_license=License.MIT, + package_info_manager=mock_package_info_manager, + ignore_packages=ignore_packages, + ) + + assert incompatible == expected_incompatible, packages + + +@pytest.mark.parametrize( + ("dependency_license", "expected_incompatible"), + [ + ("LicenseRef-CVector-Proprietary", False), + ("licenseref-cvector-proprietary", False), + ("LicenseRef-Other-Proprietary", True), + ("PROPRIETARY", True), + ], +) +def test_matching_custom_license_reference_is_compatible( + mock_package_info_manager: PackageInfoManager, + dependency_license: str, + *, + expected_incompatible: bool, +) -> None: + mock_package_info_manager.getPackages.return_value = { + PackageInfo(name="private-package", version="1.2.3", license=dependency_license) + } + + incompatible, packages = check( + requirements_paths={"requirements.txt"}, + groups=set(), + extras=set(), + this_license=License.PROPRIETARY, + this_license_text="LicenseRef-CVector-Proprietary", + package_info_manager=mock_package_info_manager, + ) + + assert incompatible == expected_incompatible, packages + + +@pytest.mark.parametrize( + ( + "dependency_license", + "allowed_license_references", + "fail_licenses", + "expected_incompatible", + ), + [ + ( + "LicenseRef-NVIDIA-Proprietary", + {"LicenseRef-NVIDIA-Proprietary"}, + None, + False, + ), + ( + "licenseref-nvidia-proprietary", + {"LicenseRef-NVIDIA-Proprietary"}, + None, + False, + ), + ( + "LicenseRef-Other-Proprietary", + {"LicenseRef-NVIDIA-Proprietary"}, + None, + True, + ), + ("PROPRIETARY", {"PROPRIETARY"}, None, True), + ( + "LicenseRef-NVIDIA-Proprietary", + {"LicenseRef-NVIDIA-Proprietary"}, + {"PROPRIETARY"}, + True, + ), + ], +) +def test_allowed_license_references_match_exact_raw_references( + mock_package_info_manager: PackageInfoManager, + dependency_license: str, + allowed_license_references: set[str], + fail_licenses: set[str] | None, + *, + expected_incompatible: bool, +) -> None: + mock_package_info_manager.getPackages.return_value = { + PackageInfo(name="private-package", version="1.2.3", license=dependency_license) + } + + incompatible, packages = check( + requirements_paths={"requirements.txt"}, + groups=set(), + extras=set(), + this_license=License.PROPRIETARY, + package_info_manager=mock_package_info_manager, + allowed_license_references=allowed_license_references, + fail_licenses=fail_licenses, + ) + + assert incompatible == expected_incompatible, packages + + +@pytest.mark.parametrize( + ("override_package", "expected_incompatible", "expected_license_source"), + [ + ("private-package==1.2.3", False, "configured override"), + ("PRIVATE_package==1.2.3", False, "configured override"), + ("private-package==1.2.4", True, None), + ], +) +def test_license_overrides_apply_only_to_exact_versions( + mock_package_info_manager: PackageInfoManager, + override_package: str, + *, + expected_incompatible: bool, + expected_license_source: str | None, +) -> None: + mock_package_info_manager.getPackages.return_value = { + PackageInfo( + name="private-package", + version="1.2.3", + license="Other/Proprietary License", + ) + } + + incompatible, packages = check( + requirements_paths={"requirements.txt"}, + groups=set(), + extras=set(), + this_license=License.MIT, + package_info_manager=mock_package_info_manager, + license_overrides={override_package: "BSD-3-Clause"}, + ) + package = packages.pop() + + assert incompatible == expected_incompatible + assert package.license == ( + "BSD-3-Clause" if expected_license_source else "Other/Proprietary License" + ) + assert package.licenseSource == expected_license_source + + +def test_license_overrides_still_obey_license_deny_rules( + mock_package_info_manager: PackageInfoManager, +) -> None: + mock_package_info_manager.getPackages.return_value = { + PackageInfo(name="private-package", version="1.2.3", license="MIT") + } + + incompatible, packages = check( + requirements_paths={"requirements.txt"}, + groups=set(), + extras=set(), + this_license=License.MIT, + package_info_manager=mock_package_info_manager, + license_overrides={"private-package==1.2.3": "GPL-3.0"}, + fail_licenses={"GPL-3.0"}, + ) + + assert incompatible, packages diff --git a/tests/test_license_matrix.py b/tests/test_license_matrix.py index 86d7670..02c83a5 100644 --- a/tests/test_license_matrix.py +++ b/tests/test_license_matrix.py @@ -90,6 +90,30 @@ def test_dualLicenseCompat() -> None: assert license_matrix.depCompatWMyLice(L.MIT, {L.GPL_2, L.MIT}) +def test_failed_license_cannot_be_masked_by_compatible_license() -> None: + assert not license_matrix.depCompatWMyLice( + L.MIT, + {L.GPL_2, L.MIT}, + failLicenses={L.GPL_2}, + ) + + +@pytest.mark.parametrize( + ("project_license", "dependency_license"), + [ + (L.UNKNOWN, L.MIT), + (L.NO_LICENSE, L.MIT), + (L.MIT, L.UNKNOWN), + (L.MIT, L.NO_LICENSE), + ], +) +def test_unknown_and_missing_licenses_fail_closed( + project_license: L, + dependency_license: L, +) -> None: + assert not license_matrix.depCompatWMyLice(project_license, {dependency_license}) + + def test_whitelistedLicenseCompat() -> None: assert license_matrix.depCompatWMyLice(L.MIT, {L.MIT}, onlyLicenses={L.MIT}) assert license_matrix.depCompatWMyLice(L.MPL, {L.MIT}, onlyLicenses={L.MIT}) diff --git a/tests/test_packageinfo.py b/tests/test_packageinfo.py index 2bc454a..21c88ed 100644 --- a/tests/test_packageinfo.py +++ b/tests/test_packageinfo.py @@ -1,6 +1,8 @@ from __future__ import annotations +from email.message import Message from pathlib import Path +from subprocess import CompletedProcess import pytest from packaging.requirements import Requirement @@ -8,9 +10,11 @@ from licensecheck.models.constants import UNKNOWN from licensecheck.models.packageinfo import PackageInfo from licensecheck.packageinforesolver import ( + IndexPackageInfo, LocalPackageInfo, PackageInfoManager, RemotePackageInfo, + _license_from_metadata, from_classifiers, normalize_license, ) @@ -38,6 +42,13 @@ def aux_packageinfo(package_name: str) -> PackageInfo: return PackageInfo(name=package_name) +def write_pyproject(directory: Path, contents: str) -> Path: + directory.mkdir(parents=True, exist_ok=True) + pyproject_path = directory / "pyproject.toml" + pyproject_path.write_text(contents.strip(), encoding="utf-8") + return pyproject_path + + requests_package = aux_packageinfo("requests") @@ -54,7 +65,434 @@ def test_getPackageInfoPypi(remote_package_info: RemotePackageInfo) -> None: assert pkg.get_name() == "requests" assert pkg.get_author() == "Kenneth Reitz" - assert pkg.get_license() == "Apache Software License" + assert pkg.get_license() == "Apache-2.0" + + +def test_remote_package_info_uses_versioned_pypi_endpoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pkg = RemotePackageInfo( + "https://packages.example", + PackageInfo(name="sample", version="1.2.3"), + ) + requested_urls: list[str] = [] + + def fake_make_req( + url: str, headers: dict[str, str] | None = None + ) -> tuple[int, dict[str, object]]: + del headers + requested_urls.append(url) + return 200, { + "info": { + "name": "sample", + "version": "1.2.3", + "license_expression": "MIT", + } + } + + monkeypatch.setattr(pkg, "make_req", fake_make_req) + + assert pkg.get_license() == "MIT" + assert requested_urls == ["https://packages.example/pypi/sample/1.2.3/json"] + + +def test_index_package_info_uses_uv_configured_indexes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pkg = IndexPackageInfo(PackageInfo(name="private-package", version="1.2.3")) + commands: list[list[str]] = [] + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + commands.append(command) + target = Path(command[command.index("--target") + 1]) + metadata_path = target / "private_package-1.2.3.dist-info" / "METADATA" + metadata_path.parent.mkdir() + metadata_path.write_text( + """ +Metadata-Version: 2.4 +Name: private-package +Version: 1.2.3 +License-Expression: LicenseRef-Example-Proprietary +""".strip(), + encoding="utf-8", + ) + return CompletedProcess(args=command, returncode=0, stdout="", stderr="") + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + assert pkg.get_name() == "private-package" + assert pkg.get_version() == "1.2.3" + assert pkg.get_license() == "LicenseRef-Example-Proprietary" + assert commands == [ + [ + "uv", + "pip", + "install", + "--color", + "never", + "--no-progress", + "--no-deps", + "--only-binary", + ":all:", + "--target", + commands[0][commands[0].index("--target") + 1], + "private-package==1.2.3", + ] + ] + + +def test_resolve_requirements_uses_project_directory_and_default_index( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path / "project", + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["private-package"] +""", + ) + calls: list[tuple[list[str], dict[str, object]]] = [] + + def fake_run(command: list[str], **kwargs: object) -> CompletedProcess[str]: + calls.append((command, kwargs)) + return CompletedProcess(command, 0, "private-package==1.2.3\n", "") + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + manager = PackageInfoManager("https://packages.example/simple") + manager.resolve_requirements({str(pyproject_path)}, set(), set(), set()) + + command, kwargs = calls[0] + assert command[command.index("--default-index") + 1] == "https://packages.example/simple" + assert "--index" not in command + assert kwargs["cwd"] == pyproject_path.parent + + +def test_package_manager_reads_metadata_from_resolved_private_source( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path / "project", + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["private-package==1.2.3"] + +[[tool.uv.index]] +name = "private" +url = "https://packages.example/simple" +explicit = true + +[tool.uv.sources] +private-package = { index = "private" } +""", + ) + commands: list[tuple[list[str], object]] = [] + + def fake_run(command: list[str], **kwargs: object) -> CompletedProcess[str]: + commands.append((command, kwargs.get("cwd"))) + if command[:3] == ["uv", "pip", "compile"]: + return CompletedProcess( + command, + 0, + """private-package==1.2.3 + # via project + # from https://packages.example/simple +""", + "", + ) + + target = Path(command[command.index("--target") + 1]) + metadata_path = target / "private_package-1.2.3.dist-info" / "METADATA" + metadata_path.parent.mkdir() + metadata_path.write_text( + """ +Metadata-Version: 2.4 +Name: private-package +Version: 1.2.3 +License-Expression: LicenseRef-Private-Proprietary +""".strip(), + encoding="utf-8", + ) + return CompletedProcess(command, 0, "", "") + + def fail_public_lookup(*_args: object, **_kwargs: object) -> tuple[int, dict[str, object]]: + pytest.fail("public metadata must not replace the resolved private artifact") + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + monkeypatch.setattr(RemotePackageInfo, "make_req", fail_public_lookup) + + package_info_manager.resolve_requirements({str(pyproject_path)}, set(), set(), set()) + package = package_info_manager.getPackages().pop() + + assert package.license == "LicenseRef-Private-Proprietary" + install_command, install_cwd = commands[1] + assert install_command[install_command.index("--index") + 1] == ( + "private=https://packages.example/simple" + ) + assert install_cwd == pyproject_path.parent + + +def test_private_index_credentials_are_not_exposed_in_command( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + private_index = "https://user:secret@packages.example/simple" + pyproject_path = write_pyproject( + tmp_path, + f""" +[project] +name = "project" +version = "1.0.0" +dependencies = ["private-package==1.2.3"] + +[tool.uv] +index-url = "{private_index}" +""", + ) + install_call: tuple[list[str], dict[str, object]] | None = None + + def fake_run(command: list[str], **kwargs: object) -> CompletedProcess[str]: + nonlocal install_call + if command[:3] == ["uv", "pip", "compile"]: + return CompletedProcess( + command, + 0, + "private-package==1.2.3\n # from https://packages.example/simple\n", + "", + ) + + install_call = command, kwargs + target = Path(command[command.index("--target") + 1]) + metadata_path = target / "private_package-1.2.3.dist-info" / "METADATA" + metadata_path.parent.mkdir() + metadata_path.write_text( + """ +Metadata-Version: 2.4 +Name: private-package +Version: 1.2.3 +License-Expression: LicenseRef-Private-Proprietary +""".strip(), + encoding="utf-8", + ) + return CompletedProcess(command, 0, "", "") + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + monkeypatch.setattr( + RemotePackageInfo, + "make_req", + lambda *_args, **_kwargs: pytest.fail("the resolved private artifact must be used"), + ) + + package_info_manager.resolve_requirements({str(pyproject_path)}, set(), set(), set()) + package = package_info_manager.getPackages().pop() + + assert package.license == "LicenseRef-Private-Proprietary" + assert install_call is not None + install_command, install_kwargs = install_call + assert "secret" not in " ".join(install_command) + install_environment = install_kwargs["env"] + assert isinstance(install_environment, dict) + assert install_environment["UV_INDEX"] == private_index + + +def test_resolved_public_source_ignores_installed_private_homonym( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path, + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["homonym==1.2.3"] +""", + ) + installed_metadata = Message() + installed_metadata["Name"] = "homonym" + installed_metadata["Version"] = "1.2.3" + installed_metadata["License-Expression"] = "LicenseRef-Private-Proprietary" + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + if command[:3] != ["uv", "pip", "compile"]: + pytest.fail("public metadata already declares a usable license") + return CompletedProcess( + command, + 0, + "homonym==1.2.3\n # from https://pypi.org/simple\n", + "", + ) + + def fake_make_req( + _self: RemotePackageInfo, + url: str, + headers: dict[str, str] | None = None, + ) -> tuple[int, dict[str, object]]: + del url, headers + return 200, { + "info": { + "name": "homonym", + "version": "1.2.3", + "license_expression": "MIT", + } + } + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + monkeypatch.setattr( + "licensecheck.packageinforesolver.metadata.metadata", + lambda _name: installed_metadata, + ) + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + + package_info_manager.resolve_requirements({str(pyproject_path)}, set(), set(), set()) + package = package_info_manager.getPackages().pop() + + assert package.version == "1.2.3" + assert package.license == "MIT" + + +def test_package_manager_uses_private_index_when_pypi_is_missing( + package_info_manager: PackageInfoManager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_make_req( + _self: RemotePackageInfo, + url: str, + headers: dict[str, str] | None = None, + ) -> tuple[int, dict[str, object]]: + del url, headers + return 404, {} + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + target = Path(command[command.index("--target") + 1]) + metadata_path = target / "private_package-1.2.3.dist-info" / "METADATA" + metadata_path.parent.mkdir() + metadata_path.write_text( + """ +Metadata-Version: 2.4 +Name: private-package +Version: 1.2.3 +License-Expression: MIT +""".strip(), + encoding="utf-8", + ) + return CompletedProcess(args=command, returncode=0, stdout="", stderr="") + + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + package_info_manager.reqs = {Requirement("private-package==1.2.3")} + + package = package_info_manager.getPackages().pop() + + assert package.name == "private-package" + assert package.version == "1.2.3" + assert package.license == "MIT" + assert package.errorCode == 0 + + +def test_package_manager_uses_exact_artifact_when_pypi_license_is_missing( + package_info_manager: PackageInfoManager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + commands: list[list[str]] = [] + + def fake_make_req( + _self: RemotePackageInfo, + url: str, + headers: dict[str, str] | None = None, + ) -> tuple[int, dict[str, object]]: + del url, headers + return 200, { + "info": { + "name": "artifact-package", + "version": "1.2.3", + } + } + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + commands.append(command) + target = Path(command[command.index("--target") + 1]) + metadata_path = target / "artifact_package-1.2.3.dist-info" / "METADATA" + metadata_path.parent.mkdir() + metadata_path.write_text( + """ +Metadata-Version: 2.4 +Name: artifact-package +Version: 1.2.3 +License-Expression: MIT +""".strip(), + encoding="utf-8", + ) + return CompletedProcess(args=command, returncode=0, stdout="", stderr="") + + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + package_info_manager.reqs = {Requirement("artifact-package==1.2.3")} + + package = package_info_manager.getPackages().pop() + + assert package.name == "artifact-package" + assert package.version == "1.2.3" + assert package.license == "MIT" + assert commands[0][-1] == "artifact-package==1.2.3" + + +def test_package_manager_ignores_installed_metadata_from_another_version( + package_info_manager: PackageInfoManager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + installed_metadata = Message() + installed_metadata["Name"] = "artifact-package" + installed_metadata["Version"] = "9.9.9" + installed_metadata["License-Expression"] = "GPL-3.0-only" + + def fake_make_req( + _self: RemotePackageInfo, + url: str, + headers: dict[str, str] | None = None, + ) -> tuple[int, dict[str, object]]: + del url, headers + return 200, { + "info": { + "name": "artifact-package", + "version": "1.2.3", + } + } + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + target = Path(command[command.index("--target") + 1]) + metadata_path = target / "artifact_package-1.2.3.dist-info" / "METADATA" + metadata_path.parent.mkdir() + metadata_path.write_text( + """ +Metadata-Version: 2.4 +Name: artifact-package +Version: 1.2.3 +License-Expression: MIT +""".strip(), + encoding="utf-8", + ) + return CompletedProcess(args=command, returncode=0, stdout="", stderr="") + + monkeypatch.setattr( + "licensecheck.packageinforesolver.metadata.metadata", lambda _name: installed_metadata + ) + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + package_info_manager.reqs = {Requirement("artifact-package==1.2.3")} + + package = package_info_manager.getPackages().pop() + + assert package.version == "1.2.3" + assert package.license == "MIT" def test_getPackageInfoLocalNotFound() -> None: @@ -68,16 +506,16 @@ def test_getPackagePypiLocalNotFound() -> None: def test_getPackages(package_info_manager: PackageInfoManager) -> None: - package_info_manager.reqs = {aux_packageinfo("requests")} + package_info_manager.reqs = {Requirement("requests")} packages = package_info_manager.getPackages() package = packages.pop() assert package.name == "requests" assert package.author == "Kenneth Reitz" - assert package.license == "Apache Software License" + assert package.license == "Apache-2.0" def test_getPackagesNotFound(package_info_manager: PackageInfoManager) -> None: - package_info_manager.reqs = {aux_packageinfo("this_package_does_not_exist")} + package_info_manager.reqs = {Requirement("this_package_does_not_exist")} packages = package_info_manager.getPackages() package = packages.pop() @@ -99,6 +537,44 @@ def test_licenseFromEmptyClassifierlist() -> None: assert licenses == [None] +@pytest.mark.parametrize( + ("license_expression", "classifier", "legacy_license", "expected"), + [ + ( + "MIT", + "License :: Other/Proprietary License", + "LicenseRef-Example-Proprietary", + "MIT", + ), + ( + None, + "License :: Other/Proprietary License", + "LicenseRef-Example-Proprietary", + "LicenseRef-Example-Proprietary", + ), + ( + None, + "License :: Other/Proprietary License", + "Apache 2.0", + "Apache-2.0", + ), + ( + None, + "License :: OSI Approved :: MIT License", + "unrecognized license text", + "MIT License", + ), + ], +) +def test_explicit_license_metadata_precedes_classifiers_when_recognizable( + license_expression: str | None, + classifier: str, + legacy_license: str, + expected: str, +) -> None: + assert _license_from_metadata(license_expression, [classifier], legacy_license) == expected + + def test_getModuleSize() -> None: local_package_info = LocalPackageInfo(aux_packageinfo("this_package_does_not_exist")) local_package_info.get_size() @@ -120,6 +596,88 @@ def test_normalize_license(lice: str, normalized: str) -> None: assert normalize_license(lice) == normalized +@pytest.mark.parametrize( + ("contents", "name", "author", "homepage"), + [ + ( + """ +[tool.poetry] +name = "poetry-local" +version = "1.2.3" +license = "MIT" +authors = ["Poetry Author "] +homepage = "https://poetry.example" +""", + "poetry-local", + "Poetry Author ", + "https://poetry.example", + ), + ( + """ +[tool.flit.metadata] +module = "flit_local" +dist-name = "flit-local" +version = "1.2.3" +license = "MIT" +author = "Flit Author" +home-page = "https://flit.example" +""", + "flit-local", + "Flit Author", + "https://flit.example", + ), + ], +) +def test_read_project_package_supports_legacy_metadata( + tmp_path: Path, + contents: str, + name: str, + author: str, + homepage: str, +) -> None: + pyproject_path = write_pyproject(tmp_path, contents) + + package = PackageInfoManager._read_project_package(pyproject_path) + + assert package is not None + assert package.name == name + assert package.version == "1.2.3" + assert package.license == "MIT" + assert package.author == author + assert package.homePage == homepage + + +@pytest.mark.parametrize( + ("license_metadata", "expected"), + [ + ('license = "MIT OR GPL-3.0-only"', "GPL-3.0-only;; MIT"), + ( + 'classifiers = ["License :: OSI Approved :: MIT License"]', + "MIT License", + ), + ], +) +def test_read_project_package_normalizes_license_metadata( + tmp_path: Path, + license_metadata: str, + expected: str, +) -> None: + pyproject_path = write_pyproject( + tmp_path, + f""" +[project] +name = "local-package" +version = "1.2.3" +{license_metadata} +""", + ) + + package = PackageInfoManager._read_project_package(pyproject_path) + + assert package is not None + assert package.license == expected + + def test_unpinned_requirement_does_not_crash(package_info_manager: PackageInfoManager) -> None: package_info_manager.reqs = {Requirement("sample")} @@ -128,3 +686,703 @@ def test_unpinned_requirement_does_not_crash(package_info_manager: PackageInfoMa assert package.name == "sample" assert package.errorCode == 0 + + +def test_resolve_requirements_audits_editable_project( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_pyproject( + tmp_path / "local_dependency", + """ +[project] +name = "local-dependency" +version = "1.2.3" +license = "LicenseRef-Example-Proprietary" +""", + ) + pyproject_path = write_pyproject( + tmp_path / "project", + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["local-dependency"] + +[tool.uv.sources] +local-dependency = { path = "../local_dependency", editable = true } +""", + ) + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + return CompletedProcess( + args=command, + returncode=0, + stdout="-e ../local_dependency\n", + stderr="", + ) + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + package = package_info_manager.getPackages().pop() + + assert {str(requirement) for requirement in package_info_manager.reqs} == { + "local-dependency==1.2.3" + } + assert package.name == "local-dependency" + assert package.version == "1.2.3" + assert package.license == "LicenseRef-Example-Proprietary" + assert package.errorCode == 0 + + +def test_resolved_requirement_version_is_preserved( + package_info_manager: PackageInfoManager, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_make_req( + _self: RemotePackageInfo, + url: str, + headers: dict[str, str] | None = None, + ) -> tuple[int, dict[str, object]]: + del url, headers + return 200, { + "info": { + "name": "sample", + "version": "1.0.0", + "license_expression": "MIT", + } + } + + def fail_artifact_fetch(*_args: object, **_kwargs: object) -> CompletedProcess[str]: + pytest.fail("artifact metadata should not be fetched when PyPI declares a license") + + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fail_artifact_fetch) + package_info_manager.reqs = {Requirement("sample==1.0.0.0")} + + package = package_info_manager.getPackages().pop() + + assert package.version == "1.0.0.0" + + +def test_resolve_requirements_handles_nested_editable_uv_sources( + package_info_manager: PackageInfoManager, tmp_path: Path +) -> None: + transitive_path = tmp_path / "transitive_dependency" + transitive_path.mkdir() + (transitive_path / "pyproject.toml").write_text( + """ +[project] +name = "transitive-dependency" +version = "1.0.0" +""".strip(), + encoding="utf-8", + ) + + nested_path = tmp_path / "nested_workspace_member" + nested_path.mkdir() + (nested_path / "pyproject.toml").write_text( + """ +[project] +name = "nested-dependency" +version = "1.0.0" +""".strip(), + encoding="utf-8", + ) + + dependency_path = tmp_path / "workspace_member" + dependency_path.mkdir() + (dependency_path / "pyproject.toml").write_text( + f""" +[project] +name = "local-dependency" +version = "1.0.0" +dependencies = [ + "nested-dependency", + "transitive-dependency @ {transitive_path.as_uri()}", +] + +[tool.uv.sources] +nested-dependency = {{ path = "../nested_workspace_member", editable = true }} +""".strip(), + encoding="utf-8", + ) + + project_path = tmp_path / "project" + project_path.mkdir() + pyproject_path = project_path / "pyproject.toml" + pyproject_path.write_text( + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["local-dependency"] + +[tool.uv.sources] +local-dependency = { path = "../workspace_member", editable = true } +""".strip(), + encoding="utf-8", + ) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {requirement.name for requirement in package_info_manager.reqs} == { + "local-dependency", + "nested-dependency", + "transitive-dependency", + } + + +def test_resolve_requirements_handles_editable_uv_sources_in_monorepo( + package_info_manager: PackageInfoManager, tmp_path: Path +) -> None: + repository_path = tmp_path / "repository" + libraries_path = repository_path / "libs" + + published_path = repository_path / "published_dependency" + published_path.mkdir(parents=True) + (published_path / "pyproject.toml").write_text( + """ +[project] +name = "published-dependency" +version = "1.0.0" +""".strip(), + encoding="utf-8", + ) + + sdk_path = libraries_path / "sdk_extensions" + sdk_path.mkdir(parents=True) + (sdk_path / "pyproject.toml").write_text( + f""" +[project] +name = "sdk-extensions" +version = "1.0.0" +dependencies = ["published-dependency @ {published_path.as_uri()}"] +""".strip(), + encoding="utf-8", + ) + + anomaly_path = libraries_path / "anomaly_detection" + anomaly_path.mkdir() + (anomaly_path / "pyproject.toml").write_text( + """ +[project] +name = "anomaly-detection" +version = "1.0.0" +""".strip(), + encoding="utf-8", + ) + + common_path = libraries_path / "service_common" + common_path.mkdir() + (common_path / "pyproject.toml").write_text( + """ +[project] +name = "service-common" +version = "1.0.0" +dependencies = ["sdk-extensions", "anomaly-detection"] + +[tool.uv.sources] +sdk-extensions = { path = "../sdk_extensions", editable = true } +anomaly-detection = { path = "../anomaly_detection", editable = true } +""".strip(), + encoding="utf-8", + ) + + function_path = repository_path / "functions" / "reconciliation" + function_path.mkdir(parents=True) + pyproject_path = function_path / "pyproject.toml" + pyproject_path.write_text( + """ +[project] +name = "reconciliation" +version = "1.0.0" +dependencies = ["sdk-extensions", "service-common", "anomaly-detection"] + +[tool.uv.sources] +sdk-extensions = { path = "../../libs/sdk_extensions", editable = true } +service-common = { path = "../../libs/service_common", editable = true } +anomaly-detection = { path = "../../libs/anomaly_detection", editable = true } + +[tool.uv] +package = false +""".strip(), + encoding="utf-8", + ) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {requirement.name for requirement in package_info_manager.reqs} == { + "anomaly-detection", + "published-dependency", + "sdk-extensions", + "service-common", + } + + +def test_resolve_requirements_does_not_skip_inactive_editable_source( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_pyproject( + tmp_path / "local-idna", + """ +[project] +name = "idna" +version = "999" +license = "LicenseRef-Local-Proprietary" +""", + ) + pyproject_path = write_pyproject( + tmp_path / "project", + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["idna==3.10"] + +[tool.uv.sources] +idna = { + path = "../local-idna", + editable = true, + marker = "python_version < '0'", +} +""", + ) + + def fake_run(*_args: object, **_kwargs: object) -> CompletedProcess[str]: + return CompletedProcess( + args=["uv", "pip", "compile"], + returncode=0, + stdout="idna==3.10\n", + stderr="", + ) + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + def fake_make_req( + _self: RemotePackageInfo, + url: str, + headers: dict[str, str] | None = None, + ) -> tuple[int, dict[str, object]]: + del url, headers + return 200, { + "info": { + "name": "idna", + "version": "3.10", + "license_expression": "BSD-3-Clause", + } + } + + monkeypatch.setattr(RemotePackageInfo, "make_req", fake_make_req) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {str(requirement) for requirement in package_info_manager.reqs} == {"idna==3.10"} + package = package_info_manager.getPackages().pop() + assert package.version == "3.10" + assert package.license == "BSD-3-Clause" + assert "idna" not in package_info_manager.local_projects + + +def test_resolve_requirements_uses_uv_prerelease_setting( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path, + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["prerelease-package"] + +[tool.uv] +prerelease = "allow" +""", + ) + commands: list[list[str]] = [] + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + commands.append(command) + return CompletedProcess( + args=command, + returncode=0, + stdout="prerelease-package==1.0.0.dev1\n", + stderr="", + ) + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert commands[0][-2:] == ["--prerelease", "allow"] + + +def test_resolve_requirements_prefers_adjacent_uv_lock( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path, + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["dependency>=1"] +""", + ) + (tmp_path / "uv.lock").write_text("version = 1", encoding="utf-8") + commands: list[list[str]] = [] + + def fake_run(command: list[str], **_kwargs: object) -> CompletedProcess[str]: + commands.append(command) + return CompletedProcess( + args=command, + returncode=0, + stdout="dependency==1.2.3\n", + stderr="", + ) + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert commands[0][:4] == ["uv", "export", "--project", tmp_path.as_posix()] + assert "--locked" in commands[0] + assert {str(requirement) for requirement in package_info_manager.reqs} == {"dependency==1.2.3"} + + +def test_resolve_requirements_keeps_package_sharing_editable_directory_name( + package_info_manager: PackageInfoManager, tmp_path: Path +) -> None: + published_path = tmp_path / "published_idna" + write_pyproject( + published_path, + """ +[project] +name = "idna" +version = "3.10" +""", + ) + + editable_path = tmp_path / "idna" + write_pyproject( + editable_path, + """ +[project] +name = "internal-helper" +version = "1.0.0" +""", + ) + + pyproject_path = write_pyproject( + tmp_path / "project", + f""" +[project] +name = "project" +version = "1.0.0" +dependencies = [ + "internal-helper", + "idna @ {published_path.as_uri()}", +] + +[tool.uv.sources] +internal-helper = {{ path = "../idna", editable = true }} +""", + ) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {requirement.name for requirement in package_info_manager.reqs} == { + "idna", + "internal-helper", + } + + +def test_resolve_requirements_handles_editable_uv_workspace_source( + package_info_manager: PackageInfoManager, tmp_path: Path +) -> None: + published_path = tmp_path / "published_dependency" + write_pyproject( + published_path, + """ +[project] +name = "published-dependency" +version = "1.0.0" +""", + ) + + workspace_path = tmp_path / "workspace" + write_pyproject( + workspace_path / "packages" / "local_dependency", + f""" +[project] +name = "local-dependency" +version = "1.0.0" +dependencies = ["published-dependency @ {published_path.as_uri()}"] +""", + ) + pyproject_path = write_pyproject( + workspace_path, + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["local-dependency"] + +[tool.uv.sources] +local-dependency = { workspace = true } + +[tool.uv.workspace] +members = ["packages/local_dependency"] +""", + ) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {requirement.name for requirement in package_info_manager.reqs} == { + "local-dependency", + "published-dependency", + } + + +def test_resolve_requirements_handles_editable_path_with_spaces( + package_info_manager: PackageInfoManager, tmp_path: Path +) -> None: + published_path = tmp_path / "published_dependency" + write_pyproject( + published_path, + """ +[project] +name = "published-dependency" +version = "1.0.0" +""", + ) + + write_pyproject( + tmp_path / "local dependency", + f""" +[project] +name = "local-dependency" +version = "1.0.0" +dependencies = ["published-dependency @ {published_path.as_uri()}"] +""", + ) + pyproject_path = write_pyproject( + tmp_path / "project", + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["local-dependency"] + +[tool.uv.sources] +local-dependency = { path = "../local dependency", editable = true } +""", + ) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {requirement.name for requirement in package_info_manager.reqs} == { + "local-dependency", + "published-dependency", + } + + +def test_resolve_requirements_uses_local_project_license_metadata( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + local_path = tmp_path / "local_dependency" + write_pyproject( + local_path, + """ +[project] +name = "local-dependency" +version = "1.2.3" +license = "LicenseRef-Example-Proprietary" +""", + ) + pyproject_path = write_pyproject( + tmp_path / "project", + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["local-dependency"] + +[tool.uv.sources] +local-dependency = { path = "../local_dependency" } +""", + ) + + def fake_run(*_args: object, **_kwargs: object) -> CompletedProcess[str]: + return CompletedProcess( + args=["uv", "pip", "compile"], + returncode=0, + stdout=f"local-dependency @ {local_path.as_uri()}\n", + stderr="", + ) + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + package = package_info_manager.getPackages().pop() + + assert package.name == "local-dependency" + assert package.version == "1.2.3" + assert package.license == "LicenseRef-Example-Proprietary" + assert package.errorCode == 0 + + +def test_resolve_requirements_falls_back_for_uv_lock( + package_info_manager: PackageInfoManager, tmp_path: Path +) -> None: + lock_path = tmp_path / "uv.lock" + lock_path.write_text( + """ +version = 1 + +[[package]] +name = "fallback-package" +version = "1.2.3" +""".strip(), + encoding="utf-8", + ) + + package_info_manager.resolve_requirements( + requirements_paths={str(lock_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + assert {str(requirement) for requirement in package_info_manager.reqs} == { + "fallback-package==1.2.3" + } + + +def test_resolve_requirements_preserves_pyproject_resolution_error( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path, + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["dependency"] +""", + ) + + def fake_run(*_args: object, **_kwargs: object) -> CompletedProcess[str]: + return CompletedProcess( + args=["uv", "pip", "compile"], + returncode=1, + stdout="", + stderr="index unavailable", + ) + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", fake_run) + + with pytest.raises(RuntimeError, match="index unavailable"): + package_info_manager.resolve_requirements( + requirements_paths={str(pyproject_path)}, + groups=set(), + extras=set(), + skip_dependencies=set(), + ) + + +def test_resolve_requirements_falls_back_when_uv_is_unavailable( + package_info_manager: PackageInfoManager, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pyproject_path = write_pyproject( + tmp_path, + """ +[project] +name = "project" +version = "1.0.0" +dependencies = ["dependency"] +""", + ) + fallback_calls: list[Path] = [] + + def missing_uv(*_args: object, **_kwargs: object) -> CompletedProcess[str]: + message = "uv" + raise FileNotFoundError(message) + + def fake_gather(**kwargs: object) -> set[Requirement]: + requirements_path = kwargs["requirementsPath"] + assert isinstance(requirements_path, Path) + fallback_calls.append(requirements_path) + return {Requirement("dependency==1.2.3")} + + monkeypatch.setattr("licensecheck.packageinforesolver.subprocess.run", missing_uv) + monkeypatch.setattr("licensecheck.packageinforesolver.gather", fake_gather) + + package_info_manager.resolve_requirements({str(pyproject_path)}, set(), set(), set()) + + assert fallback_calls == [pyproject_path] + assert {str(requirement) for requirement in package_info_manager.reqs} == {"dependency==1.2.3"}