From 1edc44d885be4c7fc4e90b9b55b2adab73553878 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Fri, 13 Mar 2026 13:19:54 +0800 Subject: [PATCH 01/73] Created "Scan Rust Package" pipeline #1767 - Get the input and extract content into the from/ codebase. - Build the source and place the built files into the to/ codebase. - Run scans. - Identify sources in from/ that are used in the build. Signed-off-by: Chin Yeung Li --- Dockerfile | 21 ++++- pyproject.toml | 1 + scanpipe/pipelines/scan_rust_package.py | 98 +++++++++++++++++++++ scanpipe/pipes/d2d.py | 109 ++++++++++++++++++++++-- scanpipe/pipes/rust.py | 50 +++++++++++ 5 files changed, 268 insertions(+), 11 deletions(-) create mode 100644 scanpipe/pipelines/scan_rust_package.py create mode 100644 scanpipe/pipes/rust.py diff --git a/Dockerfile b/Dockerfile index 0b552343bf..7ba17d50c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,15 +67,31 @@ RUN apt-get update \ wait-for-it \ universal-ctags \ gettext \ + # Added for RUST support in ScanCode, see + # https://github.com/aboutcode-org/scancode.io/issues/1767 + build-essential \ + curl \ + pkg-config \ + libssl-dev \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +# Install Rust Toolchain +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH + +# Reference: https://rust-lang.org/tools/install/ +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path \ + && chmod -R a+w $RUSTUP_HOME $CARGO_HOME + # Create the APP_USER group, user, and directory with specific UID and GID RUN groupadd --gid $APP_GID --system $APP_USER \ && useradd --uid $APP_UID --gid $APP_GID --home-dir $APP_DIR --system --create-home $APP_USER \ && chown $APP_USER:$APP_USER $APP_DIR \ && mkdir -p /var/$APP_NAME \ - && chown $APP_USER:$APP_USER /var/$APP_NAME + && chown $APP_USER:$APP_USER /var/$APP_NAME \ + && chown -R $APP_USER:$APP_USER /usr/local/cargo # Setup the work directory and the user as APP_USER for the remaining stages WORKDIR $APP_DIR @@ -87,7 +103,8 @@ RUN mkdir -p /var/$APP_NAME/static/ /var/$APP_NAME/workspace/ # Create the virtualenv RUN python -m venv $VENV_LOCATION # Enable the virtualenv, similar effect as "source activate" -ENV PATH=$VENV_LOCATION/bin:$PATH +# ENV PATH=$VENV_LOCATION/bin:$PATH +ENV PATH=$VENV_LOCATION/bin:/usr/local/cargo/bin:$PATH # Install the dependencies before the codebase COPY for proper Docker layer caching COPY --chown=$APP_USER:$APP_USER pyproject.toml $APP_DIR/ diff --git a/pyproject.toml b/pyproject.toml index bbc5757e56..2a90030284 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,6 +155,7 @@ resolve_dependencies = "scanpipe.pipelines.resolve_dependencies:ResolveDependenc scan_codebase = "scanpipe.pipelines.scan_codebase:ScanCodebase" scan_for_virus = "scanpipe.pipelines.scan_for_virus:ScanForVirus" scan_single_package = "scanpipe.pipelines.scan_single_package:ScanSinglePackage" +scan_rust_package = "scanpipe.pipelines.scan_rust_package:ScanRustPackage" [tool.setuptools.packages.find] where = ["."] diff --git a/scanpipe/pipelines/scan_rust_package.py b/scanpipe/pipelines/scan_rust_package.py new file mode 100644 index 0000000000..a79a138c37 --- /dev/null +++ b/scanpipe/pipelines/scan_rust_package.py @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +from pathlib import Path + +from scanpipe.pipelines.deploy_to_develop import DeployToDevelop +from scanpipe.pipelines.scan_codebase import ScanCodebase +from scanpipe.pipelines.scan_single_package import ScanSinglePackage +from scanpipe.pipes import d2d +from scanpipe.pipes import flag +from scanpipe.pipes import rust + +# from scanpipe.pipes.maven import update_package_license_from_resource_if_missing + + +class ScanRustPackage(ScanSinglePackage, DeployToDevelop, ScanCodebase): + """ + Download the crate’s source, build it, and run a d2d comparison between + the compiled binary and the source crate to detect any discrepancies. + + Identify the upstream source repository and verify that it matches the + contents of the source crate. + + Scan the source crate and confirm that the detected license aligns with + the license declared in Cargo.toml. + + Compare the crate’s source code against all other crates (MatchCode), + excluding itself, to detect any borrowed code from third‑party crates. + """ + + @classmethod + def steps(cls): + return ( + cls.get_input, + cls.get_package_input, + cls.collect_input_information, + cls.extract_inputs_to_codebase_directory, + cls.extract_archives, + cls.build_crates, + cls.run_scan, + cls.load_inventory_from_toolkit_scan, + cls.add_from_to_tag, + cls.identify_built_sources, + cls.flag_mapped_status, + cls.make_summary_from_scan_results, + ) + + def get_input(self): + """Get the input file for the Rust package scan pipeline.""" + from_files = list(self.project.inputs("from*")) + from_files.extend([input.path for input in self.project.inputsources.all()]) + self.from_files = from_files + self.to_files = list() + + def build_crates(self): + """ + Build the Rust crate using Cargo and put the built files under the + "to" directory. + """ + # Find the Cargo.toml file in the codebase directory + codebase_dir = Path(self.project.codebase_path) + cargo_toml_path = None + for path in codebase_dir.rglob("Cargo.toml"): + cargo_toml_path = path + break + if cargo_toml_path: + rust.build_crates(cargo_toml_path, self.project.codebase_path / "to/") + + def add_from_to_tag(self): + """Update 'from' or 'to' tag to resources based on their path.""" + d2d.update_from_to_tag(self.project) + + def identify_built_sources(self): + """Identify the built sources from the '.d' file in the "to" directory.""" + d2d.map_rust_paths(self.project) + + def flag_mapped_status(self): + """Flag the from codebase resources that were mapped.""" + flag.flag_mapped_resources(self.project) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index c8e3b64294..df29434f9b 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -1737,20 +1737,25 @@ def map_paths_resource( relations_to_create[rel_key] = relation if paths_not_mapped: to_resource.status = flag.REQUIRES_REVIEW - logger( - f"WARNING: #{len(paths_not_mapped)} {map_type} paths NOT mapped for: " - f"{to_resource.path!r}" - ) + if logger: + logger( + f"WARNING: #{len(paths_not_mapped)} {map_type} paths NOT " + f" mapped for: {to_resource.path!r}" + ) to_resource.save() if relations_to_create: rels = CodebaseRelation.objects.bulk_create(relations_to_create.values()) - logger( - f"Created {len(rels)} mappings using " - f"{', '.join(map_types)} for: {to_resource.path!r}" - ) + if logger: + logger( + f"Created {len(rels)} mappings using " + f"{', '.join(map_types)} for: {to_resource.path!r}" + ) else: - logger(f"No mappings using {', '.join(map_types)} for: {to_resource.path!r}") + if logger: + logger( + f"No mappings using {', '.join(map_types)} for: {to_resource.path!r}" + ) def process_paths_in_binary( @@ -1940,6 +1945,92 @@ def map_go_paths(project, logger=None): ) +def get_rust_file_paths(location): + """Retrieve Rust file paths.""" + file_paths = {} + rust_file_paths = parse_d_file(location) or [] + if rust_file_paths: + file_paths["rust_file_paths"] = rust_file_paths + return file_paths + + +def parse_d_file(path): + """Parse the .d file from rust package.""" + context = Path(path).read_text() + cleaned_context = context.replace("\\\n", " ") + + # Invalid .d file + if ":" not in cleaned_context: + return [] + + _, dep_paths = cleaned_context.split(":", 1) + + file_paths = [] + for file_path in dep_paths.split(): + file_path = file_path.strip() + if file_path: + file_paths.append(file_path) + + return file_paths + + +def map_rust_paths(project, logger=None): + """Map the path listed in the .d file to the source in ``project``.""" + from_resources = project.codebaseresources.files().from_codebase() + to_resources = ( + project.codebaseresources.files() + .to_codebase() + .exclude(path__contains="/deps/") + .exclude(path__contains="/build/") + .filter(path__endswith=".d") + ) + for resource in to_resources: + try: + paths = get_rust_file_paths(resource.location_path) + resource.update_extra_data(paths) + except Exception as exception: + project.add_warning( + exception=exception, + object_instance=resource, + description=f"Cannot parse file at {resource.path}", + model="map_rust_paths", + details={"path": resource.path}, + ) + + if logger: + logger( + f"Mapping {to_resources.count():,d} to/ resources using paths " + f"with {from_resources.count():,d} from/ resources." + ) + + from_resources_index = pathmap.build_index( + from_resources.values_list("id", "path"), with_subpaths=True + ) + + if logger: + logger("Done building from/ resources index.") + + resource_iterator = to_resources.iterator(chunk_size=2000) + progress = LoopProgress(to_resources.count(), logger) + for to_resource in progress.iter(resource_iterator): + map_paths_resource( + to_resource, + from_resources, + from_resources_index, + map_types=["rust_file_paths"], + logger=logger, + ) + + +def update_from_to_tag(project): + """Update 'from' or 'to' tag to resources based on their path.""" + for resource in project.codebaseresources.files(): + if resource.path.startswith("from/"): + resource.update(tag="from") + elif resource.path.startswith("to/"): + resource.update(tag="to") + + RUST_BINARY_OPTIONS = ["Rust"] ELF_BINARY_OPTIONS = ["Python", "Go", "Elf"] MACHO_BINARY_OPTIONS = ["Rust", "Go", "MacOS"] diff --git a/scanpipe/pipes/rust.py b/scanpipe/pipes/rust.py new file mode 100644 index 0000000000..de828e4387 --- /dev/null +++ b/scanpipe/pipes/rust.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import subprocess + +from scanpipe.pipes import run_command_safely + + +def build_crates(cargo_toml_path, build_dir): + """ + Build the Rust crate using Cargo to ensure that the source code compiles correctly. + + This step is crucial for validating the integrity of the source code and ensuring + that it can be successfully built. It also helps to identify any discrepancies + between the source code and the compiled binary, which can be further analyzed + in subsequent steps of the pipeline. + """ + cmd = [ + "cargo", + "build", + "--locked", + "--manifest-path", + str(cargo_toml_path), + "--target-dir", + str(build_dir), + ] + + try: + run_command_safely(cmd) + except subprocess.SubprocessError as error: + raise RuntimeError(f"Failed to build the Rust crate: {error}") From d6f63d0d4563f2d83f550a6ab61ad62452547af6 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Fri, 13 Mar 2026 17:58:44 +0800 Subject: [PATCH 02/73] Add step to compare the license declared in Cargo.toml with the licenses detected in the codebase #1767 * Introduce new "LICENSE_ISSUE" tag * License deduplication/simplification is not working well; work in progress Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/scan_rust_package.py | 9 ++ scanpipe/pipes/flag.py | 1 + scanpipe/pipes/utils.py | 101 ++++++++++++++++++ scanpipe/templates/scanpipe/package_list.html | 7 +- 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 scanpipe/pipes/utils.py diff --git a/scanpipe/pipelines/scan_rust_package.py b/scanpipe/pipelines/scan_rust_package.py index a79a138c37..324dea16ba 100644 --- a/scanpipe/pipelines/scan_rust_package.py +++ b/scanpipe/pipelines/scan_rust_package.py @@ -28,6 +28,7 @@ from scanpipe.pipes import d2d from scanpipe.pipes import flag from scanpipe.pipes import rust +from scanpipe.pipes import utils # from scanpipe.pipes.maven import update_package_license_from_resource_if_missing @@ -59,6 +60,7 @@ def steps(cls): cls.run_scan, cls.load_inventory_from_toolkit_scan, cls.add_from_to_tag, + cls.validate_package_license_integrity, cls.identify_built_sources, cls.flag_mapped_status, cls.make_summary_from_scan_results, @@ -89,6 +91,13 @@ def add_from_to_tag(self): """Update 'from' or 'to' tag to resources based on their path.""" d2d.update_from_to_tag(self.project) + def validate_package_license_integrity(self): + """ + Validate the correctness of the package license compare with the + detected license from the codebase. + """ + utils.validate_package_license_integrity(self.project) + def identify_built_sources(self): """Identify the built sources from the '.d' file in the "to" directory.""" d2d.map_rust_paths(self.project) diff --git a/scanpipe/pipes/flag.py b/scanpipe/pipes/flag.py index e8a983d40d..8f8a1a874c 100644 --- a/scanpipe/pipes/flag.py +++ b/scanpipe/pipes/flag.py @@ -66,6 +66,7 @@ REQUIRES_REVIEW = "requires-review" REVIEW_DANGLING_LEGAL_FILE = "review-dangling-legal-file" NOT_DEPLOYED = "not-deployed" +LICENSE_ISSUE = "license-mismatch-declared-vs-detected" # Target files that should be ignored during processing as those are related to the app diff --git a/scanpipe/pipes/utils.py b/scanpipe/pipes/utils.py new file mode 100644 index 0000000000..3de4e3648c --- /dev/null +++ b/scanpipe/pipes/utils.py @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +from license_expression import Licensing + +from scanpipe.pipes import flag + + +def validate_package_license_integrity(project): + """Validate the correctness of the package license.""" + # Patterns to ignore certain resources during license validation + ignore_patterns = [ + "*test*", + "*.sh", + ] + + for package in project.discoveredpackages.all(): + package_lic = package.get_declared_license_expression() + if package_lic: + if package.type == "cargo": + # A single cargo package only has one Cargo.toml file + # meaning only one package is defined. Therefore, we don't + # need to check for the package_uid + package_uid = None + else: + package_uid = package.package_uid + resources = project.codebaseresources.has_license_expression() + detected_lic_list = collect_detected_licenses( + resources, ignore_patterns, package_uid + ) + + if detected_lic_list: + lic_exp = " AND ".join(detected_lic_list) + detected_lic_exp = str(Licensing().dedup(lic_exp)) + # The package license is not in sync with detected license(s) + if detected_lic_exp != package_lic: + package.update_extra_data( + { + "issue": "License Mismatch", + "declared_license": package_lic, + "detecte_codebase_license": detected_lic_exp, + } + ) + for datafile_path in package.datafile_paths: + if not datafile_path.startswith("https://"): + data_path = project.codebaseresources.get( + path=datafile_path + ) + data_path.update(status=flag.LICENSE_ISSUE) + data_path.update_extra_data( + { + "declared_license": package_lic, + "detecte_codebase_license": detected_lic_exp, + } + ) + + +def contains_ignore_pattern(resource_path, ignore_patterns): + """Check if the resource path matches any of the ignore patterns.""" + from fnmatch import fnmatch + + for pattern in ignore_patterns: + if fnmatch(resource_path, pattern): + return True + return False + + +def collect_detected_licenses(resources, ignore_patterns, package_uid=None): + """Collect detected licenses from resources, ignoring specified patterns.""" + detected_lic_list = [] + for resource in resources: + if contains_ignore_pattern(resource.path, ignore_patterns): + continue + + # If a package_uid is provided, only consider resources linked to it + if package_uid and package_uid not in resource.for_packages: + continue + + lic = resource.detected_license_expression + if lic and lic != "unknown" and lic not in detected_lic_list: + detected_lic_list.append(lic) + return detected_lic_list diff --git a/scanpipe/templates/scanpipe/package_list.html b/scanpipe/templates/scanpipe/package_list.html index 95b2ae9220..9e707f1ba6 100644 --- a/scanpipe/templates/scanpipe/package_list.html +++ b/scanpipe/templates/scanpipe/package_list.html @@ -35,6 +35,11 @@ {% endif %} + {% if package.extra_data.issue == "License Mismatch" %} + + + + {% endif %} @@ -93,4 +98,4 @@ }); }); -{% endblock %} \ No newline at end of file +{% endblock %} From 9ca39ea483d342c231e085c8850dd9d615d44848 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Fri, 15 May 2026 13:55:54 +0800 Subject: [PATCH 03/73] Implemented the "evaluate_license_mismatch" function #1767 This commit is for testing purpose and is definitely not ready. Signed-off-by: Chin Yeung Li --- scanpipe/pipes/utils.py | 176 ++++++++++++++++++++++- scanpipe/tests/pipes/test_utils.py | 223 +++++++++++++++++++++++++++++ 2 files changed, 396 insertions(+), 3 deletions(-) create mode 100644 scanpipe/tests/pipes/test_utils.py diff --git a/scanpipe/pipes/utils.py b/scanpipe/pipes/utils.py index 3de4e3648c..092d858038 100644 --- a/scanpipe/pipes/utils.py +++ b/scanpipe/pipes/utils.py @@ -40,6 +40,10 @@ def validate_package_license_integrity(project): # A single cargo package only has one Cargo.toml file # meaning only one package is defined. Therefore, we don't # need to check for the package_uid + # In addition, the package_uid is not populated to source files: + # https://github.com/aboutcode-org/scancode.io/issues/2169 + # so we set package_uid to None to consider all resources + # in the codebase for license validation. package_uid = None else: package_uid = package.package_uid @@ -50,6 +54,8 @@ def validate_package_license_integrity(project): if detected_lic_list: lic_exp = " AND ".join(detected_lic_list) + # The dedup have bug and need to be fixed: + # https://github.com/aboutcode-org/license-expression/issues/130 detected_lic_exp = str(Licensing().dedup(lic_exp)) # The package license is not in sync with detected license(s) if detected_lic_exp != package_lic: @@ -57,7 +63,7 @@ def validate_package_license_integrity(project): { "issue": "License Mismatch", "declared_license": package_lic, - "detecte_codebase_license": detected_lic_exp, + "detected_codebase_license": detected_lic_exp, } ) for datafile_path in package.datafile_paths: @@ -69,7 +75,7 @@ def validate_package_license_integrity(project): data_path.update_extra_data( { "declared_license": package_lic, - "detecte_codebase_license": detected_lic_exp, + "detected_codebase_license": detected_lic_exp, } ) @@ -87,6 +93,8 @@ def contains_ignore_pattern(resource_path, ignore_patterns): def collect_detected_licenses(resources, ignore_patterns, package_uid=None): """Collect detected licenses from resources, ignoring specified patterns.""" detected_lic_list = [] + # Some licenses are not useful for validating package license integrity, so we ignore them. + ignored_licenses = ['free-unknown', 'unknown', 'unknown-license-reference', 'unknown-spdx'] for resource in resources: if contains_ignore_pattern(resource.path, ignore_patterns): continue @@ -96,6 +104,168 @@ def collect_detected_licenses(resources, ignore_patterns, package_uid=None): continue lic = resource.detected_license_expression - if lic and lic != "unknown" and lic not in detected_lic_list: + if lic and lic not in ignored_licenses and lic not in detected_lic_list: + # Make sure there is parentheses if the license has an 'OR' operator + # TODO: We need to parse the lic and check for ignored licenses if lic is an expression + if ' OR ' in lic and not (lic.startswith('(') and lic.endswith(')')): + lic = f'({lic})' detected_lic_list.append(lic) return detected_lic_list + + +# TODO: This may not be necessary since our focus is on license mismatches at the package level. +# If a file contains an extra license not declared in the package, we flag it at the package level rather than the file level. +# Therefore, the 'missing' and 'extra' checks may be excessive. +def evaluate_license_mismatch(package_lic, detected_lic_list): + """Check if there is a license mismatch between declared package_license and detected licenses. + + Returns: + A dictionary with mismatch information + """ + licensing = Licensing() + + # Parse expressions + package_expr = licensing.parse(package_lic) + detected_exprs = [licensing.parse(lic) for lic in detected_lic_list] + + # Combine detected licenses with AND + detected_combined = (detected_exprs[0] if len(detected_exprs) == 1 + else licensing.AND(*detected_exprs)) + + # Find what's missing from detected + missing = find_missing(package_expr, detected_combined) + + # Find extra licenses (with special handling for OR expressions) + extra = find_extra(detected_exprs, package_expr) + + return { + 'missing': sorted(missing), + 'extra': sorted(extra), + 'is_match': not missing and not extra, + 'details': format_details(missing, extra) + } + + +def find_missing(expr, detected): + """Find licenses required by package but missing from detected.""" + # Base case: single license + if is_license_symbol(expr): + return [expr.key] if not contains_license(detected, expr.key) else [] + + # WITH exception + if is_with_exception(expr): + missing = [] + if not contains_license(detected, expr.license_symbol.key): + missing.append(expr.license_symbol.key) + if not contains_license(detected, expr.exception_symbol.key): + missing.append(expr.exception_symbol.key) + return missing + + # AND/OR expressions + if hasattr(expr, 'args'): + if is_and(expr): + # AND requires all branches + return sum((find_missing(arg, detected) for arg in expr.args), []) + + if is_or(expr): + # OR requires at least one branch + # First, check if any branch is fully satisfied + for arg in expr.args: + if not find_missing(arg, detected): + return [] # Found a fully satisfied branch + + # If no branch is fully satisfied, collect ALL licenses from ALL branches + # to show everything that's missing from the OR + all_missing = [] + for arg in expr.args: + branch_missing = find_missing(arg, detected) + all_missing.extend(branch_missing) + return list(set(all_missing)) # Remove duplicates + + return [] + + +def find_extra(detected_exprs, package_expr): + """Find licenses in detected that aren't required by package.""" + extra = [] + + for detected in detected_exprs: + detected_keys = get_all_keys(detected) + + # If it's an OR and any branch is required, none of its licenses are extra + if is_or(detected) and any(is_license_required(key, package_expr) for key in detected_keys): + continue + + # Otherwise, add all keys not required by package + for key in detected_keys: + if not is_license_required(key, package_expr) and key not in extra: + extra.append(key) + + return extra + + +def is_license_required(license_key, expr): + """Check if a license key is required by the package expression.""" + if is_license_symbol(expr): + return expr.key == license_key + if is_with_exception(expr): + return (expr.license_symbol.key == license_key or + expr.exception_symbol.key == license_key) + if hasattr(expr, 'args'): + if is_and(expr): + # In AND, a license is required if it appears in any branch + return any(is_license_required(license_key, arg) for arg in expr.args) + if is_or(expr): + # In OR, a license is required if it appears in the expression + return any(is_license_required(license_key, arg) for arg in expr.args) + return False + + +def get_all_keys(expr): + """Extract all license keys from an expression.""" + if is_license_symbol(expr): + return [expr.key] + if is_with_exception(expr): + return [expr.license_symbol.key, expr.exception_symbol.key] + if hasattr(expr, 'args'): + keys = [] + for arg in expr.args: + keys.extend(get_all_keys(arg)) + return keys + return [] + + +def contains_license(expr, license_key): + """Check if an expression contains a specific license key.""" + if is_license_symbol(expr): + return expr.key == license_key + if is_with_exception(expr): + return (expr.license_symbol.key == license_key or + expr.exception_symbol.key == license_key) + if hasattr(expr, 'args'): + return any(contains_license(arg, license_key) for arg in expr.args) + return False + + +def format_details(missing, extra): + """Format the details string.""" + parts = [] + if missing: + parts.append(f"Missing: {', '.join(sorted(missing))}") + if extra: + parts.append(f"Extra: {', '.join(sorted(extra))}") + return '; '.join(parts) if parts else 'match' + + +# Helper functions for type checking +def is_license_symbol(expr): + return hasattr(expr, 'key') and not hasattr(expr, 'license_symbol') + +def is_with_exception(expr): + return hasattr(expr, 'license_symbol') and hasattr(expr, 'exception_symbol') + +def is_and(expr): + return hasattr(expr, 'operator') and 'AND' in expr.operator.strip() + +def is_or(expr): + return hasattr(expr, 'operator') and 'OR' in expr.operator.strip() diff --git a/scanpipe/tests/pipes/test_utils.py b/scanpipe/tests/pipes/test_utils.py new file mode 100644 index 0000000000..cbd2b4ba00 --- /dev/null +++ b/scanpipe/tests/pipes/test_utils.py @@ -0,0 +1,223 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from django.test import TestCase + +from scanpipe.pipes import utils + +class ScanPipeUtilsTest(TestCase): + + def test_evaluate_license_mismatch_simple_OR_condition(self): + package_lic = "mit OR apache-2.0" + detected_lic_list = ["mit", "apache-2.0"] + expected = { + 'missing': [], + 'extra': [], + 'is_match': True, + 'details': 'match' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + + def test_evaluate_license_mismatch_simple_AND_condition(self): + package_lic = "mit AND apache-2.0" + detected_lic_list = ["mit", "apache-2.0"] + expected = { + 'missing': [], + 'extra': [], + 'is_match': True, + 'details': 'match' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + + def test_evaluate_license_mismatch_simple_OR_condition_1(self): + package_lic = "mit OR apache-2.0" + detected_lic_list = ["mit"] + expected = { + 'missing': [], + 'extra': [], + 'is_match': True, + 'details': 'match' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + def test_evaluate_license_mismatch_detected_lic_missing_AND_condition(self): + package_lic = "mit AND apache-2.0" + detected_lic_list = ["mit"] + expected = { + 'missing': ["apache-2.0"], + 'extra': [], + 'is_match': False, + 'details': 'Missing: apache-2.0' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + + def test_evaluate_license_mismatch_simple_AND_condition_detected_lic_has_OR(self): + package_lic = "mit AND apache-2.0" + detected_lic_list = ["mit or bsd-new", "apache-2.0"] + expected = { + 'missing': [], + 'extra': [], + 'is_match': True, + 'details': 'match' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + + def test_evaluate_license_mismatch_detected_lic_has_extra(self): + package_lic = "mit AND apache-2.0" + detected_lic_list = ["mit AND bsd-new", "apache-2.0"] + expected = { + 'missing': [], + 'extra': ["bsd-new"], + 'is_match': False, + 'details': 'Extra: bsd-new' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + def test_evaluate_license_mismatch_detected_lic_has_extra_1(self): + package_lic = "mit AND apache-2.0" + detected_lic_list = ["mit", "bsd-new", "apache-2.0"] + expected = { + 'missing': [], + 'extra': ["bsd-new"], + 'is_match': False, + 'details': 'Extra: bsd-new' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + def test_evaluate_license_mismatch_AND_condition_with_OR(self): + package_lic = "(bsd-new OR apache-2.0) AND apache-2.0 AND mit" + detected_lic_list = ["mit", "apache-2.0"] + expected = { + 'missing': [], + 'extra': [], + 'is_match': True, + 'details': 'match' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + def test_evaluate_license_mismatch_WITH_AND_OR_Missing(self): + package_lic = "(bsd-new AND apache-2.0) AND apache-2.0 AND (mit OR public-domain)" + detected_lic_list = ["mit", "apache-2.0"] + expected = { + 'missing': ["bsd-new"], + 'extra': [], + 'is_match': False, + 'details': 'Missing: bsd-new' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + def test_evaluate_license_mismatch_WITH_AND_OR(self): + package_lic = "(bsd-new OR apache-2.0) AND apache-2.0 AND (mit OR public-domain)" + detected_lic_list = ["mit", "apache-2.0"] + expected = { + 'missing': [], + 'extra': [], + 'is_match': True, + 'details': 'match' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + def test_evaluate_license_mismatch_missing_extra_AND(self): + package_lic = "bsd-new AND apache-2.0" + detected_lic_list = ["mit"] + expected = { + 'missing': ['apache-2.0', 'bsd-new'], + 'extra': ['mit'], + 'is_match': False, + 'details': 'Missing: apache-2.0, bsd-new; Extra: mit' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + def test_evaluate_license_mismatch_missing_extra_OR(self): + package_lic = "bsd-new OR apache-2.0" + detected_lic_list = ["mit"] + expected = { + 'missing': ['apache-2.0', 'bsd-new'], + 'extra': ['mit'], + 'is_match': False, + 'details': 'Missing: apache-2.0, bsd-new; Extra: mit' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + def test_evaluate_license_mismatch_some_missing_extra(self): + package_lic = "(bsd-new OR apache-2.0) AND mit AND (apache-1.1 AND gpl-2.0)" + detected_lic_list = ["mit", "lgpl-2.1"] + expected = { + 'missing': ['apache-1.1', 'apache-2.0', 'bsd-new', 'gpl-2.0'], + 'extra': ['lgpl-2.1'], + 'is_match': False, + 'details': 'Missing: apache-1.1, apache-2.0, bsd-new, gpl-2.0; Extra: lgpl-2.1' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + def test_evaluate_license_mismatch_WITH_WITH_Exception(self): + package_lic = "bsd-new OR gpl-2.0 WITH classpath-exception-2.0" + detected_lic_list = ["bsd-new"] + expected = { + 'missing': [], + 'extra': [], + 'is_match': True, + 'details': 'match' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + def test_evaluate_license_mismatch_WITH_WITH_Exception_missing(self): + package_lic = "bsd-new AND gpl-2.0 WITH classpath-exception-2.0" + detected_lic_list = ["bsd-new", "gpl-2.0"] + expected = { + 'missing': ['classpath-exception-2.0'], + 'extra': [], + 'is_match': False, + 'details': 'Missing: classpath-exception-2.0' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected + + def test_evaluate_license_mismatch_WITH_WITH_Exception_extra(self): + package_lic = "bsd-new AND gpl-2.0" + detected_lic_list = ["bsd-new", "gpl-2.0 WITH classpath-exception-2.0"] + expected = { + 'missing': [], + 'extra': ['classpath-exception-2.0'], + 'is_match': False, + 'details': 'Extra: classpath-exception-2.0' + } + result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) + assert result == expected From cb9a90cb0153944d6f9b7bbd6b93da4d6fb3c5c4 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:38:28 +1300 Subject: [PATCH 04/73] feat!: replace plain-text DRF token with PBKDF2-hashed API token (#2087) Signed-off-by: tdruez --- docs/command-line-interface.rst | 24 ++++++-- pyproject.toml | 1 + scancodeio/settings.py | 9 ++- scancodeio/urls.py | 12 ++++ scanpipe/management/commands/create-user.py | 35 +++++++++-- scanpipe/migrations/0078_apitoken.py | 29 +++++++++ scanpipe/migrations/0079_apitoken_data.py | 60 +++++++++++++++++++ scanpipe/models.py | 15 ++--- scanpipe/templates/account/profile.html | 53 +++++++++++----- .../scanpipe/includes/navbar_header.html | 19 ++++-- .../profile_generate_api_key_modal.html | 27 +++++++++ .../modals/profile_revoke_api_key_modal.html | 27 +++++++++ scanpipe/tests/test_api.py | 4 +- scanpipe/tests/test_auth.py | 24 ++++++-- scanpipe/tests/test_commands.py | 20 ++++++- scanpipe/tests/test_models.py | 5 -- scanpipe/views.py | 17 ++++++ 17 files changed, 320 insertions(+), 61 deletions(-) create mode 100644 scanpipe/migrations/0078_apitoken.py create mode 100644 scanpipe/migrations/0079_apitoken_data.py create mode 100644 scanpipe/templates/scanpipe/modals/profile_generate_api_key_modal.html create mode 100644 scanpipe/templates/scanpipe/modals/profile_revoke_api_key_modal.html diff --git a/docs/command-line-interface.rst b/docs/command-line-interface.rst index ef2925127c..da6b5fb9e4 100644 --- a/docs/command-line-interface.rst +++ b/docs/command-line-interface.rst @@ -717,21 +717,32 @@ Optional arguments: .. note:: This command is to be used when ScanCode.io's authentication system :ref:`scancodeio_settings_require_authentication` is enabled. -Creates a user and generates an API key for authentication. +Creates a new user and optionally generates an API key for authentication. You will be prompted for a password. After you enter one, the user will be created immediately. -The API key for the new user account will be displayed on the terminal output. - .. code-block:: console - User created with API key: abcdef123456 + $ scanpipe create-user + User created. + +Use the ``--generate-api-key`` option to generate an API key for this user and print it +to the console. + +.. code-block:: console -The API key can also be retrieved from the :guilabel:`Profile settings` menu in the UI. + $ scanpipe create-user --generate-api-key + User created. + API key: 1234567890abcdef .. warning:: - Your API key is like a password and should be treated with the same care. + Treat your API key like a password and keep it secure. + For security reasons, the key is only shown once at generation time. + If you lose it, you will need to regenerate a new one. + +.. tip:: + The API key can be regenerated from the :guilabel:`Profile settings` menu in the UI. By default, this command will prompt for a password for the new user account. When run non-interactively with the ``--no-input`` option, no password will be set, @@ -741,6 +752,7 @@ API key. Optional arguments: - ``--no-input`` Does not prompt the user for input of any kind. +- ``--generate-api-key`` Generate an API key for this user and print it to the console. - ``--admin`` Specifies that the user should be created as an admin user. - ``--super`` Specifies that the user should be created as a superuser. diff --git a/pyproject.toml b/pyproject.toml index 2a90030284..990d52fe67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -96,6 +96,7 @@ dependencies = [ "aboutcode.hashid==0.2.0", # AboutCode pipeline "aboutcode.pipeline==0.2.1", + "aboutcode.api-auth==0.2.0", # ScoreCode "scorecode==0.0.4" ] diff --git a/scancodeio/settings.py b/scancodeio/settings.py index 4bb3673ec6..9a94a13383 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -194,7 +194,6 @@ "crispy_bootstrap3", # required for the djangorestframework browsable API "django_filters", "rest_framework", - "rest_framework.authtoken", "django_rq", "django_probes", "taggit", @@ -401,12 +400,12 @@ CLAMD_USE_TCP = env.bool("CLAMD_USE_TCP", default=True) CLAMD_TCP_ADDR = env.str("CLAMD_TCP_ADDR", default="clamav") -# Django restframework +# REST API + +API_TOKEN_MODEL = "scanpipe.APIToken" # noqa: S105 REST_FRAMEWORK = { - "DEFAULT_AUTHENTICATION_CLASSES": ( - "rest_framework.authentication.TokenAuthentication", - ), + "DEFAULT_AUTHENTICATION_CLASSES": ("aboutcode.api_auth.APITokenAuthentication",), "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",), "DEFAULT_RENDERER_CLASSES": ( "rest_framework.renderers.JSONRenderer", diff --git a/scancodeio/urls.py b/scancodeio/urls.py index f0e475e173..ecac3d6da8 100644 --- a/scancodeio/urls.py +++ b/scancodeio/urls.py @@ -32,6 +32,8 @@ from scanpipe.api.views import ProjectViewSet from scanpipe.api.views import RunViewSet from scanpipe.views import AccountProfileView +from scanpipe.views import GenerateAPIKeyView +from scanpipe.views import RevokeAPIKeyView api_router = DefaultRouter() api_router.register(r"projects", ProjectViewSet) @@ -45,6 +47,16 @@ name="logout", ), path("accounts/profile/", AccountProfileView.as_view(), name="account_profile"), + path( + "accounts/profile/api_key/generate/", + GenerateAPIKeyView.as_view(), + name="generate_api_key", + ), + path( + "accounts/profile/api_key/revoke/", + RevokeAPIKeyView.as_view(), + name="revoke_api_key", + ), ] diff --git a/scanpipe/management/commands/create-user.py b/scanpipe/management/commands/create-user.py index 8e537209a3..787e6c0ec1 100644 --- a/scanpipe/management/commands/create-user.py +++ b/scanpipe/management/commands/create-user.py @@ -28,7 +28,7 @@ from django.core.management.base import BaseCommand from django.core.management.base import CommandError -from rest_framework.authtoken.models import Token +from scanpipe.models import APIToken class Command(BaseCommand): @@ -43,13 +43,21 @@ def __init__(self, *args, **kwargs): ) def add_arguments(self, parser): - parser.add_argument("username", help="Specifies the username for the user.") + parser.add_argument( + "username", + help=f"Specifies the {self.UserModel.USERNAME_FIELD} for the user.", + ) parser.add_argument( "--no-input", action="store_false", dest="interactive", help="Do not prompt the user for input of any kind.", ) + parser.add_argument( + "--generate-api-key", + action="store_true", + help="Generate an API key for this user and print it to the console.", + ) parser.add_argument( "--admin", action="store_true", @@ -63,9 +71,16 @@ def add_arguments(self, parser): def handle(self, *args, **options): username = options["username"] + generate_api_key = options["generate_api_key"] is_admin = options["admin"] is_superuser = options["super"] + if options["verbosity"] <= 0 and generate_api_key: + raise CommandError( + "Cannot display the API key with verbosity disabled. " + "The key is only shown once at generation time." + ) + error_msg = self._validate_username(username) if error_msg: raise CommandError(error_msg) @@ -75,19 +90,27 @@ def handle(self, *args, **options): password = self.get_password_from_stdin(username) user_kwargs = { - "username": username, + self.UserModel.USERNAME_FIELD: username, "password": password, "is_staff": is_admin or is_superuser, "is_superuser": is_superuser, } - user = self.UserModel._default_manager.create_user(**user_kwargs) - token, _ = Token._default_manager.get_or_create(user=user) if options["verbosity"] > 0: - msg = f"User {username} created with API key: {token.key}" + msg = f"User {username} created." self.stdout.write(msg, self.style.SUCCESS) + if generate_api_key: + plain_api_key = APIToken.create_token(user=user) + self.stdout.write(f"API key: {plain_api_key}", self.style.SUCCESS) + warning_msg = ( + "Treat your API key like a password and keep it secure. " + "For security reasons, the key is only shown once at generation time. " + "If you lose it, you will need to regenerate a new one." + ) + self.stdout.write(warning_msg, self.style.WARNING) + def get_password_from_stdin(self, username): # Validators, such as UserAttributeSimilarityValidator, depends on other user's # fields data for password validation. diff --git a/scanpipe/migrations/0078_apitoken.py b/scanpipe/migrations/0078_apitoken.py new file mode 100644 index 0000000000..aa22986c15 --- /dev/null +++ b/scanpipe/migrations/0078_apitoken.py @@ -0,0 +1,29 @@ +# Generated by Django 6.0.3 on 2026-03-09 05:23 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0077_alter_discoveredpackage_children_packages'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='APIToken', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key_hash', models.CharField(max_length=128)), + ('prefix', models.CharField(db_index=True, max_length=8, unique=True)), + ('created', models.DateTimeField(auto_now_add=True, db_index=True)), + ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='api_token', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'API Token', + }, + ), + ] diff --git a/scanpipe/migrations/0079_apitoken_data.py b/scanpipe/migrations/0079_apitoken_data.py new file mode 100644 index 0000000000..22443817ef --- /dev/null +++ b/scanpipe/migrations/0079_apitoken_data.py @@ -0,0 +1,60 @@ +# Generated by Django 6.0.3 on 2026-03-10 22:09 + +from django.db import migrations +from django.contrib.auth.hashers import make_password + + +def migrate_api_tokens(apps, schema_editor): + """Migrate existing plain-text DRF tokens to the new hashed APIToken model.""" + APIToken = apps.get_model("scanpipe", "APIToken") + PREFIX_LENGTH = 8 + + with schema_editor.connection.cursor() as cursor: + cursor.execute( + "SELECT EXISTS (SELECT 1 FROM information_schema.tables " + "WHERE table_name = 'authtoken_token')" + ) + table_exists = cursor.fetchone()[0] + + if not table_exists: + return + + cursor.execute("SELECT user_id, key, created FROM authtoken_token") + rows = cursor.fetchall() + if not rows: + return + + tokens_to_create = [ + APIToken( + user_id=user_id, + prefix=key[:PREFIX_LENGTH], + key_hash=make_password(key), + created=created, + ) + for user_id, key, created in rows + ] + migrated_tokens = APIToken.objects.bulk_create(tokens_to_create, ignore_conflicts=True) + if migrated_tokens: + print(f" -> {len(migrated_tokens)} tokens migrated.") + + +def reverse_migrate_api_tokens(apps, schema_editor): + """Reverse migration: remove all migrated tokens.""" + APIToken = apps.get_model("scanpipe", "APIToken") + APIToken.objects.all().delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0078_apitoken'), + ] + + operations = [ + migrations.RunPython(migrate_api_tokens, reverse_migrate_api_tokens), + migrations.RunSQL( + sql="DROP TABLE IF EXISTS authtoken_token", + reverse_sql=migrations.RunSQL.noop, + ), + ] + diff --git a/scanpipe/models.py b/scanpipe/models.py index ac8e2da155..f065fb4bb5 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -57,7 +57,6 @@ from django.db.models import When from django.db.models.functions import Cast from django.db.models.functions import Lower -from django.dispatch import receiver from django.forms import model_to_dict from django.urls import NoReverseMatch from django.urls import reverse @@ -70,6 +69,7 @@ import redis import requests import saneyaml +from aboutcode.api_auth import AbstractAPIToken from commoncode.fileutils import parent_directory from cyclonedx import model as cyclonedx_model from cyclonedx.model import component as cyclonedx_component @@ -86,7 +86,6 @@ from packageurl.contrib.django.models import PACKAGE_URL_FIELDS from packageurl.contrib.django.models import PackageURLMixin from packageurl.contrib.django.models import PackageURLQuerySetMixin -from rest_framework.authtoken.models import Token from rq.command import send_stop_job_command from rq.exceptions import NoSuchJobError from rq.job import Job @@ -146,6 +145,11 @@ class Meta: abstract = True +class APIToken(AbstractAPIToken): + class Meta: + verbose_name = "API Token" + + class HashFieldsMixin(models.Model): """ The hash fields are not indexed by default, use the `indexes` in Meta as needed: @@ -4963,13 +4967,6 @@ def success(self): return self.response_status_code in (200, 201, 202) -@receiver(models.signals.post_save, sender=settings.AUTH_USER_MODEL) -def create_auth_token(sender, instance=None, created=False, **kwargs): - """Create an API key token on user creation, using the signal system.""" - if created: - Token.objects.create(user_id=instance.pk) - - class DiscoveredPackageScore(UUIDPKModel, PackageScoreMixin): """Represents a security or quality score for a DiscoveredPackage.""" diff --git a/scanpipe/templates/account/profile.html b/scanpipe/templates/account/profile.html index ec96eac82d..f71a732cd1 100644 --- a/scanpipe/templates/account/profile.html +++ b/scanpipe/templates/account/profile.html @@ -13,23 +13,46 @@ - -
-
- An API key is like a password and should be treated with the same care. -
-
- -
- -
- - - - +
+
+
+
+ Your personal API key provides access to the + REST API +
+ Treat it like a password and keep it secure. +
+
+ {% if request.user.api_token %} +
+ Your API key {{ request.user.api_token.prefix }}... + was generated on {{ request.user.api_token.created }}
+ For security reasons, the full key is only shown once at generation time.
+ If you lose it, you will need to regenerate a new one. +
+ {% else %} +
+ No API key created.
+ Generate one using the button below to access the REST API. +
+ {% endif %} +
+
+ + {% if request.user.api_token %} + + {% endif %} +
- + {% include 'scanpipe/modals/profile_generate_api_key_modal.html' %} + {% if request.user.api_token %} + {% include 'scanpipe/modals/profile_revoke_api_key_modal.html' %} + {% endif %}
{% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/includes/navbar_header.html b/scanpipe/templates/scanpipe/includes/navbar_header.html index caa3bc74e7..cb12597fa8 100644 --- a/scanpipe/templates/scanpipe/includes/navbar_header.html +++ b/scanpipe/templates/scanpipe/includes/navbar_header.html @@ -35,14 +35,11 @@
+ + + Profile settings -
- {% csrf_token %} - -
ScanCode.io {{ SCANCODEIO_VERSION }} @@ -50,6 +47,16 @@
ScanCode-toolkit {{ SCANCODE_TOOLKIT_VERSION }}
+
+
+ {% csrf_token %} + +
{% else %} diff --git a/scanpipe/templates/scanpipe/modals/profile_generate_api_key_modal.html b/scanpipe/templates/scanpipe/modals/profile_generate_api_key_modal.html new file mode 100644 index 0000000000..b302f25ea2 --- /dev/null +++ b/scanpipe/templates/scanpipe/modals/profile_generate_api_key_modal.html @@ -0,0 +1,27 @@ + \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/modals/profile_revoke_api_key_modal.html b/scanpipe/templates/scanpipe/modals/profile_revoke_api_key_modal.html new file mode 100644 index 0000000000..6bfa65d118 --- /dev/null +++ b/scanpipe/templates/scanpipe/modals/profile_revoke_api_key_modal.html @@ -0,0 +1,27 @@ + \ No newline at end of file diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index 8bb5f63ef3..7bb66da2f5 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -45,6 +45,7 @@ from scanpipe.api.serializers import ProjectSerializer from scanpipe.api.serializers import get_model_serializer from scanpipe.api.serializers import get_serializer_fields +from scanpipe.models import APIToken from scanpipe.models import CodebaseRelation from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency @@ -91,7 +92,8 @@ def setUp(self): self.project1_detail_url = reverse("project-detail", args=[self.project1.uuid]) self.user = User.objects.create_user("username", "e@mail.com", "secret") - self.auth = f"Token {self.user.auth_token.key}" + self.user_api_key = APIToken.create_token(user=self.user) + self.auth = f"Token {self.user_api_key}" self.csrf_client = APIClient(enforce_csrf_checks=True) self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) diff --git a/scanpipe/tests/test_auth.py b/scanpipe/tests/test_auth.py index 32e178a7d4..2f9213c6a4 100644 --- a/scanpipe/tests/test_auth.py +++ b/scanpipe/tests/test_auth.py @@ -22,6 +22,7 @@ import uuid +from django.apps import apps from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.auth.models import AnonymousUser @@ -35,6 +36,7 @@ TEST_PASSWORD = str(uuid.uuid4()) +APIToken = apps.get_model("scanpipe", "APIToken") login_url = reverse("login") project_list_url = reverse("project_list") logout_url = reverse("logout") @@ -93,10 +95,10 @@ def test_scancodeio_auth_logged_in_navbar_header(self): response = self.client.get(project_list_url) expected = 'basic_user' self.assertContains(response, expected, html=True) - expected = f'Profile settings' - self.assertContains(response, expected, html=True) expected = f'
' self.assertContains(response, expected) + self.assertContains(response, profile_url) + self.assertContains(response, "Profile settings") def test_scancodeio_auth_logout_view(self): response = self.client.get(logout_url) @@ -111,10 +113,22 @@ def test_scancodeio_auth_logout_view(self): def test_scancodeio_account_profile_view(self): self.client.login(username=self.basic_user.username, password=TEST_PASSWORD) + + expected1 = "No API key created." + expected2 = "Generate API key" + expected3 = "Revoke API key" + response = self.client.get(profile_url) - expected = '' - self.assertContains(response, expected, html=True) - self.assertContains(response, self.basic_user.auth_token.key) + self.assertContains(response, expected1) + self.assertContains(response, expected2) + self.assertNotContains(response, expected3) + + APIToken.create_token(user=self.basic_user) + response = self.client.get(profile_url) + self.assertNotContains(response, expected1) + self.assertContains(response, expected2) + self.assertContains(response, expected3) + self.assertContains(response, self.basic_user.api_token.prefix) def test_scancodeio_auth_views_are_protected(self): a_uuid = uuid.uuid4() diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 7e93bc1c64..749fc0269b 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -31,6 +31,7 @@ from django.apps import apps from django.contrib.auth import get_user_model +from django.core.exceptions import ObjectDoesNotExist from django.core.management import CommandError from django.core.management import call_command from django.core.management.base import BaseCommand @@ -928,15 +929,28 @@ def test_scanpipe_management_command_create_user(self): username = "my_username" call_command("create-user", "--no-input", username, stdout=out) - self.assertIn(f"User {username} created with API key:", out.getvalue()) + self.assertIn(f"User {username} created.", out.getvalue()) + self.assertNotIn("API key", out.getvalue()) user = get_user_model().objects.get(username=username) - self.assertTrue(user.auth_token) self.assertFalse(user.is_staff) self.assertFalse(user.is_superuser) + message = "User has no api_token" + with self.assertRaisesMessage(ObjectDoesNotExist, message): + user.api_token + + username = "verbosity_issue" + options = ["--no-input", "--generate-api-key", "--verbosity=0"] + expected = ( + "Cannot display the API key with verbosity disabled. " + "The key is only shown once at generation time." + ) + with self.assertRaisesMessage(CommandError, expected): + call_command("create-user", username, *options) + expected = "Error: That username is already taken." with self.assertRaisesMessage(CommandError, expected): - call_command("create-user", "--no-input", username) + call_command("create-user", "--no-input", user.username) username = "^&*" expected = ( diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index e4a5b4cb7d..0251e7b729 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -2635,11 +2635,6 @@ def test_scanpipe_discovered_package_model_get_author_names(self): expected = ["Debian X Strike Force", "JBoss.org Community"] self.assertEqual(expected, package1.get_author_names(roles)) - def test_scanpipe_model_create_user_creates_auth_token(self): - basic_user = User.objects.create_user(username="basic_user") - self.assertTrue(basic_user.auth_token.key) - self.assertEqual(40, len(basic_user.auth_token.key)) - def test_scanpipe_discovered_dependency_model_update_from_data(self): DiscoveredPackage.create_from_data(self.project1, package_data1) CodebaseResource.objects.create( diff --git a/scanpipe/views.py b/scanpipe/views.py index 5e657b874b..038a3e8722 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -60,6 +60,8 @@ import saneyaml import xlsxwriter +from aboutcode.api_auth.views import BaseGenerateAPIKeyView +from aboutcode.api_auth.views import BaseRevokeAPIKeyView from django_filters.views import FilterView from django_htmx.http import HttpResponseClientRedirect from licensedcode.spans import Span @@ -2835,3 +2837,18 @@ def get_context_data(self, **kwargs): context["parent_path"] = "/".join(parent_segments) return context + + +class GenerateAPIKeyView(ConditionalLoginRequired, BaseGenerateAPIKeyView): + success_url = reverse_lazy("account_profile") + success_message = ( + "Copy your API key now, it will not be shown again:" + '
'
+        '{plain_key}'
+        "
" + ) + + +class RevokeAPIKeyView(ConditionalLoginRequired, BaseRevokeAPIKeyView): + success_url = reverse_lazy("account_profile") + success_message = "API key revoked." From a0929391d704ef955c3bea9672a044bab75fee1e Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 11 Mar 2026 15:29:55 +1300 Subject: [PATCH 05/73] chore: set explicit workflow permissions and pin down actions (#2090) Signed-off-by: tdruez --- .github/workflows/generate-sboms.yml | 10 ++- .github/workflows/pr-quality.yml | 5 +- .github/workflows/publish-docker-image.yml | 25 ++++-- ...ublish-pypi-release-aboutcode-pipeline.yml | 48 ++++++++--- .github/workflows/publish-pypi-release.yml | 79 ++++++++++++++----- .github/workflows/run-unit-tests-docker.yml | 6 +- .github/workflows/run-unit-tests-macos.yml | 10 ++- .github/workflows/run-unit-tests.yml | 10 ++- .github/workflows/sca-integration-anchore.yml | 6 +- .github/workflows/sca-integration-cdxgen.yml | 6 +- .../sca-integration-cyclonedx-gomod.yml | 9 ++- .github/workflows/sca-integration-depscan.yml | 8 +- .../sca-integration-ort-package-file.yml | 22 +++--- .github/workflows/sca-integration-ort.yml | 40 +++++----- .../workflows/sca-integration-osv-scanner.yml | 7 +- .../workflows/sca-integration-sbom-tool.yml | 7 +- .github/workflows/sca-integration-trivy.yml | 8 +- 17 files changed, 202 insertions(+), 104 deletions(-) diff --git a/.github/workflows/generate-sboms.yml b/.github/workflows/generate-sboms.yml index 0cb0fc5ee6..6440dc0875 100644 --- a/.github/workflows/generate-sboms.yml +++ b/.github/workflows/generate-sboms.yml @@ -12,10 +12,14 @@ env: jobs: generate-sboms: runs-on: ubuntu-24.04 + permissions: + contents: read steps: - - name: Checkout code - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Ensure INPUTS_PATH directory exists run: mkdir -p "${{ env.INPUTS_PATH }}" @@ -32,7 +36,7 @@ jobs: find scancodeio/ -type f -name "*.ABOUT" -exec cp {} "${{ env.INPUTS_PATH }}/about-files/" \; - name: Resolve the dependencies using ScanCode-action - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "resolve_dependencies:DynamicResolver" inputs-path: ${{ env.INPUTS_PATH }} diff --git a/.github/workflows/pr-quality.yml b/.github/workflows/pr-quality.yml index 0eaa5b048b..1e938fe95e 100644 --- a/.github/workflows/pr-quality.yml +++ b/.github/workflows/pr-quality.yml @@ -6,6 +6,9 @@ permissions: pull-requests: write on: + # pull_request_target is required so the action can close/comment on fork PRs. + # This is safe because: no untrusted code is checked out, and no attacker-controlled + # values are interpolated into shell commands. All action inputs are hardcoded. pull_request_target: types: [opened, reopened] @@ -14,7 +17,7 @@ jobs: runs-on: ubuntu-24.04 name: Detects and automatically closes low-quality and AI slop PRs steps: - - uses: peakoss/anti-slop@v0 + - uses: peakoss/anti-slop@e158eeefe5c43e1d3ba8533b84e0e35d9d6761de with: # Number of check failures needed before failure actions are triggered max-failures: 3 diff --git a/.github/workflows/publish-docker-image.yml b/.github/workflows/publish-docker-image.yml index be24c863d8..9bc575214e 100644 --- a/.github/workflows/publish-docker-image.yml +++ b/.github/workflows/publish-docker-image.yml @@ -22,15 +22,19 @@ jobs: permissions: contents: read packages: write + attestations: write + id-token: write steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around # Uses the `docker/login-action` action to log in to the Container registry using # the account and password that will publish the packages. - name: Log in to the Container registry - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -42,7 +46,7 @@ jobs: # The `images` value provides the base name for the tags and labels. - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} @@ -53,11 +57,22 @@ jobs: # It uses the `tags` and `labels` parameters to tag and label the image with # the output from the "meta" step. - name: Build and push Docker image - uses: docker/build-push-action@v5 + id: push + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . push: true tags: | ${{ steps.meta.outputs.tags }} - ${{ env.REGISTRY }}/aboutcode-org/scancode.io:latest + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest labels: ${{ steps.meta.outputs.labels }} + + # This step generates an artifact attestation for the image, which is an + # unforgeable statement about where and how it was built. + # It increases supply chain security for people who consume the image. + - name: Generate artifact attestation + uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true diff --git a/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml b/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml index cef72ed191..62ff6c388c 100644 --- a/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml +++ b/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml @@ -7,32 +7,56 @@ on: - "aboutcode.pipeline/*" jobs: - build-and-publish: + build: name: Build and publish library to PyPI runs-on: ubuntu-24.04 + permissions: + contents: read steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: 3.14 - name: Install flot - run: python -m pip install flot --user + run: python -m pip install flot==0.7.2 --user - name: Build a binary wheel and a source tarball run: python -m flot --pyproject pipeline-pyproject.toml --sdist --wheel --output-dir dist/ - - name: Publish to PyPI - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Upload package distributions as GitHub workflow artifacts + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - password: ${{ secrets.PYPI_API_TOKEN_ABOUTCODE_PIPELINE }} + name: python-package-distributions + path: dist/ + + # Only set the id-token: write permission in the job that does publishing, not globally. + # Also, separate building from publishing — this makes sure that any scripts + # maliciously injected into the build or test environment won't be able to elevate + # privileges while flying under the radar. + pypi-publish: + name: Upload package distributions to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-24.04 + environment: + name: pypi + url: https://pypi.org/p/aboutcode.pipeline + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - - name: Upload built archives - uses: actions/upload-artifact@v4 + steps: + - name: Download all the dists + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: - name: pypi_archives - path: dist/* + name: python-package-distributions + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 diff --git a/.github/workflows/publish-pypi-release.yml b/.github/workflows/publish-pypi-release.yml index 7d13564a9c..2e5163a147 100644 --- a/.github/workflows/publish-pypi-release.yml +++ b/.github/workflows/publish-pypi-release.yml @@ -1,4 +1,4 @@ -name: Build Python distributions and publish on PyPI +name: Build Python distributions, publish on PyPI, and create a GH release on: workflow_dispatch: @@ -6,16 +6,24 @@ on: tags: - "v*.*.*" +env: + PYPI_PROJECT_URL: "https://pypi.org/p/scancodeio" + jobs: - build-and-publish: - name: Build and publish library to PyPI + build-python-dist: + name: Build Python distributions runs-on: ubuntu-24.04 + permissions: + contents: read steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: 3.14 @@ -23,23 +31,56 @@ jobs: run: python -m pip install build --user - name: Build a binary wheel and a source tarball - run: python -m build --sdist --wheel --outdir dist/ . + run: python -m build --sdist --wheel --outdir dist/ - - name: Publish to PyPI - if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Upload package distributions as GitHub workflow artifacts + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - password: ${{ secrets.PYPI_API_TOKEN }} + name: python-package-distributions + path: dist/ - - name: Upload built archives - uses: actions/upload-artifact@v4 + # Only set the id-token: write permission in the job that does publishing, not globally. + # Also, separate building from publishing — this makes sure that any scripts + # maliciously injected into the build or test environment won't be able to elevate + # privileges while flying under the radar. + pypi-publish: + name: Upload package distributions to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build-python-dist + runs-on: ubuntu-24.04 + environment: + name: pypi + url: ${{ env.PYPI_PROJECT_URL }} + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + steps: + - name: Download package distributions + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: - name: pypi_archives - path: dist/* + name: python-package-distributions + path: dist/ - - name: Create a GitHub release - uses: softprops/action-gh-release@v2 + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + + create-gh-release: + name: Create GitHub release + needs: + - build-python-dist + runs-on: ubuntu-24.04 + permissions: + contents: write + + steps: + - name: Download package distributions + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: - generate_release_notes: true - draft: false - files: dist/* + name: python-package-distributions + path: dist/ + + - name: Create GitHub release + run: gh release create "$GITHUB_REF_NAME" dist/* --generate-notes + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/run-unit-tests-docker.yml b/.github/workflows/run-unit-tests-docker.yml index 609fdfab09..73a20835c3 100644 --- a/.github/workflows/run-unit-tests-docker.yml +++ b/.github/workflows/run-unit-tests-docker.yml @@ -15,8 +15,10 @@ jobs: runs-on: ubuntu-24.04 steps: - - name: Checkout code - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Generate the .env file and the SECRET_KEY run: make envfile diff --git a/.github/workflows/run-unit-tests-macos.yml b/.github/workflows/run-unit-tests-macos.yml index df128bfa6a..b48dc764fd 100644 --- a/.github/workflows/run-unit-tests-macos.yml +++ b/.github/workflows/run-unit-tests-macos.yml @@ -24,16 +24,18 @@ jobs: python-version: ["3.12", "3.13", "3.14"] steps: - - name: Checkout code - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} - name: Set up PostgreSQL - uses: ikalnytskyi/action-setup-postgres@v8 + uses: ikalnytskyi/action-setup-postgres@c4dda34aae1c821e3a771b68b73b13af3198a7ee # v8 with: postgres-version: "17" database: ${{ env.POSTGRES_DB }} diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 2d8c286ca0..59c9f6657f 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -22,7 +22,7 @@ jobs: services: postgres: - image: postgres:17 + image: postgres:17.9 env: POSTGRES_DB: ${{ env.POSTGRES_DB }} POSTGRES_USER: ${{ env.POSTGRES_USER }} @@ -42,11 +42,13 @@ jobs: python-version: ["3.12", "3.13", "3.14"] steps: - - name: Checkout code - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/sca-integration-anchore.yml b/.github/workflows/sca-integration-anchore.yml index d8ac014829..f57339fd02 100644 --- a/.github/workflows/sca-integration-anchore.yml +++ b/.github/workflows/sca-integration-anchore.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Generate CycloneDX SBOM with Anchore Grype scanner - uses: anchore/scan-action@v6 + uses: anchore/scan-action@7037fa011853d5a11690026fb85feee79f4c946c # v7.3.2 with: image: ${{ env.IMAGE_REFERENCE }} output-format: cyclonedx-json @@ -36,14 +36,14 @@ jobs: fail-build: false - name: Upload SBOM as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: anchore-sbom-report path: "anchore-grype-sbom.cdx.json" retention-days: 20 - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "anchore-grype-sbom.cdx.json" diff --git a/.github/workflows/sca-integration-cdxgen.yml b/.github/workflows/sca-integration-cdxgen.yml index ad7f050fac..ded93df560 100644 --- a/.github/workflows/sca-integration-cdxgen.yml +++ b/.github/workflows/sca-integration-cdxgen.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Install CycloneDX cdxgen - run: npm install @cyclonedx/cdxgen + run: npm install @cyclonedx/cdxgen@12.1.2 - name: Generate SBOM with CycloneDX cdxgen run: | @@ -39,14 +39,14 @@ jobs: --json-pretty - name: Upload SBOM as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: cdxgen-sbom path: "cdxgen-sbom.cdx.json" retention-days: 20 - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "cdxgen-sbom.cdx.json" diff --git a/.github/workflows/sca-integration-cyclonedx-gomod.yml b/.github/workflows/sca-integration-cyclonedx-gomod.yml index bbbf724f7c..ea39bd659f 100644 --- a/.github/workflows/sca-integration-cyclonedx-gomod.yml +++ b/.github/workflows/sca-integration-cyclonedx-gomod.yml @@ -27,25 +27,26 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout minimal Go repo - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: opencontainers/runc + persist-credentials: false # do not keep the token around - name: Generate SBOM with cyclonedx-gomod - uses: CycloneDX/gh-gomod-generate-sbom@v2 + uses: CycloneDX/gh-gomod-generate-sbom@efc74245d6802c8cefd925620515442756c70d8f # v2.0.0 with: version: v1 args: mod -licenses -json -output gomod-sbom.cdx.json - name: Upload SBOM as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: sbom-report path: "gomod-sbom.cdx.json" retention-days: 20 - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "gomod-sbom.cdx.json" diff --git a/.github/workflows/sca-integration-depscan.yml b/.github/workflows/sca-integration-depscan.yml index 6b824a6a60..76cb16aace 100644 --- a/.github/workflows/sca-integration-depscan.yml +++ b/.github/workflows/sca-integration-depscan.yml @@ -29,8 +29,8 @@ jobs: steps: - name: Install OWASP dep-scan run: | - sudo npm install -g @cyclonedx/cdxgen - pip install owasp-depscan + sudo npm install -g @cyclonedx/cdxgen@12.1.2 + pip install owasp-depscan==6.1.0 - name: Generate SBOM with OWASP dep-scan run: | @@ -41,7 +41,7 @@ jobs: --explain - name: Upload SBOM as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: depscan-sbom path: reports/ @@ -51,7 +51,7 @@ jobs: run: pip uninstall --yes owasp-depscan - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "reports/sbom-docker.vdr.json" diff --git a/.github/workflows/sca-integration-ort-package-file.yml b/.github/workflows/sca-integration-ort-package-file.yml index 50cc4780dc..14ee50419d 100644 --- a/.github/workflows/sca-integration-ort-package-file.yml +++ b/.github/workflows/sca-integration-ort-package-file.yml @@ -17,49 +17,49 @@ permissions: env: SCIO_IMAGE_INPUT: "docker://osadl/alpine-docker-base-image:v3.22-latest" - ORT_VERSION: "68.1.0" + ORT_VERSION: "82.0.0" jobs: generate-and-load-sbom: runs-on: ubuntu-24.04 steps: - name: Analyze Docker image with ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "analyze_docker_image" input-urls: "${{ env.SCIO_IMAGE_INPUT }}" scancodeio-repo-branch: "main" - output-formats: "ort-package-list spdx:2.2 cyclonedx json xlsx" + output-formats: "ort-package-list" - name: Copy package-list.yml to workspace root run: | - FILE=$(ls ${{ env.PROJECT_WORK_DIRECTORY }}/output/*.package-list.yml | head -n 1) - sudo mkdir -p ${GITHUB_WORKSPACE}/ort-data/ + FILE=$(ls "${PROJECT_WORK_DIRECTORY}/output/"*.package-list.yml | head -n 1) + sudo mkdir -p "${GITHUB_WORKSPACE}/ort-data/" sudo cp "$FILE" "${GITHUB_WORKSPACE}/ort-data/package-list.yml" - sudo chmod -R 777 ${GITHUB_WORKSPACE}/ort-data/ + sudo chmod -R 777 "${GITHUB_WORKSPACE}/ort-data/" ls -lh "${GITHUB_WORKSPACE}/ort-data/" - name: Generates an ORT analyzer-result.yml file run: | - docker run --rm -v ${GITHUB_WORKSPACE}/ort-data:/data \ + docker run --rm -v "${GITHUB_WORKSPACE}/ort-data:/data" \ --entrypoint /opt/ort/bin/orth \ - ghcr.io/oss-review-toolkit/ort:${{ env.ORT_VERSION }} \ + "ghcr.io/oss-review-toolkit/ort:${ORT_VERSION}" \ create-analyzer-result-from-package-list \ --package-list-file /data/package-list.yml \ --ort-file /data/analyzer-result.yml - name: Report as CycloneDX and SPDX using the analyzer-result.yml file run: | - docker run --rm -v ${GITHUB_WORKSPACE}/ort-data:/data \ - ghcr.io/oss-review-toolkit/ort:${{ env.ORT_VERSION }} \ + docker run --rm -v "${GITHUB_WORKSPACE}/ort-data:/data" \ + "ghcr.io/oss-review-toolkit/ort:${ORT_VERSION}" \ report \ --ort-file /data/analyzer-result.yml \ --output-dir /data/results/ \ --report-formats CycloneDX,SpdxDocument - name: Upload SBOMs as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: ort-report path: "${GITHUB_WORKSPACE}/ort-data/results" diff --git a/.github/workflows/sca-integration-ort.yml b/.github/workflows/sca-integration-ort.yml index 2c777845ff..25f5d82fe4 100644 --- a/.github/workflows/sca-integration-ort.yml +++ b/.github/workflows/sca-integration-ort.yml @@ -21,11 +21,13 @@ jobs: checkout-ort-test-assets-from-scancode-io-repo: runs-on: ubuntu-24.04 steps: - - name: Checkout ScanCode.io repository - uses: actions/checkout@v5 + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # do not keep the token around - name: Upload orthw mime types example - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: npm-mime-types-2.1.26-scan-result.json path: scanpipe/tests/data/integrations-ort/orthw-example-scan-result/npm-mime-types-2.1.26-scan-result.json @@ -44,7 +46,7 @@ jobs: EOF - name: Run GitHub Action for ORT - uses: oss-review-toolkit/ort-ci-github-action@v1 + uses: oss-review-toolkit/ort-ci-github-action@1805edcf1f4f55f35ae6e4d2d9795ccfb29b6021 # v1.1.0 with: ort-cli-report-args: "-O CycloneDX=output.file.formats=json -O CycloneDX=schema.version=1.5" report-formats: "CycloneDx" @@ -55,7 +57,7 @@ jobs: reporter - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "${{ env.ORT_RESULTS_PATH }}/bom.cyclonedx.json" @@ -83,7 +85,7 @@ jobs: EOF - name: Run GitHub Action for ORT - uses: oss-review-toolkit/ort-ci-github-action@v1 + uses: oss-review-toolkit/ort-ci-github-action@1805edcf1f4f55f35ae6e4d2d9795ccfb29b6021 # v1.1.0 with: ort-cli-report-args: "-O CycloneDX=output.file.formats=json -O CycloneDX=schema.version=1.6" report-formats: "CycloneDx" @@ -94,7 +96,7 @@ jobs: reporter - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "${{ env.ORT_RESULTS_PATH }}/bom.cyclonedx.json" @@ -114,7 +116,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Download mime-type-2.1.26-scan-result file - uses: actions/download-artifact@v5 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: npm-mime-types-2.1.26-scan-result.json @@ -126,16 +128,16 @@ jobs: cat $HOME/.ort/ort-results/current-result.json - name: Run GitHub Action for ORT - uses: oss-review-toolkit/ort-ci-github-action@v1 + uses: oss-review-toolkit/ort-ci-github-action@1805edcf1f4f55f35ae6e4d2d9795ccfb29b6021 # v1.1.0 with: report-formats: "CycloneDx,SpdxDocument" run: > evaluator, advisor, reporter - - name: Upload orthw mime type example - - uses: actions/upload-artifact@v4 + - name: Upload orthw mime type example + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: npm-mime-types-2.1.26-ort-sboms path: | @@ -151,12 +153,12 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Download ORT CycloneDX JSON SBOM for mime-types 2.1.26 - uses: actions/download-artifact@v5 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: npm-mime-types-2.1.26-ort-sboms - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "bom.cyclonedx.json" @@ -177,12 +179,12 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Download ORT CycloneDX JSON SBOM for mime-types 2.1.26 - uses: actions/download-artifact@v5 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: npm-mime-types-2.1.26-ort-sboms - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "bom.cyclonedx.xml" @@ -203,12 +205,12 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Download ORT SPDX JSON SBOM for mime-types 2.1.26 - uses: actions/download-artifact@v5 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: npm-mime-types-2.1.26-ort-sboms - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "bom.spdx.json" @@ -229,12 +231,12 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Download ORT SPDX YAML SBOM for mime-types 2.1.26 - uses: actions/download-artifact@v5 + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: name: npm-mime-types-2.1.26-ort-sboms - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "bom.spdx.yml" diff --git a/.github/workflows/sca-integration-osv-scanner.yml b/.github/workflows/sca-integration-osv-scanner.yml index 0edaa49d9c..2bc594026f 100644 --- a/.github/workflows/sca-integration-osv-scanner.yml +++ b/.github/workflows/sca-integration-osv-scanner.yml @@ -19,6 +19,7 @@ permissions: env: IMAGE_REFERENCE: "python:3.13.0-slim" + OSV_SCANNER_URL: "https://github.com/google/osv-scanner/releases/download/v2.3.3/osv-scanner_linux_amd64" EXPECTED_PACKAGE: 100 EXPECTED_VULNERABLE_PACKAGE: 0 EXPECTED_DEPENDENCY: 90 @@ -29,7 +30,7 @@ jobs: steps: - name: Install OSV-Scanner run: | - curl -sLO https://github.com/google/osv-scanner/releases/latest/download/osv-scanner_linux_amd64 + curl -sLO "$OSV_SCANNER_URL" chmod +x osv-scanner_linux_amd64 sudo mv osv-scanner_linux_amd64 /usr/local/bin/osv-scanner @@ -43,14 +44,14 @@ jobs: || true - name: Upload SBOM as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: osv-scanner-sbom-report path: osv-sbom.spdx.json retention-days: 20 - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "osv-sbom.spdx.json" diff --git a/.github/workflows/sca-integration-sbom-tool.yml b/.github/workflows/sca-integration-sbom-tool.yml index 01bc2fe96d..14ef30c99b 100644 --- a/.github/workflows/sca-integration-sbom-tool.yml +++ b/.github/workflows/sca-integration-sbom-tool.yml @@ -19,6 +19,7 @@ permissions: env: IMAGE_REFERENCE: "python:3.13.0-slim" + SBOM_TOOL_URL: "https://github.com/microsoft/sbom-tool/releases/download/v4.1.5/sbom-tool-linux-x64" EXPECTED_PACKAGE: 90 EXPECTED_VULNERABLE_PACKAGE: 0 EXPECTED_DEPENDENCY: 90 @@ -29,7 +30,7 @@ jobs: steps: - name: Download SBOM tool run: | - curl -Lo $RUNNER_TEMP/sbom-tool https://github.com/microsoft/sbom-tool/releases/latest/download/sbom-tool-linux-x64 + curl -Lo $RUNNER_TEMP/sbom-tool "$SBOM_TOOL_URL" chmod +x $RUNNER_TEMP/sbom-tool - name: Generate SBOM with SBOM tool @@ -45,13 +46,13 @@ jobs: -V Verbose - name: Upload SBOM artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: sbom-output path: sbom-output - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "sbom-output/_manifest/spdx_2.2/manifest.spdx.json" diff --git a/.github/workflows/sca-integration-trivy.yml b/.github/workflows/sca-integration-trivy.yml index d135e00322..c05e538cce 100644 --- a/.github/workflows/sca-integration-trivy.yml +++ b/.github/workflows/sca-integration-trivy.yml @@ -28,24 +28,24 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Generate CycloneDX SBOM with Trivy - uses: aquasecurity/trivy-action@0.32.0 + uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0 with: scan-type: "image" image-ref: ${{ env.IMAGE_REFERENCE }} format: "cyclonedx" output: "trivy-report.sbom.json" scanners: "vuln,license" - version: "latest" + version: "v0.69.3" - name: Upload SBOM as GitHub Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: trivy-sbom-report path: "trivy-report.sbom.json" retention-days: 20 - name: Import SBOM into ScanCode.io - uses: aboutcode-org/scancode-action@main + uses: aboutcode-org/scancode-action@8adbf888f487c3cdf6c15386035769cd03a94c66 with: pipelines: "load_sbom" inputs-path: "trivy-report.sbom.json" From 731599bdfd3c8c510c832700c0b8edc0a6be5fb7 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 11 Mar 2026 18:48:53 +1300 Subject: [PATCH 06/73] chore: bump version to v37.0.0 for release (#2091) Signed-off-by: tdruez --- CHANGELOG.rst | 26 ++++++++++++++++++++------ pyproject.toml | 2 +- scancodeio/__init__.py | 2 +- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0966a61a34..6a779ef3db 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,18 +1,32 @@ Changelog ========= -**v36 Breaking Change:** PostgreSQL 17 is now required (previously 13). +**v37 Breaking Changes:** -Docker Compose users with existing data: run `./migrate-pg13-to-17.sh` before starting -the stack. -Fresh installations require no action. +- Drop support for Python3.10 and Python3.11 -v37.0.0 (unreleased) +v37.0.0 (2026-03-11) -------------------- - Upgrade Django to release 6.x -- Drop support for Python3.10 and Python3.11 +- Remove the chown service in compose file + https://github.com/aboutcode-org/scancode.io/issues/2086 + +- Replace plain-text DRF token with PBKDF2-hashed API token + https://github.com/aboutcode-org/scancode.io/issues/2087 + +- Rename "Grammar" optional step group to "Antlr" in d2d pipeline + https://github.com/aboutcode-org/scancode.io/issues/2059 + +- Fix URL-encode programming language filter values in resource list + https://github.com/aboutcode-org/scancode.io/issues/2079 + +**v36 Breaking Change:** PostgreSQL 17 is now required (previously 13). + +Docker Compose users with existing data: run `./migrate-pg13-to-17.sh` before starting +the stack. +Fresh installations require no action. v36.1.0 (2026-01-22) -------------------- diff --git a/pyproject.toml b/pyproject.toml index 990d52fe67..9437b95a85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "scancodeio" -version = "36.1.0" +version = "37.0.0" description = "Automate software composition analysis pipelines" readme = "README.rst" requires-python = ">=3.12,<3.15" diff --git a/scancodeio/__init__.py b/scancodeio/__init__.py index a2272e7bb6..1093a8a8a6 100644 --- a/scancodeio/__init__.py +++ b/scancodeio/__init__.py @@ -28,7 +28,7 @@ import git -VERSION = "36.1.0" +VERSION = "37.0.0" PROJECT_DIR = Path(__file__).resolve().parent ROOT_DIR = PROJECT_DIR.parent From a5911c736f28881e21f6173043ff6fdd06f796b2 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 11 Mar 2026 18:59:30 +1300 Subject: [PATCH 07/73] fix: add the checkout step to pypi release workflow Signed-off-by: tdruez --- .github/workflows/publish-pypi-release.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-pypi-release.yml b/.github/workflows/publish-pypi-release.yml index 2e5163a147..fb5b0f2b6f 100644 --- a/.github/workflows/publish-pypi-release.yml +++ b/.github/workflows/publish-pypi-release.yml @@ -1,4 +1,4 @@ -name: Build Python distributions, publish on PyPI, and create a GH release +name: Build Python distributions, publish on PyPI, and create a GitHub release on: workflow_dispatch: @@ -74,6 +74,11 @@ jobs: contents: write steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Download package distributions uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: From 1574a50ce6b05ec87d687425a8a385d724a190c9 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 29 Jul 2026 14:31:05 +0800 Subject: [PATCH 08/73] Reorder the command #1767 Signed-off-by: Chin Yeung Li --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9437b95a85..ace6ae7649 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,8 +155,8 @@ publish_to_federatedcode = "scanpipe.pipelines.publish_to_federatedcode:PublishT resolve_dependencies = "scanpipe.pipelines.resolve_dependencies:ResolveDependencies" scan_codebase = "scanpipe.pipelines.scan_codebase:ScanCodebase" scan_for_virus = "scanpipe.pipelines.scan_for_virus:ScanForVirus" -scan_single_package = "scanpipe.pipelines.scan_single_package:ScanSinglePackage" scan_rust_package = "scanpipe.pipelines.scan_rust_package:ScanRustPackage" +scan_single_package = "scanpipe.pipelines.scan_single_package:ScanSinglePackage" [tool.setuptools.packages.find] where = ["."] From d7c58e78dc2f100dbd16f47a74b6db5aff209d3a Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 29 Jul 2026 14:31:42 +0800 Subject: [PATCH 09/73] Set a default User-Agent #1767 Signed-off-by: Chin Yeung Li --- scanpipe/pipes/fetch.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scanpipe/pipes/fetch.py b/scanpipe/pipes/fetch.py index f1b249fec9..d05c7dbd06 100644 --- a/scanpipe/pipes/fetch.py +++ b/scanpipe/pipes/fetch.py @@ -65,6 +65,13 @@ def get_request_session(uri): """Return a Requests session setup with authentication and headers.""" session = requests.Session() + + # Set a default User-Agent to avoid 403 Forbidden errors on strict + # registries like crates.io that block default python-requests headers. + session.headers.update({ + "User-Agent": "ScanCode.io (https://github.com/aboutcode-org/scancode.io)" + }) + netloc = urlparse(uri).netloc if credentials := settings.SCANCODEIO_FETCH_BASIC_AUTH.get(netloc): From ba9326f43d03821e33261f76c1c6ddd2cc2247d9 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Thu, 30 Jul 2026 18:07:12 +0800 Subject: [PATCH 10/73] Evolve Rust pipeline (#1767) * Use Docker for building instead of Cargo * Accept only one PURL as input * Use the .rlib found in .d files for D2D mapping * Remove unnecessary code Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/scan_rust_package.py | 66 ++++++---- scanpipe/pipes/d2d.py | 44 +++++-- scanpipe/pipes/rust.py | 112 ++++++++++++++-- scanpipe/pipes/utils.py | 162 +----------------------- 4 files changed, 177 insertions(+), 207 deletions(-) diff --git a/scanpipe/pipelines/scan_rust_package.py b/scanpipe/pipelines/scan_rust_package.py index 324dea16ba..755350b165 100644 --- a/scanpipe/pipelines/scan_rust_package.py +++ b/scanpipe/pipelines/scan_rust_package.py @@ -30,7 +30,9 @@ from scanpipe.pipes import rust from scanpipe.pipes import utils -# from scanpipe.pipes.maven import update_package_license_from_resource_if_missing +from scanpipe.pipes.rust import check_input_and_return_purl, fetch_inputs + +import shutil class ScanRustPackage(ScanSinglePackage, DeployToDevelop, ScanCodebase): @@ -45,17 +47,19 @@ class ScanRustPackage(ScanSinglePackage, DeployToDevelop, ScanCodebase): the license declared in Cargo.toml. Compare the crate’s source code against all other crates (MatchCode), - excluding itself, to detect any borrowed code from third‑party crates. + excluding itself, to detect any borrowed code from third-party crates. """ + download_inputs = False + @classmethod def steps(cls): return ( - cls.get_input, - cls.get_package_input, - cls.collect_input_information, - cls.extract_inputs_to_codebase_directory, - cls.extract_archives, + cls.check_input_and_return_purl, + cls.fetch_inputs, + cls.collect_input_info, + cls.extract_input_to_codebase_directory, + cls.check_docker_command, cls.build_crates, cls.run_scan, cls.load_inventory_from_toolkit_scan, @@ -66,30 +70,40 @@ def steps(cls): cls.make_summary_from_scan_results, ) - def get_input(self): - """Get the input file for the Rust package scan pipeline.""" - from_files = list(self.project.inputs("from*")) - from_files.extend([input.path for input in self.project.inputsources.all()]) - self.from_files = from_files - self.to_files = list() + def check_input_and_return_purl(self): + """Validate the input is a PURL string and return the PURL object.""" + self.purl = check_input_and_return_purl(self.project) + + def fetch_inputs(self): + """Fetch the source of the given PURL.""" + self.from_files = fetch_inputs(self.purl) + + def collect_input_info(self): + """Collect information about the input.""" + self.input_path = self.from_files + self.collect_input_information() + + def check_docker_command(self): + self.have_docker = False + if shutil.which("docker"): + self.have_docker = True def build_crates(self): """ - Build the Rust crate using Cargo and put the built files under the + Build the Rust crate using Docker and put the built files under the "to" directory. """ - # Find the Cargo.toml file in the codebase directory - codebase_dir = Path(self.project.codebase_path) - cargo_toml_path = None - for path in codebase_dir.rglob("Cargo.toml"): - cargo_toml_path = path - break - if cargo_toml_path: - rust.build_crates(cargo_toml_path, self.project.codebase_path / "to/") + self.d2d_enable = False + if self.have_docker: + if rust.build_crates(self.project.codebase_path): + self.d2d_enable = True + else: + print("Docker command not found. Skipping crate build.") def add_from_to_tag(self): """Update 'from' or 'to' tag to resources based on their path.""" - d2d.update_from_to_tag(self.project) + if self.d2d_enable: + d2d.update_from_to_tag(self.project) def validate_package_license_integrity(self): """ @@ -100,8 +114,10 @@ def validate_package_license_integrity(self): def identify_built_sources(self): """Identify the built sources from the '.d' file in the "to" directory.""" - d2d.map_rust_paths(self.project) + if self.d2d_enable: + d2d.map_rust_paths(self.project) def flag_mapped_status(self): """Flag the from codebase resources that were mapped.""" - flag.flag_mapped_resources(self.project) + if self.d2d_enable: + flag.flag_mapped_resources(self.project) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index df29434f9b..792c7af48e 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -1948,10 +1948,10 @@ def map_go_paths(project, logger=None): def get_rust_file_paths(location): """Retrieve Rust file paths.""" file_paths = {} - rust_file_paths = parse_d_file(location) or [] + rust_lib_path, rust_file_paths = parse_d_file(location) or [] if rust_file_paths: file_paths["rust_file_paths"] = rust_file_paths - return file_paths + return rust_lib_path, file_paths def parse_d_file(path): @@ -1963,7 +1963,8 @@ def parse_d_file(path): if ":" not in cleaned_context: return [] - _, dep_paths = cleaned_context.split(":", 1) + rust_lib, dep_paths = cleaned_context.split(":", 1) + rust_lib_path = rust_lib.strip() file_paths = [] for file_path in dep_paths.split(): @@ -1971,23 +1972,48 @@ def parse_d_file(path): if file_path: file_paths.append(file_path) - return file_paths + return rust_lib_path, file_paths def map_rust_paths(project, logger=None): """Map the path listed in the .d file to the source in ``project``.""" from_resources = project.codebaseresources.files().from_codebase() - to_resources = ( + # Fetch the .d files to extract data from + data_resources = ( project.codebaseresources.files() .to_codebase() .exclude(path__contains="/deps/") .exclude(path__contains="/build/") .filter(path__endswith=".d") ) - for resource in to_resources: + target_rlib_ids = [] + for resource in data_resources: try: - paths = get_rust_file_paths(resource.location_path) - resource.update_extra_data(paths) + rlib_path, paths = get_rust_file_paths(resource.location_path) + absolute_rlib_path = Path(rlib_path) + rlib_resource = None + try: + # Docker paths start with "/codebase". Host paths start with project.codebase_path. + if str(absolute_rlib_path).startswith("/codebase/"): + clean_rlib_path = str(absolute_rlib_path.relative_to("/codebase")) + else: + clean_rlib_path = str(absolute_rlib_path.relative_to(project.codebase_path)) + + # We can now safely do an exact path match + rlib_resource = ( + project.codebaseresources.files() + .to_codebase() + .filter(path=clean_rlib_path) + .first() + ) + except ValueError: + pass + + if rlib_resource: + rlib_resource.update_extra_data(paths) + target_rlib_ids.append(rlib_resource.id) + elif logger: + logger(f"Warning: Could not find rlib file {absolute_rlib_path.name} in database.") except Exception as exception: project.add_warning( exception=exception, @@ -1997,6 +2023,8 @@ def map_rust_paths(project, logger=None): details={"path": resource.path}, ) + to_resources = project.codebaseresources.filter(id__in=target_rlib_ids) + if logger: logger( f"Mapping {to_resources.count():,d} to/ resources using paths " diff --git a/scanpipe/pipes/rust.py b/scanpipe/pipes/rust.py index de828e4387..82cae66fec 100644 --- a/scanpipe/pipes/rust.py +++ b/scanpipe/pipes/rust.py @@ -20,31 +20,117 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import shutil import subprocess +import logging +import requests +from packageurl import PackageURL +from pathlib import Path +from scanpipe.pipes import fetch from scanpipe.pipes import run_command_safely -def build_crates(cargo_toml_path, build_dir): +logger = logging.getLogger(__name__) + + +def build_crates(codebase_dir): """ - Build the Rust crate using Cargo to ensure that the source code compiles correctly. + Build the Rust crate from sources in an isolated Docker container. + + Uses the official rust image to safely sandbox the build process and + injects RUSTFLAGS to force DWARF debug symbol generation (-C debuginfo=2) + required for binary-to-source mapping. - This step is crucial for validating the integrity of the source code and ensuring - that it can be successfully built. It also helps to identify any discrepancies - between the source code and the compiled binary, which can be further analyzed - in subsequent steps of the pipeline. + Return True if build successfully, False otherwise. """ + + # Find the Cargo.toml file in the codebase directory + codebase_dir = Path(codebase_dir) + cargo_toml_path = None + for path in codebase_dir.rglob("Cargo.toml"): + cargo_toml_path = path + break + if cargo_toml_path: + to_dir = codebase_dir / "to" + else: + return False + + cargo_toml_path = Path(cargo_toml_path) + build_dir = Path(to_dir) + + # Calculate paths relative to the container's mounted /codebase directory + rel_cargo_toml = cargo_toml_path.relative_to(codebase_dir).as_posix() + rel_build_dir = build_dir.relative_to(codebase_dir).as_posix() + + container_cargo_toml = f"/codebase/{rel_cargo_toml}" + container_build_dir = f"/codebase/{rel_build_dir}" + cmd = [ - "cargo", - "build", + "docker", "run", + "--rm", # Automatically remove the container when it exits + "--volume", f"{codebase_dir}:/codebase", + "--workdir", "/codebase", + "--env", "RUSTFLAGS=-C debuginfo=2", # Force DWARF generation in release mode + "rust:latest", + "cargo", "build", + "--release", "--locked", - "--manifest-path", - str(cargo_toml_path), - "--target-dir", - str(build_dir), + "--manifest-path", container_cargo_toml, + "--target-dir", container_build_dir, ] try: run_command_safely(cmd) except subprocess.SubprocessError as error: - raise RuntimeError(f"Failed to build the Rust crate: {error}") + logger.warning(f"Failed to build the Rust crate in Docker: {error}") + return False + + from_dir = codebase_dir / "from" + from_dir.mkdir(exist_ok=True) + for item in codebase_dir.iterdir(): + if item != to_dir and item != from_dir: + shutil.move(str(item), str(from_dir / item.name)) + return True + + +def check_input_and_return_purl(project): + """Validate the input and return a cargo PURL.""" + input_sources = project.inputsources.all() + if len(input_sources) != 1: + error_msg = "Only 1 cargo purl is accepted." + raise ValueError(error_msg) + # Strip the qualifiers as this is not needed. + project_input = str(input_sources[0]).split("?")[0] + input_purl = PackageURL.from_string(project_input) + + if input_purl.type != "cargo": + error_msg = "Only cargo purl is supported." + raise ValueError(error_msg) + if not input_purl.version: + error_msg = "Version is required." + raise ValueError(error_msg) + + return input_purl + + +def fetch_inputs(purl): + """Fetch the source for the given input purl""" + purl_str = PackageURL.to_string(purl) + + purl_src_path = fetch_path(purl_str) + + if not purl_src_path: + err_msg = f"No source could be resolved for {purl}." + raise ValueError(err_msg) + + return purl_src_path + + +def fetch_path(purl): + """Fetch the purl and return the location of the fetched tarball""" + try: + return fetch.fetch_url(url=purl).path + except (ValueError, requests.RequestException) as e: + logger.warning("Failed to fetch package: %s - %s", purl, e) + return None diff --git a/scanpipe/pipes/utils.py b/scanpipe/pipes/utils.py index 092d858038..d1f050a400 100644 --- a/scanpipe/pipes/utils.py +++ b/scanpipe/pipes/utils.py @@ -54,10 +54,8 @@ def validate_package_license_integrity(project): if detected_lic_list: lic_exp = " AND ".join(detected_lic_list) - # The dedup have bug and need to be fixed: - # https://github.com/aboutcode-org/license-expression/issues/130 detected_lic_exp = str(Licensing().dedup(lic_exp)) - # The package license is not in sync with detected license(s) + if detected_lic_exp != package_lic: package.update_extra_data( { @@ -111,161 +109,3 @@ def collect_detected_licenses(resources, ignore_patterns, package_uid=None): lic = f'({lic})' detected_lic_list.append(lic) return detected_lic_list - - -# TODO: This may not be necessary since our focus is on license mismatches at the package level. -# If a file contains an extra license not declared in the package, we flag it at the package level rather than the file level. -# Therefore, the 'missing' and 'extra' checks may be excessive. -def evaluate_license_mismatch(package_lic, detected_lic_list): - """Check if there is a license mismatch between declared package_license and detected licenses. - - Returns: - A dictionary with mismatch information - """ - licensing = Licensing() - - # Parse expressions - package_expr = licensing.parse(package_lic) - detected_exprs = [licensing.parse(lic) for lic in detected_lic_list] - - # Combine detected licenses with AND - detected_combined = (detected_exprs[0] if len(detected_exprs) == 1 - else licensing.AND(*detected_exprs)) - - # Find what's missing from detected - missing = find_missing(package_expr, detected_combined) - - # Find extra licenses (with special handling for OR expressions) - extra = find_extra(detected_exprs, package_expr) - - return { - 'missing': sorted(missing), - 'extra': sorted(extra), - 'is_match': not missing and not extra, - 'details': format_details(missing, extra) - } - - -def find_missing(expr, detected): - """Find licenses required by package but missing from detected.""" - # Base case: single license - if is_license_symbol(expr): - return [expr.key] if not contains_license(detected, expr.key) else [] - - # WITH exception - if is_with_exception(expr): - missing = [] - if not contains_license(detected, expr.license_symbol.key): - missing.append(expr.license_symbol.key) - if not contains_license(detected, expr.exception_symbol.key): - missing.append(expr.exception_symbol.key) - return missing - - # AND/OR expressions - if hasattr(expr, 'args'): - if is_and(expr): - # AND requires all branches - return sum((find_missing(arg, detected) for arg in expr.args), []) - - if is_or(expr): - # OR requires at least one branch - # First, check if any branch is fully satisfied - for arg in expr.args: - if not find_missing(arg, detected): - return [] # Found a fully satisfied branch - - # If no branch is fully satisfied, collect ALL licenses from ALL branches - # to show everything that's missing from the OR - all_missing = [] - for arg in expr.args: - branch_missing = find_missing(arg, detected) - all_missing.extend(branch_missing) - return list(set(all_missing)) # Remove duplicates - - return [] - - -def find_extra(detected_exprs, package_expr): - """Find licenses in detected that aren't required by package.""" - extra = [] - - for detected in detected_exprs: - detected_keys = get_all_keys(detected) - - # If it's an OR and any branch is required, none of its licenses are extra - if is_or(detected) and any(is_license_required(key, package_expr) for key in detected_keys): - continue - - # Otherwise, add all keys not required by package - for key in detected_keys: - if not is_license_required(key, package_expr) and key not in extra: - extra.append(key) - - return extra - - -def is_license_required(license_key, expr): - """Check if a license key is required by the package expression.""" - if is_license_symbol(expr): - return expr.key == license_key - if is_with_exception(expr): - return (expr.license_symbol.key == license_key or - expr.exception_symbol.key == license_key) - if hasattr(expr, 'args'): - if is_and(expr): - # In AND, a license is required if it appears in any branch - return any(is_license_required(license_key, arg) for arg in expr.args) - if is_or(expr): - # In OR, a license is required if it appears in the expression - return any(is_license_required(license_key, arg) for arg in expr.args) - return False - - -def get_all_keys(expr): - """Extract all license keys from an expression.""" - if is_license_symbol(expr): - return [expr.key] - if is_with_exception(expr): - return [expr.license_symbol.key, expr.exception_symbol.key] - if hasattr(expr, 'args'): - keys = [] - for arg in expr.args: - keys.extend(get_all_keys(arg)) - return keys - return [] - - -def contains_license(expr, license_key): - """Check if an expression contains a specific license key.""" - if is_license_symbol(expr): - return expr.key == license_key - if is_with_exception(expr): - return (expr.license_symbol.key == license_key or - expr.exception_symbol.key == license_key) - if hasattr(expr, 'args'): - return any(contains_license(arg, license_key) for arg in expr.args) - return False - - -def format_details(missing, extra): - """Format the details string.""" - parts = [] - if missing: - parts.append(f"Missing: {', '.join(sorted(missing))}") - if extra: - parts.append(f"Extra: {', '.join(sorted(extra))}") - return '; '.join(parts) if parts else 'match' - - -# Helper functions for type checking -def is_license_symbol(expr): - return hasattr(expr, 'key') and not hasattr(expr, 'license_symbol') - -def is_with_exception(expr): - return hasattr(expr, 'license_symbol') and hasattr(expr, 'exception_symbol') - -def is_and(expr): - return hasattr(expr, 'operator') and 'AND' in expr.operator.strip() - -def is_or(expr): - return hasattr(expr, 'operator') and 'OR' in expr.operator.strip() From e10bd2ae2d484f0796a0a906537fc9a7d7b6f888 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 5 Aug 2026 17:14:09 +0800 Subject: [PATCH 11/73] Implement the rust pipeline #1767 * Add comparison logic * Add tests * Update extra_data fields * Better code organization * etc.. Signed-off-by: Chin Yeung Li --- scanpipe/pipelines/scan_rust_package.py | 105 +++- scanpipe/pipes/d2d.py | 16 +- scanpipe/pipes/fetch.py | 6 +- scanpipe/pipes/rust.py | 105 ++-- scanpipe/pipes/utils.py | 439 +++++++++++++++- scanpipe/templates/scanpipe/package_list.html | 4 +- scanpipe/tests/pipes/test_rust.py | 107 ++++ scanpipe/tests/pipes/test_utils.py | 481 +++++++++++------- 8 files changed, 976 insertions(+), 287 deletions(-) create mode 100644 scanpipe/tests/pipes/test_rust.py diff --git a/scanpipe/pipelines/scan_rust_package.py b/scanpipe/pipelines/scan_rust_package.py index 755350b165..d48bf6b63e 100644 --- a/scanpipe/pipelines/scan_rust_package.py +++ b/scanpipe/pipelines/scan_rust_package.py @@ -20,6 +20,8 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import shutil +import tempfile from pathlib import Path from scanpipe.pipelines.deploy_to_develop import DeployToDevelop @@ -27,12 +29,11 @@ from scanpipe.pipelines.scan_single_package import ScanSinglePackage from scanpipe.pipes import d2d from scanpipe.pipes import flag -from scanpipe.pipes import rust from scanpipe.pipes import utils - -from scanpipe.pipes.rust import check_input_and_return_purl, fetch_inputs - -import shutil +from scanpipe.pipes.rust import build_crates +from scanpipe.pipes.rust import check_input_and_return_purl +from scanpipe.pipes.rust import get_cargo_toml_path +from scanpipe.pipes.rust import get_repository_value_from_cargo_toml class ScanRustPackage(ScanSinglePackage, DeployToDevelop, ScanCodebase): @@ -60,6 +61,7 @@ def steps(cls): cls.collect_input_info, cls.extract_input_to_codebase_directory, cls.check_docker_command, + cls.get_cargo_toml, cls.build_crates, cls.run_scan, cls.load_inventory_from_toolkit_scan, @@ -67,6 +69,10 @@ def steps(cls): cls.validate_package_license_integrity, cls.identify_built_sources, cls.flag_mapped_status, + cls.get_src_repo_download_url, + cls.download_src_repo, + cls.compare_src_repo_with_from_codebase, + cls.update_comparison_summary, cls.make_summary_from_scan_results, ) @@ -76,7 +82,7 @@ def check_input_and_return_purl(self): def fetch_inputs(self): """Fetch the source of the given PURL.""" - self.from_files = fetch_inputs(self.purl) + self.from_files = utils.fetch_inputs(self.purl) def collect_input_info(self): """Collect information about the input.""" @@ -84,24 +90,41 @@ def collect_input_info(self): self.collect_input_information() def check_docker_command(self): + """Check if the Docker command is available.""" self.have_docker = False if shutil.which("docker"): self.have_docker = True + def get_cargo_toml(self): + """Get the Cargo.toml path from the codebase directory.""" + self.cargo_toml_path = None + self.devel_codebase_dir = None + if self.have_docker: + codebase_dir = Path(self.project.codebase_path) + self.devel_codebase_dir = codebase_dir + self.cargo_toml_path = get_cargo_toml_path(codebase_dir) + def build_crates(self): """ Build the Rust crate using Docker and put the built files under the "to" directory. """ self.d2d_enable = False - if self.have_docker: - if rust.build_crates(self.project.codebase_path): + if self.cargo_toml_path: + codebase_dir = self.devel_codebase_dir + cargo_toml_path = self.cargo_toml_path + if build_crates(codebase_dir, cargo_toml_path): self.d2d_enable = True + updated_path = cargo_toml_path.relative_to(codebase_dir) + self.cargo_toml_path = codebase_dir / "from" / updated_path + self.devel_codebase_dir = codebase_dir / "from" else: print("Docker command not found. Skipping crate build.") + else: + print("Cargo.toml is not found.") def add_from_to_tag(self): - """Update 'from' or 'to' tag to resources based on their path.""" + """Update 'from' and 'to' tag to resources based on their path.""" if self.d2d_enable: d2d.update_from_to_tag(self.project) @@ -121,3 +144,67 @@ def flag_mapped_status(self): """Flag the from codebase resources that were mapped.""" if self.d2d_enable: flag.flag_mapped_resources(self.project) + + def get_src_repo_download_url(self): + """ + Get the source repository url from Cargo.toml and determine its + download url. + """ + self.src_download_url = None + repository_url = get_repository_value_from_cargo_toml(self.cargo_toml_path) + if not repository_url: + self.project.add_warning( + description="No source repository URL found in Cargo.toml." + ) + else: + self.src_download_url = utils.get_download_url( + repository_url, self.purl.version + ) + if not self.src_download_url: + self.project.add_warning( + description=( + "Not able to determine the source repository download URL from " + "Cargo.toml." + ) + ) + + def download_src_repo(self): + """Download the source from the source repo.""" + self.src_repo_path = None + if self.src_download_url: + self.src_repo_path = utils.download_src_repo(self.src_download_url) + if not self.src_repo_path: + self.project.add_warning( + description=( + f"The source repository URL " + f"{self.src_download_url} " + f"could not be downloaded. Skipping the source " + f"crate and source repository comparison." + ) + ) + + def compare_src_repo_with_from_codebase(self): + """Compare the downloaded source repo with the from codebase.""" + self.matched_count = 0 + self.mismatches = [] + if self.src_repo_path: + with tempfile.TemporaryDirectory() as source_repo_path: + self.extract_archive(self.src_repo_path, source_repo_path) + + self.matched_count, self.mismatches = utils.compare_directories( + self.devel_codebase_dir, source_repo_path + ) + + def update_comparison_summary(self): + """Update the comparison summary in the discovered package.""" + if self.src_repo_path: + utils.update_comparison_summary( + self.project, + self.purl, + self.devel_codebase_dir, + self.src_download_url, + self.purl.name, + self.purl.version, + self.matched_count, + self.mismatches, + ) diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index 792c7af48e..d9f7ca29a7 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -1989,17 +1989,15 @@ def map_rust_paths(project, logger=None): target_rlib_ids = [] for resource in data_resources: try: - rlib_path, paths = get_rust_file_paths(resource.location_path) - absolute_rlib_path = Path(rlib_path) + rlib_path_str, paths = get_rust_file_paths(resource.location_path) + rlib_path = Path(rlib_path_str) rlib_resource = None try: - # Docker paths start with "/codebase". Host paths start with project.codebase_path. - if str(absolute_rlib_path).startswith("/codebase/"): - clean_rlib_path = str(absolute_rlib_path.relative_to("/codebase")) + if rlib_path_str.startswith("/codebase/"): + clean_rlib_path = str(rlib_path.relative_to("/codebase")) else: - clean_rlib_path = str(absolute_rlib_path.relative_to(project.codebase_path)) + clean_rlib_path = str(rlib_path.relative_to(project.codebase_path)) - # We can now safely do an exact path match rlib_resource = ( project.codebaseresources.files() .to_codebase() @@ -2013,7 +2011,9 @@ def map_rust_paths(project, logger=None): rlib_resource.update_extra_data(paths) target_rlib_ids.append(rlib_resource.id) elif logger: - logger(f"Warning: Could not find rlib file {absolute_rlib_path.name} in database.") + logger( + f"Warning: Could not find rlib file {rlib_path_str} in database." + ) except Exception as exception: project.add_warning( exception=exception, diff --git a/scanpipe/pipes/fetch.py b/scanpipe/pipes/fetch.py index d05c7dbd06..9a25262fcb 100644 --- a/scanpipe/pipes/fetch.py +++ b/scanpipe/pipes/fetch.py @@ -68,9 +68,9 @@ def get_request_session(uri): # Set a default User-Agent to avoid 403 Forbidden errors on strict # registries like crates.io that block default python-requests headers. - session.headers.update({ - "User-Agent": "ScanCode.io (https://github.com/aboutcode-org/scancode.io)" - }) + session.headers.update( + {"User-Agent": "ScanCode.io (https://github.com/aboutcode-org/scancode.io)"} + ) netloc = urlparse(uri).netloc diff --git a/scanpipe/pipes/rust.py b/scanpipe/pipes/rust.py index 82cae66fec..c50daee37e 100644 --- a/scanpipe/pipes/rust.py +++ b/scanpipe/pipes/rust.py @@ -20,64 +20,58 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import logging import shutil import subprocess -import logging -import requests +from pathlib import Path +import tomllib from packageurl import PackageURL -from pathlib import Path -from scanpipe.pipes import fetch -from scanpipe.pipes import run_command_safely +from scanpipe.pipes import run_command_safely logger = logging.getLogger(__name__) -def build_crates(codebase_dir): +def build_crates(codebase_dir, cargo_toml_path): """ - Build the Rust crate from sources in an isolated Docker container. - - Uses the official rust image to safely sandbox the build process and - injects RUSTFLAGS to force DWARF debug symbol generation (-C debuginfo=2) - required for binary-to-source mapping. - - Return True if build successfully, False otherwise. + Build the Rust crate from source in an isolated Docker container. + Use the official Rust image for the build process. + Return True if the build succeeds, False otherwise. """ - - # Find the Cargo.toml file in the codebase directory - codebase_dir = Path(codebase_dir) - cargo_toml_path = None - for path in codebase_dir.rglob("Cargo.toml"): - cargo_toml_path = path - break - if cargo_toml_path: - to_dir = codebase_dir / "to" - else: - return False - + to_dir = codebase_dir / "to" cargo_toml_path = Path(cargo_toml_path) build_dir = Path(to_dir) - # Calculate paths relative to the container's mounted /codebase directory - rel_cargo_toml = cargo_toml_path.relative_to(codebase_dir).as_posix() - rel_build_dir = build_dir.relative_to(codebase_dir).as_posix() + # Get the relative paths + relative_cargo_toml = cargo_toml_path.relative_to(codebase_dir).as_posix() + relative_build_dir = build_dir.relative_to(codebase_dir).as_posix() - container_cargo_toml = f"/codebase/{rel_cargo_toml}" - container_build_dir = f"/codebase/{rel_build_dir}" + container_cargo_toml = f"/codebase/{relative_cargo_toml}" + container_build_dir = f"/codebase/{relative_build_dir}" + # Since we will use the .d file for deployment and development file + # mapping, we will not require building with DWARF debug symbols. If we + # later decide to include DWARF, we can add the following to the + # command: + # "--env", "RUSTFLAGS=-C debuginfo=2", cmd = [ - "docker", "run", - "--rm", # Automatically remove the container when it exits - "--volume", f"{codebase_dir}:/codebase", - "--workdir", "/codebase", - "--env", "RUSTFLAGS=-C debuginfo=2", # Force DWARF generation in release mode + "docker", + "run", + "--rm", + "--volume", + f"{codebase_dir}:/codebase", + "--workdir", + "/codebase", "rust:latest", - "cargo", "build", + "cargo", + "build", "--release", "--locked", - "--manifest-path", container_cargo_toml, - "--target-dir", container_build_dir, + "--manifest-path", + container_cargo_toml, + "--target-dir", + container_build_dir, ] try: @@ -86,6 +80,7 @@ def build_crates(codebase_dir): logger.warning(f"Failed to build the Rust crate in Docker: {error}") return False + # Move the development code under the /codebase/from/ from_dir = codebase_dir / "from" from_dir.mkdir(exist_ok=True) for item in codebase_dir.iterdir(): @@ -100,7 +95,7 @@ def check_input_and_return_purl(project): if len(input_sources) != 1: error_msg = "Only 1 cargo purl is accepted." raise ValueError(error_msg) - # Strip the qualifiers as this is not needed. + # Strip the qualifiers if present as this is not needed project_input = str(input_sources[0]).split("?")[0] input_purl = PackageURL.from_string(project_input) @@ -114,23 +109,23 @@ def check_input_and_return_purl(project): return input_purl -def fetch_inputs(purl): - """Fetch the source for the given input purl""" - purl_str = PackageURL.to_string(purl) - - purl_src_path = fetch_path(purl_str) +def get_repository_value_from_cargo_toml(cargo_toml_path): + """Get the repository value from Cargo.toml.""" + path = Path(cargo_toml_path) + if not path.exists(): + raise FileNotFoundError(f"{cargo_toml_path} not found") - if not purl_src_path: - err_msg = f"No source could be resolved for {purl}." - raise ValueError(err_msg) + with path.open("rb") as f: + data = tomllib.load(f) - return purl_src_path + return data.get("package", {}).get("repository", "") -def fetch_path(purl): - """Fetch the purl and return the location of the fetched tarball""" - try: - return fetch.fetch_url(url=purl).path - except (ValueError, requests.RequestException) as e: - logger.warning("Failed to fetch package: %s - %s", purl, e) - return None +def get_cargo_toml_path(codebase_dir): + """Get the Cargo.toml path from the codebase directory.""" + cargo_toml_path = None + # There is only one "Cargo.toml" per published package + for path in codebase_dir.rglob("Cargo.toml"): + cargo_toml_path = path + break + return cargo_toml_path diff --git a/scanpipe/pipes/utils.py b/scanpipe/pipes/utils.py index d1f050a400..be37c79207 100644 --- a/scanpipe/pipes/utils.py +++ b/scanpipe/pipes/utils.py @@ -20,10 +20,23 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import hashlib +import logging +import os +from fnmatch import fnmatch +from pathlib import Path +from urllib.parse import urlparse + +import requests from license_expression import Licensing +from packageurl import PackageURL +from packageurl.contrib.purl2url import get_repo_download_url_by_package_type +from scanpipe.pipes import fetch from scanpipe.pipes import flag +logger = logging.getLogger(__name__) + def validate_package_license_integrity(project): """Validate the correctness of the package license.""" @@ -57,42 +70,101 @@ def validate_package_license_integrity(project): detected_lic_exp = str(Licensing().dedup(lic_exp)) if detected_lic_exp != package_lic: - package.update_extra_data( + package_issues = package.extra_data.get("issues", []) + + package_issues.append( { - "issue": "License Mismatch", + "issue_type": "License Mismatch", "declared_license": package_lic, "detected_codebase_license": detected_lic_exp, } ) + + package.update_extra_data({"issues": package_issues}) + for datafile_path in package.datafile_paths: if not datafile_path.startswith("https://"): data_path = project.codebaseresources.get( path=datafile_path ) data_path.update(status=flag.LICENSE_ISSUE) - data_path.update_extra_data( + + resource_issues = data_path.extra_data.get("issues", []) + resource_issues.append( { + "issue_type": "License Mismatch", "declared_license": package_lic, "detected_codebase_license": detected_lic_exp, } ) + data_path.update_extra_data({"issues": resource_issues}) + def contains_ignore_pattern(resource_path, ignore_patterns): """Check if the resource path matches any of the ignore patterns.""" - from fnmatch import fnmatch - for pattern in ignore_patterns: if fnmatch(resource_path, pattern): return True return False +def filter_ignored_licenses(license_expression, licensing): + """Filter out ignored licenses from a license expression.""" + # Some licenses are not useful for validating package license + # integrity, so we ignore them. + ignored_licenses = [ + "free-unknown", + "unknown", + "unknown-license-reference", + "unknown-spdx", + ] + + if license_expression is None: + return None + + if isinstance(license_expression, licensing.Symbol): + if ( + hasattr(license_expression, "key") + and license_expression.key in ignored_licenses + ): + return None + return license_expression + + # Handle AND operations + if isinstance(license_expression, licensing.AND): + return handle_operator_expression(license_expression, licensing, licensing.AND) + + # Handle OR operations + if isinstance(license_expression, licensing.OR): + return handle_operator_expression(license_expression, licensing, licensing.OR) + + return license_expression + + +def handle_operator_expression(expression, licensing, operator): + """ + Process AND/OR operations in a license expression, filtering out + ignored licenses. + """ + args = [] + for arg in expression.args: + filtered_arg = filter_ignored_licenses(arg, licensing) + if filtered_arg is not None: + args.append(filtered_arg) + if not args: + return None + if len(args) == 1: + return args[0] + + return operator(*args) + + def collect_detected_licenses(resources, ignore_patterns, package_uid=None): - """Collect detected licenses from resources, ignoring specified patterns.""" + """Collect detected licenses from resources, ignoring defined patterns.""" + licensing = Licensing() detected_lic_list = [] - # Some licenses are not useful for validating package license integrity, so we ignore them. - ignored_licenses = ['free-unknown', 'unknown', 'unknown-license-reference', 'unknown-spdx'] + for resource in resources: if contains_ignore_pattern(resource.path, ignore_patterns): continue @@ -101,11 +173,348 @@ def collect_detected_licenses(resources, ignore_patterns, package_uid=None): if package_uid and package_uid not in resource.for_packages: continue - lic = resource.detected_license_expression - if lic and lic not in ignored_licenses and lic not in detected_lic_list: - # Make sure there is parentheses if the license has an 'OR' operator - # TODO: We need to parse the lic and check for ignored licenses if lic is an expression - if ' OR ' in lic and not (lic.startswith('(') and lic.endswith(')')): - lic = f'({lic})' - detected_lic_list.append(lic) + license_str = resource.detected_license_expression + if not license_str: + continue + try: + parsed_lic = licensing.parse(license_str) + + # Filter out the ignored keys + filtered_license = filter_ignored_licenses(parsed_lic, licensing) + + if filtered_license is not None: + final_lic = str(filtered_license) + + if final_lic not in detected_lic_list: + # Apply parentheses so that the 'OR' expression will + # not be filtered out when doing deduplication later. + detected_lic_list.append(f"({final_lic})") + + except Exception: + logger.warning( + "Failed to parse the license expression: %s at %s", + license_str, + resource.path, + ) return detected_lic_list + + +def get_url_netloc_namespace_and_name(url): + """ + Extract netloc, namespace, and name from a URL path. + - The last path component (except for web files) is considered the name. + - Everything between netloc and name is considered the namespace. + """ + parsed = urlparse(url) + netloc = parsed.netloc + parts = parsed.path.strip("/").split("/") + + if not parts or parts == [""]: + return netloc, None, None + + if len(parts) > 1: + last_part = parts[-1].lower() + ignore_extensions = (".html", ".htm", ".php", ".jsp", ".asp", ".aspx") + if last_part.startswith("index.") or last_part.endswith(ignore_extensions): + parts.pop() + + name = parts[-1] + namespace = "/".join(parts[:-1]) if len(parts) > 1 else None + + return netloc, namespace, name + + +def download_src_repo(download_url): + try: + return fetch.fetch_url(url=download_url).path + except (ValueError, requests.RequestException): + logger.warning("Failed to download source repository: %s", download_url) + return None + + +def get_download_url(homepage_url, version): + netloc, namespace, name = get_url_netloc_namespace_and_name(homepage_url) + if netloc.endswith("github.io"): + github_page_url = github_pages_to_repo(homepage_url) + if github_page_url: + netloc, namespace, name = get_url_netloc_namespace_and_name(github_page_url) + + if netloc in ("github.com", "gitlab.com", "bitbucket.org"): + if netloc.endswith(".com"): + package_type = netloc.removesuffix(".com") + # There is an issue where the version may have a different prefix. + # For example, version can have the following prefixes: + # ["v", "V", "release-", "RELEASE-", "v-", "V-"] + clarified_version = clarify_version_tag( + package_type, namespace, name, version + ) + if clarified_version: + version = clarified_version + elif netloc.endswith(".org"): + package_type = netloc.removesuffix(".org") + download_url = get_repo_download_url_by_package_type( + type=package_type, namespace=namespace, name=name, version=version + ) + return download_url + return None + + +def clarify_version_tag(repo_type, namespace, name, version): + """Use github/gitlab API to verify the version tag""" + headers = {} + if repo_type == "github": + github_token = os.environ.get("GITHUB_TOKEN") + if github_token: + headers["Authorization"] = f"token {github_token}" + url_base = f"https://api.github.com/repos/{namespace}/{name}/git/refs/tags/{{}}" + elif repo_type == "gitlab": + gitlab_token = os.environ.get("GITLAB_TOKEN") + if gitlab_token: + headers["PRIVATE-TOKEN"] = gitlab_token + ns = namespace or "" + project_path = f"{ns}/{name}".strip("/").replace("/", "%2F") + url_base = ( + f"https://gitlab.com/api/v4/projects/{project_path}/repository/tags/{{}}" + ) + else: + return None + + potential_prefixes = ["", "v", "V", "release-", "RELEASE-", "v-", "V-"] + for prefix in potential_prefixes: + potential_tag = f"{prefix}{version}" + url = url_base.format(potential_tag) + + try: + response = requests.get(url, headers=headers, timeout=10) + except requests.RequestException: + continue + if response.status_code == 200: + return potential_tag + elif response.status_code in (403, 429): + print( + f"Rate limited by {repo_type} API while checking tag {potential_tag}." + ) + return None + + return None + + +def github_pages_to_repo(url): + """ + Try to map a GitHub Pages URL (https://{org}.github.io/{name}/) + to its corresponding GitHub repository (https://github.com/{org}/{name}). + Returns the repo URL if it exists, otherwise None. + """ + parsed = urlparse(url) + host = parsed.netloc + parts = parsed.path.strip("/").split("/") + + # Only handle {org}.github.io/{name} pattern + if not host.endswith(".github.io") or len(parts) < 1: + return None + + org = host.replace(".github.io", "") + name = parts[0] + + repo = f"https://github.com/{org}/{name}" + + # Verify existence via GitHub API + api_url = f"https://api.github.com/repos/{org}/{name}" + try: + response = requests.get(api_url, timeout=10) + if response.status_code == 200: + return repo + except requests.RequestException: + return None + + return None + + +def compute_sha1(file_path): + """Compute the SHA1 hash of a file.""" + try: + with open(file_path, "rb") as f: + return hashlib.file_digest(f, "sha1").hexdigest() + except OSError: + return None + + +def get_all_files(base_dir): + """ + Walk a directory and returns a dictionary mapping relative paths + to their filename and SHA1 hash. + """ + file_map = {} + for root, _dirs, files in os.walk(base_dir): + for file in files: + full_path = os.path.join(root, file) + hash = compute_sha1(full_path) + rel_path = Path(full_path).relative_to(base_dir).as_posix() + file_map[rel_path] = {"name": file, "hash": hash} + return file_map + + +def consolidate_unmatched(all_files, unmatched_files): + """ + Consolidate the unmatched files. If all files in a directory are + unmatched, report the directory instead of individual files. + Returns a list of tuples: (path, is_directory) + """ + matched_files = set(all_files) - set(unmatched_files) + + # Find every parent directory that contains at least one matched file. + directories_with_matches = set() + for file_path in matched_files: + for parent in Path(file_path).parents: + parent_str = parent.as_posix() + if parent_str != ".": + directories_with_matches.add(parent_str) + + consolidated_results = set() + + # For each unmatched file, check if it belongs to a fully unmatched directory. + for file_path in unmatched_files: + path_obj = Path(file_path) + target_path = file_path + is_directory = False + + # Check from top to bottom + for parent in reversed(path_obj.parents): + current_dir = parent.as_posix() + if current_dir == ".": + continue + + if current_dir not in directories_with_matches: + target_path = current_dir + is_directory = True + break # Stop at the highest possible unmatched directory level + + consolidated_results.add((target_path, is_directory)) + + # Convert the set to a list and sort it + results_list = list(consolidated_results) + results_list.sort() + + return results_list + + +def compare_directories(input_source, source_repo): + """ + Compare two directories and return the count of matched files and a + dictionary of mismatches. + """ + input_files = get_all_files(input_source) + repo_files = get_all_files(source_repo) + + matched_count = 0 + + mismatches = {"mismatches": [], "input_source_only": [], "source_repo_only": []} + + repo_unmatched = {path: data for path, data in repo_files.items()} + input_unmatched = {} + + # Check for exact path and hash match + for input_path, input_data in input_files.items(): + if input_path in repo_files: + if input_data["hash"] == repo_files[input_path]["hash"]: + matched_count += 1 + else: + mismatches["mismatches"].append(f"[File] {input_path}") + del repo_unmatched[input_path] + else: + input_unmatched[input_path] = input_data + + repo_by_hash_name = {} + for path, data in repo_unmatched.items(): + key = (data["hash"], data["name"]) + if key not in repo_by_hash_name: + repo_by_hash_name[key] = [] + repo_by_hash_name[key].append(path) + + still_unmatched_input = {} + + # Check for files with the same hash and name but different paths + for input_path, input_data in input_unmatched.items(): + hash_name_key = (input_data["hash"], input_data["name"]) + + if hash_name_key in repo_by_hash_name and repo_by_hash_name[hash_name_key]: + repo_match_path = repo_by_hash_name[hash_name_key].pop(0) + matched_count += 1 + del repo_unmatched[repo_match_path] + else: + still_unmatched_input[input_path] = input_data + + input_consolidated = consolidate_unmatched( + input_files.keys(), still_unmatched_input.keys() + ) + for path, is_directory in input_consolidated: + item_type = "Directory" if is_directory else "File" + mismatches["input_source_only"].append(f"[{item_type}] {path}") + + repo_consolidated = consolidate_unmatched(repo_files.keys(), repo_unmatched.keys()) + for path, is_directory in repo_consolidated: + item_type = "Directory" if is_directory else "File" + mismatches["source_repo_only"].append(f"[{item_type}] {path}") + + return matched_count, mismatches + + +def count_total_files(directory): + """Recursively counts all files in a given directory.""" + total_files = 0 + for _, _, files in os.walk(directory): + total_files += len(files) + return total_files + + +def update_comparison_summary( + project, + purl, + devel_codebase_dir, + src_repo_url, + package_name, + package_version, + matched_count, + mismatches, +): + total_num_files = count_total_files(devel_codebase_dir) + + package = project.discoveredpackages.filter( + name=package_name, version=package_version + ).first() + + if package: + summary_dict = { + "input_source": str(purl), + "compare_source_repository_url": str(src_repo_url), + "total_matching_files": matched_count, + "total_files_in_source_crate": total_num_files, + "mismatches": mismatches, + } + package.update_extra_data({"comparison_summary": summary_dict}) + else: + project.add_warning( + description=( + f"Could not find a discovered package matching {package_name} " + f"{package_version} to attach the summary." + ) + ) + + +def fetch_inputs(purl): + """Fetch the source for the given input purl""" + purl_str = PackageURL.to_string(purl) + purl_src_path = fetch_path(purl_str) + if not purl_src_path: + err_msg = f"No source could be resolved for {purl}." + raise ValueError(err_msg) + return purl_src_path + + +def fetch_path(purl): + """Fetch the purl and return the location of the fetched tarball""" + try: + return fetch.fetch_url(url=purl).path + except (ValueError, requests.RequestException) as e: + logger.warning("Failed to fetch package: %s - %s", purl, e) + return None diff --git a/scanpipe/templates/scanpipe/package_list.html b/scanpipe/templates/scanpipe/package_list.html index 9e707f1ba6..b07e3a2dbd 100644 --- a/scanpipe/templates/scanpipe/package_list.html +++ b/scanpipe/templates/scanpipe/package_list.html @@ -35,9 +35,9 @@ {% endif %} - {% if package.extra_data.issue == "License Mismatch" %} + {% if package.extra_data.issues %} - + {% endif %} diff --git a/scanpipe/tests/pipes/test_rust.py b/scanpipe/tests/pipes/test_rust.py new file mode 100644 index 0000000000..813383b18d --- /dev/null +++ b/scanpipe/tests/pipes/test_rust.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + + +import tempfile +from pathlib import Path +from unittest import mock + +from django.test import TestCase + +from scanpipe.pipes import rust + + +class ScanPipeRustPipesTest(TestCase): + @mock.patch("pathlib.Path.rglob") + def test_get_cargo_toml_path_found(self, mock_rglob): + mock_cargo_path = Path("/mock/Cargo.toml") + mock_rglob.return_value = [mock_cargo_path] + + found_path = rust.get_cargo_toml_path(Path("/mock")) + self.assertEqual(found_path, mock_cargo_path) + mock_rglob.assert_called_once_with("Cargo.toml") + + @mock.patch("pathlib.Path.rglob") + def test_get_cargo_toml_path_not_found(self, mock_rglob): + mock_rglob.return_value = [] + + found_path = rust.get_cargo_toml_path(Path("/mock")) + self.assertIsNone(found_path) + + @mock.patch("pathlib.Path.exists", return_value=True) + @mock.patch("pathlib.Path.open", new_callable=mock.mock_open) + @mock.patch("scanpipe.pipes.rust.tomllib.load") + def test_get_repository_value_from_cargo_toml_success( + self, mock_tomllib_load, mock_file_open, mock_exists + ): + mock_tomllib_load.return_value = { + "package": {"repository": "https://github.com/owner/repo"} + } + repo_url = rust.get_repository_value_from_cargo_toml("Cargo.toml") + self.assertEqual(repo_url, "https://github.com/owner/repo") + + @mock.patch("pathlib.Path.exists", return_value=False) + def test_get_repository_value_from_cargo_toml_missing(self, mock_exists): + with self.assertRaises(FileNotFoundError): + rust.get_repository_value_from_cargo_toml("/nonexistent/Cargo.toml") + + def test_check_input_and_return_purl_success(self): + mock_project = mock.Mock() + mock_project.inputsources.all.return_value = ["pkg:cargo/test@1.0.0"] + + purl = rust.check_input_and_return_purl(mock_project) + self.assertEqual(purl.type, "cargo") + self.assertEqual(purl.name, "test") + self.assertEqual(purl.version, "1.0.0") + + def test_check_input_and_return_purl_invalid_type(self): + mock_project = mock.Mock() + mock_project.inputsources.all.return_value = ["pkg:pypi/test@1.0.0"] + + with self.assertRaises(ValueError): + rust.check_input_and_return_purl(mock_project) + + def test_check_input_and_return_purl_missing_version(self): + mock_project = mock.Mock() + mock_project.inputsources.all.return_value = ["pkg:cargo/test"] + + with self.assertRaises(ValueError): + rust.check_input_and_return_purl(mock_project) + + @mock.patch("scanpipe.pipes.rust.run_command_safely") + def test_build_crates_success(self, mock_run_cmd): + with tempfile.TemporaryDirectory() as temp_dir: + base_path = Path(temp_dir) + cargo_path = base_path / "Cargo.toml" + cargo_path.touch() + other_file = base_path / "src_file.rs" + other_file.touch() + + # Since run_command_safely is mocked, Cargo won't create the + # 'to' dir automatically + (base_path / "to").mkdir() + + success = rust.build_crates(base_path, cargo_path) + self.assertTrue(success) + self.assertTrue((base_path / "to").exists()) + self.assertTrue((base_path / "from").exists()) + self.assertTrue((base_path / "from" / "src_file.rs").exists()) diff --git a/scanpipe/tests/pipes/test_utils.py b/scanpipe/tests/pipes/test_utils.py index cbd2b4ba00..f679b31c91 100644 --- a/scanpipe/tests/pipes/test_utils.py +++ b/scanpipe/tests/pipes/test_utils.py @@ -20,204 +20,295 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. + +import tempfile +from pathlib import Path +from unittest import mock + from django.test import TestCase +from license_expression import Licensing + +from scanpipe.pipes import flag from scanpipe.pipes import utils + class ScanPipeUtilsTest(TestCase): + def setUp(self): + self.licensing = Licensing() + + @mock.patch("scanpipe.models.CodebaseResource") + @mock.patch("scanpipe.models.DiscoveredPackage") + @mock.patch("scanpipe.models.Project") + def test_validate_package_license_integrity_mismatch( + self, mock_project_class, mock_package_class, mock_resource_class + ): + mock_project = mock_project_class() + mock_package = mock_package_class() + + mock_package.type = "pypi" + mock_package.package_uid = "pkg:pypi/test@1.0" + mock_package.get_declared_license_expression.return_value = "mit" + mock_package.datafile_paths = ["src/main.py"] + mock_package.extra_data = {} + + mock_project.discoveredpackages.all.return_value = [mock_package] + + mock_resource = mock_resource_class() + mock_resource.path = "src/main.py" + mock_resource.for_packages = ["pkg:pypi/test@1.0"] + mock_resource.detected_license_expression = "gpl-3.0" + + mock_project.codebaseresources.has_license_expression.return_value = [ + mock_resource + ] + + mock_data_path = mock_resource_class() + mock_data_path.extra_data = {} + mock_project.codebaseresources.get.return_value = mock_data_path + + utils.validate_package_license_integrity(mock_project) + + package_update_args = mock_package.update_extra_data.call_args.args[0] + self.assertEqual( + package_update_args["issues"][0]["issue_type"], "License Mismatch" + ) + self.assertEqual( + package_update_args["issues"][0]["detected_codebase_license"], "gpl-3.0" + ) + + mock_data_path.update.assert_called_once_with(status=flag.LICENSE_ISSUE) + + def test_contains_ignore_pattern(self): + ignore_patterns = ["*test*", "*.sh"] + self.assertTrue( + utils.contains_ignore_pattern("src/test_main.py", ignore_patterns) + ) + self.assertTrue( + utils.contains_ignore_pattern("scripts/build.sh", ignore_patterns) + ) + self.assertFalse(utils.contains_ignore_pattern("src/main.py", ignore_patterns)) + + def test_filter_ignored_licenses(self): + exp1 = self.licensing.parse("mit") + self.assertEqual( + str(utils.filter_ignored_licenses(exp1, self.licensing)), "mit" + ) + + exp2 = self.licensing.parse("unknown") + self.assertIsNone(utils.filter_ignored_licenses(exp2, self.licensing)) + + exp3 = self.licensing.parse("mit AND unknown") + self.assertEqual( + str(utils.filter_ignored_licenses(exp3, self.licensing)), "mit" + ) + + exp4 = self.licensing.parse("unknown-spdx OR free-unknown") + self.assertIsNone(utils.filter_ignored_licenses(exp4, self.licensing)) + + def test_collect_detected_licenses(self): + mock_resource1 = mock.Mock() + mock_resource1.path = "src/main.py" + mock_resource1.for_packages = ["pkg:pypi/test@1.0"] + mock_resource1.detected_license_expression = "mit AND unknown" + + mock_resource2 = mock.Mock() + mock_resource2.path = "test/test_main.py" + mock_resource2.for_packages = ["pkg:pypi/test@1.0"] + mock_resource2.detected_license_expression = "gpl-3.0" + + mock_resource3 = mock.Mock() + mock_resource3.path = "src/other.py" + mock_resource3.for_packages = ["pkg:pypi/test@2.0"] + mock_resource3.detected_license_expression = "apache-2.0" + + resources = [mock_resource1, mock_resource2, mock_resource3] + ignore_patterns = ["*test*"] + + result = utils.collect_detected_licenses( + resources, ignore_patterns, package_uid="pkg:pypi/test@1.0" + ) + + self.assertEqual(result, ["(mit)"]) + + def test_get_url_netloc_namespace_and_name(self): + url = "https://github.com/aboutcode-org/scancode.io/" + netloc, namespace, name = utils.get_url_netloc_namespace_and_name(url) + self.assertEqual(netloc, "github.com") + self.assertEqual(namespace, "aboutcode-org") + self.assertEqual(name, "scancode.io") + + url_web = "https://example.com/ns/project/index.html" + netloc, namespace, name = utils.get_url_netloc_namespace_and_name(url_web) + self.assertEqual(netloc, "example.com") + self.assertEqual(namespace, "ns") + self.assertEqual(name, "project") + + @mock.patch("scanpipe.pipes.utils.fetch.fetch_url") + def test_download_src_repo_success(self, mock_fetch): + mock_fetch.return_value.path = "/test/downloaded_repo" + result = utils.download_src_repo("https://example.com/repo.zip") + self.assertEqual(result, "/test/downloaded_repo") + + @mock.patch("scanpipe.pipes.utils.fetch.fetch_url") + def test_download_src_repo_failure(self, mock_fetch): + mock_fetch.side_effect = ValueError("Invalid URL") + result = utils.download_src_repo("invalid_url") + self.assertIsNone(result) + + @mock.patch("scanpipe.pipes.utils.get_repo_download_url_by_package_type") + @mock.patch("scanpipe.pipes.utils.clarify_version_tag") + def test_get_download_url(self, mock_clarify, mock_get_repo_url): + mock_clarify.return_value = "v1.0.0" + mock_get_repo_url.return_value = ( + "https://github.com/namespace/repo_name/archive/v1.0.0.zip" + ) + + url = "https://github.com/namespace/repo_name" + result = utils.get_download_url(url, "1.0.0") + + self.assertEqual( + result, "https://github.com/namespace/repo_name/archive/v1.0.0.zip" + ) + + @mock.patch("scanpipe.pipes.utils.requests.get") + def test_clarify_version_tag(self, mock_get): + # Simulate 2 requests which the first returns 404 and the second + # returns 200 + mock_get.side_effect = [mock.Mock(status_code=404), mock.Mock(status_code=200)] + + # The function tries prefixes in order: ["", "v", "V", "release-", + # "RELEASE-", "v-", "V-"] + result = utils.clarify_version_tag("github", "namespace", "name", "1.0.0") + + self.assertEqual(result, "v1.0.0") + self.assertEqual(mock_get.call_count, 2) + + @mock.patch("scanpipe.pipes.utils.requests.get") + def test_github_pages_to_repo(self, mock_get): + mock_get.return_value = mock.Mock(status_code=200) + url = "https://krumpetpirate.github.io/AAXtoMP3/" + result = utils.github_pages_to_repo(url) + self.assertEqual(result, "https://github.com/krumpetpirate/AAXtoMP3") + + # Failed mapping + mock_get.return_value = mock.Mock(status_code=404) + result_failed = utils.github_pages_to_repo(url) + self.assertIsNone(result_failed) + + def test_get_all_files_and_count(self): + with tempfile.TemporaryDirectory() as tmp_dir: + file1_path = Path(tmp_dir) / "file1.txt" + file2_path = Path(tmp_dir) / "tmp" / "file2.txt" + + file2_path.parent.mkdir() + # Use touch() to create empty files + file1_path.touch() + file2_path.touch() + + self.assertEqual(utils.count_total_files(tmp_dir), 2) + + file_map = utils.get_all_files(tmp_dir) + self.assertIn("file1.txt", file_map) + self.assertIn("tmp/file2.txt", file_map) + self.assertEqual(file_map["file1.txt"]["name"], "file1.txt") + self.assertIsNotNone(file_map["file1.txt"]["hash"]) + + def test_consolidate_unmatched(self): + all_files = ["src/main.py", "src/utils.py", "docs/readme.md", "docs/install.md"] + unmatched_files = ["docs/readme.md", "docs/install.md", "src/utils.py"] + + result = utils.consolidate_unmatched(all_files, unmatched_files) + + # 'docs' directory is entirely unmatched. + # 'src/utils.py' is unmatched, but 'src' has a matched file ('main.py'). + expected = [("docs", True), ("src/utils.py", False)] + self.assertEqual(result, sorted(expected)) + + @mock.patch("scanpipe.pipes.utils.get_all_files") + def test_compare_directories(self, mock_get_all_files): + mock_get_all_files.side_effect = [ + # input_source + { + "hello.py": {"hash": "123", "name": "hello.py"}, + "world.py": {"hash": "456", "name": "world.py"}, + }, + # source_repo + { + "hello.py": {"hash": "123", "name": "hello.py"}, + "universe.py": {"hash": "789", "name": "universe.py"}, + }, + ] + + matched_count, mismatches = utils.compare_directories("input", "repo") + self.assertEqual(matched_count, 1) + self.assertIn("[File] world.py", mismatches["input_source_only"]) + self.assertIn("[File] universe.py", mismatches["source_repo_only"]) + + @mock.patch("scanpipe.models.DiscoveredPackage") + @mock.patch("scanpipe.models.Project") + def test_update_comparison_summary_package_found( + self, mock_project_class, mock_package_class + ): + mock_project = mock_project_class() + mock_package = mock_package_class() + + mock_project.discoveredpackages.filter.return_value.first.return_value = ( + mock_package + ) + + with tempfile.TemporaryDirectory() as temp_dir: + utils.update_comparison_summary( + project=mock_project, + purl="pkg:pypi/testg@1.0", + devel_codebase_dir=temp_dir, + src_repo_url="https://github.com/test/test", + package_name="test", + package_version="1.0", + matched_count=3, + mismatches={ + "mismatches": [], + "input_source_only": [], + "source_repo_only": [], + }, + ) + + called_data = mock_package.update_extra_data.call_args.args[0] + + self.assertIn("comparison_summary", called_data) + self.assertEqual( + called_data["comparison_summary"]["total_matching_files"], 3 + ) + self.assertEqual( + called_data["comparison_summary"]["input_source"], "pkg:pypi/testg@1.0" + ) + + def test_handle_operator_expression_and(self): + expr = self.licensing.parse("mit AND apache-2.0") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.AND + ) + self.assertEqual(str(result), "mit AND apache-2.0") + + def test_handle_operator_expression_or(self): + expr = self.licensing.parse("mit OR bsd-3-clause") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.OR + ) + self.assertEqual(str(result), "mit OR bsd-3-clause") + + def test_handle_operator_expression_filters_to_single_arg(self): + # 'unknown' gets filtered out to None, leaving only 'mit' (len == 1) + expr = self.licensing.parse("mit AND unknown") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.AND + ) + self.assertEqual(str(result), "mit") - def test_evaluate_license_mismatch_simple_OR_condition(self): - package_lic = "mit OR apache-2.0" - detected_lic_list = ["mit", "apache-2.0"] - expected = { - 'missing': [], - 'extra': [], - 'is_match': True, - 'details': 'match' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - - def test_evaluate_license_mismatch_simple_AND_condition(self): - package_lic = "mit AND apache-2.0" - detected_lic_list = ["mit", "apache-2.0"] - expected = { - 'missing': [], - 'extra': [], - 'is_match': True, - 'details': 'match' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - - def test_evaluate_license_mismatch_simple_OR_condition_1(self): - package_lic = "mit OR apache-2.0" - detected_lic_list = ["mit"] - expected = { - 'missing': [], - 'extra': [], - 'is_match': True, - 'details': 'match' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - def test_evaluate_license_mismatch_detected_lic_missing_AND_condition(self): - package_lic = "mit AND apache-2.0" - detected_lic_list = ["mit"] - expected = { - 'missing': ["apache-2.0"], - 'extra': [], - 'is_match': False, - 'details': 'Missing: apache-2.0' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - - def test_evaluate_license_mismatch_simple_AND_condition_detected_lic_has_OR(self): - package_lic = "mit AND apache-2.0" - detected_lic_list = ["mit or bsd-new", "apache-2.0"] - expected = { - 'missing': [], - 'extra': [], - 'is_match': True, - 'details': 'match' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - - def test_evaluate_license_mismatch_detected_lic_has_extra(self): - package_lic = "mit AND apache-2.0" - detected_lic_list = ["mit AND bsd-new", "apache-2.0"] - expected = { - 'missing': [], - 'extra': ["bsd-new"], - 'is_match': False, - 'details': 'Extra: bsd-new' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - def test_evaluate_license_mismatch_detected_lic_has_extra_1(self): - package_lic = "mit AND apache-2.0" - detected_lic_list = ["mit", "bsd-new", "apache-2.0"] - expected = { - 'missing': [], - 'extra': ["bsd-new"], - 'is_match': False, - 'details': 'Extra: bsd-new' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - def test_evaluate_license_mismatch_AND_condition_with_OR(self): - package_lic = "(bsd-new OR apache-2.0) AND apache-2.0 AND mit" - detected_lic_list = ["mit", "apache-2.0"] - expected = { - 'missing': [], - 'extra': [], - 'is_match': True, - 'details': 'match' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - def test_evaluate_license_mismatch_WITH_AND_OR_Missing(self): - package_lic = "(bsd-new AND apache-2.0) AND apache-2.0 AND (mit OR public-domain)" - detected_lic_list = ["mit", "apache-2.0"] - expected = { - 'missing': ["bsd-new"], - 'extra': [], - 'is_match': False, - 'details': 'Missing: bsd-new' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - def test_evaluate_license_mismatch_WITH_AND_OR(self): - package_lic = "(bsd-new OR apache-2.0) AND apache-2.0 AND (mit OR public-domain)" - detected_lic_list = ["mit", "apache-2.0"] - expected = { - 'missing': [], - 'extra': [], - 'is_match': True, - 'details': 'match' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - def test_evaluate_license_mismatch_missing_extra_AND(self): - package_lic = "bsd-new AND apache-2.0" - detected_lic_list = ["mit"] - expected = { - 'missing': ['apache-2.0', 'bsd-new'], - 'extra': ['mit'], - 'is_match': False, - 'details': 'Missing: apache-2.0, bsd-new; Extra: mit' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - def test_evaluate_license_mismatch_missing_extra_OR(self): - package_lic = "bsd-new OR apache-2.0" - detected_lic_list = ["mit"] - expected = { - 'missing': ['apache-2.0', 'bsd-new'], - 'extra': ['mit'], - 'is_match': False, - 'details': 'Missing: apache-2.0, bsd-new; Extra: mit' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - def test_evaluate_license_mismatch_some_missing_extra(self): - package_lic = "(bsd-new OR apache-2.0) AND mit AND (apache-1.1 AND gpl-2.0)" - detected_lic_list = ["mit", "lgpl-2.1"] - expected = { - 'missing': ['apache-1.1', 'apache-2.0', 'bsd-new', 'gpl-2.0'], - 'extra': ['lgpl-2.1'], - 'is_match': False, - 'details': 'Missing: apache-1.1, apache-2.0, bsd-new, gpl-2.0; Extra: lgpl-2.1' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - def test_evaluate_license_mismatch_WITH_WITH_Exception(self): - package_lic = "bsd-new OR gpl-2.0 WITH classpath-exception-2.0" - detected_lic_list = ["bsd-new"] - expected = { - 'missing': [], - 'extra': [], - 'is_match': True, - 'details': 'match' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - def test_evaluate_license_mismatch_WITH_WITH_Exception_missing(self): - package_lic = "bsd-new AND gpl-2.0 WITH classpath-exception-2.0" - detected_lic_list = ["bsd-new", "gpl-2.0"] - expected = { - 'missing': ['classpath-exception-2.0'], - 'extra': [], - 'is_match': False, - 'details': 'Missing: classpath-exception-2.0' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected - - def test_evaluate_license_mismatch_WITH_WITH_Exception_extra(self): - package_lic = "bsd-new AND gpl-2.0" - detected_lic_list = ["bsd-new", "gpl-2.0 WITH classpath-exception-2.0"] - expected = { - 'missing': [], - 'extra': ['classpath-exception-2.0'], - 'is_match': False, - 'details': 'Extra: classpath-exception-2.0' - } - result = utils.evaluate_license_mismatch(package_lic, detected_lic_list) - assert result == expected + def test_handle_operator_expression_all_filtered_out(self): + # Both 'unknown' and 'free-unknown' get filtered out, leaving empty args + expr = self.licensing.parse("unknown AND free-unknown") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.AND + ) + self.assertIsNone(result) From 0d6780fb699c1a809996751190b30b9a1237a9f1 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 11 Mar 2026 15:29:55 +1300 Subject: [PATCH 12/73] chore: set explicit workflow permissions and pin down actions (#2090) Signed-off-by: tdruez --- .github/workflows/publish-pypi-release.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/publish-pypi-release.yml b/.github/workflows/publish-pypi-release.yml index fb5b0f2b6f..2e5163a147 100644 --- a/.github/workflows/publish-pypi-release.yml +++ b/.github/workflows/publish-pypi-release.yml @@ -1,4 +1,4 @@ -name: Build Python distributions, publish on PyPI, and create a GitHub release +name: Build Python distributions, publish on PyPI, and create a GH release on: workflow_dispatch: @@ -74,11 +74,6 @@ jobs: contents: write steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Download package distributions uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: From 21ab23d166bf20a2cda2fb49d3b0a8a681a51f38 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 11 Mar 2026 18:59:30 +1300 Subject: [PATCH 13/73] fix: add the checkout step to pypi release workflow Signed-off-by: tdruez --- .github/workflows/publish-pypi-release.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-pypi-release.yml b/.github/workflows/publish-pypi-release.yml index 2e5163a147..fb5b0f2b6f 100644 --- a/.github/workflows/publish-pypi-release.yml +++ b/.github/workflows/publish-pypi-release.yml @@ -1,4 +1,4 @@ -name: Build Python distributions, publish on PyPI, and create a GH release +name: Build Python distributions, publish on PyPI, and create a GitHub release on: workflow_dispatch: @@ -74,6 +74,11 @@ jobs: contents: write steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Download package distributions uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0 with: From a70388e1694b13543d4ea6c3b9ccb1ab64a30ae8 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 11 Mar 2026 21:08:26 +1300 Subject: [PATCH 14/73] chore: refine gh workflows for security and consistency Signed-off-by: tdruez --- .../publish-pypi-release-aboutcode-pipeline.yml | 13 +++++++++---- .github/workflows/publish-pypi-release.yml | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml b/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml index 62ff6c388c..51235ed27a 100644 --- a/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml +++ b/.github/workflows/publish-pypi-release-aboutcode-pipeline.yml @@ -6,6 +6,11 @@ on: tags: - "aboutcode.pipeline/*" +env: + PYPI_PROJECT_URL: "https://pypi.org/p/aboutcode.pipeline" + PYPROJECT_TOML: "pipeline-pyproject.toml" + FLOT_VERSION: "0.7.2" + jobs: build: name: Build and publish library to PyPI @@ -24,10 +29,10 @@ jobs: python-version: 3.14 - name: Install flot - run: python -m pip install flot==0.7.2 --user + run: python -m pip install "flot==${FLOT_VERSION}" --user - name: Build a binary wheel and a source tarball - run: python -m flot --pyproject pipeline-pyproject.toml --sdist --wheel --output-dir dist/ + run: python -m flot --pyproject "$PYPROJECT_TOML" --sdist --wheel --output-dir dist/ - name: Upload package distributions as GitHub workflow artifacts uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 @@ -36,7 +41,7 @@ jobs: path: dist/ # Only set the id-token: write permission in the job that does publishing, not globally. - # Also, separate building from publishing — this makes sure that any scripts + # Also, separate building from publishing, this makes sure that any scripts # maliciously injected into the build or test environment won't be able to elevate # privileges while flying under the radar. pypi-publish: @@ -47,7 +52,7 @@ jobs: runs-on: ubuntu-24.04 environment: name: pypi - url: https://pypi.org/p/aboutcode.pipeline + url: ${{ env.PYPI_PROJECT_URL }} permissions: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing diff --git a/.github/workflows/publish-pypi-release.yml b/.github/workflows/publish-pypi-release.yml index fb5b0f2b6f..6ab7203dc3 100644 --- a/.github/workflows/publish-pypi-release.yml +++ b/.github/workflows/publish-pypi-release.yml @@ -28,7 +28,7 @@ jobs: python-version: 3.14 - name: Install pypa/build - run: python -m pip install build --user + run: python -m pip install build==1.4.0 --user - name: Build a binary wheel and a source tarball run: python -m build --sdist --wheel --outdir dist/ @@ -40,7 +40,7 @@ jobs: path: dist/ # Only set the id-token: write permission in the job that does publishing, not globally. - # Also, separate building from publishing — this makes sure that any scripts + # Also, separate building from publishing, this makes sure that any scripts # maliciously injected into the build or test environment won't be able to elevate # privileges while flying under the radar. pypi-publish: From 4f2e785f24278b561c49ce029d3619cad47bb2ca Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 16 Mar 2026 17:59:37 +0400 Subject: [PATCH 15/73] feat: display scio and toolkit versions in place of django version (#2101) Signed-off-by: tdruez --- scancodeio/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scancodeio/__init__.py b/scancodeio/__init__.py index 1093a8a8a6..af281b87fa 100644 --- a/scancodeio/__init__.py +++ b/scancodeio/__init__.py @@ -95,6 +95,14 @@ def command_line(): """Command line entry point.""" from django.core.management import execute_from_command_line + # Display ScanCode.io and ScanCode-toolkit versions in place of Django version. + if "--version" in sys.argv: + from scancode_config import __version__ as scancode_toolkit_version + + print(f"ScanCode.io version: {__version__}") + print(f"ScanCode-toolkit version: v{scancode_toolkit_version}") + sys.exit(0) + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "scancodeio.settings") execute_from_command_line(sys.argv) From 15e399cfc1cc52dd48d5e8a8bf8f8e5b80ef8d30 Mon Sep 17 00:00:00 2001 From: Rishabh Rohil <130820750+rishabh23rohil@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:03:59 -0500 Subject: [PATCH 16/73] fix missing space in scan_max_file_size help text (#2097) Signed-off-by: Rishabh Rohil --- scanpipe/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanpipe/forms.py b/scanpipe/forms.py index e3f93e3af4..8329e36cf9 100644 --- a/scanpipe/forms.py +++ b/scanpipe/forms.py @@ -553,7 +553,7 @@ class ProjectSettingsForm(forms.ModelForm): label="Max file size to scan", required=False, help_text=( - "Maximum file size in bytes which should be skipped from scanning." + "Maximum file size in bytes which should be skipped from scanning. " "File size is in bytes. Example: 5 MB is 5242880 bytes." ), widget=forms.NumberInput(attrs={"class": "input"}), From 25351027d3a2ddd26eb85b2d980c1afa56a2a3c4 Mon Sep 17 00:00:00 2001 From: Rishabh Rohil <130820750+rishabh23rohil@users.noreply.github.com> Date: Mon, 16 Mar 2026 09:08:49 -0500 Subject: [PATCH 17/73] feat: add tests for chunked and get_purls utilities (#2100) Signed-off-by: Rishabh Rohil --- scanpipe/tests/pipes/test_vulnerablecode.py | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/scanpipe/tests/pipes/test_vulnerablecode.py b/scanpipe/tests/pipes/test_vulnerablecode.py index e39dbf69ed..f33a68cdca 100644 --- a/scanpipe/tests/pipes/test_vulnerablecode.py +++ b/scanpipe/tests/pipes/test_vulnerablecode.py @@ -28,8 +28,10 @@ from django.test import TestCase from scanpipe.models import Project +from scanpipe.pipes.vulnerablecode import chunked from scanpipe.pipes.vulnerablecode import fetch_vulnerabilities from scanpipe.pipes.vulnerablecode import filter_vulnerabilities +from scanpipe.pipes.vulnerablecode import get_purls from scanpipe.tests import make_package @@ -81,3 +83,29 @@ def test_scanpipe_pipes_vulnerablecode_filter_vulnerabilities(self): vulnerability2 = vulnerability_data[1] ignore_set.add(vulnerability2.get("aliases")[1]) self.assertEqual([], filter_vulnerabilities(vulnerability_data, ignore_set)) + + def test_scanpipe_pipes_vulnerablecode_chunked(self): + result = list(chunked([1, 2, 3, 4, 5], 2)) + self.assertEqual([[1, 2], [3, 4], [5]], result) + + result = list(chunked([1, 2, 3, 4, 5], 3)) + self.assertEqual([[1, 2, 3], [4, 5]], result) + + result = list(chunked([], 10)) + self.assertEqual([], result) + + result = list(chunked([1], 5)) + self.assertEqual([[1]], result) + + result = list(chunked([1, 2, 3], 3)) + self.assertEqual([[1, 2, 3]], result) + + def test_scanpipe_pipes_vulnerablecode_get_purls(self): + pkg1 = make_package(self.project1, "pkg:pypi/django@5.0") + pkg2 = make_package(self.project1, "pkg:npm/express@4.18.2") + purls = get_purls([pkg1, pkg2]) + self.assertEqual(["pkg:pypi/django@5.0", "pkg:npm/express@4.18.2"], purls) + + def test_scanpipe_pipes_vulnerablecode_get_purls_empty(self): + purls = get_purls([]) + self.assertEqual([], purls) From 239df346c6db0510b38ebc12b7c4a267aa048245 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Mon, 16 Mar 2026 18:16:20 +0400 Subject: [PATCH 18/73] feat: display layers information (created_by, comment) in tree view (#2102) Signed-off-by: tdruez --- .../templates/scanpipe/resource_tree.html | 27 ++- .../scanpipe/tree/resource_left_pane.html | 4 + .../tree/resource_left_pane_header.html | 4 +- .../tree/resource_path_breadcrumb.html | 2 +- .../scanpipe/tree/resource_table.html | 181 ++++++------------ .../tree/resource_table_resource_row.html | 78 ++++++++ scanpipe/views.py | 9 +- 7 files changed, 162 insertions(+), 143 deletions(-) create mode 100644 scanpipe/templates/scanpipe/tree/resource_left_pane.html create mode 100644 scanpipe/templates/scanpipe/tree/resource_table_resource_row.html diff --git a/scanpipe/templates/scanpipe/resource_tree.html b/scanpipe/templates/scanpipe/resource_tree.html index 73cfc75816..24d2bea087 100644 --- a/scanpipe/templates/scanpipe/resource_tree.html +++ b/scanpipe/templates/scanpipe/resource_tree.html @@ -20,24 +20,21 @@
- {% include "scanpipe/tree/resource_left_pane_header.html" only %} -
- {% include "scanpipe/tree/resource_left_pane_tree.html" with children=children path=path %} -
+ {% include 'scanpipe/tree/resource_left_pane.html' %}
-
-
+
+
diff --git a/scanpipe/templates/scanpipe/tree/resource_left_pane.html b/scanpipe/templates/scanpipe/tree/resource_left_pane.html new file mode 100644 index 0000000000..62d1bc70b5 --- /dev/null +++ b/scanpipe/templates/scanpipe/tree/resource_left_pane.html @@ -0,0 +1,4 @@ +{% include "scanpipe/tree/resource_left_pane_header.html" only %} +
+ {% include "scanpipe/tree/resource_left_pane_tree.html" with children=children path=path %} +
\ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tree/resource_left_pane_header.html b/scanpipe/templates/scanpipe/tree/resource_left_pane_header.html index 1dfbc8482c..3250d5f1e1 100644 --- a/scanpipe/templates/scanpipe/tree/resource_left_pane_header.html +++ b/scanpipe/templates/scanpipe/tree/resource_left_pane_header.html @@ -1,12 +1,12 @@
-
+
Resources
\ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tree/resource_path_breadcrumb.html b/scanpipe/templates/scanpipe/tree/resource_path_breadcrumb.html index 0a1bd943e7..766af14741 100644 --- a/scanpipe/templates/scanpipe/tree/resource_path_breadcrumb.html +++ b/scanpipe/templates/scanpipe/tree/resource_path_breadcrumb.html @@ -1,7 +1,7 @@
-{% endblock %} - -{% block scripts %} - {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/includes/admin_edit_link.html b/scanpipe/templates/scanpipe/includes/admin_edit_link.html index 932f34e41b..52e2a147fa 100644 --- a/scanpipe/templates/scanpipe/includes/admin_edit_link.html +++ b/scanpipe/templates/scanpipe/includes/admin_edit_link.html @@ -2,7 +2,7 @@ {% with object.get_admin_url as admin_url %} {% if admin_url %} - + diff --git a/scanpipe/templates/scanpipe/includes/search_field.html b/scanpipe/templates/scanpipe/includes/search_field.html index e12ced092f..bd0b7ba0d5 100644 --- a/scanpipe/templates/scanpipe/includes/search_field.html +++ b/scanpipe/templates/scanpipe/includes/search_field.html @@ -19,5 +19,6 @@ + {% include 'scanpipe/modals/search_syntax_modal.html' %} {% endif %}
\ No newline at end of file diff --git a/scanpipe/templates/scanpipe/modals/add_inputs_modal.html b/scanpipe/templates/scanpipe/modals/add_inputs_modal.html index 95afe9befc..8ea412c15a 100644 --- a/scanpipe/templates/scanpipe/modals/add_inputs_modal.html +++ b/scanpipe/templates/scanpipe/modals/add_inputs_modal.html @@ -1,7 +1,7 @@ + {{ pipelines_available_groups|json_script:"pipelines_available_groups" }} {% endblock %} -{% block scripts %} - - - {{ pipelines_available_groups|json_script:"pipelines_available_groups" }} - - - +{% block modals %} + {% include "scanpipe/modals/pipeline_help_modal.html" %} {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/project_list.html b/scanpipe/templates/scanpipe/project_list.html index 6871206137..9575f16bf4 100644 --- a/scanpipe/templates/scanpipe/project_list.html +++ b/scanpipe/templates/scanpipe/project_list.html @@ -56,7 +56,9 @@ {% endif %} +{% endblock %} +{% block modals %} {% include 'scanpipe/modals/run_modal.html' %} {% include "scanpipe/modals/projects_download_modal.html" %} {% include "scanpipe/modals/projects_report_modal.html" %} diff --git a/scanpipe/templates/scanpipe/project_settings.html b/scanpipe/templates/scanpipe/project_settings.html index 1651f7978a..e83792596f 100644 --- a/scanpipe/templates/scanpipe/project_settings.html +++ b/scanpipe/templates/scanpipe/project_settings.html @@ -21,7 +21,9 @@ +{% endblock %} +{% block modals %} {% include "scanpipe/modals/project_webhook_add_modal.html" %} {% include "scanpipe/modals/project_webhook_delete_modal.html" %} {% if not project.is_archived %} @@ -42,19 +44,5 @@ window.location.reload(); } }); - - onSubmitOverlay = function (selector) { - let element = document.querySelector(selector); - if (element) { - element.addEventListener("submit", function() { - displayOverlay(); - }); - } - }; - - onSubmitOverlay("#modal-archive form"); - onSubmitOverlay("#modal-reset form"); - onSubmitOverlay("#modal-delete form"); - onSubmitOverlay("#modal-webhook-add form"); {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/resource_detail.html b/scanpipe/templates/scanpipe/resource_detail.html index 58c23e0f55..8aebda40b5 100644 --- a/scanpipe/templates/scanpipe/resource_detail.html +++ b/scanpipe/templates/scanpipe/resource_detail.html @@ -4,8 +4,9 @@ {% block title %}ScanCode.io: {{ project.name }} - {{ object.name }}{% endblock %} {% block extrahead %} - - + + + {% endblock %} {% block content %} @@ -19,134 +20,8 @@ {% include 'scanpipe/tree/resource_path_breadcrumb.html' with path=object.path %} {% endif %} {% include 'scanpipe/tabset/tabset.html' %} + {{ detected_values|json_script:"detected_values" }} {% endblock %} -{% endblock %} - -{% block scripts %} - {{ detected_values|json_script:"detected_values" }} - {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/resource_tree.html b/scanpipe/templates/scanpipe/resource_tree.html index 24d2bea087..8c7cad2e3c 100644 --- a/scanpipe/templates/scanpipe/resource_tree.html +++ b/scanpipe/templates/scanpipe/resource_tree.html @@ -3,8 +3,10 @@ {% block title %}ScanCode.io: {{ project.name }} - Resources tree{% endblock %} {% block extrahead %} - - + + + + {% endblock %} {% block content %} @@ -17,7 +19,7 @@ -
+
{% include 'scanpipe/tree/resource_left_pane.html' %} @@ -38,126 +40,4 @@
-{% endblock %} - -{% block scripts %} - {% endblock %} \ No newline at end of file diff --git a/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html b/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html index 0d7a5e4053..e9d296b77d 100644 --- a/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html +++ b/scanpipe/templates/scanpipe/tabset/tab_vulnerabilities.html @@ -11,7 +11,7 @@ {% for vulnerability in tab_data.fields.affected_by_vulnerabilities.value %} - {% include 'scanpipe/includes/vulnerability_id.html' with vulnerability=vulnerability %} + {% include 'scanpipe/includes/vulnerability_id.html' with vulnerability=vulnerability VULNERABLECODE_URL=tab_data.VULNERABLECODE_URL %} {% include 'scanpipe/includes/vulnerability_summary.html' with vulnerability=vulnerability only %} diff --git a/scanpipe/views.py b/scanpipe/views.py index 2e1ac5f362..e5fd2bca2b 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -260,6 +260,7 @@ class TabSetMixin: "icon_class": "", "display_condition": , "disable_condition": , + "tab_context": {}, } } """ @@ -291,6 +292,8 @@ def get_tab_data(self, tab_definition): elif fields_have_no_values(fields_data): is_disabled = True + tab_context = tab_definition.get("tab_context") or {} + tab_data = { "verbose_name": tab_definition.get("verbose_name"), "icon_class": tab_definition.get("icon_class"), @@ -298,6 +301,7 @@ def get_tab_data(self, tab_definition): "fields": fields_data, "disabled": is_disabled, "label_count": self.get_label_count(fields_data), + **tab_context, } return tab_data @@ -1972,6 +1976,7 @@ def get_queryset(self): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context["object_list"] = self.project.vulnerabilities + context["VULNERABLECODE_URL"] = settings.VULNERABLECODE_URL return context @@ -2336,6 +2341,7 @@ class DiscoveredPackageDetailsView( ], "icon_class": "fa-solid fa-bug", "template": "scanpipe/tabset/tab_vulnerabilities.html", + "tab_context": {"VULNERABLECODE_URL": settings.VULNERABLECODE_URL}, }, "extra_data": { "fields": ["extra_data"], From b99006d97ae01a6f14b2d3729e12478a12ecae07 Mon Sep 17 00:00:00 2001 From: tdruez <489057+tdruez@users.noreply.github.com> Date: Wed, 18 Mar 2026 18:42:28 +0400 Subject: [PATCH 20/73] feat: remove the need for context processor and improve base template (#2106) Signed-off-by: tdruez --- scancodeio/settings.py | 1 - scanpipe/templates/account/profile.html | 3 --- scanpipe/templates/registration/login.html | 3 --- .../templates/scanpipe/app_monitoring.html | 23 ------------------- scanpipe/templates/scanpipe/base.html | 6 +++++ .../templates/scanpipe/dependency_detail.html | 2 -- .../templates/scanpipe/dependency_list.html | 1 - .../templates/scanpipe/dependency_tree.html | 1 - .../scanpipe/includes/navbar_header.html | 3 +++ .../templates/scanpipe/license_detail.html | 2 -- .../scanpipe/license_detection_detail.html | 2 -- .../scanpipe/license_detection_list.html | 1 - scanpipe/templates/scanpipe/license_list.html | 1 - scanpipe/templates/scanpipe/message_list.html | 1 - .../templates/scanpipe/package_detail.html | 4 +--- scanpipe/templates/scanpipe/package_list.html | 1 - .../templates/scanpipe/project_detail.html | 3 --- scanpipe/templates/scanpipe/project_form.html | 2 -- scanpipe/templates/scanpipe/project_list.html | 3 --- .../templates/scanpipe/project_settings.html | 2 -- .../templates/scanpipe/relation_list.html | 1 - .../templates/scanpipe/resource_detail.html | 2 -- .../templates/scanpipe/resource_list.html | 1 - .../templates/scanpipe/resource_tree.html | 1 - .../scanpipe/vulnerability_list.html | 1 - scanpipe/templatetags/__init__.py | 0 .../templatetags/scanpipe_tags.py | 18 +++++++++++---- scanpipe/urls.py | 2 -- 28 files changed, 23 insertions(+), 68 deletions(-) delete mode 100644 scanpipe/templates/scanpipe/app_monitoring.html create mode 100644 scanpipe/templatetags/__init__.py rename scancodeio/context_processors.py => scanpipe/templatetags/scanpipe_tags.py (79%) diff --git a/scancodeio/settings.py b/scancodeio/settings.py index 9a94a13383..b82fff243c 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -252,7 +252,6 @@ "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", "django.template.context_processors.request", - "scancodeio.context_processors.versions", ], }, }, diff --git a/scanpipe/templates/account/profile.html b/scanpipe/templates/account/profile.html index f71a732cd1..a2f7eeb8f2 100644 --- a/scanpipe/templates/account/profile.html +++ b/scanpipe/templates/account/profile.html @@ -2,9 +2,6 @@ {% block content %}
- {% include 'scanpipe/includes/navbar_header.html' %} -
{% include 'scanpipe/includes/messages.html' %}
-