From 9cf3b5723c98710780ee62a92b4ec87e1eca5b92 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 31 Jul 2026 13:00:04 -0700 Subject: [PATCH 1/3] Merge miners into pipes Signed-off-by: Jono Yang --- minecode_pipelines/miners/__init__.py | 8 -- minecode_pipelines/miners/cpan.py | 159 ----------------------- minecode_pipelines/miners/nix.py | 143 --------------------- minecode_pipelines/miners/npm.py | 167 ------------------------ minecode_pipelines/miners/pypi.py | 107 ---------------- minecode_pipelines/pipes/cpan.py | 148 +++++++++++++++++++++- minecode_pipelines/pipes/nix.py | 146 ++++++++++++++++++--- minecode_pipelines/pipes/npm.py | 174 +++++++++++++++++++++++--- minecode_pipelines/pipes/pypi.py | 124 +++++++++++++++--- 9 files changed, 536 insertions(+), 640 deletions(-) delete mode 100644 minecode_pipelines/miners/__init__.py delete mode 100644 minecode_pipelines/miners/cpan.py delete mode 100644 minecode_pipelines/miners/nix.py delete mode 100644 minecode_pipelines/miners/npm.py delete mode 100644 minecode_pipelines/miners/pypi.py diff --git a/minecode_pipelines/miners/__init__.py b/minecode_pipelines/miners/__init__.py deleted file mode 100644 index e1521118..00000000 --- a/minecode_pipelines/miners/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# diff --git a/minecode_pipelines/miners/cpan.py b/minecode_pipelines/miners/cpan.py deleted file mode 100644 index c006bf89..00000000 --- a/minecode_pipelines/miners/cpan.py +++ /dev/null @@ -1,159 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import gzip -import requests - -from bs4 import BeautifulSoup -from packageurl import PackageURL - -from minecode_pipelines.utils import get_temp_file - -""" -Visitors for cpan and cpan-like perl package repositories. -""" - - -CPAN_REPO = "https://www.cpan.org/" -CPAN_TYPE = "cpan" - - -def get_cpan_packages(cpan_repo=CPAN_REPO, logger=None): - """ - Get cpan package names parsed from the `02packages.details.txt` - which contains a list of all modules and their respective - package archive paths. We parse the package names and their respective - path_prefixes with author page path from this list. - """ - cpan_packages_url = cpan_repo + "modules/02packages.details.txt.gz" - packages_archive = get_temp_file(file_name="cpan_packages", extension=".gz") - packages_content = get_temp_file(file_name="cpan_packages", extension=".txt") - response = requests.get(cpan_packages_url, stream=True) - with open(packages_archive, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) - - with gzip.open(packages_archive, "rb") as f_in: - with open(packages_content, "wb") as f_out: - f_out.writelines(f_in) - - with open(packages_content, encoding="utf-8") as file: - packages_content = file.read() - - package_path_by_name = {} - - # The ``modules/02packages.details.txt`` file has the following section - # at the beginning of the file: - # - # File: 02packages.details.txt - # URL: http://www.cpan.org/modules/02packages.details.txt - # Description: Package names found in directory $CPAN/authors/id/ - # Columns: package name, version, path - # Intended-For: Automated fetch routines, namespace documentation. - # Written-By: PAUSE version 1.005 - # Line-Count: 268940 - # Last-Updated: Mon, 29 Sep 2025 22:29:02 GMT - # - # This information is there in first 10 lines, and the last line is an - # empty line, both of which we are ignoring below - - modules = packages_content.split("\n")[9:-1] - - # A sample line from this module list looks like this: - # - # Crypt::Passphrase::SHA1::Base64 0.021 L/LE/LEONT/Crypt-Passphrase-0.021.tar.gz - - for module in modules: - info = [section for section in module.split(" ") if section] - - # This is like: L/LE/LEONT/Crypt-Passphrase-0.021.tar.gz - package_path = info[-1] - path_segments = package_path.split("/") - filename = path_segments.pop() - path_prefix = "/".join(path_segments) - - name_version = filename.replace(".tar.gz", "").split("-") - _version = name_version.pop() - name = "-".join(name_version) - - # for the above example: name: Crypt-Passphrase, path_prefix: L/LE/LEONT/ - package_path_by_name[name] = path_prefix - - return package_path_by_name - - -def get_cpan_packageurls(name, path_prefix, logger=None): - """ - Given a package name and it's path_prefix (author page path) - return a list of packageURLs for that package. - - An author page (like https://www.cpan.org/authors/id/P/PT/PTC/) lists - all versions of all packages released by the author, so we can scrape - all the packageURLs from this author packages index. - """ - - author_name = path_prefix.split("/")[-1] - - packageurls = [] - - # file extensions found in cpan index - ignorable_extensions = [".meta", ".readme", ".tar.gz"] - - cpan_authors_path = "/authors/id/" - cpan_authors_url = CPAN_REPO + cpan_authors_path - - cpan_author_page_url = cpan_authors_url + path_prefix - - response = requests.get(cpan_author_page_url) - if not response.ok: - return packageurls - - if logger: - logger(f"Getting package versions for {name} from {cpan_author_page_url}") - - soup = BeautifulSoup(response.text, "html.parser") - - # We get all the listed packages in the author page index - package_list = soup.find("ul") - if not package_list: - return packageurls - - package_list_elements = package_list.text.split("\n") - - package_elements = [ - element.replace(" ", "") - for element in package_list_elements - if element and element not in {" Parent Directory", " CHECKSUMS"} - ] - - versions = [] - for package_file in package_elements: - for extension in ignorable_extensions: - if extension in package_file: - package_file = package_file.replace(extension, "") - - name_version = package_file.split("-") - version = name_version.pop() - package_name = "-".join(name_version) - if package_name != name: - continue - - versions.append(version) - - unique_versions = list(set(versions)) - for version in unique_versions: - purl = PackageURL( - type=CPAN_TYPE, - namespace=author_name, - name=name, - version=version, - ) - packageurls.append(purl.to_string()) - - return packageurls diff --git a/minecode_pipelines/miners/nix.py b/minecode_pipelines/miners/nix.py deleted file mode 100644 index 6a3b77d8..00000000 --- a/minecode_pipelines/miners/nix.py +++ /dev/null @@ -1,143 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import brotli -import json -import requests -import time - -from packageurl import PackageURL - - -NIX_TYPE = "nix" -NIXPKGS = "nixpkgs" -DELAY_MULTIPLIER = 2 -MAX_RETRIES = 3 - - -def get_all_nix_packages(channel_url, logger=None): - """Get all Nix packages""" - if logger: - logger(f"Fetching Nix packages from: {channel_url}") - response = requests.get(channel_url) - response.raise_for_status() - - try: - decompressed_data = brotli.decompress(response.content) - data = json.loads(decompressed_data) - except brotli.error: - data = response.json() - - packages_dict = data.get("packages", {}) - return packages_dict - - -def get_all_nix_packages_name(packages_dict, logger=None): - """Get all Nix packages name""" - all_package_names = [] - packages = list(packages_dict.keys()) - for attr_path in packages: - all_package_names.append(attr_path) - - return {"packages": all_package_names} - - -def load_nix_packages(packages_file): - with open(packages_file) as f: - packages_data = json.load(f) - - return packages_data.get("packages", []) - - -def get_nix_packageurls(name, packages_dict, logger=None): - packageurls = [] - data = [] - # Get all the version of a package from the following API: - # https://search.devbox.sh/v2/pkg?name={name} - url = f"https://search.devbox.sh/v2/pkg?name={name}" - - for attempt in range(MAX_RETRIES): - try: - response = requests.get(url) - response.raise_for_status() - data = response.json() - break - except requests.HTTPError as e: - if response.status_code == 429: - retry_after = response.headers.get("Retry-After") - if retry_after: - wait_time = int(retry_after) - else: - wait_time = DELAY_MULTIPLIER**attempt - if logger: - logger(f"Rate limited (429). Waiting {wait_time}s before retry...") - time.sleep(wait_time) - continue - elif response.status_code == 404: - # The package_dict contains the package's current version - # devbox.sh may have only index the top - # level, all others deeper level may not be indexed - # TODO: This can only get the current version. - version = get_version_from_package_dict(name, packages_dict) - if version: - purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name, version=version) - else: - purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name) - packageurls.append(purl.to_string()) - return packageurls - else: - if logger: - logger(f"Request failed: {e}") - logger(f"HTTP error {response.status_code}: {response.text}") - return packageurls - except requests.RequestException as e: - if logger: - logger(f"Request failed: {e}") - return packageurls - except ValueError as e: - if logger: - logger(f"JSON parsing failed: {e}") - return packageurls - if not data: - # No data collected after retries - if logger: - logger(f"Max retries reached. Skipping package: {name}") - return packageurls - - releases = data.get("releases", []) - for release in releases: - version = release.get("version") - if version: - purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name, version=version) - purl_string = purl.to_string() - if purl_string not in packageurls: - packageurls.append(purl_string) - - return packageurls - - -def get_version_from_package_dict(name, package_dict): - """Get the version data from the package_dict""" - package_info = package_dict.get(name) - if not package_info: - return None - return package_info.get("version") - - -def yield_nix_package_data(name, packageurls=[]): - for purl in packageurls: - package_url = PackageURL.from_string(purl) - package_data_url = ( - f"https://search.devbox.sh/v2/resolve?name={name}&version={package_url.version}" - ) - response = requests.get(package_data_url) - if not response.ok: - continue - yield purl, response.json() diff --git a/minecode_pipelines/miners/npm.py b/minecode_pipelines/miners/npm.py deleted file mode 100644 index 783feff4..00000000 --- a/minecode_pipelines/miners/npm.py +++ /dev/null @@ -1,167 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import json -import requests - -from packageurl import PackageURL - - -""" -Visitors for Npmjs and npmjs-like javascript package repositories. - -We have this hierarchy in npm replicate and registry index: - npm projects replicate.npmjs.com (paginated JSON) -> versions at registry.npmjs.org (JSON) -> download urls - -See https://github.com/orgs/community/discussions/152515 for information on -the latest replicate.npmjs.com API. - -https://replicate.npmjs.com/_all_docs -This NPMJS replicate API serves as an index to get all npm packages and their revision IDs -in paginated queries. - -https://replicate.npmjs.com/_changes -This NPMJS replicate API serves as a CHANGELOG of npm packages with update sequences which -can be fetched in paginated queries. - -https://registry.npmjs.org/{namespace/name} -For each npm package, a JSON containing details including the list of all releases -and archives, their URLs, and some metadata for each release. - -https://registry.npmjs.org/{namespace/name}/{version} -For each release, a JSON contains details for the released version and all the -downloads available for this release. -""" - - -NPM_REPLICATE_REPO = "https://replicate.npmjs.com/" -NPM_REGISTRY_REPO = "https://registry.npmjs.org/" -NPM_TYPE = "npm" -NPM_REPLICATE_BATCH_SIZE = 10000 - - -def get_package_names_last_key(package_data): - names = [package.get("id") for package in package_data.get("rows")] - last_key = package_data.get("rows")[-1].get("key") - return names, last_key - - -def get_package_names_last_seq(package_data): - names = [package.get("id") for package in package_data.get("results")] - last_seq = package_data.get("last_seq") - return names, last_seq - - -def get_current_last_seq(replicate_url=NPM_REPLICATE_REPO): - npm_replicate_latest_changes = replicate_url + "_changes?descending=True" - response = requests.get(npm_replicate_latest_changes) - if not response.ok: - return - - package_data = response.json() - _package_names, last_seq = get_package_names_last_seq(package_data) - return last_seq - - -def get_updated_npm_packages(last_seq, replicate_url=NPM_REPLICATE_REPO, logger=None): - all_package_names = [] - i = 0 - - while True: - if logger: - logger(f"Processing iteration: {i}: changes after seq: {last_seq}") - - npm_replicate_changes = ( - replicate_url + "_changes?" + f"limit={NPM_REPLICATE_BATCH_SIZE}" + f"&since={last_seq}" - ) - response = requests.get(npm_replicate_changes) - if not response.ok: - return all_package_names - - package_data = response.json() - package_names, last_seq = get_package_names_last_seq(package_data) - all_package_names.extend(package_names) - - # We have fetched the last set of changes if True - if len(package_names) < NPM_REPLICATE_BATCH_SIZE: - break - - i += 1 - - return {"packages": all_package_names}, last_seq - - -def get_npm_packages(replicate_url=NPM_REPLICATE_REPO, logger=None): - all_package_names = [] - - npm_replicate_all = replicate_url + "_all_docs?" + f"limit={NPM_REPLICATE_BATCH_SIZE}" - response = requests.get(npm_replicate_all) - if not response.ok: - return all_package_names - - package_data = response.json() - package_names, last_key = get_package_names_last_key(package_data) - all_package_names.extend(package_names) - - total_rows = package_data.get("total_rows") - iterations = int(total_rows / NPM_REPLICATE_BATCH_SIZE) + 1 - - for i in range(iterations): - npm_replicate_from_id = npm_replicate_all + f'&start_key="{last_key}"' - if logger: - logger(f"Processing iteration: {i}: {npm_replicate_from_id}") - - response = requests.get(npm_replicate_from_id) - if not response.ok: - raise Exception(npm_replicate_from_id, response.text) - - package_data = response.json() - package_names, last_key = get_package_names_last_key(package_data) - all_package_names.extend(package_names) - - return {"packages": all_package_names} - - -def get_npm_packageurls(name, npm_repo=NPM_REGISTRY_REPO): - packageurls = [] - - project_index_api_url = npm_repo + name - response = requests.get(project_index_api_url) - if not response.ok: - return packageurls - - project_data = response.json() - versions = project_data.get("versions") or [] - for version in versions: - purl = PackageURL( - type=NPM_TYPE, - name=name, - version=version, - ) - packageurls.append(purl.to_string()) - - return packageurls - - -def yield_npm_package_data(name, packageurls=[]): - for purl in packageurls or get_npm_packageurls(name): - package_url = PackageURL.from_string(purl) - package_data_url = NPM_REGISTRY_REPO + name + "/" + package_url.version - response = requests.get(package_data_url) - if not response.ok: - continue - yield purl, response.json() - - -def load_npm_packages(packages_file): - with open(packages_file) as f: - packages_data = json.load(f) - - return packages_data.get("packages", []) diff --git a/minecode_pipelines/miners/pypi.py b/minecode_pipelines/miners/pypi.py deleted file mode 100644 index 00f6cb6c..00000000 --- a/minecode_pipelines/miners/pypi.py +++ /dev/null @@ -1,107 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import json -import requests - -from packageurl import PackageURL - -""" -Visitors for Pypi and Pypi-like Python package repositories. - -We have this hierarchy in Pypi simple/ index: - pypi projects (JSON/HTML) -> project versions (JSON/HTML) -> download urls - -https://pypi.org/simple/ -Pypi serves a main index via JSON/HTML API that contains a list of package names -and some info on when a package was updated by releasing a new version. -See https://docs.pypi.org/api/index-api/ for more details. -This index also has a list of versions and download URLs of all -uploaded/available package archives and some basic metadata. - -https://pypi.org/pypi/{name}/json -For each package, a JSON contains details including the list of all releases -and archives, their URLs, and some metadata for each release. -For each release, a JSON contains details for the released version and all the -downloads available for this release. -""" - - -pypi_json_headers = {"Accept": "application/vnd.pypi.simple.v1+json"} - - -PYPI_SIMPLE_REPO = "https://pypi.org/simple" -PYPI_METADATA_REPO = "https://pypi.org/pypi" -PYPI_TYPE = "pypi" - - -def get_pypi_packages(pypi_repo, logger=None): - response = requests.get(pypi_repo, headers=pypi_json_headers) - if not response.ok: - return - - return response.json() - - -def get_pypi_package_versions(name): - versions = [] - - project_index_api_url = PYPI_SIMPLE_REPO + "/" + name - response = requests.get(project_index_api_url, headers=pypi_json_headers) - if not response.ok: - return versions - - project_data = response.json() - versions = project_data.get("versions", []) - return versions - - -def get_pypi_packageurls(name): - packageurls = [] - - for version in get_pypi_package_versions(name=name): - purl = PackageURL( - type=PYPI_TYPE, - name=name, - version=version, - ) - packageurls.append(purl.to_string()) - - return packageurls - - -def yield_pypi_package_data(name, packageurls=[]): - for purl in packageurls or get_pypi_packageurls(name): - package_url = PackageURL.from_string(purl) - package_data_url = ( - PYPI_METADATA_REPO + "/" + name + "/" + package_url.version + "/" + "json" - ) - response = requests.get(package_data_url, headers=pypi_json_headers) - if not response.ok: - continue - yield purl, response.json() - - -def load_pypi_packages(packages_file): - with open(packages_file) as f: - packages_data = json.load(f) - - last_serial = packages_data.get("meta").get("_last-serial") - packages = packages_data.get("projects") - - return last_serial, packages - - -def get_last_serial_from_packages(packages_file): - with open(packages_file) as f: - packages_data = json.load(f) - - last_serial = packages_data.get("meta").get("_last-serial") - return last_serial diff --git a/minecode_pipelines/pipes/cpan.py b/minecode_pipelines/pipes/cpan.py index 9f02707b..99cfb193 100644 --- a/minecode_pipelines/pipes/cpan.py +++ b/minecode_pipelines/pipes/cpan.py @@ -20,14 +20,17 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.miners.cpan import get_cpan_packages -from minecode_pipelines.miners.cpan import get_cpan_packageurls -from minecode_pipelines.miners.cpan import CPAN_REPO +import gzip +import requests -from minecode_pipelines.miners.cpan import CPAN_TYPE +from bs4 import BeautifulSoup +from packageurl import PackageURL + +from minecode_pipelines.utils import get_temp_file from minecode_pipelines.utils import grouper -from packageurl import PackageURL +CPAN_REPO = "https://www.cpan.org/" +CPAN_TYPE = "cpan" # If True, show full details on fetching packageURL for # a package name present in the index @@ -36,6 +39,141 @@ PACKAGE_BATCH_SIZE = 500 +def get_cpan_packages(cpan_repo=CPAN_REPO, logger=None): + """ + Get cpan package names parsed from the `02packages.details.txt` + which contains a list of all modules and their respective + package archive paths. We parse the package names and their respective + path_prefixes with author page path from this list. + """ + cpan_packages_url = cpan_repo + "modules/02packages.details.txt.gz" + packages_archive = get_temp_file(file_name="cpan_packages", extension=".gz") + packages_content = get_temp_file(file_name="cpan_packages", extension=".txt") + response = requests.get(cpan_packages_url, stream=True) + with open(packages_archive, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + + with gzip.open(packages_archive, "rb") as f_in: + with open(packages_content, "wb") as f_out: + f_out.writelines(f_in) + + with open(packages_content, encoding="utf-8") as file: + packages_content = file.read() + + package_path_by_name = {} + + # The ``modules/02packages.details.txt`` file has the following section + # at the beginning of the file: + # + # File: 02packages.details.txt + # URL: http://www.cpan.org/modules/02packages.details.txt + # Description: Package names found in directory $CPAN/authors/id/ + # Columns: package name, version, path + # Intended-For: Automated fetch routines, namespace documentation. + # Written-By: PAUSE version 1.005 + # Line-Count: 268940 + # Last-Updated: Mon, 29 Sep 2025 22:29:02 GMT + # + # This information is there in first 10 lines, and the last line is an + # empty line, both of which we are ignoring below + + modules = packages_content.split("\n")[9:-1] + + # A sample line from this module list looks like this: + # + # Crypt::Passphrase::SHA1::Base64 0.021 L/LE/LEONT/Crypt-Passphrase-0.021.tar.gz + + for module in modules: + info = [section for section in module.split(" ") if section] + + # This is like: L/LE/LEONT/Crypt-Passphrase-0.021.tar.gz + package_path = info[-1] + path_segments = package_path.split("/") + filename = path_segments.pop() + path_prefix = "/".join(path_segments) + + name_version = filename.replace(".tar.gz", "").split("-") + _version = name_version.pop() + name = "-".join(name_version) + + # for the above example: name: Crypt-Passphrase, path_prefix: L/LE/LEONT/ + package_path_by_name[name] = path_prefix + + return package_path_by_name + + +def get_cpan_packageurls(name, path_prefix, logger=None): + """ + Given a package name and it's path_prefix (author page path) + return a list of packageURLs for that package. + + An author page (like https://www.cpan.org/authors/id/P/PT/PTC/) lists + all versions of all packages released by the author, so we can scrape + all the packageURLs from this author packages index. + """ + + author_name = path_prefix.split("/")[-1] + + packageurls = [] + + # file extensions found in cpan index + ignorable_extensions = [".meta", ".readme", ".tar.gz"] + + cpan_authors_path = "/authors/id/" + cpan_authors_url = CPAN_REPO + cpan_authors_path + + cpan_author_page_url = cpan_authors_url + path_prefix + + response = requests.get(cpan_author_page_url) + if not response.ok: + return packageurls + + if logger: + logger(f"Getting package versions for {name} from {cpan_author_page_url}") + + soup = BeautifulSoup(response.text, "html.parser") + + # We get all the listed packages in the author page index + package_list = soup.find("ul") + if not package_list: + return packageurls + + package_list_elements = package_list.text.split("\n") + + package_elements = [ + element.replace(" ", "") + for element in package_list_elements + if element and element not in {" Parent Directory", " CHECKSUMS"} + ] + + versions = [] + for package_file in package_elements: + for extension in ignorable_extensions: + if extension in package_file: + package_file = package_file.replace(extension, "") + + name_version = package_file.split("-") + version = name_version.pop() + package_name = "-".join(name_version) + if package_name != name: + continue + + versions.append(version) + + unique_versions = list(set(versions)) + for version in unique_versions: + purl = PackageURL( + type=CPAN_TYPE, + namespace=author_name, + name=name, + version=version, + ) + packageurls.append(purl.to_string()) + + return packageurls + + def mine_cpan_packages(logger=None): if logger: logger("Getting packages from cpan index") diff --git a/minecode_pipelines/pipes/nix.py b/minecode_pipelines/pipes/nix.py index 90d187be..e0a38b1c 100644 --- a/minecode_pipelines/pipes/nix.py +++ b/minecode_pipelines/pipes/nix.py @@ -20,8 +20,15 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import json +import requests +import time from datetime import datetime +import brotli +from packageurl import PackageURL +from scanpipe.pipes.federatedcode import clone_repository +from scanpipe.pipes.federatedcode import delete_local_clone from minecode_pipelines.pipes import fetch_checkpoint_from_github from minecode_pipelines.pipes import update_checkpoints_in_github @@ -36,22 +43,12 @@ from minecode_pipelines.pipes import compress_packages_file from minecode_pipelines.pipes import decompress_packages_file from minecode_pipelines.pipes import fetch_checkpoint_by_git - -from minecode_pipelines.miners.nix import get_all_nix_packages -from minecode_pipelines.miners.nix import get_all_nix_packages_name -from minecode_pipelines.miners.nix import load_nix_packages -from minecode_pipelines.miners.nix import get_nix_packageurls -from minecode_pipelines.miners.nix import yield_nix_package_data -from minecode_pipelines.miners.nix import NIX_TYPE - - from minecode_pipelines.utils import get_temp_dir -from packageurl import PackageURL - -from scanpipe.pipes.federatedcode import clone_repository -from scanpipe.pipes.federatedcode import delete_local_clone - +NIX_TYPE = "nix" +NIXPKGS = "nixpkgs" +DELAY_MULTIPLIER = 2 +MAX_RETRIES = 3 PACKAGE_FILE_NAME = "NixPackages.json" COMPRESSED_PACKAGE_FILE_NAME = "NixPackages.json.gz" @@ -64,6 +61,127 @@ CHANNEL_URL = "https://channels.nixos.org/nixos-unstable/packages.json.br" +def get_all_nix_packages(channel_url, logger=None): + """Get all Nix packages""" + if logger: + logger(f"Fetching Nix packages from: {channel_url}") + response = requests.get(channel_url) + response.raise_for_status() + + try: + decompressed_data = brotli.decompress(response.content) + data = json.loads(decompressed_data) + except brotli.error: + data = response.json() + + packages_dict = data.get("packages", {}) + return packages_dict + + +def get_all_nix_packages_name(packages_dict, logger=None): + """Get all Nix packages name""" + all_package_names = [] + packages = list(packages_dict.keys()) + for attr_path in packages: + all_package_names.append(attr_path) + + return {"packages": all_package_names} + + +def load_nix_packages(packages_file): + with open(packages_file) as f: + packages_data = json.load(f) + + return packages_data.get("packages", []) + + +def get_nix_packageurls(name, packages_dict, logger=None): + packageurls = [] + data = [] + # Get all the version of a package from the following API: + # https://search.devbox.sh/v2/pkg?name={name} + url = f"https://search.devbox.sh/v2/pkg?name={name}" + + for attempt in range(MAX_RETRIES): + try: + response = requests.get(url) + response.raise_for_status() + data = response.json() + break + except requests.HTTPError as e: + if response.status_code == 429: + retry_after = response.headers.get("Retry-After") + if retry_after: + wait_time = int(retry_after) + else: + wait_time = DELAY_MULTIPLIER**attempt + if logger: + logger(f"Rate limited (429). Waiting {wait_time}s before retry...") + time.sleep(wait_time) + continue + elif response.status_code == 404: + # The package_dict contains the package's current version + # devbox.sh may have only index the top + # level, all others deeper level may not be indexed + # TODO: This can only get the current version. + version = get_version_from_package_dict(name, packages_dict) + if version: + purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name, version=version) + else: + purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name) + packageurls.append(purl.to_string()) + return packageurls + else: + if logger: + logger(f"Request failed: {e}") + logger(f"HTTP error {response.status_code}: {response.text}") + return packageurls + except requests.RequestException as e: + if logger: + logger(f"Request failed: {e}") + return packageurls + except ValueError as e: + if logger: + logger(f"JSON parsing failed: {e}") + return packageurls + if not data: + # No data collected after retries + if logger: + logger(f"Max retries reached. Skipping package: {name}") + return packageurls + + releases = data.get("releases", []) + for release in releases: + version = release.get("version") + if version: + purl = PackageURL(type=NIX_TYPE, namespace=NIXPKGS, name=name, version=version) + purl_string = purl.to_string() + if purl_string not in packageurls: + packageurls.append(purl_string) + + return packageurls + + +def get_version_from_package_dict(name, package_dict): + """Get the version data from the package_dict""" + package_info = package_dict.get(name) + if not package_info: + return None + return package_info.get("version") + + +def yield_nix_package_data(name, packageurls=[]): + for purl in packageurls: + package_url = PackageURL.from_string(purl) + package_data_url = ( + f"https://search.devbox.sh/v2/resolve?name={name}&version={package_url.version}" + ) + response = requests.get(package_data_url) + if not response.ok: + continue + yield purl, response.json() + + def mine_nix_packages(logger=None): """ Mine nix package names from nixos.org and save to checkpoints, diff --git a/minecode_pipelines/pipes/npm.py b/minecode_pipelines/pipes/npm.py index e5cbb084..0e02768f 100644 --- a/minecode_pipelines/pipes/npm.py +++ b/minecode_pipelines/pipes/npm.py @@ -20,8 +20,14 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import json from datetime import datetime +import requests +from packageurl import PackageURL +from scanpipe.pipes.federatedcode import clone_repository +from scanpipe.pipes.federatedcode import delete_local_clone + from minecode_pipelines.pipes import fetch_checkpoint_from_github from minecode_pipelines.pipes import update_checkpoints_in_github from minecode_pipelines.pipes import update_checkpoints_file_in_github @@ -35,25 +41,8 @@ from minecode_pipelines.pipes import compress_packages_file from minecode_pipelines.pipes import decompress_packages_file from minecode_pipelines.pipes import fetch_checkpoint_by_git - - -from minecode_pipelines.miners.npm import get_npm_packages -from minecode_pipelines.miners.npm import get_updated_npm_packages -from minecode_pipelines.miners.npm import get_current_last_seq -from minecode_pipelines.miners.npm import load_npm_packages -from minecode_pipelines.miners.npm import get_npm_packageurls -from minecode_pipelines.miners.npm import yield_npm_package_data -from minecode_pipelines.miners.npm import NPM_REPLICATE_REPO -from minecode_pipelines.miners.npm import NPM_TYPE - from minecode_pipelines.utils import get_temp_dir -from packageurl import PackageURL - -from scanpipe.pipes.federatedcode import clone_repository -from scanpipe.pipes.federatedcode import delete_local_clone - - PACKAGE_FILE_NAME = "NPMPackages.json" COMPRESSED_PACKAGE_FILE_NAME = "NPMPackages.json.gz" NPM_REPLICATE_CHECKPOINT_PATH = "npm/" + PACKAGE_FILE_NAME @@ -62,6 +51,157 @@ NPM_PACKAGES_CHECKPOINT_PATH = "npm/packages_checkpoint.json" PACKAGE_BATCH_SIZE = 700 +""" +Visitors for Npmjs and npmjs-like javascript package repositories. + +We have this hierarchy in npm replicate and registry index: + npm projects replicate.npmjs.com (paginated JSON) -> versions at registry.npmjs.org (JSON) -> download urls + +See https://github.com/orgs/community/discussions/152515 for information on +the latest replicate.npmjs.com API. + +https://replicate.npmjs.com/_all_docs +This NPMJS replicate API serves as an index to get all npm packages and their revision IDs +in paginated queries. + +https://replicate.npmjs.com/_changes +This NPMJS replicate API serves as a CHANGELOG of npm packages with update sequences which +can be fetched in paginated queries. + +https://registry.npmjs.org/{namespace/name} +For each npm package, a JSON containing details including the list of all releases +and archives, their URLs, and some metadata for each release. + +https://registry.npmjs.org/{namespace/name}/{version} +For each release, a JSON contains details for the released version and all the +downloads available for this release. +""" + +NPM_REPLICATE_REPO = "https://replicate.npmjs.com/" +NPM_REGISTRY_REPO = "https://registry.npmjs.org/" +NPM_TYPE = "npm" +NPM_REPLICATE_BATCH_SIZE = 10000 + + +def get_package_names_last_key(package_data): + names = [package.get("id") for package in package_data.get("rows")] + last_key = package_data.get("rows")[-1].get("key") + return names, last_key + + +def get_package_names_last_seq(package_data): + names = [package.get("id") for package in package_data.get("results")] + last_seq = package_data.get("last_seq") + return names, last_seq + + +def get_current_last_seq(replicate_url=NPM_REPLICATE_REPO): + npm_replicate_latest_changes = replicate_url + "_changes?descending=True" + response = requests.get(npm_replicate_latest_changes) + if not response.ok: + return + + package_data = response.json() + _package_names, last_seq = get_package_names_last_seq(package_data) + return last_seq + + +def get_updated_npm_packages(last_seq, replicate_url=NPM_REPLICATE_REPO, logger=None): + all_package_names = [] + i = 0 + + while True: + if logger: + logger(f"Processing iteration: {i}: changes after seq: {last_seq}") + + npm_replicate_changes = ( + replicate_url + "_changes?" + f"limit={NPM_REPLICATE_BATCH_SIZE}" + f"&since={last_seq}" + ) + response = requests.get(npm_replicate_changes) + if not response.ok: + return all_package_names + + package_data = response.json() + package_names, last_seq = get_package_names_last_seq(package_data) + all_package_names.extend(package_names) + + # We have fetched the last set of changes if True + if len(package_names) < NPM_REPLICATE_BATCH_SIZE: + break + + i += 1 + + return {"packages": all_package_names}, last_seq + + +def get_npm_packages(replicate_url=NPM_REPLICATE_REPO, logger=None): + all_package_names = [] + + npm_replicate_all = replicate_url + "_all_docs?" + f"limit={NPM_REPLICATE_BATCH_SIZE}" + response = requests.get(npm_replicate_all) + if not response.ok: + return all_package_names + + package_data = response.json() + package_names, last_key = get_package_names_last_key(package_data) + all_package_names.extend(package_names) + + total_rows = package_data.get("total_rows") + iterations = int(total_rows / NPM_REPLICATE_BATCH_SIZE) + 1 + + for i in range(iterations): + npm_replicate_from_id = npm_replicate_all + f'&start_key="{last_key}"' + if logger: + logger(f"Processing iteration: {i}: {npm_replicate_from_id}") + + response = requests.get(npm_replicate_from_id) + if not response.ok: + raise Exception(npm_replicate_from_id, response.text) + + package_data = response.json() + package_names, last_key = get_package_names_last_key(package_data) + all_package_names.extend(package_names) + + return {"packages": all_package_names} + + +def get_npm_packageurls(name, npm_repo=NPM_REGISTRY_REPO): + packageurls = [] + + project_index_api_url = npm_repo + name + response = requests.get(project_index_api_url) + if not response.ok: + return packageurls + + project_data = response.json() + versions = project_data.get("versions") or [] + for version in versions: + purl = PackageURL( + type=NPM_TYPE, + name=name, + version=version, + ) + packageurls.append(purl.to_string()) + + return packageurls + + +def yield_npm_package_data(name, packageurls=[]): + for purl in packageurls or get_npm_packageurls(name): + package_url = PackageURL.from_string(purl) + package_data_url = NPM_REGISTRY_REPO + name + "/" + package_url.version + response = requests.get(package_data_url) + if not response.ok: + continue + yield purl, response.json() + + +def load_npm_packages(packages_file): + with open(packages_file) as f: + packages_data = json.load(f) + + return packages_data.get("packages", []) + def mine_npm_packages(logger=None): """ diff --git a/minecode_pipelines/pipes/pypi.py b/minecode_pipelines/pipes/pypi.py index 756e0ecb..fa5a182d 100644 --- a/minecode_pipelines/pipes/pypi.py +++ b/minecode_pipelines/pipes/pypi.py @@ -20,44 +20,128 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import json from datetime import datetime +import requests +from packageurl import PackageURL +from scanpipe.pipes.federatedcode import clone_repository +from scanpipe.pipes.federatedcode import delete_local_clone + +from minecode_pipelines.pipes import INITIAL_SYNC_STATE +from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO +from minecode_pipelines.pipes import PERIODIC_SYNC_STATE from minecode_pipelines.pipes import fetch_checkpoint_from_github +from minecode_pipelines.pipes import get_mined_packages_from_checkpoint +from minecode_pipelines.pipes import get_packages_file_from_checkpoint from minecode_pipelines.pipes import update_checkpoints_in_github from minecode_pipelines.pipes import update_checkpoint_state -from minecode_pipelines.pipes import get_mined_packages_from_checkpoint from minecode_pipelines.pipes import update_mined_packages_in_checkpoint -from minecode_pipelines.pipes import get_packages_file_from_checkpoint from minecode_pipelines.pipes import write_packages_json -from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO -from minecode_pipelines.pipes import INITIAL_SYNC_STATE -from minecode_pipelines.pipes import PERIODIC_SYNC_STATE - - -from minecode_pipelines.miners.pypi import get_pypi_packages -from minecode_pipelines.miners.pypi import get_pypi_packageurls -from minecode_pipelines.miners.pypi import load_pypi_packages -from minecode_pipelines.miners.pypi import yield_pypi_package_data -from minecode_pipelines.miners.pypi import PYPI_SIMPLE_REPO - - -from minecode_pipelines.miners.pypi import PYPI_TYPE from minecode_pipelines.utils import get_temp_dir -from scanpipe.pipes.federatedcode import delete_local_clone -from packageurl import PackageURL -from scanpipe.pipes.federatedcode import clone_repository - # If True, show full details on fetching packageURL for # a package name present in the index LOG_PACKAGEURL_DETAILS = False - PACKAGE_FILE_NAME = "PypiPackages.json" PYPI_SIMPLE_CHECKPOINT_PATH = "pypi/simple_index/" + PACKAGE_FILE_NAME PYPI_CHECKPOINT_PATH = "pypi/checkpoints.json" PYPI_PACKAGES_CHECKPOINT_PATH = "pypi/packages_checkpoint.json" +""" +Visitors for Pypi and Pypi-like Python package repositories. + +We have this hierarchy in Pypi simple/ index: + pypi projects (JSON/HTML) -> project versions (JSON/HTML) -> download urls + +https://pypi.org/simple/ +Pypi serves a main index via JSON/HTML API that contains a list of package names +and some info on when a package was updated by releasing a new version. +See https://docs.pypi.org/api/index-api/ for more details. +This index also has a list of versions and download URLs of all +uploaded/available package archives and some basic metadata. + +https://pypi.org/pypi/{name}/json +For each package, a JSON contains details including the list of all releases +and archives, their URLs, and some metadata for each release. +For each release, a JSON contains details for the released version and all the +downloads available for this release. +""" + + +pypi_json_headers = {"Accept": "application/vnd.pypi.simple.v1+json"} + + +PYPI_SIMPLE_REPO = "https://pypi.org/simple" +PYPI_METADATA_REPO = "https://pypi.org/pypi" +PYPI_TYPE = "pypi" + + +def get_pypi_packages(pypi_repo, logger=None): + response = requests.get(pypi_repo, headers=pypi_json_headers) + if not response.ok: + return + + return response.json() + + +def get_pypi_package_versions(name): + versions = [] + + project_index_api_url = PYPI_SIMPLE_REPO + "/" + name + response = requests.get(project_index_api_url, headers=pypi_json_headers) + if not response.ok: + return versions + + project_data = response.json() + versions = project_data.get("versions", []) + return versions + + +def get_pypi_packageurls(name): + packageurls = [] + + for version in get_pypi_package_versions(name=name): + purl = PackageURL( + type=PYPI_TYPE, + name=name, + version=version, + ) + packageurls.append(purl.to_string()) + + return packageurls + + +def yield_pypi_package_data(name, packageurls=[]): + for purl in packageurls or get_pypi_packageurls(name): + package_url = PackageURL.from_string(purl) + package_data_url = ( + PYPI_METADATA_REPO + "/" + name + "/" + package_url.version + "/" + "json" + ) + response = requests.get(package_data_url, headers=pypi_json_headers) + if not response.ok: + continue + yield purl, response.json() + + +def load_pypi_packages(packages_file): + with open(packages_file) as f: + packages_data = json.load(f) + + last_serial = packages_data.get("meta").get("_last-serial") + packages = packages_data.get("projects") + + return last_serial, packages + + +def get_last_serial_from_packages(packages_file): + with open(packages_file) as f: + packages_data = json.load(f) + + last_serial = packages_data.get("meta").get("_last-serial") + return last_serial + def mine_pypi_packages(logger=None): """ From c9d8bea614210d89c1e457b26801aaa2d5a22b17 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 31 Jul 2026 14:40:12 -0700 Subject: [PATCH 2/3] Merge minecode_pipelines into minecode * Update pyproject-minecode_pipelines.toml Signed-off-by: Jono Yang --- ...EADME.rst => README-minecode_pipelines.rst | 0 .../management/commands/federate_packages.py | 2 +- .../management/commands/federate_sboms.py | 2 +- .../pipelines/__init__.py | 6 +- .../pipelines/mine_alpine.py | 4 +- .../pipelines/mine_apache.py | 6 +- .../pipelines/mine_cargo.py | 4 +- .../pipelines/mine_composer.py | 10 +-- .../pipelines/mine_conan.py | 4 +- .../pipelines/mine_cpan.py | 4 +- .../pipelines/mine_cran.py | 6 +- .../pipelines/mine_crates.py | 8 +-- .../pipelines/mine_debian.py | 8 +-- .../pipelines/mine_maven.py | 8 +-- .../pipelines/mine_nix.py | 6 +- .../pipelines/mine_npm.py | 6 +- .../pipelines/mine_nuget.py | 4 +- .../pipelines/mine_pypi.py | 6 +- .../pipelines/mine_swift.py | 6 +- .../pipes/__init__.py | 2 +- .../pipes/alpine.py | 2 +- .../pipes/apache.py | 8 +-- .../pipes/cargo.py | 0 .../pipes/composer.py | 4 +- .../pipes/conan.py | 0 .../pipes/cpan.py | 4 +- .../pipes/cran.py | 0 .../pipes/crates.py | 0 .../pipes/debian.py | 2 +- .../pipes/java_stream.py | 0 .../pipes/java_stream.py.ABOUT | 0 .../pipes/java_stream.py.LICENSE | 0 {minecode_pipelines => minecode}/pipes/ls.py | 0 .../pipes/maven.py | 2 +- {minecode_pipelines => minecode}/pipes/nix.py | 28 ++++---- {minecode_pipelines => minecode}/pipes/npm.py | 28 ++++---- .../pipes/nuget.py | 0 .../pipes/pypi.py | 22 +++--- .../pipes/swift.py | 0 .../tests/data/alpine/APKINDEX | 0 .../tests/data/alpine/expected_packages.json | 0 .../tests/data/cargo/c5store | 0 .../tests/data/cargo/c5store-expected.yaml | 0 .../tests/data/composer/expected_output.json | 0 .../tests/data/composer/package_details.json | 0 .../tests/data/composer/packages_list.json | 0 .../tests/data/conan/cairo/cairo-config.yml | 0 .../tests/data/conan/expected-cairo-purls.yml | 0 .../tests/data/cran/cran_db.json | 0 .../tests/data/cran/expected_abbreviate.yaml | 0 .../tests/data/cran/expected_abc.data.yaml | 0 .../tests/data/cran/expected_abc.yaml | 0 .../tests/data/directories/find-ls | 0 .../data/directories/find-ls-apache-start | 0 .../find-ls-apache-start-expected.json | 0 .../data/directories/find-ls-expected.json | 0 .../tests/data/directories/ls-lr | 0 .../data/directories/ls-lr-expected.json | 0 .../tests/data/directories/ls-lr-ubuntu | 0 .../directories/ls-lr-ubuntu-expected.json | 0 .../buggy/expected_artifacts-defaults.json | 0 .../maven/index/buggy/expected_artifacts.json | 0 .../maven/index/buggy/expected_entries.json | 0 .../data/maven/index/buggy/expected_uris.json | 0 .../index/buggy/expected_visited_uris.json | 0 .../buggy/nexus-maven-repository-index.gz | Bin .../index/expected_artifacts-all-worthy.json | 0 .../index/expected_artifacts-defaults.json | 0 .../data/maven/index/expected_artifacts.json | 0 .../data/maven/index/expected_entries.json | 0 .../tests/data/maven/index/expected_uris.json | 0 .../expected_artifacts-defaults.json | 0 .../index/increment/expected_artifacts.json | 0 .../index/increment/expected_entries.json | 0 .../increment/expected_properties_uris.json | 0 .../maven/index/increment/expected_uris.json | 0 .../nexus-maven-repository-index.445.gz | Bin .../nexus-maven-repository-index.properties | 0 .../increment2/expected_mini_package.json | 0 .../maven/index/increment2/expected_uris.json | 0 .../nexus-maven-repository-index.457.gz | Bin .../index/nexus-maven-repository-index.gz | Bin .../data/nuget/catalog/pages/page10897.json | 0 .../tests/data/swift/commits_tags1.txt | 0 .../tests/data/swift/commits_tags2.txt | 0 .../tests/data/swift/commits_tags3.txt | 0 .../tests/data/swift/commits_tags4.txt | 0 .../tests/data/swift/expected1.yaml | 0 .../tests/data/swift/expected2.yaml | 0 .../tests/data/swift/expected3.yaml | 0 .../tests/data/swift/expected4.yaml | 0 .../tests/pipes/__init__.py | 0 .../tests/pipes/test_alpine.py | 2 +- .../tests/pipes/test_apache.py | 2 +- .../tests/pipes/test_cargo.py | 2 +- .../tests/pipes/test_composer.py | 6 +- .../tests/pipes/test_conan.py | 2 +- .../tests/pipes/test_cran.py | 2 +- .../tests/pipes/test_ls.py | 2 +- .../tests/pipes/test_maven.py | 2 +- .../tests/pipes/test_nuget.py | 4 +- .../tests/pipes/test_pipes.py | 2 +- .../tests/pipes/test_pypi.py | 0 .../tests/pipes/test_swift.py | 2 +- minecode/utils.py | 17 +++++ minecode_pipelines/__init__.py | 11 --- minecode_pipelines/tests/__init__.py | 8 --- minecode_pipelines/utils.py | 67 ------------------ pyproject-minecode_pipelines.toml | 41 +++++------ setup.cfg | 16 +++++ 110 files changed, 169 insertions(+), 221 deletions(-) rename minecode_pipelines/README.rst => README-minecode_pipelines.rst (100%) rename {minecode_pipelines => minecode}/pipelines/__init__.py (98%) rename {minecode_pipelines => minecode}/pipelines/mine_alpine.py (94%) rename {minecode_pipelines => minecode}/pipelines/mine_apache.py (94%) rename {minecode_pipelines => minecode}/pipelines/mine_cargo.py (96%) rename {minecode_pipelines => minecode}/pipelines/mine_composer.py (92%) rename {minecode_pipelines => minecode}/pipelines/mine_conan.py (96%) rename {minecode_pipelines => minecode}/pipelines/mine_cpan.py (95%) rename {minecode_pipelines => minecode}/pipelines/mine_cran.py (93%) rename {minecode_pipelines => minecode}/pipelines/mine_crates.py (95%) rename {minecode_pipelines => minecode}/pipelines/mine_debian.py (94%) rename {minecode_pipelines => minecode}/pipelines/mine_maven.py (98%) rename {minecode_pipelines => minecode}/pipelines/mine_nix.py (95%) rename {minecode_pipelines => minecode}/pipelines/mine_npm.py (95%) rename {minecode_pipelines => minecode}/pipelines/mine_nuget.py (96%) rename {minecode_pipelines => minecode}/pipelines/mine_pypi.py (95%) rename {minecode_pipelines => minecode}/pipelines/mine_swift.py (92%) rename {minecode_pipelines => minecode}/pipes/__init__.py (99%) rename {minecode_pipelines => minecode}/pipes/alpine.py (99%) rename {minecode_pipelines => minecode}/pipes/apache.py (98%) rename {minecode_pipelines => minecode}/pipes/cargo.py (100%) rename {minecode_pipelines => minecode}/pipes/composer.py (97%) rename {minecode_pipelines => minecode}/pipes/conan.py (100%) rename {minecode_pipelines => minecode}/pipes/cpan.py (98%) rename {minecode_pipelines => minecode}/pipes/cran.py (100%) rename {minecode_pipelines => minecode}/pipes/crates.py (100%) rename {minecode_pipelines => minecode}/pipes/debian.py (99%) rename {minecode_pipelines => minecode}/pipes/java_stream.py (100%) rename {minecode_pipelines => minecode}/pipes/java_stream.py.ABOUT (100%) rename {minecode_pipelines => minecode}/pipes/java_stream.py.LICENSE (100%) rename {minecode_pipelines => minecode}/pipes/ls.py (100%) rename {minecode_pipelines => minecode}/pipes/maven.py (99%) rename {minecode_pipelines => minecode}/pipes/nix.py (93%) rename {minecode_pipelines => minecode}/pipes/npm.py (94%) rename {minecode_pipelines => minecode}/pipes/nuget.py (100%) rename {minecode_pipelines => minecode}/pipes/pypi.py (94%) rename {minecode_pipelines => minecode}/pipes/swift.py (100%) rename {minecode_pipelines => minecode}/tests/data/alpine/APKINDEX (100%) rename {minecode_pipelines => minecode}/tests/data/alpine/expected_packages.json (100%) rename {minecode_pipelines => minecode}/tests/data/cargo/c5store (100%) rename {minecode_pipelines => minecode}/tests/data/cargo/c5store-expected.yaml (100%) rename {minecode_pipelines => minecode}/tests/data/composer/expected_output.json (100%) rename {minecode_pipelines => minecode}/tests/data/composer/package_details.json (100%) rename {minecode_pipelines => minecode}/tests/data/composer/packages_list.json (100%) rename {minecode_pipelines => minecode}/tests/data/conan/cairo/cairo-config.yml (100%) rename {minecode_pipelines => minecode}/tests/data/conan/expected-cairo-purls.yml (100%) rename {minecode_pipelines => minecode}/tests/data/cran/cran_db.json (100%) rename {minecode_pipelines => minecode}/tests/data/cran/expected_abbreviate.yaml (100%) rename {minecode_pipelines => minecode}/tests/data/cran/expected_abc.data.yaml (100%) rename {minecode_pipelines => minecode}/tests/data/cran/expected_abc.yaml (100%) rename {minecode_pipelines => minecode}/tests/data/directories/find-ls (100%) rename {minecode_pipelines => minecode}/tests/data/directories/find-ls-apache-start (100%) rename {minecode_pipelines => minecode}/tests/data/directories/find-ls-apache-start-expected.json (100%) rename {minecode_pipelines => minecode}/tests/data/directories/find-ls-expected.json (100%) rename {minecode_pipelines => minecode}/tests/data/directories/ls-lr (100%) rename {minecode_pipelines => minecode}/tests/data/directories/ls-lr-expected.json (100%) rename {minecode_pipelines => minecode}/tests/data/directories/ls-lr-ubuntu (100%) rename {minecode_pipelines => minecode}/tests/data/directories/ls-lr-ubuntu-expected.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/buggy/expected_artifacts-defaults.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/buggy/expected_artifacts.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/buggy/expected_entries.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/buggy/expected_uris.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/buggy/expected_visited_uris.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/buggy/nexus-maven-repository-index.gz (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/expected_artifacts-all-worthy.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/expected_artifacts-defaults.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/expected_artifacts.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/expected_entries.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/expected_uris.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/increment/expected_artifacts-defaults.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/increment/expected_artifacts.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/increment/expected_entries.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/increment/expected_properties_uris.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/increment/expected_uris.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/increment/nexus-maven-repository-index.445.gz (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/increment/nexus-maven-repository-index.properties (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/increment2/expected_mini_package.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/increment2/expected_uris.json (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/increment2/nexus-maven-repository-index.457.gz (100%) rename {minecode_pipelines => minecode}/tests/data/maven/index/nexus-maven-repository-index.gz (100%) rename {minecode_pipelines => minecode}/tests/data/nuget/catalog/pages/page10897.json (100%) rename {minecode_pipelines => minecode}/tests/data/swift/commits_tags1.txt (100%) rename {minecode_pipelines => minecode}/tests/data/swift/commits_tags2.txt (100%) rename {minecode_pipelines => minecode}/tests/data/swift/commits_tags3.txt (100%) rename {minecode_pipelines => minecode}/tests/data/swift/commits_tags4.txt (100%) rename {minecode_pipelines => minecode}/tests/data/swift/expected1.yaml (100%) rename {minecode_pipelines => minecode}/tests/data/swift/expected2.yaml (100%) rename {minecode_pipelines => minecode}/tests/data/swift/expected3.yaml (100%) rename {minecode_pipelines => minecode}/tests/data/swift/expected4.yaml (100%) rename {minecode_pipelines => minecode}/tests/pipes/__init__.py (100%) rename {minecode_pipelines => minecode}/tests/pipes/test_alpine.py (96%) rename {minecode_pipelines => minecode}/tests/pipes/test_apache.py (99%) rename {minecode_pipelines => minecode}/tests/pipes/test_cargo.py (94%) rename {minecode_pipelines => minecode}/tests/pipes/test_composer.py (90%) rename {minecode_pipelines => minecode}/tests/pipes/test_conan.py (94%) rename {minecode_pipelines => minecode}/tests/pipes/test_cran.py (96%) rename {minecode_pipelines => minecode}/tests/pipes/test_ls.py (98%) rename {minecode_pipelines => minecode}/tests/pipes/test_maven.py (99%) rename {minecode_pipelines => minecode}/tests/pipes/test_nuget.py (94%) rename {minecode_pipelines => minecode}/tests/pipes/test_pipes.py (97%) rename {minecode_pipelines => minecode}/tests/pipes/test_pypi.py (100%) rename {minecode_pipelines => minecode}/tests/pipes/test_swift.py (98%) delete mode 100644 minecode_pipelines/__init__.py delete mode 100644 minecode_pipelines/tests/__init__.py delete mode 100644 minecode_pipelines/utils.py diff --git a/minecode_pipelines/README.rst b/README-minecode_pipelines.rst similarity index 100% rename from minecode_pipelines/README.rst rename to README-minecode_pipelines.rst diff --git a/minecode/management/commands/federate_packages.py b/minecode/management/commands/federate_packages.py index f9567f50..41ed5b7f 100644 --- a/minecode/management/commands/federate_packages.py +++ b/minecode/management/commands/federate_packages.py @@ -15,7 +15,7 @@ from commoncode import fileutils from minecode.management import federatedcode from minecode.management.commands import VerboseCommand -from minecode_pipelines import pipes +from minecode import pipes from packagedb import models as packagedb_models diff --git a/minecode/management/commands/federate_sboms.py b/minecode/management/commands/federate_sboms.py index 3cb18916..9886629a 100644 --- a/minecode/management/commands/federate_sboms.py +++ b/minecode/management/commands/federate_sboms.py @@ -15,7 +15,7 @@ from commoncode import fileutils from minecode.management import federatedcode from minecode.management.commands import VerboseCommand -from minecode_pipelines import pipes +from minecode import pipes from packagedb import models as packagedb_models from packagedb import sbom diff --git a/minecode_pipelines/pipelines/__init__.py b/minecode/pipelines/__init__.py similarity index 98% rename from minecode_pipelines/pipelines/__init__.py rename to minecode/pipelines/__init__.py index f1b5b1f9..15ea3ab3 100644 --- a/minecode_pipelines/pipelines/__init__.py +++ b/minecode/pipelines/__init__.py @@ -21,9 +21,9 @@ from scanpipe.pipelines import Pipeline from scanpipe.pipes import federatedcode -from minecode_pipelines import pipes -from minecode_pipelines.pipes import write_package_data_to_file -from minecode_pipelines.pipes import write_packageurls_to_file +from minecode import pipes +from minecode.pipes import write_package_data_to_file +from minecode.pipes import write_packageurls_to_file module_logger = logging.getLogger(__name__) diff --git a/minecode_pipelines/pipelines/mine_alpine.py b/minecode/pipelines/mine_alpine.py similarity index 94% rename from minecode_pipelines/pipelines/mine_alpine.py rename to minecode/pipelines/mine_alpine.py index 51bd8749..f71cdf91 100644 --- a/minecode_pipelines/pipelines/mine_alpine.py +++ b/minecode/pipelines/mine_alpine.py @@ -20,9 +20,9 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipelines import MineCodeBasePipeline +from minecode.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipes import alpine +from minecode.pipes import alpine class MineAlpine(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_apache.py b/minecode/pipelines/mine_apache.py similarity index 94% rename from minecode_pipelines/pipelines/mine_apache.py rename to minecode/pipelines/mine_apache.py index f51dda33..b5ff9191 100644 --- a/minecode_pipelines/pipelines/mine_apache.py +++ b/minecode/pipelines/mine_apache.py @@ -21,9 +21,9 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipes import apache -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode.pipes import apache +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls class MineApache(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_cargo.py b/minecode/pipelines/mine_cargo.py similarity index 96% rename from minecode_pipelines/pipelines/mine_cargo.py rename to minecode/pipelines/mine_cargo.py index fa556974..ebbb8582 100644 --- a/minecode_pipelines/pipelines/mine_cargo.py +++ b/minecode/pipelines/mine_cargo.py @@ -22,8 +22,8 @@ from pathlib import Path -from minecode_pipelines.pipes import cargo -from minecode_pipelines.pipelines import MineCodeBasePipeline +from minecode.pipes import cargo +from minecode.pipelines import MineCodeBasePipeline from scanpipe.pipes import federatedcode diff --git a/minecode_pipelines/pipelines/mine_composer.py b/minecode/pipelines/mine_composer.py similarity index 92% rename from minecode_pipelines/pipelines/mine_composer.py rename to minecode/pipelines/mine_composer.py index 5776d3aa..53b12630 100644 --- a/minecode_pipelines/pipelines/mine_composer.py +++ b/minecode/pipelines/mine_composer.py @@ -23,11 +23,11 @@ from datetime import datetime from scanpipe.pipes import federatedcode -from minecode_pipelines import pipes -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipes import composer -from minecode_pipelines.pipes.composer import mine_composer_packages, PACKAGE_BATCH_SIZE -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode import pipes +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipes import composer +from minecode.pipes.composer import mine_composer_packages, PACKAGE_BATCH_SIZE +from minecode.pipelines import _mine_and_publish_packageurls class MineComposer(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_conan.py b/minecode/pipelines/mine_conan.py similarity index 96% rename from minecode_pipelines/pipelines/mine_conan.py rename to minecode/pipelines/mine_conan.py index 1b94b749..619bb5ce 100644 --- a/minecode_pipelines/pipelines/mine_conan.py +++ b/minecode/pipelines/mine_conan.py @@ -21,8 +21,8 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. from pathlib import Path -from minecode_pipelines.pipes import conan -from minecode_pipelines.pipelines import MineCodeBasePipeline +from minecode.pipes import conan +from minecode.pipelines import MineCodeBasePipeline from scanpipe.pipes import federatedcode diff --git a/minecode_pipelines/pipelines/mine_cpan.py b/minecode/pipelines/mine_cpan.py similarity index 95% rename from minecode_pipelines/pipelines/mine_cpan.py rename to minecode/pipelines/mine_cpan.py index 813f090c..a9bda898 100644 --- a/minecode_pipelines/pipelines/mine_cpan.py +++ b/minecode/pipelines/mine_cpan.py @@ -21,8 +21,8 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipes import cpan -from minecode_pipelines.pipelines import MineCodeBasePipeline +from minecode.pipes import cpan +from minecode.pipelines import MineCodeBasePipeline class MineCpan(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_cran.py b/minecode/pipelines/mine_cran.py similarity index 93% rename from minecode_pipelines/pipelines/mine_cran.py rename to minecode/pipelines/mine_cran.py index 4de4a866..62779e51 100644 --- a/minecode_pipelines/pipelines/mine_cran.py +++ b/minecode/pipelines/mine_cran.py @@ -21,9 +21,9 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. import json -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipes import cran -from minecode_pipelines.pipes.cran import fetch_cran_db +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipes import cran +from minecode.pipes.cran import fetch_cran_db class MineCran(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_crates.py b/minecode/pipelines/mine_crates.py similarity index 95% rename from minecode_pipelines/pipelines/mine_crates.py rename to minecode/pipelines/mine_crates.py index 16a1e6bc..75ccdb80 100644 --- a/minecode_pipelines/pipelines/mine_crates.py +++ b/minecode/pipelines/mine_crates.py @@ -22,10 +22,10 @@ from scanpipe.pipes import federatedcode -from minecode_pipelines import pipes -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls -from minecode_pipelines.pipes import crates +from minecode import pipes +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls +from minecode.pipes import crates class MineCrates(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_debian.py b/minecode/pipelines/mine_debian.py similarity index 94% rename from minecode_pipelines/pipelines/mine_debian.py rename to minecode/pipelines/mine_debian.py index 54fb6969..e63af444 100644 --- a/minecode_pipelines/pipelines/mine_debian.py +++ b/minecode/pipelines/mine_debian.py @@ -22,10 +22,10 @@ from scanpipe.pipes import federatedcode -from minecode_pipelines import pipes -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls -from minecode_pipelines.pipes import debian +from minecode import pipes +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls +from minecode.pipes import debian class MineDebian(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_maven.py b/minecode/pipelines/mine_maven.py similarity index 98% rename from minecode_pipelines/pipelines/mine_maven.py rename to minecode/pipelines/mine_maven.py index da4bd4d2..dcb58ec6 100644 --- a/minecode_pipelines/pipelines/mine_maven.py +++ b/minecode/pipelines/mine_maven.py @@ -23,10 +23,10 @@ from aboutcode.pipeline import optional_step from scanpipe.pipes import federatedcode -from minecode_pipelines import pipes -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls -from minecode_pipelines.pipes import maven +from minecode import pipes +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls +from minecode.pipes import maven class MineMaven(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_nix.py b/minecode/pipelines/mine_nix.py similarity index 95% rename from minecode_pipelines/pipelines/mine_nix.py rename to minecode/pipelines/mine_nix.py index 4ebe5930..6429ecf9 100644 --- a/minecode_pipelines/pipelines/mine_nix.py +++ b/minecode/pipelines/mine_nix.py @@ -21,9 +21,9 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipes import nix -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode.pipes import nix +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls class MineNix(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_npm.py b/minecode/pipelines/mine_npm.py similarity index 95% rename from minecode_pipelines/pipelines/mine_npm.py rename to minecode/pipelines/mine_npm.py index 273b939c..57f49d49 100644 --- a/minecode_pipelines/pipelines/mine_npm.py +++ b/minecode/pipelines/mine_npm.py @@ -20,9 +20,9 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipes import npm -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode.pipes import npm +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls class MineNPM(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_nuget.py b/minecode/pipelines/mine_nuget.py similarity index 96% rename from minecode_pipelines/pipelines/mine_nuget.py rename to minecode/pipelines/mine_nuget.py index f3083e6e..61db7c79 100644 --- a/minecode_pipelines/pipelines/mine_nuget.py +++ b/minecode/pipelines/mine_nuget.py @@ -23,9 +23,9 @@ from pathlib import Path -from minecode_pipelines.pipes import nuget +from minecode.pipes import nuget -from minecode_pipelines.pipelines import MineCodeBasePipeline +from minecode.pipelines import MineCodeBasePipeline from scanpipe.pipes import federatedcode diff --git a/minecode_pipelines/pipelines/mine_pypi.py b/minecode/pipelines/mine_pypi.py similarity index 95% rename from minecode_pipelines/pipelines/mine_pypi.py rename to minecode/pipelines/mine_pypi.py index 6fd87972..08112c34 100644 --- a/minecode_pipelines/pipelines/mine_pypi.py +++ b/minecode/pipelines/mine_pypi.py @@ -20,9 +20,9 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from minecode_pipelines.pipes import pypi -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode.pipes import pypi +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipelines import _mine_and_publish_packageurls class MinePypi(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipelines/mine_swift.py b/minecode/pipelines/mine_swift.py similarity index 92% rename from minecode_pipelines/pipelines/mine_swift.py rename to minecode/pipelines/mine_swift.py index 50f789db..d0c60b59 100644 --- a/minecode_pipelines/pipelines/mine_swift.py +++ b/minecode/pipelines/mine_swift.py @@ -22,9 +22,9 @@ from scanpipe.pipes import federatedcode -from minecode_pipelines.pipelines import MineCodeBasePipeline -from minecode_pipelines.pipes.swift import mine_swift_packageurls -from minecode_pipelines.pipes.swift import load_swift_package_urls +from minecode.pipelines import MineCodeBasePipeline +from minecode.pipes.swift import mine_swift_packageurls +from minecode.pipes.swift import load_swift_package_urls class MineSwift(MineCodeBasePipeline): diff --git a/minecode_pipelines/pipes/__init__.py b/minecode/pipes/__init__.py similarity index 99% rename from minecode_pipelines/pipes/__init__.py rename to minecode/pipes/__init__.py index 4317f6a9..38832771 100644 --- a/minecode_pipelines/pipes/__init__.py +++ b/minecode/pipes/__init__.py @@ -16,7 +16,7 @@ import requests import saneyaml -from minecode_pipelines.utils import get_temp_file +from minecode.utils import get_temp_file # states: # note: a state is null when mining starts diff --git a/minecode_pipelines/pipes/alpine.py b/minecode/pipes/alpine.py similarity index 99% rename from minecode_pipelines/pipes/alpine.py rename to minecode/pipes/alpine.py index 9899e1d1..fbbaa9c1 100644 --- a/minecode_pipelines/pipes/alpine.py +++ b/minecode/pipes/alpine.py @@ -30,7 +30,7 @@ from scanpipe.pipes.fetch import fetch_http from scanpipe.pipes.scancode import extract_archives -from minecode_pipelines.pipelines import _mine_and_publish_packageurls +from minecode.pipelines import _mine_and_publish_packageurls ALPINE_CHECKPOINT_PATH = "alpine/checkpoints.json" diff --git a/minecode_pipelines/pipes/apache.py b/minecode/pipes/apache.py similarity index 98% rename from minecode_pipelines/pipes/apache.py rename to minecode/pipes/apache.py index 6357fde8..9897ac4d 100644 --- a/minecode_pipelines/pipes/apache.py +++ b/minecode/pipes/apache.py @@ -9,11 +9,11 @@ from datetime import datetime, timezone -from minecode_pipelines.pipes import fetch_checkpoint_from_github -from minecode_pipelines.pipes import update_checkpoints_in_github -from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO +from minecode.pipes import fetch_checkpoint_from_github +from minecode.pipes import update_checkpoints_in_github +from minecode.pipes import MINECODE_PIPELINES_CONFIG_REPO -from minecode_pipelines.utils import get_temp_dir +from minecode.utils import get_temp_dir from packageurl import PackageURL diff --git a/minecode_pipelines/pipes/cargo.py b/minecode/pipes/cargo.py similarity index 100% rename from minecode_pipelines/pipes/cargo.py rename to minecode/pipes/cargo.py diff --git a/minecode_pipelines/pipes/composer.py b/minecode/pipes/composer.py similarity index 97% rename from minecode_pipelines/pipes/composer.py rename to minecode/pipes/composer.py index 06f50f86..e878e8bc 100644 --- a/minecode_pipelines/pipes/composer.py +++ b/minecode/pipes/composer.py @@ -21,11 +21,11 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. import json -from minecode_pipelines.utils import get_temp_file +from minecode.utils import get_temp_file from aboutcode.hashid import get_core_purl import requests from packageurl import PackageURL -from minecode_pipelines.utils import cycle_from_index, grouper +from minecode.utils import cycle_from_index, grouper PACKAGE_BATCH_SIZE = 100 diff --git a/minecode_pipelines/pipes/conan.py b/minecode/pipes/conan.py similarity index 100% rename from minecode_pipelines/pipes/conan.py rename to minecode/pipes/conan.py diff --git a/minecode_pipelines/pipes/cpan.py b/minecode/pipes/cpan.py similarity index 98% rename from minecode_pipelines/pipes/cpan.py rename to minecode/pipes/cpan.py index 99cfb193..1c4d35b8 100644 --- a/minecode_pipelines/pipes/cpan.py +++ b/minecode/pipes/cpan.py @@ -26,8 +26,8 @@ from bs4 import BeautifulSoup from packageurl import PackageURL -from minecode_pipelines.utils import get_temp_file -from minecode_pipelines.utils import grouper +from minecode.utils import get_temp_file +from minecode.utils import grouper CPAN_REPO = "https://www.cpan.org/" CPAN_TYPE = "cpan" diff --git a/minecode_pipelines/pipes/cran.py b/minecode/pipes/cran.py similarity index 100% rename from minecode_pipelines/pipes/cran.py rename to minecode/pipes/cran.py diff --git a/minecode_pipelines/pipes/crates.py b/minecode/pipes/crates.py similarity index 100% rename from minecode_pipelines/pipes/crates.py rename to minecode/pipes/crates.py diff --git a/minecode_pipelines/pipes/debian.py b/minecode/pipes/debian.py similarity index 99% rename from minecode_pipelines/pipes/debian.py rename to minecode/pipes/debian.py index d2ccc665..7c010d6b 100644 --- a/minecode_pipelines/pipes/debian.py +++ b/minecode/pipes/debian.py @@ -32,7 +32,7 @@ from packageurl import PackageURL from scanpipe.pipes.fetch import fetch_http -from minecode_pipelines.pipes import ls +from minecode.pipes import ls DEBIAN_LSLR_URL = "http://ftp.debian.org/debian/ls-lR.gz" diff --git a/minecode_pipelines/pipes/java_stream.py b/minecode/pipes/java_stream.py similarity index 100% rename from minecode_pipelines/pipes/java_stream.py rename to minecode/pipes/java_stream.py diff --git a/minecode_pipelines/pipes/java_stream.py.ABOUT b/minecode/pipes/java_stream.py.ABOUT similarity index 100% rename from minecode_pipelines/pipes/java_stream.py.ABOUT rename to minecode/pipes/java_stream.py.ABOUT diff --git a/minecode_pipelines/pipes/java_stream.py.LICENSE b/minecode/pipes/java_stream.py.LICENSE similarity index 100% rename from minecode_pipelines/pipes/java_stream.py.LICENSE rename to minecode/pipes/java_stream.py.LICENSE diff --git a/minecode_pipelines/pipes/ls.py b/minecode/pipes/ls.py similarity index 100% rename from minecode_pipelines/pipes/ls.py rename to minecode/pipes/ls.py diff --git a/minecode_pipelines/pipes/maven.py b/minecode/pipes/maven.py similarity index 99% rename from minecode_pipelines/pipes/maven.py rename to minecode/pipes/maven.py index 30c4991f..062b64f0 100644 --- a/minecode_pipelines/pipes/maven.py +++ b/minecode/pipes/maven.py @@ -27,7 +27,7 @@ from packageurl import PackageURL -from minecode_pipelines.pipes import java_stream +from minecode.pipes import java_stream TRACE = False TRACE_DEEP = False diff --git a/minecode_pipelines/pipes/nix.py b/minecode/pipes/nix.py similarity index 93% rename from minecode_pipelines/pipes/nix.py rename to minecode/pipes/nix.py index e0a38b1c..12eb5f65 100644 --- a/minecode_pipelines/pipes/nix.py +++ b/minecode/pipes/nix.py @@ -30,20 +30,20 @@ from scanpipe.pipes.federatedcode import clone_repository from scanpipe.pipes.federatedcode import delete_local_clone -from minecode_pipelines.pipes import fetch_checkpoint_from_github -from minecode_pipelines.pipes import update_checkpoints_in_github -from minecode_pipelines.pipes import update_checkpoints_file_in_github -from minecode_pipelines.pipes import get_mined_packages_from_checkpoint -from minecode_pipelines.pipes import update_mined_packages_in_checkpoint -from minecode_pipelines.pipes import update_checkpoint_state -from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO -from minecode_pipelines.pipes import INITIAL_SYNC_STATE -from minecode_pipelines.pipes import PERIODIC_SYNC_STATE -from minecode_pipelines.pipes import write_packages_json -from minecode_pipelines.pipes import compress_packages_file -from minecode_pipelines.pipes import decompress_packages_file -from minecode_pipelines.pipes import fetch_checkpoint_by_git -from minecode_pipelines.utils import get_temp_dir +from minecode.pipes import fetch_checkpoint_from_github +from minecode.pipes import update_checkpoints_in_github +from minecode.pipes import update_checkpoints_file_in_github +from minecode.pipes import get_mined_packages_from_checkpoint +from minecode.pipes import update_mined_packages_in_checkpoint +from minecode.pipes import update_checkpoint_state +from minecode.pipes import MINECODE_PIPELINES_CONFIG_REPO +from minecode.pipes import INITIAL_SYNC_STATE +from minecode.pipes import PERIODIC_SYNC_STATE +from minecode.pipes import write_packages_json +from minecode.pipes import compress_packages_file +from minecode.pipes import decompress_packages_file +from minecode.pipes import fetch_checkpoint_by_git +from minecode.utils import get_temp_dir NIX_TYPE = "nix" NIXPKGS = "nixpkgs" diff --git a/minecode_pipelines/pipes/npm.py b/minecode/pipes/npm.py similarity index 94% rename from minecode_pipelines/pipes/npm.py rename to minecode/pipes/npm.py index 0e02768f..a5ac5978 100644 --- a/minecode_pipelines/pipes/npm.py +++ b/minecode/pipes/npm.py @@ -28,20 +28,20 @@ from scanpipe.pipes.federatedcode import clone_repository from scanpipe.pipes.federatedcode import delete_local_clone -from minecode_pipelines.pipes import fetch_checkpoint_from_github -from minecode_pipelines.pipes import update_checkpoints_in_github -from minecode_pipelines.pipes import update_checkpoints_file_in_github -from minecode_pipelines.pipes import get_mined_packages_from_checkpoint -from minecode_pipelines.pipes import update_mined_packages_in_checkpoint -from minecode_pipelines.pipes import update_checkpoint_state -from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO -from minecode_pipelines.pipes import INITIAL_SYNC_STATE -from minecode_pipelines.pipes import PERIODIC_SYNC_STATE -from minecode_pipelines.pipes import write_packages_json -from minecode_pipelines.pipes import compress_packages_file -from minecode_pipelines.pipes import decompress_packages_file -from minecode_pipelines.pipes import fetch_checkpoint_by_git -from minecode_pipelines.utils import get_temp_dir +from minecode.pipes import fetch_checkpoint_from_github +from minecode.pipes import update_checkpoints_in_github +from minecode.pipes import update_checkpoints_file_in_github +from minecode.pipes import get_mined_packages_from_checkpoint +from minecode.pipes import update_mined_packages_in_checkpoint +from minecode.pipes import update_checkpoint_state +from minecode.pipes import MINECODE_PIPELINES_CONFIG_REPO +from minecode.pipes import INITIAL_SYNC_STATE +from minecode.pipes import PERIODIC_SYNC_STATE +from minecode.pipes import write_packages_json +from minecode.pipes import compress_packages_file +from minecode.pipes import decompress_packages_file +from minecode.pipes import fetch_checkpoint_by_git +from minecode.utils import get_temp_dir PACKAGE_FILE_NAME = "NPMPackages.json" COMPRESSED_PACKAGE_FILE_NAME = "NPMPackages.json.gz" diff --git a/minecode_pipelines/pipes/nuget.py b/minecode/pipes/nuget.py similarity index 100% rename from minecode_pipelines/pipes/nuget.py rename to minecode/pipes/nuget.py diff --git a/minecode_pipelines/pipes/pypi.py b/minecode/pipes/pypi.py similarity index 94% rename from minecode_pipelines/pipes/pypi.py rename to minecode/pipes/pypi.py index fa5a182d..d11999bb 100644 --- a/minecode_pipelines/pipes/pypi.py +++ b/minecode/pipes/pypi.py @@ -28,17 +28,17 @@ from scanpipe.pipes.federatedcode import clone_repository from scanpipe.pipes.federatedcode import delete_local_clone -from minecode_pipelines.pipes import INITIAL_SYNC_STATE -from minecode_pipelines.pipes import MINECODE_PIPELINES_CONFIG_REPO -from minecode_pipelines.pipes import PERIODIC_SYNC_STATE -from minecode_pipelines.pipes import fetch_checkpoint_from_github -from minecode_pipelines.pipes import get_mined_packages_from_checkpoint -from minecode_pipelines.pipes import get_packages_file_from_checkpoint -from minecode_pipelines.pipes import update_checkpoints_in_github -from minecode_pipelines.pipes import update_checkpoint_state -from minecode_pipelines.pipes import update_mined_packages_in_checkpoint -from minecode_pipelines.pipes import write_packages_json -from minecode_pipelines.utils import get_temp_dir +from minecode.pipes import INITIAL_SYNC_STATE +from minecode.pipes import MINECODE_PIPELINES_CONFIG_REPO +from minecode.pipes import PERIODIC_SYNC_STATE +from minecode.pipes import fetch_checkpoint_from_github +from minecode.pipes import get_mined_packages_from_checkpoint +from minecode.pipes import get_packages_file_from_checkpoint +from minecode.pipes import update_checkpoints_in_github +from minecode.pipes import update_checkpoint_state +from minecode.pipes import update_mined_packages_in_checkpoint +from minecode.pipes import write_packages_json +from minecode.utils import get_temp_dir # If True, show full details on fetching packageURL for # a package name present in the index diff --git a/minecode_pipelines/pipes/swift.py b/minecode/pipes/swift.py similarity index 100% rename from minecode_pipelines/pipes/swift.py rename to minecode/pipes/swift.py diff --git a/minecode_pipelines/tests/data/alpine/APKINDEX b/minecode/tests/data/alpine/APKINDEX similarity index 100% rename from minecode_pipelines/tests/data/alpine/APKINDEX rename to minecode/tests/data/alpine/APKINDEX diff --git a/minecode_pipelines/tests/data/alpine/expected_packages.json b/minecode/tests/data/alpine/expected_packages.json similarity index 100% rename from minecode_pipelines/tests/data/alpine/expected_packages.json rename to minecode/tests/data/alpine/expected_packages.json diff --git a/minecode_pipelines/tests/data/cargo/c5store b/minecode/tests/data/cargo/c5store similarity index 100% rename from minecode_pipelines/tests/data/cargo/c5store rename to minecode/tests/data/cargo/c5store diff --git a/minecode_pipelines/tests/data/cargo/c5store-expected.yaml b/minecode/tests/data/cargo/c5store-expected.yaml similarity index 100% rename from minecode_pipelines/tests/data/cargo/c5store-expected.yaml rename to minecode/tests/data/cargo/c5store-expected.yaml diff --git a/minecode_pipelines/tests/data/composer/expected_output.json b/minecode/tests/data/composer/expected_output.json similarity index 100% rename from minecode_pipelines/tests/data/composer/expected_output.json rename to minecode/tests/data/composer/expected_output.json diff --git a/minecode_pipelines/tests/data/composer/package_details.json b/minecode/tests/data/composer/package_details.json similarity index 100% rename from minecode_pipelines/tests/data/composer/package_details.json rename to minecode/tests/data/composer/package_details.json diff --git a/minecode_pipelines/tests/data/composer/packages_list.json b/minecode/tests/data/composer/packages_list.json similarity index 100% rename from minecode_pipelines/tests/data/composer/packages_list.json rename to minecode/tests/data/composer/packages_list.json diff --git a/minecode_pipelines/tests/data/conan/cairo/cairo-config.yml b/minecode/tests/data/conan/cairo/cairo-config.yml similarity index 100% rename from minecode_pipelines/tests/data/conan/cairo/cairo-config.yml rename to minecode/tests/data/conan/cairo/cairo-config.yml diff --git a/minecode_pipelines/tests/data/conan/expected-cairo-purls.yml b/minecode/tests/data/conan/expected-cairo-purls.yml similarity index 100% rename from minecode_pipelines/tests/data/conan/expected-cairo-purls.yml rename to minecode/tests/data/conan/expected-cairo-purls.yml diff --git a/minecode_pipelines/tests/data/cran/cran_db.json b/minecode/tests/data/cran/cran_db.json similarity index 100% rename from minecode_pipelines/tests/data/cran/cran_db.json rename to minecode/tests/data/cran/cran_db.json diff --git a/minecode_pipelines/tests/data/cran/expected_abbreviate.yaml b/minecode/tests/data/cran/expected_abbreviate.yaml similarity index 100% rename from minecode_pipelines/tests/data/cran/expected_abbreviate.yaml rename to minecode/tests/data/cran/expected_abbreviate.yaml diff --git a/minecode_pipelines/tests/data/cran/expected_abc.data.yaml b/minecode/tests/data/cran/expected_abc.data.yaml similarity index 100% rename from minecode_pipelines/tests/data/cran/expected_abc.data.yaml rename to minecode/tests/data/cran/expected_abc.data.yaml diff --git a/minecode_pipelines/tests/data/cran/expected_abc.yaml b/minecode/tests/data/cran/expected_abc.yaml similarity index 100% rename from minecode_pipelines/tests/data/cran/expected_abc.yaml rename to minecode/tests/data/cran/expected_abc.yaml diff --git a/minecode_pipelines/tests/data/directories/find-ls b/minecode/tests/data/directories/find-ls similarity index 100% rename from minecode_pipelines/tests/data/directories/find-ls rename to minecode/tests/data/directories/find-ls diff --git a/minecode_pipelines/tests/data/directories/find-ls-apache-start b/minecode/tests/data/directories/find-ls-apache-start similarity index 100% rename from minecode_pipelines/tests/data/directories/find-ls-apache-start rename to minecode/tests/data/directories/find-ls-apache-start diff --git a/minecode_pipelines/tests/data/directories/find-ls-apache-start-expected.json b/minecode/tests/data/directories/find-ls-apache-start-expected.json similarity index 100% rename from minecode_pipelines/tests/data/directories/find-ls-apache-start-expected.json rename to minecode/tests/data/directories/find-ls-apache-start-expected.json diff --git a/minecode_pipelines/tests/data/directories/find-ls-expected.json b/minecode/tests/data/directories/find-ls-expected.json similarity index 100% rename from minecode_pipelines/tests/data/directories/find-ls-expected.json rename to minecode/tests/data/directories/find-ls-expected.json diff --git a/minecode_pipelines/tests/data/directories/ls-lr b/minecode/tests/data/directories/ls-lr similarity index 100% rename from minecode_pipelines/tests/data/directories/ls-lr rename to minecode/tests/data/directories/ls-lr diff --git a/minecode_pipelines/tests/data/directories/ls-lr-expected.json b/minecode/tests/data/directories/ls-lr-expected.json similarity index 100% rename from minecode_pipelines/tests/data/directories/ls-lr-expected.json rename to minecode/tests/data/directories/ls-lr-expected.json diff --git a/minecode_pipelines/tests/data/directories/ls-lr-ubuntu b/minecode/tests/data/directories/ls-lr-ubuntu similarity index 100% rename from minecode_pipelines/tests/data/directories/ls-lr-ubuntu rename to minecode/tests/data/directories/ls-lr-ubuntu diff --git a/minecode_pipelines/tests/data/directories/ls-lr-ubuntu-expected.json b/minecode/tests/data/directories/ls-lr-ubuntu-expected.json similarity index 100% rename from minecode_pipelines/tests/data/directories/ls-lr-ubuntu-expected.json rename to minecode/tests/data/directories/ls-lr-ubuntu-expected.json diff --git a/minecode_pipelines/tests/data/maven/index/buggy/expected_artifacts-defaults.json b/minecode/tests/data/maven/index/buggy/expected_artifacts-defaults.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/buggy/expected_artifacts-defaults.json rename to minecode/tests/data/maven/index/buggy/expected_artifacts-defaults.json diff --git a/minecode_pipelines/tests/data/maven/index/buggy/expected_artifacts.json b/minecode/tests/data/maven/index/buggy/expected_artifacts.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/buggy/expected_artifacts.json rename to minecode/tests/data/maven/index/buggy/expected_artifacts.json diff --git a/minecode_pipelines/tests/data/maven/index/buggy/expected_entries.json b/minecode/tests/data/maven/index/buggy/expected_entries.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/buggy/expected_entries.json rename to minecode/tests/data/maven/index/buggy/expected_entries.json diff --git a/minecode_pipelines/tests/data/maven/index/buggy/expected_uris.json b/minecode/tests/data/maven/index/buggy/expected_uris.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/buggy/expected_uris.json rename to minecode/tests/data/maven/index/buggy/expected_uris.json diff --git a/minecode_pipelines/tests/data/maven/index/buggy/expected_visited_uris.json b/minecode/tests/data/maven/index/buggy/expected_visited_uris.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/buggy/expected_visited_uris.json rename to minecode/tests/data/maven/index/buggy/expected_visited_uris.json diff --git a/minecode_pipelines/tests/data/maven/index/buggy/nexus-maven-repository-index.gz b/minecode/tests/data/maven/index/buggy/nexus-maven-repository-index.gz similarity index 100% rename from minecode_pipelines/tests/data/maven/index/buggy/nexus-maven-repository-index.gz rename to minecode/tests/data/maven/index/buggy/nexus-maven-repository-index.gz diff --git a/minecode_pipelines/tests/data/maven/index/expected_artifacts-all-worthy.json b/minecode/tests/data/maven/index/expected_artifacts-all-worthy.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/expected_artifacts-all-worthy.json rename to minecode/tests/data/maven/index/expected_artifacts-all-worthy.json diff --git a/minecode_pipelines/tests/data/maven/index/expected_artifacts-defaults.json b/minecode/tests/data/maven/index/expected_artifacts-defaults.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/expected_artifacts-defaults.json rename to minecode/tests/data/maven/index/expected_artifacts-defaults.json diff --git a/minecode_pipelines/tests/data/maven/index/expected_artifacts.json b/minecode/tests/data/maven/index/expected_artifacts.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/expected_artifacts.json rename to minecode/tests/data/maven/index/expected_artifacts.json diff --git a/minecode_pipelines/tests/data/maven/index/expected_entries.json b/minecode/tests/data/maven/index/expected_entries.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/expected_entries.json rename to minecode/tests/data/maven/index/expected_entries.json diff --git a/minecode_pipelines/tests/data/maven/index/expected_uris.json b/minecode/tests/data/maven/index/expected_uris.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/expected_uris.json rename to minecode/tests/data/maven/index/expected_uris.json diff --git a/minecode_pipelines/tests/data/maven/index/increment/expected_artifacts-defaults.json b/minecode/tests/data/maven/index/increment/expected_artifacts-defaults.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/expected_artifacts-defaults.json rename to minecode/tests/data/maven/index/increment/expected_artifacts-defaults.json diff --git a/minecode_pipelines/tests/data/maven/index/increment/expected_artifacts.json b/minecode/tests/data/maven/index/increment/expected_artifacts.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/expected_artifacts.json rename to minecode/tests/data/maven/index/increment/expected_artifacts.json diff --git a/minecode_pipelines/tests/data/maven/index/increment/expected_entries.json b/minecode/tests/data/maven/index/increment/expected_entries.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/expected_entries.json rename to minecode/tests/data/maven/index/increment/expected_entries.json diff --git a/minecode_pipelines/tests/data/maven/index/increment/expected_properties_uris.json b/minecode/tests/data/maven/index/increment/expected_properties_uris.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/expected_properties_uris.json rename to minecode/tests/data/maven/index/increment/expected_properties_uris.json diff --git a/minecode_pipelines/tests/data/maven/index/increment/expected_uris.json b/minecode/tests/data/maven/index/increment/expected_uris.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/expected_uris.json rename to minecode/tests/data/maven/index/increment/expected_uris.json diff --git a/minecode_pipelines/tests/data/maven/index/increment/nexus-maven-repository-index.445.gz b/minecode/tests/data/maven/index/increment/nexus-maven-repository-index.445.gz similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/nexus-maven-repository-index.445.gz rename to minecode/tests/data/maven/index/increment/nexus-maven-repository-index.445.gz diff --git a/minecode_pipelines/tests/data/maven/index/increment/nexus-maven-repository-index.properties b/minecode/tests/data/maven/index/increment/nexus-maven-repository-index.properties similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment/nexus-maven-repository-index.properties rename to minecode/tests/data/maven/index/increment/nexus-maven-repository-index.properties diff --git a/minecode_pipelines/tests/data/maven/index/increment2/expected_mini_package.json b/minecode/tests/data/maven/index/increment2/expected_mini_package.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment2/expected_mini_package.json rename to minecode/tests/data/maven/index/increment2/expected_mini_package.json diff --git a/minecode_pipelines/tests/data/maven/index/increment2/expected_uris.json b/minecode/tests/data/maven/index/increment2/expected_uris.json similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment2/expected_uris.json rename to minecode/tests/data/maven/index/increment2/expected_uris.json diff --git a/minecode_pipelines/tests/data/maven/index/increment2/nexus-maven-repository-index.457.gz b/minecode/tests/data/maven/index/increment2/nexus-maven-repository-index.457.gz similarity index 100% rename from minecode_pipelines/tests/data/maven/index/increment2/nexus-maven-repository-index.457.gz rename to minecode/tests/data/maven/index/increment2/nexus-maven-repository-index.457.gz diff --git a/minecode_pipelines/tests/data/maven/index/nexus-maven-repository-index.gz b/minecode/tests/data/maven/index/nexus-maven-repository-index.gz similarity index 100% rename from minecode_pipelines/tests/data/maven/index/nexus-maven-repository-index.gz rename to minecode/tests/data/maven/index/nexus-maven-repository-index.gz diff --git a/minecode_pipelines/tests/data/nuget/catalog/pages/page10897.json b/minecode/tests/data/nuget/catalog/pages/page10897.json similarity index 100% rename from minecode_pipelines/tests/data/nuget/catalog/pages/page10897.json rename to minecode/tests/data/nuget/catalog/pages/page10897.json diff --git a/minecode_pipelines/tests/data/swift/commits_tags1.txt b/minecode/tests/data/swift/commits_tags1.txt similarity index 100% rename from minecode_pipelines/tests/data/swift/commits_tags1.txt rename to minecode/tests/data/swift/commits_tags1.txt diff --git a/minecode_pipelines/tests/data/swift/commits_tags2.txt b/minecode/tests/data/swift/commits_tags2.txt similarity index 100% rename from minecode_pipelines/tests/data/swift/commits_tags2.txt rename to minecode/tests/data/swift/commits_tags2.txt diff --git a/minecode_pipelines/tests/data/swift/commits_tags3.txt b/minecode/tests/data/swift/commits_tags3.txt similarity index 100% rename from minecode_pipelines/tests/data/swift/commits_tags3.txt rename to minecode/tests/data/swift/commits_tags3.txt diff --git a/minecode_pipelines/tests/data/swift/commits_tags4.txt b/minecode/tests/data/swift/commits_tags4.txt similarity index 100% rename from minecode_pipelines/tests/data/swift/commits_tags4.txt rename to minecode/tests/data/swift/commits_tags4.txt diff --git a/minecode_pipelines/tests/data/swift/expected1.yaml b/minecode/tests/data/swift/expected1.yaml similarity index 100% rename from minecode_pipelines/tests/data/swift/expected1.yaml rename to minecode/tests/data/swift/expected1.yaml diff --git a/minecode_pipelines/tests/data/swift/expected2.yaml b/minecode/tests/data/swift/expected2.yaml similarity index 100% rename from minecode_pipelines/tests/data/swift/expected2.yaml rename to minecode/tests/data/swift/expected2.yaml diff --git a/minecode_pipelines/tests/data/swift/expected3.yaml b/minecode/tests/data/swift/expected3.yaml similarity index 100% rename from minecode_pipelines/tests/data/swift/expected3.yaml rename to minecode/tests/data/swift/expected3.yaml diff --git a/minecode_pipelines/tests/data/swift/expected4.yaml b/minecode/tests/data/swift/expected4.yaml similarity index 100% rename from minecode_pipelines/tests/data/swift/expected4.yaml rename to minecode/tests/data/swift/expected4.yaml diff --git a/minecode_pipelines/tests/pipes/__init__.py b/minecode/tests/pipes/__init__.py similarity index 100% rename from minecode_pipelines/tests/pipes/__init__.py rename to minecode/tests/pipes/__init__.py diff --git a/minecode_pipelines/tests/pipes/test_alpine.py b/minecode/tests/pipes/test_alpine.py similarity index 96% rename from minecode_pipelines/tests/pipes/test_alpine.py rename to minecode/tests/pipes/test_alpine.py index ff0b91b3..8886eceb 100644 --- a/minecode_pipelines/tests/pipes/test_alpine.py +++ b/minecode/tests/pipes/test_alpine.py @@ -12,7 +12,7 @@ from commoncode.testcase import check_against_expected_json_file from commoncode.testcase import FileBasedTesting -from minecode_pipelines.pipes import alpine +from minecode.pipes import alpine class AlpineMapperTest(FileBasedTesting): diff --git a/minecode_pipelines/tests/pipes/test_apache.py b/minecode/tests/pipes/test_apache.py similarity index 99% rename from minecode_pipelines/tests/pipes/test_apache.py rename to minecode/tests/pipes/test_apache.py index ad440fc9..06d06acf 100644 --- a/minecode_pipelines/tests/pipes/test_apache.py +++ b/minecode/tests/pipes/test_apache.py @@ -11,7 +11,7 @@ from commoncode.testcase import FileBasedTesting -from minecode_pipelines.pipes import apache +from minecode.pipes import apache class ApacheMiscTest(FileBasedTesting): diff --git a/minecode_pipelines/tests/pipes/test_cargo.py b/minecode/tests/pipes/test_cargo.py similarity index 94% rename from minecode_pipelines/tests/pipes/test_cargo.py rename to minecode/tests/pipes/test_cargo.py index 777fda74..e482373e 100644 --- a/minecode_pipelines/tests/pipes/test_cargo.py +++ b/minecode/tests/pipes/test_cargo.py @@ -14,7 +14,7 @@ import saneyaml from unittest import TestCase -from minecode_pipelines.pipes.cargo import get_cargo_packages +from minecode.pipes.cargo import get_cargo_packages DATA_DIR = Path(__file__).parent.parent / "data" / "cargo" diff --git a/minecode_pipelines/tests/pipes/test_composer.py b/minecode/tests/pipes/test_composer.py similarity index 90% rename from minecode_pipelines/tests/pipes/test_composer.py rename to minecode/tests/pipes/test_composer.py index a60211e4..986ffdd2 100644 --- a/minecode_pipelines/tests/pipes/test_composer.py +++ b/minecode/tests/pipes/test_composer.py @@ -12,9 +12,9 @@ from unittest.mock import patch, MagicMock from django.test import SimpleTestCase -from minecode_pipelines.pipes.composer import get_composer_packages -from minecode_pipelines.pipes.composer import load_composer_packages -from minecode_pipelines.pipes.composer import get_composer_purl +from minecode.pipes.composer import get_composer_packages +from minecode.pipes.composer import load_composer_packages +from minecode.pipes.composer import get_composer_purl DATA_DIR = Path(__file__).parent.parent / "data" / "composer" diff --git a/minecode_pipelines/tests/pipes/test_conan.py b/minecode/tests/pipes/test_conan.py similarity index 94% rename from minecode_pipelines/tests/pipes/test_conan.py rename to minecode/tests/pipes/test_conan.py index 38dd96d4..f025d039 100644 --- a/minecode_pipelines/tests/pipes/test_conan.py +++ b/minecode/tests/pipes/test_conan.py @@ -13,7 +13,7 @@ from django.test import TestCase -from minecode_pipelines.pipes.conan import get_conan_packages +from minecode.pipes.conan import get_conan_packages DATA_DIR = Path(__file__).parent.parent / "data" / "conan" diff --git a/minecode_pipelines/tests/pipes/test_cran.py b/minecode/tests/pipes/test_cran.py similarity index 96% rename from minecode_pipelines/tests/pipes/test_cran.py rename to minecode/tests/pipes/test_cran.py index 53dfb075..264800ea 100644 --- a/minecode_pipelines/tests/pipes/test_cran.py +++ b/minecode/tests/pipes/test_cran.py @@ -11,7 +11,7 @@ from pathlib import Path from unittest import TestCase -from minecode_pipelines.pipes.cran import mine_cran_packageurls +from minecode.pipes.cran import mine_cran_packageurls DATA_DIR = Path(__file__).parent.parent / "data" / "cran" diff --git a/minecode_pipelines/tests/pipes/test_ls.py b/minecode/tests/pipes/test_ls.py similarity index 98% rename from minecode_pipelines/tests/pipes/test_ls.py rename to minecode/tests/pipes/test_ls.py index bc87e8ab..fcb3fffe 100644 --- a/minecode_pipelines/tests/pipes/test_ls.py +++ b/minecode/tests/pipes/test_ls.py @@ -13,7 +13,7 @@ from commoncode.testcase import check_against_expected_json_file from commoncode.testcase import FileBasedTesting -from minecode_pipelines.pipes import ls +from minecode.pipes import ls class ParseDirectoryListingTest(FileBasedTesting): diff --git a/minecode_pipelines/tests/pipes/test_maven.py b/minecode/tests/pipes/test_maven.py similarity index 99% rename from minecode_pipelines/tests/pipes/test_maven.py rename to minecode/tests/pipes/test_maven.py index 199c8322..a962cfae 100644 --- a/minecode_pipelines/tests/pipes/test_maven.py +++ b/minecode/tests/pipes/test_maven.py @@ -12,7 +12,7 @@ from commoncode.testcase import check_against_expected_json_file from commoncode.testcase import FileBasedTesting -from minecode_pipelines.pipes import maven +from minecode.pipes import maven class MavenMiscTest(FileBasedTesting): diff --git a/minecode_pipelines/tests/pipes/test_nuget.py b/minecode/tests/pipes/test_nuget.py similarity index 94% rename from minecode_pipelines/tests/pipes/test_nuget.py rename to minecode/tests/pipes/test_nuget.py index 77e80c93..972b3f25 100644 --- a/minecode_pipelines/tests/pipes/test_nuget.py +++ b/minecode/tests/pipes/test_nuget.py @@ -11,8 +11,8 @@ from pathlib import Path from unittest import TestCase -from minecode_pipelines.pipes import nuget -from minecode_pipelines.tests.pipes import TestLogger +from minecode.pipes import nuget +from minecode.tests.pipes import TestLogger TEST_DIR = Path(__file__).parent.parent / "data" / "nuget" diff --git a/minecode_pipelines/tests/pipes/test_pipes.py b/minecode/tests/pipes/test_pipes.py similarity index 97% rename from minecode_pipelines/tests/pipes/test_pipes.py rename to minecode/tests/pipes/test_pipes.py index 1c8161bc..369e5520 100644 --- a/minecode_pipelines/tests/pipes/test_pipes.py +++ b/minecode/tests/pipes/test_pipes.py @@ -11,7 +11,7 @@ from unittest import TestCase from git import Repo -from minecode_pipelines.pipes import get_commit_at_distance_ahead +from minecode.pipes import get_commit_at_distance_ahead class GetCommitAtDistanceAheadIntegrationTests(TestCase): diff --git a/minecode_pipelines/tests/pipes/test_pypi.py b/minecode/tests/pipes/test_pypi.py similarity index 100% rename from minecode_pipelines/tests/pipes/test_pypi.py rename to minecode/tests/pipes/test_pypi.py diff --git a/minecode_pipelines/tests/pipes/test_swift.py b/minecode/tests/pipes/test_swift.py similarity index 98% rename from minecode_pipelines/tests/pipes/test_swift.py rename to minecode/tests/pipes/test_swift.py index f21cb67b..9f977bca 100644 --- a/minecode_pipelines/tests/pipes/test_swift.py +++ b/minecode/tests/pipes/test_swift.py @@ -10,7 +10,7 @@ from pathlib import Path from unittest import TestCase import saneyaml -from minecode_pipelines.pipes.swift import ( +from minecode.pipes.swift import ( get_tags_and_commits_from_git_output, generate_package_urls, ) diff --git a/minecode/utils.py b/minecode/utils.py index 16ac1513..3e45b753 100644 --- a/minecode/utils.py +++ b/minecode/utils.py @@ -13,6 +13,7 @@ import os import tempfile import uuid +from itertools import zip_longest from django.conf import settings from django.core import signing @@ -404,3 +405,19 @@ def get_webhook_url(view_name, user_uuid): site_url = settings.SITE_URL.rstrip("/") webhook_url = site_url + target_url return webhook_url + + +def grouper(n, iterable, padvalue=None): + """ + Produce batches of length `n` for an `iterable`. + https://docs.python.org/3.10/library/itertools.html#itertools-recipes + + #TODO: replace with `itertools.batched` added in python3.12 + """ + return zip_longest(*[iter(iterable)] * n, fillvalue=padvalue) + + +def cycle_from_index(lst, x): + """Return list rotated so it starts from index x.""" + x = x % len(lst) # handle cases where x is out of bounds + return lst[x:] + lst[:x] diff --git a/minecode_pipelines/__init__.py b/minecode_pipelines/__init__.py deleted file mode 100644 index 5b0bba7a..00000000 --- a/minecode_pipelines/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -VERSION = "0.1.1" diff --git a/minecode_pipelines/tests/__init__.py b/minecode_pipelines/tests/__init__.py deleted file mode 100644 index e1521118..00000000 --- a/minecode_pipelines/tests/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# diff --git a/minecode_pipelines/utils.py b/minecode_pipelines/utils.py deleted file mode 100644 index 29bf9f8c..00000000 --- a/minecode_pipelines/utils.py +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import tempfile -import os -from commoncode.fileutils import create_dir - -from itertools import zip_longest - - -def grouper(n, iterable, padvalue=None): - """ - Produce batches of length `n` for an `iterable`. - https://docs.python.org/3.10/library/itertools.html#itertools-recipes - - #TODO: replace with `itertools.batched` added in python3.12 - """ - return zip_longest(*[iter(iterable)] * n, fillvalue=padvalue) - - -def system_temp_dir(temp_dir=os.getenv("MINECODE_TMP")): - """Return the global temp directory..""" - if not temp_dir: - temp_dir = os.path.join(tempfile.gettempdir(), "minecode") - create_dir(temp_dir) - return temp_dir - - -def get_temp_dir(base_dir="", prefix=""): - """ - Return the path to base a new unique temporary directory, created under - the system-wide `system_temp_dir` temp directory and as a subdir of the - base_dir path, a path relative to the `system_temp_dir`. - """ - if base_dir: - base_dir = os.path.join(system_temp_dir(), base_dir) - create_dir(base_dir) - else: - base_dir = system_temp_dir() - return tempfile.mkdtemp(prefix=prefix, dir=base_dir) - - -def get_temp_file(file_name="data", extension=".file", dir_name=""): - """ - Return a file path string to a new, unique and non-existing - temporary file that can safely be created without a risk of name - collision. - """ - if extension and not extension.startswith("."): - extension = "." + extension - - file_name = file_name + extension - # create a new temp dir each time - temp_dir = get_temp_dir(dir_name) - location = os.path.join(temp_dir, file_name) - return location - - -def cycle_from_index(lst, x): - """Return list rotated so it starts from index x.""" - x = x % len(lst) # handle cases where x is out of bounds - return lst[x:] + lst[:x] diff --git a/pyproject-minecode_pipelines.toml b/pyproject-minecode_pipelines.toml index 31ba9bea..05703070 100644 --- a/pyproject-minecode_pipelines.toml +++ b/pyproject-minecode_pipelines.toml @@ -6,7 +6,7 @@ build-backend = "flot.buildapi" name = "minecode_pipelines" version = "1.0.1" description = "A library for mining packageURLs and package metadata from ecosystem repositories." -readme = "minecode_pipelines/README.rst" +readme = "README-minecode_pipelines.rst" license = { text = "Apache-2.0" } requires-python = ">=3.10" @@ -26,7 +26,6 @@ keywords = [ "data", ] - classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", @@ -51,21 +50,21 @@ dependencies = [ urls = { Homepage = "https://github.com/aboutcode-org/purldb" } [project.entry-points."scancodeio_pipelines"] -mine_pypi = "minecode_pipelines.pipelines.mine_pypi:MinePypi" -mine_npm = "minecode_pipelines.pipelines.mine_npm:MineNPM" -mine_maven = "minecode_pipelines.pipelines.mine_maven:MineMaven" -mine_cargo = "minecode_pipelines.pipelines.mine_cargo:MineCargo" -mine_debian = "minecode_pipelines.pipelines.mine_debian:MineDebian" -mine_nuget = "minecode_pipelines.pipelines.mine_nuget:MineNuGet" -mine_alpine = "minecode_pipelines.pipelines.mine_alpine:MineAlpine" -mine_conan = "minecode_pipelines.pipelines.mine_conan:MineConan" -mine_cpan = "minecode_pipelines.pipelines.mine_cpan:MineCpan" -mine_cran = "minecode_pipelines.pipelines.mine_cran:MineCran" -mine_swift = "minecode_pipelines.pipelines.mine_swift:MineSwift" -mine_composer = "minecode_pipelines.pipelines.mine_composer:MineComposer" -mine_nix = "minecode_pipelines.pipelines.mine_nix:MineNix" -mine_crates = "minecode_pipelines.pipelines.mine_crates:MineCrates" -mine_apache = "minecode_pipelines.pipelines.mine_apache:MineApache" +mine_pypi = "minecode.pipelines.mine_pypi:MinePypi" +mine_npm = "minecode.pipelines.mine_npm:MineNPM" +mine_maven = "minecode.pipelines.mine_maven:MineMaven" +mine_cargo = "minecode.pipelines.mine_cargo:MineCargo" +mine_debian = "minecode.pipelines.mine_debian:MineDebian" +mine_nuget = "minecode.pipelines.mine_nuget:MineNuGet" +mine_alpine = "minecode.pipelines.mine_alpine:MineAlpine" +mine_conan = "minecode.pipelines.mine_conan:MineConan" +mine_cpan = "minecode.pipelines.mine_cpan:MineCpan" +mine_cran = "minecode.pipelines.mine_cran:MineCran" +mine_swift = "minecode.pipelines.mine_swift:MineSwift" +mine_composer = "minecode.pipelines.mine_composer:MineComposer" +mine_nix = "minecode.pipelines.mine_nix:MineNix" +mine_crates = "minecode.pipelines.mine_crates:MineCrates" +mine_apache = "minecode.pipelines.mine_apache:MineApache" [tool.bumpversion] current_version = "1.0.1" @@ -77,7 +76,9 @@ files = [ [tool.flot] includes = [ - "minecode_pipelines/**/*", + "minecode/pipelines/*", + "minecode/pipes/*", + "minecode/utils.py", ] excludes = [ @@ -91,8 +92,8 @@ excludes = [ "**/.ve", "**/*.bak", "**/.ipynb_checkpoints/*", - "minecode_pipelines/tests/**/*", - "minecode_pipelines/notebooks/*", + "minecode/tests/**/*", + "minecode/notebooks/*", ] metadata_files = ["apache-2.0.LICENSE", "NOTICE"] diff --git a/setup.cfg b/setup.cfg index 308aec8b..9e59c196 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,6 +57,7 @@ install_requires = rubymarshal == 1.0.3 samecode == 0.5.1 urlpy == 0.5 + brotli == 1.2.0 setup_requires = setuptools_scm[toml] >= 4 @@ -90,3 +91,18 @@ console_scripts = scancodeio_pipelines = matching = matchcode.pipelines.matching:Matching d2d = scanpipe.pipelines.deploy_to_develop:DeployToDevelop + mine_pypi = minecode.pipelines.mine_pypi:MinePypi + mine_npm = minecode.pipelines.mine_npm:MineNPM + mine_maven = minecode.pipelines.mine_maven:MineMaven + mine_cargo = minecode.pipelines.mine_cargo:MineCargo + mine_debian = minecode.pipelines.mine_debian:MineDebian + mine_nuget = minecode.pipelines.mine_nuget:MineNuGet + mine_alpine = minecode.pipelines.mine_alpine:MineAlpine + mine_conan = minecode.pipelines.mine_conan:MineConan + mine_cpan = minecode.pipelines.mine_cpan:MineCpan + mine_cran = minecode.pipelines.mine_cran:MineCran + mine_swift = minecode.pipelines.mine_swift:MineSwift + mine_composer = minecode.pipelines.mine_composer:MineComposer + mine_nix = minecode.pipelines.mine_nix:MineNix + mine_crates = minecode.pipelines.mine_crates:MineCrates + mine_apache = minecode.pipelines.mine_apache:MineApache From c6c9e22fe25cb26c455be3759f4c3f39c951a2b4 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 4 Aug 2026 16:22:15 -0700 Subject: [PATCH 3/3] Bump minecode-pipelines version Signed-off-by: Jono Yang --- .../pipelines/README.rst | 10 +++++----- pyproject-minecode_pipelines.toml | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) rename README-minecode_pipelines.rst => minecode/pipelines/README.rst (83%) diff --git a/README-minecode_pipelines.rst b/minecode/pipelines/README.rst similarity index 83% rename from README-minecode_pipelines.rst rename to minecode/pipelines/README.rst index 6f42f0cb..7ed16e61 100644 --- a/README-minecode_pipelines.rst +++ b/minecode/pipelines/README.rst @@ -1,7 +1,7 @@ -minecode-pipeline -=================== +minecode-pipelines +================== -minecode-pipeline is an add-on library working with scancode.io to define pipelines to mine +minecode-pipelines is an add-on library working with scancode.io to define pipelines to mine packageURLs and package metadata from ecosystem repositories and APIs. Installation @@ -10,8 +10,8 @@ Installation Requirements ############ -* install minecode-pipeline dependencies -* `pip install minecode-pipeline` +* install minecode-pipelines dependencies +* `pip install minecode-pipelines` Funding diff --git a/pyproject-minecode_pipelines.toml b/pyproject-minecode_pipelines.toml index 05703070..395b8bbd 100644 --- a/pyproject-minecode_pipelines.toml +++ b/pyproject-minecode_pipelines.toml @@ -4,7 +4,7 @@ build-backend = "flot.buildapi" [project] name = "minecode_pipelines" -version = "1.0.1" +version = "2.0.0" description = "A library for mining packageURLs and package metadata from ecosystem repositories." readme = "README-minecode_pipelines.rst" license = { text = "Apache-2.0" } @@ -67,7 +67,7 @@ mine_crates = "minecode.pipelines.mine_crates:MineCrates" mine_apache = "minecode.pipelines.mine_apache:MineApache" [tool.bumpversion] -current_version = "1.0.1" +current_version = "2.0.0" allow_dirty = true files = [