diff --git a/.flake8 b/.flake8 index ec20d0a..e67a5a7 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] exclude = .git, .eggs, __pycache__, build, dist, docs, docker, migrations -ignore = E129, E402, F841, W504 E128 +ignore = E129, E402, F841, W504 E128 , W503, E203 max-line-length = 130 diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 76a19a6..232aade 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -57,8 +57,7 @@ metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, - help="Path to the detsination directory where to save downloaded wheels, " - "sources, ABOUT and LICENSE files..", + help="Path to the detsination directory where to save downloaded wheels, " "sources, ABOUT and LICENSE files..", ) @click.option( "-w", @@ -107,8 +106,7 @@ @click.option( "--use-cached-index", is_flag=True, - help="Use on disk cached PyPI indexes list of packages and versions and " - "do not refetch if present.", + help="Use on disk cached PyPI indexes list of packages and versions and " "do not refetch if present.", ) @click.option( "--sdist-only", @@ -186,8 +184,7 @@ def fetch_thirdparty( print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}") existing_packages_by_nv = { - (package.name, package.version): package - for package in utils_thirdparty.get_local_packages(directory=dest_dir) + (package.name, package.version): package for package in utils_thirdparty.get_local_packages(directory=dest_dir) } required_name_versions = set(existing_packages_by_nv.keys()) diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 89d0626..5fbd253 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -168,10 +168,7 @@ def from_file(cls, name, index_dir, archive_file): ) def simple_index_entry(self, base_url): - return ( - f' ' - f"{self.archive_file.name}
" - ) + return f' ' f"{self.archive_file.name}
" def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): @@ -204,11 +201,7 @@ def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi" for pkg_file in directory.iterdir(): pkg_filename = pkg_file.name - if ( - not pkg_file.is_file() - or not pkg_filename.endswith(dist_exts) - or pkg_filename.startswith(".") - ): + if not pkg_file.is_file() or not pkg_filename.endswith(dist_exts) or pkg_filename.startswith("."): continue pkg_name = get_package_name_from_filename( diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 1b87944..ae48e1f 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -33,8 +33,7 @@ def gen_requirements(): type=pathlib.Path, required=True, metavar="DIR", - help="Path to the 'site-packages' directory where wheels are installed " - "such as lib/python3.12/site-packages", + help="Path to the 'site-packages' directory where wheels are installed " "such as lib/python3.12/site-packages", ) parser.add_argument( "-r", diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index 8548205..ce9eccd 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -35,8 +35,7 @@ def gen_dev_requirements(): type=pathlib.Path, required=True, metavar="DIR", - help="Path to the 'site-packages' directory where wheels are installed " - "such as lib/python3.12/site-packages", + help="Path to the 'site-packages' directory where wheels are installed " "such as lib/python3.12/site-packages", ) parser.add_argument( "-d", @@ -52,8 +51,7 @@ def gen_dev_requirements(): type=pathlib.Path, default="requirements.txt", metavar="FILE", - help="Path to the main requirements file. Its requirements will be excluded " - "from the generated dev requirements.", + help="Path to the main requirements file. Its requirements will be excluded " "from the generated dev requirements.", ) args = parser.parse_args() diff --git a/etc/scripts/test_utils_pypi_supported_tags.py b/etc/scripts/test_utils_pypi_supported_tags.py index d291572..add6205 100644 --- a/etc/scripts/test_utils_pypi_supported_tags.py +++ b/etc/scripts/test_utils_pypi_supported_tags.py @@ -65,10 +65,7 @@ def validate_wheel_filename_for_pypi(filename): "macosx_10_15_arm64", "macosx_11_10_universal2", # A real tag used by e.g. some numpy wheels - ( - "macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64." - "macosx_10_10_intel.macosx_10_10_x86_64" - ), + ("macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64." "macosx_10_10_intel.macosx_10_10_x86_64"), ], ) def test_is_valid_pypi_wheel_return_true_for_supported_wheel(plat): diff --git a/etc/scripts/update_skeleton.py b/etc/scripts/update_skeleton.py index 374c06f..5974f23 100644 --- a/etc/scripts/update_skeleton.py +++ b/etc/scripts/update_skeleton.py @@ -87,9 +87,7 @@ def update_skeleton_files(repo_names=ABOUTCODE_PUBLIC_REPO_NAMES): os.chdir(work_dir_path / repo_name) # Add skeleton as an origin - subprocess.run( - ["git", "remote", "add", "skeleton", "git@github.com:aboutcode-org/skeleton.git"] - ) + subprocess.run(["git", "remote", "add", "skeleton", "git@github.com:aboutcode-org/skeleton.git"]) # Fetch skeleton files subprocess.run(["git", "fetch", "skeleton"]) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index bc68ac7..ed1b0fd 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -252,16 +252,12 @@ def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tu supported_wheels = list(package.get_supported_wheels(environment=environment)) if not supported_wheels: if TRACE_DEEP: - print( - f" download_wheel: No supported wheel for {name}=={version}: {environment} " - ) + print(f" download_wheel: No supported wheel for {name}=={version}: {environment} ") continue for wheel in supported_wheels: if TRACE_DEEP: - print( - f" download_wheel: Getting wheel from index (or cache): {wheel.download_url}" - ) + print(f" download_wheel: Getting wheel from index (or cache): {wheel.download_url}") fetched_wheel_filename = wheel.download(dest_dir=dest_dir) fetched_wheel_filenames.append(fetched_wheel_filename) @@ -537,19 +533,14 @@ def get_best_download_url(self, repos=tuple()): package = repo.get_package_version(name=self.name, version=self.version) if not package: if TRACE: - print( - f" get_best_download_url: {self.name}=={self.version} " - f"not found in {repo.index_url}" - ) + print(f" get_best_download_url: {self.name}=={self.version} " f"not found in {repo.index_url}") continue pypi_url = package.get_url_for_filename(self.filename) if pypi_url: return pypi_url else: if TRACE: - print( - f" get_best_download_url: {self.filename} not found in {repo.index_url}" - ) + print(f" get_best_download_url: {self.filename} not found in {repo.index_url}") def download(self, dest_dir=THIRDPARTY_DIR): """ @@ -824,7 +815,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] - extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] + extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] extra_lic_names = [ln for ln in extra_lic_names if ln] lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] @@ -846,7 +837,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from remote: {lic_url}") - except: + except Exception: try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" @@ -859,7 +850,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from licensedb: {lic_url}") - except: + except Exception: msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' print(msg) errors.append(msg) @@ -908,9 +899,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): classifiers = raw_data.get_all("Classifier") or [] - declared_license = [raw_data["License"]] + [ - c for c in classifiers if c.startswith("License") - ] + declared_license = [raw_data["License"]] + [c for c in classifiers if c.startswith("License")] license_expression = get_license_expression(declared_license) other_classifiers = [c for c in classifiers if not c.startswith("License")] @@ -956,10 +945,7 @@ def update(self, data, overwrite=False, keep_extra=True): purl_from_data = packageurl.PackageURL.from_string(package_url) purl_from_self = packageurl.PackageURL.from_string(self.package_url) if purl_from_data != purl_from_self: - print( - f"Invalid dist update attempt, no same same purl with dist: " - f"{self} using data {data}." - ) + print(f"Invalid dist update attempt, no same same purl with dist: " f"{self} using data {data}.") return data.pop("about_resource", None) @@ -1000,7 +986,7 @@ def get_license_link_for_filename(filename, urls): exception if no link is found or if there are more than one link for that file name. """ - path_or_url = [l for l in urls if l.endswith(f"/{filename}")] + path_or_url = [url for url in urls if url.endswith(f"/{filename}")] if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: @@ -1222,9 +1208,7 @@ def from_filename(cls, filename): platforms = wheel_info.group("plats").split(".") # All the tag combinations from this file - tags = { - packaging_tags.Tag(x, y, z) for x in python_versions for y in abis for z in platforms - } + tags = {packaging_tags.Tag(x, y, z) for x in python_versions for y in abis for z in platforms} return cls( filename=filename, @@ -1289,7 +1273,7 @@ def is_pure(self): def is_pure_wheel(filename): try: return Wheel.from_filename(filename).is_pure() - except: + except Exception: return False @@ -1361,18 +1345,14 @@ def package_from_dists(cls, dists): for dist in dists: if dist.normalized_name != normalized_name: if TRACE: - print( - f" Skipping inconsistent dist name: expected {normalized_name} got {dist}" - ) + print(f" Skipping inconsistent dist name: expected {normalized_name} got {dist}") continue elif dist.version != version: dv = packaging_version.parse(dist.version) v = packaging_version.parse(version) if dv != v: if TRACE: - print( - f" Skipping inconsistent dist version: expected {version} got {dist}" - ) + print(f" Skipping inconsistent dist version: expected {version} got {dist}") continue if isinstance(dist, Sdist): @@ -1614,9 +1594,7 @@ class PypiSimpleRepository: packages = attr.ib( type=dict, default=attr.Factory(lambda: defaultdict(dict)), - metadata=dict( - help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" - ), + metadata=dict(help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo"), ) fetched_package_normalized_names = attr.ib( @@ -1628,9 +1606,7 @@ class PypiSimpleRepository: use_cached_index = attr.ib( type=bool, default=False, - metadata=dict( - help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." - ), + metadata=dict(help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache."), ) def _get_package_versions_map(self, name): @@ -1647,8 +1623,7 @@ def _get_package_versions_map(self, name): links = self.fetch_links(normalized_name=normalized_name) # note that thsi is sorted so the mapping is also sorted versions = { - package.version: package - for package in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links) + package.version: package for package in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links) } self.packages[normalized_name] = versions except RemoteNotFetchedException as e: @@ -1693,7 +1668,7 @@ def fetch_links(self, normalized_name): ) links = collect_urls(text) # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] + links = [link.partition("#sha256=") for link in links] links = [url for url, _, _sha256 in links] return links @@ -1726,9 +1701,7 @@ class LinksRepository: use_cached_index = attr.ib( type=bool, default=False, - metadata=dict( - help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache." - ), + metadata=dict(help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache."), ) def __attrs_post_init__(self): @@ -1746,9 +1719,7 @@ def find_links(self, _CACHE=[]): if TRACE_DEEP: print(f"Finding links from: {links_url}") plinks_url = urllib.parse.urlparse(links_url) - base_url = urllib.parse.SplitResult( - plinks_url.scheme, plinks_url.netloc, "", "", "" - ).geturl() + base_url = urllib.parse.SplitResult(plinks_url.scheme, plinks_url.netloc, "", "", "").geturl() if TRACE_DEEP: print(f"Base URL {base_url}") @@ -1866,9 +1837,7 @@ def get_file_content(path_or_url, as_text=True): _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content - elif path_or_url.startswith("file://") or ( - path_or_url.startswith("/") and os.path.exists(path_or_url) - ): + elif path_or_url.startswith("file://") or (path_or_url.startswith("/") and os.path.exists(path_or_url)): return get_local_file_content(path=path_or_url, as_text=as_text) else: @@ -2036,11 +2005,7 @@ def get_other_dists(_package, _dist): continue # try to get another version of the same package that is not our version - other_local_packages = [ - p - for p in packages_by_name[local_package.name] - if p.version != local_package.version - ] + other_local_packages = [p for p in packages_by_name[local_package.name] if p.version != local_package.version] other_local_version = other_local_packages and other_local_packages[-1] if other_local_version: latest_local_dists = list(other_local_version.get_distributions()) @@ -2058,9 +2023,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index - ) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to fetch remotely @@ -2077,9 +2040,7 @@ def get_other_dists(_package, _dist): lpv = local_package.version lpn = local_package.name - other_remote_packages = [ - p for v, p in PYPI_SELFHOSTED_REPO.get_package_versions(lpn).items() if v != lpv - ] + other_remote_packages = [p for v, p in PYPI_SELFHOSTED_REPO.get_package_versions(lpn).items() if v != lpv] latest_version = other_remote_packages and other_remote_packages[-1] if latest_version: @@ -2098,9 +2059,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index - ) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get data from pkginfo (no license though) @@ -2133,9 +2092,7 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" - ) as process: + with subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") as process: stdouts = [] while True: line = process.stdout.readline() @@ -2198,7 +2155,7 @@ def download_wheels_with_pip( cli_args.extend(["--requirement", req_file]) if TRACE: - print(f"Downloading wheels using command:", " ".join(cli_args)) + print("Downloading wheels using command:", " ".join(cli_args)) existing = set(os.listdir(dest_dir)) error = False @@ -2282,5 +2239,5 @@ def get_license_expression(declared_licenses): return get_only_expression_from_extracted_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses - lics = [python_safe_name(l).lower() for l in declared_licenses] + lics = [python_safe_name(lic).lower() for lic in declared_licenses] return " AND ".join(lics).lower() diff --git a/src/grimoirelab_metrics/cli.py b/src/grimoirelab_metrics/cli.py index fb2c3d2..9e9eaff 100755 --- a/src/grimoirelab_metrics/cli.py +++ b/src/grimoirelab_metrics/cli.py @@ -22,7 +22,6 @@ import datetime import json import logging -import os import re import sys import time @@ -63,8 +62,7 @@ @click.option("--grimoirelab-password", help="GrimoireLab API password") @click.option("--grimoirelab-ecosystem", help="GrimoireLab will classify the data under this ecosystem name") @click.option( - "--grimoirelab-project", - help="GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem" + "--grimoirelab-project", help="GrimoireLab will classify the data under this project name. Projects are grouped by ecosystem" ) @click.option( "--opensearch-url", @@ -138,7 +136,7 @@ def grimoirelab_metrics( """Calculate metrics and the npm health score using GrimoireLab. This tools generates a sets of project health metrics and a score using a - npm health model. As input it either receives a remote Git repository or a + npm health model. As input it either receives a remote Git repository or a local SPDX SBOM file with git repositories. The data collection is scheduled by GrimoireLab and the health score is calculated on the fly. @@ -160,7 +158,7 @@ def grimoirelab_metrics( git_urls = list(set(repo for repo in packages.values() if is_valid(repo))) elif is_git_repository(source): logging.debug(f"Source is a Git repository: {source}") - packages = {"package0": source} + packages = {"package0": source} git_urls = [source] else: logging.debug(f"Source is a not either a filepath or Git repository: {source}. Exiting ...") @@ -224,7 +222,7 @@ def grimoirelab_metrics( "dev_categories_thresholds": dev_categories_thresholds, } output.write(json.dumps(package_metrics, indent=4)) - logging.info(f"Metrics and scores are calculated and written to file \"{output.name}\"") + logging.info(f'Metrics and scores are calculated and written to file "{output.name}"') except SPDXParsingError as e: logging.error(e.messages[0]) sys.exit(1) @@ -232,14 +230,17 @@ def grimoirelab_metrics( logging.error(e) sys.exit(1) + def is_git_repository(value: str) -> bool: """Return True if value looks like a Git repository URL.""" return re.match(GIT_REPO_REGEX, value) is not None + def is_sbom_file(value: str) -> bool: """Return True if value is an existing file.""" return Path(value).is_file() + def get_repository(download_location: str) -> str | None: if is_valid(download_location): git_regex = re.search(GIT_REPO_REGEX, download_location) @@ -272,11 +273,8 @@ def get_sbom_packages(file: str) -> dict[str, str]: def schedule_repositories( - repositories: list[str], - grimoirelab_client: GrimoireLabClient, - grimoirelab_ecosystem: str, - grimoirelab_project: str - ) -> None: + repositories: list[str], grimoirelab_client: GrimoireLabClient, grimoirelab_ecosystem: str, grimoirelab_project: str +) -> None: """Schedule tasks to collect data from a list of repositories. :param repositories: List of git repositories. @@ -389,8 +387,8 @@ def repository_ready( grimoirelab_ecosystem: str, grimoirelab_project: str, repository: str, - after_date: datetime.datetime - ) -> bool: + after_date: datetime.datetime, +) -> bool: """ Check if the task related to the repository has finished. @@ -416,7 +414,6 @@ def repository_ready( categories = repo_data["results"][0].get("categories", []) if not categories: return False - task = categories[0].get("task") if task["status"] == "failed": logging.warning(f"Data for '{repository}' might be incomplete, its last execution failed") @@ -440,8 +437,8 @@ def schedule_repository( grimoirelab_project: str, uri: str, datasource: str, - category: str - ) -> Any: + category: str, +) -> Any: """Schedule a task to fetch a Git repository. :param grimoirelab_client: GrimoireLab API client. @@ -458,14 +455,11 @@ def schedule_repository( "uri": uri, "datasource_type": datasource, "category": category, - "scheduler": { - "job_interval": 86400, - "job_max_retries": 3, - "force_run": False - } + "scheduler": {"job_interval": 86400, "job_max_retries": 3, "force_run": False}, } - if is_added(grimoirelab_client, grimoirelab_ecosystem, grimoirelab_project, uri): return True + if is_added(grimoirelab_client, grimoirelab_ecosystem, grimoirelab_project, uri): + return True endpoint = f"api/v1/ecosystems/{grimoirelab_ecosystem}/projects/{grimoirelab_project}/repos/" try: @@ -481,6 +475,7 @@ def schedule_repository( # If it's a different HTTP error (500, 404, 403), re-raise it raise e + def is_added(grimoirelab_client: GrimoireLabClient, grimoirelab_ecosystem: str, grimoirelab_project: str, uri: str) -> bool: """Check if the repository is already scheduled @@ -513,5 +508,6 @@ def is_added(grimoirelab_client: GrimoireLabClient, grimoirelab_ecosystem: str, else: return False + if __name__ == "__main__": grimoirelab_metrics() diff --git a/src/grimoirelab_metrics/metrics.py b/src/grimoirelab_metrics/metrics.py index fa80fde..08ea658 100644 --- a/src/grimoirelab_metrics/metrics.py +++ b/src/grimoirelab_metrics/metrics.py @@ -261,7 +261,8 @@ def get_commit_coefficient_of_variation(self): stdev = numpy.std(commits_list) try: # we need the line below because of numpy - if mean == 0: raise ZeroDivisionError + if mean == 0: + raise ZeroDivisionError cv = stdev / mean except ZeroDivisionError as e: # the activity in commits is zero, diff --git a/src/grimoirelab_metrics/metrics_model.py b/src/grimoirelab_metrics/metrics_model.py index b43dbb5..8eb1515 100644 --- a/src/grimoirelab_metrics/metrics_model.py +++ b/src/grimoirelab_metrics/metrics_model.py @@ -23,10 +23,11 @@ # These coefficients were calculated with the notebooks and data available # at https://github.com/aboutcode-org/healthycode/blob/main/model/npm/README.md + class npmModel: # Ecosystem Name - ECOSYSTEM_NAME= "npm" + ECOSYSTEM_NAME = "npm" # Model Name MODEL_NAME = "health" # Model Version @@ -34,19 +35,18 @@ class npmModel: # We have dropped the low-impact metrics, those with a coefficient close to 0 COEFFICIENTS = { - 'active_branches': -0.834069, - 'commits_over_periods_rate': -0.841944, - 'commit_size_added_lines': -0.389225, - 'contributor_growth_rate': 0.114363, - 'days_since_last_commit': 1.081051, - 'developer_categories_casual': 0.958613, - 'developer_categories_core': -1.277484, - 'developer_categories_regular': 0.193106, - 'file_types_code': -0.905957, - 'found_file_license': 0.376334, - 'returning_contributors': -1.167053, + "active_branches": -0.834069, + "commits_over_periods_rate": -0.841944, + "commit_size_added_lines": -0.389225, + "contributor_growth_rate": 0.114363, + "days_since_last_commit": 1.081051, + "developer_categories_casual": 0.958613, + "developer_categories_core": -1.277484, + "developer_categories_regular": 0.193106, + "file_types_code": -0.905957, + "found_file_license": 0.376334, + "returning_contributors": -1.167053, } - # Model Intercept Z = -1.023426 @@ -56,24 +56,20 @@ def __init__(self): def calculate_score(self, metrics: Dict[str, float]) -> float: """ - Calculates the probability of a repository being 'Unhealthy' based on + Calculates the probability of a repository being 'Unhealthy' based on the pruned logistic regression model metrics. - Parameters: metrics (dict): Dictionary containing the project feature names and values. - Returns: float: Probability score between 0.0 (Healthy) and 1.0 (Unhealthy). """ z = self.z - # Calculate the linear combination (log-odds) for metric, coef in self.coefficients.items(): # FIXME. We set by default 0 if a metric is missing. Is this safe? value = metrics.get(metric, 0.0) z += coef * value - # Apply the Sigmoid function to get the final probability try: probability = 1 / (1 + math.exp(-z)) @@ -81,12 +77,7 @@ def calculate_score(self, metrics: Dict[str, float]) -> float: # Safeguard against extreme values of z # FIXME Is this correct? probability = 0.0 if z < 0 else 1.0 - return { "value": probability, - "metadata": { - "ecosystem": self.ECOSYSTEM_NAME, - "model": self.MODEL_NAME, - "version": self.MODEL_VERSION - } + "metadata": {"ecosystem": self.ECOSYSTEM_NAME, "model": self.MODEL_NAME, "version": self.MODEL_VERSION}, } diff --git a/tests/end_to_end/base.py b/tests/end_to_end/base.py index 91ddfc8..8d4e84d 100644 --- a/tests/end_to_end/base.py +++ b/tests/end_to_end/base.py @@ -84,8 +84,8 @@ def _start_opensearch_container(self): def _start_grimoirelab(self): env = os.environ env["DJANGO_SETTINGS_MODULE"] = "grimoirelab.core.config.settings" - env["GRIMOIRELAB_REDIS_PORT"] = self.redis_container.get_exposed_port(6379) - env["GRIMOIRELAB_DB_PORT"] = self.mysql_container.get_exposed_port(3306) + env["GRIMOIRELAB_REDIS_PORT"] = str(self.redis_container.get_exposed_port(6379)) + env["GRIMOIRELAB_DB_PORT"] = str(self.mysql_container.get_exposed_port(3306)) env["GRIMOIRELAB_DB_PASSWORD"] = self.mysql_container.root_password env["GRIMOIRELAB_ARCHIVIST_STORAGE_URL"] = self.opensearch_url env["GRIMOIRELAB_USER_PASSWORD"] = "admin" diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index a2e7a13..2ad8ec2 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -22,10 +22,7 @@ import os import tempfile import unittest - import httpretty -import requests - from unittest.mock import patch from click.testing import CliRunner @@ -33,95 +30,151 @@ GRIMOIRELAB_URL = "http://localhost:8000" -OPENSEARCH_URL = "https://admin:admin@localhost:9200" +GRIMOIRELAB_USER = "admin" +GRIMOIRELAB_PASSWORD = "admin" +GRIMOIRELAB_ECOSYSTEM = "npm-training-set" +GRIMOIRELAB_PROJECT = "npm-popular-components" + +OPENSEARCH_URL = "https://localhost:9200" +OPENSEARCH_USER = "admin" +OPENSEARCH_PASSWORD = "admin" OPENSEARCH_INDEX = "events" -TASK_URL = f"{GRIMOIRELAB_URL}/datasources/add_repository" -REPOSITORIES_URL = f"{GRIMOIRELAB_URL}/datasources/repositories/" - +ERROR_GRIMOIRELAB_URL = "http://localhost:8001" + +REPOSITORIES_URL = f"{GRIMOIRELAB_URL}/api/v1/ecosystems/{GRIMOIRELAB_ECOSYSTEM}" f"/projects/{GRIMOIRELAB_PROJECT}/repos/" +ERROR_REPOSITORIES_URL = ( + f"{ERROR_GRIMOIRELAB_URL}/api/v1/ecosystems/{GRIMOIRELAB_ECOSYSTEM}" f"/projects/{GRIMOIRELAB_PROJECT}/repos/" +) + + +def command_args(source, output_path, *extra, grimoirelab_url=GRIMOIRELAB_URL): + """Build the invocation exactly like collect_and_store_grimoire_metric()""" + return [ + source, + "--grimoirelab-url", + grimoirelab_url, + "--grimoirelab-user", + GRIMOIRELAB_USER, + "--grimoirelab-password", + GRIMOIRELAB_PASSWORD, + "--grimoirelab-ecosystem", + GRIMOIRELAB_ECOSYSTEM, + "--grimoirelab-project", + GRIMOIRELAB_PROJECT, + "--opensearch-url", + OPENSEARCH_URL, + "--opensearch-index", + OPENSEARCH_INDEX, + "--opensearch-user", + OPENSEARCH_USER, + "--opensearch-password", + OPENSEARCH_PASSWORD, + "--output", + output_path, + *extra, + ] + + +def register_auth_endpoint(base_url): + """Mock authenticate endpoint""" + + def token_callback(request, uri, headers): + body = json.dumps({"token": "fake-token", "access": "fake-token", "access_token": "fake-token"}) + return (200, headers, body) -def setup_add_repository_mock_server(): - """Set up a mock HTTP server for API calls""" + httpretty.register_uri( + httpretty.POST, + f"{base_url}/token/", + responses=[httpretty.Response(body=token_callback)], + ) - http_requests = [] - def request_callback(request, uri, headers): - last_request = httpretty.last_request() - http_requests.append(last_request) - data = {"message": "Task scheduled correctly"} - body = json.dumps(data) +def repository_data(uri, status, last_run): + """Build the repositories payload returned by endpoint""" - return (200, headers, body) + return { + "count": 1, + "results": [ + { + "uri": uri, + "categories": [{"task": {"last_run": last_run, "status": status}}], + } + ], + } - def exception_callback(request, uri, headers): - last_request = httpretty.last_request() - http_requests.append(last_request) - raise requests.ConnectionError() +def setup_grimoirelab_mock_server(never_ending=False): + """Set up a GrimoireLab API mock used by the CLI.""" + register_auth_endpoint(GRIMOIRELAB_URL) - httpretty.register_uri(httpretty.POST, TASK_URL, responses=[httpretty.Response(body=request_callback)]) - httpretty.register_uri( - httpretty.POST, - "http://localhost:8001/datasources/add_repository", - responses=[httpretty.Response(body=exception_callback)], - ) + post_requests = [] + get_requests = [] + scheduled = set() - return http_requests + def get_callback(request, uri, headers): + get_requests.append(request) + repo_uri = request.querystring.get("uri", [None])[0] + if repo_uri not in scheduled: + data = {"count": 0, "results": []} + elif never_ending: + last_run = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=365) + data = repository_data(repo_uri, "running", last_run.isoformat()) + else: + last_run = datetime.datetime.now(datetime.timezone.utc) + data = repository_data(repo_uri, "completed", last_run.isoformat()) -def setup_get_repositories_mock_server(): - """Setup a mock HTTP server for repository API calls""" + return (200, headers, json.dumps(data)) - http_requests = [] + def post_callback(request, uri, headers): + try: + repo_uri = json.loads(request.body)["uri"] + except (ValueError, KeyError): + repo_uri = None - def request_callback(request, uri, headers): - last_request = httpretty.last_request() - http_requests.append(last_request) - data = { - "results": [ - { - "task": { - "last_run": datetime.datetime.now(tz=datetime.timezone.utc).isoformat(), - "status": "completed", - } - } - ] - } - body = json.dumps(data) + if repo_uri and repo_uri not in scheduled: + scheduled.add(repo_uri) + post_requests.append(request) - return 200, headers, body + return (200, headers, json.dumps({"message": "Task scheduled correctly"})) - httpretty.register_uri(httpretty.GET, REPOSITORIES_URL, responses=[httpretty.Response(body=request_callback)]) + httpretty.register_uri( + httpretty.GET, + REPOSITORIES_URL, + responses=[httpretty.Response(body=get_callback)], + ) + httpretty.register_uri( + httpretty.POST, + REPOSITORIES_URL, + responses=[httpretty.Response(body=post_callback)], + ) - return http_requests + return post_requests, get_requests -def setup_get_never_ending_repositories_mock_server(): - """Setup a mock HTTP server for repository API calls""" +def setup_grimoirelab_error_mock_server(): + """Set up a mock server whose repositories API always returns 500""" - http_requests = [] + register_auth_endpoint(ERROR_GRIMOIRELAB_URL) - def request_callback(request, uri, headers): - last_request = httpretty.last_request() - http_requests.append(last_request) - last_run = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(days=365) - data = { - "results": [ - { - "task": { - "last_run": last_run.isoformat(), - "status": "running", - } - } - ] - } - body = json.dumps(data) + post_requests = [] - return 200, headers, body + def error_callback(request, uri, headers): + return (500, headers, json.dumps({"detail": "internal server error"})) - httpretty.register_uri(httpretty.GET, REPOSITORIES_URL, responses=[httpretty.Response(body=request_callback)]) + httpretty.register_uri( + httpretty.GET, + ERROR_REPOSITORIES_URL, + responses=[httpretty.Response(body=error_callback)], + ) + httpretty.register_uri( + httpretty.POST, + ERROR_REPOSITORIES_URL, + responses=[httpretty.Response(body=error_callback)], + ) - return http_requests + return post_requests class TestCli(unittest.TestCase): @@ -138,32 +191,19 @@ def tearDown(self): def test_valid_file(self, mock_get_repository_metrics): """Check if it schedules tasks to analyze all git repositories from a valid file""" - http_requests = setup_add_repository_mock_server() - http_requests_repos = setup_get_repositories_mock_server() + httpretty.allow_net_connect = False + http_requests, http_requests_repos = setup_grimoirelab_mock_server() mock_get_repository_metrics.return_value = {"metrics": {"num_commits": 10}} runner = CliRunner() - result = runner.invoke( - grimoirelab_metrics, - [ - "./data/valid.spdx.xml", - "--grimoirelab-url", - GRIMOIRELAB_URL, - "--opensearch-url", - OPENSEARCH_URL, - "--opensearch-index", - OPENSEARCH_INDEX, - "--output", - self.temp_file.name, - ], - ) + result = runner.invoke(grimoirelab_metrics, command_args("./data/valid.spdx.xml", self.temp_file.name)) self.assertEqual(result.exit_code, 0) self.assertIn("Found 5 git repositories", result.output) - self.assertIn("Scheduling tasks", result.output) + self.assertIn("Scheduling data collection tasks", result.output) self.assertNotIn("Scheduling task to fetch commits", result.output) self.assertEqual(len(http_requests), 5) - self.assertEqual(len(http_requests_repos), 5) + self.assertEqual(len(http_requests_repos), 10) expected_packages = [ "SPDXRef-bootstrap-gnu-config.bst-0", @@ -186,232 +226,139 @@ def test_valid_file(self, mock_get_repository_metrics): def test_verbose(self, mock_get_repository_metrics): """Check if it logs all information when using '--verbose'""" - http_requests = setup_add_repository_mock_server() - http_requests_repos = setup_get_repositories_mock_server() + httpretty.allow_net_connect = False + http_requests, http_requests_repos = setup_grimoirelab_mock_server() mock_get_repository_metrics.return_value = {"metrics": {"num_commits": 10}} runner = CliRunner() - result = runner.invoke( - grimoirelab_metrics, - [ - "./data/valid.spdx.xml", - "--grimoirelab-url", - GRIMOIRELAB_URL, - "--opensearch-url", - OPENSEARCH_URL, - "--opensearch-index", - OPENSEARCH_INDEX, - "--output", - self.temp_file.name, - "--verbose", - ], - ) + result = runner.invoke(grimoirelab_metrics, command_args("./data/valid.spdx.xml", self.temp_file.name, "--verbose")) self.assertEqual(result.exit_code, 0) self.assertIn("Found 5 git repositories", result.output) - self.assertIn("Scheduling tasks", result.output) + self.assertIn("Scheduling data collection tasks", result.output) self.assertIn("Scheduling task to fetch commits", result.output) self.assertEqual(len(http_requests), 5) - self.assertEqual(len(http_requests_repos), 5) + self.assertEqual(len(http_requests_repos), 10) @httpretty.activate def test_invalid_file_type(self): """Check if it returns an error when the file type is not valid""" - http_requests = setup_add_repository_mock_server() + httpretty.allow_net_connect = False + http_requests, http_requests_repos = setup_grimoirelab_mock_server() + runner = CliRunner() - result = runner.invoke( - grimoirelab_metrics, - [ - "invalid.doc", - "--grimoirelab-url", - GRIMOIRELAB_URL, - "--opensearch-url", - OPENSEARCH_URL, - "--opensearch-index", - OPENSEARCH_INDEX, - "--output", - self.temp_file.name, - ], - ) + result = runner.invoke(grimoirelab_metrics, command_args("invalid.doc", self.temp_file.name)) - self.assertEqual(result.exit_code, 1) - self.assertIn("Unsupported SPDX file type", result.output) + self.assertEqual(result.exit_code, 0) + self.assertIn("The source is not a file and does not end with .git", result.output) self.assertEqual(len(http_requests), 0) + self.assertEqual(len(http_requests_repos), 0) @httpretty.activate def test_invalid_sbom_format(self): """Check if it returns an error when the SBoM is not formatted correctly""" - http_requests = setup_add_repository_mock_server() + httpretty.allow_net_connect = False + http_requests, http_requests_repos = setup_grimoirelab_mock_server() + runner = CliRunner() - result = runner.invoke( - grimoirelab_metrics, - [ - "./data/invalid_format.spdx.json", - "--grimoirelab-url", - GRIMOIRELAB_URL, - "--opensearch-url", - OPENSEARCH_URL, - "--opensearch-index", - OPENSEARCH_INDEX, - "--output", - self.temp_file.name, - ], - ) + result = runner.invoke(grimoirelab_metrics, command_args("./data/invalid_format.spdx.json", self.temp_file.name)) self.assertEqual(result.exit_code, 1) self.assertIn("Error while parsing document", result.output) self.assertEqual(len(http_requests), 0) + self.assertEqual(len(http_requests_repos), 0) @httpretty.activate @patch("grimoirelab_metrics.cli.get_repository_metrics") def test_no_repository(self, mock_get_repository_metrics): """Check if it returns a warning when a package does not provide a git repository""" - http_requests = setup_add_repository_mock_server() - http_requests_repos = setup_get_repositories_mock_server() + httpretty.allow_net_connect = False + http_requests, http_requests_repos = setup_grimoirelab_mock_server() mock_get_repository_metrics.return_value = {"metrics": {"num_commits": 10}} runner = CliRunner() - result = runner.invoke( - grimoirelab_metrics, - [ - "./data/missing_repo.spdx.xml", - "--grimoirelab-url", - GRIMOIRELAB_URL, - "--opensearch-url", - OPENSEARCH_URL, - "--opensearch-index", - OPENSEARCH_INDEX, - "--output", - self.temp_file.name, - ], - ) - + result = runner.invoke(grimoirelab_metrics, command_args("./data/missing_repo.spdx.xml", self.temp_file.name)) self.assertEqual(result.exit_code, 0) self.assertIn( "Could not find a git repository for SPDXRef-bootstrap-gnu-config.bst-0 (bootstrap/gnu-config.bst)", result.output, ) - self.assertEqual(len(http_requests), 4) @httpretty.activate @patch("grimoirelab_metrics.cli.get_repository_metrics") def test_invalid_git_repository(self, mock_get_repository_metrics): """Check if it returns a warning when a package URI is not a valid git repository""" - http_requests = setup_add_repository_mock_server() - http_requests_repos = setup_get_repositories_mock_server() + httpretty.allow_net_connect = False + http_requests, http_requests_repos = setup_grimoirelab_mock_server() mock_get_repository_metrics.return_value = {"metrics": {"num_commits": 10}} runner = CliRunner() - result = runner.invoke( - grimoirelab_metrics, - [ - "./data/invalid_repo.spdx.xml", - "--grimoirelab-url", - GRIMOIRELAB_URL, - "--opensearch-url", - OPENSEARCH_URL, - "--opensearch-index", - OPENSEARCH_INDEX, - "--output", - self.temp_file.name, - ], - ) - + result = runner.invoke(grimoirelab_metrics, command_args("./data/invalid_repo.spdx.xml", self.temp_file.name)) self.assertEqual(result.exit_code, 0) self.assertIn("Could not find a git repository for SPDXRef-ncurses-6.40 (bootstrap/ncurses.bst)", result.output) - self.assertEqual(len(http_requests), 4) - self.assertEqual(len(http_requests_repos), 4) @httpretty.activate def test_no_file(self): """Check if it returns an error when the file does not exist""" - http_requests = setup_add_repository_mock_server() + httpretty.allow_net_connect = False + http_requests, http_requests_repos = setup_grimoirelab_mock_server() + runner = CliRunner() - result = runner.invoke( - grimoirelab_metrics, - [ - "./data/no_file.xml", - "--grimoirelab-url", - GRIMOIRELAB_URL, - "--opensearch-url", - OPENSEARCH_URL, - "--opensearch-index", - OPENSEARCH_INDEX, - "--output", - self.temp_file.name, - ], - ) + result = runner.invoke(grimoirelab_metrics, command_args("./data/no_file.xml", self.temp_file.name)) - self.assertEqual(result.exit_code, 1) - self.assertIn("No such file or directory", result.output) + self.assertEqual(result.exit_code, 0) + self.assertIn("The source is not a file and does not end with .git", result.output) self.assertEqual(len(http_requests), 0) + self.assertEqual(len(http_requests_repos), 0) @httpretty.activate - def test_server_error(self): + @patch("grimoirelab_metrics.grimoirelab_client.time.sleep") + def test_server_error(self, mock_sleep): """Check if it returns a warning when there is a server error""" - - http_requests = setup_add_repository_mock_server() + httpretty.allow_net_connect = False + http_requests = setup_grimoirelab_error_mock_server() runner = CliRunner() result = runner.invoke( grimoirelab_metrics, - [ - "./data/valid.spdx.xml", - "--grimoirelab-url", - "http://localhost:8001", - "--opensearch-url", - OPENSEARCH_URL, - "--opensearch-index", - OPENSEARCH_INDEX, - "--output", - self.temp_file.name, - ], + command_args("./data/valid.spdx.xml", self.temp_file.name, grimoirelab_url=ERROR_GRIMOIRELAB_URL), ) self.assertEqual(result.exit_code, 1) self.assertIn("Error scheduling task", result.output) - self.assertEqual(len(http_requests), 5) + self.assertEqual(len(http_requests), 0) @httpretty.activate @patch("grimoirelab_metrics.cli.get_repository_metrics") def test_never_ending_repository(self, mock_get_repository_metrics): """Check if it returns a warning when a repository task never ends""" - http_requests = setup_add_repository_mock_server() - http_requests_repos = setup_get_never_ending_repositories_mock_server() + httpretty.allow_net_connect = False + http_requests, http_requests_repos = setup_grimoirelab_mock_server(never_ending=True) mock_get_repository_metrics.return_value = {"metrics": {"num_commits": 10}} runner = CliRunner() result = runner.invoke( grimoirelab_metrics, - [ - "./data/valid.spdx.xml", - "--grimoirelab-url", - GRIMOIRELAB_URL, - "--opensearch-url", - OPENSEARCH_URL, - "--opensearch-index", - OPENSEARCH_INDEX, - "--output", - self.temp_file.name, - "--repository-timeout", - 15, - ], + command_args("./data/valid.spdx.xml", self.temp_file.name, "--repository-timeout", "0"), ) self.assertEqual(result.exit_code, 0) - self.assertIn( - "Timeout waiting for repository https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux to be ready", - result.output, - ) + self.assertIn("Timeout waiting for repository", result.output) self.assertEqual(len(http_requests), 5) self.assertEqual(len(http_requests_repos), 10) + with open(self.temp_file.name) as f: + metrics = json.load(f) + self.assertEqual(len(metrics["packages"]), 5) + for data in metrics["packages"].values(): + self.assertIsNone(data["metrics"]) + class TestGetRepository(unittest.TestCase): def test_valid_git_repository(self): @@ -429,7 +376,7 @@ def test_valid_git_repository(self): for uri in valid_git_uris: with self.subTest(uri=uri): result = get_repository(uri) - self.assertEqual(result, "https://git.myproject.org/MyProject") + self.assertEqual(result, "https://git.myproject.org/MyProject.git") def test_invalid_git_repository(self): invalid_git_uris = [