diff --git a/config/docker/data/tast_parser.py b/config/docker/data/tast_parser.py index ec56beb75c..3f48560860 100755 --- a/config/docker/data/tast_parser.py +++ b/config/docker/data/tast_parser.py @@ -17,12 +17,12 @@ # along with this library; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from functools import partial -import subprocess -import sys -import os import json +import os import pwd +import subprocess +import sys +from functools import partial FAILED_RUN_FILE = "failed_run" STDERR_FILE = "stderr.log" @@ -69,7 +69,9 @@ def report_lava(test_data): if "measurements" in test_data: lava_test_set_start(test_data["name"]) for measurement in test_data["measurements"]: - report_lava_test_case(measurement["name"], test_data["result"], measurement) + report_lava_test_case( + measurement["name"], test_data["result"], measurement + ) lava_test_set_stop(test_data["name"]) else: report_lava_test_case(test_data["name"], test_data["result"]) @@ -178,7 +180,9 @@ def parse_test_results(): report_lava_critical("Tast tests run failed, no results") sys.exit(1) for test_data in parse_results(results): - results_chart = os.path.join(test_data["outDir"], RESULTS_CHART_FILE) + results_chart = os.path.join( + test_data["outDir"], RESULTS_CHART_FILE + ) if os.path.isfile(results_chart): with open(results_chart, "r") as rc_file: rc_data = json.load(rc_file) diff --git a/kernelci/api/__init__.py b/kernelci/api/__init__.py index e898f51747..429ca6762a 100644 --- a/kernelci/api/__init__.py +++ b/kernelci/api/__init__.py @@ -12,8 +12,8 @@ import urllib from typing import Dict, Optional, Sequence -from cloudevents.http import CloudEvent import requests +from cloudevents.http import CloudEvent from requests.adapters import HTTPAdapter from urllib3.util import Retry @@ -22,7 +22,9 @@ HTTP_ERROR_BODY_SNIPPET = 512 -def _http_error_body_snippet(response, max_length: int = HTTP_ERROR_BODY_SNIPPET): +def _http_error_body_snippet( + response, max_length: int = HTTP_ERROR_BODY_SNIPPET +): """Return a truncated response body useful for HTTP error logging. Also try to extract a more specific error message from the body if it's a JSON with a "detail", "error" or "message" field. @@ -150,7 +152,10 @@ def _get(self, path, params=None): session.mount("https://", adapter) resp = session.get( - url, params=params, headers=self.data.headers, timeout=self.data.timeout + url, + params=params, + headers=self.data.headers, + timeout=self.data.timeout, ) try: resp.raise_for_status() @@ -190,13 +195,19 @@ def _post(self, path, data=None, params=None, json_data=True): # requests.post, but in this case we have to explicitly # specify the Content-Type header if json_data: - headers = self.data.headers | {"Content-Type": "application/json"} + headers = self.data.headers | { + "Content-Type": "application/json" + } else: headers = self.data.headers | { "Content-Type": "application/x-www-form-urlencoded" } resp = session.post( - url, data, headers=headers, params=params, timeout=self.data.timeout + url, + data, + headers=headers, + params=params, + timeout=self.data.timeout, ) else: # When passing a dict to requests.post, it will @@ -325,7 +336,9 @@ def _delete(self, path): session.mount("http://", adapter) session.mount("https://", adapter) - resp = session.delete(url, headers=self.data.headers, timeout=self.data.timeout) + resp = session.delete( + url, headers=self.data.headers, timeout=self.data.timeout + ) try: resp.raise_for_status() except requests.exceptions.HTTPError as err: diff --git a/kernelci/api/helper.py b/kernelci/api/helper.py index 283dca931d..8f3f42fce2 100644 --- a/kernelci/api/helper.py +++ b/kernelci/api/helper.py @@ -7,9 +7,10 @@ """KernelCI API helpers""" -from typing import Dict import json import os +from typing import Dict + import requests from . import API @@ -36,7 +37,9 @@ def merge(primary: dict, secondary: dict): for key in primary: result[key] = primary[key] if key in secondary: - if isinstance(primary[key], dict) and isinstance(secondary[key], dict): + if isinstance(primary[key], dict) and isinstance( + secondary[key], dict + ): result[key] = merge(primary[key], secondary[key]) else: result[key] = secondary[key] @@ -62,7 +65,9 @@ def api(self): """API object""" return self._api - def subscribe_filters(self, filters=None, channel="node", promiscuous=False): + def subscribe_filters( + self, filters=None, channel="node", promiscuous=False + ): """Subscribe to a channel with some added filters""" sub_id = self.api.subscribe(channel, promiscuous) self._filters[sub_id] = filters @@ -140,7 +145,9 @@ def receive_event_node(self, sub_id): # Crude (provisional) filtering of non-node events if not node: continue - if all(self.pubsub_event_filter(sub_id, obj) for obj in [node, event]): + if all( + self.pubsub_event_filter(sub_id, obj) for obj in [node, event] + ): return node, event.get("is_hierarchy") def _find_container(self, field, node): @@ -188,7 +195,9 @@ def _is_allowed(self, rules, key, node): # the node on the basis that its rules aren; t fulfilled if not base: if len(allow) > 0: - print(f"rules[{key}]: attribute '{key}' not found in node hierarchy") + print( + f"rules[{key}]: attribute '{key}' not found in node hierarchy" + ) return False return True @@ -217,7 +226,9 @@ def _is_allowed(self, rules, key, node): found = True if not found: - print(f"rules[{key}]: {key.capitalize()} missing one of {allow}") + print( + f"rules[{key}]: {key.capitalize()} missing one of {allow}" + ) return False else: @@ -280,7 +291,10 @@ def _match_combos(self, node, combos): """ match = None for combo in combos: - if node["tree"] == combo["tree"] and node["branch"] == combo["branch"]: + if ( + node["tree"] == combo["tree"] + and node["branch"] == combo["branch"] + ): match = combo break return match @@ -415,7 +429,8 @@ def should_create_node(self, rules, node): rule_major = rules[key]["version"] rule_minor = rules[key]["patchlevel"] if key.startswith("min") and ( - (major < rule_major) or (major == rule_major and minor < rule_minor) + (major < rule_major) + or (major == rule_major and minor < rule_minor) ): print( f"rules[{key}]: Version {major}.{minor} older than minimum version " @@ -423,7 +438,8 @@ def should_create_node(self, rules, node): ) return False if key.startswith("max") and ( - (major > rule_major) or (major == rule_major and minor > rule_minor) + (major > rule_major) + or (major == rule_major and minor > rule_minor) ): print( f"rules[{key}]: Version {major}.{minor} newer than maximum version " @@ -473,7 +489,9 @@ def _is_job_filtered(self, node): # * jobfilter contains the job name suffixed with '+' # * at least one element of the node's 'path' appears in jobilfter # with a '+' suffix - for filt in (item.rstrip("+") for item in jobfilter if item.endswith("+")): + for filt in ( + item.rstrip("+") for item in jobfilter if item.endswith("+") + ): if filt in node["path"] or filt == node["name"]: return False @@ -483,7 +501,13 @@ def _is_job_filtered(self, node): # pylint: disable=too-many-arguments def create_job_node( - self, job_config, input_node, *, runtime=None, platform=None, retry_counter=0 + self, + job_config, + input_node, + *, + runtime=None, + platform=None, + retry_counter=0, ): """Create a new job node based on input and configuration""" jobfilter = input_node.get("jobfilter") @@ -525,10 +549,14 @@ def create_job_node( # Test-specific fields inherited from parent node (kbuild or # job) if available if job_config.kind == "job": - job_node["data"]["kernel_type"] = input_node["data"].get("kernel_type") + job_node["data"]["kernel_type"] = input_node["data"].get( + "kernel_type" + ) job_node["data"]["arch"] = input_node["data"].get("arch") job_node["data"]["defconfig"] = input_node["data"].get("defconfig") - job_node["data"]["config_full"] = input_node["data"].get("config_full") + job_node["data"]["config_full"] = input_node["data"].get( + "config_full" + ) job_node["data"]["compiler"] = input_node["data"].get("compiler") # This information is highly useful, as we might # extract from it the following, for example: @@ -574,7 +602,9 @@ def create_job_node( job_node = self._fsanitize_node_fields(job_node, "commit_message") try: - job_node["data"] = platform.format_params(job_node["data"], extra_args) + job_node["data"] = platform.format_params( + job_node["data"], extra_args + ) except Exception as error: print(f"Exception Error, node id: {input_node['id']}, {error}") raise error diff --git a/kernelci/api/models.py b/kernelci/api/models.py index 1a60a73b11..7ca6606e32 100644 --- a/kernelci/api/models.py +++ b/kernelci/api/models.py @@ -13,13 +13,13 @@ """KernelCI API model definitions used by client-facing endpoints""" -from datetime import datetime, timedelta -from typing import Any, Optional, Dict, List, ClassVar, Literal import enum +import json import os +from datetime import datetime, timedelta from operator import attrgetter -import json -from typing_extensions import Annotated +from typing import Any, ClassVar, Dict, List, Literal, Optional + from bson import ObjectId from pydantic import ( AnyHttpUrl, @@ -27,13 +27,15 @@ BaseModel, BeforeValidator, Field, - field_validator, StrictInt, TypeAdapter, + field_validator, ) +from typing_extensions import Annotated + from .models_base import ( - PyObjectId, DatabaseModel, + PyObjectId, ) any_url_adapter = TypeAdapter(AnyUrl) @@ -41,7 +43,9 @@ # TTL configuration for time-limited collections # Set environment variables to override defaults -EVENT_HISTORY_TTL_SECONDS = int(os.getenv("EVENT_HISTORY_TTL_SECONDS", "604800")) +EVENT_HISTORY_TTL_SECONDS = int( + os.getenv("EVENT_HISTORY_TTL_SECONDS", "604800") +) TELEMETRY_TTL_SECONDS = int(os.getenv("TELEMETRY_TTL_SECONDS", "1209600")) @@ -93,7 +97,9 @@ class ErrorCodes(str, enum.Enum): class KernelVersion(BaseModel): """Linux kernel version model""" - version: StrictInt = Field(description="Major version number e.g. 4 in 'v4.19'") + version: StrictInt = Field( + description="Major version number e.g. 4 in 'v4.19'" + ) patchlevel: StrictInt = Field( description="Minor version number or 'patch level' e.g. 19 in 'v4.19'" ) @@ -102,7 +108,8 @@ class KernelVersion(BaseModel): default=None, ) extra: Optional[str] = Field( - description="Extra version string e.g. -rc2 in 'v4.19-rc2'", default=None + description="Extra version string e.g. -rc2 in 'v4.19-rc2'", + default=None, ) name: Optional[str] = Field( description="Version name e.g. People's Front for v4.19", default=None @@ -124,16 +131,23 @@ class Revision(BaseModel): tree: str = Field(description="git tree of the revision") url: Annotated[ - str, BeforeValidator(lambda value: str(any_url_adapter.validate_python(value))) + str, + BeforeValidator( + lambda value: str(any_url_adapter.validate_python(value)) + ), ] = Field(description="git URL of the revision") branch: str = Field(description="git branch of the revision") commit: str = Field(description="git commit SHA of the revision") describe: Optional[str] = Field( default=None, description="git describe of the revision" ) - version: Optional[KernelVersion] = Field(default=None, description="Kernel version") + version: Optional[KernelVersion] = Field( + default=None, description="Kernel version" + ) patchset: Optional[str] = Field(default=None, description="Patchset hash") - commit_tags: List[str] = Field(description="List of git commit tags", default=[]) + commit_tags: List[str] = Field( + description="List of git commit tags", default=[] + ) commit_message: Optional[str] = Field( default=None, description="git commit message" ) @@ -181,7 +195,9 @@ class Node(DatabaseModel): group: Optional[str] = Field( description="Name of a group this node belongs to", default=None ) - parent: Optional[PyObjectId] = Field(description="Parent commit SHA", default=None) + parent: Optional[PyObjectId] = Field( + description="Parent commit SHA", default=None + ) state: StateValues = Field( default=StateValues.RUNNING.value, description="State of the node" ) @@ -195,12 +211,15 @@ class Node(DatabaseModel): Annotated[ str, BeforeValidator( - lambda value: str(any_http_url_adapter.validate_python(value)) + lambda value: str( + any_http_url_adapter.validate_python(value) + ) ), ], ] ] = Field( - description="Artifacts associated with the node (binaries, logs...)", default={} + description="Artifacts associated with the node (binaries, logs...)", + default={}, ) data: Optional[Dict[str, Any]] = Field( description="Arbitrary data stored in the node", default={} @@ -209,14 +228,16 @@ class Node(DatabaseModel): description="Debug info fields (for development purposes)", default={} ) jobfilter: Optional[List[str]] = Field( - description="Restrict jobs that can be scheduled by this node", default=None + description="Restrict jobs that can be scheduled by this node", + default=None, ) platform_filter: Optional[List[str]] = Field( default=[], description="Restrict test jobs to be scheduled on specific platforms", ) created: datetime = Field( - default_factory=datetime.utcnow, description="Timestamp of node creation" + default_factory=datetime.utcnow, + description="Timestamp of node creation", ) updated: datetime = Field( default_factory=datetime.utcnow, @@ -227,14 +248,19 @@ class Node(DatabaseModel): description="Node expiry timestamp", ) holdoff: Optional[datetime] = Field( - description="Node expiry timestamp while in Available state", default=None + description="Node expiry timestamp while in Available state", + default=None, + ) + owner: Optional[str] = Field( + description="Username of node owner", default=None ) - owner: Optional[str] = Field(description="Username of node owner", default=None) submitter: Optional[str] = Field( description="Token md5 hash to identify node origin(submitter token)", default=None, ) - treeid: Optional[str] = Field(description="Tree unique identifier", default=None) + treeid: Optional[str] = Field( + description="Tree unique identifier", default=None + ) user_groups: List[str] = Field( default=[], description="User groups that are permitted to update node" ) @@ -249,7 +275,12 @@ class Node(DatabaseModel): ) OBJECT_ID_FIELDS: ClassVar[list] = ["parent"] - TIMESTAMP_FIELDS: ClassVar[list] = ["created", "updated", "timeout", "holdoff"] + TIMESTAMP_FIELDS: ClassVar[list] = [ + "created", + "updated", + "timeout", + "holdoff", + ] def update(self): self.updated = datetime.utcnow() @@ -426,7 +457,9 @@ class KbuildData(BaseModel): kernel_revision: Optional[Revision] = Field( description="Kernel repo revision data", default=None ) - arch: Optional[str] = Field(description="CPU architecture family", default=None) + arch: Optional[str] = Field( + description="CPU architecture family", default=None + ) defconfig: Optional[str] = Field( description="Kernel defconfig identifier", default=None ) @@ -438,7 +471,8 @@ class KbuildData(BaseModel): ) error_msg: Optional[str] = Field(description="Error message", default=None) fragments: Optional[List[str]] = Field( - description="List of additional configuration fragments used", default=None + description="List of additional configuration fragments used", + default=None, ) config_full: Optional[str] = Field( description=( @@ -459,7 +493,8 @@ class KbuildData(BaseModel): description="Kernel image type (zimage, bzimage...)", default=None ) regression: Optional[PyObjectId] = Field( - description="Regression node related to this build instance", default=None + description="Regression node related to this build instance", + default=None, ) @@ -489,9 +524,13 @@ class TestData(BaseModel): test_source: Optional[ Annotated[ str, - BeforeValidator(lambda value: str(any_url_adapter.validate_python(value))), + BeforeValidator( + lambda value: str(any_url_adapter.validate_python(value)) + ), ] - ] = Field(description="Repository containing the test source code", default=None) + ] = Field( + description="Repository containing the test source code", default=None + ) test_revision: Optional[Revision] = Field( description="Test repo revision data", default=None ) @@ -513,7 +552,9 @@ class TestData(BaseModel): kernel_revision: Optional[Revision] = Field( description="Kernel repo revision data", default=None ) - arch: Optional[str] = Field(description="CPU architecture family", default=None) + arch: Optional[str] = Field( + description="CPU architecture family", default=None + ) defconfig: Optional[str] = Field( description="Kernel defconfig identifier", default=None ) @@ -609,7 +650,9 @@ class RegressionData(BaseModel): failed_kernel_revision: Optional[Revision] = Field( description="Kernel repo revision data of the failed job", default=None ) - arch: Optional[str] = Field(description="CPU architecture family", default=None) + arch: Optional[str] = Field( + description="CPU architecture family", default=None + ) defconfig: Optional[str] = Field( description="Kernel defconfig identifier", default=None ) @@ -714,11 +757,13 @@ def create_regression(cls, fail_node, pass_node, as_dict=False): ) if pass_node.result != "pass": raise RuntimeError( - error_msg + f"The pass node has a wrong result: {pass_node.result}" + error_msg + + f"The pass node has a wrong result: {pass_node.result}" ) if fail_node.result != "fail": raise RuntimeError( - error_msg + f"The fail node has a wrong result: {fail_node.result}" + error_msg + + f"The fail node has a wrong result: {fail_node.result}" ) # End of sanity checks data_field = { @@ -748,7 +793,9 @@ class PublishEvent(BaseModel): """API model for the data of a event""" data: Any = Field(description="Event payload", default=None) - type: Optional[str] = Field(description="Type of the event", default=None) + type: Optional[str] = Field( + description="Type of the event", default=None + ) source: Optional[str] = Field( description="Source of the event", default=None ) @@ -784,9 +831,12 @@ class EventHistory(DatabaseModel): description="Timestamp of event creation", default_factory=datetime.now ) sequence_id: Optional[int] = Field( - default=None, description="Sequential ID for pub/sub ordering (auto-generated)" + default=None, + description="Sequential ID for pub/sub ordering (auto-generated)", + ) + channel: Optional[str] = Field( + default="node", description="Pub/Sub channel name" ) - channel: Optional[str] = Field(default="node", description="Pub/Sub channel name") owner: Optional[str] = Field( default=None, description="Username of event publisher" ) @@ -802,7 +852,9 @@ def get_indexes(cls): Also creates compound index for efficient pub/sub catch-up queries. """ return [ - cls.Index("timestamp", {"expireAfterSeconds": EVENT_HISTORY_TTL_SECONDS}), + cls.Index( + "timestamp", {"expireAfterSeconds": EVENT_HISTORY_TTL_SECONDS} + ), cls.Index([("channel", 1), ("sequence_id", 1)], {}), ] @@ -830,28 +882,37 @@ class TelemetryEvent(DatabaseModel): device_id: Optional[str] = Field( default=None, description="Actual device identifier (LAVA only)" ) - job_name: Optional[str] = Field(default=None, description="Job configuration name") + job_name: Optional[str] = Field( + default=None, description="Job configuration name" + ) test_name: Optional[str] = Field( default=None, description="Individual test case name" ) job_id: Optional[str] = Field( default=None, description="Runtime job identifier (e.g. LAVA job ID)" ) - node_id: Optional[str] = Field(default=None, description="API node identifier") + node_id: Optional[str] = Field( + default=None, description="API node identifier" + ) tree: Optional[str] = Field(default=None, description="Kernel tree name") - branch: Optional[str] = Field(default=None, description="Kernel branch name") + branch: Optional[str] = Field( + default=None, description="Kernel branch name" + ) arch: Optional[str] = Field(default=None, description="CPU architecture") result: Optional[str] = Field( default=None, description="Result: pass, fail, skip, incomplete" ) is_infra_error: bool = Field( - default=False, description="Whether the failure is an infrastructure error" + default=False, + description="Whether the failure is an infrastructure error", ) error_type: Optional[str] = Field( default=None, description="Error category: online_check, submission, queue_depth, etc.", ) - error_msg: Optional[str] = Field(default=None, description="Error message details") + error_msg: Optional[str] = Field( + default=None, description="Error message details" + ) retry: int = Field(default=0, description="Retry attempt counter") extra: Dict[str, Any] = Field( default={}, description="Additional fields for future extensibility" diff --git a/kernelci/api/models_base.py b/kernelci/api/models_base.py index 2f02e96d0e..f7b2b623e7 100644 --- a/kernelci/api/models_base.py +++ b/kernelci/api/models_base.py @@ -12,13 +12,14 @@ """Common KernelCI API model definitions""" -from typing import Optional, Any, Dict, List, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union + from bson import ObjectId from pydantic import ( BaseModel, Field, - model_serializer, SerializationInfo, + model_serializer, ) from pydantic.dataclasses import dataclass from pydantic_core import core_schema @@ -44,7 +45,9 @@ def __get_pydantic_core_schema__( core_schema.chain_schema( [ core_schema.str_schema(), - core_schema.no_info_plain_validator_function(cls.validate), + core_schema.no_info_plain_validator_function( + cls.validate + ), ] ), ] diff --git a/kernelci/build.py b/kernelci/build.py index f0882962ae..5b714d279a 100644 --- a/kernelci/build.py +++ b/kernelci/build.py @@ -15,7 +15,6 @@ # along with this library; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from datetime import datetime import fnmatch import itertools import json @@ -25,14 +24,19 @@ import shutil import tarfile import time +from datetime import datetime import requests -from kernelci import shell_cmd, print_flush, __version__ as kernelci_version -import kernelci.elf + import kernelci.config +import kernelci.elf +from kernelci import __version__ as kernelci_version +from kernelci import print_flush, shell_cmd # This is used to get the mainline tags as a minimum for git describe -TORVALDS_GIT_URL = "git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" +TORVALDS_GIT_URL = ( + "git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" +) CIP_CONFIG_URL = "https://gitlab.com/cip-project/cip-kernel/cip-kernel-config/-\ /raw/master/{branch}/{config}" @@ -158,7 +162,10 @@ def update_repo(config, path, ref=None): ref_opt = "--reference={ref}".format(ref=ref) if ref else "" shell_cmd( "git clone {ref} -o {remote} {url} {path}".format( - ref=ref_opt, remote=config.tree.name, url=config.tree.url, path=path + ref=ref_opt, + remote=config.tree.name, + url=config.tree.url, + path=path, ) ) @@ -634,7 +641,9 @@ def add_artifact(self, step_name, directory, file_name, key=None): path = os.path.join(directory, file_name) return self._add_artifact(step_name, "file", path, None, key) - def add_artifact_contents(self, step_name, artifact_type, path, contents, key=None): + def add_artifact_contents( + self, step_name, artifact_type, path, contents, key=None + ): """Add meta-data for artifacts with file contents Add a meta-data entry for an artifact with a list of files as its @@ -1443,7 +1452,9 @@ def _get_modules_artifacts(self, modules_tarball): list( set( path - for path in (os.path.basename(entry.name) for entry in tarball) + for path in ( + os.path.basename(entry.name) for entry in tarball + ) if path and path.endswith(".ko") ) ) @@ -1594,7 +1605,8 @@ def _get_kselftests(self, kselftest_tarball): kselftests = set( path for path in ( - os.path.split(os.path.relpath(entry.name))[0] for entry in tarball + os.path.split(os.path.relpath(entry.name))[0] + for entry in tarball ) if path ) diff --git a/kernelci/cli/__init__.py b/kernelci/cli/__init__.py index 118736195e..0e4ee6c77b 100644 --- a/kernelci/cli/__init__.py +++ b/kernelci/cli/__init__.py @@ -17,8 +17,8 @@ import functools import json import re -import typing import traceback +import typing import click import requests @@ -41,7 +41,9 @@ class Args: # pylint: disable=too-few-public-methods help="Path to the YAML pipeline configuration", ) indent = click.option( - "--indent", type=int, help="Intentation level for structured data output" + "--indent", + type=int, + help="Intentation level for structured data output", ) page_length = click.option( "-l", "--page-length", type=int, help="Page length in paginated data" @@ -51,11 +53,17 @@ class Args: # pylint: disable=too-few-public-methods ) runtime = click.option("--runtime", help="Name of the runtime config entry") settings = click.option( - "-s", "--toml-settings", "settings", help="Path to the TOML user settings" + "-s", + "--toml-settings", + "settings", + help="Path to the TOML user settings", ) storage = click.option("--storage", help="Name of the storage config entry") verbose = click.option( - "-v", "--verbose/--no-verbose", default=None, help="Print more details output" + "-v", + "--verbose/--no-verbose", + default=None, + help="Print more details output", ) debug = click.option( "-d", @@ -228,7 +236,8 @@ def split_attributes(attributes: typing.List[str]): parsed.setdefault("".join((name, opstr)), []).append(value) return { - name: value[0] if len(value) == 1 else value for name, value in parsed.items() + name: value[0] if len(value) == 1 else value + for name, value in parsed.items() } @@ -243,15 +252,21 @@ def get_pagination(page_length: int, page_number: int): if page_length is None: page_length = 10 elif page_length < 1: - raise click.UsageError(f"Page length must be at least 1, got {page_length}") + raise click.UsageError( + f"Page length must be at least 1, got {page_length}" + ) if page_number is None: page_number = 0 elif page_number < 0: - raise click.UsageError(f"Page number must be at least 0, got {page_number}") + raise click.UsageError( + f"Page number must be at least 0, got {page_number}" + ) return page_number * page_length, page_length -def get_api(config, api, secrets: typing.Optional[kernelci.settings.Secrets] = None): +def get_api( + config, api, secrets: typing.Optional[kernelci.settings.Secrets] = None +): """Get an API object instance Return an API object based on the given `api` config name loaded from the @@ -265,7 +280,9 @@ def get_api(config, api, secrets: typing.Optional[kernelci.settings.Secrets] = N raise click.ClickException("No API section found in the toml config") api_config = api_section.get(api, None) if api_config is None: - raise click.ClickException(f"API config {api} not found in the toml config") + raise click.ClickException( + f"API config {api} not found in the toml config" + ) token = secrets.api.token if secrets else None return kernelci.api.get_api(api_config, token) diff --git a/kernelci/cli/config.py b/kernelci/cli/config.py index 6f4ac12c1e..08b7711c73 100644 --- a/kernelci/cli/config.py +++ b/kernelci/cli/config.py @@ -5,18 +5,18 @@ """Tool to manage the KernelCI YAML pipeline configuration""" +import json import os import sys -import json import click import yaml from jinja2 import Environment, FileSystemLoader -import kernelci.config import kernelci.api.helper -from . import Args, kci +import kernelci.config +from . import Args, kci REPORT_TEMPLATE_PATHS = [ "config/report", @@ -71,7 +71,11 @@ def dump(section, config, indent, recursive): data = kernelci.config.load(config) if section: for step in section.split("."): - data = data.get(step, {}) if isinstance(data, dict) else getattr(data, step) + data = ( + data.get(step, {}) + if isinstance(data, dict) + else getattr(data, step) + ) if not data: raise click.ClickException(f"Section not found: {section}") if isinstance(data, dict) and not recursive: @@ -202,7 +206,9 @@ def get_forecast_data(merged_data): if job_kind == "kbuild": # check "params" "arch" job_params = ( - merged_data.get("jobs", {}).get(job_name, {}).get("params", {}) + merged_data.get("jobs", {}) + .get(job_name, {}) + .get("params", {}) ) arch = job_params.get("arch") if checkout.get("architectures") and arch not in checkout.get( @@ -271,17 +277,23 @@ def generate_html_report(checkouts, output_dir): final_html = template.render(checkouts=checkouts) - with open(os.path.join(output_dir, "index.html"), "w", encoding="utf-8") as outfile: + with open( + os.path.join(output_dir, "index.html"), "w", encoding="utf-8" + ) as outfile: outfile.write(final_html) - print(f"Forecast report generated at: {os.path.join(output_dir, 'index.html')}") + print( + f"Forecast report generated at: {os.path.join(output_dir, 'index.html')}" + ) @kci_config.command @Args.config @Args.debug @click.option( - "--html", type=click.Path(), help="Generate HTML report in the specified directory" + "--html", + type=click.Path(), + help="Generate HTML report in the specified directory", ) def forecast(config, debug, html): """Forecast builds and tests for each tree/branch combination""" diff --git a/kernelci/cli/docker.py b/kernelci/cli/docker.py index 6a281511e1..ebcb12d2cc 100644 --- a/kernelci/cli/docker.py +++ b/kernelci/cli/docker.py @@ -10,6 +10,7 @@ import click from kernelci.docker import Docker + from . import Args, kci @@ -59,7 +60,9 @@ def name(image, fragments, arch, prefix, version): def _get_docker_args(build_arg): try: - docker_args = dict(barg.split("=") for barg in build_arg) if build_arg else {} + docker_args = ( + dict(barg.split("=") for barg in build_arg) if build_arg else {} + ) return docker_args except ValueError as exc: raise click.UsageError(f"Invalid --build-arg value: {exc}") diff --git a/kernelci/cli/job.py b/kernelci/cli/job.py index aa1d3f496e..661169ec59 100644 --- a/kernelci/cli/job.py +++ b/kernelci/cli/job.py @@ -10,6 +10,7 @@ import kernelci.config import kernelci.runtime + from . import ( Args, catch_error, @@ -75,7 +76,9 @@ def new( @kci_job.command(secrets=True) @click.argument("node-id") @click.option("--platform", help="Name of the platform to run the job") -@click.option("--output", help="Path of the directory where to generate the job data") +@click.option( + "--output", help="Path of the directory where to generate the job data" +) @Args.runtime @Args.storage @Args.config @@ -107,7 +110,9 @@ def generate( raise click.ClickException("No jobs section found in the config") job_config = jobs_section.get(job_node["name"], None) if job_config is None: - raise click.ClickException(f"Job {job_node['name']} not found in the config") + raise click.ClickException( + f"Job {job_node['name']} not found in the config" + ) job = kernelci.runtime.Job(job_node, job_config) if platform is None: if "platform" not in job_node["data"]: diff --git a/kernelci/cli/node.py b/kernelci/cli/node.py index df5ccaa2bb..80cae1f496 100644 --- a/kernelci/cli/node.py +++ b/kernelci/cli/node.py @@ -9,6 +9,7 @@ import json import click + import kernelci.config from . import ( diff --git a/kernelci/cli/storage.py b/kernelci/cli/storage.py index 60523859e7..64b865e4a1 100644 --- a/kernelci/cli/storage.py +++ b/kernelci/cli/storage.py @@ -11,7 +11,8 @@ import kernelci.config import kernelci.storage -from . import Args, kci, catch_error + +from . import Args, catch_error, kci @kci.group(name="storage") @@ -29,7 +30,9 @@ def upload(filename, path, config, storage, secrets): """Upload FILENAME to the designated storage service in PATH""" configs = kernelci.config.load(config) storage_config = configs["storage"][storage] - storage = kernelci.storage.get_storage(storage_config, secrets.storage.credentials) + storage = kernelci.storage.get_storage( + storage_config, secrets.storage.credentials + ) url = storage.upload_single( file_path=(filename, os.path.basename(filename)), dest_path=(path or "") ) diff --git a/kernelci/cli/user.py b/kernelci/cli/user.py index 79cf4bd3b1..11a219209d 100644 --- a/kernelci/cli/user.py +++ b/kernelci/cli/user.py @@ -5,8 +5,8 @@ """User management commands""" import base64 -import json import datetime +import json import time import click diff --git a/kernelci/config/__init__.py b/kernelci/config/__init__.py index 12528b8c8e..ec38bad280 100644 --- a/kernelci/config/__init__.py +++ b/kernelci/config/__init__.py @@ -8,9 +8,11 @@ import glob import importlib import os + import yaml import kernelci + from .base import default_filters_from_yaml @@ -55,12 +57,16 @@ def validate_yaml(config_paths, entries): try: for path in get_config_paths(config_paths): for yaml_path, data in iterate_yaml_files(path): - for name, value in ((k, v) for k, v in data.items() if k in entries): + for name, value in ( + (k, v) for k, v in data.items() if k in entries + ): if isinstance(value, dict): keys = value.keys() elif isinstance(value, list): keys = ( - [] if len(value) and isinstance(value[0], dict) else value + [] + if len(value) and isinstance(value[0], dict) + else value ) else: keys = [] @@ -206,7 +212,9 @@ def resolve_rootfs_params(params, rootfs_defs): f"rootfs_ref '{rootfs_ref}' used but no rootfs definitions found in config" ) if rootfs_ref not in rootfs_defs: - raise ValueError(f"rootfs_ref '{rootfs_ref}' not found in rootfs config") + raise ValueError( + f"rootfs_ref '{rootfs_ref}' not found in rootfs config" + ) for key, value in rootfs_defs[rootfs_ref].items(): if key not in params: params[key] = value diff --git a/kernelci/config/base.py b/kernelci/config/base.py index cf91af6c80..5ef06bc591 100644 --- a/kernelci/config/base.py +++ b/kernelci/config/base.py @@ -12,7 +12,6 @@ import yaml - BUILDROOT_ARCH = { "arm": "armel", "x86_64": "x86", @@ -121,7 +120,11 @@ def _kw_from_yaml(cls, data, attributes): """ return ( - {k: v for k, v in ((k, data.get(k)) for k in attributes) if v is not None} + { + k: v + for k, v in ((k, data.get(k)) for k in attributes) + if v is not None + } if data else {} ) @@ -203,7 +206,11 @@ def _kw_from_yaml(cls, data, attributes): """ return ( - {k: v for k, v in ((k, data.get(k)) for k in attributes) if v is not None} + { + k: v + for k, v in ((k, data.get(k)) for k in attributes) + if v is not None + } if data else {} ) @@ -229,7 +236,8 @@ def to_dict(self): return { attr: value for attr, value in ( - (attr, getattr(self, attr)) for attr in self._get_yaml_attributes() + (attr, getattr(self, attr)) + for attr in self._get_yaml_attributes() ) if value is not None } diff --git a/kernelci/context/__init__.py b/kernelci/context/__init__.py index febe9ccc48..791c6cfa6e 100644 --- a/kernelci/context/__init__.py +++ b/kernelci/context/__init__.py @@ -14,6 +14,7 @@ import os import sys from typing import Any, Dict, List, Optional, Union + import toml import yaml @@ -504,7 +505,9 @@ def init_storage(self, name: str) -> Optional["kernelci.storage.Storage"]: try: return kernelci.storage.get_storage(storage_config, credentials) except (AttributeError, ValueError, TypeError) as e: - print(f"Failed to initialize storage '{name}': {e}", file=sys.stderr) + print( + f"Failed to initialize storage '{name}': {e}", file=sys.stderr + ) return None def init_api(self, name: str) -> Optional[Dict[str, Any]]: @@ -652,7 +655,9 @@ def create_context( KContext instance with loaded configuration and secrets """ return KContext( - config_paths=config_paths, secrets_path=secrets_path, program_name=program_name + config_paths=config_paths, + secrets_path=secrets_path, + program_name=program_name, ) diff --git a/kernelci/elf.py b/kernelci/elf.py index b4623813ce..0e1761e899 100644 --- a/kernelci/elf.py +++ b/kernelci/elf.py @@ -21,14 +21,18 @@ """Open the ELF file and read some of its content.""" -import elftools.elf.constants as elfconst -import elftools.elf.elffile as elffile import io import os +import elftools.elf.constants as elfconst +import elftools.elf.elffile as elffile + # Default section names and their build document keys to look in the ELF file. # These are supposed to always be available. -DEFAULT_ELF_SECTIONS = [(".bss", "vmlinux_bss_size"), (".text", "vmlinux_text_size")] +DEFAULT_ELF_SECTIONS = [ + (".bss", "vmlinux_bss_size"), + (".text", "vmlinux_text_size"), +] # Write/Alloc & Alloc constant we need to check for in the ELF sections. ELF_WA_FLAG = elfconst.SH_FLAGS.SHF_WRITE | elfconst.SH_FLAGS.SHF_ALLOC diff --git a/kernelci/kbuild.py b/kernelci/kbuild.py index 981891ddc2..e1c54a8e40 100644 --- a/kernelci/kbuild.py +++ b/kernelci/kbuild.py @@ -24,24 +24,26 @@ - kselftest: false - do not build kselftest """ -from datetime import datetime, timedelta +import concurrent.futures +import json import os -import sys import re +import subprocess +import sys import tarfile -import json -import requests +import threading import time +from datetime import datetime, timedelta +from typing import Tuple + +import requests import yaml -import concurrent.futures -import threading -import subprocess + import kernelci.api import kernelci.api.helper import kernelci.config import kernelci.config.storage import kernelci.storage -from typing import Tuple CIP_CONFIG_URL = "https://gitlab.com/cip-project/cip-kernel/cip-kernel-config/-/raw/master/{branch}/{config}" # noqa CROS_CONFIG_URL = "https://chromium.googlesource.com/chromiumos/third_party/kernel/+archive/refs/heads/{branch}/chromeos/config.tar.gz" # noqa @@ -252,7 +254,9 @@ def __init__( ) self._full_artifacts = jsonobj["full_artifacts"] self._dtbs_check = jsonobj["dtbs_check"] - self._kselftest = jsonobj.get("kselftest", jsonobj.get("kfselftest")) + self._kselftest = jsonobj.get( + "kselftest", jsonobj.get("kfselftest") + ) self._coverage = jsonobj.get("coverage", False) return raise ValueError("No valid arguments provided") @@ -316,7 +320,9 @@ def init_steps(self): if self._cross_compile: self.addcomment(" cross_compile: " + self._cross_compile) if self._cross_compile_compat: - self.addcomment(" cross_compile_compat: " + self._cross_compile_compat) + self.addcomment( + " cross_compile_compat: " + self._cross_compile_compat + ) self.addspacer() # set environment variables self.addcomment("Set environment variables") @@ -325,7 +331,9 @@ def init_steps(self): if self._cross_compile: self.addcmd("export CROSS_COMPILE=" + self._cross_compile) if self._cross_compile_compat: - self.addcmd("export CROSS_COMPILE_COMPAT=" + self._cross_compile_compat) + self.addcmd( + "export CROSS_COMPILE_COMPAT=" + self._cross_compile_compat + ) self.addcmd("export INSTALL_MOD_PATH=_modules_") self.addcmd("export INSTALL_MOD_STRIP=1") self.addcmd("export INSTALL_DTBS_PATH=_dtbs_") @@ -372,7 +380,9 @@ def init_steps(self): + self._srctarball + '" -O linux.tgz' ) - self.addcmd("tar -xzf linux.tgz -C " + self._srcdir + " --strip-components=1") + self.addcmd( + "tar -xzf linux.tgz -C " + self._srcdir + " --strip-components=1" + ) def addspacer(self): """Add empty line, mostly for easier reading""" @@ -579,7 +589,9 @@ def add_fragment(self, fragname): """Get config fragment from passed fragment_configs""" if fragname not in self._fragment_configs: print(f"Fragment {fragname} not found in fragment_configs") - self.submit_failure(f"Fragment {fragname} not found in fragment_configs") + self.submit_failure( + f"Fragment {fragname} not found in fragment_configs" + ) sys.exit(1) frag = self._fragment_configs[fragname] @@ -609,15 +621,21 @@ def _parse_fragments(self, firmware=False): content = self.add_fragment(fragment) if not content: - print(f"[_parse_fragments] WARNING: Fragment {fragment} has no content") + print( + f"[_parse_fragments] WARNING: Fragment {fragment} has no content" + ) continue fragfile = os.path.join(self._fragments_dir, f"{idx}.config") with open(fragfile, "w") as f: f.write(content) - config_count = len([line for line in content.split("\n") if line.strip()]) - print(f"[_parse_fragments] Created {fragfile} ({config_count} configs)") + config_count = len( + [line for line in content.split("\n") if line.strip()] + ) + print( + f"[_parse_fragments] Created {fragfile} ({config_count} configs)" + ) fragment_files.append(fragfile) @@ -640,7 +658,9 @@ def _parse_fragments(self, firmware=False): frag_rel = os.path.relpath(fragfile, self._af_dir) self._artifacts.append(frag_rel) - print(f"[_parse_fragments] Created {len(fragment_files)} fragment files") + print( + f"[_parse_fragments] Created {len(fragment_files)} fragment files" + ) return fragment_files def _merge_frags(self, fragment_files): @@ -651,10 +671,14 @@ def _merge_frags(self, fragment_files): """ self.startjob("config_defconfig") self.addcmd("cd " + self._srcdir) - if isinstance(self._defconfig, str) and self._defconfig.startswith("cros://"): + if isinstance(self._defconfig, str) and self._defconfig.startswith( + "cros://" + ): dotconfig = os.path.join(self._srcdir, ".config") with open(dotconfig, "w") as f: - (content, self._defconfig) = self._getcrosfragment(self._defconfig) + (content, self._defconfig) = self._getcrosfragment( + self._defconfig + ) f.write(content) self.addcmd("make olddefconfig") self._config_full = self._defconfig + self._config_full @@ -672,7 +696,9 @@ def _merge_frags(self, fragment_files): # fragments self.startjob("config_fragments") for fragfile in fragment_files: - self.addcmd(f"./scripts/kconfig/merge_config.sh -m .config {fragfile}") + self.addcmd( + f"./scripts/kconfig/merge_config.sh -m .config {fragfile}" + ) # TODO: olddefconfig should be optional/configurable # TODO: log all warnings/errors of olddefconfig to separate file self.addcmd("make olddefconfig") @@ -758,7 +784,9 @@ def _build_with_tuxmake(self): print("[_build_with_tuxmake] ERROR: No fragment files available") self._fragment_files = [] - print(f"[_build_with_tuxmake] Using {len(self._fragment_files)} fragment files") + print( + f"[_build_with_tuxmake] Using {len(self._fragment_files)} fragment files" + ) # Handle defconfigs - first goes to --kconfig, rest to --kconfig-add extra_defconfigs = [] @@ -785,7 +813,9 @@ def _build_with_tuxmake(self): # Handle ChromeOS defconfig: write fragments to a file and pass # as --kconfig-add on top of defconfig if defconfig.startswith("cros://"): - print(f"[_build_with_tuxmake] Handling ChromeOS defconfig: {defconfig}") + print( + f"[_build_with_tuxmake] Handling ChromeOS defconfig: {defconfig}" + ) content, _ = self._getcrosfragment(defconfig) cros_config = os.path.join(self._af_dir, "chromeos.config") with open(cros_config, "w") as f: @@ -956,7 +986,8 @@ def _package_kimage(self): # TODO(nuclearcat): Not all images might be present for img in KERNEL_IMAGE_NAMES[self._arch]: self.addcmd( - "cp arch/" + self._arch + "/boot/" + img + " ../artifacts", False + "cp arch/" + self._arch + "/boot/" + img + " ../artifacts", + False, ) # add image to artifacts relative to artifacts dir self._artifacts.append(img) @@ -1062,7 +1093,9 @@ def serialize(self, filename): _backend, _arch, etc). The from_json() method strips underscore prefixes when loading, so _backend becomes 'backend' in jsonobj. """ - data = json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4) + data = json.dumps( + self, default=lambda o: o.__dict__, sort_keys=True, indent=4 + ) with open(filename, "w") as f: f.write(data) print(f"Serialized to {filename}") @@ -1087,7 +1120,9 @@ def verify_build(self): for file in files: if file.endswith(".dtb"): # we need to truncate abs path to relative to artifacts - file = os.path.relpath(os.path.join(root, file), self._af_dir) + file = os.path.relpath( + os.path.join(root, file), self._af_dir + ) self._artifacts.append(file) # Update manifest/metadata self._write_metadata() @@ -1140,7 +1175,9 @@ def upload_artifacts(self): ) for root, dirs, files in os.walk(self._af_dir): for file in files: - file_rel = os.path.relpath(os.path.join(root, file), self._af_dir) + file_rel = os.path.relpath( + os.path.join(root, file), self._af_dir + ) artifact_path = os.path.join(self._af_dir, file_rel) upload_tasks.append((file_rel, artifact_path)) else: @@ -1152,7 +1189,9 @@ def upload_artifacts(self): # Function to handle a single artifact upload # args: (artifact, artifact_path) # returns: (artifact, stored_url, error) - def process_and_upload_artifact(task: Tuple[str, str]) -> Tuple[str, str, str]: + def process_and_upload_artifact( + task: Tuple[str, str], + ) -> Tuple[str, str, str]: artifact, artifact_path = task compressed_file = False dst_filename = artifact @@ -1185,7 +1224,9 @@ def process_and_upload_artifact(task: Tuple[str, str]) -> Tuple[str, str, str]: if compressed_file: os.unlink(upload_path) - print(f"[_upload_artifacts] Uploaded {artifact} to {stored_url}") + print( + f"[_upload_artifacts] Uploaded {artifact} to {stored_url}" + ) return artifact, stored_url, None except Exception as e: print(f"[_upload_artifacts] Error uploading {artifact}: {e}") @@ -1198,7 +1239,9 @@ def process_and_upload_artifact(task: Tuple[str, str]) -> Tuple[str, str, str]: successful_uploads = 0 failed_uploads = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + with concurrent.futures.ThreadPoolExecutor( + max_workers=max_workers + ) as executor: # Submit all tasks future_to_task = { executor.submit(process_and_upload_artifact, task): task @@ -1247,7 +1290,9 @@ def upload_metadata(self): storage = self._get_storage() root_path = "-".join([self._apijobname, self._node["id"]]) metadata_path = os.path.join(self._af_dir, "metadata.json") - stored_url = storage.upload_single((metadata_path, "metadata.json"), root_path) + stored_url = storage.upload_single( + (metadata_path, "metadata.json"), root_path + ) print(f"[_upload_metadata] Uploaded metadata.json to {stored_url}") print("[_upload_metadata] metadata.json uploaded to storage") return stored_url @@ -1339,7 +1384,9 @@ def submit(self, retcode, dry_run=False): break if dtb_present: - af_uri = {k: v for k, v in af_uri.items() if not k.startswith("dtbs/")} + af_uri = { + k: v for k, v in af_uri.items() if not k.startswith("dtbs/") + } # if this is dtbs_check and it ran ok, we need to change job_result # to actual result of dtbs_check @@ -1421,7 +1468,9 @@ def submit(self, retcode, dry_run=False): kselftest_node["kind"] = "test" existing_path = kselftest_node.get("path") if existing_path and isinstance(existing_path, list): - kselftest_node["path"] = existing_path + [kselftest_node["name"]] + kselftest_node["path"] = existing_path + [ + kselftest_node["name"] + ] kselftest_node["parent"] = self._node["id"] kselftest_node["data"] = results["node"]["data"].copy() kselftest_node["artifacts"] = None diff --git a/kernelci/legacy/__init__.py b/kernelci/legacy/__init__.py index f7301c9829..3dd3c6628e 100644 --- a/kernelci/legacy/__init__.py +++ b/kernelci/legacy/__init__.py @@ -16,8 +16,10 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import os -import requests from urllib.parse import urljoin + +import requests + from kernelci.build import get_branch_head, git_describe, make_tarball @@ -84,7 +86,10 @@ def set_last_commit(config, api, token, commit): *commit* is the git SHA to send """ _upload_files( - api, token, config.tree.name, {_get_last_commit_file_name(config): commit} + api, + token, + config.tree.name, + {_get_last_commit_file_name(config): commit}, ) diff --git a/kernelci/legacy/cli/__init__.py b/kernelci/legacy/cli/__init__.py index 20606d4961..9896849132 100644 --- a/kernelci/legacy/cli/__init__.py +++ b/kernelci/legacy/cli/__init__.py @@ -8,12 +8,6 @@ import sys -from .base import ( - Args as Args, - Command as Command, - parse_opts as parse_opts, - sub_main as sub_main, -) from . import ( api, job, @@ -21,6 +15,18 @@ show, storage, ) +from .base import ( + Args as Args, +) +from .base import ( + Command as Command, +) +from .base import ( + parse_opts as parse_opts, +) +from .base import ( + sub_main as sub_main, +) _COMMANDS = { "api": api.main, diff --git a/kernelci/legacy/cli/base.py b/kernelci/legacy/cli/base.py index cdb400a768..ee27aaf442 100644 --- a/kernelci/legacy/cli/base.py +++ b/kernelci/legacy/cli/base.py @@ -16,13 +16,12 @@ import configparser import os.path import sys -import toml +import toml from requests.exceptions import HTTPError import kernelci.config - # ----------------------------------------------------------------------------- # Standard arguments that can be used in sub-commands # @@ -650,7 +649,9 @@ def get_from_section(self, section, option, as_list=False): else: value = None if isinstance(section, tuple): - section_data = self._settings.get(section[0], {}).get(section[1]) + section_data = self._settings.get(section[0], {}).get( + section[1] + ) else: section_data = self._settings.get(section) if section_data: diff --git a/kernelci/legacy/cli/base_api.py b/kernelci/legacy/cli/base_api.py index 44e08e89ab..f24d42cf9a 100644 --- a/kernelci/legacy/cli/base_api.py +++ b/kernelci/legacy/cli/base_api.py @@ -15,7 +15,8 @@ import re import kernelci.api -from .base import Command, Args, catch_http_error + +from .base import Args, Command, catch_http_error class APICommand(Command): # pylint: disable=too-few-public-methods diff --git a/kernelci/legacy/cli/job.py b/kernelci/legacy/cli/job.py index 61ba2522c5..2df9d2cb78 100644 --- a/kernelci/legacy/cli/job.py +++ b/kernelci/legacy/cli/job.py @@ -7,6 +7,7 @@ import kernelci.api.helper import kernelci.runtime + from .base import Args, Command, sub_main from .base_api import APICommand @@ -100,7 +101,9 @@ def _api_call(self, api, configs, args): else None ) runtime_config = configs["runtimes"][args.runtime_config] - runtime = kernelci.runtime.get_runtime(runtime_config, token=args.runtime_token) + runtime = kernelci.runtime.get_runtime( + runtime_config, token=args.runtime_token + ) params = runtime.get_params(job, api.config) job_data = runtime.generate(job, params) if args.output: @@ -132,7 +135,9 @@ class cmd_submit(Command): # pylint: disable=invalid-name def __call__(self, configs, args): runtime_config = configs["runtimes"][args.runtime_config] - runtime = kernelci.runtime.get_runtime(runtime_config, token=args.runtime_token) + runtime = kernelci.runtime.get_runtime( + runtime_config, token=args.runtime_token + ) job = runtime.submit(args.job_path) print(runtime.get_job_id(job)) if args.wait: diff --git a/kernelci/legacy/cli/sched.py b/kernelci/legacy/cli/sched.py index 6c6fee9c9e..71580808ea 100644 --- a/kernelci/legacy/cli/sched.py +++ b/kernelci/legacy/cli/sched.py @@ -10,6 +10,7 @@ import kernelci.runtime import kernelci.scheduler + from .base import Args, Command, sub_main @@ -30,7 +31,9 @@ def __call__(self, configs, args): sched = kernelci.scheduler.Scheduler(configs, runtimes) event = json.loads(sys.stdin.read()) channel = args.channel or "node" - for job, runtime, platform, _rules in sched.get_schedule(event, channel): + for job, runtime, platform, _rules in sched.get_schedule( + event, channel + ): print(f"{job.name:32} {runtime.config.name:32} {platform.name}") return True diff --git a/kernelci/legacy/cli/storage.py b/kernelci/legacy/cli/storage.py index 82d0327b3a..8890335355 100644 --- a/kernelci/legacy/cli/storage.py +++ b/kernelci/legacy/cli/storage.py @@ -8,6 +8,7 @@ import os.path import kernelci.storage + from .base import Args, Command, sub_main @@ -29,7 +30,9 @@ class cmd_upload(Command): # pylint: disable=invalid-name def __call__(self, configs, args): storage_config = configs["storage_configs"].get(args.storage_config) - storage = kernelci.storage.get_storage(storage_config, args.storage_cred) + storage = kernelci.storage.get_storage( + storage_config, args.storage_cred + ) for file_path in args.files: url = storage.upload_single( (file_path, os.path.basename(file_path)), args.upload_path or "" diff --git a/kernelci/legacy/config/__init__.py b/kernelci/legacy/config/__init__.py index 9f4f836c5d..75c51a217b 100644 --- a/kernelci/legacy/config/__init__.py +++ b/kernelci/legacy/config/__init__.py @@ -4,7 +4,11 @@ from . import ( db as db_config, +) +from . import ( rootfs as rootfs_config, +) +from . import ( test as test_config, ) diff --git a/kernelci/legacy/config/base.py b/kernelci/legacy/config/base.py index 1272417373..2416f39023 100644 --- a/kernelci/legacy/config/base.py +++ b/kernelci/legacy/config/base.py @@ -5,6 +5,10 @@ from kernelci.config.base import ( FilterFactory as FilterFactory, +) +from kernelci.config.base import ( YAMLConfigObject as YAMLConfigObject, +) +from kernelci.config.base import ( _YAMLObject as _YAMLObject, ) diff --git a/kernelci/legacy/config/rootfs.py b/kernelci/legacy/config/rootfs.py index c652932df0..cd2cba9016 100644 --- a/kernelci/legacy/config/rootfs.py +++ b/kernelci/legacy/config/rootfs.py @@ -17,6 +17,7 @@ from kernelci import sort_check + from .base import _YAMLObject @@ -257,7 +258,13 @@ def _get_yaml_attributes(cls): class RootFS_ChromiumOS(RootFS): def __init__( - self, name, rootfs_type, arch_list=None, board=None, branch=None, serial=None + self, + name, + rootfs_type, + arch_list=None, + board=None, + branch=None, + serial=None, ): super().__init__(name, rootfs_type) self._arch_list = arch_list or list() @@ -354,18 +361,26 @@ def validate(configs): def _validate_debos(name, config): err = sort_check(config.arch_list) if err: - print("Arch order broken for {}: '{}' before '{}".format(name, err[0], err[1])) + print( + "Arch order broken for {}: '{}' before '{}".format( + name, err[0], err[1] + ) + ) return False err = sort_check(config.extra_packages) if err: print( - "Packages order broken for {}: '{}' before '{}".format(name, err[0], err[1]) + "Packages order broken for {}: '{}' before '{}".format( + name, err[0], err[1] + ) ) return False err = sort_check(config.extra_packages_remove) if err: print( - "Packages order broken for {}: '{}' before '{}".format(name, err[0], err[1]) + "Packages order broken for {}: '{}' before '{}".format( + name, err[0], err[1] + ) ) return False return True @@ -374,11 +389,19 @@ def _validate_debos(name, config): def _validate_buildroot(name, config): err = sort_check(config.arch_list) if err: - print("Arch order broken for {}: '{}' before '{}".format(name, err[0], err[1])) + print( + "Arch order broken for {}: '{}' before '{}".format( + name, err[0], err[1] + ) + ) return False err = sort_check(config.frags) if err: - print("Frags order broken for {}: '{}' before '{}".format(name, err[0], err[1])) + print( + "Frags order broken for {}: '{}' before '{}".format( + name, err[0], err[1] + ) + ) return False return True @@ -386,7 +409,11 @@ def _validate_buildroot(name, config): def _validate_chromiumos(name, config): err = sort_check(config.arch_list) if err: - print("Arch order broken for {}: '{}' before '{}".format(name, err[0], err[1])) + print( + "Arch order broken for {}: '{}' before '{}".format( + name, err[0], err[1] + ) + ) return False return True diff --git a/kernelci/legacy/config/test.py b/kernelci/legacy/config/test.py index 77e05755a5..13e9d528aa 100644 --- a/kernelci/legacy/config/test.py +++ b/kernelci/legacy/config/test.py @@ -16,7 +16,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from .base import FilterFactory, _YAMLObject, YAMLConfigObject +from .base import FilterFactory, YAMLConfigObject, _YAMLObject class DeviceType(_YAMLObject): @@ -172,17 +172,23 @@ def __init__(self, name, mach, arch="riscv", *args, **kw): class DeviceType_shell(DeviceType): - def __init__(self, name, mach=None, arch=None, boot_method=None, *args, **kwargs): + def __init__( + self, name, mach=None, arch=None, boot_method=None, *args, **kwargs + ): super().__init__(name, mach, arch, boot_method, *args, **kwargs) class DeviceType_docker(DeviceType): - def __init__(self, name, mach=None, arch=None, boot_method=None, *args, **kwargs): + def __init__( + self, name, mach=None, arch=None, boot_method=None, *args, **kwargs + ): super().__init__(name, mach, arch, boot_method, *args, **kwargs) class DeviceType_kubernetes(DeviceType): - def __init__(self, name, mach=None, arch=None, boot_method=None, *args, **kwargs): + def __init__( + self, name, mach=None, arch=None, boot_method=None, *args, **kwargs + ): super().__init__(name, mach, arch, boot_method, *args, **kwargs) @@ -333,7 +339,8 @@ def from_yaml(cls, file_system_types, rootfs): { fs: url for (fs, url) in ( - (fs, rootfs.get(fs)) for fs in ["ramdisk", "nfs", "diskfile"] + (fs, rootfs.get(fs)) + for fs in ["ramdisk", "nfs", "diskfile"] ) if url } @@ -408,7 +415,9 @@ def get_url(self, fs_type, arch, variant, endian): class TestPlan(_YAMLObject): """Test plan model.""" - _pattern = "{plan}/{category}-{method}-{protocol}-{rootfs}-{plan}-template.jinja2" + _pattern = ( + "{plan}/{category}-{method}-{protocol}-{rootfs}-{plan}-template.jinja2" + ) def __init__( self, @@ -540,10 +549,14 @@ def __init__(self, device_type, test_plans, filters=None): self._filters = filters or list() @classmethod - def from_yaml(cls, test_config, device_types, test_plans, default_filters=None): + def from_yaml( + cls, test_config, device_types, test_plans, default_filters=None + ): kw = { "device_type": device_types[test_config["device_type"]], - "test_plans": [test_plans[test] for test in test_config["test_plans"]], + "test_plans": [ + test_plans[test] for test in test_config["test_plans"] + ], "filters": FilterFactory.from_data(test_config, default_filters), } @@ -561,9 +574,14 @@ def match(self, arch, flags, config, plan=None): return ( ( plan is None - or (plan in self._test_plans and self._test_plans[plan].match(config)) + or ( + plan in self._test_plans + and self._test_plans[plan].match(config) + ) + ) + and ( + self.device_type.arch is None or (self.device_type.arch == arch) ) - and (self.device_type.arch is None or (self.device_type.arch == arch)) and self.device_type.match(flags, config) and all(f.match(**config) for f in self._filters) ) diff --git a/kernelci/legacy/lava/__init__.py b/kernelci/legacy/lava/__init__.py index 50ca794116..55a9742840 100644 --- a/kernelci/legacy/lava/__init__.py +++ b/kernelci/legacy/lava/__init__.py @@ -7,11 +7,12 @@ # Author: Guillaume Tucker # Author: Michal Galka -from jinja2 import Environment, FileSystemLoader import json import os import sys +from jinja2 import Environment, FileSystemLoader + def add_kci_raise(jinja2_env): """Add a kci_raise function to use in templates @@ -74,7 +75,9 @@ def generate( ): if templates_paths is None: templates_paths = self.DEFAULT_TEMPLATE_PATHS - short_template_file = plan_config.get_template_path(device_config.boot_method) + short_template_file = plan_config.get_template_path( + device_config.boot_method + ) base_name = params["base_device_type"] priority = self._get_priority(plan_config) params.update( @@ -88,7 +91,8 @@ def generate( if callback_opts: self._add_callback_params(params, callback_opts) jinja2_env = Environment( - loader=FileSystemLoader(templates_paths), extensions=["jinja2.ext.do"] + loader=FileSystemLoader(templates_paths), + extensions=["jinja2.ext.do"], ) add_kci_raise(jinja2_env) template = jinja2_env.get_template(short_template_file) @@ -100,7 +104,9 @@ def save_file(self, job, output_path, params): output_file = os.path.join(output_path, file_name) if os.path.isfile(output_file): # print error message to stderr - print("ERROR: file already exists: %s" % output_file, file=sys.stderr) + print( + "ERROR: file already exists: %s" % output_file, file=sys.stderr + ) return None with open(output_file, "w") as output: output.write(job) diff --git a/kernelci/legacy/lava/lava_rest.py b/kernelci/legacy/lava/lava_rest.py index fd9545db6e..f496bbd039 100644 --- a/kernelci/legacy/lava/lava_rest.py +++ b/kernelci/legacy/lava/lava_rest.py @@ -5,8 +5,9 @@ # Guillaume Tucker from collections import namedtuple -import requests from urllib.parse import urljoin + +import requests import yaml from . import LavaRuntime @@ -44,7 +45,8 @@ def _get_devices(self): aliases_url = urljoin(self._server.url, "aliases") all_devices = self._get_all(devices_url) all_aliases = { - item["name"]: item["device_type"] for item in self._get_all(aliases_url) + item["name"]: item["device_type"] + for item in self._get_all(aliases_url) } device_types = {} for device in all_devices: @@ -94,7 +96,9 @@ def _submit(self, job): job_data = { "definition": yaml.dump(yaml.load(job, Loader=yaml.CLoader)), } - resp = self._server.session.post(jobs_url, json=job_data, allow_redirects=False) + resp = self._server.session.post( + jobs_url, json=job_data, allow_redirects=False + ) resp.raise_for_status() return resp.json()["job_ids"][0] diff --git a/kernelci/legacy/lava/lava_xmlrpc.py b/kernelci/legacy/lava/lava_xmlrpc.py index 59da18d616..c26b6992a8 100644 --- a/kernelci/legacy/lava/lava_xmlrpc.py +++ b/kernelci/legacy/lava/lava_xmlrpc.py @@ -7,8 +7,8 @@ # Author: Guillaume Tucker # Author: Michal Galka -import xmlrpc.client import urllib.parse +import xmlrpc.client from . import LavaRuntime @@ -64,7 +64,11 @@ def _connect(self, user=None, token=None, **kwargs): if user and token: url = urllib.parse.urlparse(self.config.url) api_url = "{scheme}://{user}:{token}@{loc}{path}".format( - scheme=url.scheme, user=user, token=token, loc=url.netloc, path=url.path + scheme=url.scheme, + user=user, + token=token, + loc=url.netloc, + path=url.path, ) else: api_url = self.config.url diff --git a/kernelci/rootfs.py b/kernelci/rootfs.py index b178c61faa..a229c1f9da 100644 --- a/kernelci/rootfs.py +++ b/kernelci/rootfs.py @@ -16,9 +16,10 @@ # along with this library; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from kernelci import shell_cmd import os +from kernelci import shell_cmd + class RootfsBuilder: def __init__(self, name): @@ -180,7 +181,9 @@ def build(name, config, data_path, arch, output): """ builder_cls = ROOTFS_BUILDERS.get(config.rootfs_type) if builder_cls is None: - raise ValueError("rootfs_type not supported: {}".format(config.rootfs_type)) + raise ValueError( + "rootfs_type not supported: {}".format(config.rootfs_type) + ) builder = builder_cls(name) return builder.build(config, data_path, arch, output) diff --git a/kernelci/runtime/__init__.py b/kernelci/runtime/__init__.py index 8acba8c5bf..4d6b1d6df2 100644 --- a/kernelci/runtime/__init__.py +++ b/kernelci/runtime/__init__.py @@ -9,10 +9,10 @@ import abc import importlib import os + import requests import yaml - -from jinja2 import Environment, FileSystemLoader, ChoiceLoader +from jinja2 import ChoiceLoader, Environment, FileSystemLoader from kernelci.config.base import get_system_arch @@ -76,7 +76,13 @@ class Runtime(abc.ABC): # pylint: disable=unused-argument,too-many-arguments def __init__( - self, config, *, user=None, token=None, custom_template_dir=None, kcictx=None + self, + config, + *, + user=None, + token=None, + custom_template_dir=None, + kcictx=None, ): """A Runtime object can be used to run jobs in a runtime environment @@ -90,7 +96,12 @@ def __init__( # Add the main custom dir self._templates.append(custom_template_dir) # Add relevant subdirectories - for subdir in ["runtime", "runtime/base", "runtime/boot", "runtime/tests"]: + for subdir in [ + "runtime", + "runtime/base", + "runtime/boot", + "runtime/tests", + ]: sub_path = ( os.path.join(custom_template_dir, subdir) if not custom_template_dir.endswith(subdir) @@ -182,7 +193,10 @@ def get_params(self, job, api_config=None): instance_callback = os.environ.get("KCI_INSTANCE_CALLBACK") device_dtb = None device_type = job.platform_config.name - if job.platform_config.base_name and len(job.platform_config.base_name) > 0: + if ( + job.platform_config.base_name + and len(job.platform_config.base_name) > 0 + ): device_type = job.platform_config.base_name if job.platform_config.dtb and len(job.platform_config.dtb) > 0: # verify if we have metadata at all @@ -206,7 +220,9 @@ def get_params(self, job, api_config=None): device_dtb = dtb break if not device_dtb: - raise ValueError(f"dtb file {job.platform_config.dtb} not found!") + raise ValueError( + f"dtb file {job.platform_config.dtb} not found!" + ) params = { "api_config": api_config or {}, @@ -272,7 +288,9 @@ def wait(self, job_object): """Wait for a job to complete and get the exit status code""" -def get_runtime(config, user=None, token=None, custom_template_dir=None, kcictx=None): +def get_runtime( + config, user=None, token=None, custom_template_dir=None, kcictx=None +): """Get the Runtime object for a given runtime config. *config* is a kernelci.config.runtime.Runtime object @@ -292,7 +310,9 @@ def get_runtime(config, user=None, token=None, custom_template_dir=None, kcictx= ) -def get_all_runtimes(runtime_configs, opts, custom_template_dir=None, kcictx=None): +def get_all_runtimes( + runtime_configs, opts, custom_template_dir=None, kcictx=None +): """Get all the Runtime objects based on the runtime configs and options This will iterate over all the runtimes configs and yield a (name, runtime) @@ -307,7 +327,8 @@ def get_all_runtimes(runtime_configs, opts, custom_template_dir=None, kcictx=Non for config_name, config in runtime_configs.items(): section = ("runtime", config_name) user, token = ( - opts.get_from_section(section, opt) for opt in ("user", "runtime_token") + opts.get_from_section(section, opt) + for opt in ("user", "runtime_token") ) runtime = get_runtime( config, @@ -345,8 +366,12 @@ def evaluate_test_suite_result(child_nodes): result_list = [child["node"]["result"] for child in child_nodes] if "fail" in result_list: result = "fail" - elif all(result == "pass" for result in result_list) or "pass" in result_list: + elif ( + all(result == "pass" for result in result_list) or "pass" in result_list + ): result = "pass" - elif all(result == "skip" for result in result_list) or "skip" in result_list: + elif ( + all(result == "skip" for result in result_list) or "skip" in result_list + ): result = "skip" return result diff --git a/kernelci/runtime/kubernetes.py b/kernelci/runtime/kubernetes.py index 2171513177..26a830349a 100644 --- a/kernelci/runtime/kubernetes.py +++ b/kernelci/runtime/kubernetes.py @@ -6,12 +6,13 @@ """Kubernetes runtime implementation""" import logging +import os import random import re import string -import os import kubernetes + from . import Runtime logger = logging.getLogger(__name__) @@ -33,7 +34,9 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.kcontext = None # Allow timeout to be configured, otherwise use default - self.api_timeout = getattr(self.config, "api_timeout", self.DEFAULT_API_TIMEOUT) + self.api_timeout = getattr( + self.config, "api_timeout", self.DEFAULT_API_TIMEOUT + ) @classmethod def _get_job_file_name(cls, params): diff --git a/kernelci/runtime/lava.py b/kernelci/runtime/lava.py index a2f0151406..454ad5513a 100644 --- a/kernelci/runtime/lava.py +++ b/kernelci/runtime/lava.py @@ -10,10 +10,10 @@ """LAVA runtime implementation""" -from collections import namedtuple import tempfile import time import uuid +from collections import namedtuple from urllib.parse import urljoin import requests @@ -65,7 +65,9 @@ def get_text(self): def get_data(self): """Get the raw log data as a list of dicts in LAVA output.yaml format""" - return [{"dt": dt, "lvl": lvl, "msg": msg} for dt, lvl, msg in self._raw_log] + return [ + {"dt": dt, "lvl": lvl, "msg": msg} for dt, lvl, msg in self._raw_log + ] class Callback: @@ -151,7 +153,9 @@ def _get_kernelmsg_case(cls, tests): # When boot has failure_retry set, there may be multiple kernel-messages checks # Unlike login, we return 'fail' if ANY attempt failed, because kernel-messages # failure means we caught kernel panic, oops, or other critical errors - kernelmsg_tests = [test for test in tests if test["name"] == "kernel-messages"] + kernelmsg_tests = [ + test for test in tests if test["name"] == "kernel-messages" + ] if not kernelmsg_tests: return None @@ -226,7 +230,9 @@ def _get_results_hierarchy(self, results): if not node["result"] or node["result"] == "pass": # Sometimes LAVA reports stage result as `pass` even if sub-tests # failed - node["result"] = evaluate_test_suite_result(item["child_nodes"]) + node["result"] = evaluate_test_suite_result( + item["child_nodes"] + ) node["kind"] = "job" elif isinstance(value, str): node["result"] = value @@ -265,7 +271,9 @@ def get_hierarchy(self, results, job_node): job_node["data"]["error_code"] = job_meta.get("error_type") job_node["data"]["error_msg"] = job_meta.get("error_msg") else: - print(f"Job failure metadata not found for node: {job_node['id']}") + print( + f"Job failure metadata not found for node: {job_node['id']}" + ) job_node["data"]["error_code"] = "Infrastructure" job_node["data"]["error_msg"] = "Unknown infrastructure error" @@ -285,7 +293,9 @@ def get_hierarchy(self, results, job_node): break if job_result == "pass": - final_job_result = self._get_job_node_result(child_nodes, job_result) + final_job_result = self._get_job_node_result( + child_nodes, job_result + ) if final_job_result != job_result: print( f"DEBUG: {job_node['id']} Transitting job node \ @@ -365,11 +375,15 @@ def _get_priority(self, job): if submitter and submitter != self.SERVICE_PIPELINE: user_priority = node.get("data", {}).get("priority") - priority = self._resolve_priority(user_priority, self.PRIORITY_HIGHEST) + priority = self._resolve_priority( + user_priority, self.PRIORITY_HIGHEST + ) else: tree_priority = node.get("data", {}).get("tree_priority") if tree_priority is not None: - priority = self._resolve_priority(tree_priority, self.PRIORITY_LOW) + priority = self._resolve_priority( + tree_priority, self.PRIORITY_LOW + ) else: priority = self._resolve_priority( job.config.priority, self.PRIORITY_LOW @@ -570,7 +584,9 @@ def _store_job_in_external_storage(self, job): raise ValueError("Upload returned no URL") print(f"Job stored at URL: {stored_url}") except Exception as exc: - raise ValueError(f"Failed to store job in external storage: {exc}") from exc + raise ValueError( + f"Failed to store job in external storage: {exc}" + ) from exc def get_runtime(runtime_config, **kwargs): diff --git a/kernelci/runtime/shell.py b/kernelci/runtime/shell.py index a4759997f3..e76b8af585 100644 --- a/kernelci/runtime/shell.py +++ b/kernelci/runtime/shell.py @@ -7,6 +7,7 @@ import os import subprocess + from . import Runtime diff --git a/kernelci/scripts/kci.py b/kernelci/scripts/kci.py index 7b4087f7e9..d438117803 100755 --- a/kernelci/scripts/kci.py +++ b/kernelci/scripts/kci.py @@ -14,17 +14,30 @@ sys.path.append(str(Path(__file__).parent.parent.parent)) sys.path.append(str(Path(__file__).parent)) -from kernelci.cli import kci # noqa: E402 - # Import subcommand modules for their Click registration side effects. from kernelci.cli import ( # pylint: disable=unused-import # noqa: F401,E402 api as kci_api, +) +from kernelci.cli import ( config as kci_config, +) +from kernelci.cli import ( docker as kci_docker, +) +from kernelci.cli import ( event as kci_event, +) +from kernelci.cli import ( job as kci_job, +) +from kernelci.cli import kci # noqa: E402 +from kernelci.cli import ( node as kci_node, +) +from kernelci.cli import ( storage as kci_storage, +) +from kernelci.cli import ( user as kci_user, ) diff --git a/kernelci/scripts/kci_rootfs.py b/kernelci/scripts/kci_rootfs.py index e92f26c868..84b71e1fbf 100755 --- a/kernelci/scripts/kci_rootfs.py +++ b/kernelci/scripts/kci_rootfs.py @@ -23,11 +23,10 @@ sys.path.append(str(Path(__file__).parent.parent.parent)) sys.path.append(str(Path(__file__).parent)) -from kernelci.legacy.cli import Args, Command, parse_opts # noqa: E402 import kernelci.config # noqa: E402 import kernelci.rootfs # noqa: E402 import kernelci.storage # noqa: E402 - +from kernelci.legacy.cli import Args, Command, parse_opts # noqa: E402 # ----------------------------------------------------------------------------- # Commands @@ -105,7 +104,9 @@ def __call__(self, configs, args, **kwargs): for config in build_configs: arch_available = set(config.arch_list) arch_set = ( - arch_requested & arch_available if arch_requested else arch_available + arch_requested & arch_available + if arch_requested + else arch_available ) for arch in arch_set: print(" ".join([config.name, arch, config.rootfs_type])) @@ -127,7 +128,9 @@ def __call__(self, config_data, args, **kwargs): entries".format(config_name) ) return False - data_path = args.data_path or "config/rootfs/{}".format(config.rootfs_type) + data_path = args.data_path or "config/rootfs/{}".format( + config.rootfs_type + ) return kernelci.rootfs.build( config_name, config, data_path, args.arch, args.output ) diff --git a/kernelci/storage/azure.py b/kernelci/storage/azure.py index ed215dbf2b..676b3a5767 100644 --- a/kernelci/storage/azure.py +++ b/kernelci/storage/azure.py @@ -5,10 +5,12 @@ """KernelCI storage implementation for Azure Files""" -from urllib.parse import urljoin import os -from azure.storage.fileshare import ShareServiceClient +from urllib.parse import urljoin + from azure.storage.blob import ContentSettings +from azure.storage.fileshare import ShareServiceClient + from . import Storage diff --git a/kernelci/storage/backend.py b/kernelci/storage/backend.py index 14a48265f0..5513d05bb9 100644 --- a/kernelci/storage/backend.py +++ b/kernelci/storage/backend.py @@ -10,7 +10,9 @@ import time from urllib.parse import urljoin + import requests + from . import Storage @@ -52,7 +54,9 @@ def _handle_http_error(self, exc, attempt, max_retries, retry_delay): ) time.sleep(retry_delay) return exc # Return exception to be saved as last_exception - print(f"Upload failed after {max_retries} attempts. Response body: {body}") + print( + f"Upload failed after {max_retries} attempts. Response body: {body}" + ) return exc # For non-5xx errors (like 4xx client errors), don't retry print(f"Upload failed with HTTP error: {exc}") diff --git a/kernelci/storage/ssh.py b/kernelci/storage/ssh.py index 2decb63d9c..2d2726c10a 100644 --- a/kernelci/storage/ssh.py +++ b/kernelci/storage/ssh.py @@ -8,7 +8,7 @@ import os -from paramiko import client, SSHClient +from paramiko import SSHClient, client from scp import SCPClient from . import Storage diff --git a/pyproject.toml b/pyproject.toml index e61acd70a3..b8f4ae8e0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,9 +60,8 @@ classifiers = [ dev = [ "mypy==1.19.1", "pre-commit==4.2.0", - "pycodestyle==2.14.0", - "pylint==4.0.4", "pytest-mock==3.15.1", + "ruff >= 0.9.0", "types-PyYAML==6.0.12.20250915", "types-requests==2.32.4.20260107", "types-urllib3==1.26.25.14", @@ -76,3 +75,30 @@ Repository = "https://github.com/kernelci/kernelci-core" [project.scripts] kci = "kernelci.scripts.kci:main" kci_rootfs = "kernelci.scripts.kci_rootfs:main" + +# Ruff — used by CI and pre-commit hooks +[tool.ruff] +target-version = "py39" +# Soft limit for formatter wrapping +line-length = 80 + +[tool.ruff.lint] +# Ruff defaults (E, F, W) plus import sorting and complexity checking +select = ["E", "F", "W", "I", "C901"] + +[tool.ruff.lint.pycodestyle] +# Hard limit — lines beyond this are errors regardless of formatter +max-line-length = 110 + +[tool.ruff.lint.per-file-ignores] +# Legacy high-complexity entrypoints, to be refactored incrementally +"examples/example_context_usage.py" = ["C901", "E501"] +"kernelci/api/__init__.py" = ["C901"] +"kernelci/api/helper.py" = ["C901"] +"kernelci/build.py" = ["C901"] +"kernelci/context/__init__.py" = ["C901"] +"kernelci/kbuild.py" = ["C901"] +"kernelci/legacy/cli/base.py" = ["C901"] +"kernelci/runtime/__init__.py" = ["C901"] +# Click subcommand modules imported for registration side effects +"kernelci/scripts/kci.py" = ["F401"] diff --git a/requirements-dev.txt b/requirements-dev.txt index d95a9d9596..de36a1cd50 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,8 @@ -r requirements.txt mypy==1.19.1 pre-commit==4.2.0 -pycodestyle==2.14.0 -pylint==4.0.4 pytest-mock==3.15.1 +ruff>=0.9.0 types-PyYAML==6.0.12.20250915 types-requests==2.32.4.20260107 types-urllib3==1.26.25.14 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 62e73811c7..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[pycodestyle] -max-line-length = 100 diff --git a/tests/api/conftest.py b/tests/api/conftest.py index b8e493771a..d917e8be1c 100644 --- a/tests/api/conftest.py +++ b/tests/api/conftest.py @@ -7,9 +7,10 @@ """pytest fixtures for APIHelper unit tests""" import json + import pytest -from cloudevents.http import CloudEvent from cloudevents.conversion import to_json +from cloudevents.http import CloudEvent from requests import Response import kernelci.config @@ -163,7 +164,9 @@ def update_kunit_child_node(self): "group": self._kunit_node["group"], "holdoff": None, "kind": self._kunit_node["kind"], - "path": (self._kunit_node["path"] + [self._kunit_child_node["name"]]), + "path": ( + self._kunit_node["path"] + [self._kunit_child_node["name"]] + ), "state": self._kunit_node["state"], "timeout": "2022-11-02T16:06:39.509000", "updated": "2022-11-01T16:07:09.633000", diff --git a/tests/api/test_apihelper.py b/tests/api/test_apihelper.py index 678b88f9ab..bb4a860fb8 100644 --- a/tests/api/test_apihelper.py +++ b/tests/api/test_apihelper.py @@ -7,8 +7,8 @@ """Test the APIHelper class""" -from kernelci.api.helper import APIHelper import kernelci.api +from kernelci.api.helper import APIHelper def test_apihelper(): diff --git a/tests/test_cli.py b/tests/test_cli.py index b0a209942a..a838087f18 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,9 +7,8 @@ import importlib -import pytest - import click +import pytest import requests import kernelci.cli @@ -90,7 +89,16 @@ def test_kci_builtin_subcommands_registered(): """Built-in kci subcommands are registered by the entrypoint imports""" importlib.reload(kernelci.scripts.kci) - expected = {"api", "config", "docker", "event", "job", "node", "storage", "user"} + expected = { + "api", + "config", + "docker", + "event", + "job", + "node", + "storage", + "user", + } assert expected.issubset(set(kernelci.cli.kci.commands)) @@ -110,7 +118,13 @@ def test_split_valid_attributes(): (["name>= value"], {"name__gte": "value"}), ( ["a=b", "c=123", "x3 = 1.2", "abc >= 4", "z != x[2]"], - {"a": "b", "c": "123", "x3": "1.2", "abc__gte": "4", "z__ne": "x[2]"}, + { + "a": "b", + "c": "123", + "x3": "1.2", + "abc__gte": "4", + "z__ne": "x[2]", + }, ), (["x>=1", "x<=3"], {"x__gte": "1", "x__lte": "3"}), (["foo=bar", "foo=baz", "foo=zap"], {"foo": ["bar", "baz", "zap"]}), diff --git a/tests/test_context.py b/tests/test_context.py index f7db140f56..048eb67de8 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -11,8 +11,9 @@ import tempfile import unittest from unittest.mock import Mock, patch -import yaml + import toml +import yaml from kernelci.context import KContext, create_context, create_context_from_args @@ -117,7 +118,9 @@ def test_get_secret(self): self.assertEqual(token, "prod-token-123") # Test default value - missing = ctx.get_secret("api.nonexistent.token", default="default-token") + missing = ctx.get_secret( + "api.nonexistent.token", default="default-token" + ) self.assertEqual(missing, "default-token") def test_get_api_config(self): @@ -140,7 +143,9 @@ def test_get_storage_config(self): self.assertIsNotNone(storage_config) self.assertEqual(storage_config["storage_type"], "backend") - self.assertEqual(storage_config["base_url"], "https://storage.kernelci.org") + self.assertEqual( + storage_config["base_url"], "https://storage.kernelci.org" + ) self.assertEqual(storage_config["storage_cred"], "storage-secret-789") def test_get_job_config(self): @@ -188,7 +193,9 @@ def test_merge_config_and_secrets(self): """Test merging configuration and secrets""" ctx = KContext(config_paths=self.yaml_path, secrets_path=self.toml_path) - merged = ctx.merge_config_and_secrets("api.production", "api.production") + merged = ctx.merge_config_and_secrets( + "api.production", "api.production" + ) self.assertIsNotNone(merged) self.assertEqual(merged["url"], "https://api.kernelci.org") @@ -196,7 +203,9 @@ def test_merge_config_and_secrets(self): def test_missing_files(self): """Test handling of missing configuration files""" - ctx = KContext(config_paths="nonexistent.yaml", secrets_path="nonexistent.toml") + ctx = KContext( + config_paths="nonexistent.yaml", secrets_path="nonexistent.toml" + ) self.assertEqual(ctx.config, {}) self.assertEqual(ctx.secrets, {}) @@ -209,7 +218,9 @@ def test_directory_loading(self): # Write multiple YAML files api_config = {"api": {"test": {"url": "https://test.com"}}} - storage_config = {"storage": {"test": {"base_url": "https://storage.test.com"}}} + storage_config = { + "storage": {"test": {"base_url": "https://storage.test.com"}} + } with open( os.path.join(config_dir, "api.yaml"), "w", encoding="utf-8" @@ -282,7 +293,9 @@ def test_parse_cli_args(self): # Check parsed values self.assertEqual(ctx.program_name, "scheduler_k8s") self.assertEqual(ctx.secrets_path, self.toml_path) - self.assertListEqual(ctx.get_runtimes(), ["k8s-gke-eu-west4", "k8s-all"]) + self.assertListEqual( + ctx.get_runtimes(), ["k8s-gke-eu-west4", "k8s-all"] + ) self.assertIsNotNone(ctx.get_cli_args()) # Check that secrets were loaded