Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 101 additions & 20 deletions config/runtime/base/python.jinja2
Original file line number Diff line number Diff line change
Expand Up @@ -55,16 +55,39 @@ STORAGE_CONFIG_YAML = """
NODE = {{ node }}
JOB_NAME = '{{ name }}'
WORKSPACE = '/tmp/kci'
READINESS_DEADLINE_SEC = 60.0
REQUEST_TIMEOUT = (5.0, 15.0)
RETRY_INTERVAL_SEC = 2.0
TRANSIENT_HTTP_STATUS_CODES = {429, 500, 502, 503, 504}
{%- endblock %}

{% block python_base_job -%}
class InfraReadinessError(RuntimeError):
"""Raised when required KernelCI network services do not become ready in time."""


def _sanitize_service_name(url_or_name):
parsed = urllib.parse.urlsplit(str(url_or_name))
if parsed.hostname:
if parsed.port:
return f"{parsed.hostname}:{parsed.port}"
return parsed.hostname
return str(url_or_name)


class BaseJob:

def __init__(self, api_config, node, workspace):
self._api_url = self._extract_api_url(api_config)
self._api = self._get_api(api_config)
self._node = node
self._workspace = workspace

def _extract_api_url(self, api_config_yaml):
data = yaml.safe_load(api_config_yaml) or {}
api_data = data.get('api', data) if isinstance(data, dict) else {}
return api_data.get('url', 'kernelci-api')

def _get_api(self, api_config_yaml):
api_config = kernelci.config.api.API.load_from_yaml(
yaml.safe_load(api_config_yaml), name='api'
Expand All @@ -87,11 +110,55 @@ class BaseJob:
storage_cred = os.getenv('KCI_STORAGE_CREDENTIALS')
return kernelci.storage.get_storage(storage_config, storage_cred)

def _call_with_retry(
self,
func,
service_url,
deadline_sec=READINESS_DEADLINE_SEC,
retry_interval=RETRY_INTERVAL_SEC,
):
service = _sanitize_service_name(service_url)
start = time.monotonic()
deadline = start + deadline_sec
last_error = "no attempts executed"

while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
break

try:
return func(remaining)
except (
requests.exceptions.Timeout,
requests.exceptions.ConnectionError,
socket.gaierror,
) as exc:
last_error = f"{type(exc).__name__}: {exc}"
except requests.exceptions.HTTPError as exc:
status = exc.response.status_code if exc.response is not None else None
if status not in TRANSIENT_HTTP_STATUS_CODES:
raise
last_error = f"HTTP {status}"

remaining = deadline - time.monotonic()
if remaining <= 0:
break
time.sleep(min(retry_interval, remaining))

elapsed = time.monotonic() - start
raise InfraReadinessError(
f"Service '{service}' unreachable after {elapsed:.1f}s ({last_error})"
)

def _get_artifact_url(self, node, artifact):
if node.get('artifacts') and artifact in node['artifacts']:
return node['artifacts'][artifact]
if node.get('parent'):
parent = self._api.node.get(node['parent'])
parent = self._call_with_retry(
lambda _rem: self._api.node.get(node['parent']),
self._api_url,
)
return self._get_artifact_url(parent, artifact)
raise ValueError(f"'{artifact}' artifact not found in node {node['id']} ancestors")

Expand All @@ -103,8 +170,17 @@ class BaseJob:
path = self._workspace
# Ensure our destination folder exists
os.makedirs(path, exist_ok=True)
resp = requests.get(url, stream=True)
resp.raise_for_status()

def _fetch(remaining):
attempt_timeout = (
min(REQUEST_TIMEOUT[0], remaining),
min(REQUEST_TIMEOUT[1], remaining),
)
r = requests.get(url, stream=True, timeout=attempt_timeout)
r.raise_for_status()
return r

resp = self._call_with_retry(_fetch, url)
tarball_name = os.path.basename(urllib.parse.urlparse(url).path)
base, ext = tarball_name.split('.tar.')
with tarfile.open(fileobj=resp.raw, mode=f'r|{ext}') as tarball:
Expand All @@ -122,14 +198,30 @@ class BaseJob:
raise NotImplementedError("_run() method required to run job")

def _submit(self, result):
node_from_id = self._api.node.get(self._node['id'])
node_from_id = self._call_with_retry(
lambda _rem: self._api.node.get(self._node['id']),
self._api_url,
)
self._node = node_from_id.copy()
self._node.update({
'result': result,
'state': 'done',
})
self._api.node.update(self._node)

def _submit_infra_failure(self, message):
try:
node = self._node.copy()
node['result'] = 'incomplete'
node['state'] = 'done'
data = dict(node.get('data') or {})
data['error_code'] = 'infra_error'
data['error_msg'] = message
node['data'] = data
self._api.node.update(node)
except Exception as exc:
print(f"Failed to report infrastructure error to API: {exc}", file=sys.stderr)

def run(self):
if self._node.get('debug') and 'dry_run' in self._node['debug']:
return self.dry_run(self._node['debug']['result'])
Expand Down Expand Up @@ -160,25 +252,14 @@ class Job(BaseJob):

{% block python_main -%}
def main(args):
# Sometimes container DNS is failing if k8s node is still booting
# This is a workaround:
# test www.google.com until it works or retry 10 times
for _ in range(30):
try:
socket.gethostbyname('www.google.com')
break
except socket.gaierror:
pass
# sleep 1 second
time.sleep(1)
else:
print("DNS resolution failed, exiting", file=sys.stderr)
sys.exit(1)

job = Job({% block python_job_constr %}api_config=API_CONFIG_YAML, node=NODE, workspace=WORKSPACE{% endblock %})
try:
results = job.run()
job.submit(results)
except InfraReadinessError as ex:
print(f"Infrastructure readiness failure: {ex}", file=sys.stderr)
job._submit_infra_failure(str(ex))
results = None
except requests.exceptions.HTTPError as ex:
print(ex, file=sys.stderr)
detail = ex.response.json().get('detail')
Expand All @@ -201,4 +282,4 @@ if __name__ == '__main__':
sys.exit(1 if results is None else 0)
{%- endif %}
{% endblock %}
{%- endblock %}
{%- endblock %}
116 changes: 87 additions & 29 deletions kernelci/kbuild.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,13 @@
import os
import re
import shlex
import socket
import subprocess
import sys
import tarfile
import threading
import time
import urllib.parse
from datetime import datetime, timedelta
from typing import Tuple

Expand Down Expand Up @@ -176,6 +178,82 @@ def _download_file(url, file_path):
return False


DEFAULT_READINESS_DEADLINE_SEC = 60.0
DEFAULT_REQUEST_TIMEOUT = (5.0, 15.0)
DEFAULT_RETRY_INTERVAL_SEC = 2.0
TRANSIENT_HTTP_STATUS_CODES = {429, 500, 502, 503, 504}


class KBuildInfraError(RuntimeError):
"""Raised when required network/infrastructure services are unreachable."""


def _sanitize_service_name(url):
"""Return scheme://host (or host) without paths, query strings, or credentials."""
parsed = urllib.parse.urlsplit(url)
if parsed.hostname:
if parsed.port:
return f"{parsed.hostname}:{parsed.port}"
return parsed.hostname
return url


def _request_with_retry(
url,
*,
deadline_sec=DEFAULT_READINESS_DEADLINE_SEC,
timeout=DEFAULT_REQUEST_TIMEOUT,
retry_interval=DEFAULT_RETRY_INTERVAL_SEC,
stream=False,
):
"""Perform an HTTP GET bounded by an explicit monotonic wall-clock deadline.

Only retries transient network/DNS errors and transient HTTP status codes.
Programming and configuration errors (such as malformed URLs) fail fast.
"""
service = _sanitize_service_name(url)
start = time.monotonic()
deadline = start + deadline_sec
last_error = "no attempts executed"

while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
break

attempt_timeout = (
min(timeout[0], remaining),
min(timeout[1], remaining),
)
try:
response = requests.get(url, stream=stream, timeout=attempt_timeout)
except (
requests.exceptions.Timeout,
requests.exceptions.ConnectionError,
socket.gaierror,
) as exc:
last_error = f"{type(exc).__name__}: {exc}"
else:
if response.status_code == 200:
return response
if response.status_code not in TRANSIENT_HTTP_STATUS_CODES:
raise KBuildInfraError(
f"Service '{service}' returned HTTP {response.status_code} "
f"after {time.monotonic() - start:.1f}s"
)
last_error = f"HTTP {response.status_code}"

remaining = deadline - time.monotonic()
if remaining <= 0:
break
time.sleep(min(retry_interval, remaining))

elapsed = time.monotonic() - start
raise KBuildInfraError(
f"Service '{service}' unreachable after {elapsed:.1f}s ({last_error})"
)


class KBuild:
"""
Build class that represents kernel build
Expand Down Expand Up @@ -562,14 +640,9 @@ def _fetch_firmware(self):
self.addcmd("cd ..")

def _download_file(self, url):
"""Download file from url and return it in buffer"""
try:
response = requests.get(url, timeout=30)
if response.status_code == 200:
return response.content
except requests.exceptions.RequestException as e:
print(f"Error downloading file: {e}")
return None
"""Download file from url with bounded network retries."""
response = _request_with_retry(url)
return response.content

def _getcipfragment(self, fragment):
"""Get CIP specific configuration fragments"""
Expand Down Expand Up @@ -633,25 +706,6 @@ def _getcrosfragment(self, fragment):

return (buffer, fragment)

def _verify_network(self):
"""Verify network connectivity"""
# TBD: Different URL? pool of urls?
retries = 10
while retries > 0:
try:
r = requests.get("https://google.com")
except Exception as e:
print(f"[_verify_network] Error: {e}")
time.sleep(5)
retries -= 1
continue
if r.status_code == 200:
return
time.sleep(5)

print("[_verify_network] Network is not available")
sys.exit(1)

def extract_config(self, frag):
"""Extract config fragments from legacy config file"""
txt = ""
Expand Down Expand Up @@ -921,8 +975,12 @@ def _generate_script(self):

def write_script(self, filename):
"""Write shell compile script to file"""
self._verify_network()
self._generate_script()
try:
self._generate_script()
except KBuildInfraError as exc:
print(f"[write_script] Infrastructure error: {exc}")
self.submit_failure(str(exc))
raise
fixed_header = """#!/bin/bash
set -eE -o pipefail # -E ⇒ trap inherits in functions/subshells

Expand Down
Loading