From b1c17b32b7c248d972bbd662de59062581400031 Mon Sep 17 00:00:00 2001 From: Aditya Grewal Date: Tue, 17 Nov 2026 23:51:22 +0100 Subject: [PATCH] kbuild: replace google.com probes with bounded retries Remove the hard-coded google.com connectivity and DNS probes from KBuild and config/runtime/base/python.jinja2, which could hang indefinitely or loop without bound when Kubernetes pod networking was still initializing. Apply bounded retries with explicit (5.0, 15.0)s connect/read timeouts, a 2.0s retry interval, and a 60.0s monotonic wall-clock deadline to the initial KernelCI API and storage requests, reporting exhausted readiness as a sanitized infrastructure error rather than calling sys.exit(). Closes: #3197 Assisted-by: Gemini Signed-off-by: Aditya Grewal --- config/runtime/base/python.jinja2 | 121 +++++++++++++++++---- kernelci/kbuild.py | 116 +++++++++++++++----- tests/test_kbuild.py | 173 ++++++++++++++++++++++++++++++ 3 files changed, 361 insertions(+), 49 deletions(-) diff --git a/config/runtime/base/python.jinja2 b/config/runtime/base/python.jinja2 index 93fe263a42..f7fb9afe5e 100644 --- a/config/runtime/base/python.jinja2 +++ b/config/runtime/base/python.jinja2 @@ -55,16 +55,39 @@ STORAGE_CONFIG_YAML = """ NODE = {{ node }} JOB_NAME = '{{ name }}' WORKSPACE = '/tmp/kci' +READINESS_DEADLINE_SEC = 60.0 +REQUEST_TIMEOUT = (5.0, 15.0) +RETRY_INTERVAL_SEC = 2.0 +TRANSIENT_HTTP_STATUS_CODES = {429, 500, 502, 503, 504} {%- endblock %} {% block python_base_job -%} +class InfraReadinessError(RuntimeError): + """Raised when required KernelCI network services do not become ready in time.""" + + +def _sanitize_service_name(url_or_name): + parsed = urllib.parse.urlsplit(str(url_or_name)) + if parsed.hostname: + if parsed.port: + return f"{parsed.hostname}:{parsed.port}" + return parsed.hostname + return str(url_or_name) + + class BaseJob: def __init__(self, api_config, node, workspace): + self._api_url = self._extract_api_url(api_config) self._api = self._get_api(api_config) self._node = node self._workspace = workspace + def _extract_api_url(self, api_config_yaml): + data = yaml.safe_load(api_config_yaml) or {} + api_data = data.get('api', data) if isinstance(data, dict) else {} + return api_data.get('url', 'kernelci-api') + def _get_api(self, api_config_yaml): api_config = kernelci.config.api.API.load_from_yaml( yaml.safe_load(api_config_yaml), name='api' @@ -87,11 +110,55 @@ class BaseJob: storage_cred = os.getenv('KCI_STORAGE_CREDENTIALS') return kernelci.storage.get_storage(storage_config, storage_cred) + def _call_with_retry( + self, + func, + service_url, + deadline_sec=READINESS_DEADLINE_SEC, + retry_interval=RETRY_INTERVAL_SEC, + ): + service = _sanitize_service_name(service_url) + start = time.monotonic() + deadline = start + deadline_sec + last_error = "no attempts executed" + + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + + try: + return func(remaining) + except ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + socket.gaierror, + ) as exc: + last_error = f"{type(exc).__name__}: {exc}" + except requests.exceptions.HTTPError as exc: + status = exc.response.status_code if exc.response is not None else None + if status not in TRANSIENT_HTTP_STATUS_CODES: + raise + last_error = f"HTTP {status}" + + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(retry_interval, remaining)) + + elapsed = time.monotonic() - start + raise InfraReadinessError( + f"Service '{service}' unreachable after {elapsed:.1f}s ({last_error})" + ) + def _get_artifact_url(self, node, artifact): if node.get('artifacts') and artifact in node['artifacts']: return node['artifacts'][artifact] if node.get('parent'): - parent = self._api.node.get(node['parent']) + parent = self._call_with_retry( + lambda _rem: self._api.node.get(node['parent']), + self._api_url, + ) return self._get_artifact_url(parent, artifact) raise ValueError(f"'{artifact}' artifact not found in node {node['id']} ancestors") @@ -103,8 +170,17 @@ class BaseJob: path = self._workspace # Ensure our destination folder exists os.makedirs(path, exist_ok=True) - resp = requests.get(url, stream=True) - resp.raise_for_status() + + def _fetch(remaining): + attempt_timeout = ( + min(REQUEST_TIMEOUT[0], remaining), + min(REQUEST_TIMEOUT[1], remaining), + ) + r = requests.get(url, stream=True, timeout=attempt_timeout) + r.raise_for_status() + return r + + resp = self._call_with_retry(_fetch, url) tarball_name = os.path.basename(urllib.parse.urlparse(url).path) base, ext = tarball_name.split('.tar.') with tarfile.open(fileobj=resp.raw, mode=f'r|{ext}') as tarball: @@ -122,7 +198,10 @@ class BaseJob: raise NotImplementedError("_run() method required to run job") def _submit(self, result): - node_from_id = self._api.node.get(self._node['id']) + node_from_id = self._call_with_retry( + lambda _rem: self._api.node.get(self._node['id']), + self._api_url, + ) self._node = node_from_id.copy() self._node.update({ 'result': result, @@ -130,6 +209,19 @@ class BaseJob: }) self._api.node.update(self._node) + def _submit_infra_failure(self, message): + try: + node = self._node.copy() + node['result'] = 'incomplete' + node['state'] = 'done' + data = dict(node.get('data') or {}) + data['error_code'] = 'infra_error' + data['error_msg'] = message + node['data'] = data + self._api.node.update(node) + except Exception as exc: + print(f"Failed to report infrastructure error to API: {exc}", file=sys.stderr) + def run(self): if self._node.get('debug') and 'dry_run' in self._node['debug']: return self.dry_run(self._node['debug']['result']) @@ -160,25 +252,14 @@ class Job(BaseJob): {% block python_main -%} def main(args): - # Sometimes container DNS is failing if k8s node is still booting - # This is a workaround: - # test www.google.com until it works or retry 10 times - for _ in range(30): - try: - socket.gethostbyname('www.google.com') - break - except socket.gaierror: - pass - # sleep 1 second - time.sleep(1) - else: - print("DNS resolution failed, exiting", file=sys.stderr) - sys.exit(1) - job = Job({% block python_job_constr %}api_config=API_CONFIG_YAML, node=NODE, workspace=WORKSPACE{% endblock %}) try: results = job.run() job.submit(results) + except InfraReadinessError as ex: + print(f"Infrastructure readiness failure: {ex}", file=sys.stderr) + job._submit_infra_failure(str(ex)) + results = None except requests.exceptions.HTTPError as ex: print(ex, file=sys.stderr) detail = ex.response.json().get('detail') @@ -201,4 +282,4 @@ if __name__ == '__main__': sys.exit(1 if results is None else 0) {%- endif %} {% endblock %} -{%- endblock %} +{%- endblock %} \ No newline at end of file diff --git a/kernelci/kbuild.py b/kernelci/kbuild.py index 4c85f8cc03..18702483a8 100644 --- a/kernelci/kbuild.py +++ b/kernelci/kbuild.py @@ -30,11 +30,13 @@ import os import re import shlex +import socket import subprocess import sys import tarfile import threading import time +import urllib.parse from datetime import datetime, timedelta from typing import Tuple @@ -176,6 +178,82 @@ def _download_file(url, file_path): return False +DEFAULT_READINESS_DEADLINE_SEC = 60.0 +DEFAULT_REQUEST_TIMEOUT = (5.0, 15.0) +DEFAULT_RETRY_INTERVAL_SEC = 2.0 +TRANSIENT_HTTP_STATUS_CODES = {429, 500, 502, 503, 504} + + +class KBuildInfraError(RuntimeError): + """Raised when required network/infrastructure services are unreachable.""" + + +def _sanitize_service_name(url): + """Return scheme://host (or host) without paths, query strings, or credentials.""" + parsed = urllib.parse.urlsplit(url) + if parsed.hostname: + if parsed.port: + return f"{parsed.hostname}:{parsed.port}" + return parsed.hostname + return url + + +def _request_with_retry( + url, + *, + deadline_sec=DEFAULT_READINESS_DEADLINE_SEC, + timeout=DEFAULT_REQUEST_TIMEOUT, + retry_interval=DEFAULT_RETRY_INTERVAL_SEC, + stream=False, +): + """Perform an HTTP GET bounded by an explicit monotonic wall-clock deadline. + + Only retries transient network/DNS errors and transient HTTP status codes. + Programming and configuration errors (such as malformed URLs) fail fast. + """ + service = _sanitize_service_name(url) + start = time.monotonic() + deadline = start + deadline_sec + last_error = "no attempts executed" + + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + + attempt_timeout = ( + min(timeout[0], remaining), + min(timeout[1], remaining), + ) + try: + response = requests.get(url, stream=stream, timeout=attempt_timeout) + except ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + socket.gaierror, + ) as exc: + last_error = f"{type(exc).__name__}: {exc}" + else: + if response.status_code == 200: + return response + if response.status_code not in TRANSIENT_HTTP_STATUS_CODES: + raise KBuildInfraError( + f"Service '{service}' returned HTTP {response.status_code} " + f"after {time.monotonic() - start:.1f}s" + ) + last_error = f"HTTP {response.status_code}" + + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(retry_interval, remaining)) + + elapsed = time.monotonic() - start + raise KBuildInfraError( + f"Service '{service}' unreachable after {elapsed:.1f}s ({last_error})" + ) + + class KBuild: """ Build class that represents kernel build @@ -562,14 +640,9 @@ def _fetch_firmware(self): self.addcmd("cd ..") def _download_file(self, url): - """Download file from url and return it in buffer""" - try: - response = requests.get(url, timeout=30) - if response.status_code == 200: - return response.content - except requests.exceptions.RequestException as e: - print(f"Error downloading file: {e}") - return None + """Download file from url with bounded network retries.""" + response = _request_with_retry(url) + return response.content def _getcipfragment(self, fragment): """Get CIP specific configuration fragments""" @@ -633,25 +706,6 @@ def _getcrosfragment(self, fragment): return (buffer, fragment) - def _verify_network(self): - """Verify network connectivity""" - # TBD: Different URL? pool of urls? - retries = 10 - while retries > 0: - try: - r = requests.get("https://google.com") - except Exception as e: - print(f"[_verify_network] Error: {e}") - time.sleep(5) - retries -= 1 - continue - if r.status_code == 200: - return - time.sleep(5) - - print("[_verify_network] Network is not available") - sys.exit(1) - def extract_config(self, frag): """Extract config fragments from legacy config file""" txt = "" @@ -921,8 +975,12 @@ def _generate_script(self): def write_script(self, filename): """Write shell compile script to file""" - self._verify_network() - self._generate_script() + try: + self._generate_script() + except KBuildInfraError as exc: + print(f"[write_script] Infrastructure error: {exc}") + self.submit_failure(str(exc)) + raise fixed_header = """#!/bin/bash set -eE -o pipefail # -E ⇒ trap inherits in functions/subshells diff --git a/tests/test_kbuild.py b/tests/test_kbuild.py index aad40b627b..1b5566dbf5 100644 --- a/tests/test_kbuild.py +++ b/tests/test_kbuild.py @@ -3,11 +3,14 @@ import json import os +import socket import subprocess import sys import types +import unittest.mock import pytest +import requests import kernelci.kbuild from kernelci.kbuild import KBuild @@ -782,3 +785,173 @@ def test_falls_back_when_cache_absent(self, tmp_path): calls = log.read_text().splitlines() assert len(calls) == 1 and self.FILES_URL in calls[0] assert (tmp_path / "linux.tgz").exists() + + +class TestBoundedNetworkRetry: + """Tests for bounded network readiness/retry handling in kernelci.kbuild.""" + + URL = "https://storage.kernelci.org/fragments/test.config?token=secret123" + + @staticmethod + def _mock_response(status_code=200, content=b"CONFIG_FOO=y\n"): + resp = unittest.mock.MagicMock() + resp.status_code = status_code + resp.content = content + return resp + + def test_immediate_success(self): + from kernelci.kbuild import _request_with_retry + + with ( + unittest.mock.patch( + "kernelci.kbuild.requests.get", + return_value=self._mock_response(200, b"ok"), + ) as mock_get, + unittest.mock.patch("kernelci.kbuild.time.sleep") as mock_sleep, + ): + resp = _request_with_retry(self.URL) + assert resp.content == b"ok" + assert mock_get.call_count == 1 + assert mock_get.call_args.kwargs["timeout"] == (5.0, 15.0) + mock_sleep.assert_not_called() + + def test_initial_failures_then_recovery(self): + from kernelci.kbuild import _request_with_retry + + side_effects = [ + socket.gaierror("Temporary failure in name resolution"), + requests.exceptions.ConnectTimeout("connect timed out"), + self._mock_response(503), + self._mock_response(200, b"recovered"), + ] + with ( + unittest.mock.patch( + "kernelci.kbuild.requests.get", side_effect=side_effects + ) as mock_get, + unittest.mock.patch("kernelci.kbuild.time.sleep") as mock_sleep, + ): + resp = _request_with_retry( + self.URL, deadline_sec=30.0, retry_interval=2.0 + ) + assert resp.content == b"recovered" + assert mock_get.call_count == 4 + assert mock_sleep.call_count == 3 + + @pytest.mark.parametrize( + "exc", + [ + socket.gaierror("Name or service not known"), + requests.exceptions.ConnectTimeout("connect timed out"), + requests.exceptions.ReadTimeout("read timed out"), + requests.exceptions.ConnectionError("connection refused"), + ], + ) + def test_transient_network_errors_exhaust_and_sanitize_url(self, exc): + from kernelci.kbuild import KBuildInfraError, _request_with_retry + + with ( + unittest.mock.patch( + "kernelci.kbuild.requests.get", side_effect=exc + ), + unittest.mock.patch( + "kernelci.kbuild.time.monotonic", + side_effect=[0.0, 0.1, 1.0, 2.0, 3.0, 6.0, 6.0], + ), + unittest.mock.patch("kernelci.kbuild.time.sleep"), + ): + with pytest.raises(KBuildInfraError) as info: + _request_with_retry( + self.URL, deadline_sec=5.0, retry_interval=2.0 + ) + msg = str(info.value) + assert "storage.kernelci.org" in msg + assert "secret123" not in msg + assert "6.0s" in msg + + def test_repeated_non_success_http_responses_bounded(self): + from kernelci.kbuild import KBuildInfraError, _request_with_retry + + with ( + unittest.mock.patch( + "kernelci.kbuild.requests.get", + return_value=self._mock_response(503), + ) as mock_get, + unittest.mock.patch( + "kernelci.kbuild.time.monotonic", + side_effect=[0.0, 0.5, 2.5, 3.0, 5.5, 5.5], + ), + unittest.mock.patch("kernelci.kbuild.time.sleep") as mock_sleep, + ): + with pytest.raises(KBuildInfraError): + _request_with_retry( + self.URL, deadline_sec=5.0, retry_interval=2.0 + ) + assert mock_get.call_count == 2 + assert mock_sleep.call_count == 1 + + def test_programming_error_not_swallowed(self): + from kernelci.kbuild import _request_with_retry + + with unittest.mock.patch( + "kernelci.kbuild.requests.get", + side_effect=requests.exceptions.MissingSchema("Invalid URL"), + ): + with pytest.raises(requests.exceptions.MissingSchema): + _request_with_retry("not-a-valid-url") + + def test_write_script_reports_infra_failure_on_network_exhaustion( + self, tmp_path + ): + from kernelci.kbuild import KBuildInfraError + + kbuild = _kbuild(tmp_path) + with ( + unittest.mock.patch.object( + kbuild, + "_generate_script", + side_effect=KBuildInfraError( + "Service 'storage.kernelci.org' unreachable after 60.0s" + ), + ), + unittest.mock.patch.object( + kbuild, "submit_failure" + ) as mock_submit_failure, + ): + with pytest.raises(KBuildInfraError): + kbuild.write_script(str(tmp_path / "build.sh")) + mock_submit_failure.assert_called_once_with( + "Service 'storage.kernelci.org' unreachable after 60.0s" + ) + + +class TestPythonTemplateNetworkReadiness: + """Verify rendered runtime template uses bounded KernelCI service checks.""" + + @staticmethod + def _render_python_template(): + from pathlib import Path + + from jinja2 import Environment, FileSystemLoader + + templates = Path(__file__).parents[1] / "config" / "runtime" + env = Environment(loader=FileSystemLoader(templates)) + env.globals["kci_yaml_dump"] = lambda data: ( + "api:\n url: https://api.kernelci.org\n" + ) + template = env.get_template("base/python.jinja2") + return template.render( + runtime="docker", + api_config={"url": "https://api.kernelci.org"}, + storage_config={}, + node={"id": "node-1", "kind": "kbuild"}, + name="kbuild-test", + always_exit_0=False, + ) + + def test_no_google_probe_and_bounded_retry_present(self): + rendered = self._render_python_template() + assert "google.com" not in rendered + assert "time.monotonic()" in rendered + assert "REQUEST_TIMEOUT = (5.0, 15.0)" in rendered + assert "InfraReadinessError" in rendered + compile(rendered, "python.jinja2", "exec")