From 0be60d002311cd794ab5e668cbd41116382425ff Mon Sep 17 00:00:00 2001 From: Bilal Tonga Date: Thu, 29 Oct 2026 08:49:05 +0100 Subject: [PATCH 1/6] Extend timeout and improve raised errors --- src/crate/testing/layer.py | 49 ++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/src/crate/testing/layer.py b/src/crate/testing/layer.py index 1cdd408e..c7db3973 100644 --- a/src/crate/testing/layer.py +++ b/src/crate/testing/layer.py @@ -93,7 +93,11 @@ def _download_and_extract(uri, directory): def wait_for_http_url(log, timeout=30, verbose=False): start = time.monotonic() while True: - line = log.readline().decode("utf-8").strip() + raw = log.readline() + if not raw: + # EOF: the process exited before publishing its HTTP address. + return None + line = raw.decode("utf-8").strip() elapsed = time.monotonic() - start if verbose: sys.stderr.write("[{:>4.1f}s]{}\n".format(elapsed, line)) @@ -107,6 +111,7 @@ def wait_for_http_url(log, timeout=30, verbose=False): class OutputMonitor: def __init__(self): self.consumers = [] + self._out_thread: Optional[threading.Thread] = None def consume(self, iterable): for line in iterable: @@ -144,6 +149,7 @@ class CrateLayer: tmpdir = tempfile.gettempdir() wait_interval = 0.2 + startup_timeout = 90 @staticmethod def from_uri( @@ -244,6 +250,7 @@ def __init__( self.http_url = http_url_from_host_port(host, port) self.process: Optional[subprocess.Popen] = None + self.monitor = OutputMonitor() self.verbose = verbose self.env = env or {} self.env.setdefault("CRATE_USE_IPV4", "true") @@ -299,6 +306,10 @@ def create_settings( transport_port, further_settings=None, ): + if os.path.isdir(crate_config): + conf_dir = crate_config + else: + conf_dir = os.path.dirname(crate_config) settings = { "discovery.type": "zen", "discovery.initial_state_timeout": 0, @@ -306,7 +317,7 @@ def create_settings( "cluster.name": cluster_name, "network.host": host, "http.port": http_port, - "path.conf": os.path.dirname(crate_config), + "path.conf": conf_dir, "transport.tcp.port": transport_port, } if further_settings: @@ -346,7 +357,9 @@ def start(self): # try to read http_url from startup logs # this is necessary if no static port is assigned self.http_url = wait_for_http_url( - self.process.stdout, verbose=self.verbose + self.process.stdout, + timeout=self.startup_timeout, + verbose=self.verbose, ) self.monitor = OutputMonitor() @@ -354,17 +367,26 @@ def start(self): if not self.http_url: self.stop() - else: - sys.stderr.write("HTTP: {}\n".format(self.http_url)) - self._wait_for_start() - self._wait_for_master() - sys.stderr.write("\nCrate instance ready.\n") + raise SystemError( + "Failed to start Crate instance: HTTP address not found in " + "startup logs within {0}s. Use verbose=True to see the " + "logs.".format(self.startup_timeout) + ) + + sys.stderr.write("HTTP: {}\n".format(self.http_url)) + self._wait_for_start() + self._wait_for_master() + sys.stderr.write("\nCrate instance ready.\n") def stop(self): self.conn_pool.clear() if self.process: self.process.terminate() - self.process.communicate(timeout=10) + try: + self.process.communicate(timeout=10) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.communicate() stdout = self.process.stdout if stdout: stdout.close() @@ -375,6 +397,13 @@ def stop(self): def tearDown(self): self.stop() + def __enter__(self): + self.start() + return self + + def __exit__(self, *exc_info): + self.stop() + def _wait_for(self, validator): start = time.monotonic() @@ -390,7 +419,7 @@ def _wait_for(self, validator): self.stop() raise e - if wait_time > 90: + if wait_time > self.startup_timeout: for line in line_buf.lines: log.error(line) self.stop() From 794fe3b88afc8220c9c98a784d85d6393d7b4065 Mon Sep 17 00:00:00 2001 From: Bilal Tonga Date: Thu, 29 Oct 2026 08:50:29 +0100 Subject: [PATCH 2/6] Fix noisy CI logs with reducing verbosity --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 13bacc48..a29c0a1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,7 +104,7 @@ non_interactive = true [tool.pytest.ini_options] -addopts = "-rA --verbosity=3 --doctest-modules" +addopts = "-ra --verbosity=1 --durations=10 --doctest-modules" doctest_optionflags = "NORMALIZE_WHITESPACE ELLIPSIS" minversion = "2.0" testpaths = [ From 8028800c4f44ad9e3456f8cf65c0f7216c4f12e4 Mon Sep 17 00:00:00 2001 From: Bilal Tonga Date: Thu, 29 Oct 2026 08:51:26 +0100 Subject: [PATCH 3/6] Add CrateDB caching and improve download reliability with retries --- .github/workflows/tests.yml | 6 ++++++ tests/conftest.py | 17 ++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3af3583c..d3100b97 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -65,6 +65,12 @@ jobs: - name: Setup env run: uv pip install --group dev -e . + - name: Cache CrateDB + uses: actions/cache@v6 + with: + path: parts/crate + key: cratedb-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/conftest.py') }} + - name: Run Linters run: | ruff check . diff --git a/tests/conftest.py b/tests/conftest.py index f6325b2c..8877ebb3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -146,12 +146,23 @@ def get_crate_url() -> str: def download_cratedb(path: Path): + if not path.exists(): + _download_and_extract(path) + + +def _download_and_extract(path: Path, attempts: int = 3): url = get_crate_url() - if path.exists(): - return if not url.startswith("https:"): raise ValueError("Invalid url") - filename, _msg = urlretrieve(url) + for attempt in range(1, attempts + 1): + try: + filename, _msg = urlretrieve(url) + break + except OSError as ex: + if attempt == attempts: + raise + log.warning("Downloading %s failed, retrying: %s", url, ex) + time.sleep(5 * attempt) if sys.platform.startswith("win32"): with zipfile.ZipFile(filename) as z: first_file = z.namelist()[0] From e11c6b456b63a4ea78a04ba5917b116972b1d411 Mon Sep 17 00:00:00 2001 From: Bilal Tonga Date: Thu, 29 Oct 2026 08:54:38 +0100 Subject: [PATCH 4/6] Refactor tests to use pytest and improve CrateDB layer handling --- tests/testing/test_layer.py | 549 +++++++++++++++++++----------------- 1 file changed, 288 insertions(+), 261 deletions(-) diff --git a/tests/testing/test_layer.py b/tests/testing/test_layer.py index 250bfde1..0d7390d7 100644 --- a/tests/testing/test_layer.py +++ b/tests/testing/test_layer.py @@ -19,16 +19,17 @@ # with Crate these terms will supersede the license and you may use the # software solely pursuant to the terms of the relevant commercial agreement. -import json import os -import tempfile +import re +import socket +import time from io import BytesIO from pathlib import Path -from unittest import TestCase, mock -import urllib3 +import pytest -import crate +import crate.testing.layer +from crate.client import connect from crate.testing.layer import ( CrateLayer, http_url_from_host_port, @@ -38,280 +39,306 @@ from tests.client.settings import crate_path from tests.conftest import download_cratedb, get_crate_url +HTTP_LOG_LINE = ( + b"[i.c.p.h.CrateNettyHttpServerTransport] [crate] " + b"publish_address {127.0.0.1:4200}" +) + -class LayerUtilsTest(TestCase): - def test_prepend_http(self): - host = prepend_http("localhost") - self.assertEqual("http://localhost", host) - host = prepend_http("http://localhost") - self.assertEqual("http://localhost", host) - host = prepend_http("https://localhost") - self.assertEqual("https://localhost", host) - host = prepend_http("http") - self.assertEqual("http://http", host) - - def test_http_url(self): - url = http_url_from_host_port(None, None) - self.assertEqual(None, url) - url = http_url_from_host_port("localhost", None) - self.assertEqual(None, url) - url = http_url_from_host_port(None, 4200) - self.assertEqual(None, url) - url = http_url_from_host_port("localhost", 4200) - self.assertEqual("http://localhost:4200", url) - url = http_url_from_host_port("https://crate", 4200) - self.assertEqual("https://crate:4200", url) - - def test_wait_for_http(self): - log = BytesIO( - b"[i.c.p.h.CrateNettyHttpServerTransport] [crate] publish_address {127.0.0.1:4200}" # noqa: E501 - ) - addr = wait_for_http_url(log) - self.assertEqual("http://127.0.0.1:4200", addr) - log = BytesIO( - b"[i.c.p.h.CrateNettyHttpServerTransport] [crate] publish_address {}" # noqa: E501 - ) - addr = wait_for_http_url(log=log, timeout=1) - self.assertEqual(None, addr) - - @mock.patch.object( +def free_port() -> int: + """ + Returns a port the OS considers free + """ + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def wait_until(predicate, timeout=30, interval=0.5) -> bool: + deadline = time.monotonic() + timeout + while not predicate(): + if time.monotonic() > deadline: + return False + time.sleep(interval) + return True + + +def query(layer, stmt): + with connect(layer.crate_servers) as conn: + cursor = conn.cursor() + cursor.execute(stmt) + return cursor.fetchall() + + +def fake_crate(tmp_path: Path, script: str) -> str: + """ + Returns a stand-in for `bin/crate` + """ + path = tmp_path / "fake-crate" + path.write_text("#!/bin/sh\n" + script) + path.chmod(0o755) + return str(path) + + +@pytest.mark.parametrize( + "host, expected", + [ + ("localhost", "http://localhost"), + ("http://localhost", "http://localhost"), + ("https://localhost", "https://localhost"), + ("http", "http://http"), + ], +) +def test_prepend_http(host, expected): + assert prepend_http(host) == expected + + +@pytest.mark.parametrize( + "host, port, expected", + [ + (None, None, None), + ("localhost", None, None), + (None, 4200, None), + ("localhost", 4200, "http://localhost:4200"), + ("localhost", "4200", "http://localhost:4200"), + ("https://crate", 4200, "https://crate:4200"), + ("localhost", "4200-4299", None), + ], +) +def test_http_url_from_host_port(host, port, expected): + assert http_url_from_host_port(host, port) == expected + + +@pytest.mark.parametrize( + "log, expected", + [ + (HTTP_LOG_LINE, "http://127.0.0.1:4200"), + (b"starting\n" + HTTP_LOG_LINE, "http://127.0.0.1:4200"), + (b"[i.c.p.h.CrateNettyHttpServerTransport] publish_address {}", None), + (b"", None), + ], +) +def test_wait_for_http_url(log, expected): + assert wait_for_http_url(BytesIO(log), timeout=1) == expected + + +def test_wait_for_http_url_verbose(capsys): + wait_for_http_url(BytesIO(b"starting\n"), timeout=1, verbose=True) + assert "starting" in capsys.readouterr().err + + +def test_default_settings(tmp_path): + layer = CrateLayer("crate", crate_home=str(tmp_path)) + assert layer.start_cmd[0] == str(tmp_path / "bin" / "crate") + for flag in [ + "-Chttp.port=4200-4299", + "-Ctransport.tcp.port=4300-4399", + "-Cnetwork.host=127.0.0.1", + "-Cnode.name=crate", + "-Ccluster.name=TestingDynamic", + "-Cpath.conf={}".format(tmp_path / "config"), + ]: + assert flag in layer.start_cmd + assert layer.crate_servers == [] + assert layer.wdPath().endswith(os.path.join("crate_layer", "crate")) + + +def test_static_ports(tmp_path): + layer = CrateLayer( + "crate", + crate_home=str(tmp_path), + port=44219, + transport_port=44319, + cluster_name="my_cluster", + ) + assert layer.crate_servers == ["http://127.0.0.1:44219"] + for flag in [ + "-Chttp.port=44219", + "-Ctransport.tcp.port=44319", + "-Ccluster.name=my_cluster", + ]: + assert flag in layer.start_cmd + + +def test_additional_settings(tmp_path): + """ + Additional settings are handed over to CrateDB using ``-C`` flags, and + take precedence over keyword arguments. + """ + layer = CrateLayer( + "custom", + str(tmp_path), + port=44401, + settings={ + "cluster.graceful_stop.min_availability": "none", + "http.port": 44402, + "lang.js.enabled": True, + }, + ) + assert layer.crate_servers == ["http://127.0.0.1:44402"] + for flag in [ + "-Ccluster.graceful_stop.min_availability=none", + "-Chttp.port=44402", + "-Clang.js.enabled=true", + ]: + assert flag in layer.start_cmd + + +def test_crate_config_directory(tmp_path): + config_dir = tmp_path / "conf" + config_dir.mkdir() + layer = CrateLayer("crate", str(tmp_path), crate_config=str(config_dir)) + assert "-Cpath.conf={}".format(config_dir) in layer.start_cmd + + +def test_crate_config_file(tmp_path): + config = tmp_path / "conf" / "crate.yml" + config.parent.mkdir() + config.touch() + layer = CrateLayer("crate", str(tmp_path), crate_config=str(config)) + assert "-Cpath.conf={}".format(config.parent) in layer.start_cmd + + +def test_crate_config_must_be_named_crate_yml(tmp_path): + config = tmp_path / "other.yml" + config.touch() + with pytest.raises(ValueError, match="crate.yml"): + CrateLayer("crate", str(tmp_path), crate_config=str(config)) + + +def test_environment_variables(tmp_path, monkeypatch): + monkeypatch.setenv("JAVA_HOME", "/opt/jdk") + layer = CrateLayer("crate", str(tmp_path), env={"CRATE_HEAP_SIZE": "300m"}) + assert layer.env == { + "CRATE_HEAP_SIZE": "300m", + "CRATE_USE_IPV4": "true", + "JAVA_HOME": "/opt/jdk", + } + + +@pytest.mark.parametrize( + "java_home, env, expected", + [ + # JAVA_HOME must not be set to `None`: It would be literally + # interpreted as a string 'None', which is an invalid path. + (None, None, ""), + ("/opt/jdk-11", None, "/opt/jdk-11"), + ("/opt/jdk-11", {"JAVA_HOME": "/opt/jdk-12"}, "/opt/jdk-12"), + ], +) +def test_java_home(tmp_path, monkeypatch, java_home, env, expected): + if java_home is None: + monkeypatch.delenv("JAVA_HOME", raising=False) + else: + monkeypatch.setenv("JAVA_HOME", java_home) + layer = CrateLayer("crate", str(tmp_path), env=env) + assert layer.env["JAVA_HOME"] == expected + + +def test_layer_from_uri(tmp_path, monkeypatch): + """ + The layer can also be created from a URI pointing to a CrateDB tarball. + """ + monkeypatch.setattr( crate.testing.layer, "_download_and_extract", lambda uri, directory: None, ) - def test_layer_from_uri(self): - """ - The CrateLayer can also be created by providing an URI that points to - a CrateDB tarball. - """ - layer = CrateLayer.from_uri(get_crate_url(), - name="crate-by-uri", - http_port=42203 - ) - self.assertIsInstance(layer, CrateLayer) - - @mock.patch.dict("os.environ", {}, clear=True) - def test_java_home_env_not_set(self): - with tempfile.TemporaryDirectory() as tmpdir: - layer = CrateLayer("java-home-test", tmpdir) - # JAVA_HOME must not be set to `None`: It would be literally - # interpreted as a string 'None', which is an invalid path. - self.assertEqual(layer.env["JAVA_HOME"], "") - - @mock.patch.dict("os.environ", {}, clear=True) - def test_java_home_env_set(self): - java_home = "/usr/lib/jvm/java-11-openjdk-amd64" - with tempfile.TemporaryDirectory() as tmpdir: - os.environ["JAVA_HOME"] = java_home - layer = CrateLayer("java-home-test", tmpdir) - self.assertEqual(layer.env["JAVA_HOME"], java_home) - - @mock.patch.dict("os.environ", {}, clear=True) - def test_java_home_env_override(self): - java_11_home = "/usr/lib/jvm/java-11-openjdk-amd64" - java_12_home = "/usr/lib/jvm/java-12-openjdk-amd64" - with tempfile.TemporaryDirectory() as tmpdir: - os.environ["JAVA_HOME"] = java_11_home - layer = CrateLayer( - "java-home-test", tmpdir, env={"JAVA_HOME": java_12_home} - ) - self.assertEqual(layer.env["JAVA_HOME"], java_12_home) - - -class LayerTest(TestCase): - @classmethod - def setup_class(cls): - download_cratedb(Path(crate_path())) - - def test_basic(self): - """ - This layer starts and stops a ``Crate`` instance on a given host, port, - a given crate node name and, optionally, a given cluster name:: - """ - - port = 44219 - transport_port = 44319 - - layer = CrateLayer( - "crate", - crate_home=crate_path(), - host="127.0.0.1", - port=port, - transport_port=transport_port, - cluster_name="my_cluster", - ) - - # The working directory is defined on layer instantiation. - # It is sometimes required to know it before starting the layer. - self.assertRegex(layer.wdPath(), r".+/crate_layer/crate") - - # Start the layer. - layer.start() - - # The urls of the crate servers to be instantiated can be obtained - # via `crate_servers`. - self.assertEqual(layer.crate_servers, ["http://127.0.0.1:44219"]) - - # Access the CrateDB instance on the HTTP interface. - - http = urllib3.PoolManager() - - stats_uri = "http://127.0.0.1:{0}/".format(port) - response = http.request("GET", stats_uri) - self.assertEqual(response.status, 200) - - # The layer can be shutdown using its `stop()` method. - layer.stop() - - def test_dynamic_http_port(self): - """ - Verify defining a port range instead of a static HTTP port. + layer = CrateLayer.from_uri( + get_crate_url(), + name="crate-by-uri", + http_port=42203, + directory=str(tmp_path), + ) + assert layer.crate_servers == ["http://127.0.0.1:42203"] - CrateDB will start with the first available port in the given range and - the test layer obtains the chosen port from the startup logs of the - CrateDB process. - Note that this feature requires a logging configuration with at least - loglevel ``INFO`` on ``http``. - """ - port = "44200-44299" - layer = CrateLayer("crate", crate_home=crate_path(), port=port) +def test_start_fails_when_process_exits(tmp_path): + exe = fake_crate(tmp_path, "echo starting\nsleep 1\n") + layer = CrateLayer("crate", str(tmp_path), crate_exec=exe) + with pytest.raises(SystemError, match="HTTP address not found"): layer.start() - self.assertRegex(layer.crate_servers[0], r"http://127.0.0.1:442\d\d") - layer.stop() + assert layer.process is None - def test_default_settings(self): - """ - Starting a CrateDB layer leaving out optional parameters will apply - the following defaults. - The default http port is the first free port in the range of - ``4200-4299``, the default transport port is the first free port in - the range of ``4300-4399``, the host defaults to ``127.0.0.1``. - - The command to call is ``bin/crate`` inside the ``crate_home`` path. - The default config file is ``config/crate.yml`` inside ``crate_home``. - The default cluster name will be auto generated using the HTTP port. - """ - layer = CrateLayer("crate_defaults", crate_home=crate_path()) +def test_start_fails_after_timeout(tmp_path): + exe = fake_crate(tmp_path, "while true; do echo waiting; sleep 0.1; done\n") + layer = CrateLayer("crate", str(tmp_path), crate_exec=exe) + layer.startup_timeout = 1 + with pytest.raises(SystemError, match="within 1s"): layer.start() - self.assertEqual(layer.crate_servers[0], "http://127.0.0.1:4200") - layer.stop() + assert layer.process is None - def test_additional_settings(self): - """ - The CrateDB test layer can be started with additional settings as well. - - Add a dictionary for keyword argument ``settings`` which contains your - settings. Those additional setting will override settings given as - keyword argument. - - The settings will be handed over to the ``Crate`` process with the - ``-C`` flag. So, the setting ``threadpool.bulk.queue_size: 100`` - becomes the command line flag: ``-Cthreadpool.bulk.queue_size=100``:: - """ - layer = CrateLayer( - "custom", - crate_path(), - port=44401, - settings={ - "cluster.graceful_stop.min_availability": "none", - "http.port": 44402, - }, - ) - layer.start() - self.assertEqual(layer.crate_servers[0], "http://127.0.0.1:44402") - self.assertIn( - "-Ccluster.graceful_stop.min_availability=none", layer.start_cmd - ) - layer.stop() - def test_verbosity(self): - """ - The test layer hides the standard output of Crate per default. +def test_stop_without_start(tmp_path): + CrateLayer("crate", str(tmp_path)).stop() - To increase the verbosity level, the additional keyword argument - ``verbose`` needs to be set to ``True``:: - """ - layer = CrateLayer("crate", crate_home=crate_path(), verbose=True) - layer.start() - self.assertTrue(layer.verbose) - layer.stop() - def test_environment_variables(self): - """ - Verify providing environment variables for the CrateDB testing layer. - """ - layer = CrateLayer( - "crate", crate_home=crate_path(), env={"CRATE_HEAP_SIZE": "300m"} - ) +@pytest.fixture(scope="module") +def crate_home(): + download_cratedb(Path(crate_path())) + return crate_path() - layer.start() - - sql_uri = layer.crate_servers[0] + "/_sql" - http = urllib3.PoolManager() - response = http.urlopen( - "POST", - sql_uri, - body='{"stmt": "select heap[\'max\'] from sys.nodes"}', - ) - json_response = json.loads(response.data.decode("utf-8")) +@pytest.fixture +def start_layer(crate_home, tmp_path, monkeypatch): + """ + Starts CrateDB layers, which are stopped after the test, also on failure. + """ + monkeypatch.setattr(CrateLayer, "tmpdir", str(tmp_path)) + layers = [] - self.assertEqual(json_response["rows"][0][0], 314572800) + def start(name="crate", **kwargs): + layer = CrateLayer(name, crate_home=crate_home, **kwargs) + layer.start() + layers.append(layer) + return layer + yield start + for layer in layers: layer.stop() - def test_cluster(self): - """ - To start a cluster of ``Crate`` instances, give each instance the same - ``cluster_name``. If you want to start instances on the same machine, - use value ``_local_`` for ``host`` and give every node different ports:: - """ - cluster_layer1 = CrateLayer( - "crate1", - crate_path(), - host="_local_", - cluster_name="my_cluster", - ) - cluster_layer2 = CrateLayer( - "crate2", - crate_path(), - host="_local_", - cluster_name="my_cluster", - settings={"discovery.initial_state_timeout": "10s"}, - ) - - # If we start both layers, they will, after a small amount of time, - # find each other, and form a cluster. - cluster_layer1.start() - cluster_layer2.start() - - # We can verify that by checking the number of nodes a node knows about. - http = urllib3.PoolManager() - - def num_cluster_nodes(crate_layer): - sql_uri = crate_layer.crate_servers[0] + "/_sql" - response = http.urlopen( - "POST", - sql_uri, - body='{"stmt":"select count(*) from sys.nodes"}', - ) - json_response = json.loads(response.data.decode("utf-8")) - return json_response["rows"][0][0] - - # We might have to wait a moment before the cluster is finally created. - num_nodes = num_cluster_nodes(cluster_layer1) - import time - - retries = 0 - while num_nodes < 2: # pragma: no cover - time.sleep(1) - num_nodes = num_cluster_nodes(cluster_layer1) - retries += 1 - if retries == 30: - break - self.assertEqual(num_nodes, 2) - - cluster_layer1.stop() - cluster_layer2.stop() + +def test_start_and_stop(crate_home, tmp_path, monkeypatch): + """ + The layer starts CrateDB on the first free port of the default range, + reads the chosen address from the startup logs, and stops it again. + """ + monkeypatch.setattr(CrateLayer, "tmpdir", str(tmp_path)) + with CrateLayer("crate", crate_home=crate_home) as layer: + assert re.match(r"http://127.0.0.1:42\d\d$", layer.crate_servers[0]) + result = query(layer, "select name from sys.cluster") + assert result == [["TestingDynamic"]] + process = layer.process + + assert layer.process is None + assert process.poll() is not None + assert not os.path.exists(layer.wdPath()) + + +def test_static_ports_and_environment(start_layer): + port = free_port() + layer = start_layer( + port=port, + transport_port=free_port(), + env={"CRATE_HEAP_SIZE": "300m"}, + ) + assert layer.crate_servers == ["http://127.0.0.1:{}".format(port)] + result = query(layer, "select heap['max'] from sys.nodes") + assert result == [[314572800]] + + +def test_cluster(start_layer): + """ + Layers with the same cluster name find each other and form a cluster. + """ + layer1 = start_layer("crate1", host="_local_", cluster_name="my_cluster") + start_layer( + "crate2", + host="_local_", + cluster_name="my_cluster", + settings={"discovery.initial_state_timeout": "10s"}, + ) + assert wait_until( + lambda: query(layer1, "select count(*) from sys.nodes") == [[2]] + ) From f4a1ce7e4216a9c6ed5b4ba6f02a378cf8562f43 Mon Sep 17 00:00:00 2001 From: Bilal Tonga Date: Thu, 29 Oct 2026 08:56:25 +0100 Subject: [PATCH 5/6] Update changelog with new fixes --- CHANGES.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 5ea4df71..02e0bf41 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,16 @@ Changes for crate ================= +Unreleased +========== + +- Testing: ``CrateLayer.start()`` now raises ``SystemError`` when CrateDB does + not report its HTTP address. The startup timeout is raised from 30s to 90s. +- Testing: ``CrateLayer`` can be used as a context manager, which starts and + stops CrateDB. +- Testing: Fixed ``CrateLayer`` using the parent of the ``crate_config`` + directory as ``path.conf``, instead of the directory itself. + 2026/09/17 2.3.0 ================ - Added ``DefaultTypeConverter`` support that decodes ``DataType.UUID`` From 5a9f0c4302511519148f153ecf8b2fdc112cbdc1 Mon Sep 17 00:00:00 2001 From: Bilal Tonga Date: Fri, 30 Oct 2026 10:30:41 +0100 Subject: [PATCH 6/6] Improve HTTP client examples and enhance connection error handling --- docs/by-example/http.rst | 14 ++++++++++---- src/crate/testing/layer.py | 4 +++- tests/conftest.py | 2 ++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/by-example/http.rst b/docs/by-example/http.rst index 55f1924c..7c6759d3 100644 --- a/docs/by-example/http.rst +++ b/docs/by-example/http.rst @@ -32,11 +32,17 @@ If no ``server`` argument (or no argument at all) is passed, the default one ['http://127.0.0.1:4200'] >>> http_client.close() -When using a list of servers, the servers are selected by round-robin: +The following examples use unreachable servers. A short connect timeout makes +them fail fast, while the read timeout leaves CrateDB enough time to respond: + >>> import urllib3 >>> invalid_host1 = "192.0.2.1:9999" >>> invalid_host2 = "192.0.2.2:9999" - >>> http_client = HttpClient([crate_host, invalid_host1, invalid_host2], timeout=0.3) + >>> timeout = urllib3.Timeout(connect=0.3, read=10) + +When using a list of servers, the servers are selected by round-robin: + + >>> http_client = HttpClient([crate_host, invalid_host1, invalid_host2], timeout=timeout) >>> http_client._get_server() 'http://127.0.0.1:44209' @@ -50,7 +56,7 @@ When using a list of servers, the servers are selected by round-robin: Servers with connection errors will be removed from the active server list: - >>> http_client = HttpClient([invalid_host1, invalid_host2, crate_host], timeout=0.3) + >>> http_client = HttpClient([invalid_host1, invalid_host2, crate_host], timeout=timeout) >>> result = http_client.sql('select name from locations') >>> http_client._active_servers ['http://127.0.0.1:44209'] @@ -70,7 +76,7 @@ sleep after the first request:: If no active servers are available and the retry interval is not reached, just use the oldest inactive one: - >>> http_client = HttpClient([invalid_host1, invalid_host2, crate_host], timeout=0.3) + >>> http_client = HttpClient([invalid_host1, invalid_host2, crate_host], timeout=timeout) >>> result = http_client.sql('select name from locations') >>> http_client._active_servers = [] >>> http_client._get_server() diff --git a/src/crate/testing/layer.py b/src/crate/testing/layer.py index c7db3973..c17ecc17 100644 --- a/src/crate/testing/layer.py +++ b/src/crate/testing/layer.py @@ -437,7 +437,9 @@ def _wait_for_start(self): # after the layer starts don't result in 503 def validator(): try: - resp = self.conn_pool.request("HEAD", self.http_url) + resp = self.conn_pool.request( + "HEAD", self.http_url, retries=False + ) return resp.status == 200 except Exception: return False diff --git a/tests/conftest.py b/tests/conftest.py index 8877ebb3..bf682516 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,8 @@ log = logging.getLogger("tests.conftest") +logging.getLogger("urllib3.connectionpool").setLevel(logging.ERROR) + REQUEST_PATH = "crate.client.http.Server.request" URL_TMPL = "https://cdn.crate.io/downloads/releases/cratedb/{arch}_{os}/crate-6.1.2.{ext}"