From 43ef6de3c5e1c4131e2ec77eaca6e9a8ac21147e Mon Sep 17 00:00:00 2001 From: Pontus Lurcock Date: Wed, 10 Jun 2026 15:47:01 +0200 Subject: [PATCH 1/5] Remove xcengine.util dependency on pystac write_stac now fails silently if pystac is not available. --- test/test_util.py | 23 +++++++++++++++++++++++ xcengine/util.py | 8 ++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index b50fedc..bd3f341 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -85,6 +85,29 @@ def test_write_stac(tmp_path, dataset, write_datasets, pre_existing_catalog): } +orig_import = __import__ +def import_mock(name, *args): + if name == 'pystac': + raise ModuleNotFoundError("No module named 'pystac'") + return orig_import(name, *args) + + +def test_write_stac_no_pystac(tmp_path, dataset): + # Import hooks are the recommended "clean" way to do this, but don't work + # in this case. + with mock.patch('builtins.__import__', side_effect=import_mock): + # pytest imports xcengine.util long before we can patch __import__, + # so we delete pystac from util's namespace (if present) instead. + # This gives a NameError on access rather than a ModuleNotFoundError + # on import, but the important thing is to break any implementation + # that tries to import pystac without checking if it's available. + import xcengine.util + xcengine.util.__dict__.pop("pystac", None) + from xcengine.util import write_stac + write_stac({"ds1": dataset}, tmp_path) + # We want nothing to happen here, so no explicit assertions. + + @pytest.mark.parametrize("eoap_mode", [False, True]) @pytest.mark.parametrize("ds2_format", [None, "zarr", "netcdf"]) def test_save_datasets(tmp_path, dataset, eoap_mode, ds2_format): diff --git a/xcengine/util.py b/xcengine/util.py index 71dd2b4..8c528c6 100644 --- a/xcengine/util.py +++ b/xcengine/util.py @@ -8,11 +8,9 @@ import shutil from typing import NamedTuple, Mapping -import pystac import xarray as xr from xarray import Dataset - def clear_directory(directory: pathlib.Path) -> None: for path in directory.iterdir(): if path.is_dir(): @@ -24,6 +22,12 @@ def clear_directory(directory: pathlib.Path) -> None: def write_stac( datasets: Mapping[str, xr.Dataset], stac_root: pathlib.Path ) -> None: + try: + import pystac + except ModuleNotFoundError: + # If pystac isn't present, we assume that stage-out is not required + # and exit quietly. + return catalog_path = stac_root / "catalog.json" if catalog_path.exists(): # Assume that the user code generated its own stage-out data From 6486bcd17888eecdcc49f159d6751638a942d00a Mon Sep 17 00:00:00 2001 From: Pontus Lurcock Date: Thu, 18 Jun 2026 16:19:56 +0200 Subject: [PATCH 2/5] Make xcube and pystac optional in created envs Related to #56 --- test/test_core.py | 47 +++++++++++++++++++++++++++++++++++------------ xcengine/core.py | 8 +++++++- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/test/test_core.py b/test/test_core.py index 29347af..09153c1 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -354,7 +354,9 @@ def test_script_creator_notebook_config(): def test_script_creator_notebook_config_http(httpserver): http_path = "/mynotebook.ipynb" nb_path = pathlib.Path(__file__).parent / "data" / "paramtest.ipynb" - httpserver.expect_request(http_path).respond_with_data(nb_path.read_bytes()) + httpserver.expect_request(http_path).respond_with_data( + nb_path.read_bytes() + ) script_creator = ScriptCreator(httpserver.url_for(http_path)) config = script_creator.nb_params.config assert config["environment_file"] == "my-environment.yml" @@ -382,12 +384,20 @@ def test_image_builder_write_dockerfile(tmp_path): @patch("docker.from_env") @pytest.mark.parametrize("env_type", ["none", "local", "http"]) @pytest.mark.parametrize("skip_build", [False, True]) +@pytest.mark.parametrize("with_eoap", [False, True]) +@pytest.mark.parametrize("with_xcube", [False, True]) +@pytest.mark.parametrize("pystac_in_deps", [False, True]) +@pytest.mark.parametrize("xcube_in_deps", [False, True]) def test_image_builder_build_dir( - from_env_mock, - tmp_path, - httpserver, - env_type, - skip_build + from_env_mock, + tmp_path, + httpserver, + env_type, + skip_build, + with_eoap, + with_xcube, + pystac_in_deps, + xcube_in_deps, ): client_mock = Mock(docker.client.DockerClient) client_mock.images.build.return_value = None, None @@ -398,16 +408,22 @@ def test_image_builder_build_dir( env_def = { "name": "foo", "channels": "bar", - "dependencies": ["python >=3.13", "baz >=42.0"], + "dependencies": ["python >=3.13", "baz >=42.0"] + + (["pystac"] if pystac_in_deps else []) + + (["xcube"] if xcube_in_deps else []), } build_env_path.write_text(yaml.safe_dump(env_def)) env_http = "/env2.yaml" match env_type: - case "none": env_param = None - case "local": env_param = build_env_path + case "none": + env_param = None + case "local": + env_param = build_env_path case "http": - httpserver.expect_request(env_http).respond_with_data(build_env_path.read_bytes()) + httpserver.expect_request(env_http).respond_with_data( + build_env_path.read_bytes() + ) env_param = httpserver.url_for(env_http) case _: raise RuntimeError(f"Unknown env type {env_type}") @@ -418,7 +434,9 @@ def test_image_builder_build_dir( build_dir, None, ) - image_builder.build(skip_build=skip_build) + image_builder.build( + skip_build=skip_build, with_eoap=with_eoap, with_xcube=with_xcube + ) if skip_build: from_env_mock.assert_not_called() else: @@ -430,7 +448,12 @@ def test_image_builder_build_dir( if env_type != "none": assert output_env["name"] == env_def["name"] assert output_env["channels"] == env_def["channels"] - assert set(output_env["dependencies"]) >= set(env_def["dependencies"]) + + output_deps = set(output_env["dependencies"]) + input_deps = set(env_def["dependencies"]) + assert output_deps >= input_deps + assert ("pystac" in output_deps) == pystac_in_deps or with_eoap + assert ("xcube" in output_deps) == xcube_in_deps or with_xcube cwl = image_builder.create_cwl() assert "cwlVersion" in cwl diff --git a/xcengine/core.py b/xcengine/core.py index 5cbd315..f084b70 100644 --- a/xcengine/core.py +++ b/xcengine/core.py @@ -241,6 +241,8 @@ def __init__( def build( self, skip_build: bool = False, + with_eoap: bool = True, + with_xcube: bool = True, ) -> Image | None: self.script_creator.convert_notebook_to_script(self.build_dir) if self.environment: @@ -254,7 +256,11 @@ def build( ) env_def = self.export_conda_env() # We need xcube for server/viewer and pystac for EOAP stage-in/out - self.add_packages_to_environment(env_def, ["xcube", "pystac"]) + self.add_packages_to_environment( + env_def, + (["xcube"] if with_xcube else []) + + (["pystac"] if with_eoap else []), + ) with open(self.build_dir / "environment.yml", "w") as fh: fh.write(yaml.safe_dump(env_def)) self.write_dockerfile(self.build_dir / "Dockerfile") From b54e5215f127c4a7972dd87072b1436a182ed774 Mon Sep 17 00:00:00 2001 From: Pontus Lurcock Date: Thu, 18 Jun 2026 16:59:00 +0200 Subject: [PATCH 3/5] Remove some redundant code from wrapper.py The call to xcube.util.plugin.init_plugins() is already made in util.start_server (where it belongs). --- xcengine/wrapper.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/xcengine/wrapper.py b/xcengine/wrapper.py index 17ea2ea..6bca37c 100644 --- a/xcengine/wrapper.py +++ b/xcengine/wrapper.py @@ -38,12 +38,8 @@ def __xce_set_params(config_locator: str | pathlib.Path = sys.argv[0]): exec(user_code) import argparse -import pathlib import xarray as xr -import xcube.util.plugin -import xcube.core.new -import xcube.webapi.viewer def main(): @@ -62,7 +58,6 @@ def main(): if args.verbose > 0: LOGGER.setLevel(logging.DEBUG) - xcube.util.plugin.init_plugins() datasets = { name: thing for name, thing in globals().copy().items() From cc89546524018fbedd017f165dd105ab7bd56c02 Mon Sep 17 00:00:00 2001 From: Pontus Lurcock Date: Fri, 19 Jun 2026 18:19:42 +0200 Subject: [PATCH 4/5] Remove xcube.util.start_server dependency on xcube start_server will now exit with a log message if xcube is not present in the environment, rather than raising an ImportError as before. Related to #56. --- test/test_util.py | 45 +++++++++++++++++++++++++++++++++++++++------ xcengine/util.py | 23 +++++++++++++++++------ 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index bd3f341..dd62852 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,3 +1,4 @@ +import argparse import logging import os import sys @@ -85,17 +86,19 @@ def test_write_stac(tmp_path, dataset, write_datasets, pre_existing_catalog): } -orig_import = __import__ -def import_mock(name, *args): - if name == 'pystac': - raise ModuleNotFoundError("No module named 'pystac'") - return orig_import(name, *args) + def test_write_stac_no_pystac(tmp_path, dataset): # Import hooks are the recommended "clean" way to do this, but don't work # in this case. - with mock.patch('builtins.__import__', side_effect=import_mock): + orig_import = __import__ + def import_mock(name, *args): + if name == "pystac": + raise ModuleNotFoundError("No module named 'pystac'") + return orig_import(name, *args) + + with mock.patch("builtins.__import__", side_effect=import_mock): # pytest imports xcengine.util long before we can patch __import__, # so we delete pystac from util's namespace (if present) instead. # This gives a NameError on access rather than a ModuleNotFoundError @@ -108,6 +111,36 @@ def test_write_stac_no_pystac(tmp_path, dataset): # We want nothing to happen here, so no explicit assertions. +def test_start_server_no_xcube(dataset): + import io + orig_import = __import__ + def import_mock(name, *args): + if name == "xcube" or name.startswith("xcube."): + raise ModuleNotFoundError(f"No module named {name}") + return orig_import(name, *args) + + with mock.patch("builtins.__import__", side_effect=import_mock): + import xcengine.util + util_vars = ( + k for k in xcengine.util.__dict__.keys() + if k == "xcube" or k.startswith("xcube.") + ) + for v in util_vars: del xcengine.util.__dict__[v] + from xcengine.util import start_server + logger = logging.getLogger("test-start-server-logger") + logger.setLevel(logging.INFO) + logger.addHandler(logging.StreamHandler(log_stream := io.StringIO())) + start_server( + {"ds1": dataset}, + {}, + argparse.Namespace(batch=False, from_saved=False), + logger + ) + logged = log_stream.getvalue() + assert "Not starting" in logged + assert "Starting server" not in logged + + @pytest.mark.parametrize("eoap_mode", [False, True]) @pytest.mark.parametrize("ds2_format", [None, "zarr", "netcdf"]) def test_save_datasets(tmp_path, dataset, eoap_mode, ds2_format): diff --git a/xcengine/util.py b/xcengine/util.py index 8c528c6..2a470cb 100644 --- a/xcengine/util.py +++ b/xcengine/util.py @@ -1,7 +1,8 @@ # Copyright (c) 2024-2026 by Brockmann Consult GmbH # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. - +import argparse +import logging from datetime import datetime import json import pathlib @@ -128,11 +129,21 @@ def save_datasets( return saved_datasets -def start_server(datasets, saved_datasets, args, logger): - import xcube.util.plugin - import xcube.webapi.viewer - from xcube.server.server import Server - from xcube.server.framework import get_framework_class +def start_server( + datasets: Mapping[str, xr.Dataset], + saved_datasets: dict[str, pathlib.Path], + args: argparse.Namespace, + logger: logging.Logger +): + try: + import xcube.util.plugin + import xcube.webapi.viewer + from xcube.server.server import Server + from xcube.server.framework import get_framework_class + except ImportError as e: + logger.info(e.msg) + logger.info("Not starting server, since xcube not available") + return xcube.util.plugin.init_plugins() server = Server(framework=get_framework_class("tornado")(), config={}) From 432940ef9252ae8a164fb0d09111e1852d9ce0f3 Mon Sep 17 00:00:00 2001 From: Pontus Lurcock Date: Tue, 7 Jul 2026 18:12:13 +0200 Subject: [PATCH 5/5] Add --no-xcube and --no-eoap CLI options Part of #56 --- test/test_cli.py | 19 +++++++++++++++++-- test/test_util.py | 18 ++++++++++++------ xcengine/cli.py | 36 +++++++++++++++++++++++++++++------- 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/test/test_cli.py b/test/test_cli.py index ad2170a..abf284e 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -66,9 +66,19 @@ def test_make_script( @pytest.mark.parametrize("specify_dir", [False, True]) @pytest.mark.parametrize("specify_env", [False, True]) @pytest.mark.parametrize("specify_eoap", [False, True]) +@pytest.mark.parametrize("skip_build", [False, True]) +@pytest.mark.parametrize("no_eoap", [False, True]) +@pytest.mark.parametrize("no_xcube", [False, True]) @patch("xcengine.cli.ImageBuilder") def test_image_build( - builder_mock, tmp_path, specify_dir, specify_env, specify_eoap + builder_mock, + tmp_path, + specify_dir, + specify_env, + specify_eoap, + skip_build, + no_eoap, + no_xcube, ): (nb_path := tmp_path / "foo.ipynb").touch() (env_path := tmp_path / "environment.yml").touch() @@ -85,6 +95,9 @@ def test_image_build( + (["--build-dir", str(build_dir)] if specify_dir else []) + (["--environment", str(env_path)] if specify_env else []) + (["--eoap", str(eoap_path)] if specify_eoap else []) + + (["--skip-build"] if skip_build else []) + + (["--no-eoap"] if no_eoap else []) + + (["--no-xcube"] if no_xcube else []) + [str(nb_path)], ) assert result.output.startswith("Built image") @@ -95,7 +108,9 @@ def test_image_build( tag=tag, build_dir=(build_dir if specify_dir else ANY), ) - instance_mock.build.assert_called_once_with(skip_build=False) + instance_mock.build.assert_called_once_with( + skip_build=skip_build, with_eoap=not no_eoap, with_xcube=not no_xcube + ) if specify_eoap: assert yaml.safe_load(eoap_path.read_text()) == cwl diff --git a/test/test_util.py b/test/test_util.py index dd62852..349db62 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -86,13 +86,11 @@ def test_write_stac(tmp_path, dataset, write_datasets, pre_existing_catalog): } - - - def test_write_stac_no_pystac(tmp_path, dataset): # Import hooks are the recommended "clean" way to do this, but don't work # in this case. orig_import = __import__ + def import_mock(name, *args): if name == "pystac": raise ModuleNotFoundError("No module named 'pystac'") @@ -105,15 +103,19 @@ def import_mock(name, *args): # on import, but the important thing is to break any implementation # that tries to import pystac without checking if it's available. import xcengine.util + xcengine.util.__dict__.pop("pystac", None) from xcengine.util import write_stac + write_stac({"ds1": dataset}, tmp_path) # We want nothing to happen here, so no explicit assertions. def test_start_server_no_xcube(dataset): import io + orig_import = __import__ + def import_mock(name, *args): if name == "xcube" or name.startswith("xcube."): raise ModuleNotFoundError(f"No module named {name}") @@ -121,12 +123,16 @@ def import_mock(name, *args): with mock.patch("builtins.__import__", side_effect=import_mock): import xcengine.util + util_vars = ( - k for k in xcengine.util.__dict__.keys() + k + for k in xcengine.util.__dict__.keys() if k == "xcube" or k.startswith("xcube.") ) - for v in util_vars: del xcengine.util.__dict__[v] + for v in util_vars: + del xcengine.util.__dict__[v] from xcengine.util import start_server + logger = logging.getLogger("test-start-server-logger") logger.setLevel(logging.INFO) logger.addHandler(logging.StreamHandler(log_stream := io.StringIO())) @@ -134,7 +140,7 @@ def import_mock(name, *args): {"ds1": dataset}, {}, argparse.Namespace(batch=False, from_saved=False), - logger + logger, ) logged = log_stream.getvalue() assert "Not starting" in logged diff --git a/xcengine/cli.py b/xcengine/cli.py index 65832df..6a8b5b6 100644 --- a/xcengine/cli.py +++ b/xcengine/cli.py @@ -151,7 +151,19 @@ def image_cli(): "--skip-build", is_flag=True, help="Prepare the Dockerfile and build context, but don't actually build " - "the image" + "the image", +) +@click.option( + "-n", + "--no-eoap", + is_flag=True, + help="Do not add EOAP functionality to built image", +) +@click.option( + "-n", + "--no-xcube", + is_flag=True, + help="Do not add xcube server/viewer functionality to built image", ) @notebook_argument def build( @@ -160,27 +172,37 @@ def build( environment: pathlib.Path, tag: str, eoap: pathlib.Path, - skip_build: bool + skip_build: bool, + no_eoap: bool, + no_xcube: bool, ) -> None: if environment is None: LOGGER.info("No environment file specified on command line.") class InitArgs(TypedDict): notebook: pathlib.Path - environment: pathlib.Path + environment: pathlib.Path | None tag: str + class BuildArgs(TypedDict): + skip_build: bool + with_eoap: bool + with_xcube: bool + init_args = InitArgs(notebook=notebook, environment=environment, tag=tag) + build_args = BuildArgs( + skip_build=skip_build, with_eoap=not no_eoap, with_xcube=not no_xcube + ) if build_dir: image_builder = ImageBuilder(build_dir=build_dir, **init_args) os.makedirs(build_dir, exist_ok=True) - image = image_builder.build(skip_build=skip_build) + image = image_builder.build(**build_args) else: with tempfile.TemporaryDirectory() as temp_dir: image_builder = ImageBuilder( build_dir=pathlib.Path(temp_dir), **init_args ) - image = image_builder.build(skip_build=skip_build) + image = image_builder.build(**build_args) if eoap: class IndentDumper(yaml.Dumper): @@ -196,8 +218,8 @@ def increase_indent(self, flow=False, indentless=False): ) print( f"Built image with tags {image.tags}" - if image is not None else - "No image built" + if image is not None + else "No image built" )