Allow creating EOAP containers without xcube, and vice versa

test/test_cli.py

@@ -66,9 +66,19 @@ def test_make_script(
 @pytest.mark.parametrize("specify_dir", [False, True])
 @pytest.mark.parametrize("specify_env", [False, True])
 @pytest.mark.parametrize("specify_eoap", [False, True])
+@pytest.mark.parametrize("skip_build", [False, True])
+@pytest.mark.parametrize("no_eoap", [False, True])
+@pytest.mark.parametrize("no_xcube", [False, True])
 @patch("xcengine.cli.ImageBuilder")
 def test_image_build(
-    builder_mock, tmp_path, specify_dir, specify_env, specify_eoap
+    builder_mock,
+    tmp_path,
+    specify_dir,
+    specify_env,
+    specify_eoap,
+    skip_build,
+    no_eoap,
+    no_xcube,
 ):
     (nb_path := tmp_path / "foo.ipynb").touch()
     (env_path := tmp_path / "environment.yml").touch()
@@ -85,6 +95,9 @@ def test_image_build(
         + (["--build-dir", str(build_dir)] if specify_dir else [])
         + (["--environment", str(env_path)] if specify_env else [])
         + (["--eoap", str(eoap_path)] if specify_eoap else [])
+        + (["--skip-build"] if skip_build else [])
+        + (["--no-eoap"] if no_eoap else [])
+        + (["--no-xcube"] if no_xcube else [])
         + [str(nb_path)],
     )
     assert result.output.startswith("Built image")
@@ -95,7 +108,9 @@ def test_image_build(
         tag=tag,
         build_dir=(build_dir if specify_dir else ANY),
     )
-    instance_mock.build.assert_called_once_with(skip_build=False)
+    instance_mock.build.assert_called_once_with(
+        skip_build=skip_build, with_eoap=not no_eoap, with_xcube=not no_xcube
+    )
     if specify_eoap:
         assert yaml.safe_load(eoap_path.read_text()) == cwl
 

test/test_core.py

@@ -354,7 +354,9 @@ def test_script_creator_notebook_config():
 def test_script_creator_notebook_config_http(httpserver):
     http_path = "/mynotebook.ipynb"
     nb_path = pathlib.Path(__file__).parent / "data" / "paramtest.ipynb"
-    httpserver.expect_request(http_path).respond_with_data(nb_path.read_bytes())
+    httpserver.expect_request(http_path).respond_with_data(
+        nb_path.read_bytes()
+    )
     script_creator = ScriptCreator(httpserver.url_for(http_path))
     config = script_creator.nb_params.config
     assert config["environment_file"] == "my-environment.yml"
@@ -382,12 +384,20 @@ def test_image_builder_write_dockerfile(tmp_path):
 @patch("docker.from_env")
 @pytest.mark.parametrize("env_type", ["none", "local", "http"])
 @pytest.mark.parametrize("skip_build", [False, True])
+@pytest.mark.parametrize("with_eoap", [False, True])
+@pytest.mark.parametrize("with_xcube", [False, True])
+@pytest.mark.parametrize("pystac_in_deps", [False, True])
+@pytest.mark.parametrize("xcube_in_deps", [False, True])
 def test_image_builder_build_dir(
-        from_env_mock,
-        tmp_path,
-        httpserver,
-        env_type,
-        skip_build
+    from_env_mock,
+    tmp_path,
+    httpserver,
+    env_type,
+    skip_build,
+    with_eoap,
+    with_xcube,
+    pystac_in_deps,
+    xcube_in_deps,
 ):
     client_mock = Mock(docker.client.DockerClient)
     client_mock.images.build.return_value = None, None
@@ -398,16 +408,22 @@ def test_image_builder_build_dir(
     env_def = {
         "name": "foo",
         "channels": "bar",
-        "dependencies": ["python >=3.13", "baz >=42.0"],
+        "dependencies": ["python >=3.13", "baz >=42.0"]
+        + (["pystac"] if pystac_in_deps else [])
+        + (["xcube"] if xcube_in_deps else []),
     }
     build_env_path.write_text(yaml.safe_dump(env_def))
     env_http = "/env2.yaml"
 
     match env_type:
-        case "none": env_param = None
-        case "local": env_param = build_env_path
+        case "none":
+            env_param = None
+        case "local":
+            env_param = build_env_path
         case "http":
-            httpserver.expect_request(env_http).respond_with_data(build_env_path.read_bytes())
+            httpserver.expect_request(env_http).respond_with_data(
+                build_env_path.read_bytes()
+            )
             env_param = httpserver.url_for(env_http)
         case _:
             raise RuntimeError(f"Unknown env type {env_type}")
@@ -418,7 +434,9 @@ def test_image_builder_build_dir(
         build_dir,
         None,
     )
-    image_builder.build(skip_build=skip_build)
+    image_builder.build(
+        skip_build=skip_build, with_eoap=with_eoap, with_xcube=with_xcube
+    )
     if skip_build:
         from_env_mock.assert_not_called()
     else:
@@ -430,7 +448,12 @@ def test_image_builder_build_dir(
     if env_type != "none":
         assert output_env["name"] == env_def["name"]
         assert output_env["channels"] == env_def["channels"]
-        assert set(output_env["dependencies"]) >= set(env_def["dependencies"])
+
+        output_deps = set(output_env["dependencies"])
+        input_deps = set(env_def["dependencies"])
+        assert output_deps >= input_deps
+        assert ("pystac" in output_deps) == pystac_in_deps or with_eoap
+        assert ("xcube" in output_deps) == xcube_in_deps or with_xcube
 
     cwl = image_builder.create_cwl()
     assert "cwlVersion" in cwl

test/test_util.py

@@ -1,3 +1,4 @@
+import argparse
 import logging
 import os
 import sys
@@ -85,6 +86,67 @@ def test_write_stac(tmp_path, dataset, write_datasets, pre_existing_catalog):
         }
 
 
+def test_write_stac_no_pystac(tmp_path, dataset):
+    # Import hooks are the recommended "clean" way to do this, but don't work
+    # in this case.
+    orig_import = __import__
+
+    def import_mock(name, *args):
+        if name == "pystac":
+            raise ModuleNotFoundError("No module named 'pystac'")
+        return orig_import(name, *args)
+
+    with mock.patch("builtins.__import__", side_effect=import_mock):
+        # pytest imports xcengine.util long before we can patch __import__,
+        # so we delete pystac from util's namespace (if present) instead.
+        # This gives a NameError on access rather than a ModuleNotFoundError
+        # on import, but the important thing is to break any implementation
+        # that tries to import pystac without checking if it's available.
+        import xcengine.util
+
+        xcengine.util.__dict__.pop("pystac", None)
+        from xcengine.util import write_stac
+
+        write_stac({"ds1": dataset}, tmp_path)
+        # We want nothing to happen here, so no explicit assertions.
+
+
+def test_start_server_no_xcube(dataset):
+    import io
+
+    orig_import = __import__
+
+    def import_mock(name, *args):
+        if name == "xcube" or name.startswith("xcube."):
+            raise ModuleNotFoundError(f"No module named {name}")
+        return orig_import(name, *args)
+
+    with mock.patch("builtins.__import__", side_effect=import_mock):
+        import xcengine.util
+
+        util_vars = (
+            k
+            for k in xcengine.util.__dict__.keys()
+            if k == "xcube" or k.startswith("xcube.")
+        )
+        for v in util_vars:
+            del xcengine.util.__dict__[v]
+        from xcengine.util import start_server
+
+        logger = logging.getLogger("test-start-server-logger")
+        logger.setLevel(logging.INFO)
+        logger.addHandler(logging.StreamHandler(log_stream := io.StringIO()))
+        start_server(
+            {"ds1": dataset},
+            {},
+            argparse.Namespace(batch=False, from_saved=False),
+            logger,
+        )
+        logged = log_stream.getvalue()
+        assert "Not starting" in logged
+        assert "Starting server" not in logged
+
+
 @pytest.mark.parametrize("eoap_mode", [False, True])
 @pytest.mark.parametrize("ds2_format", [None, "zarr", "netcdf"])
 def test_save_datasets(tmp_path, dataset, eoap_mode, ds2_format):

xcengine/cli.py

@@ -151,7 +151,19 @@ def image_cli():
     "--skip-build",
     is_flag=True,
     help="Prepare the Dockerfile and build context, but don't actually build "
-    "the image"
+    "the image",
+)
+@click.option(
+    "-n",
+    "--no-eoap",
+    is_flag=True,
+    help="Do not add EOAP functionality to built image",
+)
+@click.option(
+    "-n",
+    "--no-xcube",
+    is_flag=True,
+    help="Do not add xcube server/viewer functionality to built image",
 )
 @notebook_argument
 def build(
@@ -160,27 +172,37 @@ def build(
     environment: pathlib.Path,
     tag: str,
     eoap: pathlib.Path,
-    skip_build: bool
+    skip_build: bool,
+    no_eoap: bool,
+    no_xcube: bool,
 ) -> None:
     if environment is None:
         LOGGER.info("No environment file specified on command line.")
 
     class InitArgs(TypedDict):
         notebook: pathlib.Path
-        environment: pathlib.Path
+        environment: pathlib.Path | None
         tag: str
 
+    class BuildArgs(TypedDict):
+        skip_build: bool
+        with_eoap: bool
+        with_xcube: bool
+
     init_args = InitArgs(notebook=notebook, environment=environment, tag=tag)
+    build_args = BuildArgs(
+        skip_build=skip_build, with_eoap=not no_eoap, with_xcube=not no_xcube
+    )
     if build_dir:
         image_builder = ImageBuilder(build_dir=build_dir, **init_args)
         os.makedirs(build_dir, exist_ok=True)
-        image = image_builder.build(skip_build=skip_build)
+        image = image_builder.build(**build_args)
     else:
         with tempfile.TemporaryDirectory() as temp_dir:
             image_builder = ImageBuilder(
                 build_dir=pathlib.Path(temp_dir), **init_args
             )
-            image = image_builder.build(skip_build=skip_build)
+            image = image_builder.build(**build_args)
     if eoap:
 
         class IndentDumper(yaml.Dumper):
@@ -196,8 +218,8 @@ def increase_indent(self, flow=False, indentless=False):
         )
     print(
         f"Built image with tags {image.tags}"
-        if image is not None else
-        "No image built"
+        if image is not None
+        else "No image built"
     )
 
 

xcengine/core.py

@@ -241,6 +241,8 @@ def __init__(
     def build(
         self,
         skip_build: bool = False,
+        with_eoap: bool = True,
+        with_xcube: bool = True,
     ) -> Image | None:
         self.script_creator.convert_notebook_to_script(self.build_dir)
         if self.environment:
@@ -254,7 +256,11 @@ def build(
             )
             env_def = self.export_conda_env()
         # We need xcube for server/viewer and pystac for EOAP stage-in/out
-        self.add_packages_to_environment(env_def, ["xcube", "pystac"])
+        self.add_packages_to_environment(
+            env_def,
+            (["xcube"] if with_xcube else [])
+            + (["pystac"] if with_eoap else []),
+        )
         with open(self.build_dir / "environment.yml", "w") as fh:
             fh.write(yaml.safe_dump(env_def))
         self.write_dockerfile(self.build_dir / "Dockerfile")

xcengine/util.py

@@ -1,18 +1,17 @@
 # Copyright (c) 2024-2026 by Brockmann Consult GmbH
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
-
+import argparse
+import logging
 from datetime import datetime
 import json
 import pathlib
 import shutil
 from typing import NamedTuple, Mapping
 
-import pystac
 import xarray as xr
 from xarray import Dataset
 
-
 def clear_directory(directory: pathlib.Path) -> None:
     for path in directory.iterdir():
         if path.is_dir():
@@ -24,6 +23,12 @@ def clear_directory(directory: pathlib.Path) -> None:
 def write_stac(
     datasets: Mapping[str, xr.Dataset], stac_root: pathlib.Path
 ) -> None:
+    try:
+        import pystac
+    except ModuleNotFoundError:
+        # If pystac isn't present, we assume that stage-out is not required
+        # and exit quietly.
+        return
     catalog_path = stac_root / "catalog.json"
     if catalog_path.exists():
         # Assume that the user code generated its own stage-out data
@@ -124,11 +129,21 @@ def save_datasets(
     return saved_datasets
 
 
-def start_server(datasets, saved_datasets, args, logger):
-    import xcube.util.plugin
-    import xcube.webapi.viewer
-    from xcube.server.server import Server
-    from xcube.server.framework import get_framework_class
+def start_server(
+        datasets: Mapping[str, xr.Dataset],
+        saved_datasets: dict[str, pathlib.Path],
+        args: argparse.Namespace,
+        logger: logging.Logger
+):
+    try:
+        import xcube.util.plugin
+        import xcube.webapi.viewer
+        from xcube.server.server import Server
+        from xcube.server.framework import get_framework_class
+    except ImportError as e:
+        logger.info(e.msg)
+        logger.info("Not starting server, since xcube not available")
+        return
 
     xcube.util.plugin.init_plugins()
     server = Server(framework=get_framework_class("tornado")(), config={})

xcengine/wrapper.py

@@ -38,12 +38,8 @@ def __xce_set_params(config_locator: str | pathlib.Path = sys.argv[0]):
 exec(user_code)
 
 import argparse
-import pathlib
 
 import xarray as xr
-import xcube.util.plugin
-import xcube.core.new
-import xcube.webapi.viewer
 
 
 def main():
@@ -62,7 +58,6 @@ def main():
     if args.verbose > 0:
         LOGGER.setLevel(logging.DEBUG)
 
-    xcube.util.plugin.init_plugins()
     datasets = {
         name: thing
         for name, thing in globals().copy().items()