diff --git a/test/test_cli.py b/test/test_cli.py index ad2170a..abf284e 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -66,9 +66,19 @@ def test_make_script( @pytest.mark.parametrize("specify_dir", [False, True]) @pytest.mark.parametrize("specify_env", [False, True]) @pytest.mark.parametrize("specify_eoap", [False, True]) +@pytest.mark.parametrize("skip_build", [False, True]) +@pytest.mark.parametrize("no_eoap", [False, True]) +@pytest.mark.parametrize("no_xcube", [False, True]) @patch("xcengine.cli.ImageBuilder") def test_image_build( - builder_mock, tmp_path, specify_dir, specify_env, specify_eoap + builder_mock, + tmp_path, + specify_dir, + specify_env, + specify_eoap, + skip_build, + no_eoap, + no_xcube, ): (nb_path := tmp_path / "foo.ipynb").touch() (env_path := tmp_path / "environment.yml").touch() @@ -85,6 +95,9 @@ def test_image_build( + (["--build-dir", str(build_dir)] if specify_dir else []) + (["--environment", str(env_path)] if specify_env else []) + (["--eoap", str(eoap_path)] if specify_eoap else []) + + (["--skip-build"] if skip_build else []) + + (["--no-eoap"] if no_eoap else []) + + (["--no-xcube"] if no_xcube else []) + [str(nb_path)], ) assert result.output.startswith("Built image") @@ -95,7 +108,9 @@ def test_image_build( tag=tag, build_dir=(build_dir if specify_dir else ANY), ) - instance_mock.build.assert_called_once_with(skip_build=False) + instance_mock.build.assert_called_once_with( + skip_build=skip_build, with_eoap=not no_eoap, with_xcube=not no_xcube + ) if specify_eoap: assert yaml.safe_load(eoap_path.read_text()) == cwl diff --git a/test/test_core.py b/test/test_core.py index 29347af..09153c1 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -354,7 +354,9 @@ def test_script_creator_notebook_config(): def test_script_creator_notebook_config_http(httpserver): http_path = "/mynotebook.ipynb" nb_path = pathlib.Path(__file__).parent / "data" / "paramtest.ipynb" - httpserver.expect_request(http_path).respond_with_data(nb_path.read_bytes()) + httpserver.expect_request(http_path).respond_with_data( + nb_path.read_bytes() + ) script_creator = ScriptCreator(httpserver.url_for(http_path)) config = script_creator.nb_params.config assert config["environment_file"] == "my-environment.yml" @@ -382,12 +384,20 @@ def test_image_builder_write_dockerfile(tmp_path): @patch("docker.from_env") @pytest.mark.parametrize("env_type", ["none", "local", "http"]) @pytest.mark.parametrize("skip_build", [False, True]) +@pytest.mark.parametrize("with_eoap", [False, True]) +@pytest.mark.parametrize("with_xcube", [False, True]) +@pytest.mark.parametrize("pystac_in_deps", [False, True]) +@pytest.mark.parametrize("xcube_in_deps", [False, True]) def test_image_builder_build_dir( - from_env_mock, - tmp_path, - httpserver, - env_type, - skip_build + from_env_mock, + tmp_path, + httpserver, + env_type, + skip_build, + with_eoap, + with_xcube, + pystac_in_deps, + xcube_in_deps, ): client_mock = Mock(docker.client.DockerClient) client_mock.images.build.return_value = None, None @@ -398,16 +408,22 @@ def test_image_builder_build_dir( env_def = { "name": "foo", "channels": "bar", - "dependencies": ["python >=3.13", "baz >=42.0"], + "dependencies": ["python >=3.13", "baz >=42.0"] + + (["pystac"] if pystac_in_deps else []) + + (["xcube"] if xcube_in_deps else []), } build_env_path.write_text(yaml.safe_dump(env_def)) env_http = "/env2.yaml" match env_type: - case "none": env_param = None - case "local": env_param = build_env_path + case "none": + env_param = None + case "local": + env_param = build_env_path case "http": - httpserver.expect_request(env_http).respond_with_data(build_env_path.read_bytes()) + httpserver.expect_request(env_http).respond_with_data( + build_env_path.read_bytes() + ) env_param = httpserver.url_for(env_http) case _: raise RuntimeError(f"Unknown env type {env_type}") @@ -418,7 +434,9 @@ def test_image_builder_build_dir( build_dir, None, ) - image_builder.build(skip_build=skip_build) + image_builder.build( + skip_build=skip_build, with_eoap=with_eoap, with_xcube=with_xcube + ) if skip_build: from_env_mock.assert_not_called() else: @@ -430,7 +448,12 @@ def test_image_builder_build_dir( if env_type != "none": assert output_env["name"] == env_def["name"] assert output_env["channels"] == env_def["channels"] - assert set(output_env["dependencies"]) >= set(env_def["dependencies"]) + + output_deps = set(output_env["dependencies"]) + input_deps = set(env_def["dependencies"]) + assert output_deps >= input_deps + assert ("pystac" in output_deps) == pystac_in_deps or with_eoap + assert ("xcube" in output_deps) == xcube_in_deps or with_xcube cwl = image_builder.create_cwl() assert "cwlVersion" in cwl diff --git a/test/test_util.py b/test/test_util.py index b50fedc..349db62 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,3 +1,4 @@ +import argparse import logging import os import sys @@ -85,6 +86,67 @@ def test_write_stac(tmp_path, dataset, write_datasets, pre_existing_catalog): } +def test_write_stac_no_pystac(tmp_path, dataset): + # Import hooks are the recommended "clean" way to do this, but don't work + # in this case. + orig_import = __import__ + + def import_mock(name, *args): + if name == "pystac": + raise ModuleNotFoundError("No module named 'pystac'") + return orig_import(name, *args) + + with mock.patch("builtins.__import__", side_effect=import_mock): + # pytest imports xcengine.util long before we can patch __import__, + # so we delete pystac from util's namespace (if present) instead. + # This gives a NameError on access rather than a ModuleNotFoundError + # on import, but the important thing is to break any implementation + # that tries to import pystac without checking if it's available. + import xcengine.util + + xcengine.util.__dict__.pop("pystac", None) + from xcengine.util import write_stac + + write_stac({"ds1": dataset}, tmp_path) + # We want nothing to happen here, so no explicit assertions. + + +def test_start_server_no_xcube(dataset): + import io + + orig_import = __import__ + + def import_mock(name, *args): + if name == "xcube" or name.startswith("xcube."): + raise ModuleNotFoundError(f"No module named {name}") + return orig_import(name, *args) + + with mock.patch("builtins.__import__", side_effect=import_mock): + import xcengine.util + + util_vars = ( + k + for k in xcengine.util.__dict__.keys() + if k == "xcube" or k.startswith("xcube.") + ) + for v in util_vars: + del xcengine.util.__dict__[v] + from xcengine.util import start_server + + logger = logging.getLogger("test-start-server-logger") + logger.setLevel(logging.INFO) + logger.addHandler(logging.StreamHandler(log_stream := io.StringIO())) + start_server( + {"ds1": dataset}, + {}, + argparse.Namespace(batch=False, from_saved=False), + logger, + ) + logged = log_stream.getvalue() + assert "Not starting" in logged + assert "Starting server" not in logged + + @pytest.mark.parametrize("eoap_mode", [False, True]) @pytest.mark.parametrize("ds2_format", [None, "zarr", "netcdf"]) def test_save_datasets(tmp_path, dataset, eoap_mode, ds2_format): diff --git a/xcengine/cli.py b/xcengine/cli.py index 65832df..6a8b5b6 100644 --- a/xcengine/cli.py +++ b/xcengine/cli.py @@ -151,7 +151,19 @@ def image_cli(): "--skip-build", is_flag=True, help="Prepare the Dockerfile and build context, but don't actually build " - "the image" + "the image", +) +@click.option( + "-n", + "--no-eoap", + is_flag=True, + help="Do not add EOAP functionality to built image", +) +@click.option( + "-n", + "--no-xcube", + is_flag=True, + help="Do not add xcube server/viewer functionality to built image", ) @notebook_argument def build( @@ -160,27 +172,37 @@ def build( environment: pathlib.Path, tag: str, eoap: pathlib.Path, - skip_build: bool + skip_build: bool, + no_eoap: bool, + no_xcube: bool, ) -> None: if environment is None: LOGGER.info("No environment file specified on command line.") class InitArgs(TypedDict): notebook: pathlib.Path - environment: pathlib.Path + environment: pathlib.Path | None tag: str + class BuildArgs(TypedDict): + skip_build: bool + with_eoap: bool + with_xcube: bool + init_args = InitArgs(notebook=notebook, environment=environment, tag=tag) + build_args = BuildArgs( + skip_build=skip_build, with_eoap=not no_eoap, with_xcube=not no_xcube + ) if build_dir: image_builder = ImageBuilder(build_dir=build_dir, **init_args) os.makedirs(build_dir, exist_ok=True) - image = image_builder.build(skip_build=skip_build) + image = image_builder.build(**build_args) else: with tempfile.TemporaryDirectory() as temp_dir: image_builder = ImageBuilder( build_dir=pathlib.Path(temp_dir), **init_args ) - image = image_builder.build(skip_build=skip_build) + image = image_builder.build(**build_args) if eoap: class IndentDumper(yaml.Dumper): @@ -196,8 +218,8 @@ def increase_indent(self, flow=False, indentless=False): ) print( f"Built image with tags {image.tags}" - if image is not None else - "No image built" + if image is not None + else "No image built" ) diff --git a/xcengine/core.py b/xcengine/core.py index 5cbd315..f084b70 100644 --- a/xcengine/core.py +++ b/xcengine/core.py @@ -241,6 +241,8 @@ def __init__( def build( self, skip_build: bool = False, + with_eoap: bool = True, + with_xcube: bool = True, ) -> Image | None: self.script_creator.convert_notebook_to_script(self.build_dir) if self.environment: @@ -254,7 +256,11 @@ def build( ) env_def = self.export_conda_env() # We need xcube for server/viewer and pystac for EOAP stage-in/out - self.add_packages_to_environment(env_def, ["xcube", "pystac"]) + self.add_packages_to_environment( + env_def, + (["xcube"] if with_xcube else []) + + (["pystac"] if with_eoap else []), + ) with open(self.build_dir / "environment.yml", "w") as fh: fh.write(yaml.safe_dump(env_def)) self.write_dockerfile(self.build_dir / "Dockerfile") diff --git a/xcengine/util.py b/xcengine/util.py index 71dd2b4..2a470cb 100644 --- a/xcengine/util.py +++ b/xcengine/util.py @@ -1,18 +1,17 @@ # Copyright (c) 2024-2026 by Brockmann Consult GmbH # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. - +import argparse +import logging from datetime import datetime import json import pathlib import shutil from typing import NamedTuple, Mapping -import pystac import xarray as xr from xarray import Dataset - def clear_directory(directory: pathlib.Path) -> None: for path in directory.iterdir(): if path.is_dir(): @@ -24,6 +23,12 @@ def clear_directory(directory: pathlib.Path) -> None: def write_stac( datasets: Mapping[str, xr.Dataset], stac_root: pathlib.Path ) -> None: + try: + import pystac + except ModuleNotFoundError: + # If pystac isn't present, we assume that stage-out is not required + # and exit quietly. + return catalog_path = stac_root / "catalog.json" if catalog_path.exists(): # Assume that the user code generated its own stage-out data @@ -124,11 +129,21 @@ def save_datasets( return saved_datasets -def start_server(datasets, saved_datasets, args, logger): - import xcube.util.plugin - import xcube.webapi.viewer - from xcube.server.server import Server - from xcube.server.framework import get_framework_class +def start_server( + datasets: Mapping[str, xr.Dataset], + saved_datasets: dict[str, pathlib.Path], + args: argparse.Namespace, + logger: logging.Logger +): + try: + import xcube.util.plugin + import xcube.webapi.viewer + from xcube.server.server import Server + from xcube.server.framework import get_framework_class + except ImportError as e: + logger.info(e.msg) + logger.info("Not starting server, since xcube not available") + return xcube.util.plugin.init_plugins() server = Server(framework=get_framework_class("tornado")(), config={}) diff --git a/xcengine/wrapper.py b/xcengine/wrapper.py index 17ea2ea..6bca37c 100644 --- a/xcengine/wrapper.py +++ b/xcengine/wrapper.py @@ -38,12 +38,8 @@ def __xce_set_params(config_locator: str | pathlib.Path = sys.argv[0]): exec(user_code) import argparse -import pathlib import xarray as xr -import xcube.util.plugin -import xcube.core.new -import xcube.webapi.viewer def main(): @@ -62,7 +58,6 @@ def main(): if args.verbose > 0: LOGGER.setLevel(logging.DEBUG) - xcube.util.plugin.init_plugins() datasets = { name: thing for name, thing in globals().copy().items()