diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index b8812e292..5838f084a 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -28,7 +28,7 @@ jobs: tox: name: Tox - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: py-ver-major: [3] @@ -47,8 +47,8 @@ jobs: - name: Set up Singularity and environment-modules if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.3.5/singularity-ce_4.3.5-noble_amd64.deb + sudo apt-get install -y ./singularity-ce_4.3.5-noble_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} @@ -70,7 +70,7 @@ jobs: - name: Upgrade setuptools and install tox run: | pip install -U pip setuptools wheel - pip install "tox<4" "tox-gh-actions<3" + pip install tox tox-gh-actions - name: MyPy cache if: ${{ matrix.step == 'mypy' }} @@ -116,7 +116,7 @@ jobs: - name: Upgrade setuptools and install tox run: | pip install -U pip setuptools wheel - pip install "tox<4" "tox-gh-actions<3" + pip install tox tox-gh-actions - if: ${{ matrix.step == 'pydocstyle' && github.event_name == 'pull_request'}} name: Create local branch for diff-quality for PRs @@ -127,7 +127,7 @@ jobs: clean_working_dir: name: No leftovers - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: py-semver: "3.13" steps: @@ -137,8 +137,8 @@ jobs: - name: Set up Singularity and environment-modules run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.3.5/singularity-ce_4.3.5-noble_amd64.deb + sudo apt-get install -y ./singularity-ce_4.3.5-noble_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. run: sudo usermod -c 'CI Runner' "$(whoami)" @@ -165,7 +165,7 @@ jobs: conformance_tests: name: CWL conformance - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -186,8 +186,8 @@ jobs: - name: Set up Singularity and environment-modules if: ${{ matrix.container == 'singularity' }} run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.3.5/singularity-ce_4.3.5-noble_amd64.deb + sudo apt-get install -y ./singularity-ce_4.3.5-noble_amd64.deb environment-modules - name: Singularity cache if: ${{ matrix.container == 'singularity' }} @@ -227,15 +227,15 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} release_test: name: cwltool release test - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v6 - name: Set up Singularity and environment-modules run: | - wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.2.1/singularity-ce_4.2.1-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_4.2.1-focal_amd64.deb environment-modules + wget --no-verbose https://github.com/sylabs/singularity/releases/download/v4.3.5/singularity-ce_4.3.5-noble_amd64.deb + sudo apt-get install -y ./singularity-ce_4.3.5-noble_amd64.deb environment-modules - name: Set up Python uses: actions/setup-python@v6 @@ -290,7 +290,7 @@ jobs: - name: Upgrade setuptools and install tox run: | pip install -U pip setuptools wheel - pip install "tox<4" "tox-gh-actions<3" + pip install tox tox-gh-actions # # docker for mac install is not currently stable # - name: 'SETUP MacOS: load Homebrew cache' # uses: actions/cache@v4 diff --git a/README.rst b/README.rst index 9dc97d62a..b21c55f1a 100644 --- a/README.rst +++ b/README.rst @@ -732,14 +732,14 @@ To run the basic tests after installing `cwltool` execute the following: To run various tests in all supported Python environments, we use `tox `_. To run the test suite in all supported Python environments first clone the complete code repository (see the ``git clone`` instructions above) and then run the following in the terminal: -``pip install "tox<4"; tox -p`` +``pip install tox; tox -p`` List of all environment can be seen using: ``tox --listenvs`` and running a specific test env using: -``tox -e `` +``tox run -e `` and additionally run a specific test using this format: -``tox -e py310-unit -- -v tests/test_examples.py::test_scandeps`` +``tox run -e py310-unit -- -v tests/test_examples.py::test_scandeps`` - Running the entire suite of CWL conformance tests: diff --git a/cwltool/singularity.py b/cwltool/singularity.py index 76f1fc488..a8a9443ad 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -1,16 +1,21 @@ """Support for executing Docker format containers using Singularity {2,3}.x or Apptainer 1.x.""" +import copy +import hashlib import logging import os import os.path import re import shutil import sys +import threading from collections.abc import Callable, MutableMapping from subprocess import check_call, check_output # nosec from typing import cast +from packaging.version import Version from schema_salad.sourceline import SourceLine +from schema_salad.utils import json_dumps from spython.main import Client from spython.main.parse.parsers.docker import DockerParser from spython.main.parse.writers.singularity import SingularityWriter @@ -29,20 +34,24 @@ # This is a list containing major and minor versions as integer. # (The number of minor version digits can vary among different distributions, # therefore we need a list here.) -_SINGULARITY_VERSION: list[int] | None = None +_SINGULARITY_VERSION: Version | None = None # Cached flavor / distribution of singularity # Can be singularity, singularity-ce or apptainer _SINGULARITY_FLAVOR: str = "" -def get_version() -> tuple[list[int], str]: +_IMAGES: dict[str, str] = {} +_IMAGES_LOCK = threading.Lock() + + +def get_version() -> tuple[Version, str]: """ Parse the output of 'singularity --version' to determine the flavor and version. Both pieces of information will be cached. :returns: A tuple containing: - - A tuple with major and minor version numbers as integer. + - A parsed Version object. - A string with the name of the singularity flavor. """ global _SINGULARITY_VERSION # pylint: disable=global-statement @@ -55,7 +64,7 @@ def get_version() -> tuple[list[int], str]: raise RuntimeError("Output of 'singularity --version' not recognized.") version_string = version_match.group(2) - _SINGULARITY_VERSION = [int(i) for i in version_string.split(".")] + _SINGULARITY_VERSION = Version(version_string) _SINGULARITY_FLAVOR = version_match.group(1) _logger.debug(f"Singularity version: {version_string}" " ({_SINGULARITY_FLAVOR}.") @@ -69,18 +78,18 @@ def is_apptainer_1_or_newer() -> bool: Apptainer v1.0.0 is compatible with SingularityCE 3.9.5. See: https://github.com/apptainer/apptainer/releases """ - v = get_version() - if v[1] != "apptainer": + version, flavor = get_version() + if flavor != "apptainer": return False - return v[0][0] >= 1 + return version >= Version("1") def is_apptainer_1_1_or_newer() -> bool: """Check if apptainer singularity distribution is version 1.1 or higher.""" - v = get_version() - if v[1] != "apptainer": + version, flavor = get_version() + if flavor != "apptainer": return False - return v[0][0] >= 2 or (v[0][0] >= 1 and v[0][1] >= 1) + return version >= Version("1.1") def is_version_2_6() -> bool: @@ -89,48 +98,58 @@ def is_version_2_6() -> bool: Also returns False if the flavor is not singularity or singularity-ce. """ - v = get_version() - if v[1] != "singularity" and v[1] != "singularity-ce": + version, flavor = get_version() + if flavor not in ("singularity", "singularity-ce"): return False - return v[0][0] == 2 and v[0][1] == 6 + return version >= Version("2.6") and version < Version("2.7") def is_version_3_or_newer() -> bool: """Check if this version is singularity version 3 or newer or equivalent.""" if is_apptainer_1_or_newer(): return True # this is equivalent to singularity-ce > 3.9.5 - v = get_version() - return v[0][0] >= 3 + version, flavor = get_version() + if flavor == "apptainer": + return False + return version >= Version("3") def is_version_3_1_or_newer() -> bool: """Check if this version is singularity version 3.1 or newer or equivalent.""" if is_apptainer_1_or_newer(): return True # this is equivalent to singularity-ce > 3.9.5 - v = get_version() - return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 1) + version, flavor = get_version() + if flavor == "apptainer": + return False + return version >= Version("3.1") def is_version_3_4_or_newer() -> bool: """Detect if Singularity v3.4+ is available.""" if is_apptainer_1_or_newer(): return True # this is equivalent to singularity-ce > 3.9.5 - v = get_version() - return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 4) + version, flavor = get_version() + if flavor == "apptainer": + return False + return version >= Version("3.4") def is_version_3_9_or_newer() -> bool: """Detect if Singularity v3.9+ is available.""" if is_apptainer_1_or_newer(): return True # this is equivalent to singularity-ce > 3.9.5 - v = get_version() - return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 9) + version, flavor = get_version() + if flavor == "apptainer": + return False + return version >= Version("3.9") def is_version_3_10_or_newer() -> bool: """Detect if Singularity v3.10+ is available.""" - v = get_version() - return v[0][0] >= 4 or (v[0][0] == 3 and v[0][1] >= 10) + version, flavor = get_version() + if flavor not in ("singularity", "singularity-ce"): + return False + return version >= Version("3.10") def _normalize_image_id(string: str) -> str: @@ -180,73 +199,86 @@ def get_image( cache_folder = None debug = _logger.isEnabledFor(logging.DEBUG) + with _IMAGES_LOCK: + if "dockerImageId" in dockerRequirement: + if (d_image_id := dockerRequirement["dockerImageId"]) in _IMAGES: + if (resolved_image_id := _IMAGES[d_image_id]) != d_image_id: + dockerRequirement["dockerImage_id"] = resolved_image_id + return True + + docker_req = copy.deepcopy(dockerRequirement) # thread safety if "CWL_SINGULARITY_CACHE" in os.environ: cache_folder = os.environ["CWL_SINGULARITY_CACHE"] elif is_version_2_6() and "SINGULARITY_PULLFOLDER" in os.environ: cache_folder = os.environ["SINGULARITY_PULLFOLDER"] - if "dockerFile" in dockerRequirement: + if "dockerFile" in docker_req: if cache_folder is None: # if environment variables were not set cache_folder = create_tmp_dir(tmp_outdir_prefix) absolute_path = os.path.abspath(cache_folder) - if "dockerImageId" in dockerRequirement: - image_name = dockerRequirement["dockerImageId"] - image_path = os.path.join(absolute_path, image_name) - if os.path.exists(image_path): - found = True + if "dockerImageId" in docker_req: + image_name = docker_req["dockerImageId"] + else: + image_name = hashlib.md5( # nosec + json_dumps(dockerRequirement, separators=(",", ":"), sort_keys=True).encode( + "utf-8" + ) + ).hexdigest() + if is_version_3_or_newer(): + image_name = _normalize_sif_id(image_name) + else: + image_name = _normalize_image_id(image_name) + image_name = os.path.join(absolute_path, image_name) + docker_req["dockerImageId"] = image_name + if os.path.exists(image_name): + found = True if found is False: dockerfile_path = os.path.join(absolute_path, "Dockerfile") singularityfile_path = dockerfile_path + ".def" - # if you do not set APPTAINER_TMPDIR will crash - # WARNING: 'nodev' mount option set on /tmp, it could be a - # source of failure during build process - # FATAL: Unable to create build: 'noexec' mount option set on - # /tmp, temporary root filesystem won't be usable at this location with open(dockerfile_path, "w") as dfile: - dfile.write(dockerRequirement["dockerFile"]) + dfile.write(docker_req["dockerFile"]) - singularityfile = SingularityWriter(DockerParser(dockerfile_path).parse()).convert() + docker_recipe = DockerParser(dockerfile_path).parse() + docker_recipe["spython-base"].entrypoint = "" + singularityfile = SingularityWriter(docker_recipe).convert() with open(singularityfile_path, "w") as file: file.write(singularityfile) + # if you do not set APPTAINER_TMPDIR will crash + # WARNING: 'nodev' mount option set on /tmp, it could be a + # source of failure during build process + # FATAL: Unable to create build: 'noexec' mount option set on + # /tmp, temporary root filesystem won't be usable at this location os.environ["APPTAINER_TMPDIR"] = absolute_path singularity_options = ["--fakeroot"] if not shutil.which("proot") else [] - if "dockerImageId" in dockerRequirement: - Client.build( - recipe=singularityfile_path, - build_folder=absolute_path, - image=dockerRequirement["dockerImageId"], - sudo=False, - options=singularity_options, - ) - else: - Client.build( - recipe=singularityfile_path, - build_folder=absolute_path, - sudo=False, - options=singularity_options, - ) + Client.build( + recipe=singularityfile_path, + build_folder=absolute_path, + image=image_name, + sudo=False, + options=singularity_options, + ) found = True - elif "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement: - match = re.search(pattern=r"([a-z]*://)", string=dockerRequirement["dockerPull"]) - img_name = _normalize_image_id(dockerRequirement["dockerPull"]) + elif "dockerImageId" not in docker_req and "dockerPull" in docker_req: + match = re.search(pattern=r"([a-z]*://)", string=docker_req["dockerPull"]) + img_name = _normalize_image_id(docker_req["dockerPull"]) candidates.append(img_name) if is_version_3_or_newer(): - sif_name = _normalize_sif_id(dockerRequirement["dockerPull"]) + sif_name = _normalize_sif_id(docker_req["dockerPull"]) candidates.append(sif_name) - dockerRequirement["dockerImageId"] = sif_name + docker_req["dockerImageId"] = sif_name else: - dockerRequirement["dockerImageId"] = img_name + docker_req["dockerImageId"] = img_name if not match: - dockerRequirement["dockerPull"] = "docker://" + dockerRequirement["dockerPull"] - elif "dockerImageId" in dockerRequirement: - if os.path.isfile(dockerRequirement["dockerImageId"]): + docker_req["dockerPull"] = "docker://" + docker_req["dockerPull"] + elif "dockerImageId" in docker_req: + if os.path.isfile(docker_req["dockerImageId"]): found = True - candidates.append(dockerRequirement["dockerImageId"]) - candidates.append(_normalize_image_id(dockerRequirement["dockerImageId"])) + candidates.append(docker_req["dockerImageId"]) + candidates.append(_normalize_image_id(docker_req["dockerImageId"])) if is_version_3_or_newer(): - candidates.append(_normalize_sif_id(dockerRequirement["dockerImageId"])) + candidates.append(_normalize_sif_id(docker_req["dockerImageId"])) targets = [os.getcwd()] if "CWL_SINGULARITY_CACHE" in os.environ: @@ -263,11 +295,11 @@ def get_image( "Using local copy of Singularity image found in %s", dirpath, ) - dockerRequirement["dockerImageId"] = path + docker_req["dockerImageId"] = path found = True if (force_pull or not found) and pull_image: cmd: list[str] = [] - if "dockerPull" in dockerRequirement: + if "dockerPull" in docker_req: if cache_folder: env = os.environ.copy() if is_version_2_6(): @@ -277,8 +309,8 @@ def get_image( "pull", "--force", "--name", - dockerRequirement["dockerImageId"], - str(dockerRequirement["dockerPull"]), + docker_req["dockerImageId"], + str(docker_req["dockerPull"]), ] else: cmd = [ @@ -286,14 +318,14 @@ def get_image( "pull", "--force", "--name", - "{}/{}".format(cache_folder, dockerRequirement["dockerImageId"]), - str(dockerRequirement["dockerPull"]), + "{}/{}".format(cache_folder, docker_req["dockerImageId"]), + str(docker_req["dockerPull"]), ] _logger.info(str(cmd)) check_call(cmd, env=env, stdout=sys.stderr) # nosec - dockerRequirement["dockerImageId"] = "{}/{}".format( - cache_folder, dockerRequirement["dockerImageId"] + docker_req["dockerImageId"] = "{}/{}".format( + cache_folder, docker_req["dockerImageId"] ) found = True else: @@ -302,44 +334,47 @@ def get_image( "pull", "--force", "--name", - str(dockerRequirement["dockerImageId"]), - str(dockerRequirement["dockerPull"]), + str(docker_req["dockerImageId"]), + str(docker_req["dockerPull"]), ] _logger.info(str(cmd)) check_call(cmd, stdout=sys.stderr) # nosec found = True - elif "dockerLoad" in dockerRequirement: + elif "dockerLoad" in docker_req: if is_version_3_1_or_newer(): - if "dockerImageId" in dockerRequirement: - name = "{}.sif".format(dockerRequirement["dockerImageId"]) + if "dockerImageId" in docker_req: + name = "{}.sif".format(docker_req["dockerImageId"]) else: - name = "{}.sif".format(dockerRequirement["dockerLoad"]) + name = "{}.sif".format(docker_req["dockerLoad"]) cmd = [ "singularity", "build", name, - "docker-archive://{}".format(dockerRequirement["dockerLoad"]), + "docker-archive://{}".format(docker_req["dockerLoad"]), ] _logger.info(str(cmd)) check_call(cmd, stdout=sys.stderr) # nosec found = True - dockerRequirement["dockerImageId"] = name + docker_req["dockerImageId"] = name else: - raise SourceLine( - dockerRequirement, "dockerLoad", WorkflowException, debug - ).makeError( + raise SourceLine(docker_req, "dockerLoad", WorkflowException, debug).makeError( "dockerLoad is not currently supported when using the " "Singularity runtime (version less than 3.1) for Docker containers." ) - elif "dockerImport" in dockerRequirement: - raise SourceLine( - dockerRequirement, "dockerImport", WorkflowException, debug - ).makeError( + elif "dockerImport" in docker_req: + raise SourceLine(docker_req, "dockerImport", WorkflowException, debug).makeError( "dockerImport is not currently supported when using the " "Singularity runtime for Docker containers." ) - + if found: + with _IMAGES_LOCK: + if "dockerImageId" in dockerRequirement: + _IMAGES[dockerRequirement["dockerImageId"]] = docker_req["dockerImageId"] + dockerRequirement.clear() + dockerRequirement |= docker_req + if "dockerImageId" in docker_req: + _IMAGES[docker_req["dockerImageId"]] = docker_req["dockerImageId"] return found def get_from_requirements( @@ -486,7 +521,7 @@ def create_runtime( runtime = [ "singularity", "--quiet", - "run" if is_apptainer_1_1_or_newer() or is_version_3_10_or_newer() else "exec", + "run" if (is_apptainer_1_1_or_newer() or is_version_3_10_or_newer()) else "exec", "--contain", "--ipc", "--cleanenv", diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index a85dc81f8..6e1527c20 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -633,7 +633,7 @@ def postScatterEval(io: CWLObjectType) -> CWLObjectType | None: fs_access = getdefault(runtimeContext.make_fs_access, StdFsAccess)("") for k, v in io.items(): - if k in loadContents: + if k in loadContents and v is not None: val = cast(CWLObjectType, v) if val.get("contents") is None: with fs_access.open(cast(str, val["location"]), "rb") as f: diff --git a/tests/sing_dockerfile_named_test.cwl b/tests/sing_dockerfile_named_test.cwl new file mode 100755 index 000000000..ca1ac7b11 --- /dev/null +++ b/tests/sing_dockerfile_named_test.cwl @@ -0,0 +1,18 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + DockerRequirement: + dockerFile: | + FROM docker.io/debian:stable-slim + dockerImageId: customDebian + +inputs: + message: string + +outputs: [] + +baseCommand: echo +arguments: + - $(inputs.message) diff --git a/tests/sing_dockerfile_test.cwl b/tests/sing_dockerfile_test.cwl new file mode 100755 index 000000000..593e017a3 --- /dev/null +++ b/tests/sing_dockerfile_test.cwl @@ -0,0 +1,17 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.0 +class: CommandLineTool + +requirements: + DockerRequirement: + dockerFile: | + FROM docker.io/debian:stable-slim + +inputs: + message: string + +outputs: [] + +baseCommand: echo +arguments: + - $(inputs.message) diff --git a/tests/test_environment.py b/tests/test_environment.py index b9fa24579..901d10261 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -56,7 +56,9 @@ def assert_env_matches( if not env_accepts_null(): e.pop("LC_CTYPE", None) e.pop("__CF_USER_TEXT_ENCODING", None) - assert len(e) == 0, f"Unexpected environment variable(s): {', '.join(e.keys())}" + assert ( + len(e) == 0 + ), f"Unexpected environment variable(s): {', '.join(e.keys())} from env {env}." class CheckHolder(ABC): @@ -129,12 +131,11 @@ def PWD(v: str, env: Env) -> bool: "LD_LIBRARY_PATH": None, "PATH": None, "PS1": None, - "PWD": PWD, "TMPDIR": "/tmp", } # Singularity variables appear to be in flux somewhat. - version = Version(".".join(map(str, get_version()[0]))) + version, _ = get_version() assert version >= Version("3"), "Tests only work for Singularity 3+" sing_vars: EnvChecks = { "SINGULARITY_CONTAINER": None, @@ -142,6 +143,8 @@ def PWD(v: str, env: Env) -> bool: } if version < Version("3.5"): sing_vars["SINGULARITY_APPNAME"] = None + if version < Version("4.3"): + sing_vars["PWD"] = PWD if (version >= Version("3.5")) and (version < Version("3.6")): sing_vars["SINGULARITY_INIT"] = "1" if version >= Version("3.5"): @@ -295,6 +298,7 @@ def test_preserve_all( for vname, val in env.items(): try: + assert vname in checks, env assert_envvar_matches(checks[vname], vname, env) except KeyError: assert val == os.environ[vname] diff --git a/tests/test_singularity.py b/tests/test_singularity.py index 1139dfbc7..b60575d90 100644 --- a/tests/test_singularity.py +++ b/tests/test_singularity.py @@ -6,6 +6,7 @@ import pytest from cwltool.main import main +from cwltool.singularity import _IMAGES, _IMAGES_LOCK from .util import ( get_data, @@ -17,6 +18,12 @@ ) +@pytest.fixture(autouse=True) +def clear_singularity_image_cache() -> None: + with _IMAGES_LOCK: + _IMAGES.clear() + + @needs_singularity_2_6 def test_singularity_pullfolder(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test singularity respects SINGULARITY_PULLFOLDER.""" @@ -149,7 +156,7 @@ def test_singularity3_docker_image_id_in_tool(tmp_path: Path) -> None: "hello", ] ) - assert result_code == 0 + assert result_code == 0, stderr result_code1, stdout, stderr = get_main_output( [ "--singularity", @@ -159,3 +166,89 @@ def test_singularity3_docker_image_id_in_tool(tmp_path: Path) -> None: ] ) assert result_code1 == 0 + + +@needs_singularity_3_or_newer +def test_singularity_dockerfile_no_name_no_cache(tmp_path: Path) -> None: + workdir = tmp_path / "working_dir" + workdir.mkdir() + with working_directory(workdir): + result_code, stdout, stderr = get_main_output( + [ + "--singularity", + get_data("tests/sing_dockerfile_test.cwl"), + "--message", + "hello", + ] + ) + assert result_code == 0, stderr + assert not (workdir / "bea92b9b6910cbbd2ae602f5bb0f0f27_latest.sif").exists() + + +@needs_singularity_3_or_newer +def test_singularity_dockerfile_no_name_with_cache( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + workdir = tmp_path / "working_dir" + workdir.mkdir() + cachedir = tmp_path / "cache" + cachedir.mkdir() + monkeypatch.setenv("CWL_SINGULARITY_CACHE", str(cachedir)) + with working_directory(workdir): + result_code, stdout, stderr = get_main_output( + [ + "--singularity", + get_data("tests/sing_dockerfile_test.cwl"), + "--message", + "hello", + ] + ) + assert result_code == 0, stderr + assert not (workdir / "bea92b9b6910cbbd2ae602f5bb0f0f27_latest.sif").exists() + assert (cachedir / "bea92b9b6910cbbd2ae602f5bb0f0f27_latest.sif").exists() + + +@needs_singularity_3_or_newer +def test_singularity_dockerfile_with_name_no_cache(tmp_path: Path) -> None: + workdir = tmp_path / "working_dir" + workdir.mkdir() + with working_directory(workdir): + result_code, stdout, stderr = get_main_output( + [ + "--singularity", + get_data("tests/sing_dockerfile_named_test.cwl"), + "--message", + "hello", + ] + ) + assert result_code == 0, stderr + print(list(workdir.iterdir())) + assert not (workdir / "bea92b9b6910cbbd2ae602f5bb0f0f27_latest.sif").exists() + assert not (workdir / "customDebian_latest.sif").exists() + + +@needs_singularity_3_or_newer +def test_singularity_dockerfile_with_name_with_cache( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + workdir = tmp_path / "working_dir" + workdir.mkdir() + cachedir = tmp_path / "cache" + cachedir.mkdir() + monkeypatch.setenv("CWL_SINGULARITY_CACHE", str(cachedir)) + with working_directory(workdir): + result_code, stdout, stderr = get_main_output( + [ + "--singularity", + get_data("tests/sing_dockerfile_named_test.cwl"), + "--message", + "hello", + ] + ) + print(list(workdir.iterdir())) + print(list(cachedir.iterdir())) + assert result_code == 0, stderr + assert not (workdir / "bea92b9b6910cbbd2ae602f5bb0f0f27_latest.sif").exists() + assert not (cachedir / "bea92b9b6910cbbd2ae602f5bb0f0f27_latest.sif").exists() + assert not (workdir / "customDebian_latest.sif").exists() + assert (cachedir / "customDebian_latest.sif").exists() diff --git a/tests/test_singularity_versions.py b/tests/test_singularity_versions.py index f7b30d8a3..2f6959fe5 100644 --- a/tests/test_singularity_versions.py +++ b/tests/test_singularity_versions.py @@ -1,6 +1,7 @@ """Test singularity{,-ce} & apptainer versions.""" -from subprocess import check_output # nosec +import pytest +from packaging.version import Version import cwltool.singularity from cwltool.singularity import ( @@ -13,124 +14,123 @@ ) -def reset_singularity_version_cache() -> None: +def reset_singularity_version_cache(monkeypatch: pytest.MonkeyPatch) -> None: """Reset the cache for testing.""" - cwltool.singularity._SINGULARITY_VERSION = None - cwltool.singularity._SINGULARITY_FLAVOR = "" + monkeypatch.setattr(cwltool.singularity, "_SINGULARITY_VERSION", None) + monkeypatch.setattr(cwltool.singularity, "_SINGULARITY_FLAVOR", "") -def set_dummy_check_output(name: str, version: str) -> None: - """Mock out subprocess.check_output.""" - cwltool.singularity.check_output = ( # type: ignore[attr-defined] - lambda c, text: name + " version " + version # type: ignore[assignment] +def dummy_check_output(monkeypatch: pytest.MonkeyPatch, name: str, version: str) -> None: + monkeypatch.setattr( + cwltool.singularity, "check_output", (lambda c, text: name + " version " + version) ) -def restore_check_output() -> None: - """Undo the mock of subprocess.check_output.""" - cwltool.singularity.check_output = check_output # type: ignore[attr-defined] - - -def test_get_version() -> None: +def test_get_version(monkeypatch: pytest.MonkeyPatch) -> None: """Confirm expected types of singularity.get_version().""" - set_dummy_check_output("apptainer", "1.0.1") - reset_singularity_version_cache() - v = get_version() - assert isinstance(v, tuple) - assert isinstance(v[0], list) - assert isinstance(v[1], str) - assert cwltool.singularity._SINGULARITY_VERSION is not None # pylint: disable=protected-access - assert len(cwltool.singularity._SINGULARITY_FLAVOR) > 0 # pylint: disable=protected-access - v_cached = get_version() - assert v == v_cached - - assert v[0][0] == 1 - assert v[0][1] == 0 - assert v[0][2] == 1 - assert v[1] == "apptainer" - - set_dummy_check_output("singularity", "3.8.5") - reset_singularity_version_cache() - v = get_version() - - assert v[0][0] == 3 - assert v[0][1] == 8 - assert v[0][2] == 5 - assert v[1] == "singularity" - restore_check_output() - - -def test_version_checks() -> None: + with monkeypatch.context() as m: + dummy_check_output(m, "apptainer", "1.0.1") + reset_singularity_version_cache(m) + v = get_version() + assert isinstance(v, tuple) + assert isinstance(v[0], Version) + assert isinstance(v[1], str) + assert ( + cwltool.singularity._SINGULARITY_VERSION is not None + ) # pylint: disable=protected-access + assert len(cwltool.singularity._SINGULARITY_FLAVOR) > 0 # pylint: disable=protected-access + v_cached = get_version() + assert v == v_cached + assert v[0] == Version("1.0.1") + assert v[1] == "apptainer" + + with monkeypatch.context() as m: + dummy_check_output(m, "singularity", "3.8.5") + reset_singularity_version_cache(m) + v = get_version() + assert v[0] == Version("3.8.5") + assert v[1] == "singularity" + + +def test_version_checks(monkeypatch: pytest.MonkeyPatch) -> None: """Confirm logic in the various singularity version checks.""" - set_dummy_check_output("apptainer", "1.0.1") - reset_singularity_version_cache() - assert is_apptainer_1_or_newer() - assert not is_version_2_6() - assert is_version_3_or_newer() - assert is_version_3_1_or_newer() - assert is_version_3_4_or_newer() - - set_dummy_check_output("apptainer", "0.0.1") - reset_singularity_version_cache() - assert not is_apptainer_1_or_newer() - assert not is_version_2_6() - assert not is_version_3_or_newer() - assert not is_version_3_1_or_newer() - assert not is_version_3_4_or_newer() - - set_dummy_check_output("singularity", "0.0.1") - reset_singularity_version_cache() - assert not is_apptainer_1_or_newer() - assert not is_version_2_6() - assert not is_version_3_or_newer() - assert not is_version_3_1_or_newer() - assert not is_version_3_4_or_newer() - - set_dummy_check_output("singularity", "0.1") - reset_singularity_version_cache() - assert not is_apptainer_1_or_newer() - assert not is_version_2_6() - assert not is_version_3_or_newer() - assert not is_version_3_1_or_newer() - assert not is_version_3_4_or_newer() - - set_dummy_check_output("singularity", "2.6") - reset_singularity_version_cache() - assert not is_apptainer_1_or_newer() - assert is_version_2_6() - assert not is_version_3_or_newer() - assert not is_version_3_1_or_newer() - assert not is_version_3_4_or_newer() - - set_dummy_check_output("singularity", "3.0") - reset_singularity_version_cache() - assert not is_apptainer_1_or_newer() - assert not is_version_2_6() - assert is_version_3_or_newer() - assert not is_version_3_1_or_newer() - assert not is_version_3_4_or_newer() - - set_dummy_check_output("singularity", "3.1") - reset_singularity_version_cache() - assert not is_apptainer_1_or_newer() - assert not is_version_2_6() - assert is_version_3_or_newer() - assert is_version_3_1_or_newer() - assert not is_version_3_4_or_newer() - - set_dummy_check_output("singularity", "3.4") - reset_singularity_version_cache() - assert not is_apptainer_1_or_newer() - assert not is_version_2_6() - assert is_version_3_or_newer() - assert is_version_3_1_or_newer() - assert is_version_3_4_or_newer() - - set_dummy_check_output("singularity", "3.6.3") - reset_singularity_version_cache() - assert not is_apptainer_1_or_newer() - assert not is_version_2_6() - assert is_version_3_or_newer() - assert is_version_3_1_or_newer() - assert is_version_3_4_or_newer() - restore_check_output() + with monkeypatch.context() as m: + dummy_check_output(m, "apptainer", "1.0.1") + reset_singularity_version_cache(m) + assert is_apptainer_1_or_newer() + assert not is_version_2_6() + assert is_version_3_or_newer() + assert is_version_3_1_or_newer() + assert is_version_3_4_or_newer() + + with monkeypatch.context() as m: + dummy_check_output(m, "apptainer", "0.0.1") + reset_singularity_version_cache(m) + assert not is_apptainer_1_or_newer() + assert not is_version_2_6() + assert not is_version_3_or_newer() + assert not is_version_3_1_or_newer() + assert not is_version_3_4_or_newer() + + with monkeypatch.context() as m: + dummy_check_output(m, "singularity", "0.0.1") + reset_singularity_version_cache(m) + assert not is_apptainer_1_or_newer() + assert not is_version_2_6() + assert not is_version_3_or_newer() + assert not is_version_3_1_or_newer() + assert not is_version_3_4_or_newer() + + with monkeypatch.context() as m: + dummy_check_output(m, "singularity", "0.1") + reset_singularity_version_cache(m) + assert not is_apptainer_1_or_newer() + assert not is_version_2_6() + assert not is_version_3_or_newer() + assert not is_version_3_1_or_newer() + assert not is_version_3_4_or_newer() + + with monkeypatch.context() as m: + dummy_check_output(m, "singularity", "2.6") + reset_singularity_version_cache(m) + assert not is_apptainer_1_or_newer() + assert is_version_2_6() + assert not is_version_3_or_newer() + assert not is_version_3_1_or_newer() + assert not is_version_3_4_or_newer() + + with monkeypatch.context() as m: + dummy_check_output(m, "singularity", "3.0") + reset_singularity_version_cache(m) + assert not is_apptainer_1_or_newer() + assert not is_version_2_6() + assert is_version_3_or_newer() + assert not is_version_3_1_or_newer() + assert not is_version_3_4_or_newer() + + with monkeypatch.context() as m: + dummy_check_output(m, "singularity", "3.1") + reset_singularity_version_cache(m) + assert not is_apptainer_1_or_newer() + assert not is_version_2_6() + assert is_version_3_or_newer() + assert is_version_3_1_or_newer() + assert not is_version_3_4_or_newer() + + with monkeypatch.context() as m: + dummy_check_output(m, "singularity", "3.4") + reset_singularity_version_cache(m) + assert not is_apptainer_1_or_newer() + assert not is_version_2_6() + assert is_version_3_or_newer() + assert is_version_3_1_or_newer() + assert is_version_3_4_or_newer() + + with monkeypatch.context() as m: + dummy_check_output(m, "singularity", "3.6.3") + reset_singularity_version_cache(m) + assert not is_apptainer_1_or_newer() + assert not is_version_2_6() + assert is_version_3_or_newer() + assert is_version_3_1_or_newer() + assert is_version_3_4_or_newer() diff --git a/tests/test_udocker.py b/tests/test_udocker.py index 5a8b96304..00878828e 100644 --- a/tests/test_udocker.py +++ b/tests/test_udocker.py @@ -30,6 +30,16 @@ def udocker(tmp_path_factory: TempPathFactory) -> str: ["tar", "--strip-components=1", "-xzvf", "udocker-tarball.tgz"], ["./udocker/udocker", "install"], ] + if "UDOCKER_USER" in os.environ and "UDOCKER_PASS" in os.environ: + install_cmds.append( + [ + "./udocker/udocker", + "login", + f"--username={os.environ['UDOCKER_USER']}", + f"--password={os.environ['UDOCKER_PASS']}", + ] + ) + install_cmds.append(["./udocker/udocker", "pull", "debian:stable-slim"]) test_environ["UDOCKER_DIR"] = os.path.join(docker_install_dir, ".udocker") test_environ["HOME"] = docker_install_dir @@ -78,10 +88,13 @@ def test_udocker_usage_should_not_write_cid_file(udocker: str, tmp_path: Path) - ) def test_udocker_should_display_memory_usage(udocker: str, tmp_path: Path) -> None: """Confirm that memory ussage is logged even with udocker.""" + print(udocker) with working_directory(tmp_path): error_code, stdout, stderr = get_main_output( [ "--enable-ext", + "--timestamps", + "--debug", "--default-container=debian:stable-slim", "--user-space-docker-cmd=" + udocker, get_data("tests/wf/timelimit.cwl"), @@ -91,7 +104,10 @@ def test_udocker_should_display_memory_usage(udocker: str, tmp_path: Path) -> No ) assert "completed success" in stderr, stderr - assert "Max memory" in stderr, stderr + assert ( + "Max memory" in stderr + or "Could not collect memory usage, job ended before monitoring began." in stderr + ), stderr @pytest.mark.skipif(not LINUX, reason="LINUX only") diff --git a/tox.ini b/tox.ini index 5350201cd..305603343 100644 --- a/tox.ini +++ b/tox.ini @@ -41,6 +41,7 @@ passenv = PROOT_NO_SECCOMP APPTAINER_TMPDIR SINGULARITY_FAKEROOT + UDOCKER_* extras = py3{10,11,12,13,14}-unit: deps @@ -62,8 +63,7 @@ setenv = HOME = {envtmpdir} commands_pre = - py3{10,11,12,13}-unit: python -m pip install -U pip setuptools wheel - py312-lintreadme: python -m build --outdir {distdir} + py312-lintreadme: python -m build --outdir {pkg_dir} commands = py3{10,11,12,13,14}-unit: make coverage-report coverage.xml PYTEST_EXTRA="{posargs}" @@ -72,7 +72,7 @@ commands = py3{10,11,12,13,14}-mypy: make mypy mypyc PYTEST_EXTRA="{posargs}" py312-shellcheck: make shellcheck py312-pydocstyle: make diff_pydocstyle_report - py312-lintreadme: twine check {distdir}/* + py312-lintreadme: twine check {pkg_dir}/* skip_install = py3{10,11,12,13,14}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true