diff --git a/docs/model/HarnessMainConfig.md b/docs/model/HarnessMainConfig.md index 6b217efbf..cea5093a5 100644 --- a/docs/model/HarnessMainConfig.md +++ b/docs/model/HarnessMainConfig.md @@ -19,6 +19,7 @@ Name | Type | Description | Notes **backup** | [**BackupConfig**](BackupConfig.md) | | [optional] **name** | **str** | Base name | [optional] **task_images** | **Dict[str, object]** | | [optional] +**task_commands** | **Dict[str, object]** | | [optional] **build_hash** | **str** | | [optional] **ingress** | [**GatewayGlobalConfig**](GatewayGlobalConfig.md) | | [optional] diff --git a/libraries/cloudharness-common/cloudharness/utils/config.py b/libraries/cloudharness-common/cloudharness/utils/config.py index c6d120994..ceaba0144 100644 --- a/libraries/cloudharness-common/cloudharness/utils/config.py +++ b/libraries/cloudharness-common/cloudharness/utils/config.py @@ -97,6 +97,17 @@ def get_image_tag(cls, base_name): return base_name return cls.get_configuration()['task-images'][base_name] + @classmethod + def get_task_command(cls, base_name): + """ + Command a task image runs, derived from its Dockerfile by harness-deployment. + + Returns: + list of the command arguments, None when the image has no derived command + (an external image, a base image, or a configuration generated without it) + """ + return (cls.get_configuration().get('task-commands') or {}).get(base_name) + @classmethod def get_application_by_filter(cls, **filter): """ diff --git a/libraries/cloudharness-common/cloudharness/workflows/tasks.py b/libraries/cloudharness-common/cloudharness/workflows/tasks.py index 9dca8d606..2b5c11202 100644 --- a/libraries/cloudharness-common/cloudharness/workflows/tasks.py +++ b/libraries/cloudharness-common/cloudharness/workflows/tasks.py @@ -1,6 +1,7 @@ from cloudharness.utils import dict_merge from . import argo_service from argo_workflows.model.io_argoproj_workflow_v1alpha1_template import IoArgoprojWorkflowV1alpha1Template +from cloudharness.utils.config import CloudharnessConfig as conf from cloudharness.utils.env import get_cloudharness_variables, get_image_full_tag from .utils import WORKFLOW_NAME_VARIABLE_NAME, PodExecutionContext, affinity_spec, is_accounts_present, volume_mount_template, volume_requires_affinity @@ -186,8 +187,15 @@ class CustomTask(ContainerizedTask): def __init__(self, name, image_name, resources={}, image_pull_policy='IfNotPresent', command=None, **env_args): super().__init__(name, resources=resources, image_pull_policy=image_pull_policy, command=command, **env_args) + if command is None and not self._overrides_container_command(): + self.command = conf.get_task_command(image_name) self.__image_name = get_image_full_tag(image_name) + def _overrides_container_command(self): + """Whether the template overrides already set what the container runs, which the derived command must not replace.""" + container = self.template_overrides.get('container') or {} + return container.get('command') is not None or container.get('args') is not None + @property def image_name(self): return self.__image_name diff --git a/libraries/models/api/openapi.yaml b/libraries/models/api/openapi.yaml index 067645022..467bdaabe 100644 --- a/libraries/models/api/openapi.yaml +++ b/libraries/models/api/openapi.yaml @@ -635,6 +635,9 @@ components: task-images: $ref: '#/components/schemas/SimpleMap' description: '' + task-commands: + $ref: '#/components/schemas/SimpleMap' + description: Command each task image runs, by task name, derived from its Dockerfile build_hash: description: '' type: string diff --git a/libraries/models/cloudharness_model/models/harness_main_config.py b/libraries/models/cloudharness_model/models/harness_main_config.py index ba25128fe..41e51602c 100644 --- a/libraries/models/cloudharness_model/models/harness_main_config.py +++ b/libraries/models/cloudharness_model/models/harness_main_config.py @@ -48,10 +48,11 @@ class HarnessMainConfig(CloudHarnessBaseModel): backup: Optional[BackupConfig] = None name: Optional[StrictStr] = Field(default=None, description="Base name") task_images: Optional[Dict[str, Any]] = Field(default=None, alias="task-images") + task_commands: Optional[Dict[str, Any]] = Field(default=None, alias="task-commands") build_hash: Optional[StrictStr] = None ingress: Optional[GatewayGlobalConfig] = None additional_properties: Dict[str, Any] = {} - __properties: ClassVar[List[str]] = ["local", "secured_gatekeepers", "domain", "namespace", "mainapp", "registry", "tag", "apps", "env", "privenv", "backup", "name", "task-images", "build_hash", "ingress"] + __properties: ClassVar[List[str]] = ["local", "secured_gatekeepers", "domain", "namespace", "mainapp", "registry", "tag", "apps", "env", "privenv", "backup", "name", "task-images", "task-commands", "build_hash", "ingress"] def to_dict(self) -> Dict[str, Any]: """Return the dictionary representation of the model using alias. @@ -138,6 +139,7 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "backup": BackupConfig.from_dict(obj["backup"]) if obj.get("backup") is not None else None, "name": obj.get("name"), "task-images": obj.get("task-images"), + "task-commands": obj.get("task-commands"), "build_hash": obj.get("build_hash"), "ingress": GatewayGlobalConfig.from_dict(obj["ingress"]) if obj.get("ingress") is not None else None }) diff --git a/libraries/models/docs/HarnessMainConfig.md b/libraries/models/docs/HarnessMainConfig.md index 6b217efbf..cea5093a5 100644 --- a/libraries/models/docs/HarnessMainConfig.md +++ b/libraries/models/docs/HarnessMainConfig.md @@ -19,6 +19,7 @@ Name | Type | Description | Notes **backup** | [**BackupConfig**](BackupConfig.md) | | [optional] **name** | **str** | Base name | [optional] **task_images** | **Dict[str, object]** | | [optional] +**task_commands** | **Dict[str, object]** | | [optional] **build_hash** | **str** | | [optional] **ingress** | [**GatewayGlobalConfig**](GatewayGlobalConfig.md) | | [optional] diff --git a/tools/deployment-cli-tools/ch_cli_tools/dockerfile.py b/tools/deployment-cli-tools/ch_cli_tools/dockerfile.py new file mode 100644 index 000000000..d0ddc4839 --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/dockerfile.py @@ -0,0 +1,250 @@ +import json +import re +import shlex +from dataclasses import dataclass, field +from functools import cached_property +from pathlib import Path + +# Instructions whose argument is either a JSON exec-form array (`CMD ["gunicorn", "app:app"]`) +# or a plain shell-form string (`CMD gunicorn app:app`). +_EXEC_FORM_INSTRUCTIONS = {"RUN", "CMD", "ENTRYPOINT", "SHELL", "HEALTHCHECK"} +_DEFAULT_SHELL = ["/bin/sh", "-c"] + +_LINE_CONTINUATION = re.compile(r"\\\s*$") +_FROM_ALIAS = re.compile(r"\s+AS\s+", re.IGNORECASE) +_LEADING_FLAG = re.compile(r"^--[\w-]+(=\S*)?$") +_VAR_REFERENCE = re.compile(r"^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$") + + +@dataclass +class Instruction: + """A line of a Dockerfile.""" + + name: str + arguments: tuple[str, ...] = () + raw: str = "" + # The command ENTRYPOINT and CMD run, set once the shell in effect is known + argv: list[str] | None = None + + @property + def fields(self) -> tuple[str, ...]: + return (self.name, *self.arguments) + + +@dataclass +class Stage: + """The instructions that follow a FROM, up to the next one.""" + + reference: str + alias: str | None = None + parent: "Stage | None" = None # the earlier stage of the file this one is built FROM + instructions: list[Instruction] = field(default_factory=list) + shell: list[str] = field(default_factory=lambda: list(_DEFAULT_SHELL)) + + @property + def root_reference(self) -> str: + """The image the chain of stages ultimately starts FROM.""" + return self.parent.root_reference if self.parent else self.reference + + @property + def entrypoint(self) -> list[str] | None: + """The ENTRYPOINT set by this stage or the stages it is built FROM.""" + own = self._last_argv("ENTRYPOINT") + if own is not None or self.parent is None: + return own + return self.parent.entrypoint + + @property + def cmd(self) -> list[str] | None: + """The CMD set by this stage or the stages it is built FROM. + + Like Docker, an ENTRYPOINT clears the CMD of what the stage is built FROM, + but not one set earlier in the same stage.""" + own = self._last_argv("CMD") + if own is not None: + return own + if self._last_argv("ENTRYPOINT") is not None or self.parent is None: + return None + return self.parent.cmd + + @property + def inherits_entrypoint(self) -> bool: + """Whether no stage of the chain sets an ENTRYPOINT, leaving that of the base image.""" + return self._last_argv("ENTRYPOINT") is None and ( + self.parent is None or self.parent.inherits_entrypoint + ) + + @property + def inherits_cmd(self) -> bool: + """Whether no stage of the chain sets a CMD or an ENTRYPOINT, leaving the CMD of the base image.""" + return ( + self._last_argv("CMD") is None and + self._last_argv("ENTRYPOINT") is None and + (self.parent is None or self.parent.inherits_cmd) + ) + + def _last_argv(self, name: str) -> list[str] | None: + argvs = [i.argv for i in self.instructions if i.name == name and i.argv is not None] + return argvs[-1] if argvs else None + + +@dataclass +class ParsedDockerfile: + """What the parser makes of a Dockerfile: its instructions, and the stages they build.""" + + instructions: list[Instruction] = field(default_factory=list) + stages: list[Stage] = field(default_factory=list) + + @cached_property + def arg_names(self) -> set[str]: + """The names declared by the ARG instructions.""" + return {i.arguments[0] for i in self.instructions if i.name == "ARG" and i.arguments} + + @property + def final_stage(self) -> Stage | None: + return self.stages[-1] if self.stages else None + + @property + def arg_references(self) -> list[str]: + """The ARG names used as the image of a FROM, in order.""" + return [s.reference for s in self.stages if s.reference in self.arg_names] + + @property + def base_images(self) -> dict[str, str]: + """The default of each ARG used as the image of a FROM, by the name of the ARG.""" + defaults = { + i.arguments[0]: i.arguments[1] + for i in self.instructions + if i.name == "ARG" and len(i.arguments) > 1 + } + return {ref: defaults[ref] for ref in self.arg_references if ref in defaults} + + def resolve_commands(self, inherited_shell: list[str] | None = None) -> None: + """Set the argv of every ENTRYPOINT and CMD, shell form being run by the SHELL in effect. + + `inherited_shell` is the one the base image leaves in effect for the first stage.""" + for stage in self.stages: + stage.shell = list( + stage.parent.shell if stage.parent else inherited_shell or _DEFAULT_SHELL + ) + for instruction in stage.instructions: + if instruction.name == "SHELL" and instruction.raw: + stage.shell = DockerfileParser.parse_exec_form(instruction.raw) or stage.shell + elif instruction.name in ("ENTRYPOINT", "CMD") and instruction.raw: + argv = DockerfileParser.parse_exec_form(instruction.raw) + instruction.argv = argv if argv is not None else [*stage.shell, instruction.raw] + + +class DockerfileParser: + """Reads a Dockerfile in phases, each one enriching the model: lines, instructions, stages, commands.""" + + def __init__(self, path: Path): + self.path = path + + def parse(self) -> ParsedDockerfile: + model = ParsedDockerfile( + instructions=[ + instruction for line in self._read_lines() for instruction in self._tokenize(line) + ] + ) + self._split_stages(model) + model.resolve_commands() + return model + + def _read_lines(self) -> list[str]: + """The instructions, one per line: comments dropped, continuations joined. Empty when the file is missing.""" + if not self.path.exists(): + return [] + + lines: list[str] = [] + pending = "" + for raw_line in self.path.read_text().splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if pending: + line = f"{pending} {line}".strip() + pending = "" + if _LINE_CONTINUATION.search(line): + pending = _LINE_CONTINUATION.sub("", line).strip() + continue + lines.append(line) + return lines + + @classmethod + def _tokenize(cls, line: str) -> list[Instruction]: + """The instructions of a line: one, except for an ARG declaring several names.""" + name, _, rest = line.partition(" ") + name = name.upper() + rest = rest.strip() + if not rest: + return [Instruction(name)] + if name == "FROM": + return [Instruction(name, cls._parse_from(rest), rest)] + if name == "ARG": + return [Instruction(name, cls._parse_arg(token), token) for token in cls._split_args(rest)] + if name in _EXEC_FORM_INSTRUCTIONS and rest.startswith("["): + items = cls.parse_exec_form(rest) + return [Instruction(name, tuple(items) if items is not None else (rest,), rest)] + return [Instruction(name, tuple(cls._split_args(rest)), rest)] + + @classmethod + def _split_stages(cls, model: ParsedDockerfile) -> None: + """Group the instructions after each FROM into a stage, linked to the earlier stage it is built FROM.""" + by_alias: dict[str, Stage] = {} + for instruction in model.instructions: + if instruction.name == "FROM" and instruction.arguments: + reference, *alias = instruction.arguments + is_variable = cls.variable_reference(cls._strip_leading_flags(instruction.raw.split())[0]) is not None + parent = None if is_variable else by_alias.get(reference.lower()) + stage = Stage(reference, alias[0] if alias else None, parent) + model.stages.append(stage) + if stage.alias: + by_alias[stage.alias.lower()] = stage + elif model.stages: + model.stages[-1].instructions.append(instruction) + + @staticmethod + def variable_reference(token: str) -> str | None: + """The name in a `$NAME` or `${NAME}` token, None for anything else.""" + match = _VAR_REFERENCE.match(token) + return match.group(1) if match else None + + @staticmethod + def parse_exec_form(argument: str) -> list[str] | None: + """The items of an exec-form (JSON array) argument, None when it is not one.""" + try: + parsed = json.loads(argument) + except ValueError: + return None + return [str(item) for item in parsed] if isinstance(parsed, list) else None + + @staticmethod + def _strip_leading_flags(tokens: list[str]) -> list[str]: + while tokens and _LEADING_FLAG.match(tokens[0]): + tokens.pop(0) + return tokens + + @classmethod + def _split_args(cls, rest: str) -> list[str]: + try: + tokens = shlex.split(rest) + except ValueError: + tokens = rest.split() + return cls._strip_leading_flags(tokens) + + @classmethod + def _parse_from(cls, rest: str) -> tuple[str, ...]: + # FROM [--platform=] [AS ] + tokens = cls._strip_leading_flags(rest.split()) + parts = [part.strip() for part in _FROM_ALIAS.split(" ".join(tokens), maxsplit=1)] + reference = cls.variable_reference(parts[0]) + if reference is not None: + parts[0] = reference + return tuple(parts) + + @staticmethod + def _parse_arg(token: str) -> tuple[str, ...]: + # [=] + name, has_default, default = token.partition("=") + return (name, default) if has_default else (name,) diff --git a/tools/deployment-cli-tools/ch_cli_tools/model.py b/tools/deployment-cli-tools/ch_cli_tools/model.py index 7518d630a..26b5f011a 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/model.py +++ b/tools/deployment-cli-tools/ch_cli_tools/model.py @@ -82,9 +82,10 @@ class CHTemplate import logging import re import shutil +import sys from collections.abc import Callable from dataclasses import dataclass, field -from functools import lru_cache +from functools import cached_property, lru_cache from pathlib import Path from typing import Literal, cast @@ -92,18 +93,18 @@ class CHTemplate from ch_cli_tools.configuration.secrets import secret_definition_error # type: ignore +from .dockerfile import DockerfileParser, ParsedDockerfile from .utils import ( MERGE_DIRNAME, _merge_base_chain, content_hash, - dict_merge, # type: ignore + dict_merge, env_suffixed_path, find_files, get_git_commit_hash, get_yaml_parser, merge_with_layer, own_dockerfile_path, - parse_dockerfile, resolve_path, ) @@ -942,60 +943,122 @@ def resolve_context(self, relative_to: Path) -> CHContext: @property def base_images(self) -> dict[str, str]: """Gets the ARGS from a Dockerfile image (if ARGS is used directly in the FROM of the Dockerfile)""" - if not self.exists(): - return {} - content = self.path.read_text() - found_args = {} - args: dict[str, str] = {} - for line in content.splitlines(): - line = line.strip() - if not line: - continue - cmd, *rest = line.split() - arg = rest[0] if len(rest) > 0 else "" - match cmd: - case "ARG" if "=" in arg: - key, val = arg.split("=", maxsplit=1) - found_args[key] = val - case "FROM" if arg[1:] in found_args: # $NAME case - key = arg[1:] - args[key] = found_args[key] - case "FROM" if arg[2:-1] in found_args: # ${NAME} case - key = arg[2:-1] - args[key] = found_args[key] - case _: - continue - return args + return self.parsed.base_images - @property - def base_dependencies(self) -> list["CHApp | CHBaseImage | CHAppTask | str"]: - """Apps, base images or tasks referenced via ARG in a FROM line.""" - if not self.exists(): - return [] + @cached_property + def parsed(self) -> ParsedDockerfile: + return DockerfileParser(self.path).parse() - project = self.app.project - instructions = parse_dockerfile(self.path) - arg_names = { - instruction[1] for instruction in instructions if instruction[0] == "ARG" - } + @cached_property + def _resolved(self) -> ParsedDockerfile: + """The parsed Dockerfile with its commands resolved under the SHELL its base image leaves in effect.""" + base = self.base_dockerfile + self.parsed.resolve_commands(base.shell if base is not None else None) + return self.parsed - dependencies = [] - for instruction in instructions: - if instruction[0] != "FROM": - continue - ref = instruction[1] - if ref not in arg_names: - continue + @property + def shell(self) -> list[str] | None: + """The SHELL in effect at the end of the final stage, None when the Dockerfile has none.""" + stage = self._resolved.final_stage + return stage.shell if stage is not None else None - name = ref.lower().replace("_", "-") - entity = ( - project.scanned_apps.get(name) or - project.base_images.get(name) or - project.all_buildable_tasks().get(name) - ) + @property + def base_dependencies(self) -> list["CHDockerfile | str"]: + """Dockerfiles of the apps, base images or tasks referenced via ARG in a FROM line, or their name when unknown.""" + dependencies = [] + for reference in self.parsed.arg_references: + name = _project_image_name(reference) + entity = self.app.project.find_entity(name) dependencies.append(entity.dockerfile if entity is not None else name) return dependencies + @cached_property + def base_dockerfile(self) -> "CHDockerfile | None": + """The Dockerfile of the project image this one is built FROM, None when that image is external. + + A base that leads back to this Dockerfile is ignored, with an error.""" + base = self._find_base_dockerfile() + if base is not None and self._leads_back_to_itself(base): + logger.error( + "Dockerfile %s is built FROM itself through %s: its base is ignored", + self.path, + base.path, + ) + return None + return base + + def _find_base_dockerfile(self) -> "CHDockerfile | None": + stage = self.parsed.final_stage + if stage is None or stage.root_reference not in self.parsed.arg_names: + return None + entity = self.app.project.find_entity(_project_image_name(stage.root_reference)) + return entity.dockerfile if entity is not None else None + + def _leads_back_to_itself(self, base: "CHDockerfile") -> bool: + own = self.path.resolve() + seen: set[Path] = set() + while base is not None: + path = base.path.resolve() + if path == own: + return True + if path in seen: # a loop this Dockerfile only leads into: the Dockerfiles in it report it + return False + seen.add(path) + base = base._find_base_dockerfile() + return False + + @property + def external_base(self) -> str | None: + """The image outside the project the chain of FROMs ends in, None when there is no FROM.""" + stage = self.parsed.final_stage + if stage is None: + return None + base = self.base_dockerfile + if base is not None: + return base.external_base + return self.base_images.get(stage.root_reference, stage.root_reference) + + @property + def entrypoint(self) -> list[str] | None: + """The ENTRYPOINT of the final stage, else its base's; None when neither sets one. + + A base outside the project is assumed to set none.""" + stage = self._resolved.final_stage + if stage is None: + return None + if stage.inherits_entrypoint: + base = self.base_dockerfile + return base.entrypoint if base is not None else None + return stage.entrypoint + + @property + def cmd(self) -> list[str] | None: + """The CMD of the final stage, else its base's unless the stage sets an ENTRYPOINT (which clears it). + + A base outside the project is assumed to set none.""" + stage = self._resolved.final_stage + if stage is None: + return None + if stage.inherits_cmd: + base = self.base_dockerfile + return base.cmd if base is not None else None + return stage.cmd + + @property + def command(self) -> list[str]: + """What the container runs: the ENTRYPOINT followed by the CMD, empty when neither is set.""" + return [*(self.entrypoint or []), *(self.cmd or [])] + + +def _highlight(message: str) -> str: + """The message in bold yellow when logging to a terminal.""" + return f"\033[1;33m{message}\033[0m" if sys.stderr.isatty() else message + + +def _project_image_name(reference: str) -> str: + """The name of the project image an ARG named `reference` stands for (MY_BASE -> my-base).""" + return reference.lower().replace("_", "-") + def _default_test_unit_enabled(harness: dict) -> dict: """Defaults harness.test.unit.enabled to False when the block exists but doesn't say.""" @@ -1339,6 +1402,14 @@ def all_buildable_tasks(self): tasks.update(app._tasks) return tasks + def find_entity(self, name: str) -> "CHApp | CHBaseImage | CHAppTask | None": + """The app, base image or task of the project with that name.""" + return ( + self.scanned_apps.get(name) or + self.base_images.get(name) or + self.all_buildable_tasks().get(name) + ) + def all_source_images(self) -> dict[str, str]: """Every externally-sourced image: Dockerfile ARGs plus the source_images overrides.""" images: dict[str, str] = {} @@ -1464,18 +1535,47 @@ def qualify_image(self, entity: CHApp | CHBaseImage | CHAppTask) -> str: tag = self.config.tag return f"{qualified}:{tag}" if tag else qualified + @lru_cache + def all_task_entities(self) -> dict[str, CHApp | CHBaseImage | CHAppTask]: + """Every build dependency and task, deployed or not, by the name of its image.""" + already_deployed = tuple(app.name for app in self._concrete_involved_apps) + entities = dict(self.all_build_dependencies(already_deployed)) + for app in self._concrete_involved_apps: + entities.update({task.name: task for task in app.buildable_tasks}) + return entities + def all_task_images(self) -> dict[str, str]: """Qualified images for every build dependency and task, deployed or not.""" - already_deployed = tuple(app.name for app in self._concrete_involved_apps) - images = { - name: self.qualify_image(dependency) - for name, dependency in self.all_build_dependencies( - already_deployed - ).items() + return { + name: self.qualify_image(entity) + for name, entity in self.all_task_entities().items() } - for app in self._concrete_involved_apps: - images.update(app.task_images) - return images + + def all_task_commands(self) -> dict[str, list[str]]: + """What each task image runs, under the names of all_task_images. + + An image whose Dockerfile and bases set neither ENTRYPOINT nor CMD is left out, with a warning + when it is a task: its workflows must then pass the command explicitly.""" + commands = {} + assumed_bases: set[str] = set() + for name, entity in self.all_task_entities().items(): + dockerfile = entity.dockerfile + command = dockerfile.command + if command: + commands[name] = command + elif isinstance(entity, CHAppTask): + logger.warning( + "Task %s sets no ENTRYPOINT or CMD, so no command is derived: if its base image defines none either, workflows must pass the command explicitly", + name, + ) + if command and dockerfile.entrypoint is None and dockerfile.external_base: + assumed_bases.add(dockerfile.external_base) + if assumed_bases: + logger.warning( + _highlight("Task commands assume these external base images define no ENTRYPOINT (a task of a base that does gets a wrong command, set it explicitly in the Dockerfile or use the 'command=' parameter in your Python code launching the task): %s"), + ", ".join(sorted(assumed_bases)), + ) + return commands @lru_cache def all_values(self): @@ -1545,6 +1645,7 @@ def build_final_helm_values(self, write_on_disk=True) -> HarnessMainConfig: **project_values, "apps": apps, "task-images": self.all_task_images(), + "task-commands": self.all_task_commands(), "source_images": source_images, "local": self.config.local, "secured_gatekeepers": self.config.secured_gatekeepers, diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index f891baaee..2ebbcc066 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -55,100 +55,6 @@ def merge_with_layer(base: "CHValues", layer: "CHValues"): return {} -# Commands whose argument can be either a JSON exec-form array -# (`CMD ["gunicorn", "app:app"]`) or a plain shell-form string -# (`CMD gunicorn app:app`). -_EXEC_FORM_COMMANDS = {"RUN", "CMD", "ENTRYPOINT", "SHELL", "HEALTHCHECK"} - -_LINE_CONTINUATION = re.compile(r"\\\s*$") -_FROM_ALIAS = re.compile(r"\s+AS\s+", re.IGNORECASE) -_LEADING_FLAG = re.compile(r"^--[\w-]+(=\S*)?$") -_VAR_REFERENCE = re.compile(r"^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$") - - -def dockerfile_variable_reference(token: str) -> str | None: - match = _VAR_REFERENCE.match(token) - return match.group(1) if match else None - - -def parse_dockerfile(path: Path) -> list[tuple[str, ...]]: - if not path.exists(): - return [] - - instructions: list[tuple[str, ...]] = [] - pending = "" - for raw_line in path.read_text().splitlines(): - line = raw_line.strip() - if pending: - line = f"{pending} {line}".strip() - pending = "" - if not line or line.startswith("#"): - continue - if _LINE_CONTINUATION.search(line): - pending = _LINE_CONTINUATION.sub("", line).strip() - continue - - command, _, rest = line.partition(" ") - command = command.upper() - rest = rest.strip() - if not rest: - instructions.append((command,)) - continue - - if command == "FROM": - instructions.append((command, *_parse_from(rest))) - elif command == "ARG": - instructions.append((command, *_parse_arg(rest))) - elif command in _EXEC_FORM_COMMANDS and rest.startswith("["): - instructions.append((command, *_parse_exec_form(rest))) - else: - instructions.append((command, *_split_args(rest))) - - return instructions - - -def _strip_leading_flags(tokens: list[str]) -> list[str]: - while tokens and _LEADING_FLAG.match(tokens[0]): - tokens.pop(0) - return tokens - - -def _split_args(rest: str) -> list[str]: - try: - tokens = shlex.split(rest) - except ValueError: - tokens = rest.split() - return _strip_leading_flags(tokens) - - -def _parse_from(rest: str) -> tuple[str, ...]: - # FROM [--platform=] [AS ] - tokens = _strip_leading_flags(rest.split()) - parts = [part.strip() for part in _FROM_ALIAS.split(" ".join(tokens), maxsplit=1)] - reference = dockerfile_variable_reference(parts[0]) - if reference is not None: - parts[0] = reference - return tuple(parts) - - -def _parse_arg(rest: str) -> tuple[str, ...]: - # ARG [=] - name, has_default, default = rest.partition("=") - if has_default: - return (name.strip(), default.strip()) - return (name.strip(),) - - -def _parse_exec_form(rest: str) -> tuple[str, ...]: - try: - parsed = json.loads(rest) - except ValueError: - return (rest,) - if isinstance(parsed, list): - return tuple(str(item) for item in parsed) - return (rest,) - - CLOUD_HARNESS_DIR_NAME = "cloud-harness" diff --git a/tools/deployment-cli-tools/tests/test_dockerfile_parser.py b/tools/deployment-cli-tools/tests/test_dockerfile_parser.py new file mode 100644 index 000000000..a99f8bb19 --- /dev/null +++ b/tools/deployment-cli-tools/tests/test_dockerfile_parser.py @@ -0,0 +1,217 @@ +"""The Dockerfile parser and the model it builds, without any project around them.""" + +from pathlib import Path + +from ch_cli_tools.dockerfile import DockerfileParser + + +def _parse(tmp_path: Path, content: str): + path = tmp_path / "Dockerfile" + path.write_text(content) + return DockerfileParser(path).parse() + + +def _fields(model): + return [i.fields for i in model.instructions] + + +def test_missing_file_gives_an_empty_model(tmp_path): + model = DockerfileParser(tmp_path / "Dockerfile").parse() + + assert model.instructions == [] + assert model.stages == [] + assert model.final_stage is None + + +def test_comments_blank_lines_and_continuations_are_handled(tmp_path): + model = _parse(tmp_path, "# c\n\nFROM scratch\nRUN a \\\n b\n") + + assert _fields(model) == [("FROM", "scratch"), ("RUN", "a", "b")] + + +def test_instruction_names_are_upper_cased(tmp_path): + assert _fields(_parse(tmp_path, "from scratch\n")) == [("FROM", "scratch")] + + +def test_from_has_its_image_and_its_alias_without_flags(tmp_path): + model = _parse(tmp_path, "FROM --platform=linux/amd64 python:3.14 AS build\n") + + assert _fields(model) == [("FROM", "python:3.14", "build")] + + +def test_variable_references_are_normalized_in_from(tmp_path): + assert _fields(_parse(tmp_path, "FROM ${BASE}\n")) == [("FROM", "BASE")] + + +def test_arg_has_its_name_and_its_default(tmp_path): + model = _parse(tmp_path, "ARG A\nARG B=python:3.14\n") + + assert _fields(model) == [("ARG", "A"), ("ARG", "B", "python:3.14")] + assert model.arg_names == {"A", "B"} + + +def test_exec_form_arguments_are_the_items_and_shell_form_ones_are_split(tmp_path): + model = _parse(tmp_path, 'CMD ["a", "b"]\nRUN echo "x y"\n') + + assert _fields(model) == [("CMD", "a", "b"), ("RUN", "echo", "x y")] + + +def test_instructions_before_the_first_from_belong_to_no_stage(tmp_path): + model = _parse(tmp_path, "ARG A\nFROM scratch\nRUN x\n") + + assert len(model.stages) == 1 + assert [i.name for i in model.stages[0].instructions] == ["RUN"] + + +def test_stages_are_split_at_each_from(tmp_path): + model = _parse(tmp_path, "FROM a\nRUN 1\nFROM b\nRUN 2\nRUN 3\n") + + assert [s.reference for s in model.stages] == ["a", "b"] + assert [len(s.instructions) for s in model.stages] == [1, 2] + assert model.final_stage.reference == "b" + + +def test_stage_built_from_an_earlier_one_has_it_as_parent_whatever_the_case(tmp_path): + model = _parse(tmp_path, "FROM a AS Base\nFROM base\n") + + assert model.stages[1].parent is model.stages[0] + assert model.stages[1].root_reference == "a" + + +def test_stage_built_from_an_image_has_no_parent(tmp_path): + model = _parse(tmp_path, "FROM a AS base\nFROM python:3.14\n") + + assert model.stages[1].parent is None + assert model.stages[1].root_reference == "python:3.14" + + +def test_arg_references_are_the_args_used_as_from_images(tmp_path): + model = _parse(tmp_path, "ARG BASE\nARG OTHER\nFROM $BASE\nFROM python\n") + + assert model.arg_references == ["BASE"] + + +def test_exec_form_argv_is_used_as_written_and_shell_form_goes_through_the_shell(tmp_path): + stage = _parse(tmp_path, 'FROM a\nENTRYPOINT ["e"]\nCMD run it\n').final_stage + + assert stage.entrypoint == ["e"] + assert stage.cmd == ["/bin/sh", "-c", "run it"] + + +def test_shell_in_effect_wraps_shell_form_commands_and_is_inherited_by_child_stages(tmp_path): + model = _parse( + tmp_path, + 'FROM a AS base\nSHELL ["/bin/bash", "-c"]\nFROM base\nCMD hi\n', + ) + + assert model.final_stage.cmd == ["/bin/bash", "-c", "hi"] + + +def test_shell_applies_only_to_the_commands_after_it(tmp_path): + stage = _parse(tmp_path, 'FROM a\nCMD one\nSHELL ["/bin/bash", "-c"]\nENTRYPOINT two\n').final_stage + + assert stage.cmd == ["/bin/sh", "-c", "one"] + assert stage.entrypoint == ["/bin/bash", "-c", "two"] + + +def test_last_entrypoint_and_cmd_win(tmp_path): + stage = _parse(tmp_path, 'FROM a\nCMD ["1"]\nCMD ["2"]\nENTRYPOINT ["x"]\nENTRYPOINT ["y"]\n').final_stage + + assert stage.cmd == ["2"] + assert stage.entrypoint == ["y"] + + +def test_empty_exec_form_entrypoint_is_set_not_missing(tmp_path): + stage = _parse(tmp_path, "FROM a\nENTRYPOINT []\n").final_stage + + assert stage.entrypoint == [] + assert stage.inherits_entrypoint is False + + +def test_stage_inherits_what_it_does_not_set_from_its_parent(tmp_path): + stage = _parse(tmp_path, 'FROM a AS base\nENTRYPOINT ["e"]\nCMD ["c"]\nFROM base\nRUN x\n').final_stage + + assert stage.entrypoint == ["e"] + assert stage.cmd == ["c"] + assert stage.inherits_entrypoint is False + assert stage.inherits_cmd is False + + +def test_entrypoint_clears_the_cmd_of_the_parent_stage(tmp_path): + stage = _parse(tmp_path, 'FROM a AS base\nCMD ["a"]\nFROM base\nENTRYPOINT ["b"]\n').final_stage + + assert stage.entrypoint == ["b"] + assert stage.cmd is None + assert stage.inherits_cmd is False + + +def test_entrypoint_after_a_cmd_of_the_same_stage_keeps_it(tmp_path): + stage = _parse(tmp_path, 'FROM a\nCMD ["c"]\nENTRYPOINT ["e"]\n').final_stage + + assert stage.cmd == ["c"] + + +def test_stage_setting_nothing_inherits_the_command_of_its_base_image(tmp_path): + stage = _parse(tmp_path, "FROM a\nRUN x\n").final_stage + + assert stage.inherits_entrypoint is True + assert stage.inherits_cmd is True + assert stage.entrypoint is None + assert stage.cmd is None + + +def test_entrypoint_alone_stops_the_stage_inheriting_the_cmd_but_not_the_entrypoint_of_a_parent(tmp_path): + stage = _parse(tmp_path, 'FROM a\nENTRYPOINT ["e"]\n').final_stage + + assert stage.inherits_entrypoint is False + assert stage.inherits_cmd is False + + +def test_variable_reference_is_the_name_in_a_dollar_token(): + assert DockerfileParser.variable_reference("$NAME") == "NAME" + assert DockerfileParser.variable_reference("${NAME}") == "NAME" + assert DockerfileParser.variable_reference("python:3.12") is None + + +def test_parse_exec_form_is_none_when_the_argument_is_not_a_json_array(): + assert DockerfileParser.parse_exec_form('["a", 1]') == ["a", "1"] + assert DockerfileParser.parse_exec_form("a b") is None + assert DockerfileParser.parse_exec_form('{"a": 1}') is None + + +def test_comments_and_blank_lines_inside_a_continued_instruction_are_dropped(tmp_path): + model = _parse(tmp_path, 'FROM a\nCMD ["a", \\\n# comment\n\n "b"]\nRUN x\n') + + assert _fields(model) == [("FROM", "a"), ("CMD", "a", "b"), ("RUN", "x")] + + +def test_arg_declaring_several_names_is_one_instruction_per_name(tmp_path): + model = _parse(tmp_path, 'ARG A=1 B C="x y"\nFROM $B\n') + + assert _fields(model)[:3] == [("ARG", "A", "1"), ("ARG", "B"), ("ARG", "C", "x y")] + assert model.arg_names == {"A", "B", "C"} + assert model.arg_references == ["B"] + + +def test_variable_named_like_an_earlier_stage_is_an_image_not_that_stage(tmp_path): + model = _parse(tmp_path, "ARG BASE\nFROM x AS base\nFROM $BASE\n") + + assert model.final_stage.parent is None + assert model.final_stage.root_reference == "BASE" + + +def test_resolving_commands_again_uses_the_shell_the_base_image_leaves_in_effect(tmp_path): + model = _parse(tmp_path, "FROM a\nCMD hi\n") + + model.resolve_commands(["/bin/bash", "-c"]) + + assert model.final_stage.cmd == ["/bin/bash", "-c", "hi"] + + +def test_base_images_are_the_defaults_of_the_args_used_as_from_images(tmp_path): + model = _parse( + tmp_path, + 'ARG BASE="python:3.14"\nARG OTHER=x\nARG NODEFAULT\nFROM --platform=$BUILDPLATFORM ${BASE}\nFROM $NODEFAULT\n', + ) + + assert model.base_images == {"BASE": "python:3.14"} diff --git a/tools/deployment-cli-tools/tests/test_skaffold_internals.py b/tools/deployment-cli-tools/tests/test_skaffold_internals.py index bb05dca46..765f0f32e 100644 --- a/tools/deployment-cli-tools/tests/test_skaffold_internals.py +++ b/tools/deployment-cli-tools/tests/test_skaffold_internals.py @@ -7,7 +7,11 @@ from test_util import write_tree, write_tree_from_yaml from ch_cli_tools import CHDeployConfig, skaffold -from ch_cli_tools.utils import dockerfile_variable_reference, parse_dockerfile +from ch_cli_tools.dockerfile import DockerfileParser + + +def _instructions(path): + return [i.fields for i in DockerfileParser(path).parse().instructions] def test_build_dependency_on_app_produces_requires_entry(): @@ -492,12 +496,12 @@ def test_app_entrypoint_override_and_unit_tests_reach_skaffold_output(): def test_parse_dockerfile_missing_file_returns_empty_list(tmp_path): - assert parse_dockerfile(tmp_path / "Dockerfile") == [] + assert _instructions(tmp_path / "Dockerfile") == [] def test_parse_dockerfile_skips_comments_and_blank_lines(tmp_path): write_tree(tmp_path, {"Dockerfile": "# a comment\n\nFROM scratch\n\n# another\n"}) - assert parse_dockerfile(tmp_path / "Dockerfile") == [("FROM", "scratch")] + assert _instructions(tmp_path / "Dockerfile") == [("FROM", "scratch")] def test_parse_dockerfile_from_with_and_without_alias(tmp_path): @@ -507,7 +511,7 @@ def test_parse_dockerfile_from_with_and_without_alias(tmp_path): "Dockerfile": "FROM python:3.12 AS builder\nFROM --platform=linux/amd64 scratch\n", }, ) - assert parse_dockerfile(tmp_path / "Dockerfile") == [ + assert _instructions(tmp_path / "Dockerfile") == [ ("FROM", "python:3.12", "builder"), ("FROM", "scratch"), ] @@ -517,7 +521,7 @@ def test_parse_dockerfile_from_normalizes_variable_references(tmp_path): write_tree( tmp_path, {"Dockerfile": "FROM $BASE AS one\nFROM ${BASE}\nFROM python:3.12\n"} ) - assert parse_dockerfile(tmp_path / "Dockerfile") == [ + assert _instructions(tmp_path / "Dockerfile") == [ ("FROM", "BASE", "one"), ("FROM", "BASE"), ("FROM", "python:3.12"), # a literal image is left untouched @@ -526,7 +530,7 @@ def test_parse_dockerfile_from_normalizes_variable_references(tmp_path): def test_parse_dockerfile_arg_with_and_without_default(tmp_path): write_tree(tmp_path, {"Dockerfile": "ARG NO_DEFAULT\nARG WITH_DEFAULT=value\n"}) - assert parse_dockerfile(tmp_path / "Dockerfile") == [ + assert _instructions(tmp_path / "Dockerfile") == [ ("ARG", "NO_DEFAULT"), ("ARG", "WITH_DEFAULT", "value"), ] @@ -537,7 +541,7 @@ def test_parse_dockerfile_exec_form_vs_shell_form(tmp_path): tmp_path, {"Dockerfile": 'CMD ["gunicorn", "app:app"]\nRUN pip install gunicorn\n'}, ) - assert parse_dockerfile(tmp_path / "Dockerfile") == [ + assert _instructions(tmp_path / "Dockerfile") == [ ("CMD", "gunicorn", "app:app"), ("RUN", "pip", "install", "gunicorn"), ] @@ -548,14 +552,14 @@ def test_parse_dockerfile_line_continuation(tmp_path): tmp_path, {"Dockerfile": "RUN apt-get update && \\\n apt-get install -y curl\n"}, ) - assert parse_dockerfile(tmp_path / "Dockerfile") == [ + assert _instructions(tmp_path / "Dockerfile") == [ ("RUN", "apt-get", "update", "&&", "apt-get", "install", "-y", "curl") ] def test_parse_dockerfile_copy_strips_leading_flags_only(tmp_path): write_tree(tmp_path, {"Dockerfile": "COPY --from=builder /app/dist /app/dist\n"}) - assert parse_dockerfile(tmp_path / "Dockerfile") == [ + assert _instructions(tmp_path / "Dockerfile") == [ ("COPY", "/app/dist", "/app/dist") ] @@ -564,7 +568,7 @@ def test_parse_dockerfile_command_is_case_insensitive(): # Real fixture Dockerfile, already all-uppercase - locks in that we don't # accidentally rely on case for the common path while testing case # insensitivity synthetically below. - instructions = parse_dockerfile(RESOURCES / "applications" / "myapp" / "Dockerfile") + instructions = _instructions(RESOURCES / "applications" / "myapp" / "Dockerfile") assert instructions == [ ("ARG", "CLOUDHARNESS_FLASK"), # $NAME is normalized to the bare NAME at parse time - see @@ -575,17 +579,17 @@ def test_parse_dockerfile_command_is_case_insensitive(): def test_parse_dockerfile_lowercase_instructions_are_normalized(tmp_path): write_tree(tmp_path, {"Dockerfile": "from scratch\n"}) - assert parse_dockerfile(tmp_path / "Dockerfile") == [("FROM", "scratch")] + assert _instructions(tmp_path / "Dockerfile") == [("FROM", "scratch")] def test_dockerfile_variable_reference_dollar_and_braces_are_equivalent(): - assert dockerfile_variable_reference("$NAME") == "NAME" - assert dockerfile_variable_reference("${NAME}") == "NAME" + assert DockerfileParser.variable_reference("$NAME") == "NAME" + assert DockerfileParser.variable_reference("${NAME}") == "NAME" def test_dockerfile_variable_reference_none_for_a_literal(): - assert dockerfile_variable_reference("python:3.12") is None - assert dockerfile_variable_reference("scratch") is None + assert DockerfileParser.variable_reference("python:3.12") is None + assert DockerfileParser.variable_reference("scratch") is None def test_git_dependencies_exposes_raw_config(): diff --git a/tools/deployment-cli-tools/tests/test_task_commands.py b/tools/deployment-cli-tools/tests/test_task_commands.py new file mode 100644 index 000000000..8c467773c --- /dev/null +++ b/tools/deployment-cli-tools/tests/test_task_commands.py @@ -0,0 +1,721 @@ +"""Task commands derived from the Dockerfiles, from parsing to the Argo task that uses them. + +Argo's emissary executor reads an image's ENTRYPOINT/CMD from the registry when a container has +no command, which needs registry credentials. The commands are derived from the task Dockerfiles +at generation time and carried in the generated values instead. +""" + +import logging +from pathlib import Path + +import pytest +from conftest import CLOUDHARNESS_ROOT, chain +from test_util import write_tree + +from ch_cli_tools import CHDeployConfig +from ch_cli_tools.api import create_helm_chart +from ch_cli_tools.dockerfile import DockerfileParser +from ch_cli_tools.model import CHDockerfile +from cloudharness.utils.config import CloudharnessConfig +from cloudharness.workflows import operations, tasks +from cloudharness_model import HarnessMainConfig + + +def _instructions(path): + return [i.fields for i in DockerfileParser(path).parse().instructions] + + +# Schema + + +def _main_config(**extra): + return HarnessMainConfig.from_dict( + { + "local": False, + "secured_gatekeepers": False, + "domain": "my.local", + "namespace": "test", + "mainapp": "myapp", + "apps": {}, + **extra, + } + ) + + +def test_main_config_model_carries_task_commands(): + config = _main_config(**{"task-commands": {"t": ["a", "b"]}}) + + assert config.task_commands == {"t": ["a", "b"]} + assert config.to_dict()["task-commands"] == {"t": ["a", "b"]} + + +def test_main_config_model_does_not_require_task_commands(): + config = _main_config() + + assert config.task_commands is None + + +# Dockerfile reading + + +def _project(tmp_path: Path, tasks: dict[str, str], app_dockerfile: str = "FROM scratch\n"): + """A project with one app, myapp, whose tasks have the given Dockerfile contents.""" + write_tree( + tmp_path, + { + "deployment-configuration": {"values-template.yaml": "name: testproj\n"}, + "applications": { + "myapp": { + "Dockerfile": app_dockerfile, + "tasks": {name: {"Dockerfile": content} for name, content in tasks.items()}, + } + }, + }, + ) + return chain(tmp_path, tmp_path, config=CHDeployConfig()) + + +def _dockerfile(tmp_path: Path, content: str) -> CHDockerfile: + """The Dockerfile of a task, mytask, of a project with one app, myapp.""" + return _project(tmp_path, {"mytask": content}).all_buildable_tasks()["myapp-mytask"].dockerfile + + +def test_parse_dockerfile_still_joins_continuations_and_skips_comments(tmp_path): + path = tmp_path / "Dockerfile" + path.write_text("# comment\nFROM scratch\nRUN a \\\n b\n") + + assert _instructions(path) == [("FROM", "scratch"), ("RUN", "a", "b")] + + +def test_exec_form_cmd_is_used_as_written(tmp_path): + dockerfile = _dockerfile(tmp_path, 'FROM scratch\nCMD ["./run.sh", "--fast"]\n') + + assert dockerfile.cmd == ["./run.sh", "--fast"] + assert dockerfile.command == ["./run.sh", "--fast"] + + +def test_shell_form_cmd_is_wrapped_with_the_raw_string(tmp_path): + dockerfile = _dockerfile(tmp_path, 'FROM scratch\nCMD python a.py "$X" --opt b\n') + + assert dockerfile.command == ["/bin/sh", "-c", 'python a.py "$X" --opt b'] + + +def test_shell_form_cmd_with_line_continuation_is_joined(tmp_path): + dockerfile = _dockerfile(tmp_path, "FROM scratch\nCMD python a.py \\\n --opt b\n") + + assert dockerfile.command == ["/bin/sh", "-c", "python a.py --opt b"] + + +def test_invalid_json_cmd_is_shell_form(tmp_path): + dockerfile = _dockerfile(tmp_path, "FROM scratch\nCMD [not json\n") + + assert dockerfile.command == ["/bin/sh", "-c", "[not json"] + + +def test_entrypoint_is_followed_by_cmd(tmp_path): + dockerfile = _dockerfile(tmp_path, 'FROM scratch\nENTRYPOINT ["python"]\nCMD ["a.py"]\n') + + assert dockerfile.entrypoint == ["python"] + assert dockerfile.cmd == ["a.py"] + assert dockerfile.command == ["python", "a.py"] + + +def test_last_cmd_and_last_entrypoint_win(tmp_path): + dockerfile = _dockerfile( + tmp_path, 'FROM scratch\nENTRYPOINT ["a"]\nENTRYPOINT ["b"]\nCMD ["c"]\nCMD ["d"]\n' + ) + + assert dockerfile.command == ["b", "d"] + + +def test_dockerfile_without_command_has_an_empty_command(tmp_path): + dockerfile = _dockerfile(tmp_path, "FROM scratch\nRUN true\n") + + assert dockerfile.entrypoint is None + assert dockerfile.cmd is None + assert dockerfile.command == [] + + +def test_missing_dockerfile_has_an_empty_command(tmp_path): + dockerfile = _dockerfile(tmp_path, "FROM scratch\n") + (dockerfile.path).unlink() + + assert dockerfile.command == [] + + +def test_only_the_final_stage_counts(tmp_path): + dockerfile = _dockerfile( + tmp_path, + 'FROM a AS build\nCMD ["build-cmd"]\nFROM scratch\nCOPY --from=build /x /x\n', + ) + + assert dockerfile.command == [] + + +def test_stage_built_from_an_earlier_stage_keeps_its_command(tmp_path): + dockerfile = _dockerfile( + tmp_path, + 'FROM a AS base\nENTRYPOINT ["e"]\nCMD ["c"]\nFROM base AS final\nRUN true\n', + ) + + assert dockerfile.command == ["e", "c"] + + +def test_stage_built_from_an_earlier_stage_can_override_its_cmd(tmp_path): + dockerfile = _dockerfile( + tmp_path, + 'FROM a AS base\nENTRYPOINT ["e"]\nCMD ["c"]\nFROM base\nCMD ["own"]\n', + ) + + assert dockerfile.command == ["e", "own"] + + +def test_entrypoint_of_a_stage_clears_the_cmd_of_the_stage_it_is_built_from(tmp_path): + dockerfile = _dockerfile( + tmp_path, + 'FROM a AS base\nCMD ["a"]\nFROM base\nENTRYPOINT ["b"]\n', + ) + + assert dockerfile.command == ["b"] + + +def test_entrypoint_after_cmd_in_the_same_stage_keeps_the_cmd(tmp_path): + dockerfile = _dockerfile(tmp_path, 'FROM a\nCMD ["c"]\nENTRYPOINT ["e"]\n') + + assert dockerfile.command == ["e", "c"] + + +def test_stage_names_are_case_insensitive(tmp_path): + dockerfile = _dockerfile( + tmp_path, + 'FROM a AS Builder\nCMD ["c"]\nFROM builder\nRUN true\n', + ) + + assert dockerfile.command == ["c"] + assert dockerfile.external_base == "a" + + +def test_shell_instruction_changes_the_wrapper_of_shell_form_commands(tmp_path): + dockerfile = _dockerfile( + tmp_path, + 'FROM a\nSHELL ["/bin/bash", "-c"]\nCMD echo hi\n', + ) + + assert dockerfile.command == ["/bin/bash", "-c", "echo hi"] + + +def test_dockerfile_not_starting_with_from_has_no_base(tmp_path): + dockerfile = _dockerfile(tmp_path, "ADD x /x\n") + + assert dockerfile.base_dockerfile is None + assert dockerfile.external_base is None + + +def test_dockerfile_with_a_bare_instruction_has_no_base(tmp_path): + dockerfile = _dockerfile(tmp_path, "RUN\n") + + assert dockerfile.external_base is None + assert dockerfile.command == [] + + +# Chain following + + +def _task_dockerfile(project, name: str) -> CHDockerfile: + return project.all_buildable_tasks()[f"myapp-{name}"].dockerfile + + +def test_command_is_inherited_from_a_project_task(tmp_path): + project = _project( + tmp_path, + { + "base": 'FROM python:3.14\nENTRYPOINT ["python"]\nCMD ["-V"]\n', + "child": "ARG MYAPP_BASE\nFROM $MYAPP_BASE\nRUN true\n", + }, + ) + + child = _task_dockerfile(project, "child") + + assert child.entrypoint == ["python"] + assert child.cmd == ["-V"] + assert child.command == ["python", "-V"] + + +def test_command_is_inherited_through_several_project_images(tmp_path): + project = _project( + tmp_path, + { + "root": 'FROM python:3.14\nENTRYPOINT ["e"]\nCMD ["c"]\n', + "middle": "ARG MYAPP_ROOT\nFROM $MYAPP_ROOT\nRUN true\n", + "leaf": "ARG MYAPP_MIDDLE\nFROM ${MYAPP_MIDDLE}\nRUN true\n", + }, + ) + + assert _task_dockerfile(project, "leaf").command == ["e", "c"] + + +def test_command_is_inherited_from_an_app_image(tmp_path): + project = _project( + tmp_path, + {"child": "ARG MYAPP\nFROM $MYAPP\nRUN true\n"}, + app_dockerfile='FROM python:3.14\nCMD ["serve"]\n', + ) + + assert _task_dockerfile(project, "child").command == ["serve"] + + +def test_own_cmd_replaces_the_inherited_cmd_and_keeps_the_entrypoint(tmp_path): + project = _project( + tmp_path, + { + "base": 'FROM python:3.14\nENTRYPOINT ["e"]\nCMD ["c"]\n', + "child": 'ARG MYAPP_BASE\nFROM $MYAPP_BASE\nCMD ["own"]\n', + }, + ) + + assert _task_dockerfile(project, "child").command == ["e", "own"] + + +def test_own_entrypoint_clears_the_inherited_cmd(tmp_path): + project = _project( + tmp_path, + { + "base": 'FROM python:3.14\nENTRYPOINT ["e"]\nCMD ["c"]\n', + "child": 'ARG MYAPP_BASE\nFROM $MYAPP_BASE\nENTRYPOINT ["own"]\n', + }, + ) + + assert _task_dockerfile(project, "child").command == ["own"] + + +def test_own_entrypoint_and_cmd_replace_both_inherited_ones(tmp_path): + project = _project( + tmp_path, + { + "base": 'FROM python:3.14\nENTRYPOINT ["e"]\nCMD ["c"]\n', + "child": 'ARG MYAPP_BASE\nFROM $MYAPP_BASE\nENTRYPOINT ["own"]\nCMD ["arg"]\n', + }, + ) + + assert _task_dockerfile(project, "child").command == ["own", "arg"] + + +def test_base_dockerfile_is_the_project_image_the_dockerfile_is_built_from(tmp_path): + project = _project( + tmp_path, + { + "base": "FROM python:3.14\n", + "child": "ARG MYAPP_BASE\nFROM $MYAPP_BASE\n", + }, + ) + + assert _task_dockerfile(project, "child").base_dockerfile.path == _task_dockerfile(project, "base").path + + +def test_base_dockerfile_is_found_through_a_stage_of_the_same_file(tmp_path): + project = _project( + tmp_path, + { + "base": 'FROM python:3.14\nCMD ["c"]\n', + "child": "ARG MYAPP_BASE\nFROM $MYAPP_BASE AS build\nFROM build\nRUN true\n", + }, + ) + + child = _task_dockerfile(project, "child") + + assert child.base_dockerfile.path == _task_dockerfile(project, "base").path + assert child.command == ["c"] + + +def test_external_base_through_an_arg_has_no_base_dockerfile(tmp_path): + dockerfile = _dockerfile(tmp_path, 'ARG PYTHON=python:3.14\nFROM $PYTHON\nCMD ["a"]\n') + + assert dockerfile.base_dockerfile is None + assert dockerfile.command == ["a"] + + +def test_literal_from_is_external_even_when_named_like_a_project_image(tmp_path): + project = _project( + tmp_path, + { + "base": 'FROM python:3.14\nCMD ["c"]\n', + "child": "FROM myapp-base\nRUN true\n", + }, + ) + + child = _task_dockerfile(project, "child") + + assert child.base_dockerfile is None + assert child.command == [] + + +def test_base_dependencies_still_lists_the_project_images_referenced_via_arg(tmp_path): + project = _project( + tmp_path, + { + "base": "FROM python:3.14\n", + "child": "ARG MYAPP_BASE\nARG OTHER\nFROM $MYAPP_BASE\nFROM $OTHER\n", + }, + ) + + dependencies = _task_dockerfile(project, "child").base_dependencies + + assert dependencies[0].path == _task_dockerfile(project, "base").path + assert dependencies[1] == "other" + + +def test_project_finds_an_app_or_a_task_by_name(tmp_path): + project = _project(tmp_path, {"mytask": "FROM scratch\n"}) + + assert project.find_entity("myapp").name == "myapp" + assert project.find_entity("myapp-mytask").name == "myapp-mytask" + assert project.find_entity("unknown") is None + + +def test_dockerfiles_built_from_each_other_are_reported_and_do_not_crash(tmp_path, caplog): + project = _project( + tmp_path, + { + "a": 'ARG MYAPP_B\nFROM $MYAPP_B\nCMD ["a"]\n', + "b": 'ARG MYAPP_A\nFROM $MYAPP_A\nCMD ["b"]\n', + }, + ) + + with caplog.at_level(logging.ERROR): + command = _task_dockerfile(project, "a").command + + assert command == ["a"] + assert "built FROM itself" in caplog.text + + +def test_dockerfile_built_from_itself_is_reported_and_does_not_crash(tmp_path, caplog): + project = _project(tmp_path, {"a": 'ARG MYAPP_A\nFROM $MYAPP_A\nCMD ["a"]\n'}) + + with caplog.at_level(logging.ERROR): + command = _task_dockerfile(project, "a").command + + assert command == ["a"] + assert "built FROM itself" in caplog.text + + +def test_dockerfile_leading_into_a_loop_keeps_its_base_and_names_only_the_loop(tmp_path, caplog): + project = _project( + tmp_path, + { + "a": "ARG MYAPP_B\nFROM $MYAPP_B\nRUN true\n", + "b": 'ARG MYAPP_C\nFROM $MYAPP_C\nCMD ["b"]\n', + "c": 'ARG MYAPP_B\nFROM $MYAPP_B\n', + }, + ) + + dockerfile = _task_dockerfile(project, "a") + with caplog.at_level(logging.ERROR): + command = dockerfile.command + + assert command == ["b"] + assert dockerfile.base_dockerfile is not None + assert f"Dockerfile {dockerfile.path} is built FROM" not in caplog.text + assert "built FROM itself" in caplog.text + + +def test_shell_of_a_project_base_image_wraps_the_shell_form_command_of_its_child(tmp_path): + project = _project( + tmp_path, + { + "base": 'FROM python:3.14\nSHELL ["/bin/bash", "-c"]\n', + "child": "ARG MYAPP_BASE\nFROM $MYAPP_BASE\nCMD source env.sh && run\n", + }, + ) + + assert _task_dockerfile(project, "child").command == ["/bin/bash", "-c", "source env.sh && run"] + + +# Generation + + +def test_task_commands_are_derived_for_every_task_image(tmp_path): + project = _project( + tmp_path, + { + "exec": 'FROM python:3.14\nCMD ["./run.sh"]\n', + "shell": "FROM python:3.14\nCMD python job.py\n", + "base": 'FROM python:3.14\nENTRYPOINT ["python"]\n', + "child": 'ARG MYAPP_BASE\nFROM $MYAPP_BASE\nCMD ["job.py"]\n', + }, + ) + + assert project.all_task_commands() == { + "myapp-exec": ["./run.sh"], + "myapp-shell": ["/bin/sh", "-c", "python job.py"], + "myapp-base": ["python"], + "myapp-child": ["python", "job.py"], + } + + +def test_task_commands_are_named_like_the_task_images(tmp_path): + project = _project(tmp_path, {"job": 'FROM python:3.14\nCMD ["x"]\n'}) + + assert set(project.all_task_commands()) <= set(project.all_task_images()) + + +def test_task_without_any_command_is_left_out_with_a_warning(tmp_path, caplog): + project = _project(tmp_path, {"silent": "FROM scratch\nRUN true\n"}) + + with caplog.at_level(logging.WARNING): + commands = project.all_task_commands() + + assert "myapp-silent" not in commands + assert "Task myapp-silent sets no ENTRYPOINT or CMD" in caplog.text + + +def test_base_image_without_a_command_is_left_out_silently(tmp_path, caplog): + write_tree( + tmp_path, + { + "infrastructure": {"base-images": {"mybase": {"Dockerfile": "FROM python:3.14\n"}}}, + }, + ) + project = _project( + tmp_path, + {"job": 'FROM python:3.14\nCMD ["x"]\n'}, + app_dockerfile="ARG MYBASE\nFROM $MYBASE\n", + ) + + with caplog.at_level(logging.WARNING): + commands = project.all_task_commands() + + assert "mybase" in project.all_task_images() + assert "mybase" not in commands + assert "mybase sets no" not in caplog.text + + +def test_external_bases_the_commands_rely_on_are_reported_in_one_warning(tmp_path, caplog): + project = _project( + tmp_path, + { + "a": 'FROM python:3.14\nCMD ["a"]\n', + "b": 'ARG NODE=node:22\nFROM $NODE\nCMD ["b"]\n', + "own": 'FROM alpine:3\nENTRYPOINT ["own"]\n', + }, + ) + + with caplog.at_level(logging.WARNING): + project.all_task_commands() + + warnings = [r.message for r in caplog.records if "external base images" in r.message] + assert len(warnings) == 1 + assert "node:22" in warnings[0] and "python:3.14" in warnings[0] + assert "alpine:3" not in warnings[0] + + +def test_external_base_is_where_the_chain_of_froms_ends(tmp_path): + project = _project( + tmp_path, + { + "base": "FROM python:3.14\n", + "child": "ARG MYAPP_BASE\nFROM $MYAPP_BASE\n", + "arg": "ARG PYTHON=python:3.12\nFROM $PYTHON\n", + }, + ) + + assert _task_dockerfile(project, "child").external_base == "python:3.14" + assert _task_dockerfile(project, "arg").external_base == "python:3.12" + + +def test_generated_values_carry_task_commands_beside_task_images(tmp_path): + solution = tmp_path / "solution" + write_tree( + solution, + { + "deployment-configuration": {"values-template.yaml": "name: testproj\n"}, + "applications": { + "myapp": { + "Dockerfile": "FROM scratch\n", + "deploy": {"values.yaml": "harness:\n subdomain: myapp\n"}, + "tasks": { + "job": {"Dockerfile": "FROM python:3.14\nCMD python job.py\n"}, + "plain": {"Dockerfile": 'FROM scratch\nCMD ["x"]\n'}, + }, + } + }, + }, + ) + + values = create_helm_chart( + [CLOUDHARNESS_ROOT, solution], + output_path=tmp_path / "out", + domain="my.local", + namespace="test", + local=False, + tag="1", + registry="reg", + include=["myapp"], + ).to_dict() + + assert values["task-commands"]["myapp-job"] == ["/bin/sh", "-c", "python job.py"] + assert values["task-commands"]["myapp-plain"] == ["x"] + assert set(values["task-commands"]) <= set(values["task-images"]) + + +# Runtime + + +@pytest.fixture +def ch_config(monkeypatch): + """Installs a Cloud Harness configuration for the test, with a registry and the given values.""" + + def install(**values): + allvalues = { + "registry": {"name": "reg"}, + "apps": {}, + "task-images": {"job": "reg/job:1", "workflows-notify-queue": "reg/notify:1"}, + **values, + } + monkeypatch.setattr(CloudharnessConfig, "allvalues", allvalues) + return allvalues + + return install + + +def test_get_task_command_returns_the_derived_command(ch_config): + ch_config(**{"task-commands": {"job": ["./run.sh"]}}) + + assert CloudharnessConfig.get_task_command("job") == ["./run.sh"] + + +@pytest.mark.parametrize("values", [{"task-commands": {"other": ["x"]}}, {"task-commands": None}, {}]) +def test_get_task_command_is_none_when_there_is_none_for_the_image(ch_config, values): + ch_config(**values) + + assert CloudharnessConfig.get_task_command("job") is None + + +def test_get_task_command_is_none_without_any_configuration(monkeypatch): + monkeypatch.setattr(CloudharnessConfig, "allvalues", {}) + monkeypatch.setattr("cloudharness.utils.config.ALLVALUES_PATH", "/nonexistent/allvalues.yaml") + + assert CloudharnessConfig.get_task_command("job") is None + + +def test_custom_task_uses_the_derived_command_by_default(ch_config): + ch_config(**{"task-commands": {"job": ["./run.sh"]}}) + + spec = tasks.CustomTask("t", "job").spec() + + assert spec["container"]["command"] == ["./run.sh"] + + +def test_custom_task_explicit_command_wins(ch_config): + ch_config(**{"task-commands": {"job": ["./run.sh"]}}) + + spec = tasks.CustomTask("t", "job", command=["explicit"]).spec() + + assert spec["container"]["command"] == ["explicit"] + + +def test_custom_task_keeps_the_command_of_its_template_overrides(ch_config): + ch_config(**{"task-commands": {"job": ["./run.sh"]}}) + overrides = tasks.V1alpha1Template(container={"command": ["mine"]}) + + spec = tasks.CustomTask("t", "job", template_overrides=overrides).spec() + + assert spec["container"]["command"] == ["mine"] + + +def test_custom_task_keeps_the_args_of_its_template_overrides_without_a_derived_command(ch_config): + ch_config(**{"task-commands": {"job": ["./run.sh"]}}) + overrides = tasks.V1alpha1Template(container={"args": ["--fast"]}) + + spec = tasks.CustomTask("t", "job", template_overrides=overrides).spec() + + assert "command" not in spec["container"] + assert spec["container"]["args"] == ["--fast"] + + +def test_custom_task_still_uses_the_derived_command_with_other_template_overrides(ch_config): + ch_config(**{"task-commands": {"job": ["./run.sh"]}}) + overrides = tasks.V1alpha1Template(container={"workingDir": "/work"}) + + spec = tasks.CustomTask("t", "job", template_overrides=overrides).spec() + + assert spec["container"]["command"] == ["./run.sh"] + + +def test_custom_task_without_a_derived_command_sets_none(ch_config): + ch_config() + + spec = tasks.CustomTask("t", "job").spec() + + assert "command" not in spec["container"] + + +def _exit_template(workflow): + return next(t for t in workflow["spec"]["templates"] if t["name"] == "exit-handler") + + +def test_exit_handler_uses_the_derived_command_by_default(ch_config): + ch_config(**{"task-commands": {"workflows-notify-queue": ["notify"]}}) + + operation = operations.PipelineOperation( + "wf", [tasks.CustomTask("t", "job")], on_exit_notify={"queue": "q"} + ) + + assert _exit_template(operation.to_workflow())["container"]["command"] == ["notify"] + + +def test_exit_handler_explicit_command_wins(ch_config): + ch_config(**{"task-commands": {"workflows-notify-queue": ["notify"]}}) + + operation = operations.PipelineOperation( + "wf", [tasks.CustomTask("t", "job")], on_exit_notify={"queue": "q", "command": ["explicit"]} + ) + + assert _exit_template(operation.to_workflow())["container"]["command"] == ["explicit"] + + +# End to end + + +def _generate_values(tmp_path: Path, tasks: dict[str, str]) -> dict: + """The values harness-deployment generates for a project with one app, myapp, and the given tasks.""" + solution = tmp_path / "solution" + write_tree( + solution, + { + "deployment-configuration": {"values-template.yaml": "name: testproj\n"}, + "applications": { + "myapp": { + "Dockerfile": "FROM scratch\n", + "deploy": {"values.yaml": "harness:\n subdomain: myapp\n"}, + "tasks": {name: {"Dockerfile": content} for name, content in tasks.items()}, + } + }, + }, + ) + return create_helm_chart( + [CLOUDHARNESS_ROOT, solution], + output_path=tmp_path / "out", + domain="my.local", + namespace="test", + local=False, + tag="1", + registry="reg", + include=["myapp"], + ).to_dict() + + +def test_generated_command_reaches_the_argo_workflow(tmp_path, monkeypatch): + values = _generate_values( + tmp_path, + { + "base": 'FROM python:3.14\nENTRYPOINT ["python"]\n', + "child": 'ARG MYAPP_BASE\nFROM $MYAPP_BASE\nCMD ["job.py"]\n', + }, + ) + monkeypatch.setattr(CloudharnessConfig, "allvalues", values) + + workflow = operations.PipelineOperation("wf", [tasks.CustomTask("run", "myapp-child")]).to_workflow() + + template = next(t for t in workflow["spec"]["templates"] if t["name"] == "run") + assert template["container"]["command"] == ["python", "job.py"] + assert template["container"]["image"] == values["task-images"]["myapp-child"]