From 11887c01ef1c0ac8e364dd9e05db844058a7ec0c Mon Sep 17 00:00:00 2001 From: Simon Halvorsen Date: Tue, 10 Nov 2026 10:12:41 +0100 Subject: [PATCH 1/4] Added `cfengine test` i2, now reports a test-suite using the `assert`-promise module Runs test_*.cf files or a deployed policy's own assert: promises (or just a plain built-policy run) in disposable Docker container and reports a pytest-style pass/fail summary. In a cfbs project, each test file runs at the end of the project's real promises.cf (with its fixture applied); --test-only runs just the test bundles from a minimal generated policy instead. Adds --dockerfile/--rebuild flags for testing against a custom base image. Ticket: ENT-14444 Co-Authored-By: Claude Code Signed-off-by: Simon Halvorsen --- pyproject.toml | 2 +- .../cfengine_wrapper/cfengine_commands.py | 50 +- src/cfengine_cli/commands.py | 20 +- src/cfengine_cli/container.py | 693 +++++++++++++++++- src/cfengine_cli/docker/test-agent/Dockerfile | 15 +- .../test-agent/assert-module/promise_agent.cf | 5 + src/cfengine_cli/main.py | 45 +- src/cfengine_cli/report.py | 197 +++++ tests/unit/test_container.py | 452 ++++++++++++ tests/unit/test_report.py | 173 +++++ 10 files changed, 1620 insertions(+), 32 deletions(-) create mode 100644 src/cfengine_cli/docker/test-agent/assert-module/promise_agent.cf create mode 100644 src/cfengine_cli/report.py create mode 100644 tests/unit/test_container.py create mode 100644 tests/unit/test_report.py diff --git a/pyproject.toml b/pyproject.toml index 076c573..1dc605b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ cfengine = "cfengine_cli.main:main" license-files = [] # Workaround bug in setuptools https://github.com/astral-sh/uv/issues/9513 [tool.setuptools.package-data] -cfengine_cli = ["*.json", "docker/test-agent/Dockerfile"] # syntax-description.json, cfengine test's container +cfengine_cli = ["*.json", "docker/test-agent/Dockerfile", "docker/test-agent/assert-module/*"] # syntax-description.json, cfengine test's container (+ assert promise declaration it COPYs and lints with) [tool.pyright] include = ["src"] diff --git a/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py b/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py index a50c88b..5830af3 100644 --- a/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py +++ b/src/cfengine_cli/cfengine_wrapper/cfengine_commands.py @@ -24,7 +24,12 @@ ) from cfengine_cli.utils import UserError -from cfengine_cli.container import run_in_container +from cfengine_cli.container import ( + run_in_container, + run_files_in_container, + discover_test_files, + discover_module_files, +) from cfengine_cli.cfengine_wrapper.cfengine_objects import ( Executable, ensure_default_agent_flags, @@ -320,12 +325,45 @@ def deploy( return error -def test() -> int: - rc = build_command() - if rc != 0: - return rc +def test( + files: list[str] | None = None, + masterfiles_dir: str | None = None, + dockerfile: str | None = None, + rebuild: bool = False, + test_only: bool = False, +) -> int: + """Runs the tests, after container.prepare_test_masterfiles (and lint) + -- see there for masterfiles_dir.""" + + if files: + # Module files are libraries whether tests are auto-discovered or + # named explicitly, so a test sees the same bundles either way. + return run_files_in_container( + discover_module_files() + files, + masterfiles_dir=masterfiles_dir, + dockerfile=dockerfile, + rebuild=rebuild, + test_only=test_only, + ) + + discovered = discover_test_files() + if discovered: + return run_files_in_container( + discover_module_files() + discovered, + masterfiles_dir=masterfiles_dir, + dockerfile=dockerfile, + rebuild=rebuild, + test_only=test_only, + ) + + if masterfiles_dir is not None: + return run_in_container(masterfiles_dir, dockerfile=dockerfile, rebuild=rebuild) - return run_in_container("out/masterfiles") + raise UserError( + "Nothing to test here -- no cfbs.json project and no test_*.cf files " + "found. Add a test_*.cf file, or run `cfengine init` to start a cfbs " + "project." + ) def show(target: list[str] | None = None) -> int: diff --git a/src/cfengine_cli/commands.py b/src/cfengine_cli/commands.py index 2281576..2b6cf83 100644 --- a/src/cfengine_cli/commands.py +++ b/src/cfengine_cli/commands.py @@ -14,6 +14,10 @@ init_policy_set, init_promise_type, ) +from cfengine_cli.container import ( + assert_module_declaration_path, + prepare_test_masterfiles, +) from cf_remote.paths import cf_remote_dir from cfengine_cli.cfengine_wrapper.cfengine_commands import test as _cfengine_test from pydantic import ValidationError @@ -51,15 +55,25 @@ def lint(files, strict, syntax_path) -> int: return errors -def test(files, strict) -> int: - errors = _lint(files, strict, None) +def test( + files, strict, dockerfile=None, rebuild=False, test_only=False, masterfiles=None +) -> int: + # Adds the assert module so linting passes + lint_targets = (list(files) if files else ["."]) + [ + assert_module_declaration_path() + ] + + errors = _lint(lint_targets, strict, None) if errors != 0: plural = "error" if errors == 1 else "errors" print(f"Lint failed, {errors} {plural} in total. Skipping build/deploy/run.") return errors print("Lint passed, no errors found.") - return _cfengine_test() + rc, masterfiles_dir = prepare_test_masterfiles(masterfiles) + if rc != 0: + return rc + return _cfengine_test(files, masterfiles_dir, dockerfile, rebuild, test_only) def dev(subcommand, args) -> int: diff --git a/src/cfengine_cli/container.py b/src/cfengine_cli/container.py index 0f9a529..a9ee61b 100644 --- a/src/cfengine_cli/container.py +++ b/src/cfengine_cli/container.py @@ -1,11 +1,40 @@ +import glob +import hashlib +import json import os +import re import shutil import subprocess +import tempfile +import uuid +from typing import NamedTuple +from cfbs.commands import build_command +from cfbs.utils import is_cfbs_repo, merge_json, read_json + +from cfengine_cli import paths +from cfengine_cli.report import ( + RunResults, + run_and_parse, + print_report, + run_and_scan_asserts, + print_assert_totals, +) from cfengine_cli.utils import UserError _DOCKERFILE_DIR = os.path.join(os.path.dirname(__file__), "docker", "test-agent") _IMAGE_TAG = "cfengine-cli-test-agent:latest" +_ASSERT_MODULE_DIR = "/var/cfengine/modules/promises" +_STDLIB = "/var/cfengine/masterfiles/lib/stdlib.cf" +_PROMISE_AGENT_DECLARATION_RE = re.compile(r"promise\s+agent\s+(\w+)") +_BUNDLE_AGENT_DECLARATION_RE = re.compile(r"bundle\s+agent\s+(\w+)") +_ASSERT_SECTION_RE = re.compile(r"^\s*assert\s*:", re.MULTILINE) + + +def assert_module_declaration_path() -> str: + """Path to the `promise agent assert {...}` declaration + so calling lint on files containing the assert-promise-type will pass""" + return os.path.join(_DOCKERFILE_DIR, "assert-module", "promise_agent.cf") def require_docker() -> None: @@ -15,38 +44,664 @@ def require_docker() -> None: ) -def _ensure_image_built() -> None: +def _resolve_dockerfile(dockerfile: str | None) -> tuple: + """--dockerfile may point at a Dockerfile itself or at the directory + containing one; returns (build_context_dir, dockerfile_path) for + `docker build -f `, so a + non-default name like Dockerfile.rhel is really the one that's built. + None => use the built-in Dockerfile.""" + if dockerfile is None: + return _DOCKERFILE_DIR, os.path.join(_DOCKERFILE_DIR, "Dockerfile") + abs_path = os.path.abspath(dockerfile) + if os.path.isfile(abs_path): + return os.path.dirname(abs_path), abs_path + return abs_path, os.path.join(abs_path, "Dockerfile") + + +def _image_tag_for(dockerfile_path: str) -> str: + """The built-in Dockerfile keeps the stable, human-readable tag; any + other --dockerfile gets its own tag derived from its path, as to not + conflict with the cached image (or with a sibling Dockerfile. in + the same directory).""" + abs_path = os.path.abspath(dockerfile_path) + if abs_path == os.path.abspath(os.path.join(_DOCKERFILE_DIR, "Dockerfile")): + return _IMAGE_TAG + digest = hashlib.sha256(abs_path.encode()).hexdigest()[:12] + return f"cfengine-cli-test-agent-custom-{digest}:latest" + + +def _ensure_image_built( + context_dir: str, dockerfile_path: str, image_tag: str, rebuild: bool +) -> None: + cmd = ["docker", "build", "-q", "-t", image_tag, "-f", dockerfile_path] + if rebuild: + cmd.append("--no-cache") + cmd.append(context_dir) + result = subprocess.run(cmd) + if result.returncode != 0: + raise UserError("Failed to build the cfengine-test Docker image.") + + +def _docker_cleanup(*args: str) -> None: + """Best-effort `docker ` for rm/rmi cleanup.""" + subprocess.run(["docker", *args], capture_output=True) + + +def _run_setup_and_commit( + mount_args: list, setup_commands: list, image_tag: str +) -> str: + """Runs setup_commands once in a throwaway container and commits the + result to an image, so every test starts pre-set-up.""" + container_name = f"cfengine-cli-test-prep-{uuid.uuid4().hex[:12]}" + script = " && ".join(setup_commands) result = subprocess.run( - ["docker", "build", "-q", "-t", _IMAGE_TAG, _DOCKERFILE_DIR] + [ + "docker", + "run", + "--name", + container_name, + *mount_args, + image_tag, + "sh", + "-c", + script, + ] ) if result.returncode != 0: - raise UserError("Failed to build the cfengine-test Docker image.") + _docker_cleanup("rm", "-f", container_name) + raise UserError( + "Failed to set up the test environment (deploy step) -- see output above." + ) + + image_tag = f"cfengine-cli-test-agent-prepped:{uuid.uuid4().hex[:12]}" + commit = subprocess.run( + ["docker", "commit", container_name, image_tag], + capture_output=True, + text=True, + ) + _docker_cleanup("rm", "-f", container_name) + if commit.returncode != 0: + raise UserError( + f"Failed to commit the prepared test container: {commit.stderr.strip()}" + ) + return image_tag + + +def _remove_image(image_tag: str) -> None: + _docker_cleanup("rmi", image_tag) + + +def prepare_test_masterfiles(masterfiles: str | None = None) -> tuple: + """(rc, masterfiles_dir) for `cfengine test`: a cfbs project is built and + tested against its own out/masterfiles (whatever MPF version its + cfbs.json pins); outside one, --masterfiles names a built masterfiles + directory to use instead; otherwise there's none (None).""" + if is_cfbs_repo(): + if masterfiles is not None: + raise UserError( + "--masterfiles is for tests outside a cfbs project -- a cfbs " + "project is tested against its own build (out/masterfiles), " + "with the masterfiles version its cfbs.json pins." + ) + # Build first so any custom promise types / modules this + # project uses are shipped + rc = build_command() + return rc, "out/masterfiles" + if masterfiles is None: + return 0, None + if not os.path.isfile(os.path.join(masterfiles, "promises.cf")): + raise UserError( + f"--masterfiles '{masterfiles}' has no promises.cf -- point it at a " + "built masterfiles directory, e.g. out/masterfiles from `cfbs " + "build`, or an installed /var/cfengine/masterfiles." + ) + return 0, masterfiles -def run_in_container(masterfiles_dir: str) -> int: +def run_in_container( + masterfiles_dir: str, dockerfile: str | None = None, rebuild: bool = False +) -> int: """ - Runs cf-agent against the given built masterfiles directory inside a container + Runs cf-agent against the given built masterfiles directory inside a + container. There's no test_*.cf harness here, so there's no per-test + breakdown -- but if the deployed policy's own promises happen to emit + [ASSERT] lines (e.g. assert: promises outside the usual test_*.cf + convention), those are still counted and reported like a test run; + otherwise this just shows the agent's own output and exit code. + + dockerfile overrides the built-in Dockerfile (e.g. to test against a + different base distro); rebuild forces a cache-busting rebuild instead + of reusing whatever's already cached for it. """ require_docker() - _ensure_image_built() + context_dir, dockerfile_path = _resolve_dockerfile(dockerfile) + image_tag = _image_tag_for(dockerfile_path) + _ensure_image_built(context_dir, dockerfile_path, image_tag, rebuild) - abs_masterfiles = os.path.abspath(masterfiles_dir) - result = subprocess.run( - [ + # promises.cf is run explicitly below, so the bootstrap's own run would + # only duplicate it. + project = _project_setup(masterfiles_dir, run_policy_in_setup=False) + setup_commands = ( + ["rm -rf /var/cfengine/inputs"] + + project.deploy_commands + + project.post_deploy_commands + ) + prepped_image = _run_setup_and_commit(project.mount_args, setup_commands, image_tag) + try: + cmd = [ "docker", "run", "--rm", - "-v", - f"{abs_masterfiles}:/mnt/masterfiles:ro", - _IMAGE_TAG, + prepped_image, "sh", "-c", - "rm -rf /var/cfengine/inputs " - "&& rm -rf /var/cfengine/masterfiles " - "&& cp -r /mnt/masterfiles /var/cfengine/masterfiles " - "&& /var/cfengine/bin/cf-agent --bootstrap 127.0.0.1 " - "&& /var/cfengine/bin/cf-agent -KIf update.cf " - "&& /var/cfengine/bin/cf-agent -KI", + f"{paths.bin('cf-agent')} -KIf /var/cfengine/masterfiles/promises.cf", ] + results = run_and_scan_asserts(cmd) + finally: + _remove_image(prepped_image) + + # A failing assert alone doesn't make cf-agent exit non-zero, so a + # non-zero exit means the agent run itself broke (e.g. aborted partway + # through) -- that must fail the run even if every assert seen passed. + # Falling back to failsafe is a broken run too, despite exiting 0. + if results.fell_back_to_failsafe: + agent_rc = results.returncode or 1 + agent_problem = ( + "cf-agent ran its built-in failsafe policy instead of the " + "project's -- the policy failed validation, see the cf-promises " + "errors above" + ) + else: + agent_rc = results.returncode + agent_problem = ( + f"cf-agent exited with code {agent_rc} -- check output above for " + "a cf-agent error" + ) + + if results.passed == 0 and results.failed == 0: + if agent_rc != 0: + print(agent_problem) + return agent_rc + rc = print_assert_totals( + results.passed, results.failed, errors=1 if agent_rc != 0 else 0 ) - return result.returncode + if agent_rc != 0: + print(agent_problem) + return agent_rc + return rc + + +def _read_file(path: str) -> str: + with open(path) as file: + return file.read() + + +def _bundle_bodies_declared_in(content: str) -> list: + """[(name, body), ...] for every `bundle agent { ... }` in + content, in order, body being the brace-matched text between (and + including) its own braces.""" + bodies = [] + for match in _BUNDLE_AGENT_DECLARATION_RE.finditer(content): + name = match.group(1) + start = content.index("{", match.end()) + depth = 0 + end = start + for end in range(start, len(content)): + if content[end] == "{": + depth += 1 + elif content[end] == "}": + depth -= 1 + if depth == 0: + break + bodies.append((name, content[start : end + 1])) + return bodies + + +def _bundle_names_declared_in(path: str) -> list: + """Test (reportable) bundles in a file, in order -- a bundle only counts + as a test if it has its own `assert:` section. A bundle with no + `assert:` section (e.g. a shared bundle computing some vars) is just + available to be called via `methods:` from a test bundle, like any other + bundle in this file's inputs, without being reported as a test of its + own.""" + names = [ + name + for name, body in _bundle_bodies_declared_in(_read_file(path)) + if _ASSERT_SECTION_RE.search(body) + ] + if not names: + raise UserError(f"No bundle with an 'assert:' section found in {path}") + return names + + +def discover_test_files(tests_dir: str = "tests") -> list: + search_dir = tests_dir if os.path.isdir(tests_dir) else "." + return sorted( + os.path.join(search_dir, fname) + for fname in os.listdir(search_dir) + if fname.startswith("test_") and fname.endswith(".cf") + ) + + +def discover_module_files() -> list: + return sorted(glob.glob("*/main.cf")) + + +def _declared_promise_types(cf_file_path: str) -> set: + return set(_PROMISE_AGENT_DECLARATION_RE.findall(_read_file(cf_file_path))) + + +def _read_only_mount(host_path: str, container_path: str) -> list: + return ["-v", f"{host_path}:{container_path}:ro"] + + +def _expand_directories(paths: list, exclude_dirs: tuple = ()) -> list: + """Files are kept as given; directories are walked for .cf files, + skipping hidden directories and anything under exclude_dirs (e.g. a cfbs + project's build output, whose promises.cf/lib/*.cf are not libraries to + add to every test's inputs).""" + excluded = {os.path.abspath(d) for d in exclude_dirs} + expanded = [] + for path in paths: + if not os.path.isdir(path): + expanded.append(path) + continue + for root, dirs, files in os.walk(path, followlinks=True): + dirs[:] = [ + d + for d in dirs + if not d.startswith(".") + and os.path.abspath(os.path.join(root, d)) not in excluded + ] + expanded += sorted( + os.path.join(root, f) + for f in files + if f.endswith(".cf") and not f.startswith(".") + ) + return expanded + + +class ProjectSetup(NamedTuple): + mount_args: list + deploy_commands: list + post_deploy_commands: list + init_mount: str | None + declares_assert: bool + + +def _project_setup( + masterfiles_dir: str | None, run_policy_in_setup: bool = True +) -> ProjectSetup: + """Everything needed to make a cfbs project's masterfiles, custom promise + types and services/init.cf available in the container; empty when + there's no masterfiles_dir (a bare, cfbs-less test_*.cf file). + + run_policy_in_setup runs promises.cf once at the end of setup, so tests + can assert on the state the project's policy converged to. Callers that + run promises.cf themselves afterwards turn it off, as the policy would + otherwise run twice.""" + if masterfiles_dir is None: + return ProjectSetup([], [], [], None, False) + + abs_masterfiles = os.path.abspath(masterfiles_dir) + deploy_commands = [ + "rm -rf /var/cfengine/masterfiles", + "cp -r /mnt/project_masterfiles /var/cfengine/masterfiles", + ] + + post_deploy_commands = [] + if os.path.isdir(os.path.join(abs_masterfiles, "modules")): + post_deploy_commands.append( + "cp -r /mnt/project_masterfiles/modules/. /var/cfengine/modules/" + ) + + # Bootstrap to itself, then do what a host's first agent run does: + # update.cf, then (if wanted) promises.cf. Bootstrapping only runs the + # built-in failsafe policy, and update.cf is what installs modules into + # $(sys.workdir) (e.g. rendering the vendored package modules), so the + # bootstrap's own policy run is always skipped -- it would run before + # them. All of this happens once, in the shared setup image every test + # starts from, so it adds no per-test cost. The scheduler (cf-execd) is + # never needed: the container is committed right after this step. + cf_agent = paths.bin("cf-agent") + post_deploy_commands += [ + paths.bin("cf-serverd"), + "sleep 1", + f"{cf_agent} --bootstrap 127.0.0.1 " + "--skip-bootstrap-service-start --skip-bootstrap-policy-run", + f"{cf_agent} -KIf /var/cfengine/inputs/update.cf", + ] + if run_policy_in_setup: + post_deploy_commands.append(f"{cf_agent} -KIf /var/cfengine/inputs/promises.cf") + + init_mount = None + declares_assert = False + project_init_file = os.path.join(abs_masterfiles, "services", "init.cf") + if os.path.isfile(project_init_file): + # The prep container's /mnt/project_masterfiles mount is gone by the + # time per-test containers start from its committed image; this path + # is what's still there. + init_mount = "/var/cfengine/masterfiles/services/init.cf" + declares_assert = "assert" in _declared_promise_types(project_init_file) + + return ProjectSetup( + mount_args=_read_only_mount(abs_masterfiles, "/mnt/project_masterfiles"), + deploy_commands=deploy_commands, + post_deploy_commands=post_deploy_commands, + init_mount=init_mount, + declares_assert=declares_assert, + ) + + +def _bundlesequence_and_defs(bundle_reports: list) -> tuple: + """bundle_reports is this file's [(bundle_name, report_id), ...], one + pair per bundle agent it declares (see _bundle_names_declared_in). + Returns (bundlesequence_names, bundle_def_lines): the cftest_start/ + marker bundle names in bundlesequence order (interleaved with the real + test bundle names), and their own `bundle agent` definitions. Each + bundle gets its own CFTEST-START/DONE pair, so several independent + check bundles in one file still get attributed their own report row.""" + bundlesequence = [] + bundle_defs = [] + for i, (bundle_name, report_id) in enumerate(bundle_reports): + start, marker = f"cftest_start_{i}", f"cftest_marker_{i}" + bundlesequence += [start, bundle_name, marker] + bundle_defs += [ + f"bundle agent {start}", + "{", + " reports:", + f' "[CFTEST-START] {report_id}";', + "}", + "", + f"bundle agent {marker}", + "{", + " reports:", + f' "[CFTEST-DONE] {report_id}";', + "}", + "", + ] + return bundlesequence, bundle_defs + + +def _run_cf_content(bundle_reports: list, inputs: list) -> str: + """A standalone run.cf: its own `body common control` declaring exactly + this file's inputs and bundlesequence -- the test bundle(s) are all that + run, nothing from a real project's own policy. Used for --test-only, + and for tests outside a cfbs project; see _bundle_defs_content for the + full-policy default.""" + inputs_str = ", ".join([f'"{i}"' for i in inputs] + ["@(cftest_stdlib.inputs)"]) + bundlesequence, bundle_defs = _bundlesequence_and_defs(bundle_reports) + bundlesequence_str = ", ".join(f'"{name}"' for name in bundlesequence) + return "\n".join( + [ + # The standard library of whichever masterfiles is deployed: the + # project's own (cfbs project / --masterfiles) or the one the + # CFEngine package installed. Checked at run time, since a + # custom --dockerfile image may have none. + "bundle common cftest_stdlib", + "{", + " classes:", + f' "have_stdlib" expression => fileexists("{_STDLIB}");', + "", + " vars:", + " have_stdlib::", + f' "inputs" slist => {{ "{_STDLIB}" }};', + " !have_stdlib::", + ' "inputs" slist => { };', + "}", + "", + "body common control", + "{", + f" inputs => {{ {inputs_str} }};", + f" bundlesequence => {{ {bundlesequence_str} }};", + "}", + "", + ] + + bundle_defs + ) + + +def _bundle_defs_content(bundle_reports: list) -> str: + """Just the cftest_start/marker `bundle agent` definitions, no `body + common control` of its own -- for full-policy mode (the default in a + cfbs project), where the real project's promises.cf supplies the control + body and bundlesequence.""" + _bundlesequence, bundle_defs = _bundlesequence_and_defs(bundle_reports) + return "\n".join(bundle_defs) + + +def _merge_def_json( + masterfiles_dir: str, + fixture_path: str | None, + extra_inputs: list, + extra_bundlesequence: list, +) -> str: + """Full-policy mode runs the project's own real promises.cf as the + entry point, instead of a from-scratch run.cf -- so the test's own + inputs and bundlesequence additions have to be merged into def.json + instead of declared directly. + + The fixture is merged in the way `cfbs build` merges a module's def.json + contributions: objects merge, single values are overwritten, lists are + appended. So a fixture can never drop the project's generated wiring + (its modules' inputs and bundlesequence entries) -- but it can't shrink + a list the project sets either; use a differently named variable.""" + merged = read_json(os.path.join(masterfiles_dir, "def.json")) or {} + + if fixture_path is not None: + merged = merge_json(merged, read_json(fixture_path) or {}) + + extra = { + "inputs": extra_inputs, + "vars": {"control_common_bundlesequence_end": extra_bundlesequence}, + } + merged = merge_json(merged, extra) + + return json.dumps(merged, indent=2) + + +def _inputs_for_test( + test_mount: str, project: ProjectSetup, library_mounts: list +) -> list: + inputs = ( + [] if project.declares_assert else [f"{_ASSERT_MODULE_DIR}/promise_agent.cf"] + ) + if project.init_mount is not None: + inputs.append(project.init_mount) + inputs += library_mounts + inputs.append(test_mount) + return inputs + + +def _prepare_test_run( + runners_dir: str, + test_file: str, + bundle_reports: list, + project: ProjectSetup, + library_mounts: list, + full_policy: bool = False, + masterfiles_dir: str | None = None, +) -> tuple: + """Writes this test's run.cf (and fixture, if any) under runners_dir, a + fresh per-test tempdir. bundle_reports is this file's [(bundle_name, + report_id), ...] (see _bundlesequence_and_defs). Returns (mount_args, + run_command). + + Run after the project's own real bundlesequence has converged for real, rather + than only ever seeing whatever state the one-time setup/bootstrap step left behind. + """ + test_mount = f"/mnt/tests/{os.path.basename(test_file)}" + mount_args = _read_only_mount(test_file, test_mount) + + inputs = _inputs_for_test(test_mount, project, library_mounts) + + # Fixtures are keyed by the test file's name, not its bundle name -- + # two files can share a bundle name. + test_stem = os.path.splitext(os.path.basename(test_file))[0] + fixture = os.path.join(os.path.dirname(test_file), "fixtures", f"{test_stem}.json") + fixture_path = fixture if os.path.isfile(fixture) else None + + if full_policy: + assert masterfiles_dir is not None # enforced by run_files_in_container + wrapper_mount = "/mnt/runners/cftest_wrapper.cf" + with open(os.path.join(runners_dir, "cftest_wrapper.cf"), "w") as wrapper_cf: + wrapper_cf.write(_bundle_defs_content(bundle_reports)) + + bundlesequence, _bundle_defs = _bundlesequence_and_defs(bundle_reports) + def_json = os.path.join(runners_dir, "def.json") + with open(def_json, "w") as def_json_file: + def_json_file.write( + _merge_def_json( + masterfiles_dir, + fixture_path, + inputs + [wrapper_mount], + bundlesequence, + ) + ) + # Augments (def.json) are loaded from next to the entry file being + # parsed -- /var/cfengine/masterfiles/promises.cf here -- so the + # merged def.json has to override that exact path, on top of the + # whole-directory /mnt/runners mount below that makes the wrapper + # file (and this same def.json) reachable at all. + mount_args += _read_only_mount(def_json, "/var/cfengine/masterfiles/def.json") + run_command = ( + f"{paths.bin('cf-agent')} -KIf /var/cfengine/masterfiles/promises.cf" + ) + return mount_args, run_command + + with open(os.path.join(runners_dir, "run.cf"), "w") as run_cf: + run_cf.write(_run_cf_content(bundle_reports, inputs)) + if fixture_path is not None: + shutil.copyfile(fixture_path, os.path.join(runners_dir, "def.json")) + + run_command = f"{paths.bin('cf-agent')} -KIf /mnt/runners/run.cf" + return mount_args, run_command + + +def run_files_in_container( + files: list, + masterfiles_dir: str | None = None, + dockerfile: str | None = None, + rebuild: bool = False, + test_only: bool = False, +) -> int: + """Runs each test_*.cf file as its own isolated agent-run in its own + container, parses [ASSERT] PASS/FAIL lines out of the output, + and prints a pytest-style report, returning 0 only if everything passed. + A file may declare several bundles; each becomes its own "::" + report row within that one file's run. + + Every non-test_ file is a shared library added to every test's `inputs`. + A sibling fixtures/.json is auto-loaded as that run's + `def.json`. masterfiles_dir, if given, is a built cfbs policy set, + deployed once for every test to share. + + In a cfbs project (masterfiles_dir given), each test runs at the end of + the project's own real promises.cf (extended via augments, see + _prepare_test_run), so it sees the state a full agent run leaves behind, + with its own fixture applied. test_only runs just the test bundles from a + minimal generated policy instead; the project's policy then only ran + once, during the shared setup step and without any fixture, so every + test sees that snapshot. Without a cfbs project there's no real policy + to run, so tests always run that way.""" + require_docker() + full_policy = masterfiles_dir is not None and not test_only + context_dir, dockerfile_path = _resolve_dockerfile(dockerfile) + image_tag = _image_tag_for(dockerfile_path) + _ensure_image_built(context_dir, dockerfile_path, image_tag, rebuild) + + # cfbs builds into out/ (masterfiles_dir is out/masterfiles); none of it + # is the user's own test or library code. + exclude_dirs = ( + (os.path.dirname(os.path.abspath(masterfiles_dir)),) + if masterfiles_dir is not None + else () + ) + # dict.fromkeys: dedupe while keeping order -- the same file can be both + # auto-discovered (a module's main.cf) and inside a directory argument. + abs_files = list( + dict.fromkeys( + os.path.abspath(f) for f in _expand_directories(files, exclude_dirs) + ) + ) + test_files = [f for f in abs_files if os.path.basename(f).startswith("test_")] + library_files = [f for f in abs_files if f not in test_files] + if not test_files: + raise UserError( + "No test_*.cf files given to `cfengine test` -- nothing to run. " + "Pass at least one file whose name starts with 'test_', " + "e.g. `cfengine test tests/test_foo.cf`." + ) + + file_report_ids = [os.path.relpath(f).replace(" ", "_") for f in test_files] + bundle_reports_by_file = [ + [ + (bundle_name, f"{file_report_id}::{bundle_name}") + for bundle_name in _bundle_names_declared_in(test_file) + ] + for test_file, file_report_id in zip(test_files, file_report_ids) + ] + report_ids = [ + report_id + for bundle_reports in bundle_reports_by_file + for _bundle_name, report_id in bundle_reports + ] + + # Full-policy mode runs promises.cf again in every test's own + # container; --test-only relies on the bootstrap's run instead. + project = _project_setup(masterfiles_dir, run_policy_in_setup=not full_policy) + setup_commands = ( + ["rm -rf /var/cfengine/inputs"] + + project.deploy_commands + + project.post_deploy_commands + ) + + library_mount_args = [] + library_mounts = [] + for i, lib in enumerate(library_files): + # Indexed directory, as several libraries can share a basename -- + # every module's file is main.cf. + lib_mount = f"/mnt/lib/{i}/{os.path.basename(lib)}" + library_mount_args += _read_only_mount(lib, lib_mount) + library_mounts.append(lib_mount) + + prepped_image = _run_setup_and_commit(project.mount_args, setup_commands, image_tag) + try: + results = RunResults(set(), {}, []) + + for test_file, bundle_reports in zip(test_files, bundle_reports_by_file): + test_names = [report_id for _bundle_name, report_id in bundle_reports] + with tempfile.TemporaryDirectory( + prefix="cfengine-test-runners-" + ) as runners_dir: + test_mount_args, run_command = _prepare_test_run( + runners_dir, + test_file, + bundle_reports, + project, + library_mounts, + full_policy=full_policy, + masterfiles_dir=( + os.path.abspath(masterfiles_dir) + if masterfiles_dir is not None + else None + ), + ) + mount_args = ( + library_mount_args + + test_mount_args + + _read_only_mount(runners_dir, "/mnt/runners") + ) + + one_test_results = run_and_parse( + mount_args, run_command, test_names, image=prepped_image + ) + + results.completed.update(one_test_results.completed) + results.marks_by_test.update(one_test_results.marks_by_test) + results.failures.extend(one_test_results.failures) + finally: + _remove_image(prepped_image) + + return print_report(report_ids, results) diff --git a/src/cfengine_cli/docker/test-agent/Dockerfile b/src/cfengine_cli/docker/test-agent/Dockerfile index 2469584..0999844 100644 --- a/src/cfengine_cli/docker/test-agent/Dockerfile +++ b/src/cfengine_cli/docker/test-agent/Dockerfile @@ -3,11 +3,24 @@ ENV PATH="/root/.local/bin:${PATH}" RUN apt-get update \ && apt-get install -y --no-install-recommends \ - python3 pipx sudo procps + python3 pipx sudo procps git rsync RUN pipx install cfengine +RUN pipx install cfbs RUN cfengine install --clients localhost --edition community RUN mkdir -p /var/cfengine/bin/ \ && ln -sf "$(command -v python3)" /var/cfengine/bin/cfengine-selected-python + +RUN mkdir -p /var/cfengine/modules/promises /tmp/assert-module-fetch \ + && cd /tmp/assert-module-fetch \ + && cfbs --non-interactive --masterfiles no --git no init \ + && cfbs --non-interactive add library-for-promise-types-in-python promise-type-assert \ + && cfbs build \ + && cp out/masterfiles/modules/promises/assert_promise_type.py \ + out/masterfiles/modules/promises/cfengine_module_library.py \ + /var/cfengine/modules/promises/ \ + && cd / && rm -rf /tmp/assert-module-fetch + +COPY assert-module/promise_agent.cf /var/cfengine/modules/promises/promise_agent.cf diff --git a/src/cfengine_cli/docker/test-agent/assert-module/promise_agent.cf b/src/cfengine_cli/docker/test-agent/assert-module/promise_agent.cf new file mode 100644 index 0000000..d41638b --- /dev/null +++ b/src/cfengine_cli/docker/test-agent/assert-module/promise_agent.cf @@ -0,0 +1,5 @@ +promise agent assert +{ + path => "$(sys.workdir)/modules/promises/assert_promise_type.py"; + interpreter => "/var/cfengine/bin/cfengine-selected-python"; +} diff --git a/src/cfengine_cli/main.py b/src/cfengine_cli/main.py index d9b39c9..f93986c 100644 --- a/src/cfengine_cli/main.py +++ b/src/cfengine_cli/main.py @@ -106,14 +106,51 @@ def _get_arg_parser(): tst.add_argument( "--strict", type=str, - default="yes", - help="Strict mode for linting. Default=yes, checks for undefined promise types, bundles, bodies, functions", + default="no", + help="Strict mode for linting. Default=no: test policy commonly calls " + "the masterfiles standard library (m(), insert_lines(), ...), which " + "strict mode would report as undefined. --strict=yes checks for " + "undefined promise types, bundles, bodies, functions", ) tst.add_argument( "files", nargs="*", help="Files/Folder to lint (default is . (the entire project))", ) + tst.add_argument( + "--dockerfile", + type=str, + default=None, + help="Path to a custom Dockerfile (or its directory) to build the test " + "container from, instead of the built-in one -- e.g. to test against " + "a different base distro", + ) + tst.add_argument( + "--rebuild", + action="store_true", + help="Force a clean rebuild of the test container image, bypassing " + "Docker's build cache", + ) + tst.add_argument( + "--masterfiles", + type=str, + default=None, + help="Outside a cfbs project: a built masterfiles directory (e.g. " + "out/masterfiles from `cfbs build`) to test against, the way a cfbs " + "project uses its own build -- so tests run with the masterfiles " + "standard library (m(), insert_lines(), ...) of that exact version.", + ) + tst.add_argument( + "--test-only", + action="store_true", + help="In a cfbs project, tests normally run at the end of the " + "project's real promises.cf (added via augments, with the test's " + "fixture applied), so each test sees the state a full agent run " + "leaves behind. --test-only runs just the test bundles from a minimal " + "generated policy instead: faster, but the project's policy then " + "only ran once, during setup and without any fixture, so tests see " + "that snapshot. Tests outside a cfbs project always run this way.", + ) dev_parser = subp.add_parser( "dev", help="Utilities intended for developers / maintainers of CFEngine" @@ -289,6 +326,10 @@ def run_command_with_args(args) -> int: return commands.test( args.files, (args.strict.lower() in ("y", "ye", "yes")), + args.dockerfile, + args.rebuild, + args.test_only, + args.masterfiles, ) if args.command == "report": return cfengine_commands.report( diff --git a/src/cfengine_cli/report.py b/src/cfengine_cli/report.py new file mode 100644 index 0000000..7f8ba90 --- /dev/null +++ b/src/cfengine_cli/report.py @@ -0,0 +1,197 @@ +import re +import subprocess +import sys +from collections import Counter +from typing import NamedTuple + +_ASSERT_RE = re.compile(r"\[ASSERT\] (PASS|FAIL) (.+?)(?: # (.*))?$") +_START_RE = re.compile(r"\[CFTEST-START\] (\S+)") +_DONE_RE = re.compile(r"\[CFTEST-DONE\] (\S+)") +# An assert: promise that fails validation is skipped by cf-agent and never +# prints a verdict, so it has to be caught from the error itself +_INVALID_ASSERT_RE = re.compile( + r"error: (.*) for assert promise with promiser '([^']*)'" +) +# cf-agent prints this when cf-promises rejected the policy, and then runs +# its built-in failsafe policy instead -- still exiting 0 +_FAILSAFE_RE = re.compile(r"going to failsafe") + +_GREEN = "\033[32m" +_RED = "\033[31m" +_RESET = "\033[0m" + + +def _color(text: str, code: str) -> str: + return f"{code}{text}{_RESET}" if sys.stdout.isatty() else text + + +class RunResults(NamedTuple): + completed: set # test (bundle) names that ran to completion + marks_by_test: dict # test name -> list of "." / "F", one per assert, in order + failures: list # [(test, assertion, reason), ...], in the order they failed + + +def _failure_line(test: str, assertion: str, reason: str) -> str: + suffix = f" -> {reason}" if reason else "" + return f" {_color('FAIL', _RED)} {test}::{assertion}{suffix}" + + +def _record(results: RunResults, test: str, outcome: str, assertion: str, reason: str): + """Tallies one assert result as it streams in; a FAIL is printed right + away (and kept for the end-of-run recap), so it's visible next to the + agent output that caused it.""" + if outcome == "PASS": + results.marks_by_test[test].append(".") + return + results.marks_by_test[test].append("F") + results.failures.append((test, assertion, reason)) + print(_failure_line(test, assertion, reason), flush=True) + + +def run_and_parse( + mount_args: list, script: str, test_names: list, image: str +) -> RunResults: + proc = subprocess.Popen( + ["docker", "run", "--rm", *mount_args, image, "sh", "-c", script], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + assert proc.stdout is not None + + results = RunResults(set(), {name: [] for name in test_names}, []) + current = None # test whose cftest_start marker fired most recently + + for line in proc.stdout: + print(line, end="", flush=True) + + start_match = _START_RE.search(line) + if start_match: + current = start_match.group(1) + + assert_match = _ASSERT_RE.search(line) + if assert_match and current: + outcome, name, reason = assert_match.groups() + _record(results, current, outcome, name, reason or "") + + invalid_match = _INVALID_ASSERT_RE.search(line) + if invalid_match and current: + reason, name = invalid_match.groups() + _record(results, current, "FAIL", name, f"invalid assert: {reason}") + + done_match = _DONE_RE.search(line) + if done_match: + results.completed.add(done_match.group(1)) + current = None + + proc.wait() + return results + + +class DeployResults(NamedTuple): + returncode: ( + int # the deploy run's own exit code, used when it has no asserts at all + ) + passed: int + failed: int + fell_back_to_failsafe: bool = False + + +def run_and_scan_asserts(cmd: list) -> DeployResults: + """Like run_and_parse, but for a single plain policy run with no + CFTEST-START/DONE wrapper -- there's only one implicit "test" (the whole + run), so asserts are just counted flat, not attributed to a test name.""" + proc = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + ) + assert proc.stdout is not None + + passed = failed = 0 + fell_back_to_failsafe = False + for line in proc.stdout: + print(line, end="", flush=True) + + if _FAILSAFE_RE.search(line): + fell_back_to_failsafe = True + + assert_match = _ASSERT_RE.search(line) + if assert_match: + if assert_match.group(1) == "PASS": + passed += 1 + else: + failed += 1 + elif _INVALID_ASSERT_RE.search(line): + failed += 1 + + proc.wait() + return DeployResults(proc.returncode, passed, failed, fell_back_to_failsafe) + + +def print_assert_totals(passed: int, failed: int, errors: int = 0) -> int: + print() + print("=" * 40) + print( + f" cfengine test: {_color(f'{passed} passed', _GREEN)}, " + f"{_color(f'{failed} failed', _RED)}, {errors} errors" + ) + print("=" * 40) + return 0 if failed == 0 and errors == 0 else 1 + + +def _assert_dots(name: str, results: RunResults) -> str: + """One character per assert, in order; an incomplete test gets a + trailing "E", like pytest's "...E...." distinguishes error from fail.""" + marks = [ + _color(mark, _GREEN if mark == "." else _RED) + for mark in results.marks_by_test[name] + ] + if name not in results.completed: + marks.append(_color("E", _RED)) + return "".join(marks) + + +def _outcome_of(name: str, results: RunResults) -> str: + """cftest_marker only fires if the test bundle finished evaluating, so a + fatal abort (syntax error, missing bundle, ...) shows as ERROR rather + than FAIL.""" + if name not in results.completed: + return "ERROR" + if "F" in results.marks_by_test[name]: + return "FAIL" + return "PASS" + + +def print_report(test_names: list, results: RunResults) -> int: + outcomes = [(name, _outcome_of(name, results)) for name in test_names] + + print() + for name in test_names: + print(f"{name} {_assert_dots(name, results)}") + print("=" * 40) + # Passed/failed count individual assert: checks, like pytest counts test + # functions; errors count per file since an aborted run never reports + # the rest of its checks. + marks = Counter(mark for name in test_names for mark in results.marks_by_test[name]) + errored_files = sum(1 for _, outcome in outcomes if outcome == "ERROR") + passed = marks["."] + failed = marks["F"] + print( + f" cfengine test: {_color(f'{passed} passed', _GREEN)}, " + f"{_color(f'{failed} failed', _RED)}, " + f"{errored_files} errors" + ) + print("=" * 40) + + if all(outcome == "PASS" for _, outcome in outcomes): + return 0 + + for test, assertion, reason in results.failures: + print(_failure_line(test, assertion, reason)) + for name, outcome in outcomes: + if outcome == "ERROR": + print( + f" {_color('ERROR', _RED)} {name} -> test did not run to completion -- check " + "output above for a cf-agent error (syntax error, missing bundle, ...)" + ) + print() + return 1 diff --git a/tests/unit/test_container.py b/tests/unit/test_container.py new file mode 100644 index 0000000..b87df13 --- /dev/null +++ b/tests/unit/test_container.py @@ -0,0 +1,452 @@ +import pytest + +import cfengine_cli.container as container +from cfengine_cli.report import DeployResults +from cfengine_cli.utils import UserError + + +@pytest.fixture +def no_real_docker(monkeypatch): + """run_files_in_container/run_in_container always start by checking for + and building the docker image, then setting up and committing a prepped + image -- stub all of that out so tests don't need a real docker daemon.""" + monkeypatch.setattr(container, "require_docker", lambda: None) + monkeypatch.setattr(container, "_ensure_image_built", lambda *a, **k: None) + monkeypatch.setattr( + container, "_run_setup_and_commit", lambda *a, **k: "prepped-image:latest" + ) + monkeypatch.setattr(container, "_remove_image", lambda *a, **k: None) + + +# --------------------------------------------------------------------------- +# require_docker +# --------------------------------------------------------------------------- + + +def test_require_docker_raises_when_missing(monkeypatch): + monkeypatch.setattr(container.shutil, "which", lambda name: None) + with pytest.raises(UserError): + container.require_docker() + + +def test_require_docker_passes_when_present(monkeypatch): + monkeypatch.setattr(container.shutil, "which", lambda name: "/usr/bin/docker") + container.require_docker() # doesn't raise + + +# --------------------------------------------------------------------------- +# _resolve_dockerfile_dir / _image_tag_for +# --------------------------------------------------------------------------- + + +def test_resolve_dockerfile_defaults_to_builtin(): + context_dir, dockerfile_path = container._resolve_dockerfile(None) + assert context_dir == container._DOCKERFILE_DIR + assert dockerfile_path == f"{container._DOCKERFILE_DIR}/Dockerfile" + + +def test_resolve_dockerfile_accepts_a_directory(tmp_path): + assert container._resolve_dockerfile(str(tmp_path)) == ( + str(tmp_path), + str(tmp_path / "Dockerfile"), + ) + + +def test_resolve_dockerfile_keeps_a_non_default_dockerfile_name(tmp_path): + dockerfile = tmp_path / "Dockerfile.rhel" + dockerfile.write_text("FROM rockylinux:9\n") + assert container._resolve_dockerfile(str(dockerfile)) == ( + str(tmp_path), + str(dockerfile), + ) + + +def test_image_tag_for_builtin_dockerfile_is_the_stable_tag(): + _context_dir, dockerfile_path = container._resolve_dockerfile(None) + assert container._image_tag_for(dockerfile_path) == container._IMAGE_TAG + + +def test_image_tag_for_custom_dockerfile_is_distinct_and_deterministic(tmp_path): + tag_a = container._image_tag_for(str(tmp_path / "Dockerfile")) + tag_b = container._image_tag_for(str(tmp_path / "Dockerfile")) + assert tag_a == tag_b + assert tag_a != container._IMAGE_TAG + assert tag_a.startswith("cfengine-cli-test-agent-custom-") + + +def test_image_tag_for_sibling_dockerfiles_differ(tmp_path): + tag_default = container._image_tag_for(str(tmp_path / "Dockerfile")) + tag_rhel = container._image_tag_for(str(tmp_path / "Dockerfile.rhel")) + assert tag_default != tag_rhel + + +# --------------------------------------------------------------------------- +# _ensure_image_built +# --------------------------------------------------------------------------- + + +def _record_runs(monkeypatch, returncode=0): + seen_cmds = [] + monkeypatch.setattr( + container.subprocess, + "run", + lambda cmd, **k: seen_cmds.append(cmd) + or container.subprocess.CompletedProcess(cmd, returncode), + ) + return seen_cmds + + +def test_ensure_image_built_passes_no_cache_only_when_rebuilding(monkeypatch): + seen_cmds = _record_runs(monkeypatch) + + container._ensure_image_built( + "/some/dir", "/some/dir/Dockerfile", "some-tag:latest", rebuild=False + ) + container._ensure_image_built( + "/some/dir", "/some/dir/Dockerfile", "some-tag:latest", rebuild=True + ) + + assert "--no-cache" not in seen_cmds[0] + assert "--no-cache" in seen_cmds[1] + + +def test_ensure_image_built_builds_the_given_dockerfile(monkeypatch): + seen_cmds = _record_runs(monkeypatch) + + container._ensure_image_built( + "/some/dir", "/some/dir/Dockerfile.rhel", "some-tag:latest", rebuild=False + ) + + cmd = seen_cmds[0] + assert cmd[cmd.index("-f") + 1] == "/some/dir/Dockerfile.rhel" + assert cmd[-1] == "/some/dir" + + +def test_ensure_image_built_raises_on_build_failure(monkeypatch): + _record_runs(monkeypatch, returncode=1) + with pytest.raises(UserError): + container._ensure_image_built( + "/some/dir", "/some/dir/Dockerfile", "some-tag:latest", rebuild=False + ) + + +# --------------------------------------------------------------------------- +# _run_setup_and_commit +# --------------------------------------------------------------------------- + + +def test_run_setup_and_commit_raises_when_setup_fails(monkeypatch): + monkeypatch.setattr( + container.subprocess, + "run", + lambda *a, **k: container.subprocess.CompletedProcess(a, 1), + ) + monkeypatch.setattr(container, "_docker_cleanup", lambda *a: None) + + with pytest.raises(UserError): + container._run_setup_and_commit([], ["false"], "some-tag:latest") + + +def test_run_setup_and_commit_raises_user_error_and_cleans_up_when_commit_fails( + monkeypatch, +): + def fake_run(cmd, **k): + returncode = 1 if cmd[1] == "commit" else 0 + return container.subprocess.CompletedProcess(cmd, returncode, "", "disk full") + + cleaned = [] + monkeypatch.setattr(container.subprocess, "run", fake_run) + monkeypatch.setattr(container, "_docker_cleanup", lambda *a: cleaned.append(a)) + + with pytest.raises(UserError, match="disk full"): + container._run_setup_and_commit([], ["true"], "some-tag:latest") + assert cleaned and cleaned[0][0] == "rm" + + +# --------------------------------------------------------------------------- +# run_in_container +# --------------------------------------------------------------------------- + + +def test_run_in_container_reports_totals_when_asserts_are_present( + no_real_docker, monkeypatch, tmp_path, capsys +): + monkeypatch.setattr( + container, + "run_and_scan_asserts", + lambda cmd: DeployResults(returncode=1, passed=2, failed=1), + ) + + rc = container.run_in_container(str(tmp_path)) + out = capsys.readouterr().out + + assert rc == 1 + assert "2 passed" in out + assert "1 failed" in out + + +def test_run_in_container_returns_raw_exit_code_without_asserts( + no_real_docker, monkeypatch, tmp_path +): + monkeypatch.setattr( + container, + "run_and_scan_asserts", + lambda cmd: DeployResults(returncode=7, passed=0, failed=0), + ) + + assert container.run_in_container(str(tmp_path)) == 7 + + +def test_run_in_container_fails_when_agent_fails_despite_passing_asserts( + no_real_docker, monkeypatch, tmp_path, capsys +): + monkeypatch.setattr( + container, + "run_and_scan_asserts", + lambda cmd: DeployResults(returncode=2, passed=1, failed=0), + ) + + rc = container.run_in_container(str(tmp_path)) + out = capsys.readouterr().out + + assert rc == 2 + assert "1 passed, 0 failed, 1 errors" in out + + +def test_run_in_container_fails_when_agent_fell_back_to_failsafe( + no_real_docker, monkeypatch, tmp_path, capsys +): + monkeypatch.setattr( + container, + "run_and_scan_asserts", + lambda cmd: DeployResults( + returncode=0, passed=0, failed=0, fell_back_to_failsafe=True + ), + ) + + rc = container.run_in_container(str(tmp_path)) + + assert rc != 0 + assert "failsafe" in capsys.readouterr().out + + +# --------------------------------------------------------------------------- +# _expand_directories +# --------------------------------------------------------------------------- + + +def test_expand_directories_skips_excluded_and_hidden_dirs(tmp_path): + (tmp_path / "tests").mkdir() + (tmp_path / "tests" / "test_a.cf").write_text("") + (tmp_path / ".git").mkdir() + (tmp_path / ".git" / "x.cf").write_text("") + (tmp_path / "out" / "masterfiles").mkdir(parents=True) + (tmp_path / "out" / "masterfiles" / "promises.cf").write_text("") + + expanded = container._expand_directories( + [str(tmp_path)], exclude_dirs=(str(tmp_path / "out"),) + ) + + assert expanded == [str(tmp_path / "tests" / "test_a.cf")] + + +# --------------------------------------------------------------------------- +# _bundle_names_declared_in +# --------------------------------------------------------------------------- + + +def test_bundle_names_declared_in_single_bundle(tmp_path): + path = tmp_path / "test_foo.cf" + path.write_text('bundle agent test_foo\n{\n assert:\n "x" pass => "true";\n}\n') + assert container._bundle_names_declared_in(str(path)) == ["test_foo"] + + +def test_bundle_names_declared_in_excludes_bundles_without_assert_section(tmp_path): + path = tmp_path / "test_foo.cf" + path.write_text( + 'bundle agent variables\n{\n vars:\n "x" string => "1";\n}\n' + '\nbundle agent test_foo\n{\n assert:\n "x" pass => "true";\n}\n' + ) + assert container._bundle_names_declared_in(str(path)) == ["test_foo"] + + +def test_bundle_names_declared_in_raises_without_declaration(tmp_path): + path = tmp_path / "test_foo.cf" + path.write_text("# no bundle here\n") + with pytest.raises(UserError): + container._bundle_names_declared_in(str(path)) + + +def test_bundle_names_declared_in_raises_when_no_bundle_has_assert_section(tmp_path): + path = tmp_path / "test_foo.cf" + path.write_text('bundle agent variables\n{\n vars:\n "x" string => "1";\n}\n') + with pytest.raises(UserError): + container._bundle_names_declared_in(str(path)) + + +# --------------------------------------------------------------------------- +# run_files_in_container +# --------------------------------------------------------------------------- + + +def test_run_files_in_container_raises_without_test_files(no_real_docker, tmp_path): + lib = tmp_path / "lib.cf" + lib.write_text("bundle agent lib {}\n") + + with pytest.raises(UserError): + container.run_files_in_container([str(lib)]) + + +_TEST_X = 'bundle agent test_x\n{\n assert:\n "x" pass => "true";\n}\n' + + +def _capture_run_and_parse(monkeypatch): + calls = [] + + def fake_run_and_parse(mount_args, script, test_names, image): + calls.append(mount_args) + return container.RunResults(set(test_names), {n: [] for n in test_names}, []) + + monkeypatch.setattr(container, "run_and_parse", fake_run_and_parse) + monkeypatch.setattr(container, "print_report", lambda *a: 0) + return calls + + +def _library_mount_targets(mount_args): + return [arg.split(":")[1] for arg in mount_args if ":/mnt/lib/" in arg] + + +def test_run_files_in_container_mounts_same_named_libraries_apart( + no_real_docker, monkeypatch, tmp_path +): + for mod in ("mod_a", "mod_b"): + (tmp_path / mod).mkdir() + (tmp_path / mod / "main.cf").write_text(f"bundle agent {mod} {{}}\n") + test = tmp_path / "test_x.cf" + test.write_text(_TEST_X) + calls = _capture_run_and_parse(monkeypatch) + + container.run_files_in_container( + [ + str(tmp_path / "mod_a" / "main.cf"), + str(tmp_path / "mod_b" / "main.cf"), + str(test), + ] + ) + + targets = _library_mount_targets(calls[0]) + assert len(targets) == 2 + assert len(set(targets)) == 2 + + +def test_run_files_in_container_dedupes_files(no_real_docker, monkeypatch, tmp_path): + lib = tmp_path / "lib.cf" + lib.write_text("bundle agent lib {}\n") + (tmp_path / "test_x.cf").write_text(_TEST_X) + calls = _capture_run_and_parse(monkeypatch) + + container.run_files_in_container([str(lib), str(tmp_path)]) + + assert len(_library_mount_targets(calls[0])) == 1 + + +def test_run_files_in_container_skips_cfbs_build_output( + no_real_docker, monkeypatch, tmp_path +): + (tmp_path / "out" / "masterfiles").mkdir(parents=True) + (tmp_path / "out" / "masterfiles" / "promises.cf").write_text("") + (tmp_path / "test_x.cf").write_text(_TEST_X) + monkeypatch.setattr( + container, + "_project_setup", + lambda *a, **k: container.ProjectSetup([], [], [], None, False), + ) + calls = _capture_run_and_parse(monkeypatch) + + container.run_files_in_container( + [str(tmp_path)], masterfiles_dir=str(tmp_path / "out" / "masterfiles") + ) + + assert _library_mount_targets(calls[0]) == [] + + +def _agent_runs(project): + """The policy files setup runs cf-agent on, in order.""" + return [c.rsplit("/", 1)[-1] for c in project.post_deploy_commands if " -KIf " in c] + + +def test_project_setup_runs_update_then_promises_like_a_first_agent_run(tmp_path): + project = container._project_setup(str(tmp_path)) + [bootstrap] = [c for c in project.post_deploy_commands if "--bootstrap" in c] + assert "--skip-bootstrap-policy-run" in bootstrap + assert "--skip-bootstrap-service-start" in bootstrap + assert _agent_runs(project) == ["update.cf", "promises.cf"] + + +def test_project_setup_can_skip_the_policy_run(tmp_path): + project = container._project_setup(str(tmp_path), run_policy_in_setup=False) + assert _agent_runs(project) == ["update.cf"] + + +def test_run_files_in_container_runs_policy_in_setup_only_for_test_only( + no_real_docker, monkeypatch, tmp_path +): + (tmp_path / "test_x.cf").write_text(_TEST_X) + (tmp_path / "out" / "masterfiles").mkdir(parents=True) + _capture_run_and_parse(monkeypatch) + seen = [] + real_project_setup = container._project_setup + monkeypatch.setattr( + container, + "_project_setup", + lambda *a, **k: seen.append(k.get("run_policy_in_setup")) + or real_project_setup(*a, **k), + ) + monkeypatch.setattr(container, "_merge_def_json", lambda *a, **k: "{}") + + masterfiles = str(tmp_path / "out" / "masterfiles") + container.run_files_in_container([str(tmp_path / "test_x.cf")], masterfiles) + container.run_files_in_container( + [str(tmp_path / "test_x.cf")], masterfiles, test_only=True + ) + + assert seen == [False, True] + + +def test_run_cf_content_loads_stdlib_only_if_present(): + content = container._run_cf_content([("test_x", "f.cf::test_x")], ["/mnt/t.cf"]) + assert 'fileexists("/var/cfengine/masterfiles/lib/stdlib.cf")' in content + assert 'inputs => { "/mnt/t.cf", @(cftest_stdlib.inputs) };' in content + + +# --------------------------------------------------------------------------- +# prepare_test_masterfiles +# --------------------------------------------------------------------------- + + +def test_prepare_test_masterfiles_without_project_or_flag(monkeypatch): + monkeypatch.setattr(container, "is_cfbs_repo", lambda: False) + assert container.prepare_test_masterfiles(None) == (0, None) + + +def test_prepare_test_masterfiles_accepts_built_masterfiles(monkeypatch, tmp_path): + monkeypatch.setattr(container, "is_cfbs_repo", lambda: False) + (tmp_path / "promises.cf").write_text("") + assert container.prepare_test_masterfiles(str(tmp_path)) == ( + 0, + str(tmp_path), + ) + + +def test_prepare_test_masterfiles_rejects_dir_without_promises_cf( + monkeypatch, tmp_path +): + monkeypatch.setattr(container, "is_cfbs_repo", lambda: False) + with pytest.raises(UserError, match="promises.cf"): + container.prepare_test_masterfiles(str(tmp_path)) + + +def test_prepare_test_masterfiles_rejects_flag_in_cfbs_project(monkeypatch, tmp_path): + monkeypatch.setattr(container, "is_cfbs_repo", lambda: True) + with pytest.raises(UserError, match="cfbs project"): + container.prepare_test_masterfiles(str(tmp_path)) diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py new file mode 100644 index 0000000..776ee0c --- /dev/null +++ b/tests/unit/test_report.py @@ -0,0 +1,173 @@ +from cfengine_cli import report +from cfengine_cli.report import RunResults + + +class _FakeProc: + """Stands in for subprocess.Popen: stdout is pre-canned lines, no real + process behind it.""" + + def __init__(self, lines, returncode=0): + self.stdout = iter(lines) + self.returncode = returncode + + def wait(self): + return self.returncode + + +def test_run_and_parse_attributes_pass_and_fail_to_current_test(monkeypatch, capsys): + lines = [ + "[CFTEST-START] file.cf::bundle_a\n", + " info: [ASSERT] PASS a check\n", + " error: [ASSERT] FAIL another check # it broke\n", + "[CFTEST-DONE] file.cf::bundle_a\n", + ] + monkeypatch.setattr(report.subprocess, "Popen", lambda *a, **k: _FakeProc(lines)) + + results = report.run_and_parse([], "script", ["file.cf::bundle_a"], image="img") + + assert results.failures == [("file.cf::bundle_a", "another check", "it broke")] + assert results.completed == {"file.cf::bundle_a"} + assert results.marks_by_test["file.cf::bundle_a"] == [".", "F"] + # The FAIL is printed as soon as it's parsed, not only in the recap + out = capsys.readouterr().out + assert "FAIL file.cf::bundle_a::another check -> it broke" in out + + +def test_run_and_parse_records_invalid_assert_as_fail(monkeypatch): + lines = [ + "[CFTEST-START] file.cf::bundle_a\n", + "error: wrong type for 'int' for assert promise with promiser 'oops'\n", + "[CFTEST-DONE] file.cf::bundle_a\n", + ] + monkeypatch.setattr(report.subprocess, "Popen", lambda *a, **k: _FakeProc(lines)) + + results = report.run_and_parse([], "script", ["file.cf::bundle_a"], image="img") + + [(_test, name, reason)] = results.failures + assert name == "oops" + assert "invalid assert: wrong type for 'int'" == reason + assert results.marks_by_test["file.cf::bundle_a"] == ["F"] + + +def test_run_and_scan_asserts_counts_pass_fail_and_invalid(monkeypatch): + lines = [ + " info: [ASSERT] PASS one\n", + " error: [ASSERT] FAIL two # bad\n", + "error: wrong type for 'int' for assert promise with promiser 'three'\n", + ] + monkeypatch.setattr( + report.subprocess, "Popen", lambda *a, **k: _FakeProc(lines, returncode=1) + ) + + results = report.run_and_scan_asserts(["docker", "run"]) + + assert results.passed == 1 + assert results.failed == 2 + assert results.returncode == 1 + + +def test_run_and_scan_asserts_notices_failsafe_fallback(monkeypatch): + lines = [ + " error: CFEngine was not able to get confirmation of promises " + "from cf-promises, so going to failsafe\n", + ] + monkeypatch.setattr(report.subprocess, "Popen", lambda *a, **k: _FakeProc(lines)) + + results = report.run_and_scan_asserts(["docker", "run"]) + + assert results.fell_back_to_failsafe + assert results.returncode == 0 + + +def test_print_assert_totals_reports_zero_failures_as_success(capsys): + rc = report.print_assert_totals(passed=3, failed=0) + out = capsys.readouterr().out + assert rc == 0 + assert "3 passed" in out + assert "0 failed" in out + + +def test_print_assert_totals_reports_failures_as_failure(capsys): + rc = report.print_assert_totals(passed=1, failed=2) + assert rc == 1 + + +def test_print_assert_totals_reports_errors_as_failure(capsys): + rc = report.print_assert_totals(passed=1, failed=0, errors=1) + out = capsys.readouterr().out + assert rc == 1 + assert "1 errors" in out + + +def test_run_and_parse_keeps_every_result_for_a_repeated_promiser(monkeypatch): + lines = [ + "[CFTEST-START] file.cf::bundle_a\n", + " error: [ASSERT] FAIL check # first one broke\n", + " info: [ASSERT] PASS check\n", + "[CFTEST-DONE] file.cf::bundle_a\n", + ] + monkeypatch.setattr(report.subprocess, "Popen", lambda *a, **k: _FakeProc(lines)) + + results = report.run_and_parse([], "script", ["file.cf::bundle_a"], image="img") + + assert results.marks_by_test["file.cf::bundle_a"] == ["F", "."] + assert len(results.failures) == 1 + assert report._outcome_of("file.cf::bundle_a", results) == "FAIL" + + +def test_print_report_counts_every_result_for_a_repeated_promiser(capsys): + results = RunResults( + completed={"t"}, + marks_by_test={"t": ["F", "."]}, + failures=[("t", "check", "broke")], + ) + rc = report.print_report(["t"], results) + out = capsys.readouterr().out + assert rc == 1 + assert "1 passed, 1 failed, 0 errors" in out + assert "FAIL t::check -> broke" in out + + +def test_assert_dots_marks_incomplete_test_with_trailing_e(): + results = RunResults(completed=set(), marks_by_test={"t": [".", "."]}, failures=[]) + assert report._assert_dots("t", results) == "..E" + + +def test_outcome_of_is_error_when_test_never_completed(): + results = RunResults(completed=set(), marks_by_test={"t": []}, failures=[]) + assert report._outcome_of("t", results) == "ERROR" + + +def test_print_report_returns_zero_when_everything_passes(capsys): + results = RunResults( + completed={"t"}, + marks_by_test={"t": ["."]}, + failures=[], + ) + rc = report.print_report(["t"], results) + assert rc == 0 + + +def test_print_report_prints_error_line_for_incomplete_test(capsys): + results = RunResults(completed=set(), marks_by_test={"t": []}, failures=[]) + rc = report.print_report(["t"], results) + out = capsys.readouterr().out + assert rc == 1 + assert "ERROR" in out + assert "test did not run to completion" in out + + +def test_run_and_parse_ignores_asserts_after_done_marker(monkeypatch): + lines = [ + "[CFTEST-START] file.cf::bundle_a\n", + " info: [ASSERT] PASS a check\n", + "[CFTEST-DONE] file.cf::bundle_a\n", + " error: [ASSERT] FAIL not part of any test # printed after DONE\n", + ] + monkeypatch.setattr(report.subprocess, "Popen", lambda *a, **k: _FakeProc(lines)) + + results = report.run_and_parse([], "script", ["file.cf::bundle_a"], image="img") + + assert results.marks_by_test["file.cf::bundle_a"] == ["."] + assert results.failures == [] + assert report._outcome_of("file.cf::bundle_a", results) == "PASS" From 17d2672c5ca42b0a667c62470280173a3ee4b884 Mon Sep 17 00:00:00 2001 From: Simon Halvorsen Date: Tue, 10 Nov 2026 10:12:41 +0100 Subject: [PATCH 2/4] Added shell tests for `cfengine test` Covers the assert promise vocabulary, a standalone test suite (auto-discovery and an explicit directory argument), a cfbs project suite with a local module, and the full-policy default versus --test-only. Ticket: ENT-14444 Co-Authored-By: Claude Code Signed-off-by: Simon Halvorsen --- tests/shell/009-test-assert-vocab.sh | 46 ++++++++++ tests/shell/010-test-suite-standalone.sh | 57 ++++++++++++ tests/shell/011-test-suite-cfbs-project.sh | 40 +++++++++ tests/shell/012-test-full-policy.sh | 40 +++++++++ .../test_assert_vocab.cf | 90 +++++++++++++++++++ .../tests/fixtures/test_service_config.json | 3 + .../tests/test_computed_metrics.cf | 31 +++++++ .../tests/test_service_config.cf | 41 +++++++++ .../app_module/main.cf | 8 ++ .../011-test-suite-cfbs-project/cfbs.json | 17 ++++ .../tests/test_app_config.cf | 14 +++ .../fixtures/012-test-full-policy/cfbs.json | 27 ++++++ .../012-test-full-policy/overrides/def.json | 3 + .../overrides/services/init.cf | 8 ++ .../tests/fixtures/test_instance_a.json | 3 + .../tests/fixtures/test_instance_b.json | 3 + .../tests/test_instance_a.cf | 8 ++ .../tests/test_instance_b.cf | 8 ++ 18 files changed, 447 insertions(+) create mode 100755 tests/shell/009-test-assert-vocab.sh create mode 100755 tests/shell/010-test-suite-standalone.sh create mode 100755 tests/shell/011-test-suite-cfbs-project.sh create mode 100755 tests/shell/012-test-full-policy.sh create mode 100644 tests/shell/fixtures/009-test-assert-vocab/test_assert_vocab.cf create mode 100644 tests/shell/fixtures/010-test-suite-standalone/tests/fixtures/test_service_config.json create mode 100644 tests/shell/fixtures/010-test-suite-standalone/tests/test_computed_metrics.cf create mode 100644 tests/shell/fixtures/010-test-suite-standalone/tests/test_service_config.cf create mode 100644 tests/shell/fixtures/011-test-suite-cfbs-project/app_module/main.cf create mode 100644 tests/shell/fixtures/011-test-suite-cfbs-project/cfbs.json create mode 100644 tests/shell/fixtures/011-test-suite-cfbs-project/tests/test_app_config.cf create mode 100644 tests/shell/fixtures/012-test-full-policy/cfbs.json create mode 100644 tests/shell/fixtures/012-test-full-policy/overrides/def.json create mode 100644 tests/shell/fixtures/012-test-full-policy/overrides/services/init.cf create mode 100644 tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_a.json create mode 100644 tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_b.json create mode 100644 tests/shell/fixtures/012-test-full-policy/tests/test_instance_a.cf create mode 100644 tests/shell/fixtures/012-test-full-policy/tests/test_instance_b.cf diff --git a/tests/shell/009-test-assert-vocab.sh b/tests/shell/009-test-assert-vocab.sh new file mode 100755 index 0000000..8ace9c8 --- /dev/null +++ b/tests/shell/009-test-assert-vocab.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -e +set -x + +tmpdir=$(mktemp -d) +output_file=$(mktemp) +trap "rm -rf $tmpdir $output_file" EXIT + +cp tests/shell/fixtures/009-test-assert-vocab/test_assert_vocab.cf "$tmpdir/test_assert_vocab.cf" +cd "$tmpdir" + +if cfengine test test_assert_vocab.cf >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected test_assert_vocab.cf to fail (it has deliberately-failing assertions)" + exit 1 +fi +cat "$output_file" + +grep -qF "PASS: test_assert_vocab.cf" "$output_file" +grep -qF "Lint passed, no errors found." "$output_file" + +expected_fails=( + "FAIL test_assert_vocab.cf::test_assert_vocab::int_equals_fail -> expected 6, got 5" + "FAIL test_assert_vocab.cf::test_assert_vocab::int_less_than_fail -> 5 is not less than 1" + "FAIL test_assert_vocab.cf::test_assert_vocab::str_not_equals_fail -> expected not 'hello', but got it" + "FAIL test_assert_vocab.cf::test_assert_vocab::file_contents_fail -> /tmp/assert_vocab_fixture.txt contents did not match the expected contents" + "FAIL test_assert_vocab.cf::test_assert_vocab::dir_exists_fail -> expected exists=True, but exists=False for /this/path/should/not/exist/hopefully" + "FAIL test_assert_vocab.cf::test_assert_vocab::slist_contains_fail -> ['apple', 'banana', 'cherry'] does not contain 'durian'" +) +for line in "${expected_fails[@]}"; do + grep -qF "$line" "$output_file" || { + echo "FAIL: expected line not found in output: $line" + exit 1 + } +done + +# Exactly these six should fail -- counted in the end-of-run recap only, since +# each FAIL is also printed live as it happens +fail_count=$(sed -n '/cfengine test: /,$p' "$output_file" | grep -c "^ FAIL test_assert_vocab.cf::") +if [ "$fail_count" -ne 6 ]; then + echo "FAIL: expected exactly 6 failing assertions, got $fail_count" + exit 1 +fi + +grep -qF "cfengine test: 11 passed, 6 failed, 0 errors" "$output_file" diff --git a/tests/shell/010-test-suite-standalone.sh b/tests/shell/010-test-suite-standalone.sh new file mode 100755 index 0000000..d992005 --- /dev/null +++ b/tests/shell/010-test-suite-standalone.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -e +set -x + +tmpdir=$(mktemp -d) +output_file=$(mktemp) +trap "rm -rf $tmpdir $output_file" EXIT + +cp -r tests/shell/fixtures/010-test-suite-standalone/. "$tmpdir/" +cd "$tmpdir" + +# --- Auto-discovery: bare `cfengine test`, no arguments --- +# Two files, each with two check bundles sharing the same acted-upon data +if cfengine test >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected the suite to fail (it has deliberately-failing assertions)" + exit 1 +fi +cat "$output_file" + +grep -qF "cfengine test: 5 passed, 2 failed, 0 errors" "$output_file" + +expected_rows=( + "tests/test_service_config.cf::test_service_config_contents" + "tests/test_service_config.cf::test_service_config_permissions" + "tests/test_computed_metrics.cf::test_computed_metrics_equality" + "tests/test_computed_metrics.cf::test_computed_metrics_comparison" +) +for row in "${expected_rows[@]}"; do + grep -qF "$row" "$output_file" || { + echo "FAIL: expected report row not found: $row" + exit 1 + } +done + +expected_fails=( + "FAIL tests/test_service_config.cf::test_service_config_permissions::config file is world-writable (should fail) -> /tmp/cfengine_test_suite_service.conf has permissions 640, expected 777" + "FAIL tests/test_computed_metrics.cf::test_computed_metrics_comparison::doubled value is less than five (should fail) -> 7.0 is not less than 5.0" +) +for line in "${expected_fails[@]}"; do + grep -qF "$line" "$output_file" || { + echo "FAIL: expected line not found in output: $line" + exit 1 + } +done + +# --- Explicit directory argument --- +# Not a repeat of the run above: a directory argument goes through directory +# expansion (recursive walk, non-test_ files become libraries), not +# auto-discovery, and must still find exactly the same suite. +if cfengine test tests/ >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected the suite to fail when passed as an explicit directory too" + exit 1 +fi +grep -qF "cfengine test: 5 passed, 2 failed, 0 errors" "$output_file" diff --git a/tests/shell/011-test-suite-cfbs-project.sh b/tests/shell/011-test-suite-cfbs-project.sh new file mode 100755 index 0000000..b795dbe --- /dev/null +++ b/tests/shell/011-test-suite-cfbs-project.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e +set -x + +tmpdir=$(mktemp -d) +output_file=$(mktemp) +trap "rm -rf $tmpdir $output_file" EXIT + +# A plain cfbs policy-set project built on the standard community +# masterfiles -- no assert-specific setup needed, `cfengine test` auto-injects +# the assert: promise type into every test run on its own. It has a +# locally-authored module (discovered via */main.cf, same as a real cfbs +# project's own custom modules), and a single test showing the setup(act)/ +# assert flow: act by calling into the project's own bundle, then assert on +# the state it left behind. The module uses the masterfiles standard library +# (m()), which must be available when running. +cp -r tests/shell/fixtures/011-test-suite-cfbs-project/. "$tmpdir/" +cd "$tmpdir" + +if cfengine test >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected the cfbs-project suite to fail (it has a deliberately-failing assertion)" + exit 1 +fi +cat "$output_file" + +grep -qF "cfengine test: 1 passed, 1 failed, 0 errors" "$output_file" +grep -qF "tests/test_app_config.cf::test_app_config" "$output_file" +grep -qF "FAIL tests/test_app_config.cf::test_app_config::app config file is world-writable (should fail) -> /tmp/cfengine_test_suite_app.conf has permissions 644, expected 777" "$output_file" + +# --- --test-only: same result, with the standard library loaded by the +# generated policy instead of the project's promises.cf --- +if cfengine test --test-only >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected the cfbs-project suite to fail with --test-only too" + exit 1 +fi +cat "$output_file" +grep -qF "cfengine test: 1 passed, 1 failed, 0 errors" "$output_file" diff --git a/tests/shell/012-test-full-policy.sh b/tests/shell/012-test-full-policy.sh new file mode 100755 index 0000000..3977d1e --- /dev/null +++ b/tests/shell/012-test-full-policy.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e +set -x + +tmpdir=$(mktemp -d) +output_file=$(mktemp) +trap "rm -rf $tmpdir $output_file" EXIT + +cp -r tests/shell/fixtures/012-test-full-policy/. "$tmpdir/" +cd "$tmpdir" + +# The project's real policy (services/init.cf, wired into the default +# bundlesequence via def.json's control_common_bundlesequence_end) writes a +# file whose contents depend on a variable with no project-level default -- +# only a per-test fixture supplies it. By default, the real policy converges +# fresh inside each test's own container, using that test's own fixture. +# With --test-only, that bundle only runs once, during the shared +# setup/bootstrap step, with no fixture in scope, so the variable is never +# resolved. + +# --- Default: the real bundle converges fresh per test file, using that +# test's own fixture, so both pass. --- +if ! cfengine test >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected the suite to pass by default (full policy run)" + exit 1 +fi +cat "$output_file" +grep -qF "cfengine test: 2 passed, 0 failed, 0 errors" "$output_file" + +# --- With --test-only: the real bundle only ran once, with no per-test +# fixture in scope, so neither test's expected content matches. --- +if cfengine test --test-only >"$output_file" 2>&1; then + cat "$output_file" + echo "FAIL: expected the suite to fail with --test-only" + exit 1 +fi +cat "$output_file" +grep -qF "cfengine test: 0 passed, 2 failed, 0 errors" "$output_file" diff --git a/tests/shell/fixtures/009-test-assert-vocab/test_assert_vocab.cf b/tests/shell/fixtures/009-test-assert-vocab/test_assert_vocab.cf new file mode 100644 index 0000000..30a836e --- /dev/null +++ b/tests/shell/fixtures/009-test-assert-vocab/test_assert_vocab.cf @@ -0,0 +1,90 @@ +bundle agent test_assert_vocab +{ + vars: + "fruits" slist => { "apple", "banana", "cherry" }; + "counts" ilist => { "1", "2", "3" }; + "prices" rlist => { "1.50", "2.000000", "3.75" }; + + files: + "/tmp/assert_vocab_fixture.txt" + create => "true", + content => "hello world"; + + assert: + # int + "int_equals_pass" + int => "5", + equals => "5"; + + "int_equals_fail" + int => "5", + equals => "6"; + + "int_greater_than_pass" + int => "5", + greater_than => "1"; + + "int_less_than_fail" + int => "5", + less_than => "1"; + + # real + "real_equals_pass" + real => "3.140000", + equals => "3.14"; + + # str + "str_equals_pass" + str => "hello", + equals => "hello"; + + "str_not_equals_fail" + str => "hello", + not_equals => "hello"; + + "str_contains_pass" + str => "hello world", + contains => "world"; + + # file + "file_exists_pass" + file => "/tmp/assert_vocab_fixture.txt", + exists => "true"; + + "file_contents_fail" + file => "/tmp/assert_vocab_fixture.txt", + contents => "wrong contents"; + + # dir + "dir_exists_pass" + dir => "/tmp", + exists => "true"; + + "dir_exists_fail" + dir => "/this/path/should/not/exist/hopefully", + exists => "true"; + + # command + "command_output_pass" + command => "echo hello", + output => "hello"; + + # slist + "slist_equals_pass" + slist => "@(fruits)", + equals => { "apple", "banana", "cherry" }; + + "slist_contains_fail" + slist => "@(fruits)", + contains => "durian"; + + # ilist (numeric-aware comparison) + "ilist_contains_pass" + ilist => "@(counts)", + contains => "2"; + + # rlist (numeric-aware: "2" should match "2.000000") + "rlist_contains_pass" + rlist => "@(prices)", + contains => "2"; +} diff --git a/tests/shell/fixtures/010-test-suite-standalone/tests/fixtures/test_service_config.json b/tests/shell/fixtures/010-test-suite-standalone/tests/fixtures/test_service_config.json new file mode 100644 index 0000000..6902c69 --- /dev/null +++ b/tests/shell/fixtures/010-test-suite-standalone/tests/fixtures/test_service_config.json @@ -0,0 +1,3 @@ +{ + "vars": { "expected_contents": "mode=standalone\nretries=3\n" } +} diff --git a/tests/shell/fixtures/010-test-suite-standalone/tests/test_computed_metrics.cf b/tests/shell/fixtures/010-test-suite-standalone/tests/test_computed_metrics.cf new file mode 100644 index 0000000..220039f --- /dev/null +++ b/tests/shell/fixtures/010-test-suite-standalone/tests/test_computed_metrics.cf @@ -0,0 +1,31 @@ +bundle agent variables +{ + vars: + "doubled" string => eval("3.5 * 2", "math", "infix"); +} + +bundle agent test_computed_metrics_equality +{ + methods: + "variables"; + + assert: + "doubled value equals seven" + real => "$(variables.doubled)", + equals => "7"; +} + +bundle agent test_computed_metrics_comparison +{ + methods: + "variables"; + + assert: + "doubled value is greater than five" + real => "$(variables.doubled)", + greater_than => "5"; + + "doubled value is less than five (should fail)" + real => "$(variables.doubled)", + less_than => "5"; +} diff --git a/tests/shell/fixtures/010-test-suite-standalone/tests/test_service_config.cf b/tests/shell/fixtures/010-test-suite-standalone/tests/test_service_config.cf new file mode 100644 index 0000000..210da8b --- /dev/null +++ b/tests/shell/fixtures/010-test-suite-standalone/tests/test_service_config.cf @@ -0,0 +1,41 @@ +bundle agent test_service_config_contents +{ + files: + "/tmp/cfengine_test_suite_service.conf" + create => "true", + content => "$(def.expected_contents)", + perms => demo_perms; + + assert: + "config file exists" + file => "/tmp/cfengine_test_suite_service.conf", + exists => "true"; + + "config file has the fixture's contents" + file => "/tmp/cfengine_test_suite_service.conf", + contents => "$(def.expected_contents)"; +} + +bundle agent test_service_config_permissions +{ + files: + "/tmp/cfengine_test_suite_service.conf" + create => "true", + content => "$(def.expected_contents)", + perms => demo_perms; + + assert: + "config file has mode 0640" + file => "/tmp/cfengine_test_suite_service.conf", + perms => "0640"; + + "config file is world-writable (should fail)" + file => "/tmp/cfengine_test_suite_service.conf", + perms => "0777"; +} + +body perms demo_perms +{ + mode => "0640"; + rxdirs => "false"; +} diff --git a/tests/shell/fixtures/011-test-suite-cfbs-project/app_module/main.cf b/tests/shell/fixtures/011-test-suite-cfbs-project/app_module/main.cf new file mode 100644 index 0000000..be0910b --- /dev/null +++ b/tests/shell/fixtures/011-test-suite-cfbs-project/app_module/main.cf @@ -0,0 +1,8 @@ +bundle agent configure_app +{ + files: + "/tmp/cfengine_test_suite_app.conf" + create => "true", + content => "listen_port=8080", + perms => m("644"); +} diff --git a/tests/shell/fixtures/011-test-suite-cfbs-project/cfbs.json b/tests/shell/fixtures/011-test-suite-cfbs-project/cfbs.json new file mode 100644 index 0000000..77f157f --- /dev/null +++ b/tests/shell/fixtures/011-test-suite-cfbs-project/cfbs.json @@ -0,0 +1,17 @@ +{ + "name": "Example project", + "description": "Example description", + "type": "policy-set", + "git": false, + "build": [ + { + "name": "masterfiles", + "description": "Official CFEngine Masterfiles Policy Framework (MPF)", + "url": "https://github.com/cfengine/masterfiles", + "commit": "1ac0cef8c592bfca1ce0e842744b354f90b5759b", + "branch": "master", + "added_by": "cfbs init", + "steps": ["run ./prepare.sh -y", "copy ./ ./"] + } + ] +} diff --git a/tests/shell/fixtures/011-test-suite-cfbs-project/tests/test_app_config.cf b/tests/shell/fixtures/011-test-suite-cfbs-project/tests/test_app_config.cf new file mode 100644 index 0000000..50f3e0e --- /dev/null +++ b/tests/shell/fixtures/011-test-suite-cfbs-project/tests/test_app_config.cf @@ -0,0 +1,14 @@ +bundle agent test_app_config +{ + methods: + "configure" usebundle => configure_app; + + assert: + "app config file exists" + file => "/tmp/cfengine_test_suite_app.conf", + exists => "true"; + + "app config file is world-writable (should fail)" + file => "/tmp/cfengine_test_suite_app.conf", + perms => "0777"; +} diff --git a/tests/shell/fixtures/012-test-full-policy/cfbs.json b/tests/shell/fixtures/012-test-full-policy/cfbs.json new file mode 100644 index 0000000..fcbb05d --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/cfbs.json @@ -0,0 +1,27 @@ +{ + "name": "Full policy example", + "description": "Example description", + "type": "policy-set", + "git": false, + "build": [ + { + "name": "masterfiles", + "description": "Official CFEngine Masterfiles Policy Framework (MPF)", + "url": "https://github.com/cfengine/masterfiles", + "commit": "1ac0cef8c592bfca1ce0e842744b354f90b5759b", + "branch": "master", + "added_by": "cfbs init", + "steps": ["run ./prepare.sh -y", "copy ./ ./"] + }, + { + "name": "./overrides/", + "description": "Local subdirectory added using cfbs command line", + "tags": ["local"], + "added_by": "cfbs add", + "steps": [ + "copy ./def.json def.json", + "copy ./services/init.cf services/init.cf" + ] + } + ] +} diff --git a/tests/shell/fixtures/012-test-full-policy/overrides/def.json b/tests/shell/fixtures/012-test-full-policy/overrides/def.json new file mode 100644 index 0000000..320ac1a --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/overrides/def.json @@ -0,0 +1,3 @@ +{ + "vars": { "control_common_bundlesequence_end": ["real_policy_work"] } +} diff --git a/tests/shell/fixtures/012-test-full-policy/overrides/services/init.cf b/tests/shell/fixtures/012-test-full-policy/overrides/services/init.cf new file mode 100644 index 0000000..7faff0d --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/overrides/services/init.cf @@ -0,0 +1,8 @@ +bundle agent real_policy_work +{ + files: + "/tmp/cfengine_test_suite_full_policy.txt" + create => "true", + content => "ran for $(def.instance_id) +"; +} diff --git a/tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_a.json b/tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_a.json new file mode 100644 index 0000000..c2cee3a --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_a.json @@ -0,0 +1,3 @@ +{ + "vars": { "instance_id": "A" } +} diff --git a/tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_b.json b/tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_b.json new file mode 100644 index 0000000..86ec666 --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/tests/fixtures/test_instance_b.json @@ -0,0 +1,3 @@ +{ + "vars": { "instance_id": "B" } +} diff --git a/tests/shell/fixtures/012-test-full-policy/tests/test_instance_a.cf b/tests/shell/fixtures/012-test-full-policy/tests/test_instance_a.cf new file mode 100644 index 0000000..bfef3fd --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/tests/test_instance_a.cf @@ -0,0 +1,8 @@ +bundle agent test_instance_a +{ + assert: + "real policy ran for instance A" + file => "/tmp/cfengine_test_suite_full_policy.txt", + contents => "ran for A +"; +} diff --git a/tests/shell/fixtures/012-test-full-policy/tests/test_instance_b.cf b/tests/shell/fixtures/012-test-full-policy/tests/test_instance_b.cf new file mode 100644 index 0000000..21e5bcf --- /dev/null +++ b/tests/shell/fixtures/012-test-full-policy/tests/test_instance_b.cf @@ -0,0 +1,8 @@ +bundle agent test_instance_b +{ + assert: + "real policy ran for instance B" + file => "/tmp/cfengine_test_suite_full_policy.txt", + contents => "ran for B +"; +} From 3e45a905883ca0387d3cf806fe770b75b2611bea Mon Sep 17 00:00:00 2001 From: Simon Halvorsen Date: Tue, 10 Nov 2026 10:12:41 +0100 Subject: [PATCH 3/4] Documented `cfengine test` in CHEATSHEET.md Ticket: ENT-14444 Co-Authored-By: Claude Code Signed-off-by: Simon Halvorsen --- CHEATSHEET.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/CHEATSHEET.md b/CHEATSHEET.md index a9303da..a4082a2 100644 --- a/CHEATSHEET.md +++ b/CHEATSHEET.md @@ -60,6 +60,37 @@ cfengine build --hub myhub --non-interactive cfengine deploy --hub myhub ``` +## Testing policy with `cfengine test` + +Runs `test_*.cf` files (bundles that contain their own `assert:` section) in disposable Docker containers and reports a pytest-style pass/fail summary. + +```bash +# Auto-discover and run tests/test_*.cf +cfengine test + +# Run specific files or directories instead +cfengine test tests/test_foo.cf +cfengine test tests/ + +# Inside a cfbs project: builds first, then runs each test file at the end of +# the project's real promises.cf (with that test's fixture applied), so the +# tests see the state a full agent run leaves behind. A fixture +# (tests/fixtures/test_.json) is merged into the project's def.json +cfengine test + +# Test against a custom base image (e.g. a different distro), bypassing the build cache +cfengine test --dockerfile ./my-ubuntu.Dockerfile --rebuild + +# Inside a cfbs project: run only the test bundles, from a minimal generated +# policy. Faster, but the project's policy then only ran once during setup, +# without any fixture, and every test sees that snapshot +cfengine test --test-only + +# Outside a cfbs project: test against a built masterfiles directory, e.g. to +# use the standard library (m(), insert_lines(), ...) of that version +cfengine test --masterfiles ./out/masterfiles +``` + ## Running the CFEngine agent ```bash From b052f2f709e4de4c4cda2d86d3f82a2be86dc779 Mon Sep 17 00:00:00 2001 From: Simon Halvorsen Date: Tue, 10 Nov 2026 11:18:34 +0100 Subject: [PATCH 4/4] Find `cfengine test` bundles via the syntax tree instead of a regex Ticket: ENT-14444 Co-Authored-By: Claude Code Signed-off-by: Simon Halvorsen --- src/cfengine_cli/container.py | 97 +++++++++++++++++++++++++---------- tests/unit/test_container.py | 35 +++++++++++++ 2 files changed, 105 insertions(+), 27 deletions(-) diff --git a/src/cfengine_cli/container.py b/src/cfengine_cli/container.py index a9ee61b..6986377 100644 --- a/src/cfengine_cli/container.py +++ b/src/cfengine_cli/container.py @@ -20,6 +20,7 @@ run_and_scan_asserts, print_assert_totals, ) +from cfengine_cli.lint import _parse_policy_file from cfengine_cli.utils import UserError _DOCKERFILE_DIR = os.path.join(os.path.dirname(__file__), "docker", "test-agent") @@ -27,8 +28,6 @@ _ASSERT_MODULE_DIR = "/var/cfengine/modules/promises" _STDLIB = "/var/cfengine/masterfiles/lib/stdlib.cf" _PROMISE_AGENT_DECLARATION_RE = re.compile(r"promise\s+agent\s+(\w+)") -_BUNDLE_AGENT_DECLARATION_RE = re.compile(r"bundle\s+agent\s+(\w+)") -_ASSERT_SECTION_RE = re.compile(r"^\s*assert\s*:", re.MULTILINE) def assert_module_declaration_path() -> str: @@ -237,25 +236,42 @@ def _read_file(path: str) -> str: return file.read() -def _bundle_bodies_declared_in(content: str) -> list: - """[(name, body), ...] for every `bundle agent { ... }` in - content, in order, body being the brace-matched text between (and - including) its own braces.""" - bodies = [] - for match in _BUNDLE_AGENT_DECLARATION_RE.finditer(content): - name = match.group(1) - start = content.index("{", match.end()) - depth = 0 - end = start - for end in range(start, len(content)): - if content[end] == "{": - depth += 1 - elif content[end] == "}": - depth -= 1 - if depth == 0: - break - bodies.append((name, content[start : end + 1])) - return bodies +def _node_text(node) -> str: + return node.text.decode() + + +def _file_control_namespace(body_block) -> str | None: + """The namespace a `body file control` block sets, if any.""" + kinds = {child.type: _node_text(child) for child in body_block.children} + if (kinds.get("body_block_type"), kinds.get("body_block_name")) != ( + "file", + "control", + ): + return None + for body in body_block.children: + if body.type != "body_block_body": + continue + for attribute in body.children: + if attribute.type != "attribute": + continue + name, *rest = attribute.children + if _node_text(name) == "namespace" and rest: + return _node_text(rest[-1]).strip("\"'") + return None + + +def _has_assert_section(bundle_block) -> bool: + for body in bundle_block.children: + if body.type != "bundle_block_body": + continue + for section in body.children: + if section.type != "bundle_section": + continue + guard = section.children[0] if section.children else None + if guard is not None and guard.type == "promise_guard": + if _node_text(guard).rstrip(":").strip() == "assert": + return True + return False def _bundle_names_declared_in(path: str) -> list: @@ -264,12 +280,39 @@ def _bundle_names_declared_in(path: str) -> list: `assert:` section (e.g. a shared bundle computing some vars) is just available to be called via `methods:` from a test bundle, like any other bundle in this file's inputs, without being reported as a test of its - own.""" - names = [ - name - for name, body in _bundle_bodies_declared_in(_read_file(path)) - if _ASSERT_SECTION_RE.search(body) - ] + own. + + Uses the same syntax tree as `cfengine lint`, so comments and strings + can't fake a bundle or a section. A bundle after a `body file control` + namespace is returned namespaced (`ns:name`), as the bundlesequence + needs it.""" + tree, _lines, _data = _parse_policy_file(path) + if tree.root_node.has_error: + raise UserError(f"Could not parse {path} -- check it with `cfengine lint`") + + names = [] + namespace = "default" + for node in tree.root_node.children: + if node.type == "body_block": + namespace = _file_control_namespace(node) or namespace + continue + if node.type != "bundle_block": + continue + kinds = {child.type: child for child in node.children} + if _node_text(kinds["bundle_block_type"]) != "agent": + continue + if not _has_assert_section(node): + continue + name = _node_text(kinds["bundle_block_name"]) + if "parameter_list" in kinds: + raise UserError( + f"Test bundle '{name}' in {path} takes parameters -- a bundle " + "with an assert: section is run directly, so it can't take " + "any. Move the asserts into a bundle without parameters, or " + "call this one from such a bundle via methods:." + ) + names.append(name if namespace == "default" else f"{namespace}:{name}") + if not names: raise UserError(f"No bundle with an 'assert:' section found in {path}") return names diff --git a/tests/unit/test_container.py b/tests/unit/test_container.py index b87df13..9cbfa6d 100644 --- a/tests/unit/test_container.py +++ b/tests/unit/test_container.py @@ -270,6 +270,41 @@ def test_bundle_names_declared_in_excludes_bundles_without_assert_section(tmp_pa assert container._bundle_names_declared_in(str(path)) == ["test_foo"] +def test_bundle_names_declared_in_ignores_comments_and_strings(tmp_path): + path = tmp_path / "test_foo.cf" + path.write_text( + "# bundle agent commented_out\n" + 'bundle agent helper\n{\n reports:\n "assert: { in a string";\n}\n' + '\nbundle agent test_foo\n{\n assert:\n "x" pass => "true";\n}\n' + ) + assert container._bundle_names_declared_in(str(path)) == ["test_foo"] + + +def test_bundle_names_declared_in_namespaces_bundles(tmp_path): + path = tmp_path / "test_foo.cf" + path.write_text( + 'body file control\n{\n namespace => "myns";\n}\n' + '\nbundle agent test_foo\n{\n assert:\n "x" pass => "true";\n}\n' + ) + assert container._bundle_names_declared_in(str(path)) == ["myns:test_foo"] + + +def test_bundle_names_declared_in_rejects_test_bundle_with_parameters(tmp_path): + path = tmp_path / "test_foo.cf" + path.write_text( + 'bundle agent test_foo(arg)\n{\n assert:\n "x" pass => "true";\n}\n' + ) + with pytest.raises(UserError, match="takes parameters"): + container._bundle_names_declared_in(str(path)) + + +def test_bundle_names_declared_in_raises_user_error_on_unparsable_file(tmp_path): + path = tmp_path / "test_foo.cf" + path.write_text("bundle agent test_foo\n") + with pytest.raises(UserError, match="Could not parse"): + container._bundle_names_declared_in(str(path)) + + def test_bundle_names_declared_in_raises_without_declaration(tmp_path): path = tmp_path / "test_foo.cf" path.write_text("# no bundle here\n")