From 3df6b36f97200cf30534be89fd9f60466d3be0e5 Mon Sep 17 00:00:00 2001 From: tadhgdowdall Date: Tue, 9 Jun 2026 19:22:48 +0100 Subject: [PATCH 01/10] feat(ir): add deferred pipeline evaluation types --- crates/hm-pipeline-ir/src/graph.rs | 63 +++++++- crates/hm-pipeline-ir/src/lib.rs | 8 +- .../hm-pipeline-ir/tests/schema_snapshot.rs | 6 + ...apshot__command_step_schema_is_stable.snap | 3 +- ...pshot__pipeline_step_schema_is_stable.snap | 139 ++++++++++++++++++ 5 files changed, 210 insertions(+), 9 deletions(-) create mode 100644 crates/hm-pipeline-ir/tests/snapshots/schema_snapshot__pipeline_step_schema_is_stable.snap diff --git a/crates/hm-pipeline-ir/src/graph.rs b/crates/hm-pipeline-ir/src/graph.rs index e1297547..832fc654 100644 --- a/crates/hm-pipeline-ir/src/graph.rs +++ b/crates/hm-pipeline-ir/src/graph.rs @@ -5,11 +5,11 @@ use daggy::Dag; use schemars::JsonSchema as DeriveJsonSchema; use serde::{Deserialize, Serialize}; -/// A single build command within a pipeline. +/// A concrete build command passed to a step runner. /// -/// Serialized as a JSON object inside each graph node's `step` field. -/// The `key` is the unique identifier used to reference this step in -/// edges and log output. +/// Pipeline graph nodes use [`PipelineStep`]. The scheduler converts +/// `StepEval::Cmd` nodes to this runner protocol type immediately before +/// execution. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeriveJsonSchema)] pub struct CommandStep { /// Unique identifier for this step within the pipeline. @@ -42,6 +42,61 @@ pub struct CommandStep { pub runner_args: Option, } +/// Evaluation performed when the scheduler reaches a pipeline node. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeriveJsonSchema)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum StepEval { + /// Execute a concrete shell command. + Cmd { cmd: String }, + /// Invoke a registered DSL target and lower its result to a graph fragment. + Dynamic { target_name: String }, +} + +/// A node in the pipeline graph. +/// +/// Graph nodes always carry a [`StepEval`]. Concrete command nodes are +/// converted to [`CommandStep`] only at the step-runner boundary. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeriveJsonSchema)] +pub struct PipelineStep { + pub key: String, + #[serde(default)] + pub label: Option, + pub eval: StepEval, + #[serde(default)] + pub image: Option, + #[serde(default)] + pub env: Option>, + #[serde(default)] + pub timeout_seconds: Option, + #[serde(default)] + pub cache: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runner: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub runner_args: Option, +} + +impl PipelineStep { + /// Convert a baked command node into the runner protocol type. + #[must_use] + pub fn into_command(self) -> Option { + let StepEval::Cmd { cmd } = self.eval else { + return None; + }; + Some(CommandStep { + key: self.key, + label: self.label, + cmd, + image: self.image, + env: self.env, + timeout_seconds: self.timeout_seconds, + cache: self.cache, + runner: self.runner, + runner_args: self.runner_args, + }) + } +} + /// Snapshot cache configuration for a step. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeriveJsonSchema)] pub struct Cache { diff --git a/crates/hm-pipeline-ir/src/lib.rs b/crates/hm-pipeline-ir/src/lib.rs index cd55c2a9..dae2407d 100644 --- a/crates/hm-pipeline-ir/src/lib.rs +++ b/crates/hm-pipeline-ir/src/lib.rs @@ -1,12 +1,12 @@ //! Pipeline IR, the v0 wire format consumed by the `hm` binary. //! -//! The wire format is a petgraph-serde graph. Nodes carry -//! `CommandStep` + resolved env; edges are `EdgeKind` (`BuildsIn` or -//! `DependsOn`). `PipelineGraph` is the top-level type. +//! The wire format is a petgraph-serde graph. Nodes carry a [`PipelineStep`] +//! plus resolved env; edges are `EdgeKind` (`BuildsIn` or `DependsOn`). +//! `PipelineGraph` is the top-level type. #![forbid(unsafe_code)] #![allow(clippy::multiple_crate_versions, clippy::cargo_common_metadata)] mod graph; -pub use graph::{Cache, CommandStep, EdgeKind, PipelineGraph, Transition}; +pub use graph::{Cache, CommandStep, EdgeKind, PipelineGraph, PipelineStep, StepEval, Transition}; diff --git a/crates/hm-pipeline-ir/tests/schema_snapshot.rs b/crates/hm-pipeline-ir/tests/schema_snapshot.rs index 816f34dc..c5fe6f9b 100644 --- a/crates/hm-pipeline-ir/tests/schema_snapshot.rs +++ b/crates/hm-pipeline-ir/tests/schema_snapshot.rs @@ -5,3 +5,9 @@ fn command_step_schema_is_stable() { let schema = schemars::schema_for!(hm_pipeline_ir::CommandStep); insta::assert_json_snapshot!(schema); } + +#[test] +fn pipeline_step_schema_is_stable() { + let schema = schemars::schema_for!(hm_pipeline_ir::PipelineStep); + insta::assert_json_snapshot!(schema); +} diff --git a/crates/hm-pipeline-ir/tests/snapshots/schema_snapshot__command_step_schema_is_stable.snap b/crates/hm-pipeline-ir/tests/snapshots/schema_snapshot__command_step_schema_is_stable.snap index f3d4d5cf..5b786768 100644 --- a/crates/hm-pipeline-ir/tests/snapshots/schema_snapshot__command_step_schema_is_stable.snap +++ b/crates/hm-pipeline-ir/tests/snapshots/schema_snapshot__command_step_schema_is_stable.snap @@ -1,11 +1,12 @@ --- source: crates/hm-pipeline-ir/tests/schema_snapshot.rs +assertion_line: 6 expression: schema --- { "$schema": "http://json-schema.org/draft-07/schema#", "title": "CommandStep", - "description": "A single build command within a pipeline.\n\nSerialized as a JSON object inside each graph node's `step` field. The `key` is the unique identifier used to reference this step in edges and log output.", + "description": "A concrete build command passed to a step runner.\n\nPipeline graph nodes use [`PipelineStep`]. The scheduler converts `StepEval::Cmd` nodes to this runner protocol type immediately before execution.", "type": "object", "required": [ "cmd", diff --git a/crates/hm-pipeline-ir/tests/snapshots/schema_snapshot__pipeline_step_schema_is_stable.snap b/crates/hm-pipeline-ir/tests/snapshots/schema_snapshot__pipeline_step_schema_is_stable.snap new file mode 100644 index 00000000..520a9fed --- /dev/null +++ b/crates/hm-pipeline-ir/tests/snapshots/schema_snapshot__pipeline_step_schema_is_stable.snap @@ -0,0 +1,139 @@ +--- +source: crates/hm-pipeline-ir/tests/schema_snapshot.rs +expression: schema +--- +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "PipelineStep", + "description": "A node in the pipeline graph.\n\nGraph nodes always carry a [`StepEval`]. Concrete command nodes are converted to [`CommandStep`] only at the step-runner boundary.", + "type": "object", + "required": [ + "eval", + "key" + ], + "properties": { + "key": { + "type": "string" + }, + "label": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "eval": { + "$ref": "#/definitions/StepEval" + }, + "image": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "env": { + "default": null, + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "string" + } + }, + "timeout_seconds": { + "default": null, + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + }, + "cache": { + "default": null, + "anyOf": [ + { + "$ref": "#/definitions/Cache" + }, + { + "type": "null" + } + ] + }, + "runner": { + "type": [ + "string", + "null" + ] + }, + "runner_args": true + }, + "definitions": { + "StepEval": { + "description": "Evaluation performed when the scheduler reaches a pipeline node.", + "oneOf": [ + { + "description": "Execute a concrete shell command.", + "type": "object", + "required": [ + "cmd", + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "cmd" + ] + }, + "cmd": { + "type": "string" + } + } + }, + { + "description": "Invoke a registered DSL target and lower its result to a graph fragment.", + "type": "object", + "required": [ + "target_name", + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "dynamic" + ] + }, + "target_name": { + "type": "string" + } + } + } + ] + }, + "Cache": { + "description": "Snapshot cache configuration for a step.", + "type": "object", + "required": [ + "policy" + ], + "properties": { + "policy": { + "description": "Cache policy name (e.g. `\"content-hash\"`).", + "type": "string" + }, + "key": { + "description": "Explicit cache key override; derived from the step if absent.", + "default": null, + "type": [ + "string", + "null" + ] + } + } + } + } +} From b2f5dec0924249a6ae83910e4c74bb6c7ef54672 Mon Sep 17 00:00:00 2001 From: tadhgdowdall Date: Tue, 9 Jun 2026 19:23:12 +0100 Subject: [PATCH 02/10] refactor(ir): remove legacy command node shape --- .../harmont-py/harmont/_pipeline.py | 6 +- .../harmont-py/harmont/keygen.py | 3 +- .../harmont-py/tests/test_cmake.py | 4 +- .../harmont-py/tests/test_e2e_fixtures.py | 3 +- .../harmont-py/tests/test_elixir.py | 10 +-- .../harmont-py/tests/test_envelope.py | 12 +-- .../hm-dsl-engine/harmont-py/tests/test_go.py | 6 +- .../harmont-py/tests/test_har_28_example.py | 6 +- .../harmont-py/tests/test_json_emit.py | 2 +- .../harmont-py/tests/test_keygen.py | 28 +++--- .../tests/test_pipeline_fixtures.py | 8 +- .../tests/test_pipeline_lowering.py | 2 +- .../harmont-py/tests/test_py_uv.py | 6 +- .../harmont-py/tests/test_python.py | 6 +- .../harmont-py/tests/test_rust.py | 14 +-- .../tests/test_target_cross_module.py | 2 +- .../tests/test_toolchain_compose.py | 4 +- .../harmont-py/tests/test_zig.py | 10 +-- crates/hm-dsl-engine/harmont-ts/CLAUDE.md | 2 +- crates/hm-dsl-engine/harmont-ts/src/keygen.ts | 3 +- .../hm-dsl-engine/harmont-ts/src/pipeline.ts | 5 +- .../harmont-ts/tests/examples.test.ts | 2 +- .../harmont-ts/tests/pipeline.test.ts | 2 +- .../harmont-ts/tests/toolchains/rust.test.ts | 11 ++- .../tests/toolchains/shared.test.ts | 2 +- crates/hm-exec/src/local/scheduler.rs | 4 +- crates/hm-exec/src/request.rs | 42 ++++++++- crates/hm-pipeline-ir/src/graph.rs | 4 +- crates/hm-pipeline-ir/tests/e2e_fixtures.rs | 3 +- crates/hm-pipeline-ir/tests/graph_build.rs | 18 ++-- crates/hm-pipeline-ir/tests/graph_serde.rs | 83 ++++++++++++++++-- .../graph_serde__pipeline_graph_snapshot.snap | 17 +++- crates/hm/tests/default_image_inheritance.rs | 12 +-- tests/e2e/fixtures/python/cmake-advanced.json | 30 +++++-- tests/e2e/fixtures/python/kitchen-sink.json | 50 ++++++++--- tests/e2e/fixtures/python/monorepo-ci.json | 85 +++++++++++++++---- tests/e2e/fixtures/python/rust-release.json | 35 ++++++-- .../fixtures/python/zig-node-polyglot.json | 55 +++++++++--- tests/e2e/fixtures/ts/cmake-advanced.json | 30 +++++-- tests/e2e/fixtures/ts/kitchen-sink.json | 50 ++++++++--- tests/e2e/fixtures/ts/monorepo-ci.json | 85 +++++++++++++++---- tests/e2e/fixtures/ts/rust-release.json | 35 ++++++-- tests/e2e/fixtures/ts/zig-node-polyglot.json | 55 +++++++++--- 43 files changed, 643 insertions(+), 209 deletions(-) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_pipeline.py b/crates/hm-dsl-engine/harmont-py/harmont/_pipeline.py index cf601d86..bce1eaba 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_pipeline.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_pipeline.py @@ -114,10 +114,12 @@ def _lower_to_graph( node_idx = idx_by_id[id(s)] step_key = keys[id(s)] - # Build the CommandStep dict (no "type" or "builds_in" fields). step_dict: dict[str, Any] = { "key": step_key, - "cmd": s.cmd, + "eval": { + "type": "cmd", + "cmd": s.cmd, + }, } if s.label is not None: step_dict["label"] = s.label diff --git a/crates/hm-dsl-engine/harmont-py/harmont/keygen.py b/crates/hm-dsl-engine/harmont-py/harmont/keygen.py index f7285c33..fceb3bd6 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/keygen.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/keygen.py @@ -61,7 +61,8 @@ def resolve_pipeline_keys( cache = step.get("cache") if not cache or cache["policy"] == "none": continue - cmd = step.get("cmd", "") + evaluation = step["eval"] + cmd = evaluation["cmd"] if evaluation["type"] == "cmd" else "" parent = parent_key_map.get(step["key"]) parent_resolved = _lookup_parent(parent, resolved) policy_res = _resolve_policy(cache, cmd, now, base_path, env) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_cmake.py b/crates/hm-dsl-engine/harmont-py/tests/test_cmake.py index a17e1cc5..4d9802ec 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_cmake.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_cmake.py @@ -8,7 +8,7 @@ def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] + return [n["step"]["eval"]["cmd"] for n in p["graph"]["nodes"]] # --------------------------------------------------------------------------- @@ -226,7 +226,7 @@ def test_vcpkg_step_has_on_change_cache_policy(self): proj = hm.cmake(path="svc", deps="vcpkg") p = hm.pipeline([proj.built], default_image="ubuntu:24.04") nodes = p["graph"]["nodes"] - vcpkg_node = next(n for n in nodes if "bootstrap-vcpkg" in n["step"]["cmd"]) + vcpkg_node = next(n for n in nodes if "bootstrap-vcpkg" in n["step"]["eval"]["cmd"]) assert vcpkg_node["step"]["cache"]["policy"] == "on_change" diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_e2e_fixtures.py b/crates/hm-dsl-engine/harmont-py/tests/test_e2e_fixtures.py index 417046ec..ba16f476 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_e2e_fixtures.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_e2e_fixtures.py @@ -160,7 +160,8 @@ def test_e2e_fixture(name: str) -> None: for node in ir["graph"]["nodes"]: assert "key" in node["step"] - assert "cmd" in node["step"] + assert node["step"]["eval"]["type"] == "cmd" + assert "cmd" in node["step"]["eval"] assert isinstance(node["env"], dict) for src, dst, kind in ir["graph"]["edges"]: diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_elixir.py b/crates/hm-dsl-engine/harmont-py/tests/test_elixir.py index 5589999e..ffb14f0c 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_elixir.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_elixir.py @@ -8,12 +8,12 @@ def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] + return [n["step"]["eval"]["cmd"] for n in p["graph"]["nodes"]] def _step_by_substring(p: dict, needle: str) -> dict: for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): + if needle in (n["step"].get("eval", {}).get("cmd") or ""): return n["step"] msg = f"no command step containing {needle!r}" raise AssertionError(msg) @@ -56,8 +56,8 @@ def test_elixir_version_in_install_cmd(): ex = hm.elixir(elixir_version="1.18.3", otp_version="27.3.3") p = hm.pipeline([ex.compile()]) elixir_step = _step_by_substring(p, "elixir-otp") - assert "1.18.3" in elixir_step["cmd"] - assert "27" in elixir_step["cmd"] + assert "1.18.3" in elixir_step["eval"]["cmd"] + assert "27" in elixir_step["eval"]["cmd"] def test_elixir_invalid_version_rejected(): @@ -98,7 +98,7 @@ def test_elixir_plt_cached_on_lock(): assert step.label == ":ex: plt" p = hm.pipeline([step]) plt_ir = next( - n["step"] for n in p["graph"]["nodes"] if "dialyzer --plt" in (n["step"].get("cmd") or "") + n["step"] for n in p["graph"]["nodes"] if "dialyzer --plt" in (n["step"].get("eval", {}).get("cmd") or "") ) assert plt_ir["cache"]["policy"] == "on_change" assert "./mix.lock" in plt_ir["cache"]["paths"] diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_envelope.py b/crates/hm-dsl-engine/harmont-py/tests/test_envelope.py index 430f93be..f4744e3d 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_envelope.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_envelope.py @@ -30,7 +30,7 @@ def _graph_edges(definition): def _step_cmds(definition): - return [n["step"].get("cmd") for n in _graph_nodes(definition)] + return [n["step"].get("eval", {}).get("cmd") for n in _graph_nodes(definition)] def _builds_in_children(definition, parent_key): @@ -72,7 +72,7 @@ def ci() -> hm.Step: assert definition["version"] == "0" nodes = _graph_nodes(definition) assert len(nodes) == 1 - assert nodes[0]["step"]["cmd"] == "echo hi" + assert nodes[0]["step"]["eval"]["cmd"] == "echo hi" assert nodes[0]["step"]["label"] == "hi" @@ -107,7 +107,7 @@ def ci() -> hm.Pipeline: out = json.loads(hm.dump_registry_json()) p = out["pipelines"][0] - cmds = sorted(n["step"]["cmd"] for n in _graph_nodes(p["definition"])) + cmds = sorted(n["step"]["eval"]["cmd"] for n in _graph_nodes(p["definition"])) assert cmds == ["a", "b"] @@ -152,7 +152,7 @@ def ci(): out = json.loads(hm.dump_registry_json()) nodes = _graph_nodes(out["pipelines"][0]["definition"]) - cmds = [n["step"].get("cmd") for n in nodes] + cmds = [n["step"].get("eval", {}).get("cmd") for n in nodes] assert any("go build" in (c or "") for c in cmds) @@ -176,11 +176,11 @@ def ci() -> tuple[hm.Step, ...]: out = json.loads(hm.dump_registry_json()) definition = out["pipelines"][0]["definition"] nodes = _graph_nodes(definition) - apt_nodes = [n for n in nodes if n["step"].get("cmd") == "apt-get update"] + apt_nodes = [n for n in nodes if n["step"].get("eval", {}).get("cmd") == "apt-get update"] assert len(apt_nodes) == 1 # deduplicated via target memoization children = _builds_in_children(definition, apt_nodes[0]["step"]["key"]) assert len(children) == 2 - child_cmds = sorted(n["step"]["cmd"] for n in children) + child_cmds = sorted(n["step"]["eval"]["cmd"] for n in children) assert child_cmds == ["cabal build", "pytest"] diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_go.py b/crates/hm-dsl-engine/harmont-py/tests/test_go.py index 4b0b6fe4..c0ed0ffe 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_go.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_go.py @@ -8,12 +8,12 @@ def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] + return [n["step"]["eval"]["cmd"] for n in p["graph"]["nodes"]] def _step_by_substring(p: dict, needle: str) -> dict: for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): + if needle in (n["step"].get("eval", {}).get("cmd") or ""): return n["step"] msg = f"no command step containing {needle!r}" raise AssertionError(msg) @@ -50,7 +50,7 @@ def test_go_version_in_install_cmd(): go = hm.go(path=".", version="1.23.2") p = hm.pipeline([go.build()]) install = _step_by_substring(p, "go.dev/dl/") - assert "go1.23.2" in install["cmd"] + assert "go1.23.2" in install["eval"]["cmd"] def test_go_invalid_version_rejected(): diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_har_28_example.py b/crates/hm-dsl-engine/harmont-py/tests/test_har_28_example.py index 2e65fbcb..5c3c2196 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_har_28_example.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_har_28_example.py @@ -58,13 +58,13 @@ def ci(): p = out["pipelines"][0] nodes = _graph_nodes(p["definition"]) - cmds = [n["step"].get("cmd") for n in nodes] + cmds = [n["step"].get("eval", {}).get("cmd") for n in nodes] assert any("pytest -v" in (c or "") for c in cmds) assert any("go build" in (c or "") for c in cmds) assert any("npm" in (c or "") for c in cmds) # apt-base used by the venv chain appears exactly once (memoized). - apt_update_nodes = [n for n in nodes if n["step"].get("cmd") == "apt-get update"] + apt_update_nodes = [n for n in nodes if n["step"].get("eval", {}).get("cmd") == "apt-get update"] assert len(apt_update_nodes) == 1 @@ -75,5 +75,5 @@ def ci(): out = json.loads(hm.dump_registry_json()) nodes = _graph_nodes(out["pipelines"][0]["definition"]) - cmds = [n["step"]["cmd"] for n in nodes] + cmds = [n["step"]["eval"]["cmd"] for n in nodes] assert "cd cidsl/py && pytest -v" in cmds diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_json_emit.py b/crates/hm-dsl-engine/harmont-py/tests/test_json_emit.py index 59b27705..30182186 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_json_emit.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_json_emit.py @@ -69,7 +69,7 @@ def test_minimal_command(): step = _nodes(out)[0]["step"] assert step["key"] == "hello" assert step["label"] == "hello" - assert step["cmd"] == "echo hi" + assert step["eval"]["cmd"] == "echo hi" # No "type" or "builds_in" field on step dicts. assert "type" not in step assert "builds_in" not in step diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_keygen.py b/crates/hm-dsl-engine/harmont-py/tests/test_keygen.py index 38a30e92..d831c196 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_keygen.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_keygen.py @@ -34,7 +34,7 @@ def test_none_policy_emits_no_key(): graph = _make_graph( [ { - "step": {"key": "a", "cmd": "echo", "cache": {"policy": "none"}}, + "step": {"key": "a", "eval": {"type": "cmd", "cmd": "echo"}, "cache": {"policy": "none"}}, "env": {}, }, ] @@ -56,7 +56,7 @@ def test_forever_policy_key_matches_scheme_formula(): { "step": { "key": "a", - "cmd": "echo hi", + "eval": {"type": "cmd", "cmd": "echo hi"}, "cache": {"policy": "forever", "env_keys": []}, }, "env": {}, @@ -85,7 +85,7 @@ def test_ttl_policy_key_includes_bucket(): { "step": { "key": "a", - "cmd": "x", + "eval": {"type": "cmd", "cmd": "x"}, "cache": {"policy": "ttl", "duration_seconds": 3600, "env_keys": []}, }, "env": {}, @@ -117,7 +117,7 @@ def test_on_change_reads_file_contents(): { "step": { "key": "a", - "cmd": "make", + "eval": {"type": "cmd", "cmd": "make"}, "cache": {"policy": "on_change", "paths": ["file.txt"]}, }, "env": {}, @@ -158,7 +158,7 @@ def test_on_change_handles_directory_paths(): { "step": { "key": "s", - "cmd": "make", + "eval": {"type": "cmd", "cmd": "make"}, "cache": {"policy": "on_change", "paths": ["dir/"]}, }, "env": {}, @@ -181,7 +181,7 @@ def test_on_change_handles_directory_paths(): { "step": { "key": "s", - "cmd": "make", + "eval": {"type": "cmd", "cmd": "make"}, "cache": {"policy": "on_change", "paths": ["dir/"]}, }, "env": {}, @@ -205,7 +205,7 @@ def test_on_change_handles_directory_paths(): { "step": { "key": "s", - "cmd": "make", + "eval": {"type": "cmd", "cmd": "make"}, "cache": {"policy": "on_change", "paths": ["dir/"]}, }, "env": {}, @@ -230,7 +230,7 @@ def test_on_change_missing_path_skipped(): { "step": { "key": "s", - "cmd": "make", + "eval": {"type": "cmd", "cmd": "make"}, "cache": {"policy": "on_change", "paths": ["nope/"]}, }, "env": {}, @@ -254,7 +254,7 @@ def test_env_keys_are_sorted_and_picked_up(): { "step": { "key": "a", - "cmd": "echo", + "eval": {"type": "cmd", "cmd": "echo"}, "cache": {"policy": "forever", "env_keys": ["BAR", "FOO"]}, }, "env": {}, @@ -284,7 +284,7 @@ def test_parent_key_chains_through_resolved_cache_keys(): { "step": { "key": "a", - "cmd": "x", + "eval": {"type": "cmd", "cmd": "x"}, "cache": {"policy": "forever", "env_keys": []}, }, "env": {}, @@ -292,7 +292,7 @@ def test_parent_key_chains_through_resolved_cache_keys(): { "step": { "key": "b", - "cmd": "y", + "eval": {"type": "cmd", "cmd": "y"}, "cache": {"policy": "forever", "env_keys": []}, }, "env": {}, @@ -323,7 +323,7 @@ def test_compose_concatenates_subpolicies(): { "step": { "key": "a", - "cmd": "z", + "eval": {"type": "cmd", "cmd": "z"}, "cache": { "policy": "compose", "sub_policies": [ @@ -359,13 +359,13 @@ def test_parent_without_cache_is_planerror(): graph = _make_graph( [ { - "step": {"key": "a", "cmd": "x"}, + "step": {"key": "a", "eval": {"type": "cmd", "cmd": "x"}}, "env": {}, }, { "step": { "key": "b", - "cmd": "y", + "eval": {"type": "cmd", "cmd": "y"}, "cache": {"policy": "forever", "env_keys": []}, }, "env": {}, diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_fixtures.py b/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_fixtures.py index 037827b2..77f940a7 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_fixtures.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_fixtures.py @@ -31,7 +31,7 @@ def ci() -> hm.Step: out = json.loads(hm.dump_registry_json()) nodes = _graph_nodes(out["pipelines"][0]["definition"]) - assert any(n["step"].get("cmd") == "echo hi" for n in nodes) + assert any(n["step"].get("eval", {}).get("cmd") == "echo hi" for n in nodes) def test_pipeline_receives_target_as_param(): @@ -45,7 +45,7 @@ def ci(apt_base: hm.Target[hm.Step]) -> hm.Step: out = json.loads(hm.dump_registry_json()) nodes = _graph_nodes(out["pipelines"][0]["definition"]) - cmds = [n["step"].get("cmd") for n in nodes] + cmds = [n["step"].get("eval", {}).get("cmd") for n in nodes] assert "apt-get update" in cmds assert "smoke" in cmds @@ -72,9 +72,9 @@ def ci( out = json.loads(hm.dump_registry_json()) nodes = _graph_nodes(out["pipelines"][0]["definition"]) - apt = [n for n in nodes if n["step"].get("cmd") == "apt-get update"] + apt = [n for n in nodes if n["step"].get("eval", {}).get("cmd") == "apt-get update"] assert len(apt) == 1 # apt_base deduped via target memoization - cmds = sorted(n["step"].get("cmd") for n in nodes) + cmds = sorted(n["step"].get("eval", {}).get("cmd") for n in nodes) assert "cabal build" in cmds assert "pytest" in cmds diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_lowering.py b/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_lowering.py index a02a55ad..84b5fbac 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_lowering.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_lowering.py @@ -118,7 +118,7 @@ def test_command_omits_optional_fields_when_unset(): step = graph["nodes"][0]["step"] # Required fields present. assert "key" in step - assert "cmd" in step + assert step["eval"] == {"type": "cmd", "cmd": "make"} # No "type" or "builds_in" fields in the new format. assert "type" not in step assert "builds_in" not in step diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_py_uv.py b/crates/hm-dsl-engine/harmont-py/tests/test_py_uv.py index c6f4d300..0a736ee0 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_py_uv.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_py_uv.py @@ -9,12 +9,12 @@ def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] + return [n["step"]["eval"]["cmd"] for n in p["graph"]["nodes"]] def _step_by_substring(p: dict, needle: str) -> dict: for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): + if needle in (n["step"].get("eval", {}).get("cmd") or ""): return n["step"] msg = f"no command step containing {needle!r}" raise AssertionError(msg) @@ -167,7 +167,7 @@ def test_pinned_version(self): proj = hm.py.uv(path=".", version="0.4.18") p = hm.pipeline([proj.test()]) install = _step_by_substring(p, "astral.sh/uv/install.sh") - assert "UV_VERSION=0.4.18" in install["cmd"] + assert "UV_VERSION=0.4.18" in install["eval"]["cmd"] def test_invalid_version_rejected(self): with pytest.raises(ValueError, match="invalid version"): diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_python.py b/crates/hm-dsl-engine/harmont-py/tests/test_python.py index 55cf98f0..bfc90d6a 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_python.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_python.py @@ -9,12 +9,12 @@ def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] + return [n["step"]["eval"]["cmd"] for n in p["graph"]["nodes"]] def _step_by_substring(p: dict, needle: str) -> dict: for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): + if needle in (n["step"].get("eval", {}).get("cmd") or ""): return n["step"] msg = f"no command step containing {needle!r}" raise AssertionError(msg) @@ -142,7 +142,7 @@ def test_python_uv_version_in_install_cmd(): py = hm.python(path=".", uv_version="0.4.18") p = hm.pipeline([py.test()]) install = _step_by_substring(p, "astral.sh/uv/install.sh") - assert "UV_VERSION=0.4.18" in install["cmd"] + assert "UV_VERSION=0.4.18" in install["eval"]["cmd"] def test_python_invalid_uv_version_rejected(): diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_rust.py b/crates/hm-dsl-engine/harmont-py/tests/test_rust.py index fe65c060..fe997429 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_rust.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_rust.py @@ -9,12 +9,12 @@ def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] + return [n["step"]["eval"]["cmd"] for n in p["graph"]["nodes"]] def _step_by_substring(p: dict, needle: str) -> dict: for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): + if needle in (n["step"].get("eval", {}).get("cmd") or ""): return n["step"] msg = f"no command step containing {needle!r}" raise AssertionError(msg) @@ -62,20 +62,20 @@ def test_default_components(self): tc = hm.rust.toolchain(path=".") p = hm.pipeline([tc.build()]) rustup = _step_by_substring(p, "sh.rustup.rs") - assert "--component clippy,rustfmt" in rustup["cmd"] + assert "--component clippy,rustfmt" in rustup["eval"]["cmd"] def test_components_override(self): tc = hm.rust.toolchain(path=".", components=("clippy",)) p = hm.pipeline([tc.build()]) rustup = _step_by_substring(p, "sh.rustup.rs") - assert "--component clippy" in rustup["cmd"] - assert "rustfmt" not in rustup["cmd"] + assert "--component clippy" in rustup["eval"]["cmd"] + assert "rustfmt" not in rustup["eval"]["cmd"] def test_version_in_rustup_cmd(self): tc = hm.rust.toolchain(path=".", version="1.81.0") p = hm.pipeline([tc.build()]) rustup = _step_by_substring(p, "sh.rustup.rs") - assert "--default-toolchain 1.81.0" in rustup["cmd"] + assert "--default-toolchain 1.81.0" in rustup["eval"]["cmd"] def test_invalid_version_rejected(self): with pytest.raises(ValueError, match="version"): @@ -259,4 +259,4 @@ def test_version_forwarded(self): proj = hm.rust.project(path=".", version="1.81.0") p = hm.pipeline([proj.test()]) rustup = _step_by_substring(p, "sh.rustup.rs") - assert "--default-toolchain 1.81.0" in rustup["cmd"] + assert "--default-toolchain 1.81.0" in rustup["eval"]["cmd"] diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_target_cross_module.py b/crates/hm-dsl-engine/harmont-py/tests/test_target_cross_module.py index ac5fcd90..c3cf6f90 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_target_cross_module.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_target_cross_module.py @@ -42,7 +42,7 @@ def ci(py_test: hm.Target[hm.Step]) -> hm.Step: out = json.loads(hm.dump_registry_json()) nodes = out["pipelines"][0]["definition"]["graph"]["nodes"] - cmds = sorted(n["step"].get("cmd") for n in nodes) + cmds = sorted(n["step"].get("eval", {}).get("cmd") for n in nodes) assert "apt-get update" in cmds assert "cd cidsl/py && pytest -v" in cmds diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_toolchain_compose.py b/crates/hm-dsl-engine/harmont-py/tests/test_toolchain_compose.py index d64a7e36..2d2e6c0f 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_toolchain_compose.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_toolchain_compose.py @@ -6,7 +6,7 @@ def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] + return [n["step"]["eval"]["cmd"] for n in p["graph"]["nodes"]] def test_stack_npm_on_spec_step(): @@ -57,7 +57,7 @@ def test_mixed_pipeline_compiles(): def _step_by_substring(p: dict, needle: str) -> dict: for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): + if needle in (n["step"].get("eval", {}).get("cmd") or ""): return n["step"] msg = f"no command step containing {needle!r}" raise AssertionError(msg) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_zig.py b/crates/hm-dsl-engine/harmont-py/tests/test_zig.py index 4bb55296..ca9c2906 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_zig.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_zig.py @@ -8,12 +8,12 @@ def _cmds(p: dict) -> list[str]: - return [n["step"]["cmd"] for n in p["graph"]["nodes"]] + return [n["step"]["eval"]["cmd"] for n in p["graph"]["nodes"]] def _step_by_substring(p: dict, needle: str) -> dict: for n in p["graph"]["nodes"]: - if needle in (n["step"].get("cmd") or ""): + if needle in (n["step"].get("eval", {}).get("cmd") or ""): return n["step"] raise AssertionError(needle) @@ -39,7 +39,7 @@ def test_zig_version_in_install_cmd(): z = hm.zig(path=".", version="0.14.1") p = hm.pipeline([z.build()]) install = _step_by_substring(p, "ziglang.org") - assert "0.14.1" in install["cmd"] + assert "0.14.1" in install["eval"]["cmd"] def test_zig_invalid_version_rejected(): @@ -66,7 +66,7 @@ def test_zig_old_version_uses_old_url_format(): z = hm.zig(path=".", version="0.13.0") p = hm.pipeline([z.build()]) install = _step_by_substring(p, "ziglang.org") - assert "zig-linux-x86_64-0.13.0" in install["cmd"] + assert "zig-linux-x86_64-0.13.0" in install["eval"]["cmd"] def test_zig_new_version_uses_new_url_format(): @@ -74,7 +74,7 @@ def test_zig_new_version_uses_new_url_format(): z = hm.zig(path=".", version="0.14.1") p = hm.pipeline([z.build()]) install = _step_by_substring(p, "ziglang.org") - assert "zig-x86_64-linux-0.14.1" in install["cmd"] + assert "zig-x86_64-linux-0.14.1" in install["eval"]["cmd"] def test_zig_with_base_skips_apt(): diff --git a/crates/hm-dsl-engine/harmont-ts/CLAUDE.md b/crates/hm-dsl-engine/harmont-ts/CLAUDE.md index 51730277..a64250e3 100644 --- a/crates/hm-dsl-engine/harmont-ts/CLAUDE.md +++ b/crates/hm-dsl-engine/harmont-ts/CLAUDE.md @@ -22,7 +22,7 @@ TypeScript pipeline DSL — equivalent of `harmont-py/` (sibling directory). ## IR Compatibility Output must match the v0 IR that `crates/hm-pipeline-ir/` deserializes. -The Rust `CommandStep` accepts: key, cmd, label?, image?, env?, timeout_seconds?, cache?, runner?, runner_args?. +The Rust `PipelineStep` accepts: key, eval, label?, image?, env?, timeout_seconds?, cache?, runner?, runner_args?. Static commands use `eval: { type: "cmd", cmd }`. The Rust `Cache` accepts: policy, key?. Edge kinds: `builds_in`, `depends_on`. Envelope: `{ schema_version: "1", pipelines: [...] }`. diff --git a/crates/hm-dsl-engine/harmont-ts/src/keygen.ts b/crates/hm-dsl-engine/harmont-ts/src/keygen.ts index 2867b3c5..75c80a2f 100644 --- a/crates/hm-dsl-engine/harmont-ts/src/keygen.ts +++ b/crates/hm-dsl-engine/harmont-ts/src/keygen.ts @@ -44,7 +44,8 @@ export function resolvePipelineCacheKeys( const cache = step.cache as Record | undefined; if (!cache || cache.policy === "none") continue; - const cmd = (step.cmd as string) ?? ""; + const evaluation = step.eval as Record; + const cmd = evaluation.type === "cmd" ? (evaluation.cmd as string) : ""; const stepKey = step.key as string; const parentStepKey = parentKeyMap.get(stepKey); const parentResolved = lookupParent(parentStepKey, resolved); diff --git a/crates/hm-dsl-engine/harmont-ts/src/pipeline.ts b/crates/hm-dsl-engine/harmont-ts/src/pipeline.ts index a531fb92..6133e0e8 100644 --- a/crates/hm-dsl-engine/harmont-ts/src/pipeline.ts +++ b/crates/hm-dsl-engine/harmont-ts/src/pipeline.ts @@ -84,7 +84,10 @@ function lowerToGraph( const stepDict: Record = { key: stepKey, - cmd: s._cmd, + eval: { + type: "cmd", + cmd: s._cmd, + }, }; if (s._label != null) stepDict.label = s._label; if (s._cache != null) stepDict.cache = cachePolicyToDict(s._cache); diff --git a/crates/hm-dsl-engine/harmont-ts/tests/examples.test.ts b/crates/hm-dsl-engine/harmont-ts/tests/examples.test.ts index 71288277..8c58ecb6 100644 --- a/crates/hm-dsl-engine/harmont-ts/tests/examples.test.ts +++ b/crates/hm-dsl-engine/harmont-ts/tests/examples.test.ts @@ -49,7 +49,7 @@ describe.skipIf(examples.length === 0)("examples render to v0 IR", () => { // Verify all nodes have required fields for (const node of ci.pipeline.graph.nodes) { expect(node.step.key).toBeDefined(); - expect(node.step.cmd).toBeDefined(); + expect(node.step.eval).toMatchObject({ type: "cmd" }); expect(typeof node.env).toBe("object"); } diff --git a/crates/hm-dsl-engine/harmont-ts/tests/pipeline.test.ts b/crates/hm-dsl-engine/harmont-ts/tests/pipeline.test.ts index 10764699..d3e684f1 100644 --- a/crates/hm-dsl-engine/harmont-ts/tests/pipeline.test.ts +++ b/crates/hm-dsl-engine/harmont-ts/tests/pipeline.test.ts @@ -135,7 +135,7 @@ describe("lowering: optional fields", () => { const ir = pipeline([s]); const step = ir.graph.nodes[0].step; expect(step.key).toBeDefined(); - expect(step.cmd).toBe("make"); + expect(step.eval).toEqual({ type: "cmd", cmd: "make" }); expect("label" in step).toBe(false); expect("timeout_seconds" in step).toBe(false); expect("cache" in step).toBe(false); diff --git a/crates/hm-dsl-engine/harmont-ts/tests/toolchains/rust.test.ts b/crates/hm-dsl-engine/harmont-ts/tests/toolchains/rust.test.ts index f12a3ef9..e51768ec 100644 --- a/crates/hm-dsl-engine/harmont-ts/tests/toolchains/rust.test.ts +++ b/crates/hm-dsl-engine/harmont-ts/tests/toolchains/rust.test.ts @@ -4,11 +4,14 @@ import { sh, timeout } from "../../src/step.js"; import { pipeline } from "../../src/pipeline.js"; const cmds = (ir: ReturnType) => - ir.graph.nodes.map((n: { step: { cmd: string } }) => n.step.cmd); + ir.graph.nodes.map( + (n: { step: { eval: { cmd: string } } }) => n.step.eval.cmd, + ); const stepBySubstring = (ir: ReturnType, needle: string) => { - const node = ir.graph.nodes.find((n: { step: { cmd: string } }) => - n.step.cmd.includes(needle), + const node = ir.graph.nodes.find( + (n: { step: { eval: { cmd: string } } }) => + n.step.eval.cmd.includes(needle), ); if (!node) throw new Error(`no command step containing "${needle}"`); return node.step; @@ -268,6 +271,6 @@ describe("rust.project", () => { const proj = rust.project({ path: ".", version: "1.81.0" }); const ir = pipeline([proj.test()]); const rustup = stepBySubstring(ir, "sh.rustup.rs"); - expect(rustup.cmd).toContain("--default-toolchain 1.81.0"); + expect(rustup.eval.cmd).toContain("--default-toolchain 1.81.0"); }); }); diff --git a/crates/hm-dsl-engine/harmont-ts/tests/toolchains/shared.test.ts b/crates/hm-dsl-engine/harmont-ts/tests/toolchains/shared.test.ts index 4bacbf62..b24245e8 100644 --- a/crates/hm-dsl-engine/harmont-ts/tests/toolchains/shared.test.ts +++ b/crates/hm-dsl-engine/harmont-ts/tests/toolchains/shared.test.ts @@ -52,7 +52,7 @@ describe("aptBase", () => { const p = uv({ path: "dsls/harmont-py", base }); const ir = pipeline([r.build(), p.test()], { defaultImage: "ubuntu:24.04" }); const cmds = ir.graph.nodes.map( - (n: { step: { cmd: string } }) => n.step.cmd, + (n: { step: { eval: { cmd: string } } }) => n.step.eval.cmd, ); const aptSteps = cmds.filter((c: string) => c.includes("apt-get install")); expect(aptSteps).toHaveLength(1); diff --git a/crates/hm-exec/src/local/scheduler.rs b/crates/hm-exec/src/local/scheduler.rs index 551e74ec..68ea09d1 100644 --- a/crates/hm-exec/src/local/scheduler.rs +++ b/crates/hm-exec/src/local/scheduler.rs @@ -362,7 +362,9 @@ async fn execute_step( cancel: CancellationToken, keep_going: bool, ) -> anyhow::Result { - let step_wire = transition.step; + let step_wire = transition.step.into_command().ok_or_else(|| { + anyhow::anyhow!("dynamic step expansion is not implemented by the local backend yet") + })?; let step_key = step_wire.key.clone(); let display_name = step_wire.label.clone().unwrap_or_else(|| { let cmd = step_wire.cmd.trim(); diff --git a/crates/hm-exec/src/request.rs b/crates/hm-exec/src/request.rs index faf8aa5e..fdb683a0 100644 --- a/crates/hm-exec/src/request.rs +++ b/crates/hm-exec/src/request.rs @@ -118,8 +118,8 @@ mod tests { "default_image": "ubuntu:24.04", "graph": { "nodes": [ - {"step": {"key": "a", "cmd": "echo a", "image": "ubuntu:24.04"}, "env": {}}, - {"step": {"key": "b", "cmd": "echo b"}, "env": {}} + {"step": {"key": "a", "eval": {"type": "cmd", "cmd": "echo a"}, "image": "ubuntu:24.04"}, "env": {}}, + {"step": {"key": "b", "eval": {"type": "cmd", "cmd": "echo b"}}, "env": {}} ], "node_holes": [], "edge_property": "directed", @@ -143,8 +143,8 @@ mod tests { "version": "0", "graph": { "nodes": [ - {"step": {"key": "a", "cmd": "echo a", "image": "ubuntu:24.04"}, "env": {}}, - {"step": {"key": "b", "cmd": "echo b", "image": "ubuntu:24.04"}, "env": {}} + {"step": {"key": "a", "eval": {"type": "cmd", "cmd": "echo a"}, "image": "ubuntu:24.04"}, "env": {}}, + {"step": {"key": "b", "eval": {"type": "cmd", "cmd": "echo b"}, "image": "ubuntu:24.04"}, "env": {}} ], "node_holes": [], "edge_property": "directed", @@ -158,6 +158,40 @@ mod tests { assert_eq!(plan.summary.chain_count, 2); } + #[test] + fn plan_accepts_dynamic_node_and_keeps_verbatim_json() { + let json = r#"{ + "version": "0", + "graph": { + "nodes": [ + { + "step": { + "key": "choose-build", + "label": "Choose build", + "eval": { + "type": "dynamic", + "target_name": "choose_build" + } + }, + "env": {} + } + ], + "node_holes": [], + "edge_property": "directed", + "edges": [] + } + }"# + .to_string(); + + let plan = Plan::parse(json.clone()).expect("parse"); + assert_eq!(plan.ir_json, json); + assert_eq!(plan.summary.step_count, 1); + assert_eq!( + plan.graph.dag()[daggy::NodeIndex::new(0)].step.key, + "choose-build" + ); + } + #[test] fn invalid_ir_returns_rejected_error() { let err = Plan::parse("not json at all".to_string()).unwrap_err(); diff --git a/crates/hm-pipeline-ir/src/graph.rs b/crates/hm-pipeline-ir/src/graph.rs index 832fc654..51bec1dc 100644 --- a/crates/hm-pipeline-ir/src/graph.rs +++ b/crates/hm-pipeline-ir/src/graph.rs @@ -107,13 +107,13 @@ pub struct Cache { pub key: Option, } -/// A graph node: a [`CommandStep`] paired with its resolved environment. +/// A graph node paired with its resolved environment. /// /// The `env` map is the final merged result of pipeline-level defaults /// and per-step overrides — ready to hand to the executor as-is. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Transition { - pub step: CommandStep, + pub step: PipelineStep, pub env: BTreeMap, } diff --git a/crates/hm-pipeline-ir/tests/e2e_fixtures.rs b/crates/hm-pipeline-ir/tests/e2e_fixtures.rs index e8c93112..4bc1e545 100644 --- a/crates/hm-pipeline-ir/tests/e2e_fixtures.rs +++ b/crates/hm-pipeline-ir/tests/e2e_fixtures.rs @@ -149,8 +149,9 @@ fn all_fixtures_have_valid_structure() { for (_, t) in g.dag().graph().node_references() { assert!(!t.step.key.is_empty(), "{dsl}/{scenario}: empty key"); + let step = t.step.clone().into_command().expect("fixture command"); assert!( - !t.step.cmd.is_empty(), + !step.cmd.is_empty(), "{dsl}/{scenario}: empty cmd for {}", t.step.key, ); diff --git a/crates/hm-pipeline-ir/tests/graph_build.rs b/crates/hm-pipeline-ir/tests/graph_build.rs index 4a777931..1e481894 100644 --- a/crates/hm-pipeline-ir/tests/graph_build.rs +++ b/crates/hm-pipeline-ir/tests/graph_build.rs @@ -32,9 +32,9 @@ fn builds_simple_chain() { "default_image": "ubuntu:24.04", "graph": { "nodes": [ - {"step": {"key": "a", "cmd": "echo a", "image": "ubuntu:24.04"}, "env": {}}, - {"step": {"key": "b", "cmd": "echo b"}, "env": {}}, - {"step": {"key": "c", "cmd": "echo c"}, "env": {}} + {"step": {"key": "a", "eval": {"type": "cmd", "cmd": "echo a"}, "image": "ubuntu:24.04"}, "env": {}}, + {"step": {"key": "b", "eval": {"type": "cmd", "cmd": "echo b"}}, "env": {}}, + {"step": {"key": "c", "eval": {"type": "cmd", "cmd": "echo c"}}, "env": {}} ], "edge_property": "directed", "edges": [ @@ -56,7 +56,7 @@ fn root_inherits_default_image() { "default_image": "ubuntu:24.04", "graph": { "nodes": [ - {"step": {"key": "a", "cmd": "echo a", "image": "ubuntu:24.04"}, "env": {}} + {"step": {"key": "a", "eval": {"type": "cmd", "cmd": "echo a"}, "image": "ubuntu:24.04"}, "env": {}} ], "edge_property": "directed", "edges": [] @@ -75,8 +75,8 @@ fn child_does_not_inherit_default_image() { "default_image": "ubuntu:24.04", "graph": { "nodes": [ - {"step": {"key": "a", "cmd": "echo a", "image": "ubuntu:24.04"}, "env": {}}, - {"step": {"key": "b", "cmd": "echo b"}, "env": {}} + {"step": {"key": "a", "eval": {"type": "cmd", "cmd": "echo a"}, "image": "ubuntu:24.04"}, "env": {}}, + {"step": {"key": "b", "eval": {"type": "cmd", "cmd": "echo b"}}, "env": {}} ], "edge_property": "directed", "edges": [ @@ -96,9 +96,9 @@ fn wait_inserts_implicit_deps() { "version": "0", "graph": { "nodes": [ - {"step": {"key": "a", "cmd": "echo a"}, "env": {}}, - {"step": {"key": "b", "cmd": "echo b"}, "env": {}}, - {"step": {"key": "c", "cmd": "echo c"}, "env": {}} + {"step": {"key": "a", "eval": {"type": "cmd", "cmd": "echo a"}}, "env": {}}, + {"step": {"key": "b", "eval": {"type": "cmd", "cmd": "echo b"}}, "env": {}}, + {"step": {"key": "c", "eval": {"type": "cmd", "cmd": "echo c"}}, "env": {}} ], "edge_property": "directed", "edges": [ diff --git a/crates/hm-pipeline-ir/tests/graph_serde.rs b/crates/hm-pipeline-ir/tests/graph_serde.rs index 52251dbc..90246a33 100644 --- a/crates/hm-pipeline-ir/tests/graph_serde.rs +++ b/crates/hm-pipeline-ir/tests/graph_serde.rs @@ -8,15 +8,17 @@ use std::collections::BTreeMap; -use hm_pipeline_ir::{CommandStep, EdgeKind, Transition}; +use hm_pipeline_ir::{EdgeKind, PipelineStep, StepEval, Transition}; #[test] fn transition_round_trips() { let nw = Transition { - step: CommandStep { + step: PipelineStep { key: "a".into(), label: Some("step A".into()), - cmd: "echo a".into(), + eval: StepEval::Cmd { + cmd: "echo a".into(), + }, image: Some("ubuntu:24.04".into()), env: None, timeout_seconds: None, @@ -32,6 +34,75 @@ fn transition_round_trips() { assert_eq!(back.env.get("FOO").unwrap(), "bar"); } +#[test] +fn command_transition_uses_eval_wire_shape() { + let transition = Transition { + step: PipelineStep { + key: "build".into(), + label: None, + eval: StepEval::Cmd { + cmd: "cargo build".into(), + }, + image: None, + env: None, + timeout_seconds: None, + cache: None, + runner: None, + runner_args: None, + }, + env: BTreeMap::new(), + }; + + let json = serde_json::to_value(transition).unwrap(); + assert_eq!(json["step"]["eval"]["type"], "cmd"); + assert_eq!(json["step"]["eval"]["cmd"], "cargo build"); + assert!(json["step"].get("cmd").is_none()); +} + +#[test] +fn dynamic_transition_round_trips() { + let transition = Transition { + step: PipelineStep { + key: "choose-build".into(), + label: Some("Choose build".into()), + eval: StepEval::Dynamic { + target_name: "choose_build".into(), + }, + image: None, + env: None, + timeout_seconds: None, + cache: None, + runner: None, + runner_args: None, + }, + env: BTreeMap::new(), + }; + + let json = serde_json::to_value(&transition).unwrap(); + assert_eq!(json["step"]["eval"]["type"], "dynamic"); + assert_eq!(json["step"]["eval"]["target_name"], "choose_build"); + + let back: Transition = serde_json::from_value(json).unwrap(); + assert!(matches!( + back.step.eval, + StepEval::Dynamic { target_name } if target_name == "choose_build" + )); +} + +#[test] +fn legacy_command_shape_is_rejected() { + let json = serde_json::json!({ + "step": { + "key": "build", + "cmd": "cargo build" + }, + "env": {} + }); + + let error = serde_json::from_value::(json).unwrap_err(); + assert!(error.to_string().contains("eval")); +} + #[test] fn edge_kind_serializes_as_snake_case() { assert_eq!( @@ -60,9 +131,9 @@ fn build_test_graph() -> PipelineGraph { "default_image": "ubuntu:24.04", "graph": { "nodes": [ - {"step": {"key": "a", "cmd": "echo a", "image": "ubuntu:24.04"}, "env": {}}, - {"step": {"key": "b", "cmd": "echo b"}, "env": {}}, - {"step": {"key": "c", "cmd": "echo c", "image": "ubuntu:24.04"}, "env": {}} + {"step": {"key": "a", "eval": {"type": "cmd", "cmd": "echo a"}, "image": "ubuntu:24.04"}, "env": {}}, + {"step": {"key": "b", "eval": {"type": "cmd", "cmd": "echo b"}}, "env": {}}, + {"step": {"key": "c", "eval": {"type": "cmd", "cmd": "echo c"}, "image": "ubuntu:24.04"}, "env": {}} ], "node_holes": [], "edge_property": "directed", diff --git a/crates/hm-pipeline-ir/tests/snapshots/graph_serde__pipeline_graph_snapshot.snap b/crates/hm-pipeline-ir/tests/snapshots/graph_serde__pipeline_graph_snapshot.snap index 4a02d56c..ab226064 100644 --- a/crates/hm-pipeline-ir/tests/snapshots/graph_serde__pipeline_graph_snapshot.snap +++ b/crates/hm-pipeline-ir/tests/snapshots/graph_serde__pipeline_graph_snapshot.snap @@ -1,6 +1,6 @@ --- source: crates/hm-pipeline-ir/tests/graph_serde.rs -assertion_line: 88 +assertion_line: 181 expression: json --- { @@ -20,8 +20,11 @@ expression: json "env": {}, "step": { "cache": null, - "cmd": "echo a", "env": null, + "eval": { + "cmd": "echo a", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "a", "label": null, @@ -32,8 +35,11 @@ expression: json "env": {}, "step": { "cache": null, - "cmd": "echo b", "env": null, + "eval": { + "cmd": "echo b", + "type": "cmd" + }, "image": null, "key": "b", "label": null, @@ -44,8 +50,11 @@ expression: json "env": {}, "step": { "cache": null, - "cmd": "echo c", "env": null, + "eval": { + "cmd": "echo c", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "c", "label": null, diff --git a/crates/hm/tests/default_image_inheritance.rs b/crates/hm/tests/default_image_inheritance.rs index 9c706436..0a6a22f7 100644 --- a/crates/hm/tests/default_image_inheritance.rs +++ b/crates/hm/tests/default_image_inheritance.rs @@ -20,7 +20,7 @@ fn decode(json: &[u8]) -> PipelineGraph { serde_json::from_slice::(json).unwrap() } -fn find_step<'a>(g: &'a PipelineGraph, key: &str) -> &'a hm_pipeline_ir::CommandStep { +fn find_step<'a>(g: &'a PipelineGraph, key: &str) -> &'a hm_pipeline_ir::PipelineStep { let dag = g.dag(); let (_, t) = dag .graph() @@ -37,7 +37,7 @@ fn root_step_inherits_default_image() { "default_image": "ubuntu:24.04", "graph": { "nodes": [ - {"step": {"key": "apt-base", "cmd": "apt-get update", "image": "ubuntu:24.04"}, "env": {}} + {"step": {"key": "apt-base", "eval": {"type": "cmd", "cmd": "apt-get update"}, "image": "ubuntu:24.04"}, "env": {}} ], "edge_property": "directed", "edges": [] @@ -59,7 +59,7 @@ fn root_step_explicit_image_wins() { "default_image": "ubuntu:24.04", "graph": { "nodes": [ - {"step": {"key": "rust", "cmd": "cargo build", "image": "rust:1.82"}, "env": {}} + {"step": {"key": "rust", "eval": {"type": "cmd", "cmd": "cargo build"}, "image": "rust:1.82"}, "env": {}} ], "edge_property": "directed", "edges": [] @@ -85,8 +85,8 @@ fn child_step_unchanged_by_default_image() { "default_image": "ubuntu:24.04", "graph": { "nodes": [ - {"step": {"key": "parent", "cmd": "echo p", "image": "ubuntu:24.04"}, "env": {}}, - {"step": {"key": "child", "cmd": "echo c"}, "env": {}} + {"step": {"key": "parent", "eval": {"type": "cmd", "cmd": "echo p"}, "image": "ubuntu:24.04"}, "env": {}}, + {"step": {"key": "child", "eval": {"type": "cmd", "cmd": "echo c"}}, "env": {}} ], "edge_property": "directed", "edges": [ @@ -109,7 +109,7 @@ fn no_default_image_leaves_root_alone() { "version": "0", "graph": { "nodes": [ - {"step": {"key": "k", "cmd": "true"}, "env": {}} + {"step": {"key": "k", "eval": {"type": "cmd", "cmd": "true"}}, "env": {}} ], "edge_property": "directed", "edges": [] diff --git a/tests/e2e/fixtures/python/cmake-advanced.json b/tests/e2e/fixtures/python/cmake-advanced.json index 5a923649..8749aab9 100644 --- a/tests/e2e/fixtures/python/cmake-advanced.json +++ b/tests/e2e/fixtures/python/cmake-advanced.json @@ -43,7 +43,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y cmake build-essential pkg-config ninja-build ccache clang-format clang-tidy clang-18 lld-18", + "eval": { + "cmd": "apt-get update && apt-get install -y cmake build-essential pkg-config ninja-build ccache clang-format clang-tidy clang-18 lld-18", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "apt-base", "label": ":cmake: apt-base" @@ -60,7 +63,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "cmake --version && ninja --version && ccache --version && clang-18 --version", + "eval": { + "cmd": "cmake --version && ninja --version && ccache --version && clang-18 --version", + "type": "cmd" + }, "key": "verify", "label": ":cmake: verify" } @@ -78,7 +84,10 @@ ], "policy": "on_change" }, - "cmd": "cd . && cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=20 && cmake --build build --parallel $(nproc)", + "eval": { + "cmd": "cd . && cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=20 && cmake --build build --parallel $(nproc)", + "type": "cmd" + }, "key": "build", "label": ":cmake: build" } @@ -90,7 +99,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cmake --build ./build --parallel $(nproc) && ctest --test-dir ./build --output-on-failure --parallel $(nproc)", + "eval": { + "cmd": "cmake --build ./build --parallel $(nproc) && ctest --test-dir ./build --output-on-failure --parallel $(nproc)", + "type": "cmd" + }, "key": "test", "label": ":cmake: test" } @@ -102,7 +114,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd . && run-clang-tidy -p build", + "eval": { + "cmd": "cd . && run-clang-tidy -p build", + "type": "cmd" + }, "key": "lint", "label": ":cmake: lint" } @@ -114,7 +129,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd . && find . -not -path './build/*' \\( -name '*.c' -o -name '*.h' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.cc' -o -name '*.cxx' \\) | xargs clang-format --dry-run --Werror", + "eval": { + "cmd": "cd . && find . -not -path './build/*' \\( -name '*.c' -o -name '*.h' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.cc' -o -name '*.cxx' \\) | xargs clang-format --dry-run --Werror", + "type": "cmd" + }, "key": "fmt", "label": ":cmake: fmt" } diff --git a/tests/e2e/fixtures/python/kitchen-sink.json b/tests/e2e/fixtures/python/kitchen-sink.json index 028331bc..cbefc5ba 100644 --- a/tests/e2e/fixtures/python/kitchen-sink.json +++ b/tests/e2e/fixtures/python/kitchen-sink.json @@ -58,7 +58,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y cmake build-essential pkg-config ninja-build ccache clang-format clang-tidy", + "eval": { + "cmd": "apt-get update && apt-get install -y cmake build-essential pkg-config ninja-build ccache clang-format clang-tidy", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "fdcc2fd62363", "label": ":cmake: apt-base" @@ -75,7 +78,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "cmake --version && ninja --version && ccache --version", + "eval": { + "cmd": "cmake --version && ninja --version && ccache --version", + "type": "cmd" + }, "key": "verify", "label": ":cmake: verify" } @@ -93,7 +99,10 @@ ], "policy": "on_change" }, - "cmd": "cd infra/agent && cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache && cmake --build build --parallel $(nproc)", + "eval": { + "cmd": "cd infra/agent && cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache && cmake --build build --parallel $(nproc)", + "type": "cmd" + }, "key": "build", "label": ":cmake: build" } @@ -105,7 +114,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cmake --build infra/agent/build --parallel $(nproc) && ctest --test-dir infra/agent/build --output-on-failure --parallel $(nproc)", + "eval": { + "cmd": "cmake --build infra/agent/build --parallel $(nproc) && ctest --test-dir infra/agent/build --output-on-failure --parallel $(nproc)", + "type": "cmd" + }, "key": "431f35b84318", "label": ":cmake: test" } @@ -117,7 +129,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd infra/agent && find . -not -path './build/*' \\( -name '*.c' -o -name '*.h' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.cc' -o -name '*.cxx' \\) | xargs clang-format --dry-run --Werror", + "eval": { + "cmd": "cd infra/agent && find . -not -path './build/*' \\( -name '*.c' -o -name '*.h' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.cc' -o -name '*.cxx' \\) | xargs clang-format --dry-run --Werror", + "type": "cmd" + }, "key": "fmt", "label": ":cmake: fmt" } @@ -134,7 +149,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates python3 python3-venv", + "eval": { + "cmd": "apt-get update && apt-get install -y curl ca-certificates python3 python3-venv", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "c8d9fda86ff3", "label": ":python: apt-base" @@ -151,7 +169,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl -LsSf https://astral.sh/uv/install.sh | sh && ln -sf /root/.local/bin/uv /usr/local/bin/uv && uv --version", + "eval": { + "cmd": "curl -LsSf https://astral.sh/uv/install.sh | sh && ln -sf /root/.local/bin/uv /usr/local/bin/uv && uv --version", + "type": "cmd" + }, "key": "uv-install", "label": ":python: uv-install" } @@ -170,7 +191,10 @@ ], "policy": "on_change" }, - "cmd": "cd services/web && uv sync --all-extras", + "eval": { + "cmd": "cd services/web && uv sync --all-extras", + "type": "cmd" + }, "key": "uv-sync", "label": ":python: uv-sync" } @@ -182,7 +206,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/web && uv run pytest", + "eval": { + "cmd": "cd services/web && uv run pytest", + "type": "cmd" + }, "key": "239c4926568b", "label": ":python: test" } @@ -194,7 +221,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/web && uv run ruff check .", + "eval": { + "cmd": "cd services/web && uv run ruff check .", + "type": "cmd" + }, "key": "lint", "label": ":python: lint" } diff --git a/tests/e2e/fixtures/python/monorepo-ci.json b/tests/e2e/fixtures/python/monorepo-ci.json index e84a4f3b..bd938758 100644 --- a/tests/e2e/fixtures/python/monorepo-ci.json +++ b/tests/e2e/fixtures/python/monorepo-ci.json @@ -88,7 +88,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates git", + "eval": { + "cmd": "apt-get update && apt-get install -y curl ca-certificates git", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "334b29e96b76", "label": ":go: apt-base" @@ -105,7 +108,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl -fsSL https://go.dev/dl/go1.23.2.linux-amd64.tar.gz -o /tmp/go.tgz && rm -rf /usr/local/go && tar -C /usr/local -xzf /tmp/go.tgz && ln -sf /usr/local/go/bin/go /usr/local/bin/go && ln -sf /usr/local/go/bin/gofmt /usr/local/bin/gofmt && go version", + "eval": { + "cmd": "curl -fsSL https://go.dev/dl/go1.23.2.linux-amd64.tar.gz -o /tmp/go.tgz && rm -rf /usr/local/go && tar -C /usr/local -xzf /tmp/go.tgz && ln -sf /usr/local/go/bin/go /usr/local/bin/go && ln -sf /usr/local/go/bin/gofmt /usr/local/bin/gofmt && go version", + "type": "cmd" + }, "key": "e0b494124562", "label": ":go: install" } @@ -117,7 +123,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/api && go build ./...", + "eval": { + "cmd": "cd services/api && go build ./...", + "type": "cmd" + }, "key": "6f9493b7219f", "label": ":go: build" } @@ -129,7 +138,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/api && go test ./...", + "eval": { + "cmd": "cd services/api && go test ./...", + "type": "cmd" + }, "key": "1ad6d86b2c0a", "label": ":go: test" } @@ -141,7 +153,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/api && go vet ./...", + "eval": { + "cmd": "cd services/api && go vet ./...", + "type": "cmd" + }, "key": "vet", "label": ":go: vet" } @@ -158,7 +173,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates python3 python3-venv", + "eval": { + "cmd": "apt-get update && apt-get install -y curl ca-certificates python3 python3-venv", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "c8d9fda86ff3", "label": ":python: apt-base" @@ -175,7 +193,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl -LsSf https://astral.sh/uv/install.sh | sh && ln -sf /root/.local/bin/uv /usr/local/bin/uv && uv --version", + "eval": { + "cmd": "curl -LsSf https://astral.sh/uv/install.sh | sh && ln -sf /root/.local/bin/uv /usr/local/bin/uv && uv --version", + "type": "cmd" + }, "key": "uv-install", "label": ":python: uv-install" } @@ -194,7 +215,10 @@ ], "policy": "on_change" }, - "cmd": "cd services/ml && uv sync --all-extras", + "eval": { + "cmd": "cd services/ml && uv sync --all-extras", + "type": "cmd" + }, "key": "uv-sync", "label": ":python: uv-sync" } @@ -206,7 +230,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/ml && uv run pytest", + "eval": { + "cmd": "cd services/ml && uv run pytest", + "type": "cmd" + }, "key": "847020e744bc", "label": ":python: test" } @@ -218,7 +245,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/ml && uv run ruff check .", + "eval": { + "cmd": "cd services/ml && uv run ruff check .", + "type": "cmd" + }, "key": "6c48498afb84", "label": ":python: lint" } @@ -230,7 +260,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/ml && uv run ty check .", + "eval": { + "cmd": "cd services/ml && uv run ty check .", + "type": "cmd" + }, "key": "typecheck", "label": ":python: typecheck" } @@ -247,7 +280,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates", + "eval": { + "cmd": "apt-get update && apt-get install -y curl ca-certificates", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "3c2cfedcad46", "label": ":node: apt-base" @@ -264,7 +300,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && apt-get install -y nodejs", + "eval": { + "cmd": "curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && apt-get install -y nodejs", + "type": "cmd" + }, "key": "ed974519b390", "label": ":node: install" } @@ -282,7 +321,10 @@ ], "policy": "on_change" }, - "cmd": "cd web && npm ci", + "eval": { + "cmd": "cd web && npm ci", + "type": "cmd" + }, "key": "deps", "label": ":node: deps" } @@ -294,7 +336,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd web && npm run build", + "eval": { + "cmd": "cd web && npm run build", + "type": "cmd" + }, "key": "a94a0f84e711", "label": ":node: build" } @@ -306,7 +351,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd web && npm run test", + "eval": { + "cmd": "cd web && npm run test", + "type": "cmd" + }, "key": "d2438adde70d", "label": ":node: test" } @@ -318,7 +366,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd web && npm run lint", + "eval": { + "cmd": "cd web && npm run lint", + "type": "cmd" + }, "key": "74c52c9e5ef6", "label": ":node: lint" } diff --git a/tests/e2e/fixtures/python/rust-release.json b/tests/e2e/fixtures/python/rust-release.json index cd40ff9d..e5789e7b 100644 --- a/tests/e2e/fixtures/python/rust-release.json +++ b/tests/e2e/fixtures/python/rust-release.json @@ -48,7 +48,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates build-essential pkg-config libssl-dev", + "eval": { + "cmd": "apt-get update && apt-get install -y curl ca-certificates build-essential pkg-config libssl-dev", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "apt-base", "label": ":rust: apt-base" @@ -65,7 +68,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal --component clippy,rustfmt && . $HOME/.cargo/env && rustc --version && cargo --version", + "eval": { + "cmd": "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal --component clippy,rustfmt && . $HOME/.cargo/env && rustc --version && cargo --version", + "type": "cmd" + }, "key": "rustup", "label": ":rust: rustup" } @@ -77,7 +83,10 @@ "TERM": "dumb" }, "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo build", + "eval": { + "cmd": ". $HOME/.cargo/env && cd . && cargo build", + "type": "cmd" + }, "key": "build", "label": ":rust: build" } @@ -89,7 +98,10 @@ "TERM": "dumb" }, "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo test", + "eval": { + "cmd": ". $HOME/.cargo/env && cd . && cargo test", + "type": "cmd" + }, "key": "test", "label": ":rust: test" } @@ -101,7 +113,10 @@ "TERM": "dumb" }, "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo clippy --all-targets -- -D warnings", + "eval": { + "cmd": ". $HOME/.cargo/env && cd . && cargo clippy --all-targets -- -D warnings", + "type": "cmd" + }, "key": "clippy", "label": ":rust: clippy" } @@ -113,7 +128,10 @@ "TERM": "dumb" }, "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo fmt --check", + "eval": { + "cmd": ". $HOME/.cargo/env && cd . && cargo fmt --check", + "type": "cmd" + }, "key": "fmt", "label": ":rust: fmt" } @@ -125,7 +143,10 @@ "TERM": "dumb" }, "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo doc --no-deps", + "eval": { + "cmd": ". $HOME/.cargo/env && cd . && cargo doc --no-deps", + "type": "cmd" + }, "key": "doc", "label": ":rust: doc" } diff --git a/tests/e2e/fixtures/python/zig-node-polyglot.json b/tests/e2e/fixtures/python/zig-node-polyglot.json index 76406024..af9787d1 100644 --- a/tests/e2e/fixtures/python/zig-node-polyglot.json +++ b/tests/e2e/fixtures/python/zig-node-polyglot.json @@ -68,7 +68,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y --no-install-recommends curl ca-certificates xz-utils", + "eval": { + "cmd": "apt-get update && apt-get install -y --no-install-recommends curl ca-certificates xz-utils", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "base", "label": ":apt: base" @@ -85,7 +88,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl -fsSL https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz -o /tmp/zig.tar.xz && rm -rf /usr/local/zig && mkdir -p /usr/local/zig && tar -xJf /tmp/zig.tar.xz -C /usr/local/zig --strip-components=1 && ln -sf /usr/local/zig/zig /usr/local/bin/zig && zig version", + "eval": { + "cmd": "curl -fsSL https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz -o /tmp/zig.tar.xz && rm -rf /usr/local/zig && mkdir -p /usr/local/zig && tar -xJf /tmp/zig.tar.xz -C /usr/local/zig --strip-components=1 && ln -sf /usr/local/zig/zig /usr/local/bin/zig && zig version", + "type": "cmd" + }, "key": "3083a531d11a", "label": ":zig: install" } @@ -97,7 +103,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd zig-a && zig build", + "eval": { + "cmd": "cd zig-a && zig build", + "type": "cmd" + }, "key": "zig-a-build", "label": ":zig: zig-a build" } @@ -109,7 +118,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd zig-a && zig build test", + "eval": { + "cmd": "cd zig-a && zig build test", + "type": "cmd" + }, "key": "zig-a-test", "label": ":zig: zig-a test" } @@ -121,7 +133,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd zig-b && zig build", + "eval": { + "cmd": "cd zig-b && zig build", + "type": "cmd" + }, "key": "zig-b-build", "label": ":zig: zig-b build" } @@ -133,7 +148,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd zig-b && zig build test", + "eval": { + "cmd": "cd zig-b && zig build test", + "type": "cmd" + }, "key": "zig-b-test", "label": ":zig: zig-b test" } @@ -149,7 +167,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && apt-get install -y nodejs", + "eval": { + "cmd": "curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && apt-get install -y nodejs", + "type": "cmd" + }, "key": "a8f0d9d99460", "label": ":node: install" } @@ -167,7 +188,10 @@ ], "policy": "on_change" }, - "cmd": "cd web && npm ci", + "eval": { + "cmd": "cd web && npm ci", + "type": "cmd" + }, "key": "deps", "label": ":node: deps" } @@ -179,7 +203,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd web && npm run build", + "eval": { + "cmd": "cd web && npm run build", + "type": "cmd" + }, "key": "build", "label": ":node: build" } @@ -191,7 +218,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd web && npm run test", + "eval": { + "cmd": "cd web && npm run test", + "type": "cmd" + }, "key": "test", "label": ":node: test" } @@ -203,7 +233,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd web && npm run lint", + "eval": { + "cmd": "cd web && npm run lint", + "type": "cmd" + }, "key": "lint", "label": ":node: lint" } diff --git a/tests/e2e/fixtures/ts/cmake-advanced.json b/tests/e2e/fixtures/ts/cmake-advanced.json index 5a923649..8749aab9 100644 --- a/tests/e2e/fixtures/ts/cmake-advanced.json +++ b/tests/e2e/fixtures/ts/cmake-advanced.json @@ -43,7 +43,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y cmake build-essential pkg-config ninja-build ccache clang-format clang-tidy clang-18 lld-18", + "eval": { + "cmd": "apt-get update && apt-get install -y cmake build-essential pkg-config ninja-build ccache clang-format clang-tidy clang-18 lld-18", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "apt-base", "label": ":cmake: apt-base" @@ -60,7 +63,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "cmake --version && ninja --version && ccache --version && clang-18 --version", + "eval": { + "cmd": "cmake --version && ninja --version && ccache --version && clang-18 --version", + "type": "cmd" + }, "key": "verify", "label": ":cmake: verify" } @@ -78,7 +84,10 @@ ], "policy": "on_change" }, - "cmd": "cd . && cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=20 && cmake --build build --parallel $(nproc)", + "eval": { + "cmd": "cd . && cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=20 && cmake --build build --parallel $(nproc)", + "type": "cmd" + }, "key": "build", "label": ":cmake: build" } @@ -90,7 +99,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cmake --build ./build --parallel $(nproc) && ctest --test-dir ./build --output-on-failure --parallel $(nproc)", + "eval": { + "cmd": "cmake --build ./build --parallel $(nproc) && ctest --test-dir ./build --output-on-failure --parallel $(nproc)", + "type": "cmd" + }, "key": "test", "label": ":cmake: test" } @@ -102,7 +114,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd . && run-clang-tidy -p build", + "eval": { + "cmd": "cd . && run-clang-tidy -p build", + "type": "cmd" + }, "key": "lint", "label": ":cmake: lint" } @@ -114,7 +129,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd . && find . -not -path './build/*' \\( -name '*.c' -o -name '*.h' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.cc' -o -name '*.cxx' \\) | xargs clang-format --dry-run --Werror", + "eval": { + "cmd": "cd . && find . -not -path './build/*' \\( -name '*.c' -o -name '*.h' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.cc' -o -name '*.cxx' \\) | xargs clang-format --dry-run --Werror", + "type": "cmd" + }, "key": "fmt", "label": ":cmake: fmt" } diff --git a/tests/e2e/fixtures/ts/kitchen-sink.json b/tests/e2e/fixtures/ts/kitchen-sink.json index 028331bc..cbefc5ba 100644 --- a/tests/e2e/fixtures/ts/kitchen-sink.json +++ b/tests/e2e/fixtures/ts/kitchen-sink.json @@ -58,7 +58,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y cmake build-essential pkg-config ninja-build ccache clang-format clang-tidy", + "eval": { + "cmd": "apt-get update && apt-get install -y cmake build-essential pkg-config ninja-build ccache clang-format clang-tidy", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "fdcc2fd62363", "label": ":cmake: apt-base" @@ -75,7 +78,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "cmake --version && ninja --version && ccache --version", + "eval": { + "cmd": "cmake --version && ninja --version && ccache --version", + "type": "cmd" + }, "key": "verify", "label": ":cmake: verify" } @@ -93,7 +99,10 @@ ], "policy": "on_change" }, - "cmd": "cd infra/agent && cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache && cmake --build build --parallel $(nproc)", + "eval": { + "cmd": "cd infra/agent && cmake -S . -B build -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache && cmake --build build --parallel $(nproc)", + "type": "cmd" + }, "key": "build", "label": ":cmake: build" } @@ -105,7 +114,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cmake --build infra/agent/build --parallel $(nproc) && ctest --test-dir infra/agent/build --output-on-failure --parallel $(nproc)", + "eval": { + "cmd": "cmake --build infra/agent/build --parallel $(nproc) && ctest --test-dir infra/agent/build --output-on-failure --parallel $(nproc)", + "type": "cmd" + }, "key": "431f35b84318", "label": ":cmake: test" } @@ -117,7 +129,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd infra/agent && find . -not -path './build/*' \\( -name '*.c' -o -name '*.h' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.cc' -o -name '*.cxx' \\) | xargs clang-format --dry-run --Werror", + "eval": { + "cmd": "cd infra/agent && find . -not -path './build/*' \\( -name '*.c' -o -name '*.h' -o -name '*.cpp' -o -name '*.hpp' -o -name '*.cc' -o -name '*.cxx' \\) | xargs clang-format --dry-run --Werror", + "type": "cmd" + }, "key": "fmt", "label": ":cmake: fmt" } @@ -134,7 +149,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates python3 python3-venv", + "eval": { + "cmd": "apt-get update && apt-get install -y curl ca-certificates python3 python3-venv", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "c8d9fda86ff3", "label": ":python: apt-base" @@ -151,7 +169,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl -LsSf https://astral.sh/uv/install.sh | sh && ln -sf /root/.local/bin/uv /usr/local/bin/uv && uv --version", + "eval": { + "cmd": "curl -LsSf https://astral.sh/uv/install.sh | sh && ln -sf /root/.local/bin/uv /usr/local/bin/uv && uv --version", + "type": "cmd" + }, "key": "uv-install", "label": ":python: uv-install" } @@ -170,7 +191,10 @@ ], "policy": "on_change" }, - "cmd": "cd services/web && uv sync --all-extras", + "eval": { + "cmd": "cd services/web && uv sync --all-extras", + "type": "cmd" + }, "key": "uv-sync", "label": ":python: uv-sync" } @@ -182,7 +206,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/web && uv run pytest", + "eval": { + "cmd": "cd services/web && uv run pytest", + "type": "cmd" + }, "key": "239c4926568b", "label": ":python: test" } @@ -194,7 +221,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/web && uv run ruff check .", + "eval": { + "cmd": "cd services/web && uv run ruff check .", + "type": "cmd" + }, "key": "lint", "label": ":python: lint" } diff --git a/tests/e2e/fixtures/ts/monorepo-ci.json b/tests/e2e/fixtures/ts/monorepo-ci.json index e84a4f3b..bd938758 100644 --- a/tests/e2e/fixtures/ts/monorepo-ci.json +++ b/tests/e2e/fixtures/ts/monorepo-ci.json @@ -88,7 +88,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates git", + "eval": { + "cmd": "apt-get update && apt-get install -y curl ca-certificates git", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "334b29e96b76", "label": ":go: apt-base" @@ -105,7 +108,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl -fsSL https://go.dev/dl/go1.23.2.linux-amd64.tar.gz -o /tmp/go.tgz && rm -rf /usr/local/go && tar -C /usr/local -xzf /tmp/go.tgz && ln -sf /usr/local/go/bin/go /usr/local/bin/go && ln -sf /usr/local/go/bin/gofmt /usr/local/bin/gofmt && go version", + "eval": { + "cmd": "curl -fsSL https://go.dev/dl/go1.23.2.linux-amd64.tar.gz -o /tmp/go.tgz && rm -rf /usr/local/go && tar -C /usr/local -xzf /tmp/go.tgz && ln -sf /usr/local/go/bin/go /usr/local/bin/go && ln -sf /usr/local/go/bin/gofmt /usr/local/bin/gofmt && go version", + "type": "cmd" + }, "key": "e0b494124562", "label": ":go: install" } @@ -117,7 +123,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/api && go build ./...", + "eval": { + "cmd": "cd services/api && go build ./...", + "type": "cmd" + }, "key": "6f9493b7219f", "label": ":go: build" } @@ -129,7 +138,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/api && go test ./...", + "eval": { + "cmd": "cd services/api && go test ./...", + "type": "cmd" + }, "key": "1ad6d86b2c0a", "label": ":go: test" } @@ -141,7 +153,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/api && go vet ./...", + "eval": { + "cmd": "cd services/api && go vet ./...", + "type": "cmd" + }, "key": "vet", "label": ":go: vet" } @@ -158,7 +173,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates python3 python3-venv", + "eval": { + "cmd": "apt-get update && apt-get install -y curl ca-certificates python3 python3-venv", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "c8d9fda86ff3", "label": ":python: apt-base" @@ -175,7 +193,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl -LsSf https://astral.sh/uv/install.sh | sh && ln -sf /root/.local/bin/uv /usr/local/bin/uv && uv --version", + "eval": { + "cmd": "curl -LsSf https://astral.sh/uv/install.sh | sh && ln -sf /root/.local/bin/uv /usr/local/bin/uv && uv --version", + "type": "cmd" + }, "key": "uv-install", "label": ":python: uv-install" } @@ -194,7 +215,10 @@ ], "policy": "on_change" }, - "cmd": "cd services/ml && uv sync --all-extras", + "eval": { + "cmd": "cd services/ml && uv sync --all-extras", + "type": "cmd" + }, "key": "uv-sync", "label": ":python: uv-sync" } @@ -206,7 +230,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/ml && uv run pytest", + "eval": { + "cmd": "cd services/ml && uv run pytest", + "type": "cmd" + }, "key": "847020e744bc", "label": ":python: test" } @@ -218,7 +245,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/ml && uv run ruff check .", + "eval": { + "cmd": "cd services/ml && uv run ruff check .", + "type": "cmd" + }, "key": "6c48498afb84", "label": ":python: lint" } @@ -230,7 +260,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd services/ml && uv run ty check .", + "eval": { + "cmd": "cd services/ml && uv run ty check .", + "type": "cmd" + }, "key": "typecheck", "label": ":python: typecheck" } @@ -247,7 +280,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates", + "eval": { + "cmd": "apt-get update && apt-get install -y curl ca-certificates", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "3c2cfedcad46", "label": ":node: apt-base" @@ -264,7 +300,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && apt-get install -y nodejs", + "eval": { + "cmd": "curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && apt-get install -y nodejs", + "type": "cmd" + }, "key": "ed974519b390", "label": ":node: install" } @@ -282,7 +321,10 @@ ], "policy": "on_change" }, - "cmd": "cd web && npm ci", + "eval": { + "cmd": "cd web && npm ci", + "type": "cmd" + }, "key": "deps", "label": ":node: deps" } @@ -294,7 +336,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd web && npm run build", + "eval": { + "cmd": "cd web && npm run build", + "type": "cmd" + }, "key": "a94a0f84e711", "label": ":node: build" } @@ -306,7 +351,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd web && npm run test", + "eval": { + "cmd": "cd web && npm run test", + "type": "cmd" + }, "key": "d2438adde70d", "label": ":node: test" } @@ -318,7 +366,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd web && npm run lint", + "eval": { + "cmd": "cd web && npm run lint", + "type": "cmd" + }, "key": "74c52c9e5ef6", "label": ":node: lint" } diff --git a/tests/e2e/fixtures/ts/rust-release.json b/tests/e2e/fixtures/ts/rust-release.json index cd40ff9d..e5789e7b 100644 --- a/tests/e2e/fixtures/ts/rust-release.json +++ b/tests/e2e/fixtures/ts/rust-release.json @@ -48,7 +48,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y curl ca-certificates build-essential pkg-config libssl-dev", + "eval": { + "cmd": "apt-get update && apt-get install -y curl ca-certificates build-essential pkg-config libssl-dev", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "apt-base", "label": ":rust: apt-base" @@ -65,7 +68,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal --component clippy,rustfmt && . $HOME/.cargo/env && rustc --version && cargo --version", + "eval": { + "cmd": "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal --component clippy,rustfmt && . $HOME/.cargo/env && rustc --version && cargo --version", + "type": "cmd" + }, "key": "rustup", "label": ":rust: rustup" } @@ -77,7 +83,10 @@ "TERM": "dumb" }, "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo build", + "eval": { + "cmd": ". $HOME/.cargo/env && cd . && cargo build", + "type": "cmd" + }, "key": "build", "label": ":rust: build" } @@ -89,7 +98,10 @@ "TERM": "dumb" }, "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo test", + "eval": { + "cmd": ". $HOME/.cargo/env && cd . && cargo test", + "type": "cmd" + }, "key": "test", "label": ":rust: test" } @@ -101,7 +113,10 @@ "TERM": "dumb" }, "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo clippy --all-targets -- -D warnings", + "eval": { + "cmd": ". $HOME/.cargo/env && cd . && cargo clippy --all-targets -- -D warnings", + "type": "cmd" + }, "key": "clippy", "label": ":rust: clippy" } @@ -113,7 +128,10 @@ "TERM": "dumb" }, "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo fmt --check", + "eval": { + "cmd": ". $HOME/.cargo/env && cd . && cargo fmt --check", + "type": "cmd" + }, "key": "fmt", "label": ":rust: fmt" } @@ -125,7 +143,10 @@ "TERM": "dumb" }, "step": { - "cmd": ". $HOME/.cargo/env && cd . && cargo doc --no-deps", + "eval": { + "cmd": ". $HOME/.cargo/env && cd . && cargo doc --no-deps", + "type": "cmd" + }, "key": "doc", "label": ":rust: doc" } diff --git a/tests/e2e/fixtures/ts/zig-node-polyglot.json b/tests/e2e/fixtures/ts/zig-node-polyglot.json index 76406024..af9787d1 100644 --- a/tests/e2e/fixtures/ts/zig-node-polyglot.json +++ b/tests/e2e/fixtures/ts/zig-node-polyglot.json @@ -68,7 +68,10 @@ "env_keys": [], "policy": "ttl" }, - "cmd": "apt-get update && apt-get install -y --no-install-recommends curl ca-certificates xz-utils", + "eval": { + "cmd": "apt-get update && apt-get install -y --no-install-recommends curl ca-certificates xz-utils", + "type": "cmd" + }, "image": "ubuntu:24.04", "key": "base", "label": ":apt: base" @@ -85,7 +88,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl -fsSL https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz -o /tmp/zig.tar.xz && rm -rf /usr/local/zig && mkdir -p /usr/local/zig && tar -xJf /tmp/zig.tar.xz -C /usr/local/zig --strip-components=1 && ln -sf /usr/local/zig/zig /usr/local/bin/zig && zig version", + "eval": { + "cmd": "curl -fsSL https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz -o /tmp/zig.tar.xz && rm -rf /usr/local/zig && mkdir -p /usr/local/zig && tar -xJf /tmp/zig.tar.xz -C /usr/local/zig --strip-components=1 && ln -sf /usr/local/zig/zig /usr/local/bin/zig && zig version", + "type": "cmd" + }, "key": "3083a531d11a", "label": ":zig: install" } @@ -97,7 +103,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd zig-a && zig build", + "eval": { + "cmd": "cd zig-a && zig build", + "type": "cmd" + }, "key": "zig-a-build", "label": ":zig: zig-a build" } @@ -109,7 +118,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd zig-a && zig build test", + "eval": { + "cmd": "cd zig-a && zig build test", + "type": "cmd" + }, "key": "zig-a-test", "label": ":zig: zig-a test" } @@ -121,7 +133,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd zig-b && zig build", + "eval": { + "cmd": "cd zig-b && zig build", + "type": "cmd" + }, "key": "zig-b-build", "label": ":zig: zig-b build" } @@ -133,7 +148,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd zig-b && zig build test", + "eval": { + "cmd": "cd zig-b && zig build test", + "type": "cmd" + }, "key": "zig-b-test", "label": ":zig: zig-b test" } @@ -149,7 +167,10 @@ "env_keys": [], "policy": "forever" }, - "cmd": "curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && apt-get install -y nodejs", + "eval": { + "cmd": "curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && apt-get install -y nodejs", + "type": "cmd" + }, "key": "a8f0d9d99460", "label": ":node: install" } @@ -167,7 +188,10 @@ ], "policy": "on_change" }, - "cmd": "cd web && npm ci", + "eval": { + "cmd": "cd web && npm ci", + "type": "cmd" + }, "key": "deps", "label": ":node: deps" } @@ -179,7 +203,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd web && npm run build", + "eval": { + "cmd": "cd web && npm run build", + "type": "cmd" + }, "key": "build", "label": ":node: build" } @@ -191,7 +218,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd web && npm run test", + "eval": { + "cmd": "cd web && npm run test", + "type": "cmd" + }, "key": "test", "label": ":node: test" } @@ -203,7 +233,10 @@ "TERM": "dumb" }, "step": { - "cmd": "cd web && npm run lint", + "eval": { + "cmd": "cd web && npm run lint", + "type": "cmd" + }, "key": "lint", "label": ":node: lint" } From 3dd9250a333fe10523736f9e735a5d8d985ca06e Mon Sep 17 00:00:00 2001 From: tadhgdowdall Date: Tue, 9 Jun 2026 20:10:19 +0100 Subject: [PATCH 03/10] feat(dsl): emit dynamic target placeholders --- .../hm-dsl-engine/harmont-py/harmont/_keys.py | 3 +- .../harmont-py/harmont/_pipeline.py | 34 ++++++++++------ .../hm-dsl-engine/harmont-py/harmont/_step.py | 10 ++++- .../harmont-py/harmont/_target.py | 39 +++++++++++++++++++ .../tests/test_pipeline_lowering.py | 28 +++++++++++++ .../harmont-py/tests/test_target.py | 37 +++++++++++++++++- 6 files changed, 137 insertions(+), 14 deletions(-) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_keys.py b/crates/hm-dsl-engine/harmont-py/harmont/_keys.py index 0e440001..73589c39 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_keys.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_keys.py @@ -117,5 +117,6 @@ def resolve_keys(steps: Iterable[Step]) -> dict[int, str]: parent_key = "" if s.parent is not None and id(s.parent) in keys: parent_key = keys[id(s.parent)] - keys[sid] = hash_key(parent_key, s.cmd or "", position) + key_material = s.cmd or s.dynamic_target_name or "" + keys[sid] = hash_key(parent_key, key_material, position) return keys diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_pipeline.py b/crates/hm-dsl-engine/harmont-py/harmont/_pipeline.py index bce1eaba..1513ede7 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_pipeline.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_pipeline.py @@ -78,12 +78,16 @@ def _lower_to_graph( explicit ``depends_on`` edges. """ ordered = _topo_collect(leaves) - command_steps = [s for s in ordered if s.cmd is not None and not s.is_wait] - keys = resolve_keys(command_steps) + emitted_steps = [ + s + for s in ordered + if (s.cmd is not None or s.dynamic_target_name is not None) and not s.is_wait + ] + keys = resolve_keys(emitted_steps) # Assign integer node indices (dense, in emission order). idx_by_id: dict[int, int] = {} - for i, s in enumerate(command_steps): + for i, s in enumerate(emitted_steps): idx_by_id[id(s)] = i # Track which node indices have a builds_in parent (for default_image). @@ -107,20 +111,26 @@ def _lower_to_graph( pre_wait_indices = [] continue - if s.cmd is None: + if s.cmd is None and s.dynamic_target_name is None: # scratch or fork — passthrough, not emitted. continue node_idx = idx_by_id[id(s)] step_key = keys[id(s)] - step_dict: dict[str, Any] = { - "key": step_key, - "eval": { + step_eval: dict[str, str] + if s.dynamic_target_name is not None: + step_eval = { + "type": "dynamic", + "target_name": s.dynamic_target_name, + } + else: + assert s.cmd is not None + step_eval = { "type": "cmd", "cmd": s.cmd, - }, - } + } + step_dict: dict[str, Any] = {"key": step_key, "eval": step_eval} if s.label is not None: step_dict["label"] = s.label if s.cache is not None: @@ -149,7 +159,7 @@ def _lower_to_graph( # builds_in edge from parent. parent_key = _resolved_parent_key(s, keys) if parent_key is not None: - parent_idx = _find_idx_by_key(parent_key, command_steps, keys, idx_by_id) + parent_idx = _find_idx_by_key(parent_key, emitted_steps, keys, idx_by_id) edges.append([parent_idx, node_idx, "builds_in"]) has_builds_in_parent.add(node_idx) @@ -218,7 +228,9 @@ def _resolved_parent_key(s: Step, keys: dict[int, str]) -> str | None: """Walk back through scratch/fork nodes to the nearest emitted ancestor.""" node = s.parent while node is not None: - if node.cmd is not None and not node.is_wait: + if ( + node.cmd is not None or node.dynamic_target_name is not None + ) and not node.is_wait: return keys[id(node)] node = node.parent return None diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_step.py b/crates/hm-dsl-engine/harmont-py/harmont/_step.py index b3e1bea2..53c4196c 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_step.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_step.py @@ -24,6 +24,8 @@ class Step: """ cmd: str | None = None + dynamic_target_name: str | None = None + """Registered target to evaluate when execution reaches this node.""" parent: Step | None = None """In-tree pointer used by the lowering pass to walk back to the nearest emitted ancestor. Distinct from the wire-format @@ -107,7 +109,13 @@ def sh( # chain has a real cmd, inheritance stops — keeps wire format # identical for normal chains. effective_image = ( - image if image is not None else (self.image if self.cmd is None else None) + image + if image is not None + else ( + self.image + if self.cmd is None and self.dynamic_target_name is None + else None + ) ) return Step( cmd=effective_cmd, diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_target.py b/crates/hm-dsl-engine/harmont-py/harmont/_target.py index b30130d4..a5ddfbb0 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_target.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_target.py @@ -49,6 +49,7 @@ def venv() -> hm.Step: _TARGET_CACHE: dict[Callable[..., Any], Any] = {} +_DYNAMIC_TARGETS_BY_NAME: dict[str, Callable[..., Any]] = {} def clear_target_memo() -> None: @@ -71,12 +72,39 @@ def clear_target_cache() -> None: is wiped via ``clear_target_memo()``. """ _TARGET_CACHE.clear() + _DYNAMIC_TARGETS_BY_NAME.clear() clear_target_names() +def evaluate_dynamic_target(name: str) -> Any: + """Evaluate a registered dynamic target for runtime graph expansion. + + Dynamic groups are intentionally rejected for the first implementation: + a single leaf gives the scheduler one unambiguous snapshot from which + subsequent ``builds_in`` steps can continue. + """ + try: + fn = _DYNAMIC_TARGETS_BY_NAME[name] + except KeyError as e: + raise KeyError(f"hm: dynamic target {name!r} not found") from e + + from ._unwrap import as_leaves + + leaves = as_leaves(call_with_deps(fn)) + if len(leaves) != 1: + msg = ( + f"hm: dynamic target {name!r} returned {len(leaves)} leaves\n" + " → dynamic targets must currently return exactly one leaf; " + "group continuation semantics are not yet defined" + ) + raise ValueError(msg) + return leaves[0] + + def target( *, name: str | None = None, + dynamic: bool = False, ) -> Callable[[Callable[..., Any]], Callable[[], Any]]: """Mark a function as a reusable, memoized pipeline building block. @@ -89,6 +117,8 @@ def target( name: Registry key for this target. Defaults to the decorated function's name. Override when the name collides with another target or a more human-readable key is preferred. + dynamic: Defer this target's evaluation until pipeline execution. + The initial bake emits a dynamic placeholder referencing its name. Returns: A decorator that registers and memoizes the wrapped function. @@ -106,11 +136,20 @@ def decorator(fn: Callable[..., Any]) -> Callable[[], Any]: @wraps(fn) def wrapper() -> Any: + if dynamic: + from ._step import Step + + return Step( + dynamic_target_name=target_name, + key_override=target_name, + ) if fn not in _TARGET_CACHE: _TARGET_CACHE[fn] = call_with_deps(fn) return _TARGET_CACHE[fn] register_named_target(target_name, wrapper) + if dynamic: + _DYNAMIC_TARGETS_BY_NAME[target_name] = fn return wrapper return decorator diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_lowering.py b/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_lowering.py index 84b5fbac..2d4aaaa7 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_lowering.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_pipeline_lowering.py @@ -128,6 +128,34 @@ def test_command_omits_optional_fields_when_unset(): assert "cache" not in step +def test_dynamic_target_emits_deferred_eval_node(): + @hm.target(dynamic=True) + def choose_build() -> hm.Step: + raise AssertionError("dynamic target must not run during initial bake") + + graph = _lower_to_graph([choose_build()]) + + assert graph["nodes"][0]["step"] == { + "key": "choose_build", + "eval": { + "type": "dynamic", + "target_name": "choose_build", + }, + } + + +def test_command_can_continue_from_dynamic_placeholder(): + @hm.target(dynamic=True) + def choose_compile() -> hm.Step: + return hm.sh("cargo build") + + after = choose_compile().sh("cargo test", label="test") + graph = _lower_to_graph([after]) + + assert _step_keys(graph) == ["choose_compile", "test"] + assert _builds_in_edges(graph) == [(0, 1)] + + def test_pipeline_factory_collects_reachable_via_parent(): base = scratch().sh("install", label="install") leaf_a = base.fork(label="a").sh("test-a", label="test-a") diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_target.py b/crates/hm-dsl-engine/harmont-py/tests/test_target.py index e445e512..1ca43076 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_target.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_target.py @@ -6,7 +6,7 @@ import harmont as hm from harmont._deps import clear_target_names -from harmont._target import clear_target_cache +from harmont._target import clear_target_cache, evaluate_dynamic_target @pytest.fixture(autouse=True) @@ -105,3 +105,38 @@ def venv() -> hm.Step: v1 = venv() v2 = venv() assert v1 is v2 + + +def test_dynamic_target_returns_placeholder_without_evaluating_body(): + call_count = 0 + + @hm.target(dynamic=True) + def choose_build() -> hm.Step: + nonlocal call_count + call_count += 1 + return hm.sh("cargo test") + + placeholder = choose_build() + + assert call_count == 0 + assert placeholder.dynamic_target_name == "choose_build" + assert placeholder.cmd is None + + +def test_dynamic_target_body_can_be_evaluated_by_name(): + @hm.target(dynamic=True) + def choose_build() -> hm.Step: + return hm.sh("cargo test") + + leaf = evaluate_dynamic_target("choose_build") + + assert leaf.cmd == "cargo test" + + +def test_dynamic_target_rejects_group_until_continuation_is_defined(): + @hm.target(dynamic=True) + def checks() -> tuple[hm.Step, ...]: + return hm.group([hm.sh("cargo test"), hm.sh("cargo clippy")]) + + with pytest.raises(ValueError, match="must currently return exactly one leaf"): + evaluate_dynamic_target("checks") From 11fd14bc97c0a0435bfe740e2ea6e94f233adcf7 Mon Sep 17 00:00:00 2001 From: tadhgdowdall Date: Tue, 9 Jun 2026 20:41:38 +0100 Subject: [PATCH 04/10] feat(dsl-engine): render dynamic Python targets --- .../harmont-py/harmont/_target.py | 19 +++++ .../harmont-py/tests/test_target.py | 22 +++++- crates/hm-dsl-engine/src/lib.rs | 18 ++++- crates/hm-dsl-engine/src/python_engine.rs | 38 +++++++++- crates/hm-dsl-engine/src/ts_engine.rs | 11 ++- .../hm-dsl-engine/tests/python_engine_test.rs | 69 +++++++++++++++++++ crates/hm-dsl-engine/tests/ts_engine_test.rs | 20 ++++++ 7 files changed, 193 insertions(+), 4 deletions(-) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_target.py b/crates/hm-dsl-engine/harmont-py/harmont/_target.py index a5ddfbb0..5dca8b42 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_target.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_target.py @@ -101,6 +101,25 @@ def evaluate_dynamic_target(name: str) -> Any: return leaves[0] +def render_dynamic_target_json( + name: str, + *, + env: dict[str, str] | None = None, +) -> str: + """Evaluate one dynamic target and serialize its concrete graph fragment.""" + from ._pipeline import pipeline, pipeline_to_json + + clear_target_memo() + runtime_env = env if env is not None else {} + leaf = evaluate_dynamic_target(name) + fragment = pipeline([leaf], env=runtime_env) + return pipeline_to_json( + fragment, + pipeline_slug=name, + env=runtime_env, + ) + + def target( *, name: str | None = None, diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_target.py b/crates/hm-dsl-engine/harmont-py/tests/test_target.py index 1ca43076..7876284d 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_target.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_target.py @@ -2,11 +2,17 @@ from __future__ import annotations +import json + import pytest import harmont as hm from harmont._deps import clear_target_names -from harmont._target import clear_target_cache, evaluate_dynamic_target +from harmont._target import ( + clear_target_cache, + evaluate_dynamic_target, + render_dynamic_target_json, +) @pytest.fixture(autouse=True) @@ -140,3 +146,17 @@ def checks() -> tuple[hm.Step, ...]: with pytest.raises(ValueError, match="must currently return exactly one leaf"): evaluate_dynamic_target("checks") + + +def test_dynamic_target_json_contains_concrete_fragment_and_runtime_env(): + @hm.target(dynamic=True) + def choose_build() -> hm.Step: + return hm.sh("cargo test", label="selected build") + + fragment = json.loads( + render_dynamic_target_json("choose_build", env={"PROFILE": "release"}) + ) + node = fragment["graph"]["nodes"][0] + + assert node["step"]["eval"] == {"type": "cmd", "cmd": "cargo test"} + assert node["env"]["PROFILE"] == "release" diff --git a/crates/hm-dsl-engine/src/lib.rs b/crates/hm-dsl-engine/src/lib.rs index 2d80324f..4681e2da 100644 --- a/crates/hm-dsl-engine/src/lib.rs +++ b/crates/hm-dsl-engine/src/lib.rs @@ -1,7 +1,8 @@ +use std::collections::BTreeMap; use std::path::Path; use async_trait::async_trait; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; pub mod detect; pub mod python_engine; @@ -21,10 +22,25 @@ pub struct PipelineMeta { pub name: String, } +/// Runtime values available while evaluating a deferred DSL target. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct DynamicContext { + /// Explicit environment supplied for the build. + pub env: BTreeMap, +} + #[async_trait] pub trait DslEngine: Send + Sync { async fn list_pipelines(&self, project_dir: &Path) -> anyhow::Result>; async fn render_pipeline_json(&self, project_dir: &Path, slug: &str) -> anyhow::Result; + /// Evaluate one registered dynamic target and return its v0 IR graph + /// fragment. Implementations must not evaluate unrelated dynamic targets. + async fn render_target_json( + &self, + project_dir: &Path, + target_name: &str, + context: &DynamicContext, + ) -> anyhow::Result; /// Emit the full discovery envelope JSON for every pipeline in the repo: /// `{"schema_version": "...", "pipelines": [{slug, name, allow_manual, /// triggers, definition}, ...]}`. Returned verbatim from the DSL runtime so diff --git a/crates/hm-dsl-engine/src/python_engine.rs b/crates/hm-dsl-engine/src/python_engine.rs index c97e778b..315e7748 100644 --- a/crates/hm-dsl-engine/src/python_engine.rs +++ b/crates/hm-dsl-engine/src/python_engine.rs @@ -6,7 +6,7 @@ use async_trait::async_trait; use tracing::debug; use crate::bundled_sources; -use crate::{DslEngine, PipelineMeta}; +use crate::{DslEngine, DynamicContext, PipelineMeta}; const LIST_PIPELINES_SCRIPT: &str = "\ import sys, json, pathlib, importlib.util @@ -58,6 +58,25 @@ if match is None: print(json.dumps(match['definition'])) "; +const RENDER_TARGET_SCRIPT: &str = "\ +import sys, json, os, pathlib, importlib.util +try: + import harmont as hm + from harmont._target import render_dynamic_target_json +except ImportError as e: + print(f'error: {e}', file=sys.stderr) + sys.exit(1) +target_name = sys.argv[1] +context = json.loads(sys.argv[2]) +runtime_env = context.get('env', {}) +os.environ.update(runtime_env) +for p in sorted(pathlib.Path('.hm').glob('*.py')): + spec = importlib.util.spec_from_file_location(f'_harmont_{p.stem}', p) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) +sys.stdout.write(render_dynamic_target_json(target_name, env=runtime_env)) +"; + #[derive(Debug)] pub struct SubprocessPythonEngine { python_bin: std::path::PathBuf, @@ -129,6 +148,23 @@ impl DslEngine for SubprocessPythonEngine { .context("rendering pipeline via python3") } + async fn render_target_json( + &self, + project_dir: &Path, + target_name: &str, + context: &DynamicContext, + ) -> Result { + let context_json = + serde_json::to_string(context).context("encoding dynamic target context")?; + self.run_script( + project_dir, + RENDER_TARGET_SCRIPT, + &[target_name, &context_json], + ) + .await + .with_context(|| format!("rendering dynamic target {target_name:?} via python3")) + } + async fn registry_json(&self, project_dir: &Path) -> Result { self.run_script(project_dir, REGISTRY_JSON_SCRIPT, &[]) .await diff --git a/crates/hm-dsl-engine/src/ts_engine.rs b/crates/hm-dsl-engine/src/ts_engine.rs index 2d49c459..13247aa2 100644 --- a/crates/hm-dsl-engine/src/ts_engine.rs +++ b/crates/hm-dsl-engine/src/ts_engine.rs @@ -6,7 +6,7 @@ use async_trait::async_trait; use tracing::debug; use crate::bundled_sources; -use crate::{DslEngine, PipelineMeta}; +use crate::{DslEngine, DynamicContext, PipelineMeta}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum JsRuntime { @@ -262,6 +262,15 @@ impl DslEngine for SubprocessTsEngine { .context("rendering pipeline via JS runtime") } + async fn render_target_json( + &self, + _project_dir: &Path, + target_name: &str, + _context: &DynamicContext, + ) -> Result { + bail!("dynamic target {target_name:?} cannot be rendered from TypeScript pipelines yet") + } + async fn registry_json(&self, _project_dir: &Path) -> Result { bail!( "the discovery envelope (hm pipelines) is not yet supported for \ diff --git a/crates/hm-dsl-engine/tests/python_engine_test.rs b/crates/hm-dsl-engine/tests/python_engine_test.rs index ee37a116..ca5cb178 100644 --- a/crates/hm-dsl-engine/tests/python_engine_test.rs +++ b/crates/hm-dsl-engine/tests/python_engine_test.rs @@ -73,3 +73,72 @@ def ci() -> hm.Step: assert_eq!(p["triggers"][0]["branches"][0], "main"); assert_eq!(p["definition"]["version"], "0"); } + +#[tokio::test] +async fn python_renders_dynamic_target_with_runtime_environment() { + if which::which("python3").is_err() { + eprintln!("skipping: python3 not on PATH"); + return; + } + + let dir = tempfile::tempdir().unwrap(); + let harmont = dir.path().join(".hm"); + std::fs::create_dir_all(&harmont).unwrap(); + std::fs::write( + harmont.join("ci.py"), + r#"import os +import harmont as hm + +@hm.target(dynamic=True) +def choose_build() -> hm.Step: + command = 'go test ./...' if os.environ.get('LANGUAGE') == 'go' else 'cargo test' + return hm.sh(command, label='selected build') + +@hm.pipeline('ci') +def ci() -> hm.Step: + return choose_build() +"#, + ) + .unwrap(); + + let engine = hm_dsl_engine::engine_for(hm_dsl_engine::DslLanguage::Python).unwrap(); + let mut context = hm_dsl_engine::DynamicContext::default(); + context.env.insert("LANGUAGE".into(), "go".into()); + + let json = engine + .render_target_json(dir.path(), "choose_build", &context) + .await + .unwrap(); + let fragment: serde_json::Value = serde_json::from_str(&json).unwrap(); + let node = &fragment["graph"]["nodes"][0]; + + assert_eq!(fragment["version"], "0"); + assert_eq!(node["step"]["eval"]["type"], "cmd"); + assert_eq!(node["step"]["eval"]["cmd"], "go test ./..."); + assert_eq!(node["env"]["LANGUAGE"], "go"); +} + +#[tokio::test] +async fn python_dynamic_target_reports_unknown_name() { + if which::which("python3").is_err() { + eprintln!("skipping: python3 not on PATH"); + return; + } + + let dir = tempfile::tempdir().unwrap(); + let harmont = dir.path().join(".hm"); + std::fs::create_dir_all(&harmont).unwrap(); + std::fs::write(harmont.join("ci.py"), "import harmont as hm\n").unwrap(); + + let engine = hm_dsl_engine::engine_for(hm_dsl_engine::DslLanguage::Python).unwrap(); + let error = engine + .render_target_json( + dir.path(), + "missing_target", + &hm_dsl_engine::DynamicContext::default(), + ) + .await + .unwrap_err(); + + assert!(format!("{error:#}").contains("dynamic target 'missing_target' not found")); +} diff --git a/crates/hm-dsl-engine/tests/ts_engine_test.rs b/crates/hm-dsl-engine/tests/ts_engine_test.rs index 8adade17..f5bbcf6a 100644 --- a/crates/hm-dsl-engine/tests/ts_engine_test.rs +++ b/crates/hm-dsl-engine/tests/ts_engine_test.rs @@ -79,3 +79,23 @@ export const pipelines: PipelineDefinition[] = [ let v: serde_json::Value = serde_json::from_str(&json_str).unwrap(); assert_eq!(v["version"], "0"); } + +#[tokio::test] +async fn typescript_dynamic_target_rendering_is_explicitly_unsupported() { + if which::which("bun").is_err() && which::which("node").is_err() { + eprintln!("skipping: no JS runtime on PATH"); + return; + } + + let engine = hm_dsl_engine::engine_for(hm_dsl_engine::DslLanguage::TypeScript).unwrap(); + let error = engine + .render_target_json( + std::path::Path::new("."), + "choose_build", + &hm_dsl_engine::DynamicContext::default(), + ) + .await + .unwrap_err(); + + assert!(format!("{error:#}").contains("cannot be rendered from TypeScript")); +} From 7e31c869bfc89519762dd992ac23af37454a947e Mon Sep 17 00:00:00 2001 From: tadhgdowdall Date: Wed, 10 Jun 2026 17:55:02 +0100 Subject: [PATCH 05/10] feat(exec): evaluate dynamic targets locally --- crates/hm-dsl-engine/src/lib.rs | 2 +- crates/hm-exec/src/lib.rs | 2 +- crates/hm-exec/src/local/backend.rs | 10 +- crates/hm-exec/src/local/mod.rs | 1 + crates/hm-exec/src/local/scheduler.rs | 243 ++++++++++++++++++++++- crates/hm-exec/src/request.rs | 17 ++ crates/hm-exec/tests/backend_contract.rs | 1 + crates/hm/src/commands/run/mod.rs | 57 +++++- 8 files changed, 311 insertions(+), 22 deletions(-) diff --git a/crates/hm-dsl-engine/src/lib.rs b/crates/hm-dsl-engine/src/lib.rs index 4681e2da..6be70ac2 100644 --- a/crates/hm-dsl-engine/src/lib.rs +++ b/crates/hm-dsl-engine/src/lib.rs @@ -30,7 +30,7 @@ pub struct DynamicContext { } #[async_trait] -pub trait DslEngine: Send + Sync { +pub trait DslEngine: Send + Sync + std::fmt::Debug { async fn list_pipelines(&self, project_dir: &Path) -> anyhow::Result>; async fn render_pipeline_json(&self, project_dir: &Path, slug: &str) -> anyhow::Result; /// Evaluate one registered dynamic target and return its v0 IR graph diff --git a/crates/hm-exec/src/lib.rs b/crates/hm-exec/src/lib.rs index 4072613f..68e6df23 100644 --- a/crates/hm-exec/src/lib.rs +++ b/crates/hm-exec/src/lib.rs @@ -30,7 +30,7 @@ mod error; pub use error::{BackendError, Result}; mod request; -pub use request::{Plan, RunOptions, RunRequest, SourceMeta}; +pub use request::{DynamicEvaluator, Plan, RunOptions, RunRequest, SourceMeta}; mod outcome; pub use outcome::{BuildOutcome, BuildStatus, StepResultSummary, StepStatus}; diff --git a/crates/hm-exec/src/local/backend.rs b/crates/hm-exec/src/local/backend.rs index 53ae5371..940893fd 100644 --- a/crates/hm-exec/src/local/backend.rs +++ b/crates/hm-exec/src/local/backend.rs @@ -11,7 +11,7 @@ use tokio_util::sync::CancellationToken; use hm_vm::{HmVm, ImageRegistry, VmBackend, VmConfig}; -use crate::local::{RunnerRegistry, VmRunner}; +use crate::local::{LocalRunContext, RunnerRegistry, VmRunner}; use crate::{BackendError, BackendHandle, Capabilities, ExecutionBackend, Result, RunRequest}; /// Number of cached snapshots the image registry retains before evicting @@ -87,8 +87,12 @@ impl ExecutionBackend for LocalBackend { let join = tokio::spawn(async move { crate::local::run( req.plan.graph, - req.repo_root, - req.pipeline_slug, + LocalRunContext { + repo_root: req.repo_root, + pipeline_slug: req.pipeline_slug, + runtime_env: req.env, + dynamic_evaluator: req.dynamic_evaluator, + }, parallelism, registry, tx, diff --git a/crates/hm-exec/src/local/mod.rs b/crates/hm-exec/src/local/mod.rs index 2d48fe26..636149cd 100644 --- a/crates/hm-exec/src/local/mod.rs +++ b/crates/hm-exec/src/local/mod.rs @@ -15,6 +15,7 @@ mod source; pub use backend::LocalBackend; pub(crate) use runner::RunnerRegistry; // intra-crate: local/backend.rs via crate::local:: pub(crate) use runner::vm::VmRunner; // intra-crate: local/backend.rs via crate::local:: +pub(crate) use scheduler::LocalRunContext; pub(crate) use scheduler::chain_count; pub(crate) use scheduler::run; pub(crate) use source::build_archive_bytes; // intra-crate: cloud/backend.rs via crate::local:: diff --git a/crates/hm-exec/src/local/scheduler.rs b/crates/hm-exec/src/local/scheduler.rs index 68ea09d1..92b89b5e 100644 --- a/crates/hm-exec/src/local/scheduler.rs +++ b/crates/hm-exec/src/local/scheduler.rs @@ -22,7 +22,7 @@ clippy::missing_panics_doc )] -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::path::PathBuf; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -38,11 +38,11 @@ use hm_plugin_protocol::{ }; use uuid::Uuid; -use hm_pipeline_ir::{EdgeKind, PipelineGraph, Transition}; +use hm_pipeline_ir::{EdgeKind, PipelineGraph, StepEval, Transition}; use crate::local::runner::{RunnerRegistry, StepContext}; use crate::local::source::build_archive_bytes; -use crate::{BuildOutcome, BuildStatus, StepResultSummary, StepStatus}; +use crate::{BuildOutcome, BuildStatus, DynamicEvaluator, StepResultSummary, StepStatus}; use super::archive::ArchiveStore; use super::cache; @@ -63,6 +63,75 @@ struct StepOutcome { type StepFuture = futures::future::Shared>; +pub(crate) struct LocalRunContext { + pub repo_root: PathBuf, + pub pipeline_slug: String, + pub runtime_env: BTreeMap, + pub dynamic_evaluator: Option>, +} + +async fn resolve_dynamic_transition( + transition: Transition, + repo_root: &std::path::Path, + runtime_env: &BTreeMap, + evaluator: Option<&dyn DynamicEvaluator>, +) -> anyhow::Result { + let StepEval::Dynamic { target_name } = &transition.step.eval else { + return Ok(transition); + }; + let evaluator = evaluator.ok_or_else(|| { + anyhow::anyhow!("dynamic target {target_name:?} cannot run without a DSL evaluator") + })?; + + let fragment = evaluator + .evaluate(repo_root, target_name, runtime_env) + .await + .map_err(|e| anyhow::anyhow!("{e:#}"))?; + if fragment.node_count() != 1 { + anyhow::bail!( + "dynamic target {target_name:?} produced {} steps; local execution currently requires exactly one concrete step", + fragment.node_count() + ); + } + + let node = fragment + .dag() + .graph() + .node_indices() + .next() + .expect("one-node fragment"); + let mut concrete = fragment.dag()[node].clone(); + if !matches!(concrete.step.eval, StepEval::Cmd { .. }) { + anyhow::bail!("dynamic target {target_name:?} returned another dynamic step"); + } + + let placeholder = transition.step; + concrete.step.key = placeholder.key; + if concrete.step.label.is_none() { + concrete.step.label = placeholder.label; + } + if concrete.step.image.is_none() { + concrete.step.image = placeholder.image; + } + if concrete.step.timeout_seconds.is_none() { + concrete.step.timeout_seconds = placeholder.timeout_seconds; + } + if concrete.step.cache.is_none() { + concrete.step.cache = placeholder.cache; + } + if concrete.step.runner.is_none() { + concrete.step.runner = placeholder.runner; + } + if concrete.step.runner_args.is_none() { + concrete.step.runner_args = placeholder.runner_args; + } + + let mut env = transition.env; + env.extend(concrete.env); + concrete.env = env; + Ok(concrete) +} + /// Entry point: run a parsed pipeline locally end-to-end. /// /// Emits every [`BuildEvent`] to `tx` (via an internal broadcast bus that @@ -79,14 +148,19 @@ type StepFuture = futures::future::Shared>; /// surfaced via the returned [`BuildOutcome`], not as an `Err`. pub(crate) async fn run( graph: PipelineGraph, - repo_root: PathBuf, - pipeline_slug: String, + context: LocalRunContext, parallelism: usize, runner_registry: Arc, tx: tokio::sync::mpsc::Sender, cancel: CancellationToken, keep_going: bool, ) -> crate::Result { + let LocalRunContext { + repo_root, + pipeline_slug, + runtime_env, + dynamic_evaluator, + } = context; // Set up per-run state. let bus = EventBus::new(); let archives = Arc::new(ArchiveStore::new()); @@ -180,6 +254,9 @@ pub(crate) async fn run( let bus = bus.clone(); let cancel = cancel.clone(); let run_ctx = run_ctx.clone(); + let repo_root = repo_root.clone(); + let runtime_env = runtime_env.clone(); + let dynamic_evaluator = dynamic_evaluator.clone(); let fut: StepFuture = async move { // Await all predecessors. @@ -233,6 +310,9 @@ pub(crate) async fn run( bus, cancel, keep_going, + repo_root, + runtime_env, + dynamic_evaluator, ) .await { @@ -361,10 +441,21 @@ async fn execute_step( bus: Arc, cancel: CancellationToken, keep_going: bool, + repo_root: PathBuf, + runtime_env: BTreeMap, + dynamic_evaluator: Option>, ) -> anyhow::Result { - let step_wire = transition.step.into_command().ok_or_else(|| { - anyhow::anyhow!("dynamic step expansion is not implemented by the local backend yet") - })?; + let transition = resolve_dynamic_transition( + transition, + &repo_root, + &runtime_env, + dynamic_evaluator.as_deref(), + ) + .await?; + let step_wire = transition + .step + .into_command() + .ok_or_else(|| anyhow::anyhow!("dynamic target returned another dynamic step"))?; let step_key = step_wire.key.clone(); let display_name = step_wire.label.clone().unwrap_or_else(|| { let cmd = step_wire.cmd.trim(); @@ -623,3 +714,139 @@ fn compute_chain_info(dag: &Dag) -> ChainInfo { node_chain_pos, } } + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used)] +mod dynamic_tests { + use super::*; + use hm_pipeline_ir::{PipelineStep, StepEval}; + + #[derive(Debug)] + struct FakeEvaluator { + graph: PipelineGraph, + } + + #[async_trait::async_trait] + impl DynamicEvaluator for FakeEvaluator { + async fn evaluate( + &self, + _repo_root: &std::path::Path, + _target_name: &str, + _env: &BTreeMap, + ) -> crate::Result { + Ok(self.graph.clone()) + } + } + + fn graph(json: &str) -> PipelineGraph { + serde_json::from_str(json).unwrap() + } + + fn dynamic_transition() -> Transition { + Transition { + step: PipelineStep { + key: "choose-build".into(), + label: Some("Choose build".into()), + eval: StepEval::Dynamic { + target_name: "choose_build".into(), + }, + image: Some("ubuntu:24.04".into()), + env: None, + timeout_seconds: None, + cache: None, + runner: None, + runner_args: None, + }, + env: BTreeMap::from([("PIPELINE_ENV".into(), "present".into())]), + } + } + + #[tokio::test] + async fn dynamic_transition_resolves_to_one_concrete_step() { + let evaluator = FakeEvaluator { + graph: graph( + r#"{ + "version":"0", + "graph":{ + "nodes":[ + {"step":{"key":"generated","eval":{"type":"cmd","cmd":"go test ./..."}}, "env":{"LANGUAGE":"go"}} + ], + "node_holes":[], + "edge_property":"directed", + "edges":[] + } + }"#, + ), + }; + + let resolved = resolve_dynamic_transition( + dynamic_transition(), + std::path::Path::new("/repo"), + &BTreeMap::from([("LANGUAGE".into(), "go".into())]), + Some(&evaluator), + ) + .await + .unwrap(); + + assert_eq!(resolved.step.key, "choose-build"); + assert_eq!(resolved.step.label.as_deref(), Some("Choose build")); + assert_eq!(resolved.step.image.as_deref(), Some("ubuntu:24.04")); + assert_eq!( + resolved.step.eval, + StepEval::Cmd { + cmd: "go test ./...".into() + } + ); + assert_eq!(resolved.env["PIPELINE_ENV"], "present"); + assert_eq!(resolved.env["LANGUAGE"], "go"); + } + + #[tokio::test] + async fn dynamic_transition_requires_an_evaluator() { + let error = resolve_dynamic_transition( + dynamic_transition(), + std::path::Path::new("/repo"), + &BTreeMap::new(), + None, + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("without a DSL evaluator")); + } + + #[tokio::test] + async fn dynamic_transition_rejects_multi_step_fragment() { + let evaluator = FakeEvaluator { + graph: graph( + r#"{ + "version":"0", + "graph":{ + "nodes":[ + {"step":{"key":"a","eval":{"type":"cmd","cmd":"echo a"}}, "env":{}}, + {"step":{"key":"b","eval":{"type":"cmd","cmd":"echo b"}}, "env":{}} + ], + "node_holes":[], + "edge_property":"directed", + "edges":[[0,1,"builds_in"]] + } + }"#, + ), + }; + + let error = resolve_dynamic_transition( + dynamic_transition(), + std::path::Path::new("/repo"), + &BTreeMap::new(), + Some(&evaluator), + ) + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("currently requires exactly one concrete step") + ); + } +} diff --git a/crates/hm-exec/src/request.rs b/crates/hm-exec/src/request.rs index fdb683a0..ddf3083a 100644 --- a/crates/hm-exec/src/request.rs +++ b/crates/hm-exec/src/request.rs @@ -7,6 +7,21 @@ use std::time::Duration; use hm_pipeline_ir::PipelineGraph; use hm_plugin_protocol::events::PlanSummary; +/// Evaluates a deferred target in the source DSL. +/// +/// The execution layer owns this narrow contract so local scheduling does not +/// depend on Python, TypeScript, or `hm-dsl-engine` implementation details. +#[async_trait::async_trait] +pub trait DynamicEvaluator: Send + Sync + std::fmt::Debug { + /// Evaluate `target_name` using explicit runtime environment values. + async fn evaluate( + &self, + repo_root: &std::path::Path, + target_name: &str, + env: &BTreeMap, + ) -> crate::Result; +} + /// A rendered, ready-to-run pipeline. /// /// Carries both the typed graph (for client-scheduling backends like local) and @@ -85,6 +100,8 @@ pub struct RunRequest { pub repo_root: PathBuf, pub pipeline_slug: String, pub env: BTreeMap, + /// Runtime target evaluator used by client-scheduled backends. + pub dynamic_evaluator: Option>, pub source: SourceMeta, pub options: RunOptions, } diff --git a/crates/hm-exec/tests/backend_contract.rs b/crates/hm-exec/tests/backend_contract.rs index 7b1eac53..af9b0732 100644 --- a/crates/hm-exec/tests/backend_contract.rs +++ b/crates/hm-exec/tests/backend_contract.rs @@ -122,6 +122,7 @@ fn fake_request() -> RunRequest { repo_root: std::path::PathBuf::from("/tmp"), pipeline_slug: "p".into(), env: Default::default(), + dynamic_evaluator: None, source: SourceMeta { branch: "main".into(), commit: "0".repeat(40), diff --git a/crates/hm/src/commands/run/mod.rs b/crates/hm/src/commands/run/mod.rs index 84602ae1..f234151f 100644 --- a/crates/hm/src/commands/run/mod.rs +++ b/crates/hm/src/commands/run/mod.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::sync::Arc; use anyhow::{Context, Result}; @@ -8,6 +9,29 @@ use crate::cli::RunArgs; use crate::context::RunContext; use crate::error::{ErrorCategory, HmError}; +#[derive(Debug)] +struct DslDynamicEvaluator { + engine: Arc, +} + +#[async_trait::async_trait] +impl hm_exec::DynamicEvaluator for DslDynamicEvaluator { + async fn evaluate( + &self, + repo_root: &std::path::Path, + target_name: &str, + env: &std::collections::BTreeMap, + ) -> hm_exec::Result { + let context = hm_dsl_engine::DynamicContext { env: env.clone() }; + let json = self + .engine + .render_target_json(repo_root, target_name, &context) + .await + .map_err(|e| hm_exec::BackendError::Local(format!("{e:#}")))?; + hm_exec::Plan::parse(json).map(|plan| plan.graph) + } +} + /// Top-level driver for `hm run`. /// /// Runs the local worktree on the selected execution backend: `docker` @@ -20,10 +44,12 @@ use crate::error::{ErrorCategory, HmError}; /// - neither → `ctx.config.backend` (figment-layered, default `docker`) /// /// This is a THIN driver over the `hm-exec` backends: it builds an -/// [`hm_exec::ExecutionBackend`], renders the pipeline to v0 IR once, starts -/// the build, drives its event stream through an `hm_render` renderer, owns -/// Ctrl-C, and returns the build's process exit code. Cloud authentication is -/// resolved BEFORE the (local) render work so a missing token fails fast. +/// [`hm_exec::ExecutionBackend`], renders the initial pipeline to v0 IR, +/// starts the build, drives its event stream through an `hm_render` renderer, +/// owns Ctrl-C, and returns the build's process exit code. Local runs retain +/// the DSL engine so deferred targets can be evaluated when reached. Cloud +/// authentication is resolved BEFORE the (local) render work so a missing +/// token fails fast. /// /// # Errors /// @@ -95,14 +121,21 @@ pub async fn handle(args: RunArgs, ctx: RunContext) -> Result { } // 3. Render + parse the plan once (shared by every backend). - let (repo_root, slug, ir_json) = render_pipeline(&args, &ctx).await?; + let (repo_root, slug, ir_json, dsl_engine) = render_pipeline(&args, &ctx).await?; let plan = hm_exec::Plan::parse(ir_json).map_err(|e| backend_anyhow(&e))?; let (branch, commit) = git_metadata(&repo_root, args.branch.clone()); + let dynamic_evaluator = if backend_name == "cloud" { + None + } else { + Some(Arc::new(DslDynamicEvaluator { engine: dsl_engine }) + as Arc) + }; let req = hm_exec::RunRequest { plan, repo_root, pipeline_slug: slug, env: parse_env(&args.env).into_iter().collect(), + dynamic_evaluator, source: hm_exec::SourceMeta { branch, commit, @@ -192,7 +225,12 @@ fn git_metadata(root: &std::path::Path, branch_override: Option) -> (Str async fn render_pipeline( args: &RunArgs, _ctx: &RunContext, -) -> Result<(std::path::PathBuf, String, String)> { +) -> Result<( + std::path::PathBuf, + String, + String, + Arc, +)> { let repo_root = match args.dir.clone() { Some(p) => p, None => std::env::current_dir().context("cannot determine current directory")?, @@ -200,8 +238,9 @@ async fn render_pipeline( let lang = detect::detect_language(&repo_root).map_err(|e| HmError::DslEngine(format!("{e:#}")))?; - let engine = - hm_dsl_engine::engine_for(lang).map_err(|e| HmError::DslEngine(format!("{e:#}")))?; + let engine: Arc = Arc::from( + hm_dsl_engine::engine_for(lang).map_err(|e| HmError::DslEngine(format!("{e:#}")))?, + ); let slug = if let Some(s) = &args.pipeline { s.clone() @@ -229,7 +268,7 @@ async fn render_pipeline( .await .map_err(|e| HmError::PipelineRender(format!("{e:#}")))?; - Ok((repo_root, slug, json_str)) + Ok((repo_root, slug, json_str, engine)) } /// Convert an [`hm_exec::BackendError`] into an [`anyhow::Error`] that carries From 1de06ae5ab6392ea0b9196608e27631acaa5f0aa Mon Sep 17 00:00:00 2001 From: tadhgdowdall Date: Wed, 10 Jun 2026 18:04:32 +0100 Subject: [PATCH 06/10] test(dsl): migrate cross-sdk cache fixtures to eval --- crates/hm-dsl-engine/harmont-py/tests/test_keygen.py | 9 ++++++--- crates/hm-dsl-engine/harmont-ts/tests/keygen.test.ts | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_keygen.py b/crates/hm-dsl-engine/harmont-py/tests/test_keygen.py index d831c196..2904824d 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_keygen.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_keygen.py @@ -395,7 +395,7 @@ def test_golden_hash_cross_sdk_reference_pipeline(): { "step": { "key": "build", - "cmd": "make build", + "eval": {"type": "cmd", "cmd": "make build"}, "cache": {"policy": "forever", "env_keys": []}, }, "env": {}, @@ -428,7 +428,10 @@ def test_golden_hash_cross_sdk_chained_pipeline(): { "step": { "key": "setup", - "cmd": "apt-get update && apt-get install -y gcc", + "eval": { + "type": "cmd", + "cmd": "apt-get update && apt-get install -y gcc", + }, "cache": {"policy": "forever", "env_keys": []}, }, "env": {}, @@ -436,7 +439,7 @@ def test_golden_hash_cross_sdk_chained_pipeline(): { "step": { "key": "compile", - "cmd": "gcc -o main main.c", + "eval": {"type": "cmd", "cmd": "gcc -o main main.c"}, "cache": {"policy": "forever", "env_keys": []}, }, "env": {}, diff --git a/crates/hm-dsl-engine/harmont-ts/tests/keygen.test.ts b/crates/hm-dsl-engine/harmont-ts/tests/keygen.test.ts index 9804a8a0..8a32b78d 100644 --- a/crates/hm-dsl-engine/harmont-ts/tests/keygen.test.ts +++ b/crates/hm-dsl-engine/harmont-ts/tests/keygen.test.ts @@ -173,7 +173,7 @@ describe("resolvePipelineCacheKeys", () => { { step: { key: "build", - cmd: "make build", + eval: { type: "cmd", cmd: "make build" }, cache: { policy: "forever", env_keys: [] }, }, env: {}, @@ -209,7 +209,10 @@ describe("resolvePipelineCacheKeys", () => { { step: { key: "setup", - cmd: "apt-get update && apt-get install -y gcc", + eval: { + type: "cmd", + cmd: "apt-get update && apt-get install -y gcc", + }, cache: { policy: "forever", env_keys: [] }, }, env: {}, @@ -217,7 +220,7 @@ describe("resolvePipelineCacheKeys", () => { { step: { key: "compile", - cmd: "gcc -o main main.c", + eval: { type: "cmd", cmd: "gcc -o main main.c" }, cache: { policy: "forever", env_keys: [] }, }, env: {}, From e72a055ea7eae968edfa683feabe366c85dac47c Mon Sep 17 00:00:00 2001 From: tadhgdowdall Date: Wed, 10 Jun 2026 18:15:34 +0100 Subject: [PATCH 07/10] feat(exec): run dynamic command chains --- crates/hm-exec/src/local/scheduler.rs | 250 +++++++++++++++++++------- 1 file changed, 186 insertions(+), 64 deletions(-) diff --git a/crates/hm-exec/src/local/scheduler.rs b/crates/hm-exec/src/local/scheduler.rs index 92b89b5e..efba7e28 100644 --- a/crates/hm-exec/src/local/scheduler.rs +++ b/crates/hm-exec/src/local/scheduler.rs @@ -56,9 +56,7 @@ use tokio_util::sync::CancellationToken; struct StepOutcome { exit_code: i32, snapshot: Option, - /// `None` only for steps short-circuited because a predecessor failed - /// or the build was cancelled before they could run. - summary: Option, + summaries: Vec, } type StepFuture = futures::future::Shared>; @@ -70,14 +68,14 @@ pub(crate) struct LocalRunContext { pub dynamic_evaluator: Option>, } -async fn resolve_dynamic_transition( +async fn resolve_dynamic_transitions( transition: Transition, repo_root: &std::path::Path, runtime_env: &BTreeMap, evaluator: Option<&dyn DynamicEvaluator>, -) -> anyhow::Result { +) -> anyhow::Result> { let StepEval::Dynamic { target_name } = &transition.step.eval else { - return Ok(transition); + return Ok(vec![transition]); }; let evaluator = evaluator.ok_or_else(|| { anyhow::anyhow!("dynamic target {target_name:?} cannot run without a DSL evaluator") @@ -87,49 +85,75 @@ async fn resolve_dynamic_transition( .evaluate(repo_root, target_name, runtime_env) .await .map_err(|e| anyhow::anyhow!("{e:#}"))?; - if fragment.node_count() != 1 { - anyhow::bail!( - "dynamic target {target_name:?} produced {} steps; local execution currently requires exactly one concrete step", - fragment.node_count() - ); + if fragment.node_count() == 0 { + anyhow::bail!("dynamic target {target_name:?} produced no steps"); } - let node = fragment - .dag() - .graph() - .node_indices() - .next() - .expect("one-node fragment"); - let mut concrete = fragment.dag()[node].clone(); - if !matches!(concrete.step.eval, StepEval::Cmd { .. }) { - anyhow::bail!("dynamic target {target_name:?} returned another dynamic step"); + let dag = fragment.dag(); + let order = toposort(dag.graph(), None) + .map_err(|_| anyhow::anyhow!("dynamic target {target_name:?} produced a cyclic graph"))?; + for (position, &node) in order.iter().enumerate() { + if !matches!(dag[node].step.eval, StepEval::Cmd { .. }) { + anyhow::bail!("dynamic target {target_name:?} returned another dynamic step"); + } + + let parents: Vec<(EdgeKind, NodeIndex)> = dag + .parents(node) + .iter(dag) + .map(|(edge, parent)| (*dag.edge_weight(edge).expect("edge in dynamic DAG"), parent)) + .collect(); + let valid = if position == 0 { + parents.is_empty() + } else { + parents.as_slice() == [(EdgeKind::BuildsIn, order[position - 1])] + }; + if !valid { + anyhow::bail!( + "dynamic target {target_name:?} produced a branched or grouped graph; only one linear command chain is supported" + ); + } } let placeholder = transition.step; - concrete.step.key = placeholder.key; - if concrete.step.label.is_none() { - concrete.step.label = placeholder.label; - } - if concrete.step.image.is_none() { - concrete.step.image = placeholder.image; - } - if concrete.step.timeout_seconds.is_none() { - concrete.step.timeout_seconds = placeholder.timeout_seconds; - } - if concrete.step.cache.is_none() { - concrete.step.cache = placeholder.cache; - } - if concrete.step.runner.is_none() { - concrete.step.runner = placeholder.runner; - } - if concrete.step.runner_args.is_none() { - concrete.step.runner_args = placeholder.runner_args; - } + let terminal = order.len() - 1; + let mut transitions = Vec::with_capacity(order.len()); + for (position, node) in order.into_iter().enumerate() { + let mut concrete = dag[node].clone(); + concrete.step.key = if position == terminal { + placeholder.key.clone() + } else { + format!("{}/{}", placeholder.key, concrete.step.key) + }; + if position == terminal { + if concrete.step.label.is_none() { + concrete.step.label.clone_from(&placeholder.label); + } + if concrete.step.timeout_seconds.is_none() { + concrete.step.timeout_seconds = placeholder.timeout_seconds; + } + if concrete.step.cache.is_none() { + concrete.step.cache.clone_from(&placeholder.cache); + } + if concrete.step.runner.is_none() { + concrete.step.runner.clone_from(&placeholder.runner); + } + if concrete.step.runner_args.is_none() { + concrete + .step + .runner_args + .clone_from(&placeholder.runner_args); + } + } + if position == 0 && concrete.step.image.is_none() { + concrete.step.image.clone_from(&placeholder.image); + } - let mut env = transition.env; - env.extend(concrete.env); - concrete.env = env; - Ok(concrete) + let mut env = transition.env.clone(); + env.extend(concrete.env); + concrete.env = env; + transitions.push(concrete); + } + Ok(transitions) } /// Entry point: run a parsed pipeline locally end-to-end. @@ -273,13 +297,13 @@ pub(crate) async fn run( return StepOutcome { exit_code: 0, snapshot: None, - summary: Some(StepResultSummary { + summaries: vec![StepResultSummary { step_id: Uuid::new_v4(), key: node_key, status, exit_code: None, duration_ms: 0, - }), + }], }; } @@ -322,13 +346,13 @@ pub(crate) async fn run( StepOutcome { exit_code: 1, snapshot: None, - summary: Some(StepResultSummary { + summaries: vec![StepResultSummary { step_id: Uuid::new_v4(), key: node_key, status: StepStatus::Failed, exit_code: Some(1), duration_ms: 0, - }), + }], } } } @@ -387,7 +411,10 @@ pub(crate) async fn run( ); } - let steps: Vec = outcomes.iter().filter_map(|o| o.summary.clone()).collect(); + let steps: Vec = outcomes + .iter() + .flat_map(|outcome| outcome.summaries.clone()) + .collect(); let dur = started_total.elapsed().as_millis() as u64; @@ -430,10 +457,10 @@ pub(crate) async fn run( async fn execute_step( _node_idx: NodeIndex, transition: Transition, - parent_snapshot: Option, + mut parent_snapshot: Option, chain_id: usize, chain_pos: usize, - parent_key: Option, + mut parent_key: Option, archive_id: ArchiveId, run_id: Uuid, run_ctx: StepContext, @@ -445,13 +472,66 @@ async fn execute_step( runtime_env: BTreeMap, dynamic_evaluator: Option>, ) -> anyhow::Result { - let transition = resolve_dynamic_transition( + let transitions = resolve_dynamic_transitions( transition, &repo_root, &runtime_env, dynamic_evaluator.as_deref(), ) .await?; + let mut summaries = Vec::with_capacity(transitions.len()); + + for (offset, transition) in transitions.into_iter().enumerate() { + let step_key = transition.step.key.clone(); + let outcome = execute_command( + transition, + parent_snapshot, + chain_id, + chain_pos + offset, + parent_key, + archive_id, + run_id, + run_ctx.clone(), + runner_registry.clone(), + bus.clone(), + cancel.clone(), + keep_going, + ) + .await?; + summaries.extend(outcome.summaries); + if outcome.exit_code != 0 { + return Ok(StepOutcome { + exit_code: outcome.exit_code, + snapshot: None, + summaries, + }); + } + parent_snapshot = outcome.snapshot; + parent_key = Some(step_key); + } + + Ok(StepOutcome { + exit_code: 0, + snapshot: parent_snapshot, + summaries, + }) +} + +#[allow(clippy::too_many_arguments)] +async fn execute_command( + transition: Transition, + parent_snapshot: Option, + chain_id: usize, + chain_pos: usize, + parent_key: Option, + archive_id: ArchiveId, + run_id: Uuid, + run_ctx: StepContext, + runner_registry: Arc, + bus: Arc, + cancel: CancellationToken, + keep_going: bool, +) -> anyhow::Result { let step_wire = transition .step .into_command() @@ -565,13 +645,13 @@ async fn execute_step( return Ok(StepOutcome { exit_code: 124, snapshot: None, - summary: Some(StepResultSummary { + summaries: vec![StepResultSummary { step_id, key: step_key.clone(), status: StepStatus::TimedOut, exit_code: Some(124), duration_ms: dur_ms, - }), + }], }); } } @@ -611,13 +691,13 @@ async fn execute_step( Ok(StepOutcome { exit_code: sr.exit_code, snapshot: sr.committed_snapshot, - summary: Some(StepResultSummary { + summaries: vec![StepResultSummary { step_id, key: step_key.clone(), status, exit_code: Some(sr.exit_code), duration_ms: dur_ms, - }), + }], }) } Err(e) => { @@ -779,7 +859,7 @@ mod dynamic_tests { ), }; - let resolved = resolve_dynamic_transition( + let resolved = resolve_dynamic_transitions( dynamic_transition(), std::path::Path::new("/repo"), &BTreeMap::from([("LANGUAGE".into(), "go".into())]), @@ -787,6 +867,7 @@ mod dynamic_tests { ) .await .unwrap(); + let resolved = &resolved[0]; assert_eq!(resolved.step.key, "choose-build"); assert_eq!(resolved.step.label.as_deref(), Some("Choose build")); @@ -803,7 +884,7 @@ mod dynamic_tests { #[tokio::test] async fn dynamic_transition_requires_an_evaluator() { - let error = resolve_dynamic_transition( + let error = resolve_dynamic_transitions( dynamic_transition(), std::path::Path::new("/repo"), &BTreeMap::new(), @@ -816,7 +897,7 @@ mod dynamic_tests { } #[tokio::test] - async fn dynamic_transition_rejects_multi_step_fragment() { + async fn dynamic_transition_accepts_linear_multi_step_fragment() { let evaluator = FakeEvaluator { graph: graph( r#"{ @@ -834,19 +915,60 @@ mod dynamic_tests { ), }; - let error = resolve_dynamic_transition( + let resolved = resolve_dynamic_transitions( dynamic_transition(), std::path::Path::new("/repo"), &BTreeMap::new(), Some(&evaluator), ) .await - .unwrap_err(); + .unwrap(); - assert!( - error - .to_string() - .contains("currently requires exactly one concrete step") + assert_eq!(resolved.len(), 2); + assert_eq!(resolved[0].step.key, "choose-build/a"); + assert_eq!(resolved[1].step.key, "choose-build"); + assert_eq!( + resolved[0].step.eval, + StepEval::Cmd { + cmd: "echo a".into() + } + ); + assert_eq!( + resolved[1].step.eval, + StepEval::Cmd { + cmd: "echo b".into() + } ); } + + #[tokio::test] + async fn dynamic_transition_rejects_branched_fragment() { + let evaluator = FakeEvaluator { + graph: graph( + r#"{ + "version":"0", + "graph":{ + "nodes":[ + {"step":{"key":"a","eval":{"type":"cmd","cmd":"echo a"}}, "env":{}}, + {"step":{"key":"b","eval":{"type":"cmd","cmd":"echo b"}}, "env":{}} + ], + "node_holes":[], + "edge_property":"directed", + "edges":[] + } + }"#, + ), + }; + + let error = resolve_dynamic_transitions( + dynamic_transition(), + std::path::Path::new("/repo"), + &BTreeMap::new(), + Some(&evaluator), + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("branched or grouped graph")); + } } From 6f514214d4ba518fac73064693fafd283ae1a79b Mon Sep 17 00:00:00 2001 From: tadhgdowdall Date: Wed, 10 Jun 2026 19:57:58 +0100 Subject: [PATCH 08/10] feat(exec): support dynamic target groups --- .../harmont-py/harmont/_target.py | 21 +- .../harmont-py/tests/test_target.py | 40 ++- crates/hm-exec/src/local/scheduler.rs | 257 ++++++++++++++---- 3 files changed, 247 insertions(+), 71 deletions(-) diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_target.py b/crates/hm-dsl-engine/harmont-py/harmont/_target.py index 5dca8b42..91170f90 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/_target.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/_target.py @@ -77,12 +77,7 @@ def clear_target_cache() -> None: def evaluate_dynamic_target(name: str) -> Any: - """Evaluate a registered dynamic target for runtime graph expansion. - - Dynamic groups are intentionally rejected for the first implementation: - a single leaf gives the scheduler one unambiguous snapshot from which - subsequent ``builds_in`` steps can continue. - """ + """Evaluate a registered dynamic target for runtime graph expansion.""" try: fn = _DYNAMIC_TARGETS_BY_NAME[name] except KeyError as e: @@ -90,15 +85,7 @@ def evaluate_dynamic_target(name: str) -> Any: from ._unwrap import as_leaves - leaves = as_leaves(call_with_deps(fn)) - if len(leaves) != 1: - msg = ( - f"hm: dynamic target {name!r} returned {len(leaves)} leaves\n" - " → dynamic targets must currently return exactly one leaf; " - "group continuation semantics are not yet defined" - ) - raise ValueError(msg) - return leaves[0] + return as_leaves(call_with_deps(fn)) def render_dynamic_target_json( @@ -111,8 +98,8 @@ def render_dynamic_target_json( clear_target_memo() runtime_env = env if env is not None else {} - leaf = evaluate_dynamic_target(name) - fragment = pipeline([leaf], env=runtime_env) + leaves = evaluate_dynamic_target(name) + fragment = pipeline(leaves, env=runtime_env) return pipeline_to_json( fragment, pipeline_slug=name, diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_target.py b/crates/hm-dsl-engine/harmont-py/tests/test_target.py index 7876284d..beabb80c 100644 --- a/crates/hm-dsl-engine/harmont-py/tests/test_target.py +++ b/crates/hm-dsl-engine/harmont-py/tests/test_target.py @@ -134,18 +134,48 @@ def test_dynamic_target_body_can_be_evaluated_by_name(): def choose_build() -> hm.Step: return hm.sh("cargo test") - leaf = evaluate_dynamic_target("choose_build") + leaves = evaluate_dynamic_target("choose_build") - assert leaf.cmd == "cargo test" + assert len(leaves) == 1 + assert leaves[0].cmd == "cargo test" -def test_dynamic_target_rejects_group_until_continuation_is_defined(): +def test_dynamic_target_returns_group_leaves(): @hm.target(dynamic=True) def checks() -> tuple[hm.Step, ...]: return hm.group([hm.sh("cargo test"), hm.sh("cargo clippy")]) - with pytest.raises(ValueError, match="must currently return exactly one leaf"): - evaluate_dynamic_target("checks") + leaves = evaluate_dynamic_target("checks") + + assert [leaf.cmd for leaf in leaves] == ["cargo test", "cargo clippy"] + + fragment = json.loads(render_dynamic_target_json("checks")) + assert len(fragment["graph"]["nodes"]) == 2 + assert fragment["graph"]["edges"] == [] + + +def test_dynamic_group_can_define_explicit_continuation(): + @hm.target(dynamic=True) + def checks() -> tuple[hm.Step, ...]: + return hm.group( + [ + hm.sh("cargo test", label="test"), + hm.sh("cargo clippy", label="clippy"), + hm.wait(), + hm.sh("prepare deploy", label="merge"), + ] + ) + + fragment = json.loads(render_dynamic_target_json("checks")) + keys = [node["step"]["key"] for node in fragment["graph"]["nodes"]] + merge = keys.index("merge") + depends_on = { + (source, target) + for source, target, kind in fragment["graph"]["edges"] + if kind == "depends_on" + } + + assert depends_on == {(keys.index("test"), merge), (keys.index("clippy"), merge)} def test_dynamic_target_json_contains_concrete_fragment_and_runtime_env(): diff --git a/crates/hm-exec/src/local/scheduler.rs b/crates/hm-exec/src/local/scheduler.rs index efba7e28..fa9f590b 100644 --- a/crates/hm-exec/src/local/scheduler.rs +++ b/crates/hm-exec/src/local/scheduler.rs @@ -68,14 +68,25 @@ pub(crate) struct LocalRunContext { pub dynamic_evaluator: Option>, } -async fn resolve_dynamic_transitions( +#[derive(Debug)] +struct ExecutionBatch { + transitions: Vec, + parents: Vec>, + terminals: Vec, +} + +async fn resolve_execution_batch( transition: Transition, repo_root: &std::path::Path, runtime_env: &BTreeMap, evaluator: Option<&dyn DynamicEvaluator>, -) -> anyhow::Result> { +) -> anyhow::Result { let StepEval::Dynamic { target_name } = &transition.step.eval else { - return Ok(vec![transition]); + return Ok(ExecutionBatch { + transitions: vec![transition], + parents: vec![Vec::new()], + terminals: vec![0], + }); }; let evaluator = evaluator.ok_or_else(|| { anyhow::anyhow!("dynamic target {target_name:?} cannot run without a DSL evaluator") @@ -92,39 +103,60 @@ async fn resolve_dynamic_transitions( let dag = fragment.dag(); let order = toposort(dag.graph(), None) .map_err(|_| anyhow::anyhow!("dynamic target {target_name:?} produced a cyclic graph"))?; - for (position, &node) in order.iter().enumerate() { + let position_by_node: HashMap = order + .iter() + .enumerate() + .map(|(position, &node)| (node, position)) + .collect(); + let mut parents = Vec::with_capacity(order.len()); + let mut terminals = Vec::new(); + + for &node in &order { if !matches!(dag[node].step.eval, StepEval::Cmd { .. }) { anyhow::bail!("dynamic target {target_name:?} returned another dynamic step"); } - let parents: Vec<(EdgeKind, NodeIndex)> = dag + let node_parents: Vec<(EdgeKind, usize)> = dag .parents(node) .iter(dag) - .map(|(edge, parent)| (*dag.edge_weight(edge).expect("edge in dynamic DAG"), parent)) + .map(|(edge, parent)| { + ( + *dag.edge_weight(edge).expect("edge in dynamic DAG"), + position_by_node[&parent], + ) + }) .collect(); - let valid = if position == 0 { - parents.is_empty() - } else { - parents.as_slice() == [(EdgeKind::BuildsIn, order[position - 1])] - }; - if !valid { + if node_parents + .iter() + .filter(|(kind, _)| *kind == EdgeKind::BuildsIn) + .count() + > 1 + { anyhow::bail!( - "dynamic target {target_name:?} produced a branched or grouped graph; only one linear command chain is supported" + "dynamic target {target_name:?} produced a step with multiple snapshot parents" ); } + parents.push(node_parents); + if dag.children(node).iter(dag).next().is_none() { + terminals.push(position_by_node[&node]); + } } let placeholder = transition.step; - let terminal = order.len() - 1; + let single_terminal = terminals + .as_slice() + .first() + .copied() + .filter(|_| terminals.len() == 1); let mut transitions = Vec::with_capacity(order.len()); for (position, node) in order.into_iter().enumerate() { let mut concrete = dag[node].clone(); - concrete.step.key = if position == terminal { + concrete.step.key = if Some(position) == single_terminal { placeholder.key.clone() } else { format!("{}/{}", placeholder.key, concrete.step.key) }; - if position == terminal { + if Some(position) == single_terminal { if concrete.step.label.is_none() { concrete.step.label.clone_from(&placeholder.label); } @@ -144,7 +176,7 @@ async fn resolve_dynamic_transitions( .clone_from(&placeholder.runner_args); } } - if position == 0 && concrete.step.image.is_none() { + if parents[position].is_empty() && concrete.step.image.is_none() { concrete.step.image.clone_from(&placeholder.image); } @@ -153,7 +185,11 @@ async fn resolve_dynamic_transitions( concrete.env = env; transitions.push(concrete); } - Ok(transitions) + Ok(ExecutionBatch { + transitions, + parents, + terminals, + }) } /// Entry point: run a parsed pipeline locally end-to-end. @@ -234,6 +270,7 @@ pub(crate) async fn run( let semaphore = Arc::new(tokio::sync::Semaphore::new(parallelism)); + let default_image = graph.default_image().map(str::to_owned); let dag = graph.dag(); let pipeline_timeout = graph.timeout_seconds(); let chain_info = compute_chain_info(dag); @@ -281,6 +318,7 @@ pub(crate) async fn run( let repo_root = repo_root.clone(); let runtime_env = runtime_env.clone(); let dynamic_evaluator = dynamic_evaluator.clone(); + let default_image = default_image.clone(); let fut: StepFuture = async move { // Await all predecessors. @@ -337,6 +375,7 @@ pub(crate) async fn run( repo_root, runtime_env, dynamic_evaluator, + default_image, ) .await { @@ -457,10 +496,10 @@ pub(crate) async fn run( async fn execute_step( _node_idx: NodeIndex, transition: Transition, - mut parent_snapshot: Option, + parent_snapshot: Option, chain_id: usize, chain_pos: usize, - mut parent_key: Option, + parent_key: Option, archive_id: ArchiveId, run_id: Uuid, run_ctx: StepContext, @@ -471,24 +510,88 @@ async fn execute_step( repo_root: PathBuf, runtime_env: BTreeMap, dynamic_evaluator: Option>, + default_image: Option, ) -> anyhow::Result { - let transitions = resolve_dynamic_transitions( + let batch = resolve_execution_batch( transition, &repo_root, &runtime_env, dynamic_evaluator.as_deref(), ) .await?; - let mut summaries = Vec::with_capacity(transitions.len()); + let mut outcomes: Vec> = vec![None; batch.transitions.len()]; + let mut summaries = Vec::with_capacity(batch.transitions.len()); + let mut first_failure = None; + let step_keys: Vec = batch + .transitions + .iter() + .map(|transition| transition.step.key.clone()) + .collect(); - for (offset, transition) in transitions.into_iter().enumerate() { + for (offset, mut transition) in batch.transitions.into_iter().enumerate() { + let node_parents = &batch.parents[offset]; let step_key = transition.step.key.clone(); + let blocked = cancel.is_cancelled() + || node_parents.iter().any(|(_, parent)| { + outcomes[*parent] + .as_ref() + .is_some_and(|outcome| outcome.exit_code != 0) + }); + if blocked { + let status = if cancel.is_cancelled() { + StepStatus::Canceled + } else { + StepStatus::Skipped + }; + let summary = StepResultSummary { + step_id: Uuid::new_v4(), + key: step_key, + status, + exit_code: None, + duration_ms: 0, + }; + summaries.push(summary.clone()); + outcomes[offset] = Some(StepOutcome { + exit_code: 0, + snapshot: None, + summaries: vec![summary], + }); + continue; + } + + let builds_in_parent = node_parents + .iter() + .find(|(kind, _)| *kind == EdgeKind::BuildsIn) + .map(|(_, parent)| *parent); + let node_parent_snapshot = builds_in_parent + .and_then(|parent| outcomes[parent].as_ref()) + .and_then(|outcome| outcome.snapshot.clone()) + .or_else(|| { + if node_parents.is_empty() { + parent_snapshot.clone() + } else { + None + } + }); + let node_parent_key = builds_in_parent + .map(|parent| step_keys[parent].clone()) + .or_else(|| { + if node_parents.is_empty() { + parent_key.clone() + } else { + None + } + }); + if node_parent_snapshot.is_none() && transition.step.image.is_none() { + transition.step.image.clone_from(&default_image); + } + let outcome = execute_command( transition, - parent_snapshot, + node_parent_snapshot, chain_id, chain_pos + offset, - parent_key, + node_parent_key, archive_id, run_id, run_ctx.clone(), @@ -498,21 +601,23 @@ async fn execute_step( keep_going, ) .await?; - summaries.extend(outcome.summaries); - if outcome.exit_code != 0 { - return Ok(StepOutcome { - exit_code: outcome.exit_code, - snapshot: None, - summaries, - }); + summaries.extend(outcome.summaries.clone()); + if outcome.exit_code != 0 && first_failure.is_none() { + first_failure = Some(outcome.exit_code); } - parent_snapshot = outcome.snapshot; - parent_key = Some(step_key); + outcomes[offset] = Some(outcome); } + let snapshot = if batch.terminals.len() == 1 { + outcomes[batch.terminals[0]] + .as_ref() + .and_then(|outcome| outcome.snapshot.clone()) + } else { + None + }; Ok(StepOutcome { - exit_code: 0, - snapshot: parent_snapshot, + exit_code: first_failure.unwrap_or(0), + snapshot, summaries, }) } @@ -859,7 +964,7 @@ mod dynamic_tests { ), }; - let resolved = resolve_dynamic_transitions( + let resolved = resolve_execution_batch( dynamic_transition(), std::path::Path::new("/repo"), &BTreeMap::from([("LANGUAGE".into(), "go".into())]), @@ -867,7 +972,7 @@ mod dynamic_tests { ) .await .unwrap(); - let resolved = &resolved[0]; + let resolved = &resolved.transitions[0]; assert_eq!(resolved.step.key, "choose-build"); assert_eq!(resolved.step.label.as_deref(), Some("Choose build")); @@ -884,7 +989,7 @@ mod dynamic_tests { #[tokio::test] async fn dynamic_transition_requires_an_evaluator() { - let error = resolve_dynamic_transitions( + let error = resolve_execution_batch( dynamic_transition(), std::path::Path::new("/repo"), &BTreeMap::new(), @@ -915,7 +1020,7 @@ mod dynamic_tests { ), }; - let resolved = resolve_dynamic_transitions( + let resolved = resolve_execution_batch( dynamic_transition(), std::path::Path::new("/repo"), &BTreeMap::new(), @@ -924,17 +1029,22 @@ mod dynamic_tests { .await .unwrap(); - assert_eq!(resolved.len(), 2); - assert_eq!(resolved[0].step.key, "choose-build/a"); - assert_eq!(resolved[1].step.key, "choose-build"); + assert_eq!(resolved.transitions.len(), 2); + assert_eq!(resolved.transitions[0].step.key, "choose-build/a"); + assert_eq!(resolved.transitions[1].step.key, "choose-build"); + assert_eq!( + resolved.parents, + vec![vec![], vec![(EdgeKind::BuildsIn, 0)]] + ); + assert_eq!(resolved.terminals, vec![1]); assert_eq!( - resolved[0].step.eval, + resolved.transitions[0].step.eval, StepEval::Cmd { cmd: "echo a".into() } ); assert_eq!( - resolved[1].step.eval, + resolved.transitions[1].step.eval, StepEval::Cmd { cmd: "echo b".into() } @@ -942,7 +1052,7 @@ mod dynamic_tests { } #[tokio::test] - async fn dynamic_transition_rejects_branched_fragment() { + async fn dynamic_transition_accepts_group_with_multiple_terminals() { let evaluator = FakeEvaluator { graph: graph( r#"{ @@ -960,15 +1070,64 @@ mod dynamic_tests { ), }; - let error = resolve_dynamic_transitions( + let resolved = resolve_execution_batch( dynamic_transition(), std::path::Path::new("/repo"), &BTreeMap::new(), Some(&evaluator), ) .await - .unwrap_err(); + .unwrap(); + + assert_eq!(resolved.transitions.len(), 2); + let mut keys: Vec<&str> = resolved + .transitions + .iter() + .map(|transition| transition.step.key.as_str()) + .collect(); + keys.sort_unstable(); + assert_eq!(keys, vec!["choose-build/a", "choose-build/b"]); + assert_eq!(resolved.parents, vec![vec![], vec![]]); + assert_eq!(resolved.terminals, vec![0, 1]); + } + + #[tokio::test] + async fn dynamic_transition_accepts_explicit_group_continuation() { + let evaluator = FakeEvaluator { + graph: graph( + r#"{ + "version":"0", + "graph":{ + "nodes":[ + {"step":{"key":"a","eval":{"type":"cmd","cmd":"echo a"}}, "env":{}}, + {"step":{"key":"b","eval":{"type":"cmd","cmd":"echo b"}}, "env":{}}, + {"step":{"key":"merge","eval":{"type":"cmd","cmd":"echo merge"}}, "env":{}} + ], + "node_holes":[], + "edge_property":"directed", + "edges":[[0,2,"depends_on"],[1,2,"depends_on"]] + } + }"#, + ), + }; - assert!(error.to_string().contains("branched or grouped graph")); + let resolved = resolve_execution_batch( + dynamic_transition(), + std::path::Path::new("/repo"), + &BTreeMap::new(), + Some(&evaluator), + ) + .await + .unwrap(); + + let terminal = resolved.terminals[0]; + assert_eq!(resolved.terminals.len(), 1); + assert_eq!(resolved.transitions[terminal].step.key, "choose-build"); + assert_eq!(resolved.parents[terminal].len(), 2); + assert!( + resolved.parents[terminal] + .iter() + .all(|(kind, _)| *kind == EdgeKind::DependsOn) + ); } } From a133efb620cc6e79b5f940760cb4ab55d8343a90 Mon Sep 17 00:00:00 2001 From: tadhgdowdall Date: Wed, 10 Jun 2026 20:14:13 +0100 Subject: [PATCH 09/10] feat(dsl): support dynamic TypeScript targets --- crates/hm-dsl-engine/harmont-ts/src/index.ts | 7 ++- .../hm-dsl-engine/harmont-ts/src/pipeline.ts | 27 ++++++---- crates/hm-dsl-engine/harmont-ts/src/step.ts | 14 +++++ crates/hm-dsl-engine/harmont-ts/src/target.ts | 47 +++++++++++++++- .../harmont-ts/tests/pipeline.test.ts | 35 ++++++++++++ .../harmont-ts/tests/target.test.ts | 49 ++++++++++++++++- crates/hm-dsl-engine/src/ts_engine.rs | 43 ++++++++++++--- crates/hm-dsl-engine/tests/ts_engine_test.rs | 54 +++++++++++++++---- 8 files changed, 244 insertions(+), 32 deletions(-) diff --git a/crates/hm-dsl-engine/harmont-ts/src/index.ts b/crates/hm-dsl-engine/harmont-ts/src/index.ts index c950c850..e0368bc0 100644 --- a/crates/hm-dsl-engine/harmont-ts/src/index.ts +++ b/crates/hm-dsl-engine/harmont-ts/src/index.ts @@ -18,7 +18,12 @@ export { pullRequest, } from "./triggers.js"; export { pipeline, type PipelineIR, type PipelineOptions } from "./pipeline.js"; -export { target, clearTargetCache } from "./target.js"; +export { + target, + clearTargetCache, + renderDynamicTarget, + type TargetOptions, +} from "./target.js"; export { aptBase } from "./toolchains/shared.js"; export { renderEnvelope, diff --git a/crates/hm-dsl-engine/harmont-ts/src/pipeline.ts b/crates/hm-dsl-engine/harmont-ts/src/pipeline.ts index 6133e0e8..976bca5b 100644 --- a/crates/hm-dsl-engine/harmont-ts/src/pipeline.ts +++ b/crates/hm-dsl-engine/harmont-ts/src/pipeline.ts @@ -55,12 +55,14 @@ function lowerToGraph( opts?: PipelineOptions, ): PipelineIR["graph"] { const ordered = topoCollect(leaves); - const commandSteps = ordered.filter((s) => s._cmd !== null && !s._isWait); - const keys = resolveKeys(commandSteps); + const emittedSteps = ordered.filter( + (s) => (s._cmd !== null || s._dynamicTargetName != null) && !s._isWait, + ); + const keys = resolveKeys(emittedSteps); const idxById = new Map(); - for (let i = 0; i < commandSteps.length; i++) { - idxById.set(commandSteps[i]._id, i); + for (let i = 0; i < emittedSteps.length; i++) { + idxById.set(emittedSteps[i]._id, i); } const hasBuildsInParent = new Set(); @@ -77,17 +79,17 @@ function lowerToGraph( continue; } - if (s._cmd === null) continue; + if (s._cmd === null && s._dynamicTargetName == null) continue; const nodeIdx = idxById.get(s._id)!; const stepKey = keys.get(s._id)!; const stepDict: Record = { key: stepKey, - eval: { - type: "cmd", - cmd: s._cmd, - }, + eval: + s._dynamicTargetName != null + ? { type: "dynamic", target_name: s._dynamicTargetName } + : { type: "cmd", cmd: s._cmd }, }; if (s._label != null) stepDict.label = s._label; if (s._cache != null) stepDict.cache = cachePolicyToDict(s._cache); @@ -107,7 +109,7 @@ function lowerToGraph( const parentKey = resolvedParentKey(s, keys); if (parentKey !== null) { - const parentIdx = findIdxByKey(parentKey, commandSteps, keys, idxById); + const parentIdx = findIdxByKey(parentKey, emittedSteps, keys, idxById); edges.push([parentIdx, nodeIdx, "builds_in"]); hasBuildsInParent.add(nodeIdx); } @@ -168,7 +170,10 @@ function resolvedParentKey( ): string | null { let node = s._parent; while (node !== null) { - if (node._cmd !== null && !node._isWait) { + if ( + (node._cmd !== null || node._dynamicTargetName != null) && + !node._isWait + ) { return keys.get(node._id) ?? null; } node = node._parent; diff --git a/crates/hm-dsl-engine/harmont-ts/src/step.ts b/crates/hm-dsl-engine/harmont-ts/src/step.ts index 5803c0cf..bea8bfd4 100644 --- a/crates/hm-dsl-engine/harmont-ts/src/step.ts +++ b/crates/hm-dsl-engine/harmont-ts/src/step.ts @@ -28,6 +28,7 @@ export class Step { readonly _runner: string | undefined; readonly _runnerArgs: Readonly> | undefined; readonly _keyOverride: string | undefined; + readonly _dynamicTargetName: string | undefined; /** @internal */ constructor(init: { @@ -43,6 +44,7 @@ export class Step { runner?: string; runnerArgs?: Record; keyOverride?: string; + dynamicTargetName?: string; }) { this._id = nextId++; this._cmd = init.cmd; @@ -57,6 +59,7 @@ export class Step { this._runner = init.runner; this._runnerArgs = init.runnerArgs; this._keyOverride = init.keyOverride; + this._dynamicTargetName = init.dynamicTargetName; } sh(cmd: string, opts?: StepOptions): Step { @@ -108,10 +111,21 @@ export class Step { runner: this._runner, runnerArgs: this._runnerArgs as Record | undefined, keyOverride: this._keyOverride, + dynamicTargetName: this._dynamicTargetName, }); } } +/** @internal */ +export function dynamicTarget(name: string): Step { + return new Step({ + cmd: null, + parent: null, + keyOverride: name, + dynamicTargetName: name, + }); +} + export function scratch(opts?: { image?: string }): Step { return new Step({ cmd: null, parent: null, image: opts?.image }); } diff --git a/crates/hm-dsl-engine/harmont-ts/src/target.ts b/crates/hm-dsl-engine/harmont-ts/src/target.ts index 0aa289d4..541611e0 100644 --- a/crates/hm-dsl-engine/harmont-ts/src/target.ts +++ b/crates/hm-dsl-engine/harmont-ts/src/target.ts @@ -1,7 +1,34 @@ +import { pipeline, type PipelineIR } from "./pipeline.js"; +import { dynamicTarget, type Step } from "./step.js"; + const cache = new Map(); +const dynamicTargets = new Map Step | readonly Step[]>(); + +export interface TargetOptions { + readonly dynamic?: boolean; +} + +export function target( + name: string, + fn: () => T, + opts?: { readonly dynamic?: false }, +): () => T; +export function target( + name: string, + fn: () => Step | readonly Step[], + opts: { readonly dynamic: true }, +): () => Step; +export function target( + name: string, + fn: () => T, + opts?: TargetOptions, +): () => T | Step { + const key = Symbol(name); + if (opts?.dynamic) { + dynamicTargets.set(name, fn as () => Step | readonly Step[]); + return () => dynamicTarget(name); + } -export function target(_name: string, fn: () => T): () => T { - const key = Symbol(_name); return () => { if (!cache.has(key)) { cache.set(key, fn()); @@ -12,4 +39,20 @@ export function target(_name: string, fn: () => T): () => T { export function clearTargetCache(): void { cache.clear(); + dynamicTargets.clear(); +} + +export function renderDynamicTarget( + name: string, + env: Readonly> = {}, +): PipelineIR { + const fn = dynamicTargets.get(name); + if (fn == null) { + throw new Error(`hm: dynamic target '${name}' not found`); + } + + cache.clear(); + const result = fn(); + const leaves = Array.isArray(result) ? [...result] : [result as Step]; + return pipeline(leaves, { env }); } diff --git a/crates/hm-dsl-engine/harmont-ts/tests/pipeline.test.ts b/crates/hm-dsl-engine/harmont-ts/tests/pipeline.test.ts index d3e684f1..8a8cf66f 100644 --- a/crates/hm-dsl-engine/harmont-ts/tests/pipeline.test.ts +++ b/crates/hm-dsl-engine/harmont-ts/tests/pipeline.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { pipeline } from "../src/pipeline.js"; import { scratch, sh, wait, timeout } from "../src/step.js"; import { forever, onChange } from "../src/cache.js"; +import { target } from "../src/target.js"; function stepKeys(ir: any): string[] { return ir.graph.nodes.map((n: any) => n.step.key); @@ -154,6 +155,40 @@ describe("lowering: optional fields", () => { }); }); +describe("lowering: dynamic target", () => { + it("emits a deferred target placeholder", () => { + const chooseBuild = target( + "choose-build", + () => sh("npm run build"), + { dynamic: true }, + ); + + const ir = pipeline([chooseBuild()]); + expect(ir.graph.nodes[0].step).toMatchObject({ + key: "choose-build", + eval: { + type: "dynamic", + target_name: "choose-build", + }, + }); + }); + + it("connects subsequent static steps to the placeholder", () => { + const chooseBuild = target( + "choose-build-chain", + () => sh("npm run build"), + { dynamic: true }, + ); + + const ir = pipeline([chooseBuild().sh("npm test")]); + expect(buildsInEdges(ir)).toEqual([[0, 1]]); + expect(ir.graph.nodes[1].step.eval).toEqual({ + type: "cmd", + cmd: "npm test", + }); + }); +}); + describe("lowering: cache serialization", () => { it("serializes forever cache", () => { const s = sh("echo", { cache: forever({ envKeys: ["CI"] }) }); diff --git a/crates/hm-dsl-engine/harmont-ts/tests/target.test.ts b/crates/hm-dsl-engine/harmont-ts/tests/target.test.ts index 5460ee6f..a35a69d8 100644 --- a/crates/hm-dsl-engine/harmont-ts/tests/target.test.ts +++ b/crates/hm-dsl-engine/harmont-ts/tests/target.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it, beforeEach } from "vitest"; -import { target, clearTargetCache } from "../src/target.js"; +import { + target, + clearTargetCache, + renderDynamicTarget, +} from "../src/target.js"; import { sh } from "../src/step.js"; import { forever } from "../src/cache.js"; @@ -69,4 +73,47 @@ describe("target", () => { expect(a).toBe(b); expect(a.value).toBe(b.value); }); + + it("defers dynamic target evaluation", () => { + let callCount = 0; + const chooseBuild = target( + "choose-build", + () => { + callCount++; + return sh("npm run build"); + }, + { dynamic: true }, + ); + + const placeholder = chooseBuild(); + expect(callCount).toBe(0); + expect(placeholder._dynamicTargetName).toBe("choose-build"); + + const rendered = renderDynamicTarget("choose-build"); + expect(callCount).toBe(1); + expect(rendered.graph.nodes[0].step.eval).toEqual({ + type: "cmd", + cmd: "npm run build", + }); + }); + + it("renders dynamic target groups", () => { + target( + "checks", + () => [sh("npm test"), sh("npm run lint")], + { dynamic: true }, + ); + + const rendered = renderDynamicTarget("checks"); + expect(rendered.graph.nodes.map((node) => node.step.eval)).toEqual([ + { type: "cmd", cmd: "npm test" }, + { type: "cmd", cmd: "npm run lint" }, + ]); + }); + + it("rejects unknown dynamic target names", () => { + expect(() => renderDynamicTarget("missing")).toThrow( + "dynamic target 'missing' not found", + ); + }); }); diff --git a/crates/hm-dsl-engine/src/ts_engine.rs b/crates/hm-dsl-engine/src/ts_engine.rs index 13247aa2..aab4eb6a 100644 --- a/crates/hm-dsl-engine/src/ts_engine.rs +++ b/crates/hm-dsl-engine/src/ts_engine.rs @@ -34,10 +34,13 @@ import { readdirSync } from 'node:fs'; import { join, resolve } from 'node:path'; const projectDir = process.argv[2]; -const mode = process.argv[3]; // "list" or "render" +const mode = process.argv[3]; // "list", "render", or "render-target" const slug = process.argv[4] || null; +const context = JSON.parse(process.argv[5] || '{}'); const harmontDir = join(projectDir, '.hm'); +if (context.env) Object.assign(process.env, context.env); + const tsFiles = readdirSync(harmontDir) .filter(f => f.endsWith('.ts')) .sort(); @@ -56,7 +59,13 @@ for (const file of tsFiles) { else if (d) defs.push(d); } -const { renderEnvelope } = await import('@harmont/hm'); +const { renderDynamicTarget, renderEnvelope } = await import('@harmont/hm'); + +if (mode === 'render-target') { + process.stdout.write(JSON.stringify(renderDynamicTarget(slug, context.env))); + process.exit(0); +} + const envelope = JSON.parse(renderEnvelope(defs, { basePath: projectDir })); if (mode === 'render') { @@ -154,7 +163,13 @@ impl SubprocessTsEngine { } } - async fn run(&self, project_dir: &Path, mode: &str, slug: Option<&str>) -> Result { + async fn run( + &self, + project_dir: &Path, + mode: &str, + slug: Option<&str>, + context: Option<&str>, + ) -> Result { let tmp = self.setup_temp()?; let runner_path = tmp.path().join("runner.mjs"); @@ -223,6 +238,9 @@ impl SubprocessTsEngine { if let Some(s) = slug { cmd.arg(s); } + if let Some(context) = context { + cmd.arg(context); + } cmd.env("NODE_PATH", tmp.path().join("node_modules")) .stdin(Stdio::null()) @@ -247,7 +265,7 @@ impl SubprocessTsEngine { impl DslEngine for SubprocessTsEngine { async fn list_pipelines(&self, project_dir: &Path) -> Result> { let stdout = self - .run(project_dir, "list", None) + .run(project_dir, "list", None, None) .await .context("listing pipelines via JS runtime")?; @@ -257,18 +275,27 @@ impl DslEngine for SubprocessTsEngine { } async fn render_pipeline_json(&self, project_dir: &Path, slug: &str) -> Result { - self.run(project_dir, "render", Some(slug)) + self.run(project_dir, "render", Some(slug), None) .await .context("rendering pipeline via JS runtime") } async fn render_target_json( &self, - _project_dir: &Path, + project_dir: &Path, target_name: &str, - _context: &DynamicContext, + context: &DynamicContext, ) -> Result { - bail!("dynamic target {target_name:?} cannot be rendered from TypeScript pipelines yet") + let context_json = + serde_json::to_string(context).context("encoding dynamic target context")?; + self.run( + project_dir, + "render-target", + Some(target_name), + Some(&context_json), + ) + .await + .with_context(|| format!("rendering dynamic target {target_name:?} via JS runtime")) } async fn registry_json(&self, _project_dir: &Path) -> Result { diff --git a/crates/hm-dsl-engine/tests/ts_engine_test.rs b/crates/hm-dsl-engine/tests/ts_engine_test.rs index f5bbcf6a..7003698d 100644 --- a/crates/hm-dsl-engine/tests/ts_engine_test.rs +++ b/crates/hm-dsl-engine/tests/ts_engine_test.rs @@ -81,21 +81,57 @@ export const pipelines: PipelineDefinition[] = [ } #[tokio::test] -async fn typescript_dynamic_target_rendering_is_explicitly_unsupported() { +async fn typescript_renders_dynamic_target_with_runtime_environment() { if which::which("bun").is_err() && which::which("node").is_err() { eprintln!("skipping: no JS runtime on PATH"); return; } + let dir = tempfile::tempdir().unwrap(); + let harmont = dir.path().join(".hm"); + std::fs::create_dir_all(&harmont).unwrap(); + std::fs::write( + harmont.join("ci.ts"), + r#"import { pipeline, sh, target, type PipelineDefinition } from '@harmont/hm'; + +const chooseBuild = target( + 'choose-build', + () => process.env.BUILD_KIND === 'release' + ? sh('npm run build:release') + : sh('npm run build:debug'), + { dynamic: true }, +); + +export default [{ + slug: 'ci', + pipeline: pipeline([chooseBuild()]), +}] satisfies PipelineDefinition[]; +"#, + ) + .unwrap(); + let engine = hm_dsl_engine::engine_for(hm_dsl_engine::DslLanguage::TypeScript).unwrap(); - let error = engine - .render_target_json( - std::path::Path::new("."), - "choose_build", - &hm_dsl_engine::DynamicContext::default(), - ) + let pipeline_json = engine.render_pipeline_json(dir.path(), "ci").await.unwrap(); + let pipeline: serde_json::Value = serde_json::from_str(&pipeline_json).unwrap(); + assert_eq!( + pipeline["graph"]["nodes"][0]["step"]["eval"]["type"], + "dynamic" + ); + assert_eq!( + pipeline["graph"]["nodes"][0]["step"]["eval"]["target_name"], + "choose-build" + ); + + let mut context = hm_dsl_engine::DynamicContext::default(); + context.env.insert("BUILD_KIND".into(), "release".into()); + let rendered = engine + .render_target_json(dir.path(), "choose-build", &context) .await - .unwrap_err(); + .unwrap(); - assert!(format!("{error:#}").contains("cannot be rendered from TypeScript")); + let fragment: serde_json::Value = serde_json::from_str(&rendered).unwrap(); + assert_eq!( + fragment["graph"]["nodes"][0]["step"]["eval"]["cmd"], + "npm run build:release" + ); } From 742a6e31cbdefbee447d4fb65980613ad5027677 Mon Sep 17 00:00:00 2001 From: tadhgdowdall Date: Wed, 10 Jun 2026 20:18:42 +0100 Subject: [PATCH 10/10] feat(dsl): add cross-sdk environment helper --- .../harmont-py/harmont/__init__.py | 1 + .../hm-dsl-engine/harmont-py/harmont/_env.py | 10 +++++++++ .../harmont-py/tests/test_env.py | 16 ++++++++++++++ crates/hm-dsl-engine/harmont-ts/src/env.ts | 8 +++++++ crates/hm-dsl-engine/harmont-ts/src/index.ts | 1 + .../harmont-ts/tests/env.test.ts | 21 +++++++++++++++++++ .../harmont-ts/tests/integration.test.ts | 2 ++ .../hm-dsl-engine/tests/python_engine_test.rs | 5 ++--- crates/hm-dsl-engine/tests/ts_engine_test.rs | 4 ++-- 9 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 crates/hm-dsl-engine/harmont-py/harmont/_env.py create mode 100644 crates/hm-dsl-engine/harmont-py/tests/test_env.py create mode 100644 crates/hm-dsl-engine/harmont-ts/src/env.ts create mode 100644 crates/hm-dsl-engine/harmont-ts/tests/env.test.ts diff --git a/crates/hm-dsl-engine/harmont-py/harmont/__init__.py b/crates/hm-dsl-engine/harmont-py/harmont/__init__.py index f8b7c37e..cb38123c 100644 --- a/crates/hm-dsl-engine/harmont-py/harmont/__init__.py +++ b/crates/hm-dsl-engine/harmont-py/harmont/__init__.py @@ -34,6 +34,7 @@ from ._duration import parse_duration as _parse_duration from ._elixir import elixir from ._envelope import dump_registry_json +from ._env import env from ._go import go from ._js import JsProject, js, ts from ._pipeline import pipeline as _pipeline_factory diff --git a/crates/hm-dsl-engine/harmont-py/harmont/_env.py b/crates/hm-dsl-engine/harmont-py/harmont/_env.py new file mode 100644 index 00000000..08532e5d --- /dev/null +++ b/crates/hm-dsl-engine/harmont-py/harmont/_env.py @@ -0,0 +1,10 @@ +"""Environment access shared by static and dynamic pipeline evaluation.""" + +from __future__ import annotations + +import os + + +def env(name: str, default: str | None = None) -> str | None: + """Return an environment variable, or ``default`` when it is unset.""" + return os.environ.get(name, default) diff --git a/crates/hm-dsl-engine/harmont-py/tests/test_env.py b/crates/hm-dsl-engine/harmont-py/tests/test_env.py new file mode 100644 index 00000000..af8c9ef5 --- /dev/null +++ b/crates/hm-dsl-engine/harmont-py/tests/test_env.py @@ -0,0 +1,16 @@ +import harmont as hm + + +def test_env_returns_value(monkeypatch): + monkeypatch.setenv("HARMONT_TEST_ENV", "configured") + assert hm.env("HARMONT_TEST_ENV") == "configured" + + +def test_env_returns_default_when_unset(monkeypatch): + monkeypatch.delenv("HARMONT_TEST_ENV", raising=False) + assert hm.env("HARMONT_TEST_ENV", "fallback") == "fallback" + + +def test_env_returns_none_without_default(monkeypatch): + monkeypatch.delenv("HARMONT_TEST_ENV", raising=False) + assert hm.env("HARMONT_TEST_ENV") is None diff --git a/crates/hm-dsl-engine/harmont-ts/src/env.ts b/crates/hm-dsl-engine/harmont-ts/src/env.ts new file mode 100644 index 00000000..65811e3a --- /dev/null +++ b/crates/hm-dsl-engine/harmont-ts/src/env.ts @@ -0,0 +1,8 @@ +export function env(name: string): string | undefined; +export function env(name: string, defaultValue: string): string; +export function env( + name: string, + defaultValue?: string, +): string | undefined { + return process.env[name] ?? defaultValue; +} diff --git a/crates/hm-dsl-engine/harmont-ts/src/index.ts b/crates/hm-dsl-engine/harmont-ts/src/index.ts index e0368bc0..ec7a68ca 100644 --- a/crates/hm-dsl-engine/harmont-ts/src/index.ts +++ b/crates/hm-dsl-engine/harmont-ts/src/index.ts @@ -1,4 +1,5 @@ export { Step, scratch, sh, timeout, wait, type StepOptions } from "./step.js"; +export { env } from "./env.js"; export { type CachePolicy, type CacheForever, diff --git a/crates/hm-dsl-engine/harmont-ts/tests/env.test.ts b/crates/hm-dsl-engine/harmont-ts/tests/env.test.ts new file mode 100644 index 00000000..84f74836 --- /dev/null +++ b/crates/hm-dsl-engine/harmont-ts/tests/env.test.ts @@ -0,0 +1,21 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { env } from "../src/env.js"; + +afterEach(() => { + delete process.env.HARMONT_TEST_ENV; +}); + +describe("env", () => { + it("returns the environment value", () => { + process.env.HARMONT_TEST_ENV = "configured"; + expect(env("HARMONT_TEST_ENV")).toBe("configured"); + }); + + it("returns the default when unset", () => { + expect(env("HARMONT_TEST_ENV", "fallback")).toBe("fallback"); + }); + + it("returns undefined without a default", () => { + expect(env("HARMONT_TEST_ENV")).toBeUndefined(); + }); +}); diff --git a/crates/hm-dsl-engine/harmont-ts/tests/integration.test.ts b/crates/hm-dsl-engine/harmont-ts/tests/integration.test.ts index 8cec87b6..36b77aaa 100644 --- a/crates/hm-dsl-engine/harmont-ts/tests/integration.test.ts +++ b/crates/hm-dsl-engine/harmont-ts/tests/integration.test.ts @@ -9,6 +9,7 @@ import { ttl, onChange, compose, + env, pipeline, target, clearTargetCache, @@ -215,6 +216,7 @@ describe("public API completeness", () => { expect(typeof ttl).toBe("function"); expect(typeof onChange).toBe("function"); expect(typeof compose).toBe("function"); + expect(typeof env).toBe("function"); expect(typeof pipeline).toBe("function"); expect(typeof target).toBe("function"); expect(typeof clearTargetCache).toBe("function"); diff --git a/crates/hm-dsl-engine/tests/python_engine_test.rs b/crates/hm-dsl-engine/tests/python_engine_test.rs index ca5cb178..c2ac298f 100644 --- a/crates/hm-dsl-engine/tests/python_engine_test.rs +++ b/crates/hm-dsl-engine/tests/python_engine_test.rs @@ -86,12 +86,11 @@ async fn python_renders_dynamic_target_with_runtime_environment() { std::fs::create_dir_all(&harmont).unwrap(); std::fs::write( harmont.join("ci.py"), - r#"import os -import harmont as hm + r#"import harmont as hm @hm.target(dynamic=True) def choose_build() -> hm.Step: - command = 'go test ./...' if os.environ.get('LANGUAGE') == 'go' else 'cargo test' + command = 'go test ./...' if hm.env('LANGUAGE') == 'go' else 'cargo test' return hm.sh(command, label='selected build') @hm.pipeline('ci') diff --git a/crates/hm-dsl-engine/tests/ts_engine_test.rs b/crates/hm-dsl-engine/tests/ts_engine_test.rs index 7003698d..4ace0f5b 100644 --- a/crates/hm-dsl-engine/tests/ts_engine_test.rs +++ b/crates/hm-dsl-engine/tests/ts_engine_test.rs @@ -92,11 +92,11 @@ async fn typescript_renders_dynamic_target_with_runtime_environment() { std::fs::create_dir_all(&harmont).unwrap(); std::fs::write( harmont.join("ci.ts"), - r#"import { pipeline, sh, target, type PipelineDefinition } from '@harmont/hm'; + r#"import { env, pipeline, sh, target, type PipelineDefinition } from '@harmont/hm'; const chooseBuild = target( 'choose-build', - () => process.env.BUILD_KIND === 'release' + () => env('BUILD_KIND') === 'release' ? sh('npm run build:release') : sh('npm run build:debug'), { dynamic: true },