test(smoke): smoke test in rust

.githooks/pre-push

@@ -27,7 +27,7 @@ if ! cargo clippy --locked --all-targets --all-features -- -D warnings; then
 fi
 
 # Tests
-if ! cargo test --locked --workspace --all-features; then
+if ! cargo test --locked --workspace --features pluto-eth2api/integration; then
     echo "❌ Test failures detected"
     exit 1
 fi

.github/workflows/smoke-tests.yml

@@ -1,20 +1,18 @@
 name: Compose smoke tests
 
-# Manual only: each scenario stands up a full docker-compose cluster (N nodes +
-# relay + prometheus) and observes it for 2 minutes. The resource-heavy
-# very_large scenario remains local-only. Too heavy to attach to push or
-# pull_request.
+# Manual only: each scenario runs a full docker-compose cluster for two
+# minutes, too heavy for push or pull_request. very_large stays local-only.
 on:
   workflow_dispatch:
     inputs:
       scenarios:
-        description: "Scenario filter (go test -run regex). Empty runs the CI matrix; very_large is excluded."
+        description: "Space-separated scenario names (e.g. `default_alpha pluto_dkg`). Empty runs the CI matrix; very_large is always excluded."
         type: string
         default: ""
-      go_timeout:
-        description: "go test -timeout. Must exceed the sum of the selected scenarios."
-        type: string
-        default: "50m"
+      smoke_timeout:
+        description: "Minutes allowed for the smoke run itself (image and harness builds are separate steps). Must exceed the sum of the selected scenarios' windows."
+        type: number
+        default: 50
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -24,72 +22,71 @@ permissions:
   contents: read
   actions: read
 
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: 0
+  RUSTFLAGS: "-Dwarnings -C debuginfo=0"
+
 jobs:
   smoke:
     name: Compose smoke tests
     runs-on: ubuntu-24.04
-    # Covers the pluto image build (release build of pluto-cli inside docker,
-    # uncached on a fresh runner) plus the scenario matrix.
-    timeout-minutes: 90
+    timeout-minutes: 100
 
     steps:
       - name: Checkout
         uses: actions/checkout@v6
 
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: test-infra/compose/go.mod
-          cache-dependency-path: test-infra/compose/go.sum
+      - name: Cache cargo registry and target
+        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+
+      - name: Install `oas3-gen`
+        run: cargo install oas3-gen@0.24.0 --locked
 
       - name: Build pluto image
-        # Built here rather than letting the harness do it inside `go test`, so
-        # the release compile does not consume the -timeout budget (which should
-        # bound observation, not compilation) and a build break fails in its own
-        # step. The harness still calls `docker build` during its define step;
-        # replicating the tag and build-arg exactly makes that a cache hit.
-        #
-        # Deliberately plain `docker build`, not buildx: setup-buildx-action's
-        # docker-container driver keeps a separate cache that the harness's
-        # `docker build` would not see, so the image would be compiled twice.
+        # Built in its own step so the release compile is not charged to the smoke
+        # budget; the harness's own `docker build` then hits the cache.
         timeout-minutes: 40
         run: |
           docker build -t pluto:local \
             --build-arg "GIT_COMMIT_HASH_SHORT=$(git rev-parse --short=7 HEAD)" .
 
+      - name: Build smoke harness
+        run: cargo test --locked -p pluto-test-compose --test smoke --features smoke --no-run
+
       - name: Run smoke tests
-        working-directory: test-infra/compose
-        # Inputs are passed as env vars, never interpolated into the script:
-        # `${{ inputs.* }}` inside `run:` is substituted before the shell sees
-        # it, so a crafted value would execute as shell.
+        timeout-minutes: ${{ fromJSON(inputs.smoke_timeout) }}
+        # Inputs are passed through env, never interpolated into the script.
         env:
-          # The pluto image is built from this checkout during the define step.
           PLUTO_REPO: ${{ github.workspace }}
           SCENARIOS: ${{ inputs.scenarios }}
-          GO_TIMEOUT: ${{ inputs.go_timeout }}
-          LOG_DIR: ${{ runner.temp }}/smoke-logs
+          SMOKE_LOG_DIR: ${{ runner.temp }}/smoke-logs
+          # Containers run as root; without this the runner cannot clean up.
+          SMOKE_SUDO_PERMS: "1"
         run: |
-          mkdir -p "$LOG_DIR"
-
-          args=(
-            ./smoke -v -integration
-            "-timeout=$GO_TIMEOUT"
-            "-log-dir=$LOG_DIR"
-            # Requires more CPU than a GitHub-hosted runner provides reliably.
-            "-skip=^TestSmoke/very_large$"
-            # Containers run as root, so the artefacts they leave in the compose
-            # dir are root-owned; without this the runner cannot clean them up.
-            -sudo-perms
-          )
+          mkdir -p "$SMOKE_LOG_DIR"
+
+          # very_large needs more CPU than a hosted runner has.
+          args=(--nocapture --test-threads=1 --skip 'scenario::very_large')
           if [ -n "$SCENARIOS" ]; then
-            args+=(-run "$SCENARIOS")
+            # Exact names: `dkg` alone would also select pluto_dkg.
+            args+=(--exact)
+            for name in $SCENARIOS; do
+              args+=("scenario::$name")
+            done
+
+            # libtest passes with 0 tests for an unknown name; every name must select one.
+            wanted=$(echo "$SCENARIOS" | wc -w | tr -d ' ')
+            found=$(cargo test --locked -p pluto-test-compose --test smoke --features smoke -- "${args[@]}" --list | grep -c ': test$' || true)
+            if [ "$found" -ne "$wanted" ]; then
+              echo "::error::$found of $wanted scenario names select a test (very_large is always excluded): $SCENARIOS"
+              exit 1
+            fi
           fi
 
-          go test "${args[@]}"
+          cargo test --locked -p pluto-test-compose --test smoke --features smoke -- "${args[@]}"
 
       - name: Upload scenario logs
-        # Always: a passing run's logs are the baseline for triaging the next
-        # failure, and these clusters are expensive to reproduce.
         if: always()
         uses: actions/upload-artifact@v4
         with:

.github/workflows/test.yml

@@ -66,8 +66,11 @@ jobs:
           packages: protobuf-compiler=3.21.12*
           version: 3.21.12
 
-      - name: Build (all features)
-        run: cargo test --locked --workspace --all-features --no-run
+      # Enables eth2api's docker-based integration tests; NOT pluto-test-compose's
+      # `smoke` feature, whose tests are docker-compose clusters run manually via
+      # smoke-tests.yml.
+      - name: Build (integration features)
+        run: cargo test --locked --workspace --features pluto-eth2api/integration --no-run
 
       # Slash-less `charon/priority/2.0.0` Charon interop relies on the vendored
       # multistream-select fork ([patch.crates-io] -> third_party/). If a libp2p
@@ -83,5 +86,5 @@ jobs:
           fi
           echo "multistream-select fork patch applied ✓"
 
-      - name: Run tests (all features)
-        run: cargo test --locked --workspace --all-features
+      - name: Run tests (integration features)
+        run: cargo test --locked --workspace --features pluto-eth2api/integration

.gitignore

@@ -34,5 +34,5 @@ test-infra/sszfixtures/sszfixtures
 .claude/scheduled_tasks.lock
 test-cluster
 
-# Smoke-test docker-compose logs (go test -log-dir)
-test-infra/compose/**/*.log
+# Compose smoke-test logs (SMOKE_LOG_DIR lands in the test cwd, crates/test-compose/)
+*.log

AGENTS.md

@@ -27,6 +27,7 @@ pluto/
     p2p/                   # P2P networking (libp2p)
     peerinfo/              # Peer info utilities
     relay-server/          # Relay server implementation
+    test-compose/          # Docker-compose smoke-test harness (test infrastructure, not shipped)
     testutil/              # Test helpers/fixtures (workspace-internal)
     tracing/               # Observability/tracing utilities
   test-infra/              # Docker-compose and local infra for integration testing/observability
@@ -56,6 +57,7 @@ Commands (run from `pluto/`):
 ```bash
 cargo +nightly fmt --all --check
 cargo clippy --workspace --all-targets --all-features -- -D warnings
-cargo test --workspace --all-features
+cargo test --workspace --features pluto-eth2api/integration
+# Smoke tests are opt-in (--features smoke); see crates/test-compose/README.md.
 cargo deny check --hide-inclusion-graph
 ```

CONTRIBUTING.md

@@ -39,9 +39,11 @@ cargo build --workspace --all-features
 To run all tests - unit and integration - run:
 
 ```sh
-cargo test --workspace --all-features
+cargo test --workspace --features pluto-eth2api/integration
 ```
 
+Smoke tests are opt-in (`--features smoke`); see `crates/test-compose/README.md`.
+
 ## Running the Rust Documentation Locally
 To build the documentation locally:
 

Cargo.toml

@@ -24,6 +24,7 @@ members = [
     "crates/frost",
     "crates/priority",
     "crates/infosync",
+    "crates/test-compose",
 ]
 # Vendored fork consumed only via [patch.crates-io]; excluded so it builds/tests
 # standalone (its upstream code isn't written to this workspace's lints) without
@@ -59,6 +60,7 @@ futures-timer = "3.0"
 backon = "1.6.0"
 hex = { version = "0.4.3" }
 hex-literal = "0.4"
+nix = { version = "0.30", features = ["user"] }
 prost = "0.14"
 prost-build = "0.14"
 prost-types = "0.14"
@@ -171,6 +173,7 @@ pluto-peerinfo = { path = "crates/peerinfo" }
 pluto-frost = { path = "crates/frost" }
 pluto-priority = { path = "crates/priority" }
 pluto-infosync = { path = "crates/infosync" }
+pluto-test-compose = { path = "crates/test-compose" }
 
 [workspace.lints.rust]
 missing_docs = "deny"

crates/cli/src/commands/create_cluster.rs

@@ -2862,7 +2862,7 @@ mod tests {
     /// `CHARON_*` env var. Charon binds env for all commands generically
     /// (viper `SetEnvPrefix`+`AutomaticEnv`), so tooling that configures a
     /// cluster purely through the environment — the compose harness in
-    /// `test-infra/compose` — works against charon and pluto alike.
+    /// `crates/test-compose` — works against charon and pluto alike.
     #[test]
     fn create_cluster_flags_use_charon_env_prefix() {
         use clap::CommandFactory as _;

crates/test-compose/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+name = "pluto-test-compose"
+description = "Docker-compose smoke-test harness for pluto and charon clusters. Test infrastructure, not shipped."
+version.workspace = true
+edition.workspace = true
+repository.workspace = true
+license.workspace = true
+publish.workspace = true
+
+[features]
+smoke = []
+
+[[test]]
+name = "smoke"
+path = "tests/smoke.rs"
+required-features = ["smoke"]
+
+[dependencies]
+k256.workspace = true
+nix.workspace = true
+pluto-eth2util.workspace = true
+pluto-k1util.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+tokio-util.workspace = true
+tracing.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true
+test-case.workspace = true
+tokio = { workspace = true, features = ["test-util"] }
+tracing-subscriber.workspace = true
+
+[lints]
+workspace = true

crates/test-compose/README.md

@@ -0,0 +1,81 @@
+# Pluto Compose
+
+Docker-compose smoke-test harness for pluto and charon clusters, adapted from
+charon's `testutil/compose`. Test infrastructure: nothing here ships in the
+`pluto` binary.
+
+A cluster is produced in steps (`define` → `lock` → `run`), each
+rewriting `docker-compose.yml` from `config.json`. `auto` chains the steps
+against a docker daemon, brings the cluster up and watches Prometheus for
+alerts. Nodes are charon or pluto per `node_impls`; key generation follows
+`key_gen_impl`.
+
+## Smoke tests
+
+`tests/smoke.rs` holds one test per scenario, named `scenario::<name>`, gated
+behind the `smoke` feature so a plain `cargo test --workspace` never builds or
+runs them. Each stands up a cluster for two minutes and fails on any firing
+alert. Prerequisites: docker with compose v2, and `oas3-gen` from
+`CONTRIBUTING.md` (the harness links `pluto-eth2util`, whose API types are
+generated at build time).
+
+```bash
+# one or more scenarios
+PLUTO_REPO=$PWD cargo test -p pluto-test-compose --test smoke --features smoke -- --nocapture --exact scenario::default_alpha scenario::pluto_dkg
+# the CI matrix (very_large needs a big machine)
+cargo test -p pluto-test-compose --test smoke --features smoke -- --nocapture --test-threads=1 --skip scenario::very_large
+# keep per-scenario logs
+SMOKE_LOG_DIR=. cargo test -p pluto-test-compose --test smoke --features smoke -- --nocapture --exact scenario::default_alpha
+```
+
+Scenarios run one at a time whatever `--test-threads` says: a mutex in
+`tests/smoke.rs` serializes them, since clusters competing for CPU and memory
+produce duty timeouts a sequential run never sees. `--test-threads=1` only
+keeps the output orderly.
+
+
+| Variable               | Effect                                                                                                  |
+| ---------------------- | ------------------------------------------------------------------------------------------------------- |
+| `PLUTO_REPO`           | Repository root the `pluto:local` image is built from. Scenarios that run pluto are skipped when unset. |
+| `SMOKE_SUDO_PERMS`     | Set to `1` when containers run as root, so the harness can `sudo chown` its artefacts.                  |
+| `SMOKE_LOG_DIR`        | Write `<dir>/<scenario>.log` with the `docker compose up` output.                                       |
+| `SMOKE_EXTERNAL_RELAY` | Use this relay URL instead of the in-cluster relay.                                                     |
+
+
+The CI workflow (`.github/workflows/smoke-tests.yml`) is manual-only and runs
+the same command.
+
+## Alert criteria vs. charon
+
+Adapted from charon's `testutil/compose` alert rules, but the gate is corrected and the
+criteria calibrated to actually fire: charon's collector matches Prometheus alert state
+`"active"`, which is never emitted (only `inactive` / `pending` / `firing`), so upstream
+nothing is ever gated. This harness matches `"firing"`, so several rules necessarily differ:
+
+
+| Rule                       | Charon v1.7.1                                                 | Pluto                                                                                          | Change & why                                                                                                                                           |
+| -------------------------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `Pluto Down`               | `up == 0`                                                     | `up == 0`                                                                                      | identical                                                                                                                                              |
+| `Validator API Error Rate` | `increase(…{endpoint!="proxy"}[30s]) > 1`                     | same                                                                                           | identical                                                                                                                                              |
+| `Proxy API Error Rate`     | `increase(…{endpoint="proxy"}[30s]) > 5`                      | same                                                                                           | identical                                                                                                                                              |
+| `Warn Log Rate`            | `increase(app_log_warn_total[30s]) > 2`                       | same + `{topic!~"vmock|tracker"}`                                                              | exclude charon mock-noise topics (vmock has no builder-registration handler; the beacon mock never includes broadcasts on-chain)                       |
+| `Error Log Rate`           | `app_log_error_total > 0`                                     | `increase(app_log_error_total[30s]) > 0`                                                       | windowed — an absolute counter can't recover from the inherent cold-start consensus timeout (mock-VC startup delay → no randao); a window + warmup can |
+| `Broadcast Duty Rate`      | `increase(core_bcast_broadcast_total[30s]) < 0.5`             | `(sum by (job) (increase(…{job=~"node[0-9]+"}[30s])) or on (job) max by (job) (0 * up)) < 0.5` | per-node sum + absent-series fallback, so a node emitting *no* broadcast series fails (charon's per-series form missed it)                             |
+| `Outstanding Duty Rate`    | `core_bcast_broadcast_total − core_scheduler_duty_total > 50` | *removed*                                                                                      | dead rule — a duty is broadcast at most as often as scheduled, so it can never be positive                                                             |
+| *gate (alert state)*       | `"active"` — never emitted                                    | `"firing"` + readiness wait + 60s warmup allowlist                                             | charon's gate is vacuous; pluto's enforces                                                                                                             |
+
+
+Scenarios that intentionally degrade the cluster tune the gate via config, not the code:
+
+
+| Config knob           | Effect                                                          | Used by                                                                                                                           |
+| --------------------- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
+| `alert_exclude_jobs`  | exempt a node from the per-node rules (never from `Pluto Down`) | `node_1_of_4_down`, `node_1_of_3_down`                                                                                                      |
+| `alert_disable_rules` | drop an entire rule                                             | `node_1_of_3_down` (disables the error-rate gates — a downed round-1 leader makes every third proposer duty unrecoverable on the mock) |
+
+
+
+
+## Versioning
+
+The charon image tag is `CHARON_IMAGE_TAG` in `src/smoke.rs`.