diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 556d856b39..6d4daf9b05 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -395,7 +395,7 @@ jobs: secrets: INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} with: - require-power: ${{ inputs.require-power }} + require-power: ${{ inputs.require-power || (matrix.config['require-power'] == true && matrix.config['eval-only'] != true) }} config: ${{ toJSON(matrix.config) }} klaud-run: ${{ inputs.klaud-run }} runner: ${{ matrix.config.runner }} @@ -459,7 +459,7 @@ jobs: duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} agentx-fast: ${{ inputs.agentx-fast }} run-eval: false - require-power: ${{ inputs.require-power }} + require-power: ${{ inputs.require-power || (matrix.config['require-power'] == true && matrix.config['eval-only'] != true) }} power-producer-sha: ${{ inputs.power-producer-sha }} scenario-type: agentic-coding ref: ${{ needs.get-jobs.outputs.benchmark-ref }} diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 469a141a1b..36e797951a 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -392,6 +392,7 @@ jobs: dp-attn: ${{ matrix.config.dp-attn }} run-eval: false scenario-type: ${{ matrix.config['scenario-type'] || 'fixed-seq-len' }} + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} duration: ${{ matrix.config.duration || '3600' }} agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }} @@ -419,6 +420,7 @@ jobs: duration: ${{ matrix.config.duration }} run-eval: false scenario-type: agentic-coding + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }} sweep-multi-node-1k1k: @@ -548,6 +550,7 @@ jobs: duration: ${{ matrix.config.duration }} run-eval: false scenario-type: agentic-coding + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }} sweep-multi-node-agentic: @@ -582,6 +585,7 @@ jobs: duration: ${{ matrix.config.duration }} run-eval: false scenario-type: agentic-coding + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }} sweep-evals: diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index 047db43d58..265ef2747b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -53,7 +53,7 @@ All referenced recipes must be checked in: srt-slurm 2 ships curated examples in Active recipes, fixed-sequence and AgentX, single- and multi-node, are fragments: native srt-slurm YAML holding only recipe-specific settings. At launch the fragment is composed and bound: -1. The lane's shared block in [`configs/srt-recipes/`](../../../configs/srt-recipes) (`fixed-sequence-{single,multi}.yaml` or `agentic.yaml`) is merged under the fragment (under `base` for bundles). A multi-node row with `power: true` also gets [`telemetry-dcgm.yaml`](../../../configs/srt-recipes/telemetry-dcgm.yaml), its exporter on the cluster's `srt-slurm.power-exporter-port`; the lane must allow power (`POWER_LANES`). The fragment wins: mappings merge and lists replace, so a fragment may keep telemetry tunables such as `collector_join_timeout_seconds`. Native single-node benchmarks also get [`telemetry-native.yaml`](../../../configs/srt-recipes/telemetry-native.yaml) before variant selection. That block enables best-effort power collection with the cluster's `default_gpu_exporter`; eval-only points omit collection. The binder sets `telemetry.required` from `REQUIRE_POWER`, so a single-node fragment may not set it, and gives AgentX clients matching power switches. +1. The lane's shared block in [`configs/srt-recipes/`](../../../configs/srt-recipes) (`fixed-sequence-{single,multi}.yaml` or `agentic.yaml`) is merged under the fragment (under `base` for bundles). A multi-node row with `power: true`, or an ordinary GB200/GB300 NVL72 point, also gets [`telemetry-dcgm.yaml`](../../../configs/srt-recipes/telemetry-dcgm.yaml) with its exporter on the cluster's `srt-slurm.power-exporter-port`. NVL72 points also bind the cluster's Grace CPU exporter. Other multi-node lanes still require an allowed `power: true` row (`POWER_LANES`). The fragment wins: mappings merge and lists replace, so a fragment may keep telemetry tunables such as `collector_join_timeout_seconds`. Native single-node benchmarks also get [`telemetry-native.yaml`](../../../configs/srt-recipes/telemetry-native.yaml) before variant selection. That block enables best-effort power collection with the cluster's `default_gpu_exporter`; eval-only points omit collection. The binder sets `telemetry.required` from `REQUIRE_POWER`, so a single-node fragment may not set it, and gives AgentX clients matching power switches. 2. The row's selector picks the variant. A single-node variant may name its `benchmark.env.CONC`, which keeps that concurrency paired with its tuning. 3. The binder ([`workload.py`](../../../infx/srt_slurm/workload.py)) writes the matrix point into the selected recipe: `model.path: hf:`, `model.container: `, `model.precision`, `identity.container.image` (as a registry reference) and `identity.model.repo` when the fragment declares `identity.container`/`identity.model`, and `benchmark.concurrencies` when telemetry is enabled. Fixed-sequence recipes also get `benchmark.env.ISL`/`OSL`; single-node recipes get `MODEL` and `CONC`, and fixed-sequence ones `RANDOM_RANGE_RATIO` and `USE_CHAT_TEMPLATE` (`true` exactly when the recipe speculates). Multi-node AgentX recipes get the client's `RESULT_DIR`, `AIPERF_DATASET_MMAP_CACHE_DIR` and `HF_HUB_CACHE` from the launcher, which derives them from the volumes it mounts (`volume-mounts`, `agentic-volume-mounts`, lane mounts); a fragment that sets `HF_HOME` keeps its own cache. They also get `IS_MULTINODE: "true"` and `ENABLE_AGENTX_POWER`, `"1"` exactly when the launcher collects the point's AgentX power (a `power: true` row whose lane rule collects AgentX or adapter power) and `"0"` otherwise, and `REQUIRE_POWER`, `"1"` when the workflow requires power or the lane rule marks the point `require` (GB200 and GB300 Kimi-K3 AgentX), so the client fails a point whose power is invalid. The multi-node client reads `CONC_LIST`, `CONC`, `MODEL` and the other matrix inputs, prefill and decode topology included, from the job environment, so fragments do not copy them (`TP`, `PP_SIZE`, `PCP_SIZE` and `EP_SIZE` describe single-node points only); single-node AgentX points get them as runtime arguments. 4. A component a repository setup script installs gets the master config's version. When `setup_script`, or a service `preamble` that runs `/configs/