edwingao28 · Oct 2, 2026 · Oct 2, 2026 · Oct 2, 2026 · Oct 5, 2026 · Oct 14, 2026
@@ -395,7 +395,7 @@ jobs:
         secrets:
             INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }}
         with:
-            require-power: ${{ inputs.require-power }}
+            require-power: ${{ inputs.require-power || (matrix.config['require-power'] == true && matrix.config['eval-only'] != true) }}
             config: ${{ toJSON(matrix.config) }}
             klaud-run: ${{ inputs.klaud-run }}
             runner: ${{ matrix.config.runner }}
@@ -459,7 +459,7 @@ jobs:
             duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }}
             agentx-fast: ${{ inputs.agentx-fast }}
             run-eval: false
-            require-power: ${{ inputs.require-power }}
+            require-power: ${{ inputs.require-power || (matrix.config['require-power'] == true && matrix.config['eval-only'] != true) }}
             power-producer-sha: ${{ inputs.power-producer-sha }}
             scenario-type: agentic-coding
             ref: ${{ needs.get-jobs.outputs.benchmark-ref }}

@@ -392,6 +392,7 @@ jobs:
             dp-attn: ${{ matrix.config.dp-attn }}
             run-eval: false
             scenario-type: ${{ matrix.config['scenario-type'] || 'fixed-seq-len' }}
+            require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }}
             duration: ${{ matrix.config.duration || '3600' }}
             agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }}
 
@@ -419,6 +420,7 @@ jobs:
             duration: ${{ matrix.config.duration }}
             run-eval: false
             scenario-type: agentic-coding
+            require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }}
             agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }}
 
     sweep-multi-node-1k1k:
@@ -548,6 +550,7 @@ jobs:
             duration: ${{ matrix.config.duration }}
             run-eval: false
             scenario-type: agentic-coding
+            require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }}
             agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }}
 
     sweep-multi-node-agentic:
@@ -582,6 +585,7 @@ jobs:
             duration: ${{ matrix.config.duration }}
             run-eval: false
             scenario-type: agentic-coding
+            require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }}
             agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }}
 
     sweep-evals:

@@ -53,7 +53,7 @@ All referenced recipes must be checked in: srt-slurm 2 ships curated examples in
 
 Active recipes, fixed-sequence and AgentX, single- and multi-node, are fragments: native srt-slurm YAML holding only recipe-specific settings. At launch the fragment is composed and bound:
 
-1. The lane's shared block in [`configs/srt-recipes/`](../../../configs/srt-recipes) (`fixed-sequence-{single,multi}.yaml` or `agentic.yaml`) is merged under the fragment (under `base` for bundles). A multi-node row with `power: true` also gets [`telemetry-dcgm.yaml`](../../../configs/srt-recipes/telemetry-dcgm.yaml), its exporter on the cluster's `srt-slurm.power-exporter-port`; the lane must allow power (`POWER_LANES`). The fragment wins: mappings merge and lists replace, so a fragment may keep telemetry tunables such as `collector_join_timeout_seconds`. Native single-node benchmarks also get [`telemetry-native.yaml`](../../../configs/srt-recipes/telemetry-native.yaml) before variant selection. That block enables best-effort power collection with the cluster's `default_gpu_exporter`; eval-only points omit collection. The binder sets `telemetry.required` from `REQUIRE_POWER`, so a single-node fragment may not set it, and gives AgentX clients matching power switches.
+1. The lane's shared block in [`configs/srt-recipes/`](../../../configs/srt-recipes) (`fixed-sequence-{single,multi}.yaml` or `agentic.yaml`) is merged under the fragment (under `base` for bundles). A multi-node row with `power: true`, or an ordinary GB200/GB300 NVL72 point, also gets [`telemetry-dcgm.yaml`](../../../configs/srt-recipes/telemetry-dcgm.yaml) with its exporter on the cluster's `srt-slurm.power-exporter-port`. NVL72 points also bind the cluster's Grace CPU exporter. Other multi-node lanes still require an allowed `power: true` row (`POWER_LANES`). The fragment wins: mappings merge and lists replace, so a fragment may keep telemetry tunables such as `collector_join_timeout_seconds`. Native single-node benchmarks also get [`telemetry-native.yaml`](../../../configs/srt-recipes/telemetry-native.yaml) before variant selection. That block enables best-effort power collection with the cluster's `default_gpu_exporter`; eval-only points omit collection. The binder sets `telemetry.required` from `REQUIRE_POWER`, so a single-node fragment may not set it, and gives AgentX clients matching power switches.
 2. The row's selector picks the variant. A single-node variant may name its `benchmark.env.CONC`, which keeps that concurrency paired with its tuning.
 3. The binder ([`workload.py`](../../../infx/srt_slurm/workload.py)) writes the matrix point into the selected recipe: `model.path: hf:<model>`, `model.container: <image>`, `model.precision`, `identity.container.image` (as a registry reference) and `identity.model.repo` when the fragment declares `identity.container`/`identity.model`, and `benchmark.concurrencies` when telemetry is enabled. Fixed-sequence recipes also get `benchmark.env.ISL`/`OSL`; single-node recipes get `MODEL` and `CONC`, and fixed-sequence ones `RANDOM_RANGE_RATIO` and `USE_CHAT_TEMPLATE` (`true` exactly when the recipe speculates). Multi-node AgentX recipes get the client's `RESULT_DIR`, `AIPERF_DATASET_MMAP_CACHE_DIR` and `HF_HUB_CACHE` from the launcher, which derives them from the volumes it mounts (`volume-mounts`, `agentic-volume-mounts`, lane mounts); a fragment that sets `HF_HOME` keeps its own cache. They also get `IS_MULTINODE: "true"` and `ENABLE_AGENTX_POWER`, `"1"` exactly when the launcher collects the point's AgentX power (a `power: true` row whose lane rule collects AgentX or adapter power) and `"0"` otherwise, and `REQUIRE_POWER`, `"1"` when the workflow requires power or the lane rule marks the point `require` (GB200 and GB300 Kimi-K3 AgentX), so the client fails a point whose power is invalid. The multi-node client reads `CONC_LIST`, `CONC`, `MODEL` and the other matrix inputs, prefill and decode topology included, from the job environment, so fragments do not copy them (`TP`, `PP_SIZE`, `PCP_SIZE` and `EP_SIZE` describe single-node points only); single-node AgentX points get them as runtime arguments.
 4. A component a repository setup script installs gets the master config's version. When `setup_script`, or a service `preamble` that runs `/configs/<script>`, is one of the installers in the binder's `INSTALLERS`, the binder writes the point's `router` or `kv-offload-backend` version as `ROUTER_VERSION` or `KV_OFFLOAD_BACKEND_VERSION` into the top-level `environment` (workers and static routers) or into that service's `env` (services do not inherit `environment`). A point that does not declare the version fails, and a `vllm-router==` pin in `SETUP_PIP_PACKAGES` must equal the master router version.
@@ -101,7 +101,7 @@ The initial migration also resolves compatibility issues that `srtctl migrate` c
 
 - SGLang Model Gateway recipes use `frontend.type: sglang-router`; in v2.36.0, `sglang` selects a direct worker without a router.
 - Duplicate YAML keys retain the value selected by the former PyYAML loader.
-- DCGM telemetry uses `collect_interval_ms: 1000` instead of `provider` and `default_frequency`. The collector derives its shutdown budget; an explicit ten-second budget is too short for the current validator. Dedicated discovery-service placement is preserved from the original recipes. The pinned upstream runtime rejects telemetry with dedicated infrastructure nodes; this remains a power compatibility blocker rather than changing the original topology to satisfy validation.
+- DCGM telemetry uses `collect_interval_ms: 1000` instead of `provider` and `default_frequency`. The collector derives its shutdown budget; an explicit ten-second budget is too short for the current validator. GB200/GB300 NVL72 multi-node recipes place the client, frontend, and discovery services on the head node so the pinned runtime can collect power. This changes host contention and needs fresh performance results.
 - DeepSeek-V4 vLLM benchmarks use the supported `custom_tokenizer` loader. Retired `warmup_req_rate: inf` fields are removed; the current upstream client uses its fixed warmup rate of 250 requests per second.
 - The power reader accepts both generations of samples CSV while validating utilization values and continuing to compute board energy from watts.
 - Post-eval selection uses native `post_eval.command` and `post_eval.passthrough_env` with [`srt_eval.sh`](../srt_eval.sh). TRT AgentX recipes declare their existing Dynamo fork with `dynamo.source.git`; launchers no longer rewrite the srt-slurm source.

@@ -72,7 +72,13 @@ base:
         stream_interval: 100
         num_postprocess_workers: 4
 
+  benchmark:
+    placement:
+      node: head
+
   frontend:
+    placement:
+      node: head
     type: "dynamo"
     nginx_container: "nginx-sqsh"
 
@@ -86,11 +92,11 @@ base:
     - name: etcd
       type: etcd
       placement:
-        node: dedicated
+        node: head
     - name: nats
       type: nats
       placement:
-        node: dedicated
+        node: head
 
 override_1p4d_tep8_b8_eplb0_mtp3:
   roles:

@@ -67,7 +67,13 @@ base:
         print_iter_log: true
         stream_interval: 100
 
+  benchmark:
+    placement:
+      node: head
+
   frontend:
+    placement:
+      node: head
     type: "dynamo"
     nginx_container: "nginx-sqsh"
 
@@ -85,11 +91,11 @@ base:
     - name: etcd
       type: etcd
       placement:
-        node: dedicated
+        node: head
     - name: nats
       type: nats
       placement:
-        node: dedicated
+        node: head
 
 override_3p1d_dep8_b64_mtp3_c666:
   roles:

@@ -74,7 +74,13 @@ base:
         stream_interval: 100
         num_postprocess_workers: 4
 
+  benchmark:
+    placement:
+      node: head
+
   frontend:
+    placement:
+      node: head
     type: "dynamo"
     enable_multiple_frontends: false
 
@@ -88,11 +94,11 @@ base:
     - name: etcd
       type: etcd
       placement:
-        node: dedicated
+        node: head
     - name: nats
       type: nats
       placement:
-        node: dedicated
+        node: head
 
 override_1p3d_tep8_b8_eplb0_mtp3:
   roles:

@@ -67,7 +67,13 @@ base:
         print_iter_log: true
         stream_interval: 100
 
+  benchmark:
+    placement:
+      node: head
+
   frontend:
+    placement:
+      node: head
     type: "dynamo"
     enable_multiple_frontends: false
 
@@ -85,11 +91,11 @@ base:
     - name: etcd
       type: etcd
       placement:
-        node: dedicated
+        node: head
     - name: nats
       type: nats
       placement:
-        node: dedicated
+        node: head
 
 override_1p4d_tep8_b1_mtp3_c8:
   roles:

@@ -27,11 +27,11 @@ base:
     - name: etcd
       type: etcd
       placement:
-        node: dedicated
+        node: head
     - name: nats
       type: nats
       placement:
-        node: dedicated
+        node: head
       options:
         max_payload_mb: 32
     - name: mooncake-master
@@ -83,7 +83,12 @@ base:
       readiness:
         port: 8803
         timeout_seconds: 600
+  benchmark:
+    placement:
+      node: head
   frontend:
+    placement:
+      node: head
     type: dynamo
     nginx_session_affinity: true
     nginx_session_affinity_header: X-Dynamo-Session-ID
@@ -261,10 +266,8 @@ base:
     mem: "0"
     container-remap-root: ""
   benchmark:
-    # The replay client holds ~300 GB of host memory; on the default head node it
-    # shares prefill_0's node and OOMs it. Put it on the etcd/nats node instead.
     placement:
-      node: dedicated
+      node: head
     env:
       AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
       AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"

@@ -102,9 +102,6 @@ srun_options:
   # The custom benchmark installs its isolated AIPerf environment at runtime.
   container-remap-root: ""
 
-telemetry:
-  required: false
-
 benchmark:
   env:
     AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"

@@ -144,7 +144,7 @@ base:
       active-prefill-tokens-threshold: None
       active-prefill-tokens-threshold-frac: None
     placement:
-      node: first_decode
+      node: head
   benchmark:
     env:
       AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: '14400'
@@ -153,7 +153,7 @@ base:
       AIPERF_REQUIRED_SERVER_METRIC_PREFIX: trtllm_kv_cache_utilization
       SERVED_MODEL_NAME: DeepSeek-V4-Pro
     placement:
-      node: last_decode
+      node: head
 
 override_1p4d_dep4_tep8_c4_b1:
   name: dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp

@@ -19,14 +19,19 @@ base:
     - name: etcd
       type: etcd
       placement:
-        node: dedicated
+        node: head
     - name: nats
       type: nats
       placement:
-        node: dedicated
+        node: head
       options:
         max_payload_mb: 32
+  benchmark:
+    placement:
+      node: head
   frontend:
+    placement:
+      node: head
     type: dynamo
     enable_multiple_frontends: false
   engine:

@@ -193,7 +193,7 @@ base:
       AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true'
       AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0'
     placement:
-      node: dedicated
+      node: head
 
 override_1p1d_tp8_c1_b1_mtp5:
   name: dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5

@@ -18,14 +18,19 @@ base:
     - name: etcd
       type: etcd
       placement:
-        node: dedicated
+        node: head
     - name: nats
       type: nats
       placement:
-        node: dedicated
+        node: head
       options:
         max_payload_mb: 8
+  benchmark:
+    placement:
+      node: head
   frontend:
+    placement:
+      node: head
     type: dynamo
     nginx_session_affinity: true
     enable_multiple_frontends: true

@@ -18,14 +18,19 @@ base:
     - name: etcd
       type: etcd
       placement:
-        node: dedicated
+        node: head
     - name: nats
       type: nats
       placement:
-        node: dedicated
+        node: head
       options:
         max_payload_mb: 8
+  benchmark:
+    placement:
+      node: head
   frontend:
+    placement:
+      node: head
     type: dynamo
     nginx_session_affinity: true
     enable_multiple_frontends: true

@@ -98,7 +98,7 @@ base:
     env:
       AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'
       # Native single-node binding verifies these against the generated point.
-      TOTAL_CPU_DRAM_GB: '225'
+      TOTAL_CPU_DRAM_GB: '241'
       KV_OFFLOADING: dram
 
 # Only client concurrency changes; serving/admission/graph/cache settings stay fixed.