From 03eaa88a48b204e78b84f4cb62c3e62d818f4671 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Fri, 2 Oct 2026 01:29:22 -0700 Subject: [PATCH 1/5] feat: enable NVL72 power collection by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：为 NVL72 基准测试默认启用 GPU 与 CPU 侧功耗采集。 --- .github/workflows/benchmark-tmpl.yml | 2 + inferencex-e2e/benchmarks/benchmark_lib.sh | 10 +- .../disagg-10p1d-dep16-b256-eplb256-mtp0.yaml | 5 +- .../disagg-11p1d-dep16-b256-eplb256-mtp1.yaml | 5 +- .../8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml | 5 +- .../8k1k/disagg-1p4d-tep8-b16-eplb0-mtp0.yaml | 5 +- .../8k1k/disagg-1p4d-tep8-b8-eplb0-mtp3.yaml | 5 +- .../8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0.yaml | 5 +- .../8k1k/disagg-3p1d-dep32-b4-eplb0-mtp3.yaml | 5 +- .../disagg-7p1d-dep16-b64-eplb256-mtp1.yaml | 5 +- .../disagg-7p1d-dep32-b32-eplb0-mtp0.yaml | 5 +- .../disagg-8p1d-dep16-b128-eplb0-mtp0.yaml | 5 +- .../disagg-8p1d-dep32-b16-eplb0-mtp3.yaml | 5 +- .../disagg-1p3d-tep8-b1-eplb0-mtp0-c6.yaml | 5 +- .../disagg-1p3d-tep8-b16-eplb0-mtp0-c63.yaml | 5 +- .../disagg-1p3d-tep8-b2-eplb0-mtp3-c6.yaml | 5 +- .../disagg-1p3d-tep8-b4-eplb0-mtp0-c18.yaml | 5 +- .../disagg-1p3d-tep8-b4-eplb0-mtp3-c15.yaml | 5 +- .../disagg-2p1d-dep32-b2-eplb0-mtp3-c90.yaml | 5 +- .../disagg-2p1d-dep32-b8-eplb0-mtp0-c333.yaml | 5 +- ...disagg-3p1d-dep16-b16-eplb0-mtp3-c333.yaml | 5 +- ...disagg-3p1d-dep16-b32-eplb0-mtp0-c615.yaml | 5 +- .../disagg-3p1d-dep8-b64-eplb0-mtp3-c666.yaml | 5 +- ...disagg-4p1d-dep32-b16-eplb0-mtp0-c666.yaml | 5 +- .../disagg-4p1d-dep32-b8-eplb0-mtp3-c333.yaml | 5 +- ...disagg-5p1d-dep16-b32-eplb0-mtp3-c666.yaml | 5 +- ...isagg-5p1d-dep16-b64-eplb0-mtp0-c1229.yaml | 5 +- .../disagg-10p1d-dep16-b32-eplb0-mtp3.yaml | 5 +- .../disagg-10p1d-dep8-b256-eplb0-mtp1.yaml | 5 +- .../disagg-11p3d-dep4-b256-eplb0-mtp0.yaml | 5 +- .../disagg-13p1d-dep16-b64-eplb256-mtp3.yaml | 5 +- .../disagg-14p1d-dep16-b128-eplb0-mtp0.yaml | 5 +- .../8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml | 5 +- .../8k1k/disagg-1p3d-tep8-b8-eplb0-mtp3.yaml | 5 +- .../8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml | 5 +- .../8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml | 5 +- .../8k1k/disagg-1p4d-tep8-b2-eplb0-mtp0.yaml | 5 +- .../8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml | 5 +- .../8k1k/disagg-1p5d-tep4-b4-eplb0-mtp0.yaml | 5 +- .../8k1k/disagg-4p1d-dep32-b4-eplb0-mtp3.yaml | 5 +- .../disagg-7p1d-dep32-b16-eplb0-mtp0.yaml | 5 +- .../8k1k/disagg-8p1d-dep32-b8-eplb0-mtp3.yaml | 5 +- .../disagg-9p1d-dep16-b64-eplb0-mtp0.yaml | 5 +- ...sagg-10p1d-dep16-b64-eplb0-mtp1-c1229.yaml | 5 +- .../disagg-1p4d-tep8-b1-eplb0-mtp0-c4.yaml | 5 +- .../disagg-1p4d-tep8-b1-eplb0-mtp3-c8.yaml | 5 +- .../disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml | 5 +- .../disagg-1p4d-tep8-b4-eplb0-mtp3-c24.yaml | 5 +- .../disagg-1p4d-tep8-b8-eplb0-mtp0-c36.yaml | 5 +- ...disagg-4p1d-dep16-b32-eplb0-mtp0-c666.yaml | 5 +- ...disagg-6p1d-dep32-b16-eplb0-mtp0-c512.yaml | 5 +- .../disagg-6p1d-dep32-b8-eplb0-mtp3-c333.yaml | 5 +- ...isagg-7p1d-dep16-b64-eplb0-mtp0-c1229.yaml | 5 +- ...isagg-7p1d-dep8-b128-eplb0-mtp1-c1229.yaml | 5 +- ...isagg-7p1d-dep8-b256-eplb0-mtp0-c2151.yaml | 5 +- ...disagg-8p1d-dep16-b32-eplb0-mtp3-c666.yaml | 5 +- .../gb300-fp4/agentx/disagg-variants.yaml | 5 +- .../gb300-fp4/agentx/disagg-variants.yaml | 4 +- .../agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml | 7 +- .../gb300-fp4/agentx/disagg-variants.yaml | 5 +- .../gb300-fp4/agentx/disagg-variants.yaml | 4 +- .../agentx/disagg-pp-pareto-variants.yaml | 5 +- .../gb300-fp4/agentx/disagg-variants.yaml | 5 +- .../single_node/srt_fixed_sequence.sh | 10 +- inferencex-e2e/configs/runners.yaml | 9 ++ .../docs/configuration-procedures.md | 4 + .../docs/configuration-procedures_zh.md | 4 + inferencex-e2e/docs/results-and-ingestion.md | 2 + .../docs/results-and-ingestion_zh.md | 2 + inferencex-e2e/infx/clusters/slurm.py | 9 ++ inferencex-e2e/infx/launch/artifacts.py | 4 +- .../infx/launch/drivers/srt/__init__.py | 54 +++++++- .../infx/launch/drivers/srt/collect.py | 51 ++++++- .../infx/launch/drivers/srt/power.py | 89 ++++++++++--- inferencex-e2e/infx/launch/request.py | 2 +- .../infx/results/agentic/power_adapter.py | 18 ++- inferencex-e2e/infx/results/fixed_sequence.py | 15 ++- .../infx/tests/launch/fake_slurm.py | 3 + .../tests/launch/test_launch_artifacts.py | 6 +- .../tests/launch/test_nvl72_power_defaults.py | 126 ++++++++++++++++++ .../infx/tests/launch/test_srt_driver.py | 101 ++++++++++++++ .../results/agentic/test_power_adapter.py | 30 +++++ .../results/agentic/test_power_lifecycle.py | 5 +- .../results/power/test_process_result.py | 29 ++++ .../srt_slurm/test_fixed_sequence_client.py | 59 ++++++++ inferencex-e2e/perf-changelog.yaml | 58 ++++++++ 86 files changed, 828 insertions(+), 179 deletions(-) create mode 100644 inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 500d331658..a9bee7f811 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -432,6 +432,8 @@ jobs: ${{ env.INFERENCEX_E2E_ROOT }}/results/agentic_power_window.json ${{ env.INFERENCEX_E2E_ROOT }}/results/agentic_power_timezone_offset.txt ${{ env.INFERENCEX_E2E_ROOT }}/results/power_validation.json + ${{ env.INFERENCEX_E2E_ROOT }}/LOGS/power + ${{ env.INFERENCEX_E2E_ROOT }}/power-producer-sha.txt if-no-files-found: ignore - name: Upload eval results (if any) diff --git a/inferencex-e2e/benchmarks/benchmark_lib.sh b/inferencex-e2e/benchmarks/benchmark_lib.sh index 98750ea1a0..c68d7af732 100644 --- a/inferencex-e2e/benchmarks/benchmark_lib.sh +++ b/inferencex-e2e/benchmarks/benchmark_lib.sh @@ -3411,12 +3411,10 @@ run_agentic_replay_and_write_outputs() ( case "${ENABLE_AGENTX_POWER}" in 1|true|TRUE|yes|YES) - if [ "${IS_MULTINODE}" = "true" ]; then - if [ -n "${SRT_MEASUREMENT_WINDOW_DIR:-}" ]; then - agentx_multinode_power_enabled=1 - else - agentx_multinode_contract_missing=1 - fi + if [ -n "${SRT_MEASUREMENT_WINDOW_DIR:-}" ]; then + agentx_multinode_power_enabled=1 + elif [ "${IS_MULTINODE}" = "true" ]; then + agentx_multinode_contract_missing=1 else check_env_vars TP PP_SIZE PCP_SIZE agentx_power_enabled=1 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-10p1d-dep16-b256-eplb256-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-10p1d-dep16-b256-eplb256-mtp0.yaml index a45058ab37..7c76a0e5f2 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-10p1d-dep16-b256-eplb256-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-10p1d-dep16-b256-eplb256-mtp0.yaml @@ -141,6 +141,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -154,8 +155,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-11p1d-dep16-b256-eplb256-mtp1.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-11p1d-dep16-b256-eplb256-mtp1.yaml index bf3b5ff060..9e6698bc6c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-11p1d-dep16-b256-eplb256-mtp1.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-11p1d-dep16-b256-eplb256-mtp1.yaml @@ -147,6 +147,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -160,8 +161,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml index edffadc0c5..b50e1e7d95 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml @@ -107,6 +107,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -120,8 +121,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b16-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b16-eplb0-mtp0.yaml index eccb9d7f50..4b045ba9dc 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b16-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b16-eplb0-mtp0.yaml @@ -116,6 +116,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -129,8 +130,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp3.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp3.yaml index 3008438e9d..a5ea04a63b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp3.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp3.yaml @@ -118,6 +118,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -131,8 +132,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0.yaml index 38111f6241..4dd21496c0 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0.yaml @@ -107,6 +107,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -120,8 +121,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-3p1d-dep32-b4-eplb0-mtp3.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-3p1d-dep32-b4-eplb0-mtp3.yaml index 4e8fdd2090..775c30626e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-3p1d-dep32-b4-eplb0-mtp3.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-3p1d-dep32-b4-eplb0-mtp3.yaml @@ -112,6 +112,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -125,8 +126,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep16-b64-eplb256-mtp1.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep16-b64-eplb256-mtp1.yaml index 127ebcdeab..f76ad878ba 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep16-b64-eplb256-mtp1.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep16-b64-eplb256-mtp1.yaml @@ -123,6 +123,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -136,8 +137,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep32-b32-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep32-b32-eplb0-mtp0.yaml index 3d647fa582..97badfd490 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep32-b32-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-7p1d-dep32-b32-eplb0-mtp0.yaml @@ -110,6 +110,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -123,8 +124,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep16-b128-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep16-b128-eplb0-mtp0.yaml index 566f18fa77..b4cc679708 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep16-b128-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep16-b128-eplb0-mtp0.yaml @@ -122,6 +122,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -135,8 +136,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep32-b16-eplb0-mtp3.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep32-b16-eplb0-mtp3.yaml index 5dc7fc1880..972dbbe23d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep32-b16-eplb0-mtp3.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp4/8k1k/disagg-8p1d-dep32-b16-eplb0-mtp3.yaml @@ -114,6 +114,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -127,8 +128,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0-c6.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0-c6.yaml index 278bc1a392..fba525948c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0-c6.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b1-eplb0-mtp0-c6.yaml @@ -105,6 +105,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -122,8 +123,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0-c63.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0-c63.yaml index 45cc85d155..0ea8e656b1 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0-c63.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0-c63.yaml @@ -107,6 +107,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -124,8 +125,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b2-eplb0-mtp3-c6.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b2-eplb0-mtp3-c6.yaml index c9aa401adf..1f0e733af3 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b2-eplb0-mtp3-c6.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b2-eplb0-mtp3-c6.yaml @@ -111,6 +111,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -128,8 +129,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp0-c18.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp0-c18.yaml index e945474a20..33d74e9b01 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp0-c18.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp0-c18.yaml @@ -105,6 +105,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -122,8 +123,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp3-c15.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp3-c15.yaml index efa89624f8..7b0cbbcfaa 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp3-c15.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-1p3d-tep8-b4-eplb0-mtp3-c15.yaml @@ -111,6 +111,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -128,8 +129,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b2-eplb0-mtp3-c90.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b2-eplb0-mtp3-c90.yaml index 540f22ff1e..6539859962 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b2-eplb0-mtp3-c90.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b2-eplb0-mtp3-c90.yaml @@ -110,6 +110,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -127,8 +128,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0-c333.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0-c333.yaml index 3a89196592..82edad44b2 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0-c333.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-2p1d-dep32-b8-eplb0-mtp0-c333.yaml @@ -105,6 +105,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -122,8 +123,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b16-eplb0-mtp3-c333.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b16-eplb0-mtp3-c333.yaml index 5a38c5f195..1e0f9f0db5 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b16-eplb0-mtp3-c333.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b16-eplb0-mtp3-c333.yaml @@ -112,6 +112,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -129,8 +130,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b32-eplb0-mtp0-c615.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b32-eplb0-mtp0-c615.yaml index 4f6e29a060..148ee64399 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b32-eplb0-mtp0-c615.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep16-b32-eplb0-mtp0-c615.yaml @@ -108,6 +108,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -125,8 +126,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3-c666.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3-c666.yaml index 2c6f5bc526..b251d30abd 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3-c666.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-3p1d-dep8-b64-eplb0-mtp3-c666.yaml @@ -118,6 +118,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -135,8 +136,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b16-eplb0-mtp0-c666.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b16-eplb0-mtp0-c666.yaml index 8948a6ebb1..f09d3ff14d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b16-eplb0-mtp0-c666.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b16-eplb0-mtp0-c666.yaml @@ -106,6 +106,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -123,8 +124,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b8-eplb0-mtp3-c333.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b8-eplb0-mtp3-c333.yaml index 41144c6ee4..606f0bc794 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b8-eplb0-mtp3-c333.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-4p1d-dep32-b8-eplb0-mtp3-c333.yaml @@ -111,6 +111,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -128,8 +129,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b32-eplb0-mtp3-c666.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b32-eplb0-mtp3-c666.yaml index 2b40da4227..1d7e9a8dcc 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b32-eplb0-mtp3-c666.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b32-eplb0-mtp3-c666.yaml @@ -114,6 +114,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -131,8 +132,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b64-eplb0-mtp0-c1229.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b64-eplb0-mtp0-c1229.yaml index 12821044c2..1946fa9ec4 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b64-eplb0-mtp0-c1229.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb200-fp8/8k1k/disagg-5p1d-dep16-b64-eplb0-mtp0-c1229.yaml @@ -112,6 +112,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" nginx_container: "nginx-sqsh" @@ -129,8 +130,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep16-b32-eplb0-mtp3.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep16-b32-eplb0-mtp3.yaml index 078b5b7f5a..e42a12155b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep16-b32-eplb0-mtp3.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep16-b32-eplb0-mtp3.yaml @@ -118,6 +118,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -131,8 +132,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml index f84a51fe2b..70af6e14b7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-10p1d-dep8-b256-eplb0-mtp1.yaml @@ -146,6 +146,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -159,8 +160,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-11p3d-dep4-b256-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-11p3d-dep4-b256-eplb0-mtp0.yaml index 8a56d45e8e..465525e622 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-11p3d-dep4-b256-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-11p3d-dep4-b256-eplb0-mtp0.yaml @@ -141,6 +141,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -154,8 +155,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-13p1d-dep16-b64-eplb256-mtp3.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-13p1d-dep16-b64-eplb256-mtp3.yaml index cd1411710f..e965c4009d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-13p1d-dep16-b64-eplb256-mtp3.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-13p1d-dep16-b64-eplb256-mtp3.yaml @@ -126,6 +126,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -139,8 +140,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-14p1d-dep16-b128-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-14p1d-dep16-b128-eplb0-mtp0.yaml index 3b0805b157..3757509611 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-14p1d-dep16-b128-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-14p1d-dep16-b128-eplb0-mtp0.yaml @@ -124,6 +124,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -137,8 +138,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml index a415c4fcd1..928c7e0da2 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b16-eplb0-mtp0.yaml @@ -112,6 +112,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -125,8 +126,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b8-eplb0-mtp3.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b8-eplb0-mtp3.yaml index 94b6e1beac..025c74f87b 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b8-eplb0-mtp3.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p3d-tep8-b8-eplb0-mtp3.yaml @@ -117,6 +117,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -130,8 +131,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml index f4e25793d4..8092913c6c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0.yaml @@ -108,6 +108,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -121,8 +122,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml index 75cfcb536c..4f90525531 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3.yaml @@ -114,6 +114,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -127,8 +128,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp0.yaml index 587730a370..bc0a0df63e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b2-eplb0-mtp0.yaml @@ -109,6 +109,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -122,8 +123,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml index 23a9a2b7d3..780592c8df 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3.yaml @@ -117,6 +117,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -130,8 +131,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p5d-tep4-b4-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p5d-tep4-b4-eplb0-mtp0.yaml index e1d03a7e91..cb8e90e145 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p5d-tep4-b4-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-1p5d-tep4-b4-eplb0-mtp0.yaml @@ -109,6 +109,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -122,8 +123,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep32-b4-eplb0-mtp3.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep32-b4-eplb0-mtp3.yaml index d06871d2d5..06892f93bc 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep32-b4-eplb0-mtp3.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-4p1d-dep32-b4-eplb0-mtp3.yaml @@ -114,6 +114,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -127,8 +128,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep32-b16-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep32-b16-eplb0-mtp0.yaml index 0073549e73..826f574c1d 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep32-b16-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-7p1d-dep32-b16-eplb0-mtp0.yaml @@ -111,6 +111,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -124,8 +125,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep32-b8-eplb0-mtp3.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep32-b8-eplb0-mtp3.yaml index 75ccd2569f..9a6451f3c1 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep32-b8-eplb0-mtp3.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-8p1d-dep32-b8-eplb0-mtp3.yaml @@ -115,6 +115,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -128,8 +129,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep16-b64-eplb0-mtp0.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep16-b64-eplb0-mtp0.yaml index 07471dcd6c..9197efb550 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep16-b64-eplb0-mtp0.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp4/8k1k/disagg-9p1d-dep16-b64-eplb0-mtp0.yaml @@ -117,6 +117,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -130,8 +131,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-10p1d-dep16-b64-eplb0-mtp1-c1229.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-10p1d-dep16-b64-eplb0-mtp1-c1229.yaml index 4b843d19a8..d0c1997dcf 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-10p1d-dep16-b64-eplb0-mtp1-c1229.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-10p1d-dep16-b64-eplb0-mtp1-c1229.yaml @@ -120,6 +120,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -138,8 +139,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c4.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c4.yaml index b3e159aafa..6aacb21161 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c4.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp0-c4.yaml @@ -107,6 +107,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -125,8 +126,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c8.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c8.yaml index c2959e0323..1ac6803386 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c8.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b1-eplb0-mtp3-c8.yaml @@ -113,6 +113,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -131,8 +132,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml index 5dd289e8db..aaadd13bb5 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp0-c24.yaml @@ -107,6 +107,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -125,8 +126,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3-c24.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3-c24.yaml index 8b5e7330c0..e9823384d3 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3-c24.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b4-eplb0-mtp3-c24.yaml @@ -113,6 +113,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -131,8 +132,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp0-c36.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp0-c36.yaml index 1b2a6c604d..2e957409fe 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp0-c36.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-1p4d-tep8-b8-eplb0-mtp0-c36.yaml @@ -108,6 +108,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -126,8 +127,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-4p1d-dep16-b32-eplb0-mtp0-c666.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-4p1d-dep16-b32-eplb0-mtp0-c666.yaml index 7e8d26bf0b..c8c60aab7f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-4p1d-dep16-b32-eplb0-mtp0-c666.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-4p1d-dep16-b32-eplb0-mtp0-c666.yaml @@ -110,6 +110,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -128,8 +129,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b16-eplb0-mtp0-c512.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b16-eplb0-mtp0-c512.yaml index aefa142921..81b6d1c3a4 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b16-eplb0-mtp0-c512.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b16-eplb0-mtp0-c512.yaml @@ -108,6 +108,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -126,8 +127,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b8-eplb0-mtp3-c333.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b8-eplb0-mtp3-c333.yaml index df10364df2..a1258d6e14 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b8-eplb0-mtp3-c333.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-6p1d-dep32-b8-eplb0-mtp3-c333.yaml @@ -113,6 +113,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -131,8 +132,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep16-b64-eplb0-mtp0-c1229.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep16-b64-eplb0-mtp0-c1229.yaml index c7857fcc0d..ac20ce8f56 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep16-b64-eplb0-mtp0-c1229.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep16-b64-eplb0-mtp0-c1229.yaml @@ -114,6 +114,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -132,8 +133,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b128-eplb0-mtp1-c1229.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b128-eplb0-mtp1-c1229.yaml index 037b04b0a8..a8ed456449 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b128-eplb0-mtp1-c1229.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b128-eplb0-mtp1-c1229.yaml @@ -128,6 +128,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -146,8 +147,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b256-eplb0-mtp0-c2151.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b256-eplb0-mtp0-c2151.yaml index 7e82280000..4e33f6786c 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b256-eplb0-mtp0-c2151.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-7p1d-dep8-b256-eplb0-mtp0-c2151.yaml @@ -138,6 +138,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -156,8 +157,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-8p1d-dep16-b32-eplb0-mtp3-c666.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-8p1d-dep16-b32-eplb0-mtp3-c666.yaml index c3ff792986..8a879a59ae 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-8p1d-dep16-b32-eplb0-mtp3-c666.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsr1/trtllm/gb300-fp8/8k1k/disagg-8p1d-dep16-b32-eplb0-mtp3-c666.yaml @@ -116,6 +116,7 @@ benchmark: OSL: '1024' frontend: + dedicated_node: true type: "dynamo" enable_multiple_frontends: false @@ -134,8 +135,8 @@ services: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml index ad2af4be37..7f7834c074 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -30,14 +30,15 @@ base: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head options: max_payload_mb: 32 frontend: + dedicated_node: true type: dynamo nginx_session_affinity: true nginx_session_affinity_header: X-Dynamo-Session-ID diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml index d78570c07f..136dad37df 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -159,7 +159,7 @@ base: active-prefill-tokens-threshold: None active-prefill-tokens-threshold-frac: None placement: - node: first_decode + node: head benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh @@ -183,7 +183,7 @@ base: WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 SERVED_MODEL_NAME: DeepSeek-V4-Pro placement: - node: last_decode + node: head override_1p4d_dep4_tep8_c4_b1: name: dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml index 02c44990af..4d74d6e36f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml @@ -103,7 +103,7 @@ frontend: active-prefill-tokens-threshold-frac: "None" tokenizer: "fastokens" placement: - node: first_decode + node: head engine: type: vllm connector: @@ -188,8 +188,7 @@ srun_options: container-remap-root: "" benchmark: - # The AgentX client uses localhost:8000, so colocate it with the Dynamo - # frontend launched on the first decode node. + # Keep the client, frontend and telemetry collector on the head clock. type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh env: @@ -231,4 +230,4 @@ benchmark: HF_WEKA_DATASET: "semianalysisai/cc-traces-weka-062126" PUBLIC_DATASET: "semianalysis_cc_traces_weka_062126" placement: - node: first_decode + node: head diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml index fc0aae1a06..6df09c5cb7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml @@ -26,14 +26,15 @@ base: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head options: max_payload_mb: 32 frontend: + dedicated_node: true type: dynamo enable_multiple_frontends: false engine: diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml index c8e5cfd844..7016a426ec 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml @@ -172,7 +172,7 @@ base: active-prefill-tokens-threshold: None active-prefill-tokens-threshold-frac: None placement: - node: head + node: dedicated benchmark: type: custom command: bash /infmax-workspace/benchmarks/srt_agentic.sh @@ -193,7 +193,7 @@ base: OPENAI_API_KEY: EMPTY KV_OFFLOADING: dram placement: - node: dedicated + node: head override_1p1d_tp8_c1_b1_mtp5: name: dynamo-disagg-gb300-1p1d-tp8-c1-b1-mtp5 diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml index 3caef43cbc..3ec5ed07d7 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-pp-pareto-variants.yaml @@ -25,14 +25,15 @@ base: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head options: max_payload_mb: 8 frontend: + dedicated_node: true type: dynamo nginx_session_affinity: true enable_multiple_frontends: true diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml index b38e93698f..0347f87c2e 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp4/agentx/disagg-variants.yaml @@ -25,14 +25,15 @@ base: - name: etcd type: etcd placement: - node: dedicated + node: head - name: nats type: nats placement: - node: dedicated + node: head options: max_payload_mb: 8 frontend: + dedicated_node: true type: dynamo nginx_session_affinity: true enable_multiple_frontends: true diff --git a/inferencex-e2e/benchmarks/single_node/srt_fixed_sequence.sh b/inferencex-e2e/benchmarks/single_node/srt_fixed_sequence.sh index 0bc79b9ab4..d782280875 100644 --- a/inferencex-e2e/benchmarks/single_node/srt_fixed_sequence.sh +++ b/inferencex-e2e/benchmarks/single_node/srt_fixed_sequence.sh @@ -46,8 +46,10 @@ source "$(dirname "${BASH_SOURCE[0]}")/../benchmark_lib.sh" cd "$INFERENCEX_REPO_ROOT" pip3 install --break-system-packages sentencepiece datasets pandas -start_gpu_monitor --output "$RESULT_DIR/gpu_metrics.csv" --interval "$SRT_MONITOR_INTERVAL" -trap 'rc=$?; stop_gpu_monitor; exit "$rc"' EXIT +if [[ -z "${SRT_MEASUREMENT_WINDOW_DIR:-}" ]]; then + start_gpu_monitor --output "$RESULT_DIR/gpu_metrics.csv" --interval "$SRT_MONITOR_INTERVAL" + trap 'rc=$?; stop_gpu_monitor; exit "$rc"' EXIT +fi run_benchmark_serving \ --model "$MODEL" \ @@ -62,3 +64,7 @@ run_benchmark_serving \ --result-filename "$RESULT_FILENAME" \ --result-dir "$RESULT_DIR" \ "${CLIENT_ARGS[@]}" + +if [[ -n "${SRT_MEASUREMENT_WINDOW_DIR:-}" ]]; then + python3 -m infx.results.power.window "$RESULT_DIR/$RESULT_FILENAME.json" "$CONC" +fi diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index 7cd6fa2651..1d4bbce674 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -705,6 +705,11 @@ clusters: lock-timeout-s: 1800 single-node-import: true srt-slurm: + env: + CPU_POWER_EXPORTER_RELEASE: v2.40.2 + power-telemetry: + dcgm-port: 9401 + cpu-source: acpi extra: default_gpu_exporter: container_image: nvcr.io#nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless @@ -756,6 +761,9 @@ clusters: import-step-args: [--exclusive] single-node-import: true srt-slurm: + power-telemetry: + dcgm-port: 19401 + cpu-source: acpi extra: default_gpu_exporter: container_image: nvcr.io#nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless @@ -767,6 +775,7 @@ clusters: container-aliases: [dynamo-trtllm, dynamo-sglang, v0.5.11, v0.5.13.post1] nginx-aliases: [nginx-sqsh] env: + CPU_POWER_EXPORTER_RELEASE: v2.40.2 ENROOT_ROOTFS_WRITABLE: "1" volume-mounts: aiperf-cache: /aiperf_mmap_cache diff --git a/inferencex-e2e/docs/configuration-procedures.md b/inferencex-e2e/docs/configuration-procedures.md index 4e8a7d0d23..bbdd062a8f 100644 --- a/inferencex-e2e/docs/configuration-procedures.md +++ b/inferencex-e2e/docs/configuration-procedures.md @@ -70,6 +70,10 @@ omitted. This does not enable telemetry in opted-out recipes or qualify Tachomet metrics as validated PowerX results. The existing 1000 ms Tachometer / 100 ms power-exporter collection intervals and port 9401 are preserved. +GB200/GB300 profiles set `power-telemetry` to collect noprof GPU and ACPI CPU power for every benchmark recipe, including native single-node and AgentX, with `CPU_POWER_EXPORTER_RELEASE=v2.40.2`; native CLI overrides carry these settings through preflight into the retained effective recipe. Eval-only jobs omit collection; GPU and CPU validity stay independent, all-in estimates require both, and existing strict power qualification rejects missing CPU data. + +Telemetry requires the benchmark client and collector on the first allocation host: recipes with reserved infra or clients place the frontend and client together on that reserved head, and affected DSV4 recipes move their frontend/client from decode hosts to head. Worker and allocation counts stay unchanged, but this placement change needs fresh performance results. + Keep model selection, cache preparation, and workload-dependent time limits in the srt driver's tables ([`lanes.py`](../infx/launch/drivers/srt/lanes.py), [`models.py`](../infx/launch/drivers/srt/models.py), diff --git a/inferencex-e2e/docs/configuration-procedures_zh.md b/inferencex-e2e/docs/configuration-procedures_zh.md index efbf9347ba..74854f4002 100644 --- a/inferencex-e2e/docs/configuration-procedures_zh.md +++ b/inferencex-e2e/docs/configuration-procedures_zh.md @@ -54,6 +54,10 @@ srt 驱动（[`infx/launch/drivers/srt/`](../infx/launch/drivers/srt)）在 `mak 缓存挂载、时间上限，以及功耗作业所需的 DCGM exporter 镜像。取值以 YAML 数据写入， 绝不替换进 shell 或 YAML 文本，`extra` 也不能覆盖类型化的键。 +GB200/GB300 集群通过 `power-telemetry` 为所有 benchmark 配方（包括原生单节点和 AgentX）默认采集 noprof GPU 与 ACPI CPU 功耗，并固定 `CPU_POWER_EXPORTER_RELEASE=v2.40.2`；原生 CLI 覆盖项贯穿预检并保存在实际运行配方中。纯评估作业不采集；GPU 和 CPU 有效性保持独立，all-in 估算要求两者均有效，已有严格功耗资格检查仍拒绝缺失的 CPU 数据。 + +遥测要求 benchmark 客户端与采集器使用分配首节点的时钟：原本预留 infra 或客户端节点的配方改为预留 frontend/head，并将客户端放在同一节点；相关 DSV4 配方将 frontend/客户端从 decode 节点移至 head。worker 和总节点数不变，但此放置变化需要新的性能结果。 + 模型选择、缓存准备以及依赖工作负载的时间上限保留在 srt 驱动的表中 （[`lanes.py`](../infx/launch/drivers/srt/lanes.py)、[`models.py`](../infx/launch/drivers/srt/models.py)、[`power.py`](../infx/launch/drivers/srt/power.py)），不写进集群记录。 diff --git a/inferencex-e2e/docs/results-and-ingestion.md b/inferencex-e2e/docs/results-and-ingestion.md index 08d4e62226..c72d286d85 100644 --- a/inferencex-e2e/docs/results-and-ingestion.md +++ b/inferencex-e2e/docs/results-and-ingestion.md @@ -456,6 +456,8 @@ supply P75 or P90. The validation sidecar records `power_percentile_method`. ## Measured Grace CPU-side power (NVL72) +GB200/GB300 cluster defaults enable GPU and ACPI CPU collection for ordinary fixed-sequence and AgentX submissions, including native single-node recipes; eval-only jobs have no benchmark power window. The Grace socket total includes CPU/SoC, LPDDR5X and regulator losses, not a separate RAM measurement; validated totals supply the measured part of system-power estimates, while unmeasured rack components remain modeled. + GB200 and GB300 NVL72 recipes that enable srt-slurm's `telemetry.cpu_power_exporter` leg write `LOGS/power/cpu/samples.csv` and a non-authoritative `cpu_manifest.json` beside the GPU DCGM package. The multinode validator accepts the historical v2.2.1 long format (one row per sensor diff --git a/inferencex-e2e/docs/results-and-ingestion_zh.md b/inferencex-e2e/docs/results-and-ingestion_zh.md index 5c7eee8b4c..1499ec0b99 100644 --- a/inferencex-e2e/docs/results-and-ingestion_zh.md +++ b/inferencex-e2e/docs/results-and-ingestion_zh.md @@ -440,6 +440,8 @@ rm -rf -- "$tmp" ## NVL72 Grace CPU 侧实测功耗 +GB200/GB300 集群默认配置为普通 fixed-sequence 和 AgentX 提交（包括原生单节点配方）启用 GPU 与 ACPI CPU 采集；纯评估作业没有 benchmark 功耗窗口。Grace socket 总功耗包含 CPU/SoC、LPDDR5X 和稳压损耗，并非单独的 RAM 测量；有效总量构成系统功耗估算的实测部分，未测量的机架组件仍采用模型。 + 启用了 srt-slurm `telemetry.cpu_power_exporter` 的 GB200 和 GB300 NVL72 recipe 会在 GPU DCGM 产物包旁写入 `LOGS/power/cpu/samples.csv` 和一份非权威的 `cpu_manifest.json`。多节点校验器接受 历史版本 v2.2.1 写出的长格式（每个传感器读数一行，表头为 diff --git a/inferencex-e2e/infx/clusters/slurm.py b/inferencex-e2e/infx/clusters/slurm.py index 72bafde63e..87dfdbfabf 100644 --- a/inferencex-e2e/infx/clusters/slurm.py +++ b/inferencex-e2e/infx/clusters/slurm.py @@ -150,6 +150,14 @@ def _repository_relative(cls, script: PurePosixPath) -> PurePosixPath: return script +class PowerTelemetrySettings(Record): + """Cluster power sensors used by normal SRT benchmark submissions.""" + + dcgm_port: int = Field(alias="dcgm-port", ge=1, le=65535) + cpu_port: int = Field(default=9405, alias="cpu-port", ge=1, le=65535) + cpu_source: Literal["acpi", "dcgm"] = Field(alias="cpu-source") + + class SrtSlurmSettings(Record): """Cluster-owned srtslurm.yaml facts; job-specific values are added by the driver.""" @@ -168,6 +176,7 @@ class SrtSlurmSettings(Record): container_aliases: tuple[str, ...] = Field(default=(), alias="container-aliases") nginx_aliases: tuple[str, ...] = Field(default=(), alias="nginx-aliases") host_setup: HostSetup | None = Field(default=None, alias="host-setup") + power_telemetry: PowerTelemetrySettings | None = Field(default=None, alias="power-telemetry") env: dict[str, str] = Field(default_factory=dict) outputs: HostPath | None = None shared_run_root: HostPath | None = Field(default=None, alias="shared-run-root") diff --git a/inferencex-e2e/infx/launch/artifacts.py b/inferencex-e2e/infx/launch/artifacts.py index ba66f5a38a..c0f1a707c9 100644 --- a/inferencex-e2e/infx/launch/artifacts.py +++ b/inferencex-e2e/infx/launch/artifacts.py @@ -154,6 +154,7 @@ def collect_agentic_power_results( *, results_python: str | None, expected_cpu_source: str | None = None, + require_power: bool = True, ) -> int: """Stage the AgentX power audit inputs and validate each concurrency's power window. @@ -176,7 +177,8 @@ def collect_agentic_power_results( if step := validate_agentic_power( logs_dir, workspace, result_filename, producer_sha, concurrencies, - results_python=results_python, require_power=True, expected_cpu_source=expected_cpu_source, + results_python=results_python, require_power=require_power, + expected_cpu_source=expected_cpu_source, ): # fmt: skip rc = step return rc diff --git a/inferencex-e2e/infx/launch/drivers/srt/__init__.py b/inferencex-e2e/infx/launch/drivers/srt/__init__.py index ec2838f2d1..f81ae4dfb9 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/__init__.py +++ b/inferencex-e2e/infx/launch/drivers/srt/__init__.py @@ -43,16 +43,38 @@ def run_single_node(launch: Launch) -> int: model_path = models.single_node_model_path(launch.cluster, request) staged = {"MODEL_PATH": model_path} if model_path.startswith("/") else {} run = SrtRun.create(launch, request, staged) + decision = power.resolve_power( + launch.cluster.id, launch.path, request, settings=run.srt.power_telemetry + ) hf_cache = models.single_node_hf_cache(run.cluster, request) time_limit = lanes.srt_time_limit(run.cluster.id, request, None, run.srt) root = Path(tempfile.mkdtemp(prefix="srt-single.", dir=run.workspace)) - checkout = prepare_checkout(run, root / "checkout", power=False) + checkout = prepare_checkout(run, root / "checkout", power=decision.dcgm) install_srtctl(run, checkout) if (options := config.srun_options(run.backend.settings)) is not None: run.env["SRT_SRUN_OPTIONS"] = options if rc := submit.bind_point(run, checkout, root / "arguments"): return rc selected, runtime_args = submit.bound_arguments(root / "arguments") + if request.eval_only and run.srt.power_telemetry is not None: + runtime_args += ["--set", "telemetry.enabled=false"] + containers = {} + if decision.dcgm: + exporter = run.backend.stage_image(config.DCGM_EXPORTER_IMAGE, helper="dcgm-exporter") + (run.workspace / config.EXPORTER_PROVENANCE).write_text( + f"{run.backend.image_provenance(exporter)}\n" + ) + containers["dcgm-exporter"] = exporter.reference + runtime_args += power.telemetry_arguments(run.srt.power_telemetry, [request.conc]) + runtime_args += ["--set", 'benchmark.env.ENABLE_AGENTX_POWER="1"'] + runtime_args += ["--set", f'benchmark.env.REQUIRE_POWER="{int(request.require_power)}"'] + if github_env := request.env.get("GITHUB_ENV"): + with Path(github_env).open("a") as handle: + handle.write( + "POWER_ARTIFACT_DIR=LOGS/power\nPOWER_RESULT_ROOT=LOGS\n" + f"POWER_EXPECTED_CPU_SOURCE={decision.expected_cpu_source or ''}\n" + f"POWER_PRODUCER_SHA={checkout.commit}\n" + ) job_config = config.SrtJob( srtctl_root=checkout.root, workspace=run.workspace, @@ -60,6 +82,7 @@ def run_single_node(launch: Launch) -> int: image=request.image, container=run.backend.stage_image(request.image, single_node=True).reference, nginx=config.NGINX_IMAGE if run.srt.nginx_aliases else None, + containers=containers, model_paths={f"hf:{request.model}": model_path}, mounts=[(str(hf_cache), request.hf_hub_cache)], single_node=True, @@ -72,7 +95,7 @@ def run_single_node(launch: Launch) -> int: fetched = root / "fetched-outputs" submitted = submit.Submitted(manifest=run.workspace / submit.SINGLE_NODE_SUBMISSION) - run.life.callback(collect.finish_single_node, run, submitted, fetched) + run.life.callback(collect.finish_single_node, run, submitted, fetched, decision) eval_args = submit.eval_args(run.env, submit.SINGLE_NODE_EVAL_COMMAND) arguments = ["--json", "--yes", "--output", str(root / "outputs"), *runtime_args, *eval_args] applied = submit.apply(run, checkout, selected, arguments, stdout=submitted.manifest) @@ -84,9 +107,13 @@ def run_single_node(launch: Launch) -> int: run.backend.stream_logs(job) except BackendError: return 1 - if not run.backend.state(job).succeeded: - return 1 - return collect.check_single_node(run, run.backend.fetch_outputs(job, fetched) / "logs") + status = run.backend.state(job) + logs = run.backend.fetch_outputs(job, fetched) / "logs" + if decision.dcgm: + (logs / "power").mkdir(parents=True, exist_ok=True) + (logs / "power/native-job-status.txt").write_text(f"{job.id}|{status.raw}\n") + rc = collect.check_single_node(run, logs, decision, checkout.commit) + return rc or int(not status.succeeded) def run_batch(launch: Launch) -> int: @@ -123,7 +150,9 @@ def run_multinode(launch: Launch) -> int: request = SrtRequest.from_env(launch.request.env) lanes.check_request(lane, request) config_file = lanes.config_file(request) - decision = power.resolve_power(launch.cluster.id, launch.path, request) + settings = _srt_settings(launch.cluster) + telemetry = settings.power_telemetry if settings else None + decision = power.resolve_power(launch.cluster.id, launch.path, request, settings=telemetry) if github_env := request.env.get("GITHUB_ENV"): with Path(github_env).open("a") as handle: handle.write(f"POWER_EXPECTED_CPU_SOURCE={decision.expected_cpu_source or ''}\n") @@ -138,6 +167,17 @@ def run_multinode(launch: Launch) -> int: checkout = prepare_checkout(run, checkout_dir(run, shared=shared), power=decision.dcgm) overrides = eval_overrides(checkout.root / "recipes", lane, request) + if telemetry is not None: + if request.eval_only: + overrides += ["--set", "telemetry.enabled=false"] + else: + overrides += power.telemetry_arguments(telemetry, request.conc_list) + if request.is_agentic: + overrides += ["--set", 'benchmark.env.ENABLE_AGENTX_POWER="1"'] + overrides += [ + "--set", + f'benchmark.env.REQUIRE_POWER="{int(decision.require_power)}"', + ] system_python = ( "/usr/bin/python3" if shared and os.access("/usr/bin/python3", os.X_OK) else None ) @@ -148,7 +188,7 @@ def run_multinode(launch: Launch) -> int: infmax = compute_workspace(run, checkout, shared=shared) run.env["INFMAX_WORKSPACE"] = str(infmax) - conc_list = request.env.get("CONC_LIST", "") if decision.dcgm else None + conc_list = request.env.get("CONC_LIST", "") if decision.dcgm and telemetry is None else None job_name = srtctl_job_name(request.runner_name) prepare_recipe(checkout.root, config_file, job_name, run.srt.dist_timeout_s, conc_list) arguments = submit.multinode_arguments(run, lane, config_file, overrides, preflight=preflight) diff --git a/inferencex-e2e/infx/launch/drivers/srt/collect.py b/inferencex-e2e/infx/launch/drivers/srt/collect.py index 8b89281506..973bd2c405 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/collect.py +++ b/inferencex-e2e/infx/launch/drivers/srt/collect.py @@ -48,7 +48,9 @@ def _copy_tree_into(source: Path, destination: Path) -> None: shutil.copytree(source, destination, symlinks=True, dirs_exist_ok=True) -def finish_single_node(run: SrtRun, submitted: Submitted, fetched: Path) -> int: +def finish_single_node( + run: SrtRun, submitted: Submitted, fetched: Path, power: PowerDecision | None = None +) -> int: """Exit cleanup of a single-node point: cancel a live job, then stage its artifacts.""" job = submitted.recover(run.backend) if job is None: @@ -58,8 +60,18 @@ def finish_single_node(run: SrtRun, submitted: Submitted, fetched: Path) -> int: if not output.is_dir(): return 0 rc = 0 - bundle_server_logs(output, run.workspace / SINGLE_NODE_LOGS) logs = output / "logs" + if power is not None and power.dcgm: + power_dir = logs / "power" + power_dir.mkdir(parents=True, exist_ok=True) + try: + for name in (EXPORTER_PROVENANCE, "power-producer-sha.txt"): + shutil.copyfile(run.workspace / name, power_dir / name) + shutil.copytree(logs, run.workspace / "LOGS", dirs_exist_ok=True) + except OSError as error: + print(f"ERROR: failed to stage power artifacts: {error}", file=sys.stderr) + rc = 1 + bundle_server_logs(output, run.workspace / SINGLE_NODE_LOGS) result = logs / f"{run.request.result_filename}.json" for artifact in [result, *sorted(logs.glob("gpu_metrics*"))]: if artifact.is_file(): @@ -77,12 +89,31 @@ def finish_single_node(run: SrtRun, submitted: Submitted, fetched: Path) -> int: return rc -def check_single_node(run: SrtRun, logs: Path) -> int: +def check_single_node( + run: SrtRun, logs: Path, power: PowerDecision | None = None, producer_sha: str = "" +) -> int: """Fail unless each requested eval succeeded and the benchmark result exists. srt-slurm treats a failed post-benchmark eval as non-fatal; InferenceX does not. """ request = run.request + rc = 0 + if not request.eval_only and power is not None and power.agentx: + require(request, "INFERENCEX_RESULTS_PYTHON", "GPU_COUNT") + argv = [ + request.inferencex_results_python, "-m", "infx.results.agentic.power_adapter", + "--result-dir", str(logs / "agentic"), + "--agg-result", str(logs / f"{request.result_filename}.json"), + "--power-dir", str(logs / "power"), + "--logs-root", str(logs), + "--expected-producer-sha", producer_sha, + "--expected-num-gpus", request.env["GPU_COUNT"], + "--audit-source", "results/power_validation.json", + *(["--require-power"] if request.require_power else []), + *(["--expected-cpu-source", power.expected_cpu_source] + if power.expected_cpu_source else []), + ] # fmt: skip + rc = proc.run(argv, env=run.env, cwd=run.workspace).returncode if request.run_eval or request.eval_only: exit_file = logs / "infx-eval-exit-code" if not exit_file.is_file() or exit_file.read_text().rstrip("\n") != "0": @@ -93,7 +124,7 @@ def check_single_node(run: SrtRun, logs: Path) -> int: if not result.is_file() or result.stat().st_size == 0: print(f"ERROR: benchmark result {result} is missing or empty", file=sys.stderr) return 1 - return 0 + return rc def _stage_logs(run: SrtRun, logs: Path, power: PowerDecision) -> None: @@ -104,11 +135,16 @@ def _stage_logs(run: SrtRun, logs: Path, power: PowerDecision) -> None: if power.dcgm: power_dir = logs / "power" power_dir.mkdir(parents=True, exist_ok=True) - for name in (EXPORTER_PROVENANCE, "power-producer-sha.txt"): + receipts = [ + *logs.parent.glob("config*.yaml"), + workspace / EXPORTER_PROVENANCE, + workspace / "power-producer-sha.txt", + ] + for receipt in receipts: try: - shutil.copyfile(workspace / name, power_dir / name) + shutil.copyfile(receipt, power_dir / receipt.name) except OSError as error: - print(f"WARNING: could not stage {name}: {error}", file=sys.stderr) + print(f"WARNING: could not stage {receipt.name}: {error}", file=sys.stderr) try: _copy_tree_into(logs, workspace / "LOGS") except OSError as error: @@ -155,6 +191,7 @@ def stage_logs_on_exit() -> None: *audit, results_python=python, expected_cpu_source=power.expected_cpu_source, + require_power=power.require_power, ) else: power_rc = validate_agentic_power( diff --git a/inferencex-e2e/infx/launch/drivers/srt/power.py b/inferencex-e2e/infx/launch/drivers/srt/power.py index 6985dd987e..4430f1a195 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/power.py +++ b/inferencex-e2e/infx/launch/drivers/srt/power.py @@ -1,12 +1,10 @@ -"""DCGM power eligibility of the srt-slurm multi-node lanes. - -A recipe asks for DCGM power in its top-level ``telemetry:`` mapping; each lane lists the -requests and recipes that may have it. -""" +"""Cluster power defaults and legacy recipe opt-ins for srt-slurm launches.""" from __future__ import annotations import fnmatch +import json +from collections.abc import Sequence from dataclasses import dataclass, replace from typing import TYPE_CHECKING @@ -17,6 +15,7 @@ from infx.launch.policy import LaunchPath, Match, any_of if TYPE_CHECKING: + from infx.clusters.slurm import PowerTelemetrySettings from infx.launch.request import LaunchRequest @@ -160,6 +159,7 @@ class PowerDecision: agentx: bool adapter: bool = False expected_cpu_source: str | None = None + require_power: bool = False NO_POWER = PowerDecision(dcgm=False, agentx=False) @@ -179,30 +179,54 @@ def decide_power( if rule.when(request) and ( rule.recipe_glob is None or fnmatch.fnmatchcase(recipe, rule.recipe_glob) ): - return PowerDecision(dcgm=True, agentx=rule.agentx, adapter=rule.adapter) + return PowerDecision( + dcgm=True, + agentx=rule.agentx, + adapter=rule.adapter, + require_power=request.require_power or rule.agentx, + ) message = lane.agentic_error if request.is_agentic and lane.agentic_error else lane.error raise PowerPolicyError(message) -def resolve_power(cluster_id: str, path: LaunchPath, request: LaunchRequest) -> PowerDecision: - """Detect dcgm in the workspace mirror of the recipe the lane inspects, and decide. - - A recipe only upstream (no mirror) stays non-power. - """ +def resolve_power( + cluster_id: str, + path: LaunchPath, + request: LaunchRequest, + *, + settings: PowerTelemetrySettings | None = None, +) -> PowerDecision: + """Use cluster defaults when configured; otherwise inspect the mirrored recipe opt-in.""" lane = POWER_LANES.get((cluster_id, path)) - if lane is None: + if settings is not None and request.eval_only: + return NO_POWER + if lane is None and settings is None: return NO_POWER config_file = request.config_file - if lane.eval_recipe_when_eval_only and request.eval_only and request.eval_config_file: + if lane and lane.eval_recipe_when_eval_only and request.eval_only and request.eval_config_file: config_file = request.eval_config_file - if not config_file: - return NO_POWER - mirror = recipe_mirror_path(request.workspace, config_file) - text = mirror.read_text() if mirror.is_file() else "" + mirror = recipe_mirror_path(request.workspace, config_file) if config_file else None + text = mirror.read_text() if mirror and mirror.is_file() else "" dcgm = recipe_enables_dcgm_power(text) - decision = decide_power( - cluster_id, path, dcgm=dcgm, request=request, recipe=recipe_relpath(config_file) - ) + try: + decision = decide_power( + cluster_id, + path, + dcgm=dcgm, + request=request, + recipe=recipe_relpath(config_file) if config_file else "", + ) + except PowerPolicyError: + if settings is None: + raise + decision = NO_POWER + if settings is not None: + return PowerDecision( + dcgm=True, + agentx=request.is_agentic, + expected_cpu_source=settings.cpu_source, + require_power=decision.require_power or request.require_power, + ) if decision.dcgm: telemetry = yaml.safe_load(text)["telemetry"] cpu = telemetry.get("cpu_power_exporter") @@ -210,3 +234,28 @@ def resolve_power(cluster_id: str, path: LaunchPath, request: LaunchRequest) -> if source in {"acpi", "dcgm"}: decision = replace(decision, expected_cpu_source=source) return decision + + +def telemetry_arguments( + settings: PowerTelemetrySettings, concurrencies: Sequence[int] +) -> list[str]: + """Bind cluster sensors to the selected native recipe, after variant expansion.""" + if not concurrencies or any(value <= 0 for value in concurrencies): + raise PowerPolicyError("power telemetry requires positive benchmark concurrencies") + values = { + "telemetry.enabled": True, + "telemetry.collect_interval_ms": 1000, + "telemetry.storage_subdir": "power", + "telemetry.startup_timeout_seconds": 120, + "telemetry.request_timeout_seconds": 2, + "telemetry.dcgm_exporter.container_image": "dcgm-exporter", + "telemetry.dcgm_exporter.port": settings.dcgm_port, + "telemetry.dcgm_exporter.command": ( + "dcgm-exporter --collect-interval=100 --address :{port} " + "-f /configs/dcgm-counters-noprof.csv" + ), + "telemetry.cpu_power_exporter.port": settings.cpu_port, + "telemetry.cpu_power_exporter.source": settings.cpu_source, + "benchmark.concurrencies": list(concurrencies), + } + return [arg for key, value in values.items() for arg in ("--set", f"{key}={json.dumps(value)}")] diff --git a/inferencex-e2e/infx/launch/request.py b/inferencex-e2e/infx/launch/request.py index f293892211..ab2a449847 100644 --- a/inferencex-e2e/infx/launch/request.py +++ b/inferencex-e2e/infx/launch/request.py @@ -75,6 +75,7 @@ class LaunchRequest(BaseModel): bench_script_override: str | None = Field(None, alias="BENCH_SCRIPT_OVERRIDE") batch_reentry: OneFlag = Field(False, alias=BATCH_REENTRY_ENV) conc: int | None = Field(None, alias="CONC") + require_power: PowerFlag = Field(False, alias="REQUIRE_POWER") run_eval: TrueFlag = Field(False, alias="RUN_EVAL") eval_only: TrueFlag = Field(False, alias="EVAL_ONLY") salloc_time_limit: int | None = Field(None, alias="SALLOC_TIME_LIMIT") @@ -115,7 +116,6 @@ class SrtRequest(LaunchRequest): eval_only: TrueFlag = Field(alias="EVAL_ONLY") thinking_mode: str | None = Field(None, alias="THINKING_MODE") conc_list: IntList = Field(default_factory=list, alias="CONC_LIST") - require_power: PowerFlag = Field(False, alias="REQUIRE_POWER") inferencex_results_python: str | None = Field(None, alias="INFERENCEX_RESULTS_PYTHON") eval_conc: str | None = Field(None, alias="EVAL_CONC") diff --git a/inferencex-e2e/infx/results/agentic/power_adapter.py b/inferencex-e2e/infx/results/agentic/power_adapter.py index 2d32dfa925..eef6ab8d86 100644 --- a/inferencex-e2e/infx/results/agentic/power_adapter.py +++ b/inferencex-e2e/infx/results/agentic/power_adapter.py @@ -422,6 +422,8 @@ def run_multinode_agentic_power( expected_producer_sha: str, require_power: bool = False, expected_cpu_source: str | None = None, + expected_num_gpus: int | None = None, + audit_source: str | None = None, ) -> int: """Join one AgentX aggregate to the finalized central multinode package.""" validation_result = result_dir / "power_validation.json" @@ -439,6 +441,15 @@ def run_multinode_agentic_power( prefill_gpus = _gpu_count(aggregate.get("num_prefill_gpu")) decode_gpus = _gpu_count(aggregate.get("num_decode_gpu")) disagg = aggregate.get("disagg") + if expected_num_gpus is not None: + if ( + expected_num_gpus <= 0 + or aggregate.get("is_multinode") is not False + or disagg is not False + or _gpu_count(aggregate.get("num_gpus")) != expected_num_gpus + ): + reasons.append("agentic_gpu_topology_invalid") + prefill_gpus, decode_gpus = expected_num_gpus, 0 if ( not isinstance(disagg, bool) or prefill_gpus is None @@ -498,7 +509,9 @@ def run_multinode_agentic_power( try: result = json.loads(agg_result.read_text()) validation = json.loads(validation_result.read_text()) - source = "LOGS/" + validation_result.resolve().relative_to(logs_root.resolve()).as_posix() + source = audit_source or ( + "LOGS/" + validation_result.resolve().relative_to(logs_root.resolve()).as_posix() + ) result.update(audit_summary(validation, source)) _write_json_atomic(agg_result, result) except (OSError, ValueError, TypeError) as exc: @@ -518,6 +531,7 @@ def main() -> int: parser.add_argument("--power-dir", type=Path) parser.add_argument("--logs-root", type=Path) parser.add_argument("--expected-producer-sha") + parser.add_argument("--audit-source") parser.add_argument( "--require-power", action="store_true", @@ -575,6 +589,8 @@ def main() -> int: expected_producer_sha=args.expected_producer_sha, require_power=args.require_power, expected_cpu_source=args.expected_cpu_source, + expected_num_gpus=args.expected_num_gpus, + audit_source=args.audit_source, ) if args.agg_result is None: parser.error("--agg-result is required for single-node AgentX power") diff --git a/inferencex-e2e/infx/results/fixed_sequence.py b/inferencex-e2e/infx/results/fixed_sequence.py index edf9f23bb2..a825755d3b 100644 --- a/inferencex-e2e/infx/results/fixed_sequence.py +++ b/inferencex-e2e/infx/results/fixed_sequence.py @@ -267,11 +267,14 @@ def aggregate_power_result( require_power = env.get("REQUIRE_POWER", "").lower() in {"1", "true", "yes"} validation_path = Path(f"power_validation_{env['RESULT_FILENAME']}.json") is_multinode = env.get("IS_MULTINODE", "false").lower() == "true" - if is_multinode: + srt_power = bool(env.get("POWER_ARTIFACT_DIR")) + if is_multinode or srt_power: source = Path(env.get("POWER_ARTIFACT_DIR", "LOGS/power")) - prefill_gpus = int(env["PREFILL_GPUS"]) - decode_gpus = int(env["DECODE_GPUS"]) - aggregate_gpus = int(env.get("AGGREGATE_GPUS", "0")) + prefill_gpus = int(env["PREFILL_GPUS"]) if is_multinode else 0 + decode_gpus = int(env["DECODE_GPUS"]) if is_multinode else 0 + aggregate_gpus = ( + int(env.get("AGGREGATE_GPUS", "0")) if is_multinode else int(env["GPU_COUNT"]) + ) expected_num_gpus = prefill_gpus + decode_gpus + aggregate_gpus else: candidates = [ @@ -287,9 +290,9 @@ def aggregate_power_result( int(env["TP"]) * int(env.get("PP_SIZE", "1")) * int(env.get("PCP_SIZE", "1")) ) try: - if is_multinode: + if is_multinode or srt_power: native_dir = Path(env.get("POWERX_NATIVE_DIR", "LOGS/native_power")) - if env.get("POWERX_NATIVE_DIR") or native_dir.is_dir(): + if is_multinode and (env.get("POWERX_NATIVE_DIR") or native_dir.is_dir()): if source.is_dir() and source != native_dir: raise ValueError("Both native and SRT power packages are present") source = native_dir diff --git a/inferencex-e2e/infx/tests/launch/fake_slurm.py b/inferencex-e2e/infx/tests/launch/fake_slurm.py index c64e645f9f..89c004e837 100644 --- a/inferencex-e2e/infx/tests/launch/fake_slurm.py +++ b/inferencex-e2e/infx/tests/launch/fake_slurm.py @@ -134,6 +134,9 @@ (logs / "infx-eval-exit-code").write_text("0\n") if os.environ.get("FAKE_ACTIVE"): pathlib.Path(os.environ["FAKE_ACTIVE"]).touch() +if fixture := os.environ.get("FAKE_SRT_OUTPUT_FIXTURE"): + import shutil + shutil.copytree(fixture, logs, dirs_exist_ok=True) if "--json" in argv: print(json.dumps({"status": "submitted", "slurm_job_id": "42", "output_dir": str(output)})) else: diff --git a/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py b/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py index 1f5ec3e094..302fb2c140 100644 --- a/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py +++ b/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py @@ -116,9 +116,10 @@ def test_power_collection_records_the_job_status_and_stages_results_either_way(t assert (workspace / "point_conc4.json").exists() +@pytest.mark.parametrize("require_power", [False, True]) @pytest.mark.parametrize("has_cpu", [False, True]) @pytest.mark.parametrize("disagg", [False, True]) -def test_agentic_collection_requires_declared_cpu_and_keeps_gpu_valid(tmp_path, has_cpu, disagg): +def test_agentic_collection_requires_declared_cpu_and_keeps_gpu_valid(tmp_path, has_cpu, disagg, require_power): from infx.tests.results.power.test_aggregate_power_multinode import ( PRODUCER_SHA, _cpu_rows, @@ -173,7 +174,8 @@ def test_agentic_collection_requires_declared_cpu_and_keeps_gpu_valid(tmp_path, [4], results_python=sys.executable, expected_cpu_source="acpi", - ) == int(not has_cpu) + require_power=require_power, + ) == int(require_power and not has_cpu) result = json.loads((workspace / "point_conc4.json").read_text()) assert result["power_valid"] == 1 assert result["cpu_power_valid"] == int(has_cpu) diff --git a/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py b/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py new file mode 100644 index 0000000000..7b0481dd22 --- /dev/null +++ b/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py @@ -0,0 +1,126 @@ +"""Cluster telemetry survives new models, recipe variants, and ordinary run policy.""" + +import copy +import sys +from pathlib import Path + +import pytest +import yaml + +from infx.clusters import load_inventory +from infx.launch.drivers.srt.power import resolve_power, telemetry_arguments +from infx.launch.policy import LaunchPath +from infx.launch.request import LaunchRequest + +ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(ROOT / "utils/srt-slurm/src")) +from srtctl.core.config import generate_override_configs, resolve_config_with_defaults +from srtctl.core.overrides import apply_overrides_to_recipe, parse_overrides +from srtctl.core.schema import SrtConfig + + +@pytest.mark.parametrize("cluster", ["gb200-nv", "gb300-nv"]) +@pytest.mark.parametrize("agentic,framework,path", [ + (False, "dynamo-trt", LaunchPath.SRT_MULTI), + (True, "dynamo-trt", LaunchPath.SRT_MULTI), + (True, "dynamo-vllm", LaunchPath.SRT_MULTI), + (True, "vllm", LaunchPath.SRT_SINGLE), +]) +def test_new_models_collect_without_recipe_or_model_allowlists(cluster, agentic, framework, path): + settings = load_inventory(yaml.safe_load((ROOT / "configs/runners.yaml").read_text())).clusters[ + cluster + ].scheduler_settings.srt_slurm + request = LaunchRequest.from_env({ + "RUNNER_NAME": "fixture_0", "IS_AGENTIC": str(int(agentic)), + "MODEL_PREFIX": "future-model", "FRAMEWORK": framework, + }) + decision = resolve_power(cluster, path, request, settings=settings.power_telemetry) + assert decision.dcgm and decision.agentx is agentic + assert decision.expected_cpu_source == "acpi" + assert not decision.require_power + assert settings.env["CPU_POWER_EXPORTER_RELEASE"] == "v2.40.2" + evaluation = LaunchRequest.from_env({**request.env, "EVAL_ONLY": "true"}) + assert not resolve_power(cluster, path, evaluation, settings=settings.power_telemetry).dcgm + + +@pytest.mark.parametrize("variant_file", [False, True]) +def test_native_overrides_materialize_cpu_and_noprof_without_rewriting_workload(variant_file): + settings = load_inventory(yaml.safe_load((ROOT / "configs/runners.yaml").read_text())).clusters[ + "gb300-nv" + ].scheduler_settings.srt_slurm.power_telemetry + recipe = yaml.safe_load((ROOT / ( + "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/" + "disagg-8p1d-dep4-dep16-stp.yaml" + )).read_text()) + recipe.pop("telemetry") + original = copy.deepcopy(recipe) + document = {"base": recipe, "override_point": {"benchmark": {"concurrencies": [1]}}} if variant_file else recipe + arguments = telemetry_arguments(settings, [2048, 4096]) + apply_overrides_to_recipe(document, parse_overrides(arguments[1::2], None)) + effective = generate_override_configs(document, selector="override_point")[0][1] if variant_file else document + config = SrtConfig.Schema().load(resolve_config_with_defaults(effective, None)) + assert config.telemetry.enabled + assert config.telemetry.cpu_power_exporter.source == "acpi" + assert config.telemetry.dcgm_exporter.port == 19401 + assert "dcgm-counters-noprof.csv" in config.telemetry.dcgm_exporter.command + assert config.benchmark.get_concurrency_list() == [2048, 4096] + assert effective["roles"] == original["roles"] + assert effective["benchmark"]["command"] == original["benchmark"]["command"] + + +def test_existing_strict_agentx_recipe_remains_strict(): + settings = load_inventory(yaml.safe_load((ROOT / "configs/runners.yaml").read_text())).clusters[ + "gb200-nv" + ].scheduler_settings.srt_slurm.power_telemetry + request = LaunchRequest.from_env({ + "RUNNER_NAME": "fixture_0", "GITHUB_WORKSPACE": str(ROOT), + "IS_AGENTIC": "1", "MODEL_PREFIX": "kimik3", "PRECISION": "fp4", + "FRAMEWORK": "dynamo-vllm", + "CONFIG_FILE": "recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml", + }) + assert resolve_power("gb200-nv", LaunchPath.SRT_MULTI, request, settings=settings).require_power + + +@pytest.mark.parametrize('relative,selector,total,workers', [ + ('dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml', None, 9, 9), + ('dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml', 'override_1p4d_dep4_tep8_c4_b1', 9, 9), + ('glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml', 'override_1p1d_tp8_c1_b1_mtp5', 4, 3), +]) +def test_client_and_collector_share_head_without_changing_worker_budget(monkeypatch, relative, selector, total, workers): + from srtctl.cli.submit import planned_total_nodes + from srtctl.core.runtime import Nodes + settings = load_inventory(yaml.safe_load((ROOT / 'configs/runners.yaml').read_text())).clusters[ + 'gb300-nv' + ].scheduler_settings.srt_slurm.power_telemetry + raw = yaml.safe_load((ROOT / 'benchmarks/multi_node/srt-slurm-recipes' / relative).read_text()) + apply_overrides_to_recipe(raw, parse_overrides(telemetry_arguments(settings, [4])[1::2], None)) + effective = generate_override_configs(raw, selector)[0][1] if selector else raw + config = SrtConfig.Schema().load(resolve_config_with_defaults(effective, None)) + assert planned_total_nodes(config) == total + assert config.engine_node_count == workers + assert not config.pool_services + monkeypatch.setattr('srtctl.core.runtime.get_slurm_nodelist', lambda: [f'node-{i}' for i in range(total)]) + monkeypatch.setattr('srtctl.core.runtime.get_slurm_het_nodelists', lambda: None) + nodes = Nodes.from_slurm( + frontend_dedicated_node=config.frontend.dedicated_node, + client_dedicated_node=config.benchmark.client_dedicated_node, + etcd_nats_dedicated_node=config.infra.etcd_nats_dedicated_node, + colocate_dedicated_nodes=config.benchmark.colocate_with_frontend, + ) + assert nodes.head == nodes.bench == 'node-0' + assert len(nodes.compute) == len(nodes.worker) == workers + + +def test_multinode_power_bundle_retains_effective_cluster_overrides(tmp_path): + from types import SimpleNamespace + from infx.launch.drivers.srt.collect import _stage_logs + from infx.launch.drivers.srt.power import PowerDecision + logs, workspace = tmp_path / 'job/logs', tmp_path / 'workspace' + logs.mkdir(parents=True) + workspace.mkdir() + for name in ('power-exporter-image.txt', 'power-producer-sha.txt'): + (workspace / name).write_text('fixture\n') + effective = 'telemetry:\n enabled: true\n cpu_power_exporter:\n source: acpi\n' + (logs.parent / 'config_resolved.yaml').write_text(effective) + _stage_logs(SimpleNamespace(workspace=workspace), logs, PowerDecision(dcgm=True, agentx=True)) + assert (workspace / 'LOGS/power/config_resolved.yaml').read_text() == effective diff --git a/inferencex-e2e/infx/tests/launch/test_srt_driver.py b/inferencex-e2e/infx/tests/launch/test_srt_driver.py index 33b539e7bc..ed1f0b3a71 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_driver.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_driver.py @@ -173,6 +173,107 @@ def test_single_node_failed_allocation_fails_the_launch(harness): assert (harness.workspace / "point-identity.json").is_file() +@pytest.mark.parametrize( + ("gpu_count", "has_cpu", "require_power"), + [(2, True, True), (4, True, True), (4, False, True), (4, False, False)], +) +def test_native_agentx_collects_declared_cpu_with_serving_gpu_count( + harness, monkeypatch, gpu_count, has_cpu, require_power +): + from infx.tests.results.power import test_aggregate_power_multinode as packages + + monkeypatch.setattr( + packages, "DEVICES", + tuple(("node-native", index, "agg", 0, 300.0) for index in range(gpu_count)), + ) + monkeypatch.setattr(packages, "CPU_HOSTS", ("node-native",)) + pkg = packages.build_package(harness.tmp / "captured") + result_dir = pkg.logs_root / "agentic" + result_dir.mkdir() + formal_result = result_dir / "agentic_power_concurrency_4.json" + formal_result.write_bytes(pkg.original_result.read_bytes()) + [window_path] = pkg.windows_dir.glob("*.json") + window = json.loads(window_path.read_text()) + window["result_path"] = str(formal_result.relative_to(pkg.logs_root)) + window_path.unlink() + (pkg.windows_dir / formal_result.name).write_text(json.dumps(window)) + manifest_path = pkg.power_dir / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["window_validations"][0]["window_file"] = f"windows/{formal_result.name}" + manifest_path.write_text(json.dumps(manifest)) + if has_cpu: + packages.add_cpu_package( + pkg, packages._cpu_rows({"node-native": {"grace": 500.0}}, hosts=("node-native",)) + ) + (pkg.logs_root / "point-identity.json").write_text(json.dumps({ + "disagg": False, "is_multinode": False, "num_gpus": gpu_count, + })) + + env = single_node_env( + harness, "gb200-nv", IS_AGENTIC="1", ISL="0", OSL="0", CONC="4", DURATION="60", + TP=str(gpu_count), GPU_COUNT=str(gpu_count), + REQUIRE_POWER=str(int(require_power)), INFERENCEX_RESULTS_PYTHON=sys.executable, + FAKE_SRT_COMMIT=packages.PRODUCER_SHA, FAKE_SRT_OUTPUT_FIXTURE=str(pkg.logs_root), + GITHUB_ENV=str(harness.tmp / "github-env"), + ) + recipe_path = harness.workspace / "recipe.yaml" + recipe = yaml.safe_load(recipe_path.read_text()) + recipe["base"]["benchmark"]["command"] = "bash /infmax-workspace/benchmarks/srt_agentic.sh" + recipe["base"]["roles"]["agg"]["gpus"] = gpu_count + recipe["base"]["roles"]["agg"]["args"]["tensor-parallel-size"] = gpu_count + recipe_path.write_text(yaml.safe_dump(recipe)) + config_data = yaml.safe_load(harness.config.read_text()) + config_data["clusters"]["gb200-nv"]["slurm"]["srt-slurm"]["power-telemetry"] = { + "dcgm-port": 9401, "cpu-port": 9405, "cpu-source": "acpi", + } + harness.config.write_text(yaml.safe_dump(config_data)) + + completed = launch(env, harness.config, harness.workspace) + assert completed.returncode == int(require_power and not has_cpu), completed.stderr + result = json.loads((harness.workspace / "point-identity.json").read_text()) + assert result["is_multinode"] is False + assert result["num_gpus"] == gpu_count + assert result["power_valid"] == 1 + assert result["cpu_power_valid"] == int(has_cpu) + assert result["total_gpu_energy_j"] == 18000.0 * gpu_count + assert (harness.workspace / "LOGS/agentic/power_validation.json").is_file() + assert result["power_audit"]["source"] == "results/power_validation.json" + assert (harness.workspace / result["power_audit"]["source"]).is_file() + cpu = result["power_audit"]["cpu"] + if has_cpu: + assert cpu["sensor_kind"] == "grace_socket" + assert cpu["source"] == "acpi" + assert cpu["expected_sockets"] == cpu["observed_sockets"] == 2 + assert cpu["reason_codes"] == [] + else: + assert cpu["reason_codes"] == ["cpu_artifacts_missing"] + assert (harness.workspace / "LOGS/power/native-job-status.txt").read_text() == "42|COMPLETED|0:0\n" + assert (harness.workspace / "LOGS/power/power-producer-sha.txt").read_text() == packages.PRODUCER_SHA + "\n" + exporter = srtslurm(harness.workspace)["containers"]["dcgm-exporter"] + assert (harness.workspace / "LOGS/power/exporter-image.sha256").read_text().endswith(f" {exporter}\n") + [submitted] = srtctl_calls(harness.logs) + assert "benchmark.concurrencies=[4]" in submitted["argv"] + assert 'benchmark.env.ENABLE_AGENTX_POWER="1"' in submitted["argv"] + assert 'telemetry.cpu_power_exporter.source="acpi"' in submitted["argv"] + + +def test_native_eval_only_disables_default_power_collection(harness): + env = single_node_env( + harness, "gb300-nv", EVAL_ONLY="true", MAX_MODEL_LEN="1024", REQUIRE_POWER="1" + ) + config_data = yaml.safe_load(harness.config.read_text()) + config_data["clusters"]["gb300-nv"]["slurm"]["srt-slurm"]["power-telemetry"] = { + "dcgm-port": 9401, "cpu-port": 9405, "cpu-source": "acpi", + } + harness.config.write_text(yaml.safe_dump(config_data)) + + assert_ok(launch(env, harness.config, harness.workspace)) + [submitted] = srtctl_calls(harness.logs) + assert "telemetry.enabled=false" in submitted["argv"] + assert not (harness.workspace / "power-producer-sha.txt").exists() + assert "dcgm-exporter" not in srtslurm(harness.workspace)["containers"] + + LABS = { "lab-a": dict( lane=SrtLane( diff --git a/inferencex-e2e/infx/tests/results/agentic/test_power_adapter.py b/inferencex-e2e/infx/tests/results/agentic/test_power_adapter.py index 7a4b644a54..719b81fe53 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_power_adapter.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_power_adapter.py @@ -87,6 +87,36 @@ def _record( } +def test_native_srt_package_rejects_aggregate_gpu_count_mismatch(tmp_path): + from infx.results.agentic.power_adapter import run_multinode_agentic_power + + result_dir = tmp_path / "agentic" + result_dir.mkdir() + (result_dir / "agentic_power_concurrency_4.json").write_text("{}") + aggregate = tmp_path / "point.json" + aggregate.write_text(json.dumps({ + "is_multinode": False, "disagg": False, "num_gpus": 2, + "power_valid": 1, "avg_power_w": 900, + })) + + assert run_multinode_agentic_power( + result_dir=result_dir, + agg_result=aggregate, + power_dir=tmp_path / "power", + logs_root=tmp_path, + expected_producer_sha="a" * 40, + expected_num_gpus=4, + require_power=True, + expected_cpu_source="acpi", + ) == 1 + assert json.loads((result_dir / "power_validation.json").read_text())["reasons"] == [ + "agentic_gpu_topology_invalid" + ] + verdict = json.loads(aggregate.read_text()) + assert verdict["power_valid"] == 0 + assert "avg_power_w" not in verdict + + def _write_artifacts( tmp_path: Path, *, diff --git a/inferencex-e2e/infx/tests/results/agentic/test_power_lifecycle.py b/inferencex-e2e/infx/tests/results/agentic/test_power_lifecycle.py index 35f16c7bc5..ea9004aa8c 100644 --- a/inferencex-e2e/infx/tests/results/agentic/test_power_lifecycle.py +++ b/inferencex-e2e/infx/tests/results/agentic/test_power_lifecycle.py @@ -235,10 +235,11 @@ def test_nvidia_monitor_preserves_identity_without_requiring_it( assert "Stopped" in result.stdout -def test_multinode_formal_window_wraps_replay_without_local_monitor(tmp_path: Path): +@pytest.mark.parametrize("is_multinode", [False, True]) +def test_srt_formal_window_wraps_replay_without_local_monitor(tmp_path: Path, is_multinode): result = _run_lifecycle( tmp_path, - is_multinode=True, + is_multinode=is_multinode, formal_multinode_power=True, require_power=True, ) diff --git a/inferencex-e2e/infx/tests/results/power/test_process_result.py b/inferencex-e2e/infx/tests/results/power/test_process_result.py index 601bbcc04e..b40940e284 100644 --- a/inferencex-e2e/infx/tests/results/power/test_process_result.py +++ b/inferencex-e2e/infx/tests/results/power/test_process_result.py @@ -1757,3 +1757,32 @@ def test_agentic_collector_preserves_archive_when_result_python_is_missing( else: assert "INFERENCEX_RESULTS_PYTHON" in capfd.readouterr().err assert aggregate == raw_result + + +@pytest.mark.parametrize('cpu_present,strict', [(True, True), (False, True), (False, False)]) +def test_native_fixed_sequence_uses_srt_cpu_package_without_multinode_metadata( + tmp_path, monkeypatch, single_node_env_vars, cpu_present, strict +): + from infx.tests.results.power import test_aggregate_power_multinode as fixture + + monkeypatch.setattr(fixture, 'DEVICES', tuple( + ('node-d', index, 'agg', None, 300.0) for index in range(4) + )) + pkg = fixture.build_package(tmp_path, bench_extra=TestMultinodePower.BENCH_EXTRA) + if cpu_present: + fixture.add_cpu_package(pkg, fixture._cpu_rows(hosts=('node-d',))) + env = { + **single_node_env_vars, 'TP': '4', 'GPU_COUNT': '4', + 'POWER_ARTIFACT_DIR': 'LOGS/power', 'POWER_RESULT_ROOT': 'LOGS', + 'POWER_PRODUCER_SHA': PRODUCER_SHA, 'POWER_EXPECTED_CPU_SOURCE': 'acpi', + 'REQUIRE_POWER': str(int(strict)), + } + result = run_script(tmp_path, env, json.loads(pkg.original_result.read_text())) + assert result.returncode == int(strict and not cpu_present), result.stderr + aggregate = json.loads((tmp_path / 'agg_benchmark_result.json').read_text()) + assert aggregate['is_multinode'] is False + assert aggregate['tp'] == 4 + assert aggregate['power_valid'] == 1 + assert aggregate['cpu_power_valid'] == int(cpu_present) + if cpu_present: + assert aggregate['avg_total_cpu_power_w'] == 1000 diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_fixed_sequence_client.py b/inferencex-e2e/infx/tests/srt_slurm/test_fixed_sequence_client.py index 2704393c5a..6b73a61a94 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_fixed_sequence_client.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_fixed_sequence_client.py @@ -9,6 +9,65 @@ import pytest +def _run_native_fixed_client(tmp_path, script): + """Run the real wrapper and window writer with only external clients stubbed.""" + root = Path(__file__).resolve().parents[3] + binaries = tmp_path / "bin" + binaries.mkdir() + logs = tmp_path / "logs" + windows = logs / "power/windows" + windows.mkdir(parents=True) + scripts = { + "python3": ( + f"#!{sys.executable}\n" + "import json, os, sys\nfrom pathlib import Path\n" + "if sys.argv[1:3] == ['-m', 'infx.bench_serving.benchmark_serving']:\n" + " result = Path(sys.argv[sys.argv.index('--result-dir') + 1])\n" + " result /= sys.argv[sys.argv.index('--result-filename') + 1]\n" + " result.write_text(json.dumps({'benchmark_start_time_unix': 1000.0, " + "'benchmark_end_time_unix': 1060.0, 'duration': 60.0}))\n" + "else:\n" + f" os.execv({sys.executable!r}, [{sys.executable!r}, *sys.argv[1:]])\n" + ), + "pip3": "#!/bin/sh\nexit 0\n", + "nvidia-smi": '#!/bin/sh\nprintf called > "$MONITOR_CALLED_FILE"\n', + } + for name, source in scripts.items(): + binary = binaries / name + binary.write_text(source) + binary.chmod(0o755) + env = { + "PATH": f"{binaries}{os.pathsep}/usr/bin:/bin", + "PYTHONPATH": str(root), + "PYTHONPYCACHEPREFIX": str(tmp_path / "pycache"), + "MONITOR_CALLED_FILE": str(tmp_path / "monitor-called"), + "MODEL": "test/model", "CONC": "4", "ISL": "128", "OSL": "64", + "RANDOM_RANGE_RATIO": "0.5", "RESULT_FILENAME": "native-point", + "RESULT_DIR": str(logs), "SRT_MEASUREMENT_WINDOW_DIR": str(windows), + "SRT_FRONTEND_HOST": "router", "SRT_FRONTEND_PORT": "8123", + "RUN_EVAL": "false", "EVAL_ONLY": "false", "GPU_MONITOR_INTERVAL": "3", + "USE_CHAT_TEMPLATE": "false", "FRAMEWORK": "sglang", "IS_AGENTIC": "0", + } + return subprocess.run( + ["bash", str(script)], env=env, cwd=tmp_path, capture_output=True, text=True + ) + + +def test_native_fixed_client_publishes_formal_window_without_local_sampler(tmp_path): + script = Path(__file__).resolve().parents[3] / "benchmarks/single_node/srt_fixed_sequence.sh" + completed = _run_native_fixed_client(tmp_path, script) + assert completed.returncode == 0, completed.stderr + window = json.loads((tmp_path / "logs/power/windows/native-point.json").read_text()) + assert window == { + "schema_version": 1, "benchmark_type": "custom", "clock_source": "head_node_unix_clock", + "status": "completed", "reason": None, "result_path": "native-point.json", + "concurrency": 4, "benchmark_start_time_unix": 1000.0, + "benchmark_end_time_unix": 1060.0, "duration": 60.0, + } + assert not (tmp_path / "monitor-called").exists() + assert not (tmp_path / "logs/gpu_metrics.csv").exists() + + @pytest.mark.parametrize("explicit", [True, False]) def test_client_model_discovery_and_explicit_request_count(tmp_path, explicit): binaries = tmp_path / "bin" diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 2bad1e01f8..77e4219aee 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9277,3 +9277,61 @@ - "Collect ACPI CPU power on Qwen3.5 NVL72 4P1D and 8P1D recipes, including noprof DCGM collection on 8P1D, with the pinned v2.40.2 CPU exporter. Place the 8P1D frontend, benchmark client, and infra on the reserved head so power and benchmark timestamps share its clock; retain 13 allocated nodes and 12 GPU workers. Require valid power for all selected Qwen3.5 fixed-sequence points." - "During strict power qualification, require the CPU source declared by each selected recipe; reject missing or invalid CPU totals without changing the independent GPU verdict. Preserve best-effort behavior when no CPU source is declared." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3296 + +- config-keys: + - dsr1-fp4-gb200-dynamo-trt + - dsr1-fp8-gb200-dynamo-trt + - dsr1-fp8-gb200-dynamo-sglang + - dsr1-fp8-gb300-dynamo-sglang + - dsr1-fp4-gb200-dynamo-sglang + - dsr1-fp4-gb300-dynamo-trt + - dsr1-fp4-gb300-dynamo-sglang + - dsr1-fp8-gb300-dynamo-trt + - qwen3.5-fp8-gb200-dynamo-sglang + - qwen3.5-fp4-gb300-dynamo-sglang + - qwen3.5-fp4-gb300-dynamo-trt + - qwen3.5-fp4-gb300-dynamo-trt-mtp + - qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-pp-pareto + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg-pareto + - dsv4-fp4-gb300-dynamo-vllm-agentic + - qwen3.5-fp4-gb200-dynamo-sglang-agentic-mtp + - minimaxm3-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp + - minimaxm3-fp4-gb200-dynamo-vllm-agentic-disagg-mtp + - minimaxm3-fp4-gb200-dynamo-trt-agentic-agg-mtp + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-agg + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp2-disagg + - kimik3-fp4-gb200-dynamo-vllm-agentic + - kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg + - kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg + - dsv4-fp4-gb300-dynamo-trt-agentx + - kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2 + - dsv4-fp4-gb300-dynamo-sglang-agentic-agg + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + - qwen3.5-fp8-gb300-dynamo-sglang + - glm5.2-fp4-gb200-dynamo-sglang-agentic-agg + - glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg + - glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp + - glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp-agg + - glm5.2-fp4-gb300-dynamo-sglang-agentic-agg + - glm5.2-fp4-gb300-dynamo-sglang-agentic-disagg + - glm5.2-fp4-gb300-dynamo-trt-agentic-disagg-mtp + - qwen3.5-fp8-gb200-dynamo-sglang-mtp + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - qwen3.5-fp8-gb300-dynamo-sglang-mtp + - dsv41flash-fp4-gb200-vllm-agentic-dspark + - dsv41flash-fp4-gb200-sglang-agentic-dspark + - dsv41flash-fp4-gb300-vllm-agentic-dspark + - dsv41flash-fp4-gb300-sglang-agentic-dspark + description: + - "Enable noprof GPU and ACPI CPU collection by default for GB200/GB300 benchmark recipes, including native single-node and AgentX, with cpu-power-exporter v2.40.2. Preserve independent GPU/CPU validity and existing strict qualification; eval-only jobs omit collection." + - "Place benchmark clients and telemetry on the head clock without changing worker or allocation counts. Native single-node results retain their real topology and power audit. Placement changes require fresh performance results." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3670 From 64e7e77a87caacd93c2836607b96c9dbb812dfa7 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Fri, 2 Oct 2026 13:50:53 -0700 Subject: [PATCH 2/5] fix: keep optional power staging failures nonfatal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：可选功耗产物暂存失败不再导致基准测试失败。 --- .../multi_node/srt-slurm-recipes/RECIPES.md | 3 +- .../srt-slurm-recipes/RECIPES_zh.md | 3 +- .../infx/launch/drivers/srt/collect.py | 9 +- .../tests/launch/test_nvl72_power_defaults.py | 105 ++++++++---------- .../infx/tests/launch/test_srt_driver.py | 55 +++++++++ 5 files changed, 113 insertions(+), 62 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md index 42c3c5bb73..3b271fe9e8 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES.md @@ -70,7 +70,8 @@ The initial migration also resolves compatibility issues that `srtctl migrate` c - SGLang Model Gateway recipes use `frontend.type: sglang-router`; in v2.36.0, `sglang` selects a direct worker without a router. - Duplicate YAML keys retain the value selected by the former PyYAML loader. -- DCGM telemetry uses `collect_interval_ms: 1000` instead of `provider` and `default_frequency`. The collector derives its shutdown budget; an explicit ten-second budget is too short for the current validator. Dedicated discovery-service placement is preserved from the original recipes. The pinned upstream runtime rejects telemetry with dedicated infrastructure nodes; this remains a power compatibility blocker rather than changing the original topology to satisfy validation. H200 custom recipes declare a default concurrency that the launcher replaces before submission. +- DCGM telemetry uses `collect_interval_ms: 1000` instead of `provider` and `default_frequency`. The collector derives its shutdown budget; an explicit ten-second budget is too short for the current validator. H200 custom recipes declare a default concurrency that the launcher replaces before submission. +- Active GB200/GB300 recipes colocate the benchmark client and telemetry collector on the allocation's head: dedicated discovery/client reservations become dedicated frontend heads, and affected DSV4 frontend/client placements move from decode nodes to head. Allocation and GPU-worker counts stay unchanged; changed host contention requires fresh hardware performance results. - DeepSeek-V4 vLLM benchmarks use the supported `custom_tokenizer` loader. Retired `warmup_req_rate: inf` fields are removed; the current upstream client uses its fixed warmup rate of 250 requests per second. - The power reader accepts both generations of samples CSV while validating utilization values and continuing to compute board energy from watts. - Post-eval selection uses native `post_eval.command` and `post_eval.passthrough_env` with [`srt_eval.sh`](../srt_eval.sh). TRT AgentX recipes declare their existing Dynamo fork with `dynamo.source.git`; launchers no longer rewrite the srt-slurm source. diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md index 41d431845f..94fa99a59f 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/RECIPES_zh.md @@ -70,7 +70,8 @@ python -m infx.matrix.generate full-sweep \ - SGLang Model Gateway 配置使用 `frontend.type: sglang-router`；在 v2.36.0 中，`sglang` 表示不经过路由器的独立工作进程。 - 对重复的 YAML 键，保留原 PyYAML 加载器实际采用的值。 -- DCGM 遥测使用 `collect_interval_ms: 1000`，替代 `provider` 和 `default_frequency`。采集器自动推导退出等待时间；原先显式设置的十秒不满足当前校验要求。保留原配置中服务发现进程的专用节点部署方式。固定的上游版本不支持在专用基础设施节点上启用遥测；该功耗兼容性问题仍待解决，不通过改变原有拓扑来绕过校验。H200 自定义配置声明默认并发数，提交前由启动器替换。 +- DCGM 遥测使用 `collect_interval_ms: 1000`，替代 `provider` 和 `default_frequency`。采集器自动推导退出等待时间；原先显式设置的十秒不满足当前校验要求。H200 自定义配置声明默认并发数，提交前由启动器替换。 +- 当前启用的 GB200/GB300 配方将 benchmark 客户端与遥测采集器放在分配的 head 节点：原本为服务发现或客户端预留的节点改为专用 frontend/head，相关 DSV4 配方的 frontend/客户端也从 decode 节点移至 head。总分配节点数与 GPU worker 节点数不变；主机资源竞争发生变化，仍需新的硬件性能结果验证。 - DeepSeek-V4 vLLM 基准测试使用受支持的 `custom_tokenizer` 加载器。删除已废弃的 `warmup_req_rate: inf` 字段；当前上游客户端的预热速率固定为每秒 250 个请求。 - 功耗读取器兼容两代 samples CSV，校验利用率字段，并继续根据瓦特数计算 GPU 板级能耗。 - 评估选择通过原生 `post_eval.command` 和 `post_eval.passthrough_env` 调用 [`srt_eval.sh`](../srt_eval.sh)。TRT AgentX 配置通过 `dynamo.source.git` 声明原有的 Dynamo 分支仓库，启动器不再改写 srt-slurm 源码。 diff --git a/inferencex-e2e/infx/launch/drivers/srt/collect.py b/inferencex-e2e/infx/launch/drivers/srt/collect.py index 973bd2c405..a7c8c84794 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/collect.py +++ b/inferencex-e2e/infx/launch/drivers/srt/collect.py @@ -63,14 +63,15 @@ def finish_single_node( logs = output / "logs" if power is not None and power.dcgm: power_dir = logs / "power" - power_dir.mkdir(parents=True, exist_ok=True) try: + power_dir.mkdir(parents=True, exist_ok=True) for name in (EXPORTER_PROVENANCE, "power-producer-sha.txt"): shutil.copyfile(run.workspace / name, power_dir / name) shutil.copytree(logs, run.workspace / "LOGS", dirs_exist_ok=True) except OSError as error: - print(f"ERROR: failed to stage power artifacts: {error}", file=sys.stderr) - rc = 1 + level = "ERROR" if power.require_power else "WARNING" + print(f"{level}: failed to stage power artifacts: {error}", file=sys.stderr) + rc = int(power.require_power) bundle_server_logs(output, run.workspace / SINGLE_NODE_LOGS) result = logs / f"{run.request.result_filename}.json" for artifact in [result, *sorted(logs.glob("gpu_metrics*"))]: @@ -92,7 +93,7 @@ def finish_single_node( def check_single_node( run: SrtRun, logs: Path, power: PowerDecision | None = None, producer_sha: str = "" ) -> int: - """Fail unless each requested eval succeeded and the benchmark result exists. + """Validate requested evals, benchmark output and AgentX power requirements. srt-slurm treats a failed post-benchmark eval as non-fatal; InferenceX does not. """ diff --git a/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py b/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py index 7b0481dd22..559c6af38c 100644 --- a/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py +++ b/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py @@ -7,8 +7,9 @@ import pytest import yaml -from infx.clusters import load_inventory +from infx.clusters.slurm import PowerTelemetrySettings from infx.launch.drivers.srt.power import resolve_power, telemetry_arguments +from infx.launch.drivers.srt.recipe import recipe_mirror_path from infx.launch.policy import LaunchPath from infx.launch.request import LaunchRequest @@ -19,6 +20,11 @@ from srtctl.core.schema import SrtConfig +@pytest.fixture(params=["acpi", "dcgm"]) +def settings(request): + return PowerTelemetrySettings(dcgm_port=19317, cpu_port=19318, cpu_source=request.param) + + @pytest.mark.parametrize("cluster", ["gb200-nv", "gb300-nv"]) @pytest.mark.parametrize("agentic,framework,path", [ (False, "dynamo-trt", LaunchPath.SRT_MULTI), @@ -26,80 +32,53 @@ (True, "dynamo-vllm", LaunchPath.SRT_MULTI), (True, "vllm", LaunchPath.SRT_SINGLE), ]) -def test_new_models_collect_without_recipe_or_model_allowlists(cluster, agentic, framework, path): - settings = load_inventory(yaml.safe_load((ROOT / "configs/runners.yaml").read_text())).clusters[ - cluster - ].scheduler_settings.srt_slurm +def test_new_models_collect_without_recipe_or_model_allowlists(settings, cluster, agentic, framework, path): request = LaunchRequest.from_env({ "RUNNER_NAME": "fixture_0", "IS_AGENTIC": str(int(agentic)), "MODEL_PREFIX": "future-model", "FRAMEWORK": framework, }) - decision = resolve_power(cluster, path, request, settings=settings.power_telemetry) + decision = resolve_power(cluster, path, request, settings=settings) assert decision.dcgm and decision.agentx is agentic - assert decision.expected_cpu_source == "acpi" + assert decision.expected_cpu_source == settings.cpu_source assert not decision.require_power - assert settings.env["CPU_POWER_EXPORTER_RELEASE"] == "v2.40.2" evaluation = LaunchRequest.from_env({**request.env, "EVAL_ONLY": "true"}) - assert not resolve_power(cluster, path, evaluation, settings=settings.power_telemetry).dcgm + assert not resolve_power(cluster, path, evaluation, settings=settings).dcgm @pytest.mark.parametrize("variant_file", [False, True]) -def test_native_overrides_materialize_cpu_and_noprof_without_rewriting_workload(variant_file): - settings = load_inventory(yaml.safe_load((ROOT / "configs/runners.yaml").read_text())).clusters[ - "gb300-nv" - ].scheduler_settings.srt_slurm.power_telemetry - recipe = yaml.safe_load((ROOT / ( - "benchmarks/multi_node/srt-slurm-recipes/qwen3.5/sglang/gb300-fp8/8k1k/" - "disagg-8p1d-dep4-dep16-stp.yaml" - )).read_text()) - recipe.pop("telemetry") +@pytest.mark.parametrize("dedicated_frontend", [False, True]) +def test_native_overrides_preserve_workload_and_worker_budget(settings, variant_file, dedicated_frontend, monkeypatch): + from srtctl.cli.submit import planned_total_nodes + from srtctl.core.runtime import Nodes + + recipe = { + "schema": 2, "name": "fixture", "engine": "sglang", + "model": {"path": "hf:test/model", "container": "test:tag", "precision": "fp8"}, + "resources": {"gpus_per_node": 4}, + "frontend": {"type": "sglang-router", "dedicated_node": dedicated_frontend}, + "roles": {"prefill": {"nodes": 2, "workers": 2}, "decode": {"nodes": 1, "workers": 1}}, + "benchmark": {"type": "custom", "command": "python /configs/client.py", "concurrencies": [1]}, + } original = copy.deepcopy(recipe) + before = SrtConfig.Schema().load(resolve_config_with_defaults(original, None)) document = {"base": recipe, "override_point": {"benchmark": {"concurrencies": [1]}}} if variant_file else recipe arguments = telemetry_arguments(settings, [2048, 4096]) apply_overrides_to_recipe(document, parse_overrides(arguments[1::2], None)) effective = generate_override_configs(document, selector="override_point")[0][1] if variant_file else document config = SrtConfig.Schema().load(resolve_config_with_defaults(effective, None)) assert config.telemetry.enabled - assert config.telemetry.cpu_power_exporter.source == "acpi" - assert config.telemetry.dcgm_exporter.port == 19401 + assert config.telemetry.cpu_power_exporter.source == settings.cpu_source + assert config.telemetry.dcgm_exporter.port == settings.dcgm_port + assert config.telemetry.cpu_power_exporter.port == settings.cpu_port assert "dcgm-counters-noprof.csv" in config.telemetry.dcgm_exporter.command assert config.benchmark.get_concurrency_list() == [2048, 4096] assert effective["roles"] == original["roles"] assert effective["benchmark"]["command"] == original["benchmark"]["command"] - - -def test_existing_strict_agentx_recipe_remains_strict(): - settings = load_inventory(yaml.safe_load((ROOT / "configs/runners.yaml").read_text())).clusters[ - "gb200-nv" - ].scheduler_settings.srt_slurm.power_telemetry - request = LaunchRequest.from_env({ - "RUNNER_NAME": "fixture_0", "GITHUB_WORKSPACE": str(ROOT), - "IS_AGENTIC": "1", "MODEL_PREFIX": "kimik3", "PRECISION": "fp4", - "FRAMEWORK": "dynamo-vllm", - "CONFIG_FILE": "recipes/kimik3/vllm/gb200-fp4/agentx/agg-dcp16-dspark4-maxseq2-mooncake.yaml", - }) - assert resolve_power("gb200-nv", LaunchPath.SRT_MULTI, request, settings=settings).require_power - - -@pytest.mark.parametrize('relative,selector,total,workers', [ - ('dsv4/vllm/gb300-fp4/agentx/disagg-1p4d-dep4-tp8-c4-mtp.yaml', None, 9, 9), - ('dsv4/trtllm/gb300-fp4/agentx/disagg-variants.yaml', 'override_1p4d_dep4_tep8_c4_b1', 9, 9), - ('glm5.2/trtllm/gb300-fp4/agentx/disagg-variants.yaml', 'override_1p1d_tp8_c1_b1_mtp5', 4, 3), -]) -def test_client_and_collector_share_head_without_changing_worker_budget(monkeypatch, relative, selector, total, workers): - from srtctl.cli.submit import planned_total_nodes - from srtctl.core.runtime import Nodes - settings = load_inventory(yaml.safe_load((ROOT / 'configs/runners.yaml').read_text())).clusters[ - 'gb300-nv' - ].scheduler_settings.srt_slurm.power_telemetry - raw = yaml.safe_load((ROOT / 'benchmarks/multi_node/srt-slurm-recipes' / relative).read_text()) - apply_overrides_to_recipe(raw, parse_overrides(telemetry_arguments(settings, [4])[1::2], None)) - effective = generate_override_configs(raw, selector)[0][1] if selector else raw - config = SrtConfig.Schema().load(resolve_config_with_defaults(effective, None)) - assert planned_total_nodes(config) == total - assert config.engine_node_count == workers - assert not config.pool_services - monkeypatch.setattr('srtctl.core.runtime.get_slurm_nodelist', lambda: [f'node-{i}' for i in range(total)]) + assert planned_total_nodes(config) == planned_total_nodes(before) + assert config.engine_node_count == before.engine_node_count + monkeypatch.setattr('srtctl.core.runtime.get_slurm_nodelist', lambda: [ + f'node-{i}' for i in range(planned_total_nodes(config)) + ]) monkeypatch.setattr('srtctl.core.runtime.get_slurm_het_nodelists', lambda: None) nodes = Nodes.from_slurm( frontend_dedicated_node=config.frontend.dedicated_node, @@ -107,8 +86,22 @@ def test_client_and_collector_share_head_without_changing_worker_budget(monkeypa etcd_nats_dedicated_node=config.infra.etcd_nats_dedicated_node, colocate_dedicated_nodes=config.benchmark.colocate_with_frontend, ) - assert nodes.head == nodes.bench == 'node-0' - assert len(nodes.compute) == len(nodes.worker) == workers + assert nodes.head == nodes.bench + assert len(nodes.compute) == len(nodes.worker) == config.engine_node_count + + +def test_existing_strict_agentx_recipe_remains_strict(settings, tmp_path): + config_file = "recipes/kimik3/vllm/gb200-fp4/agentx/fixture.yaml" + mirror = recipe_mirror_path(tmp_path, config_file) + mirror.parent.mkdir(parents=True) + mirror.write_text(yaml.safe_dump({"telemetry": {"enabled": True, "dcgm_exporter": {}}})) + request = LaunchRequest.from_env({ + "RUNNER_NAME": "fixture_0", "GITHUB_WORKSPACE": str(tmp_path), + "IS_AGENTIC": "1", "MODEL_PREFIX": "kimik3", "PRECISION": "fp4", + "FRAMEWORK": "dynamo-vllm", + "CONFIG_FILE": config_file, + }) + assert resolve_power("gb200-nv", LaunchPath.SRT_MULTI, request, settings=settings).require_power def test_multinode_power_bundle_retains_effective_cluster_overrides(tmp_path): diff --git a/inferencex-e2e/infx/tests/launch/test_srt_driver.py b/inferencex-e2e/infx/tests/launch/test_srt_driver.py index ed1f0b3a71..f57f76d60f 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_driver.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_driver.py @@ -173,6 +173,61 @@ def test_single_node_failed_allocation_fails_the_launch(harness): assert (harness.workspace / "point-identity.json").is_file() +@pytest.mark.parametrize( + ("failure", "require_power", "expected_rc"), + [ + ("power_dir", False, 0), ("power_dir", True, 1), + ("provenance", False, 0), ("provenance", True, 1), + ("power_tree", False, 0), ("power_tree", True, 1), + ("result", False, 1), ("agentic", False, 1), + ], +) +def test_native_cleanup_keeps_optional_power_failures_best_effort( + tmp_path, failure, require_power, expected_rc +): + from infx.launch.backends.base import Job + from infx.launch.drivers.srt.collect import finish_single_node + from infx.launch.drivers.srt.power import PowerDecision + from infx.launch.drivers.srt.submit import Submitted + from infx.launch.lifecycle import Lifecycle + + workspace = tmp_path / "workspace" + workspace.mkdir() + output = tmp_path / "output" + logs = output / "logs" + (logs / "agentic").mkdir(parents=True) + (logs / "point.json").write_text('{"completed": 2}') + (logs / "agentic/replay.json").write_text('{"requests": 2}') + (workspace / "power-producer-sha.txt").write_text("a" * 40) + if failure != "provenance": + (workspace / "exporter-image.sha256").write_text("exporter digest") + if failure == "power_dir": + (logs / "power").write_text("blocks directory creation") + elif failure == "power_tree": + (workspace / "LOGS").write_text("blocks tree copy") + elif failure == "result": + (workspace / "point.json").symlink_to(tmp_path / "missing/point.json") + elif failure == "agentic": + (workspace / "results").write_text("blocks AgentX copy") + run = SimpleNamespace( + workspace=workspace, + request=SimpleNamespace(result_filename="point", require_power=False), + backend=SimpleNamespace(cancel=lambda job: None, fetch_outputs=lambda *args: output), + ) + submitted = Submitted(manifest=tmp_path / "submission.json", job=Job("42")) + with Lifecycle() as life: + life.callback( + finish_single_node, run, submitted, tmp_path / "fetched", + PowerDecision(dcgm=True, agentx=True, require_power=require_power), + ) + + assert life.returncode == expected_rc + if failure != "result": + assert json.loads((workspace / "point.json").read_text()) == {"completed": 2} + if failure != "agentic": + assert json.loads((workspace / "results/replay.json").read_text()) == {"requests": 2} + + @pytest.mark.parametrize( ("gpu_count", "has_cpu", "require_power"), [(2, True, True), (4, True, True), (4, False, True), (4, False, False)], From 61fded28bd363a77a9d934ee5afb4bc4a428002b Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Fri, 2 Oct 2026 15:43:27 -0700 Subject: [PATCH 3/5] fix: preserve power samples when provenance staging fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：来源信息暂存失败时仍保留原始功耗采样。 --- .../dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml | 4 ++-- .../benchmarks/single_node/srt_fixed_sequence.sh | 3 ++- inferencex-e2e/docs/results-and-ingestion.md | 6 +++++- inferencex-e2e/docs/results-and-ingestion_zh.md | 5 ++++- inferencex-e2e/infx/launch/drivers/srt/collect.py | 5 +++++ inferencex-e2e/infx/launch/drivers/srt/power.py | 2 +- inferencex-e2e/infx/tests/launch/test_srt_driver.py | 7 ++++++- 7 files changed, 25 insertions(+), 7 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml index 6df09c5cb7..59c1fb4c13 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/vllm/gb300-fp4/agentx/disagg-variants.yaml @@ -148,7 +148,7 @@ base: # Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-1p6d-dep4-tp4-stp.yaml. # Topology is identical (1 prefill DEP=4 + 6 decode TP=4, 28 GPUs across 7 -# GB300 nodes + 1 dedicated NATS/etcd infra node) so we can compare against +# GB300 nodes + 1 dedicated frontend/client head node) so we can compare against # the fixed-seq-len 1p6d baseline at the same concurrency point (192). # # Divergence vs the 8k1k sibling: @@ -182,7 +182,7 @@ override_1p6d_dep4_tp4: # Agentic-coding variant of dsv4/vllm/gb300-fp4/8k1k/disagg-4p1d-dep4-dep8-24-c4096-stp.yaml. # Max-throughput shape: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 GB300 -# nodes (4P + 2D = 24 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra +# nodes (4P + 2D = 24 GPUs at 4 GPUs/node) plus a dedicated frontend/client head # node. Sized for concurrency 4096 with deep_gemm_mega_moe on both workers. # # Divergence vs the 8k1k sibling: diff --git a/inferencex-e2e/benchmarks/single_node/srt_fixed_sequence.sh b/inferencex-e2e/benchmarks/single_node/srt_fixed_sequence.sh index d782280875..cd1a308d7c 100644 --- a/inferencex-e2e/benchmarks/single_node/srt_fixed_sequence.sh +++ b/inferencex-e2e/benchmarks/single_node/srt_fixed_sequence.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# SRT owns the server lifecycle; retain the existing InferenceX client and sampler. +# SRT owns the server lifecycle and, when telemetry is enabled, the power sampler. +# Keep the InferenceX client and use its local sampler only without native telemetry. set -eo pipefail source "$(dirname "${BASH_SOURCE[0]}")/../benchmark_lib.sh" --validation-only check_env_vars MODEL CONC ISL OSL RANDOM_RANGE_RATIO RESULT_FILENAME RESULT_DIR \ diff --git a/inferencex-e2e/docs/results-and-ingestion.md b/inferencex-e2e/docs/results-and-ingestion.md index c72d286d85..331ba18e21 100644 --- a/inferencex-e2e/docs/results-and-ingestion.md +++ b/inferencex-e2e/docs/results-and-ingestion.md @@ -494,7 +494,11 @@ The CPU verdict is independent of `power_valid`: it borrows only the bound formal window and the worker-host topology from the GPU package, and no GPU verdict reaches it, so an unpinned producer or failed GPU coverage withholds GPU energy while `cpu_power_valid` still judges the CPU samples on their own. Any CPU-leg failure records `cpu_power_valid: 0` with no CPU -keys and leaves every GPU field unchanged. With `REQUIRE_POWER=1`, a recipe-declared CPU source must be valid; ACPI requires complete Grace socket or module totals, and missing CPU artifacts produce an independent invalid CPU audit. GPU-only recipes retain their existing behavior. Reason codes: +keys and leaves every GPU field unchanged. When power is required by `REQUIRE_POWER=1` or a strict +lane policy, a configured CPU source must be valid; ACPI requires complete Grace socket or module +totals, and missing CPU artifacts produce an independent invalid CPU audit. On `gb200-nv` and +`gb300-nv`, the cluster declares `acpi`, so every strict point requires a valid CPU audit. +GPU-only recipes on other clusters retain their existing behavior. Reason codes: `cpu_artifacts_missing`, `cpu_sensor_source_mismatch`, `cpu_samples_missing`, `cpu_samples_header_mismatch`, `cpu_samples_malformed`, `cpu_manifest_invalid`, `cpu_socket_count_mismatch`, `cpu_sensor_kind_mixed`, diff --git a/inferencex-e2e/docs/results-and-ingestion_zh.md b/inferencex-e2e/docs/results-and-ingestion_zh.md index 1499ec0b99..bcd7b3dc65 100644 --- a/inferencex-e2e/docs/results-and-ingestion_zh.md +++ b/inferencex-e2e/docs/results-and-ingestion_zh.md @@ -473,7 +473,10 @@ HBM、LPDDR5X 及稳压损耗）；`Grace Power Socket N` 或 `CPU:cpuSidePow 该测量环节尽力而为，其结论与 `power_valid` 相互独立：它只从 GPU 产物包借用已绑定的正式测量窗口和 worker 主机拓扑，GPU 侧的任何结论都不会传导过来，因此 producer 固定版本校验失败或 GPU 覆盖不足只会 使 GPU 能耗不予发布，`cpu_power_valid` 仍按 CPU 采样自身给出结论。CPU 侧的任何失败都会记录 -`cpu_power_valid: 0` 且不输出 CPU 字段，所有 GPU 字段保持不变。设置 `REQUIRE_POWER=1` 时，配方声明的 CPU 来源必须有效；ACPI 必须提供完整的 Grace socket 或模块总功耗，缺失 CPU 产物会生成独立的无效 CPU 审计。仅采集 GPU 的配方保留原有行为。 +`cpu_power_valid: 0` 且不输出 CPU 字段，所有 GPU 字段保持不变。当 `REQUIRE_POWER=1` 或严格的 +执行路径策略要求功耗有效时，已配置的 CPU 来源也必须有效；ACPI 必须提供完整的 Grace socket 或 +模块总功耗，缺失 CPU 产物会生成独立的无效 CPU 审计。`gb200-nv` 和 `gb300-nv` 在集群级声明 +`acpi`，因此每个严格校验的数据点都必须有有效的 CPU 审计。其他集群中仅采集 GPU 的配方保留原有行为。 原因码包括 `cpu_artifacts_missing`、`cpu_sensor_source_mismatch`、`cpu_samples_missing`、`cpu_samples_header_mismatch`、`cpu_samples_malformed`、 `cpu_manifest_invalid`、`cpu_socket_count_mismatch`、`cpu_sensor_kind_mixed`、 `cpu_sample_gap_exceeded`、`cpu_window_not_bracketed` 以及 `cpu_window_unavailable`（没有已完成的窗口 diff --git a/inferencex-e2e/infx/launch/drivers/srt/collect.py b/inferencex-e2e/infx/launch/drivers/srt/collect.py index a7c8c84794..5f1a361681 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/collect.py +++ b/inferencex-e2e/infx/launch/drivers/srt/collect.py @@ -67,6 +67,11 @@ def finish_single_node( power_dir.mkdir(parents=True, exist_ok=True) for name in (EXPORTER_PROVENANCE, "power-producer-sha.txt"): shutil.copyfile(run.workspace / name, power_dir / name) + except OSError as error: + level = "ERROR" if power.require_power else "WARNING" + print(f"{level}: failed to stage power provenance: {error}", file=sys.stderr) + rc = int(power.require_power) + try: shutil.copytree(logs, run.workspace / "LOGS", dirs_exist_ok=True) except OSError as error: level = "ERROR" if power.require_power else "WARNING" diff --git a/inferencex-e2e/infx/launch/drivers/srt/power.py b/inferencex-e2e/infx/launch/drivers/srt/power.py index 4430f1a195..49c1e975f0 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/power.py +++ b/inferencex-e2e/infx/launch/drivers/srt/power.py @@ -196,7 +196,7 @@ def resolve_power( *, settings: PowerTelemetrySettings | None = None, ) -> PowerDecision: - """Use cluster defaults when configured; otherwise inspect the mirrored recipe opt-in.""" + """Resolve recipe power policy, then apply cluster defaults while preserving strictness.""" lane = POWER_LANES.get((cluster_id, path)) if settings is not None and request.eval_only: return NO_POWER diff --git a/inferencex-e2e/infx/tests/launch/test_srt_driver.py b/inferencex-e2e/infx/tests/launch/test_srt_driver.py index f57f76d60f..42e169a5b4 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_driver.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_driver.py @@ -203,7 +203,10 @@ def test_native_cleanup_keeps_optional_power_failures_best_effort( (workspace / "exporter-image.sha256").write_text("exporter digest") if failure == "power_dir": (logs / "power").write_text("blocks directory creation") - elif failure == "power_tree": + else: + (logs / "power").mkdir() + (logs / "power/samples.csv").write_text("timestamp,power_w\n1,120\n") + if failure == "power_tree": (workspace / "LOGS").write_text("blocks tree copy") elif failure == "result": (workspace / "point.json").symlink_to(tmp_path / "missing/point.json") @@ -226,6 +229,8 @@ def test_native_cleanup_keeps_optional_power_failures_best_effort( assert json.loads((workspace / "point.json").read_text()) == {"completed": 2} if failure != "agentic": assert json.loads((workspace / "results/replay.json").read_text()) == {"requests": 2} + if failure not in {"power_dir", "power_tree"}: + assert (workspace / "LOGS/power/samples.csv").read_text() == "timestamp,power_w\n1,120\n" @pytest.mark.parametrize( From 7f40a272723caf949efae040deea0937da17a749 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 17 Oct 2026 01:19:30 -0700 Subject: [PATCH 4/5] fix: bind AgentX power policy and require named NVL72 curves --- .github/workflows/e2e-tests.yml | 4 ++-- .github/workflows/run-sweep.yml | 4 ++++ inferencex-e2e/configs/nvidia-master.yaml | 18 +++++++++++---- .../configs/srt-recipes/telemetry-dcgm.yaml | 3 --- inferencex-e2e/infx/launch/artifacts.py | 3 ++- .../infx/launch/drivers/srt/collect.py | 3 ++- .../infx/launch/drivers/srt/config.py | 5 +++-- .../infx/launch/drivers/srt/power.py | 9 +++++--- inferencex-e2e/infx/matrix/generate.py | 4 ++++ inferencex-e2e/infx/matrix/validation.py | 3 +++ inferencex-e2e/infx/srt_slurm/workload.py | 15 ++++++++----- .../tests/launch/test_launch_artifacts.py | 22 ++++++++++++++----- .../tests/launch/test_nvl72_power_defaults.py | 16 +++++++++++++- .../infx/tests/launch/test_srt_config.py | 2 +- .../infx/tests/launch/test_srt_driver.py | 3 ++- .../matrix/test_generate_sweep_configs.py | 16 ++++++++++++++ .../results/power/test_process_result.py | 2 +- .../infx/tests/srt_slurm/test_generate.py | 2 +- .../infx/tests/srt_slurm/test_workload.py | 22 ++++++++++++++++++- inferencex-e2e/perf-changelog.yaml | 3 ++- 20 files changed, 126 insertions(+), 33 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 556d856b39..6d4daf9b05 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -395,7 +395,7 @@ jobs: secrets: INFERENCEX_OFFICIAL_RO_HF_TOKEN: ${{ secrets.INFERENCEX_OFFICIAL_RO_HF_TOKEN }} with: - require-power: ${{ inputs.require-power }} + require-power: ${{ inputs.require-power || (matrix.config['require-power'] == true && matrix.config['eval-only'] != true) }} config: ${{ toJSON(matrix.config) }} klaud-run: ${{ inputs.klaud-run }} runner: ${{ matrix.config.runner }} @@ -459,7 +459,7 @@ jobs: duration: ${{ inputs.agentx-fast && '1200' || (inputs.duration-override != '' && inputs.duration-override || matrix.config.duration) }} agentx-fast: ${{ inputs.agentx-fast }} run-eval: false - require-power: ${{ inputs.require-power }} + require-power: ${{ inputs.require-power || (matrix.config['require-power'] == true && matrix.config['eval-only'] != true) }} power-producer-sha: ${{ inputs.power-producer-sha }} scenario-type: agentic-coding ref: ${{ needs.get-jobs.outputs.benchmark-ref }} diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index 469a141a1b..36e797951a 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -392,6 +392,7 @@ jobs: dp-attn: ${{ matrix.config.dp-attn }} run-eval: false scenario-type: ${{ matrix.config['scenario-type'] || 'fixed-seq-len' }} + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} duration: ${{ matrix.config.duration || '3600' }} agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }} @@ -419,6 +420,7 @@ jobs: duration: ${{ matrix.config.duration }} run-eval: false scenario-type: agentic-coding + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }} sweep-multi-node-1k1k: @@ -548,6 +550,7 @@ jobs: duration: ${{ matrix.config.duration }} run-eval: false scenario-type: agentic-coding + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }} sweep-multi-node-agentic: @@ -582,6 +585,7 @@ jobs: duration: ${{ matrix.config.duration }} run-eval: false scenario-type: agentic-coding + require-power: ${{ matrix.config['require-power'] == true && matrix.config['eval-only'] != true }} agentx-fast: ${{ contains(github.event.pull_request.labels.*.name, 'agentx-fast') }} sweep-evals: diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index cbf732aadc..f894f3c7b8 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6003,11 +6003,13 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: srt-recipe-dir: dsv4/vllm/gb200-fp4/agentx scenarios: agentic-coding: - - search-space: + - require-power: true + search-space: # Match the B200 TP8 latency points for direct normalized-interactivity # comparison, then use the public vLLM GB200 DEP8 topology for throughput. - spec-decoding: mtp conc-list: [1, 2, 4, 6, 8] + power: true num-nodes: 2 srt-recipe: agg-mtp-variants.yaml:override_tp8 worker: @@ -6017,6 +6019,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: dp-attn: false - spec-decoding: mtp conc-list: [8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80] + power: true num-nodes: 2 srt-recipe: agg-mtp-variants.yaml:override_dep8 worker: @@ -7428,7 +7431,8 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-agg: srt-recipe-dir: glm5.2/sglang/gb200-fp4/agentx scenarios: agentic-coding: - - dram-utilization: 0.80 + - require-power: true + dram-utilization: 0.80 search-space: - spec-decoding: mtp conc-list: [1, 2, 4, 8, 12, 16] @@ -7457,10 +7461,12 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-disagg: srt-recipe-dir: glm5.2/sglang/gb200-fp4/agentx scenarios: agentic-coding: - - dram-utilization: 0.80 + - require-power: true + dram-utilization: 0.80 search-space: - spec-decoding: mtp conc-list: [10, 12] + power: true kv-offloading: dram kv-offload-backend: { name: hicache } srt-recipe: disagg-mtp-variants.yaml:zip_override_mtp_agentx_frontier[0] @@ -7493,11 +7499,13 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: srt-recipe-dir: glm5.2/sglang/gb200-fp4/agentx scenarios: agentic-coding: - - dram-utilization: 0.8 + - require-power: true + dram-utilization: 0.8 search-space: - spec-decoding: mtp conc-list: - 45 + power: true kv-offloading: dram kv-offload-backend: name: hicache @@ -7515,6 +7523,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: - spec-decoding: mtp conc-list: - 48 + power: true kv-offloading: dram kv-offload-backend: name: hicache @@ -7532,6 +7541,7 @@ glm5.2-fp4-gb200-dynamo-sglang-agentic-mtp: - spec-decoding: mtp conc-list: - 128 + power: true kv-offloading: dram kv-offload-backend: name: hicache diff --git a/inferencex-e2e/configs/srt-recipes/telemetry-dcgm.yaml b/inferencex-e2e/configs/srt-recipes/telemetry-dcgm.yaml index f6327f349f..5545c90baa 100644 --- a/inferencex-e2e/configs/srt-recipes/telemetry-dcgm.yaml +++ b/inferencex-e2e/configs/srt-recipes/telemetry-dcgm.yaml @@ -1,10 +1,7 @@ -# Merged under the fragment of a point whose master row sets `power: true`. The launcher -# adds the cluster's power-exporter-port and stages the dcgm-exporter image. telemetry: enabled: true collect_interval_ms: 1000 storage_subdir: power - required: true startup_timeout_seconds: 120 request_timeout_seconds: 2 dcgm_exporter: diff --git a/inferencex-e2e/infx/launch/artifacts.py b/inferencex-e2e/infx/launch/artifacts.py index 08f2bd3729..41218e201b 100644 --- a/inferencex-e2e/infx/launch/artifacts.py +++ b/inferencex-e2e/infx/launch/artifacts.py @@ -153,6 +153,7 @@ def collect_agentic_power_results( concurrencies: Sequence[int], *, results_python: str | None, + require_power: bool, expected_cpu: bool = False, ) -> int: """Stage the AgentX power audit inputs and validate each concurrency's power window. @@ -176,7 +177,7 @@ def collect_agentic_power_results( if step := validate_agentic_power( logs_dir, workspace, result_filename, producer_sha, concurrencies, - results_python=results_python, require_power=True, expected_cpu=expected_cpu, + results_python=results_python, require_power=require_power, expected_cpu=expected_cpu, ): # fmt: skip rc = step return rc diff --git a/inferencex-e2e/infx/launch/drivers/srt/collect.py b/inferencex-e2e/infx/launch/drivers/srt/collect.py index 6f521b9378..74d7885029 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/collect.py +++ b/inferencex-e2e/infx/launch/drivers/srt/collect.py @@ -189,6 +189,7 @@ def stage_logs_on_exit() -> None: infmax, *audit, results_python=python, + require_power=power.require, expected_cpu=power.cpu, ) else: @@ -196,7 +197,7 @@ def stage_logs_on_exit() -> None: logs, *audit, results_python=python, - require_power=request.require_power, + require_power=power.require, expected_cpu=power.cpu, ) if power_rc: diff --git a/inferencex-e2e/infx/launch/drivers/srt/config.py b/inferencex-e2e/infx/launch/drivers/srt/config.py index 837fd25845..79532a1fb7 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/config.py +++ b/inferencex-e2e/infx/launch/drivers/srt/config.py @@ -263,8 +263,9 @@ def binder_inputs( f"cluster {cluster.id!r} declares no srt-slurm.cpu-power-exporter for cpu-power" ) cpu_source = srt.cpu_power_exporter.source + required = "1" if decision.require else "0" if not request.is_agentic: - return port, cpu_source, {} + return port, cpu_source, {"REQUIRE_POWER": required} return ( port, cpu_source, @@ -272,7 +273,7 @@ def binder_inputs( **agentic_client_env(cluster, srt, lane, request), "IS_MULTINODE": str(request.is_multinode).lower(), "ENABLE_AGENTX_POWER": "1" if decision.agentx or decision.adapter else "0", - "REQUIRE_POWER": "1" if decision.require or request.require_power else "0", + "REQUIRE_POWER": required, }, ) diff --git a/inferencex-e2e/infx/launch/drivers/srt/power.py b/inferencex-e2e/infx/launch/drivers/srt/power.py index 34af419b46..7c9a21fd51 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/power.py +++ b/inferencex-e2e/infx/launch/drivers/srt/power.py @@ -153,9 +153,11 @@ def decide_power(cluster_id: str, path: LaunchPath, request: MultiNodeRequest) - Raises ``PowerPolicyError`` for a combination the lane does not allow. """ default = cluster_id in NVL72_POWER_CLUSTERS and path == LaunchPath.SRT_MULTI - if default and request.eval_only: + if request.eval_only: return NO_POWER if not request.power and not default: + if request.require_power: + raise PowerPolicyError("require-power needs an eligible power collector") return NO_POWER lane = POWER_LANES.get((cluster_id, path)) if default: @@ -164,7 +166,8 @@ def decide_power(cluster_id: str, path: LaunchPath, request: MultiNodeRequest) - dcgm=True, agentx=request.is_agentic, adapter=matched.adapter if matched else False, - require=matched.require if request.power and matched else False, + require=request.require_power + or (matched.require if request.power and matched else False), cpu=True, ) if lane is None: @@ -175,7 +178,7 @@ def decide_power(cluster_id: str, path: LaunchPath, request: MultiNodeRequest) - dcgm=True, agentx=rule.agentx, adapter=rule.adapter, - require=rule.require, + require=request.require_power or rule.require, cpu=request.cpu_power, ) message = lane.agentic_error if request.is_agentic and lane.agentic_error else lane.error diff --git a/inferencex-e2e/infx/matrix/generate.py b/inferencex-e2e/infx/matrix/generate.py index 4bea185cb6..ef3a6592bc 100644 --- a/inferencex-e2e/infx/matrix/generate.py +++ b/inferencex-e2e/infx/matrix/generate.py @@ -1113,6 +1113,9 @@ def _agentic_entries( """ is_multinode = config.get(Fields.MULTINODE.value, False) disagg = config.get(Fields.DISAGG.value, False) + require_power = scenario.get(Fields.REQUIRE_POWER.value) + if require_power is None: + require_power = False model_code = config[Fields.MODEL_PREFIX.value] if is_multinode: prefill, decode = multinode_worker_pair(benchmark, disagg) @@ -1197,6 +1200,7 @@ def _agentic_entries( + (f"_spec-{spec_decoding}" if spec_decoding != "none" else "") ) entry.update(srt_recipe_fields(config, benchmark)) + entry[Fields.REQUIRE_POWER.value] = require_power if benchmark.get(Fields.POWER.value): entry[Fields.POWER.value] = True if benchmark.get(Fields.CPU_POWER.value): diff --git a/inferencex-e2e/infx/matrix/validation.py b/inferencex-e2e/infx/matrix/validation.py index 01c118d0bb..6eccc7a7c0 100644 --- a/inferencex-e2e/infx/matrix/validation.py +++ b/inferencex-e2e/infx/matrix/validation.py @@ -335,6 +335,7 @@ class SingleNodeAgenticMatrixEntry(BaseModel): precision: str framework: str runner: str + require_power: bool = Field(default=False, alias=Fields.REQUIRE_POWER.value, strict=True) tp: int pp: int = Field(gt=0, strict=True) dcp_size: int = Field(alias=Fields.DCP_SIZE.value, gt=0, strict=True) @@ -385,6 +386,7 @@ class MultiNodeAgenticMatrixEntry(BaseModel): framework: str spec_decoding: Literal["mtp", "draft_model", "none"] = Field(alias=Fields.SPEC_DECODING.value) runner: str + require_power: bool = Field(default=False, alias=Fields.REQUIRE_POWER.value, strict=True) srt_recipe: str | None = Field(default=None, alias=Fields.SRT_RECIPE.value, min_length=1) eval_srt_recipe: str | None = Field( default=None, alias=Fields.EVAL_SRT_RECIPE.value, min_length=1 @@ -792,6 +794,7 @@ class AgenticCodingConfig(BaseModel): model_config = ConfigDict(extra="forbid", populate_by_name=True) search_space: list[AgenticCodingSearchSpaceEntry] = Field(alias=Fields.SEARCH_SPACE.value) + require_power: bool | None = Field(default=None, alias=Fields.REQUIRE_POWER.value, strict=True) dram_utilization: float | None = Field( default=None, alias=Fields.DRAM_UTILIZATION.value, gt=0, le=1 ) diff --git a/inferencex-e2e/infx/srt_slurm/workload.py b/inferencex-e2e/infx/srt_slurm/workload.py index 95e4ac57c9..a778f87a30 100644 --- a/inferencex-e2e/infx/srt_slurm/workload.py +++ b/inferencex-e2e/infx/srt_slurm/workload.py @@ -137,7 +137,7 @@ def check_fragment(raw: Mapping[str, Any], source: Path, *, agentic: bool, multi keys = ( *_POINT_KEYS, *(("benchmark", "env", name) for name in BOUND_ENV[agentic, multinode]), - *(() if multinode else (("telemetry", "required"),)), + ("telemetry", "required"), ) if "base" in raw: blocks = [(name, block) for name, block in raw.items() if name != "schema"] @@ -337,10 +337,15 @@ def bind_workload( ) if (bound.get("telemetry") or {}).get("enabled") is True: benchmark["concurrencies"] = concurrencies - if not multinode: - required = environment.get("REQUIRE_POWER", "0") in {"1", "true", "TRUE", "yes", "YES"} - bound["telemetry"]["required"] = required - workload["REQUIRE_POWER"] = "1" if required else "0" + required = workload.get("REQUIRE_POWER", environment.get("REQUIRE_POWER", "0")) in { + "1", + "true", + "TRUE", + "yes", + "YES", + } + bound["telemetry"]["required"] = required + workload["REQUIRE_POWER"] = "1" if required else "0" if agentic and not multinode: workload["ENABLE_AGENTX_POWER"] = ( "1" if (bound.get("telemetry") or {}).get("enabled") is True else "0" diff --git a/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py b/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py index b0e34fbef6..6709093760 100644 --- a/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py +++ b/inferencex-e2e/infx/tests/launch/test_launch_artifacts.py @@ -109,14 +109,19 @@ def test_power_collection_records_the_job_status_and_stages_results_either_way(t (source / "point_conc4.json").write_text("{}") assert collect_agentic_power_results( - status, "42", logs, source, workspace, "point", "sha", [4], results_python="true", + status, "42", logs, source, workspace, "point", "sha", [4], + results_python="true", require_power=False, ) == rc # fmt: skip assert (logs / "power" / "native-job-status.txt").read_text() == f"42|{status.raw}\n" assert (workspace / "point_conc4.json").exists() -def test_agentic_collection_requires_declared_cpu_and_keeps_gpu_valid(tmp_path): +@pytest.mark.parametrize(("require_power", "expected_rc"), [(False, 0), (True, 1)]) +@pytest.mark.parametrize("missing_gpu", [False, True]) +def test_agentic_collection_enforces_only_required_invalid_power( + tmp_path, require_power, expected_rc, missing_gpu +): from infx.tests.results.power.test_aggregate_power_multinode import ( PRODUCER_SHA, build_package, @@ -139,6 +144,8 @@ def test_agentic_collection_requires_declared_cpu_and_keeps_gpu_valid(tmp_path): for device in manifest["expected_devices"]: device["assignments"][0].update(worker_role="agg", het_group=0) manifest_path.write_text(json.dumps(manifest)) + if missing_gpu: + (pkg.power_dir / "samples.csv").unlink() source, workspace = tmp_path / "source", tmp_path / "output" source.mkdir() workspace.mkdir() @@ -156,11 +163,16 @@ def test_agentic_collection_requires_declared_cpu_and_keeps_gpu_valid(tmp_path): PRODUCER_SHA, [4], results_python=sys.executable, + require_power=require_power, expected_cpu=True, - ) == 1 + ) == expected_rc result = json.loads((workspace / "point_conc4.json").read_text()) - assert result["power_valid"] == 1 + assert result["power_valid"] == int(not missing_gpu) assert result["cpu_power_valid"] == 0 - assert result["total_gpu_energy_j"] == 84000.0 + if missing_gpu: + assert "total_gpu_energy_j" not in result + assert "package_recompute_invalid" in result["power_invalid_reasons"] + else: + assert result["total_gpu_energy_j"] == 84000.0 assert result["power_audit"]["source"] == "LOGS/agentic/conc_4/power_validation.json" assert result["power_audit"]["cpu"]["reason_codes"] == ["cpu_artifacts_missing"] diff --git a/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py b/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py index e84d933a18..5c560fefc7 100644 --- a/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py +++ b/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py @@ -4,7 +4,7 @@ import pytest -from infx.launch.drivers.srt.power import decide_power +from infx.launch.drivers.srt.power import PowerPolicyError, decide_power from infx.launch.policy import LaunchPath from infx.launch.request import MultiNodeRequest from infx.srt_slurm.workload import compose_recipe @@ -56,6 +56,20 @@ def test_existing_strict_agentx_rule_stays_strict(): assert decision.dcgm and decision.agentx and decision.cpu and decision.require +def test_explicit_requirement_uses_default_nvl72_collector_and_skips_evals(): + strict = request(REQUIRE_POWER="1", POWER="0") + decision = decide_power("gb200-nv", LaunchPath.SRT_MULTI, strict) + assert decision.dcgm and decision.agentx and decision.cpu and decision.require + evaluation = decide_power("gb200-nv", LaunchPath.SRT_MULTI, + request(REQUIRE_POWER="1", EVAL_ONLY="true")) # fmt: skip + assert not evaluation.dcgm and not evaluation.require + + +def test_explicit_requirement_without_collector_fails(): + with pytest.raises(PowerPolicyError, match="eligible power collector"): + decide_power("h200-dgxc", LaunchPath.SRT_MULTI, request(REQUIRE_POWER="1", POWER="0")) + + @pytest.mark.parametrize("hardware", ["gb200", "gb300"]) @pytest.mark.parametrize("precision", ["fp4", "fp8"]) def test_composed_dsr1_recipe_places_power_client_on_head(hardware, precision): diff --git a/inferencex-e2e/infx/tests/launch/test_srt_config.py b/inferencex-e2e/infx/tests/launch/test_srt_config.py index 95ab96c798..656b173c89 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_config.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_config.py @@ -278,7 +278,7 @@ def rsync(source: Path, staging: Path, *, exclude: tuple[str, ...]) -> Path: ("1", PowerDecision(dcgm=True, agentx=True), (9401, "true", "1", "0")), ("1", PowerDecision(dcgm=True, agentx=True, require=True), (9401, "true", "1", "1")), ("1", PowerDecision(dcgm=True, agentx=False, adapter=True), (9401, "true", "1", "0")), - ("0", PowerDecision(dcgm=True, agentx=False), (9401, None, None, None)), + ("0", PowerDecision(dcgm=True, agentx=False), (9401, None, None, "0")), ]) # fmt: skip def test_the_agentx_client_measures_power_exactly_when_the_launcher_collects_it( tmp_path, agentic, decision, expected diff --git a/inferencex-e2e/infx/tests/launch/test_srt_driver.py b/inferencex-e2e/infx/tests/launch/test_srt_driver.py index 8dfff525f2..1984d932ba 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_driver.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_driver.py @@ -375,6 +375,7 @@ def test_power_lane_stages_provenance_and_validates_each_concurrency( assert bound["benchmark"]["concurrencies"] == [4] assert bound["benchmark"]["env"]["ENABLE_AGENTX_POWER"] == "1" assert bound["benchmark"]["env"]["REQUIRE_POWER"] == require_power + assert bound["telemetry"]["required"] is (require_power == "1") assert bound["telemetry"]["collector_join_timeout_seconds"] == 12 assert (bound["telemetry"]["enabled"], bound["telemetry"]["dcgm_exporter"]["port"]) == (True, 9401) assert "cpu_power_exporter" not in bound["telemetry"] @@ -383,7 +384,7 @@ def test_power_lane_stages_provenance_and_validates_each_concurrency( runs = lines(harness.logs, "adapter") assert [run.split("--result-dir ")[1].split()[0].rsplit("/", 1)[1] for run in runs] == ["conc_4"] assert all(f"--expected-producer-sha {commit}" in run for run in runs) - required = lane == "agentx" or require_power == "1" + required = require_power == "1" assert all(run.endswith("--require-power") is required for run in runs) if lane == "agentx": status = (workspace / "LOGS/power/native-job-status.txt").read_text() diff --git a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py index f32bce0f9e..53b0183fc8 100644 --- a/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py +++ b/inferencex-e2e/infx/tests/matrix/test_generate_sweep_configs.py @@ -2526,6 +2526,22 @@ def agentic_config(request, sample_single_node_config): class TestAgenticGeneration: + @pytest.mark.parametrize("required", [None, False, True]) + def test_agentic_power_requirement_reaches_rows_and_not_evals( + self, agentic_config, sample_runner_config, generate_agentic_sweep, required + ): + from infx.matrix.generate import select_matrix_evals + + config, benchmark = agentic_config + benchmark["conc-list"] = [4] + scenario = next(iter(config.values()))["scenarios"]["agentic-coding"][0] + if required is not None: + scenario["require-power"] = required + rows = generate_agentic_sweep(config, sample_runner_config) + assert rows and all(row["require-power"] is (required is True) for row in rows) + evals = select_matrix_evals(copy.deepcopy(rows), mode="subset") + assert evals and all("require-power" not in row for row in evals) + def test_point_order_and_input_preservation( self, agentic_config, sample_runner_config, generate_agentic_sweep, ): diff --git a/inferencex-e2e/infx/tests/results/power/test_process_result.py b/inferencex-e2e/infx/tests/results/power/test_process_result.py index b812e38c4b..5d0909530d 100644 --- a/inferencex-e2e/infx/tests/results/power/test_process_result.py +++ b/inferencex-e2e/infx/tests/results/power/test_process_result.py @@ -1148,7 +1148,7 @@ def test_agentic_collector_preserves_archive_when_result_python_is_missing( completed = JobStatus(JobState.SUCCEEDED, "COMPLETED|0:0", 0) rc = collect_agentic_power_results( completed, "12345", pkg.logs_root, source, workspace, "point", PRODUCER_SHA, [4], - results_python=result_python, + results_python=result_python, require_power=True, ) # fmt: skip archive_name = "server-logs.tar.gz" else: diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py index 0159220589..8895d01260 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_generate.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_generate.py @@ -188,7 +188,7 @@ def test_a_multinode_agentx_power_point_gets_its_clusters_exporter_port_and_clie [(record, recipe)] = written(output) assert record["cluster"] == "fixture" - assert recipe["telemetry"] == {"enabled": True, "dcgm_exporter": { + assert recipe["telemetry"] == {"enabled": True, "required": False, "dcgm_exporter": { "container_image": "dcgm-exporter", "command": "dcgm-exporter --address :{port}", "port": 9555, }} # fmt: skip assert recipe["benchmark"]["env"] == { diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py index b01668a3b0..199711fdcb 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_workload.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_workload.py @@ -166,10 +166,15 @@ def test_a_power_point_gets_telemetry_and_its_concurrencies_on_a_bundle_variant( bound = bind_workload(variant, AGENTX_ENV, agentic=True, multinode=True) assert bound["telemetry"] == { - "collector_join_timeout_seconds": 12, "enabled": True, "required": True, + "collector_join_timeout_seconds": 12, "enabled": True, "required": False, "dcgm_exporter": {"container_image": "dcgm-exporter", "port": 19401}, } # fmt: skip assert bound["benchmark"]["concurrencies"] == [8] + strict = bind_workload( + variant, {**AGENTX_ENV, "REQUIRE_POWER": "1"}, agentic=True, multinode=True + ) + assert strict["telemetry"]["required"] is True + assert strict["benchmark"]["env"]["REQUIRE_POWER"] == "1" with_cpu = compose_recipe( path, agentic=True, multinode=True, root=project, power_port=19401, cpu_power_source="acpi" ) @@ -180,6 +185,21 @@ def test_a_power_point_gets_telemetry_and_its_concurrencies_on_a_bundle_variant( ] +def test_cpu_only_telemetry_keeps_its_source_without_gpu_exporter(): + recipe = { + "benchmark": {}, + "telemetry": {"enabled": True, "cpu_power_exporter": {"source": "acpi"}}, + } + bound = bind_workload(recipe, AGENTX_ENV, agentic=True, multinode=True) + assert bound["telemetry"] == { + "enabled": True, + "required": False, + "cpu_power_exporter": {"source": "acpi"}, + } + assert bound["benchmark"]["concurrencies"] == [8] + assert bound["benchmark"]["env"]["REQUIRE_POWER"] == "0" + + @pytest.mark.parametrize(("environment", "message"), [ ({**MULTI_ENV, "IMAGE": ""}, "Missing workload input: IMAGE"), ({**MULTI_ENV, "OSL": "1k"}, "OSL must be a positive integer: '1k'"), diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index eb1470be4c..515af356d7 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9895,6 +9895,7 @@ - dsv41flash-fp4-gb300-vllm-agentic-dspark - dsv41flash-fp4-gb300-sglang-agentic-dspark description: - - "Enable noprof GPU and ACPI CPU collection by default for GB200/GB300 benchmarks, including native single-node and AgentX, with the pinned cpu-power-exporter v2.51.1. Preserve independent GPU/CPU validity and existing strict qualification; eval-only jobs omit collection." + - "Enable best-effort noprof GPU and ACPI Grace CPU collection by default for GB200/GB300 benchmarks, including native single-node and AgentX, with the pinned cpu-power-exporter v2.51.1. Explicit required-power AgentX points fail on invalid GPU or expected CPU evidence; ordinary points retain valid benchmark results and independent GPU/CPU validity. Eval-only jobs omit collection." - "Place benchmark clients and telemetry on the head clock without changing worker or allocation counts. Native single-node results retain their real topology and power audit. Placement changes require fresh performance results." + - "Require validated power on the 11 GLM-5.2 GB200 aggregate/disaggregated/MTP AgentX points and 24 DSV4 GB200 aggregate MTP TP8/DEP8 points, retaining their current images, selectors and topology." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3670 From 58191c74c77c4bebb5d112e5154984c77f60a1c3 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 17 Oct 2026 01:33:44 -0700 Subject: [PATCH 5/5] fix: align power rows and recipes across affected curves --- .../agentx/agg-tp8-mtp-kvoffload.yaml | 3 --- .../sglang/gb300-fp4-mtp/agentic.yaml | 2 +- inferencex-e2e/configs/nvidia-master.yaml | 7 +++++- .../infx/launch/drivers/srt/power.py | 2 -- .../infx/tests/launch/test_launch_policy.py | 2 +- .../tests/launch/test_nvl72_power_defaults.py | 10 ++++++-- inferencex-e2e/perf-changelog.yaml | 25 ++++++++++++++++++- 7 files changed, 40 insertions(+), 11 deletions(-) diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml index 9d0487660c..93e7162639 100644 --- a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/h200-fp8/agentx/agg-tp8-mtp-kvoffload.yaml @@ -102,9 +102,6 @@ srun_options: # The custom benchmark installs its isolated AIPerf environment at runtime. container-remap-root: "" -telemetry: - required: false - benchmark: env: AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml index 3c21ee1306..119a4e953e 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/qwen3.8next/sglang/gb300-fp4-mtp/agentic.yaml @@ -98,7 +98,7 @@ base: env: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' # Native single-node binding verifies these against the generated point. - TOTAL_CPU_DRAM_GB: '225' + TOTAL_CPU_DRAM_GB: '241' KV_OFFLOADING: dram # Only client concurrency changes; serving/admission/graph/cache settings stay fixed. diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 9d5008acc9..23d65549dc 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -6198,6 +6198,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg: scenarios: agentic-coding: - dram-utilization: 0.1065 + require-power: true search-space: - spec-decoding: mtp kv-offloading: dram @@ -6227,6 +6228,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg: scenarios: agentic-coding: - dram-utilization: 0.1065 + require-power: true search-space: - kv-offloading: dram kv-offload-backend: { name: mooncake, version: "0.3.12.post1" } @@ -6363,6 +6365,7 @@ kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2: scenarios: agentic-coding: - dram-utilization: 0.8 + require-power: true search-space: - spec-decoding: mtp kv-offloading: dram @@ -7889,6 +7892,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg: # 0.1775 resolves to 160 GB of host DRAM per 4-GPU server node on # cluster:gb300-nv, matching the recipe's Mooncake segment size. - dram-utilization: 0.1775 + require-power: true search-space: - spec-decoding: mtp kv-offloading: dram @@ -7960,6 +7964,7 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg: scenarios: agentic-coding: - dram-utilization: 0.1775 + require-power: true search-space: - spec-decoding: mtp kv-offloading: dram @@ -9227,7 +9232,7 @@ qwen3.8next-fp4-gb300-sglang-agentic-mtp: scenarios: agentic-coding: # Use the GB300 profile's proportional per-worker host DRAM allowance. - # Generated allowance: 225 GB. Recipe H224 reserves alignment headroom. + # Generated allowance: 241 GB. Recipe H224 retains its existing cache size. - dram-utilization: 1.0 search-space: - tp: 1 diff --git a/inferencex-e2e/infx/launch/drivers/srt/power.py b/inferencex-e2e/infx/launch/drivers/srt/power.py index 7c9a21fd51..1818ed3d0c 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/power.py +++ b/inferencex-e2e/infx/launch/drivers/srt/power.py @@ -49,7 +49,6 @@ class PowerLane: PowerRule( Match(any_of("kimik3"), any_of("fp4"), any_of("dynamo-vllm"), agentic=True), agentx=True, - require=True, ), PowerRule(Match(frameworks=any_of("dynamo-sglang"), agentic=False), agentx=False), ), @@ -61,7 +60,6 @@ class PowerLane: PowerRule( Match(any_of("kimik3"), any_of("fp4"), any_of("dynamo-vllm"), agentic=True), agentx=True, - require=True, ), PowerRule(Match(frameworks=any_of("dynamo-sglang")), agentx=False), ), diff --git a/inferencex-e2e/infx/tests/launch/test_launch_policy.py b/inferencex-e2e/infx/tests/launch/test_launch_policy.py index 5d7ba4649b..481f6cb7ba 100644 --- a/inferencex-e2e/infx/tests/launch/test_launch_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_launch_policy.py @@ -73,7 +73,7 @@ def test_power_eligibility_of_each_cluster_lane( assert (decision.dcgm, decision.agentx, decision.adapter) == ( True, expected == AGENTX, expected == ADAPTER, ) # fmt: skip - assert decision.require is (prefix == "kimik3" and cluster in ("gb200-nv", "gb300-nv")) + assert not decision.require assert decide_power(cluster, path, _multinode(**env, POWER="0")).dcgm is ( cluster in power.NVL72_POWER_CLUSTERS ) diff --git a/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py b/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py index 5c560fefc7..273a4269fc 100644 --- a/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py +++ b/inferencex-e2e/infx/tests/launch/test_nvl72_power_defaults.py @@ -47,13 +47,19 @@ def test_other_cluster_keeps_explicit_power_policy(): assert not decide_power("h200-dgxc", LaunchPath.SRT_MULTI, request(POWER="0")).dcgm -def test_existing_strict_agentx_rule_stays_strict(): +def test_existing_kimi_agentx_row_requires_explicit_flag(): decision = decide_power( "gb200-nv", LaunchPath.SRT_MULTI, request(POWER="1", MODEL_PREFIX="kimik3", FRAMEWORK="dynamo-vllm"), ) - assert decision.dcgm and decision.agentx and decision.cpu and decision.require + assert decision.dcgm and decision.agentx and decision.cpu and not decision.require + strict = decide_power( + "gb200-nv", + LaunchPath.SRT_MULTI, + request(REQUIRE_POWER="1", MODEL_PREFIX="kimik3", FRAMEWORK="dynamo-vllm"), + ) + assert strict.require def test_explicit_requirement_uses_default_nvl72_collector_and_skips_evals(): diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index abdf4a4084..acfac63727 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9963,8 +9963,31 @@ - dsv41flash-fp4-gb200-sglang-agentic-dspark - dsv41flash-fp4-gb300-vllm-agentic-dspark - dsv41flash-fp4-gb300-sglang-agentic-dspark + - qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-pareto + - qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-pareto + - qwen3.5-fp8-gb300-dynamo-sglang-agentic-pp-midcurve + - qwen3.5-fp8-gb300-dynamo-sglang-agentic-agg-midcurve + - dsv4-fp4-gb200-dynamo-sglang-agentic-agg + - dsv4-fp4-gb200-dynamo-sglang-agentic-disagg + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg + - dsv41flash-fp4-gb300-dynamo-sglang-agentic-disagg + - dsv41flash-fp4-gb300-dynamo-vllm-agentic-dspark-agg + - dsv41flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg + - dsv4-fp4-gb200-dynamo-vllm-agentic-dspark-agg + - dsv4-fp4-gb200-dynamo-vllm-agentic-dspark-disagg + - qwen3.8next-fp4-gb300-sglang-agentic-mtp + - kimik3-fp4-h200-vllm-agentic-latency + - kimik3-fp4-h200-vllm-agentic-balanced + - kimik3-fp4-h200-vllm-agentic-simple + - dsv4-fp8-h200-dynamo-sglang-agentic-agg + - glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d + - glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-1p1d-hicache + - glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-agg + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + - qwen3.5-fp8-b200-dynamo-sglang-agentic-disagg-mtp description: - "Enable best-effort noprof GPU and ACPI Grace CPU collection by default for GB200/GB300 benchmarks, including native single-node and AgentX, with the pinned cpu-power-exporter v2.51.1. Explicit required-power AgentX points fail on invalid GPU or expected CPU evidence; ordinary points retain valid benchmark results and independent GPU/CPU validity. Eval-only jobs omit collection." - "Place benchmark clients and telemetry on the head clock without changing worker or allocation counts. Native single-node results retain their real topology and power audit. Placement changes require fresh performance results." - - "Require validated power on the 11 GLM-5.2 GB200 aggregate/disaggregated/MTP AgentX points and 24 DSV4 GB200 aggregate MTP TP8/DEP8 points, retaining their current images, selectors and topology." + - "Require validated power on the 11 GLM-5.2 GB200 aggregate/disaggregated/MTP AgentX points and 24 DSV4 GB200 aggregate MTP TP8/DEP8 points. Declare the five existing Kimi-K3 GB200/GB300 measured-power arms strict in their matrix rows instead of a cluster-wide Kimi rule. Retain their current images, selectors and topology." + - "Forward the selected required-power flag through existing H200/B200 AgentX clients, telemetry, and result collection. Align the Qwen3.8-Next GB300 recipe's declared host DRAM with the current runner's 241 GB allowance while retaining its H224 cache setting." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3670