From a0202361b32f259c974f00f5a8ee8d8cce64aeaf Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Mon, 3 Aug 2026 10:18:42 -0700 Subject: [PATCH 1/3] perf(agentx): refresh Qwen3.5 GB300 Pareto recipes Refresh the Qwen3.5-397B-A17B NVFP4 GB300 AgentX frontier with the fourteen-point locally-validated curve: aggregate TP2 (c1/c24/c32/c40/ c48/c52/c64), disaggregate TP2/TP2 (c72), and disaggregate TP4/TP4 (c8/c16/c32/c64/c96/c128), all MTP/NEXTN + hicache with X-Dynamo-Session-ID affinity. --- ...g-gb300-tp2-c1-mtp-hicache-jid2530006.yaml | 89 ++++++ ...-gb300-tp2-c24-mtp-hicache-jid2530012.yaml | 89 ++++++ ...-gb300-tp2-c32-mtp-hicache-jid2530013.yaml | 89 ++++++ ...-gb300-tp2-c40-mtp-hicache-jid2530015.yaml | 89 ++++++ ...-gb300-tp2-c48-mtp-hicache-jid2530017.yaml | 89 ++++++ ...-gb300-tp2-c52-mtp-hicache-jid2527406.yaml | 89 ++++++ ...-gb300-tp2-c64-mtp-hicache-jid2527410.yaml | 89 ++++++ ...g-gb300-tp4-c1-mtp-hicache-jid2191933.yaml | 114 -------- ...-gb300-tp4-c96-mtp-hicache-jid2195211.yaml | 114 -------- ...p2-c72-mtp-hicache-session-jid2527415.yaml | 203 ++++++++++++++ ...-c128-mtp-hicache-session-jid2527417.yaml} | 257 ++++++++---------- ...4-c16-mtp-hicache-session-jid2530027.yaml} | 233 ++++++++-------- ...4-c32-mtp-hicache-session-jid2530028.yaml} | 233 ++++++++-------- ...-tp4-tp4-c64-mtp-convaware-jid2214439.yaml | 237 ---------------- ...p4-c64-mtp-hicache-session-jid2530029.yaml | 198 ++++++++++++++ ...p4-c8-mtp-hicache-session-jid2530030.yaml} | 249 ++++++++--------- ...p4-c96-mtp-hicache-session-jid2527409.yaml | 198 ++++++++++++++ ...c256-mtp-hicache-convaware-jid2228613.yaml | 215 --------------- ...c384-mtp-hicache-convaware-jid2239740.yaml | 215 --------------- configs/nvidia-master.yaml | 213 +++++++++------ perf-changelog.yaml | 9 + runners/launch_gb300-nv.sh | 6 +- 22 files changed, 1821 insertions(+), 1496 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml rename benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/{disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml => disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml} (70%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/{disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml => disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml} (80%) rename benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/{disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml => disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml} (79%) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml rename benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/{disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml => disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml} (72%) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml new file mode 100644 index 0000000000..9a2c54927d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml @@ -0,0 +1,89 @@ +name: agg-gb300-tp2-c1-mtp-hicache-jid2530006 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml new file mode 100644 index 0000000000..5c90cfd0ed --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml @@ -0,0 +1,89 @@ +name: agg-gb300-tp2-c24-mtp-hicache-jid2530012 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml new file mode 100644 index 0000000000..1c2c2bc7b2 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml @@ -0,0 +1,89 @@ +name: agg-gb300-tp2-c32-mtp-hicache-jid2530013 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml new file mode 100644 index 0000000000..203846e7b7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml @@ -0,0 +1,89 @@ +name: agg-gb300-tp2-c40-mtp-hicache-jid2530015 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml new file mode 100644 index 0000000000..da0adf6fdb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml @@ -0,0 +1,89 @@ +name: agg-gb300-tp2-c48-mtp-hicache-jid2530017 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml new file mode 100644 index 0000000000..4efc58a5a3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml @@ -0,0 +1,89 @@ +name: agg-gb300-tp2-c52-mtp-hicache-jid2527406 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml new file mode 100644 index 0000000000..a7b6dd0727 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml @@ -0,0 +1,89 @@ +name: agg-gb300-tp2-c64-mtp-hicache-jid2527410 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml deleted file mode 100644 index 02c88cc299..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml +++ /dev/null @@ -1,114 +0,0 @@ -name: "agg-gb300-tp4-c1-mtp-hicache-jid2191933" - -# Agentic-coding SGLang aggregated recipe for Qwen3.5-397B-A17B-NVFP4 on GB300 -# (single aggregated worker, TP4, NEXTN MTP + hierarchical cache). -# -# Ported from the manually-run srtctl base:/zip_override_conc: starter to the -# flat single-variant agentic schema. Pure sglang (no dynamo): frontend.type -# sglang serves directly. Concurrency comes from the GHA matrix (exported as -# CONC into agentic_srt.sh), not the recipe. Serving tuning preserved verbatim -# from the source run; only CI scaffolding is added. - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - nats_max_payload_mb: 8 - -frontend: - type: sglang - -backend: - type: sglang - aggregated_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - NCCL_NVLS_ENABLE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 256 - max-running-requests: 1024 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.8 - max-mamba-cache-size: 1024 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 1.01 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user into the container as a non-root uid; - # agentic_srt.sh's apt-get install git step needs EUID 0. Remap to uid 0 inside - # the container. srt-slurm renders empty-string values as flag-only srun args. - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - # Aggregated: one worker serves both prefill and decode, so GPU accounting is - # the single-worker form num_gpus = TP (not the disagg prefill+decode sum). - # Force the single-node post-proc path; TP must match sglang_config tp-size. - IS_MULTINODE: "false" - TP: "4" - # Match the source run's dataset (256k cc-traces-with-subagents variant). - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - # Container-side path of the aiperf mmap dataset cache; host-side mount wired - # via launch_gb300-nv.sh srtslurm.yaml default_mounts. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also via default_mounts). - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml deleted file mode 100644 index 5891df5b32..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml +++ /dev/null @@ -1,114 +0,0 @@ -name: "agg-gb300-tp4-c96-mtp-hicache-jid2195211" - -# Agentic-coding SGLang aggregated recipe for Qwen3.5-397B-A17B-NVFP4 on GB300 -# (single aggregated worker, TP4, NEXTN MTP + hierarchical cache). -# -# Ported from the manually-run srtctl base:/zip_override_conc: starter to the -# flat single-variant agentic schema. Pure sglang (no dynamo): frontend.type -# sglang serves directly. Concurrency comes from the GHA matrix (exported as -# CONC into agentic_srt.sh), not the recipe. Serving tuning preserved verbatim -# from the source run; only CI scaffolding is added. - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - nats_max_payload_mb: 8 - -frontend: - type: sglang - -backend: - type: sglang - aggregated_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - NCCL_NVLS_ENABLE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 256 - max-running-requests: 1024 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.8 - max-mamba-cache-size: 1024 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 1.05 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user into the container as a non-root uid; - # agentic_srt.sh's apt-get install git step needs EUID 0. Remap to uid 0 inside - # the container. srt-slurm renders empty-string values as flag-only srun args. - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - # Aggregated: one worker serves both prefill and decode, so GPU accounting is - # the single-worker form num_gpus = TP (not the disagg prefill+decode sum). - # Force the single-node post-proc path; TP must match sglang_config tp-size. - IS_MULTINODE: "false" - TP: "4" - # Match the source run's dataset (256k cc-traces-with-subagents variant). - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - # Container-side path of the aiperf mmap dataset cache; host-side mount wired - # via launch_gb300-nv.sh srtslurm.yaml default_mounts. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also via default_mounts). - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml new file mode 100644 index 0000000000..a5473b511f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml @@ -0,0 +1,203 @@ +name: disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + decode_nodes: 1 + decode_workers: 1 + gpu_type: gb300 + gpus_per_decode: 2 + gpus_per_node: 2 + gpus_per_prefill: 2 + prefill_nodes: 1 + prefill_workers: 1 +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +backend: + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 80 + max-running-requests: 80 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + decode-log-interval: 10 + mamba-max-states-per-path: -1 + prefill: + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + max-mamba-cache-size: 320 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml similarity index 70% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml index 6449be871c..2b481ddc3c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml @@ -1,168 +1,122 @@ -name: "disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562" - -# Agentic-coding SGLang disaggregated 2P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300, ported to the flat single-variant agentic schema. Concurrency is -# NOT a recipe field: the GHA matrix fans out one job per concurrency and -# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, -# frontend, dynamo hash, topology) is preserved verbatim from the source run. - +name: disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417 model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 dynamo: install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a slurm: - time_limit: "8:00:00" - + time_limit: '8:00:00' health_check: max_attempts: 1440 interval_seconds: 10 - resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 decode_nodes: 1 - prefill_workers: 2 decode_workers: 1 - gpus_per_prefill: 4 + gpu_type: gb300 gpus_per_decode: 4 - + gpus_per_node: 4 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 infra: etcd_nats_dedicated_node: true nats_max_payload_mb: 8 - frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - num_additional_frontends: 0 + enable_multiple_frontends: true + num_additional_frontends: 4 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" + PIP_BREAK_SYSTEM_PACKAGES: '1' args: router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - PIP_BREAK_SYSTEM_PACKAGES: "1" - + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false prefill-round-robin-balance: true mamba-scheduler-strategy: no_buffer mamba-track-interval: 128 @@ -172,44 +126,73 @@ backend: disaggregation-bootstrap-port: 31000 page-size: 64 attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl + moe-runner-backend: flashinfer_trtllm linear-attn-decode-backend: flashinfer disable-shared-experts-fusion: true - moe-a2a-backend: deepep - deepep-mode: low_latency ep-dispatch-algorithm: static eplb-algorithm: deepseek speculative-algorithm: NEXTN speculative-num-steps: 3 speculative-eagle-topk: 1 speculative-num-draft-tokens: 4 - speculative-moe-runner-backend: deep_gemm - speculative-moe-a2a-backend: deepep chunked-prefill-size: 4096 mem-fraction-static: 0.75 max-mamba-cache-size: 1024 max-running-requests: 512 - cuda-graph-max-bs: 64 + cuda-graph-max-bs: 160 watchdog-timeout: 1000000 decode-log-interval: 10 - + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + type: sglang sbatch_directives: - mem: "0" - cpus-per-task: "144" - + mem: '0' + cpus-per-task: '144' srun_options: - mem: "0" - container-remap-root: "" - + mem: '0' + container-remap-root: '' benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" + PORT: '8000' + IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml similarity index 80% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml index cbf7157955..d1c3481093 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml @@ -1,161 +1,113 @@ -name: "disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783" - -# Agentic-coding SGLang disaggregated 1P1D recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300, ported to the flat single-variant agentic schema. Concurrency is -# NOT a recipe field: the GHA matrix fans out one job per concurrency and -# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, -# frontend, dynamo hash, topology) is preserved verbatim from the source run. - +name: disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027 model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 dynamo: install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a slurm: - time_limit: "8:00:00" - + time_limit: '8:00:00' health_check: max_attempts: 1440 interval_seconds: 10 - resources: + decode_nodes: 1 + decode_workers: 1 gpu_type: gb300 + gpus_per_decode: 4 gpus_per_node: 4 + gpus_per_prefill: 4 prefill_nodes: 1 - decode_nodes: 1 prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - infra: etcd_nats_dedicated_node: true nats_max_payload_mb: 8 - frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" + PIP_BREAK_SYSTEM_PACKAGES: '1' args: router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - PIP_BREAK_SYSTEM_PACKAGES: "1" - + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 @@ -190,24 +142,57 @@ backend: cuda-graph-max-bs: 64 watchdog-timeout: 1000000 decode-log-interval: 10 - + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + type: sglang sbatch_directives: - mem: "0" - cpus-per-task: "144" - + mem: '0' + cpus-per-task: '144' srun_options: - mem: "0" - container-remap-root: "" - + mem: '0' + container-remap-root: '' benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" + PORT: '8000' + IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml similarity index 79% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml index 4f093f5ec8..f992b34126 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml @@ -1,161 +1,113 @@ -name: "disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440" - -# Agentic-coding SGLang disaggregated 1P1D recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300, ported to the flat single-variant agentic schema. Concurrency is -# NOT a recipe field: the GHA matrix fans out one job per concurrency and -# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, -# frontend, dynamo hash, topology) is preserved verbatim from the source run. - +name: disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028 model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 dynamo: install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a slurm: - time_limit: "8:00:00" - + time_limit: '8:00:00' health_check: max_attempts: 1440 interval_seconds: 10 - resources: + decode_nodes: 1 + decode_workers: 1 gpu_type: gb300 + gpus_per_decode: 4 gpus_per_node: 4 + gpus_per_prefill: 4 prefill_nodes: 1 - decode_nodes: 1 prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - infra: etcd_nats_dedicated_node: true nats_max_payload_mb: 8 - frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" + PIP_BREAK_SYSTEM_PACKAGES: '1' args: router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - PIP_BREAK_SYSTEM_PACKAGES: "1" - + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: false - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 @@ -190,24 +142,57 @@ backend: cuda-graph-max-bs: 64 watchdog-timeout: 1000000 decode-log-interval: 10 - + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + type: sglang sbatch_directives: - mem: "0" - cpus-per-task: "144" - + mem: '0' + cpus-per-task: '144' srun_options: - mem: "0" - container-remap-root: "" - + mem: '0' + container-remap-root: '' benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" + PORT: '8000' + IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml deleted file mode 100644 index 52aabae1ef..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml +++ /dev/null @@ -1,237 +0,0 @@ -name: "disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439" - -# Agentic-coding SGLang disaggregated 1P1D recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300 (TP4 prefill / TP4 decode, NEXTN MTP + hierarchical cache). -# -# Ported from the manually-run srtctl base:/zip_override_conc: starter to the -# flat single-variant schema the agentic CI flow expects (modeled on the -# proven sglang/deepseek-v4/agentic recipes). Concurrency is NOT a recipe -# field here: the GHA matrix fans out one job per concurrency from the -# master-config conc-list, exporting CONC into agentic_srt.sh. Serving tuning -# (sglang_config, prefill/decode env, frontend, dynamo hash, topology) is -# preserved verbatim from the source run; only the CI scaffolding is added. - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -infra: - # Dedicated etcd/nats node — under the dynamo router + multiple frontends - # the infra services need their own node (matches the proven agentic recipes). - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - # The sglang image is PEP 668 externally-managed; the runtime dynamo - # install (dynamo.hash source build) runs pip and fails with - # "externally-managed-environment" without this. - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. - PIP_BREAK_SYSTEM_PACKAGES: "1" - - sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: false - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user into the container as a non-root - # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to - # uid 0 inside the container. srt-slurm renders empty-string values as - # flag-only srun args. - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Match the source run's dataset (256k cc-traces-with-subagents variant). - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - # Per-session affinity so multi-turn KV cache is reused across turns - # (recipe name: "convaware"); build_replay_cmd adds - # --use-dynamo-conv-aware-routing for dynamo-* frameworks. - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Container-side path of the aiperf mmap dataset cache; host-side mount - # wired via launch_gb300-nv.sh srtslurm.yaml default_mounts. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also via default_mounts) so the trace dataset - # isn't re-downloaded each run. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml new file mode 100644 index 0000000000..5bd9695e09 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml @@ -0,0 +1,198 @@ +name: disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + decode_nodes: 1 + decode_workers: 1 + gpu_type: gb300 + gpus_per_decode: 4 + gpus_per_node: 4 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +backend: + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 80 + watchdog-timeout: 1000000 + decode-log-interval: 10 + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml similarity index 72% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml index 75e0dddc08..598ac987f1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml @@ -1,168 +1,122 @@ -name: "disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685" - -# Agentic-coding SGLang disaggregated 1P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300, ported to the flat single-variant agentic schema. Concurrency is -# NOT a recipe field: the GHA matrix fans out one job per concurrency and -# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, -# frontend, dynamo hash, topology) is preserved verbatim from the source run. - +name: disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030 model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 dynamo: install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a slurm: - time_limit: "8:00:00" - + time_limit: '8:00:00' health_check: max_attempts: 1440 interval_seconds: 10 - resources: + decode_nodes: 1 + decode_workers: 1 gpu_type: gb300 + gpus_per_decode: 4 gpus_per_node: 4 + gpus_per_prefill: 4 prefill_nodes: 1 - decode_nodes: 1 prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - infra: etcd_nats_dedicated_node: true nats_max_payload_mb: 8 - frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" + PIP_BREAK_SYSTEM_PACKAGES: '1' args: router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - PIP_BREAK_SYSTEM_PACKAGES: "1" - + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false prefill-round-robin-balance: true mamba-scheduler-strategy: no_buffer mamba-track-interval: 128 @@ -172,19 +126,15 @@ backend: disaggregation-bootstrap-port: 31000 page-size: 64 attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl + moe-runner-backend: flashinfer_trtllm linear-attn-decode-backend: flashinfer disable-shared-experts-fusion: true - moe-a2a-backend: deepep - deepep-mode: low_latency ep-dispatch-algorithm: static eplb-algorithm: deepseek speculative-algorithm: NEXTN speculative-num-steps: 3 speculative-eagle-topk: 1 speculative-num-draft-tokens: 4 - speculative-moe-runner-backend: deep_gemm - speculative-moe-a2a-backend: deepep chunked-prefill-size: 4096 mem-fraction-static: 0.75 max-mamba-cache-size: 1024 @@ -192,24 +142,57 @@ backend: cuda-graph-max-bs: 64 watchdog-timeout: 1000000 decode-log-interval: 10 - + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + type: sglang sbatch_directives: - mem: "0" - cpus-per-task: "144" - + mem: '0' + cpus-per-task: '144' srun_options: - mem: "0" - container-remap-root: "" - + mem: '0' + container-remap-root: '' benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" + PORT: '8000' + IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml new file mode 100644 index 0000000000..1ca34c2d88 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml @@ -0,0 +1,198 @@ +name: disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + decode_nodes: 1 + decode_workers: 1 + gpu_type: gb300 + gpus_per_decode: 4 + gpus_per_node: 4 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +backend: + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + decode-log-interval: 10 + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml deleted file mode 100644 index 62f641193f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml +++ /dev/null @@ -1,215 +0,0 @@ -name: "disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613" - -# Agentic-coding SGLang disaggregated 3P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300, ported to the flat single-variant agentic schema. Concurrency is -# NOT a recipe field: the GHA matrix fans out one job per concurrency and -# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, -# frontend, dynamo hash, topology) is preserved verbatim from the source run. - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - num_additional_frontends: 0 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - PIP_BREAK_SYSTEM_PACKAGES: "1" - - sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - moe-a2a-backend: deepep - deepep-mode: low_latency - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - speculative-moe-runner-backend: deep_gemm - speculative-moe-a2a-backend: deepep - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml deleted file mode 100644 index 66a8a650b0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml +++ /dev/null @@ -1,215 +0,0 @@ -name: "disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740" - -# Agentic-coding SGLang disaggregated 4P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300, ported to the flat single-variant agentic schema. Concurrency is -# NOT a recipe field: the GHA matrix fans out one job per concurrency and -# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, -# frontend, dynamo hash, topology) is preserved verbatim from the source run. - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 4 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - num_additional_frontends: 0 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - PIP_BREAK_SYSTEM_PACKAGES: "1" - - sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - moe-a2a-backend: deepep - deepep-mode: low_latency - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - speculative-moe-runner-backend: deep_gemm - speculative-moe-a2a-backend: deepep - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 90e54195b6..0b521719fd 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7090,6 +7090,10 @@ qwen3.5-fp4-gb300-dynamo-sglang: ep: 16 dp-attn: true +# GB300 SGLang AgentX, re-swept across low/mid/high concurrency after aiperf +# restored real inter-tool-call pacing. The seven aggregate frontier points use +# one TP2 worker with MTP/NEXTN + hierarchical KV cache. CONFIG_FILE carries the +# full srtctl recipe; the matrix topology stays in sync for naming/GPU accounting. qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1 model: nvidia/Qwen3.5-397B-A17B-NVFP4 @@ -7101,42 +7105,110 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: disagg: false scenarios: agentic-coding: - - dram-utilization: 0.80 - search-space: - - spec-decoding: "mtp" + - search-space: + - spec-decoding: mtp conc-list: [1] - kv-offloading: dram - kv-offload-backend: { name: hicache } prefill: num-worker: 1 - tp: 4 + tp: 2 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml" decode: - num-worker: 0 - tp: 4 + num-worker: 1 + tp: 2 ep: 1 dp-attn: false - - spec-decoding: "mtp" - conc-list: [96] - kv-offloading: dram - kv-offload-backend: { name: hicache } + - spec-decoding: mtp + conc-list: [24] prefill: num-worker: 1 - tp: 4 + tp: 2 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml" decode: - num-worker: 0 - tp: 4 + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [32] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml" + decode: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [40] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml" + decode: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [48] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml" + decode: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [52] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml" + decode: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [64] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml" + decode: + num-worker: 1 + tp: 2 ep: 1 dp-attn: false +# The seven disaggregated frontier points use six TP4 shapes plus one TP2 +# shape. Stable X-Dynamo-Session-ID affinity replaces the removed conv-aware +# routing message path. qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1 model: nvidia/Qwen3.5-397B-A17B-NVFP4 @@ -7144,129 +7216,110 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" } kv-p2p-transfer: mooncake multinode: true disagg: true scenarios: agentic-coding: - - dram-utilization: 0.80 - search-space: - # 1P1D STP (TP4 prefill / TP4 decode). 2 GB300 nodes. - - spec-decoding: "mtp" - conc-list: [32] - kv-offloading: dram - kv-offload-backend: { name: hicache } + - search-space: + - spec-decoding: mtp + conc-list: [8] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false - - spec-decoding: "mtp" - conc-list: [64] - kv-offloading: none + - spec-decoding: mtp + conc-list: [16] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false - - spec-decoding: "mtp" - conc-list: [96] - kv-offloading: none + - spec-decoding: mtp + conc-list: [32] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false - # 1P1D wide-EP (DEP4 prefill / DEP4 decode). 2 GB300 nodes. - - spec-decoding: "mtp" - conc-list: [128] - kv-offloading: dram - kv-offload-backend: { name: hicache } + - spec-decoding: mtp + conc-list: [64] prefill: num-worker: 1 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml" decode: num-worker: 1 tp: 4 - ep: 4 - dp-attn: true - # 2P1D wide-EP (2x DEP4 prefill / DEP4 decode). 3 GB300 nodes. - - spec-decoding: "mtp" - conc-list: [192] - kv-offloading: dram - kv-offload-backend: { name: hicache } + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [96] prefill: - num-worker: 2 + num-worker: 1 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml" decode: num-worker: 1 tp: 4 - ep: 4 - dp-attn: true - # 3P1D wide-EP (3x DEP4 prefill / DEP8 decode). 5 GB300 nodes. - - spec-decoding: "mtp" - conc-list: [256] - kv-offloading: dram - kv-offload-backend: { name: hicache } + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [128] prefill: - num-worker: 3 + num-worker: 1 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # 4P1D wide-EP (4x DEP4 prefill / DEP8 decode). 6 GB300 nodes. - - spec-decoding: "mtp" - conc-list: [384] - kv-offloading: dram - kv-offload-backend: { name: hicache } - prefill: - num-worker: 4 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [72] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true + tp: 2 + ep: 1 + dp-attn: false # ---------- 1k1k high-throughput (wide-EP decode, EAGLE MTP) ---------- qwen3.5-fp8-b300-sglang-agentic-hicache: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7a064f8090..c4c026b923 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5374,3 +5374,12 @@ - "Image: lmsysorg/sglang:v0.5.16-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2420 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Refresh the Qwen3.5 GB300 AgentX frontier after aiperf 381758a restored real conversation pacing; replace the stale nine-point sweep with a complete fourteen-point locally validated curve: disaggregate TP4/TP4 c8/c16/c32/c64/c96/c128, disaggregate TP2/TP2 c72, and aggregate TP2 c1/c24/c32/c40/c48/c52/c64." + - "The 30-minute local re-sweep covered low, mid, and high concurrency across aggregate TP2, disaggregate TP2/TP2, and disaggregate TP4/TP4. All 33 new-client steady-state samples completed with zero request errors and zero retract events; using the same historical output-token-throughput-per-user p50 metric as the old curve, the fourteen submitted points are the strict non-dominated set. The InferenceMAX sweep keeps the canonical 60-minute AgentX benchmark duration." + - "Align both configurations on the 2026-07-24 SGLang container, current aiperf be758d6, the merged Dynamo #12081 commit 5a638087, X-Dynamo-Session-ID affinity with the removed legacy conv-aware CLI path explicitly disabled, and NVIDIA/srt-slurm v1.0.36 while keeping the recipe and nvidia-master entries synchronized." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2477 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 7232174d92..5a41924a06 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -139,8 +139,8 @@ SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then - # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.22: the two features the - # cquil11 fork was pinned for are merged upstream (present in v1.0.22) — + # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.36: the two features the + # cquil11 fork was pinned for are merged upstream (present in v1.0.36) — # - `srtctl apply --no-preflight` (skip the in-process model FS check): # model.path resolves to /scratch/models/Qwen3.5-397B-A17B-NVFP4 # (compute-node-only NVMe), which the GHA runner pod can't stat, so @@ -151,7 +151,7 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == # reach the agentic_srt.sh srun). git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout v1.0.22 + git checkout v1.0.36 mkdir -p recipes/sglang/qwen3.5 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" \ recipes/sglang/qwen3.5 From fe019b126b0fb209bedd4fc0ec73982dc0206c71 Mon Sep 17 00:00:00 2001 From: Sahithi Chigurupati Date: Mon, 3 Aug 2026 15:28:34 -0700 Subject: [PATCH 2/3] fix(agentx): set agg decode num-worker to 0 for qwen3.5 GB300 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 7 aggregated (disagg: false) agg entries run a single 2-GPU worker serving both prefill and decode. With decode.num-worker: 1, process_agentic_result.py counts num_gpus=4/tp=4 instead of 2/2 — halving reported per-GPU throughput and mislabeling TP4. Set decode num-worker back to 0 to match the sibling aggregated configs. --- configs/nvidia-master.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 0b521719fd..e30979b901 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7116,7 +7116,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: additional-settings: - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml" decode: - num-worker: 1 + num-worker: 0 tp: 2 ep: 1 dp-attn: false @@ -7130,7 +7130,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: additional-settings: - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml" decode: - num-worker: 1 + num-worker: 0 tp: 2 ep: 1 dp-attn: false @@ -7144,7 +7144,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: additional-settings: - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml" decode: - num-worker: 1 + num-worker: 0 tp: 2 ep: 1 dp-attn: false @@ -7158,7 +7158,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: additional-settings: - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml" decode: - num-worker: 1 + num-worker: 0 tp: 2 ep: 1 dp-attn: false @@ -7172,7 +7172,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: additional-settings: - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml" decode: - num-worker: 1 + num-worker: 0 tp: 2 ep: 1 dp-attn: false @@ -7186,7 +7186,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: additional-settings: - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml" decode: - num-worker: 1 + num-worker: 0 tp: 2 ep: 1 dp-attn: false @@ -7200,7 +7200,7 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: additional-settings: - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml" decode: - num-worker: 1 + num-worker: 0 tp: 2 ep: 1 dp-attn: false From eb1e4107ac078fa736735066c90b2ef457babab0 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 5 Aug 2026 15:05:20 -0500 Subject: [PATCH 3/3] fix(agentx): align GB300 metrics and offload metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use srt-slurm v1.0.38 and enable SGLang backend metrics for the 14-point Qwen3.5 GB300 sweep. Record the recipes' existing HiCache DRAM tier and pinned Dynamo router so generated metadata matches the deployed topology. 使用 srt-slurm v1.0.38，并为 Qwen3.5 GB300 的 14 个扫描点启用 SGLang 后端指标。记录配方中现有的 HiCache DRAM 层和固定的 Dynamo 路由器，使生成的元数据与实际部署拓扑一致。 --- benchmarks/benchmark_lib.sh | 32 +++++++++++++++++ ...g-gb300-tp2-c1-mtp-hicache-jid2530006.yaml | 2 ++ ...-gb300-tp2-c24-mtp-hicache-jid2530012.yaml | 2 ++ ...-gb300-tp2-c32-mtp-hicache-jid2530013.yaml | 2 ++ ...-gb300-tp2-c40-mtp-hicache-jid2530015.yaml | 2 ++ ...-gb300-tp2-c48-mtp-hicache-jid2530017.yaml | 2 ++ ...-gb300-tp2-c52-mtp-hicache-jid2527406.yaml | 2 ++ ...-gb300-tp2-c64-mtp-hicache-jid2527410.yaml | 2 ++ ...p2-c72-mtp-hicache-session-jid2527415.yaml | 3 ++ ...4-c128-mtp-hicache-session-jid2527417.yaml | 3 ++ ...p4-c16-mtp-hicache-session-jid2530027.yaml | 3 ++ ...p4-c32-mtp-hicache-session-jid2530028.yaml | 3 ++ ...p4-c64-mtp-hicache-session-jid2530029.yaml | 3 ++ ...tp4-c8-mtp-hicache-session-jid2530030.yaml | 3 ++ ...p4-c96-mtp-hicache-session-jid2527409.yaml | 3 ++ configs/nvidia-master.yaml | 35 +++++++++++++++++-- perf-changelog.yaml | 4 ++- runners/launch_gb300-nv.sh | 8 +++-- 18 files changed, 109 insertions(+), 5 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 6979ca102f..81e646d12b 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -2006,6 +2006,36 @@ write_agentic_result_json() { "$AIPERF_PYTHON" "$INFMAX_CONTAINER_WORKSPACE/utils/generate_aiperf_plots.py" "$result_dir" 2>&1 || true } +validate_required_agentic_server_metrics() { + local result_dir="$1" + local required_prefix="${AIPERF_REQUIRED_SERVER_METRIC_PREFIX:-}" + local metrics_dir="$result_dir/aiperf_artifacts" + local metrics_json="$metrics_dir/server_metrics_export.json" + local metrics_csv="$metrics_dir/server_metrics_export.csv" + + # Opt-in so existing AgentX configurations retain their current contract. + # Recipes that require trace charts set a metric prefix (for example + # `sglang:`) and fail loudly instead of publishing a partial trace artifact. + if [ -z "$required_prefix" ]; then + return 0 + fi + + if [ ! -s "$metrics_json" ] || [ ! -s "$metrics_csv" ]; then + echo "ERROR: required AIPerf server metrics artifacts are missing or empty in $metrics_dir" >&2 + return 1 + fi + + # Avoid parsing the potentially multi-GiB JSON into memory. AIPerf writes + # metric names as JSON object keys, so a fixed-string scan establishes that + # backend engine metrics—not only frontend/router metrics—were captured. + if ! grep -F -m 1 -q "\"${required_prefix}" "$metrics_json"; then + echo "ERROR: $metrics_json contains no metric with required prefix '$required_prefix'" >&2 + return 1 + fi + + echo "Validated required AIPerf server metrics prefix '$required_prefix'" +} + run_agentic_replay_and_write_outputs() { local result_dir="$1" local replay_rc @@ -2044,4 +2074,6 @@ run_agentic_replay_and_write_outputs() { echo "ERROR: agentic trace replay produced invalid results after writing available artifacts" >&2 return "$validation_rc" fi + + validate_required_agentic_server_metrics "$result_dir" } diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml index 9a2c54927d..1ff01cb879 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml @@ -32,6 +32,7 @@ backend: aggregated: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true tensor-parallel-size: 2 @@ -87,3 +88,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml index 5c90cfd0ed..a3e1e47e50 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml @@ -32,6 +32,7 @@ backend: aggregated: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true tensor-parallel-size: 2 @@ -87,3 +88,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml index 1c2c2bc7b2..77324cb75d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml @@ -32,6 +32,7 @@ backend: aggregated: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true tensor-parallel-size: 2 @@ -87,3 +88,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml index 203846e7b7..aca9321b4c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml @@ -32,6 +32,7 @@ backend: aggregated: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true tensor-parallel-size: 2 @@ -87,3 +88,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml index da0adf6fdb..9aed4b026f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml @@ -32,6 +32,7 @@ backend: aggregated: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true tensor-parallel-size: 2 @@ -87,3 +88,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml index 4efc58a5a3..17c2095906 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml @@ -32,6 +32,7 @@ backend: aggregated: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true tensor-parallel-size: 2 @@ -87,3 +88,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml index a7b6dd0727..d5a14ddf4d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml @@ -32,6 +32,7 @@ backend: aggregated: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true tensor-parallel-size: 2 @@ -87,3 +88,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml index a5473b511f..626f1e21b3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml @@ -108,6 +108,7 @@ backend: decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 @@ -150,6 +151,7 @@ backend: speculative-num-draft-tokens: 4 served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 @@ -201,3 +203,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml index 2b481ddc3c..bf0b6dd7e2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml @@ -108,6 +108,7 @@ backend: decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 @@ -145,6 +146,7 @@ backend: prefill: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 @@ -196,3 +198,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml index d1c3481093..1657ffb134 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml @@ -108,6 +108,7 @@ backend: decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 @@ -145,6 +146,7 @@ backend: prefill: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 @@ -196,3 +198,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml index f992b34126..67bfe4f61c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml @@ -108,6 +108,7 @@ backend: decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 @@ -145,6 +146,7 @@ backend: prefill: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 @@ -196,3 +198,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml index 5bd9695e09..14012bd254 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml @@ -108,6 +108,7 @@ backend: decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 @@ -145,6 +146,7 @@ backend: prefill: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 @@ -196,3 +198,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml index 598ac987f1..f33326871d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml @@ -108,6 +108,7 @@ backend: decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 @@ -145,6 +146,7 @@ backend: prefill: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 @@ -196,3 +198,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml index 1ca34c2d88..0584b59017 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml @@ -108,6 +108,7 @@ backend: decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 @@ -145,6 +146,7 @@ backend: prefill: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 enable-cache-report: true + enable-metrics: true model-path: /model/ quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 @@ -196,3 +198,4 @@ benchmark: HF_HUB_CACHE: /hf_hub_cache AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 118ebfa55a..52e896f93b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6940,8 +6940,11 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: disagg: false scenarios: agentic-coding: - - search-space: + - dram-utilization: 0.90 + search-space: - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [1] prefill: num-worker: 1 @@ -6956,6 +6959,8 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [24] prefill: num-worker: 1 @@ -6970,6 +6975,8 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [32] prefill: num-worker: 1 @@ -6984,6 +6991,8 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [40] prefill: num-worker: 1 @@ -6998,6 +7007,8 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [48] prefill: num-worker: 1 @@ -7012,6 +7023,8 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [52] prefill: num-worker: 1 @@ -7026,6 +7039,8 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: ep: 1 dp-attn: false - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [64] prefill: num-worker: 1 @@ -7051,13 +7066,17 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang + router: { name: dynamo-router, version: "5a638087d82c990d35c69cb8e41c2c2582e9ef9a" } kv-p2p-transfer: mooncake multinode: true disagg: true scenarios: agentic-coding: - - search-space: + - dram-utilization: 0.90 + search-space: - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [8] prefill: num-worker: 1 @@ -7072,6 +7091,8 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [16] prefill: num-worker: 1 @@ -7086,6 +7107,8 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [32] prefill: num-worker: 1 @@ -7100,6 +7123,8 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [64] prefill: num-worker: 1 @@ -7114,6 +7139,8 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [96] prefill: num-worker: 1 @@ -7128,6 +7155,8 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [128] prefill: num-worker: 1 @@ -7142,6 +7171,8 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: ep: 1 dp-attn: false - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [72] prefill: num-worker: 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index da3eaf2357..dc5481915b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5466,5 +5466,7 @@ description: - "Refresh the Qwen3.5 GB300 AgentX frontier after aiperf 381758a restored real conversation pacing; replace the stale nine-point sweep with a complete fourteen-point locally validated curve: disaggregate TP4/TP4 c8/c16/c32/c64/c96/c128, disaggregate TP2/TP2 c72, and aggregate TP2 c1/c24/c32/c40/c48/c52/c64." - "The 30-minute local re-sweep covered low, mid, and high concurrency across aggregate TP2, disaggregate TP2/TP2, and disaggregate TP4/TP4. All 33 new-client steady-state samples completed with zero request errors and zero retract events; using the same historical output-token-throughput-per-user p50 metric as the old curve, the fourteen submitted points are the strict non-dominated set. The InferenceMAX sweep keeps the canonical 60-minute AgentX benchmark duration." - - "Align both configurations on the 2026-07-24 SGLang container, current aiperf be758d6, the merged Dynamo #12081 commit 5a638087, X-Dynamo-Session-ID affinity with the removed legacy conv-aware CLI path explicitly disabled, and NVIDIA/srt-slurm v1.0.36 while keeping the recipe and nvidia-master entries synchronized." + - "Align both configurations on the 2026-07-24 SGLang container, current aiperf be758d6, the merged Dynamo #12081 commit 5a638087, X-Dynamo-Session-ID affinity with the removed legacy conv-aware CLI path explicitly disabled, and NVIDIA/srt-slurm v1.0.38 with logical worker metric endpoint injection." + - "Enable SGLang backend metrics on every aggregate, prefill, and decode engine, and fail before publishing a partial trace artifact if required sglang: metrics are absent." + - "Record the recipes' active HiCache host-DRAM tier and, for disaggregated points, Dynamo router commit 5a638087 in nvidia-master metadata so generated artifacts no longer report kv_offloading=none, allocated_cpu_dram_gb=0, or a null router." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2477 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index b8d4ea3df2..f31714039b 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -185,7 +185,7 @@ SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then - # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.36: the two features the + # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.38: the two features the # cquil11 fork was pinned for are merged upstream (present in v1.0.36) — # - `srtctl apply --no-preflight` (skip the in-process model FS check): # model.path resolves to /scratch/models/Qwen3.5-397B-A17B-NVFP4 @@ -195,9 +195,13 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == # genuinely missing on the compute node. # - benchmark_stage propagates srun_options (container-remap-root must # reach the agentic_srt.sh srun). + # v1.0.38 additionally injects AIPERF_SERVER_METRICS_URLS for custom + # benchmarks using each logical SGLang worker leader. This is required for + # complete AgentX trace artifacts; the public frontend alone may expose no + # Prometheus endpoint or only Dynamo frontend metrics. git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout v1.0.36 + git checkout v1.0.38 mkdir -p recipes/sglang/qwen3.5 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" \ recipes/sglang/qwen3.5