diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 6979ca102f..81e646d12b 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -2006,6 +2006,36 @@ write_agentic_result_json() { "$AIPERF_PYTHON" "$INFMAX_CONTAINER_WORKSPACE/utils/generate_aiperf_plots.py" "$result_dir" 2>&1 || true } +validate_required_agentic_server_metrics() { + local result_dir="$1" + local required_prefix="${AIPERF_REQUIRED_SERVER_METRIC_PREFIX:-}" + local metrics_dir="$result_dir/aiperf_artifacts" + local metrics_json="$metrics_dir/server_metrics_export.json" + local metrics_csv="$metrics_dir/server_metrics_export.csv" + + # Opt-in so existing AgentX configurations retain their current contract. + # Recipes that require trace charts set a metric prefix (for example + # `sglang:`) and fail loudly instead of publishing a partial trace artifact. + if [ -z "$required_prefix" ]; then + return 0 + fi + + if [ ! -s "$metrics_json" ] || [ ! -s "$metrics_csv" ]; then + echo "ERROR: required AIPerf server metrics artifacts are missing or empty in $metrics_dir" >&2 + return 1 + fi + + # Avoid parsing the potentially multi-GiB JSON into memory. AIPerf writes + # metric names as JSON object keys, so a fixed-string scan establishes that + # backend engine metrics—not only frontend/router metrics—were captured. + if ! grep -F -m 1 -q "\"${required_prefix}" "$metrics_json"; then + echo "ERROR: $metrics_json contains no metric with required prefix '$required_prefix'" >&2 + return 1 + fi + + echo "Validated required AIPerf server metrics prefix '$required_prefix'" +} + run_agentic_replay_and_write_outputs() { local result_dir="$1" local replay_rc @@ -2044,4 +2074,6 @@ run_agentic_replay_and_write_outputs() { echo "ERROR: agentic trace replay produced invalid results after writing available artifacts" >&2 return "$validation_rc" fi + + validate_required_agentic_server_metrics "$result_dir" } diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml new file mode 100644 index 0000000000..1ff01cb879 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml @@ -0,0 +1,91 @@ +name: agg-gb300-tp2-c1-mtp-hicache-jid2530006 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml new file mode 100644 index 0000000000..a3e1e47e50 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml @@ -0,0 +1,91 @@ +name: agg-gb300-tp2-c24-mtp-hicache-jid2530012 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml new file mode 100644 index 0000000000..77324cb75d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml @@ -0,0 +1,91 @@ +name: agg-gb300-tp2-c32-mtp-hicache-jid2530013 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml new file mode 100644 index 0000000000..aca9321b4c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml @@ -0,0 +1,91 @@ +name: agg-gb300-tp2-c40-mtp-hicache-jid2530015 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml new file mode 100644 index 0000000000..9aed4b026f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml @@ -0,0 +1,91 @@ +name: agg-gb300-tp2-c48-mtp-hicache-jid2530017 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml new file mode 100644 index 0000000000..17c2095906 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml @@ -0,0 +1,91 @@ +name: agg-gb300-tp2-c52-mtp-hicache-jid2527406 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml new file mode 100644 index 0000000000..d5a14ddf4d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml @@ -0,0 +1,91 @@ +name: agg-gb300-tp2-c64-mtp-hicache-jid2527410 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 2 + agg_nodes: 1 + agg_workers: 1 + gpus_per_agg: 2 +infra: + nats_max_payload_mb: 8 +frontend: + type: sglang +backend: + aggregated_environment: + NCCL_NVLS_ENABLE: '1' + PYTHONNOUSERSITE: '1' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGL_ENABLE_JIT_DEEPGEMM: 'false' + TORCH_CUDA_ARCH_LIST: '10.0' + sglang_config: + aggregated: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-symm-mem: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + mamba-ssm-dtype: bfloat16 + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 8192 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + cuda-graph-max-bs: 80 + max-running-requests: 72 + max-prefill-tokens: 16384 + chunked-prefill-size: 16384 + mem-fraction-static: 0.75 + max-mamba-cache-size: 360 + allow-auto-truncate: true + stream-interval: 50 + scheduler-recv-interval: 10 + tokenizer-worker-num: 6 + page-size: 64 + enable-hierarchical-cache: true + hicache-ratio: 0.9 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-write-policy: write_back + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'false' + TP: '2' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml deleted file mode 100644 index 02c88cc299..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml +++ /dev/null @@ -1,114 +0,0 @@ -name: "agg-gb300-tp4-c1-mtp-hicache-jid2191933" - -# Agentic-coding SGLang aggregated recipe for Qwen3.5-397B-A17B-NVFP4 on GB300 -# (single aggregated worker, TP4, NEXTN MTP + hierarchical cache). -# -# Ported from the manually-run srtctl base:/zip_override_conc: starter to the -# flat single-variant agentic schema. Pure sglang (no dynamo): frontend.type -# sglang serves directly. Concurrency comes from the GHA matrix (exported as -# CONC into agentic_srt.sh), not the recipe. Serving tuning preserved verbatim -# from the source run; only CI scaffolding is added. - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - nats_max_payload_mb: 8 - -frontend: - type: sglang - -backend: - type: sglang - aggregated_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - NCCL_NVLS_ENABLE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 256 - max-running-requests: 1024 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.8 - max-mamba-cache-size: 1024 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 1.01 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user into the container as a non-root uid; - # agentic_srt.sh's apt-get install git step needs EUID 0. Remap to uid 0 inside - # the container. srt-slurm renders empty-string values as flag-only srun args. - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - # Aggregated: one worker serves both prefill and decode, so GPU accounting is - # the single-worker form num_gpus = TP (not the disagg prefill+decode sum). - # Force the single-node post-proc path; TP must match sglang_config tp-size. - IS_MULTINODE: "false" - TP: "4" - # Match the source run's dataset (256k cc-traces-with-subagents variant). - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - # Container-side path of the aiperf mmap dataset cache; host-side mount wired - # via launch_gb300-nv.sh srtslurm.yaml default_mounts. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also via default_mounts). - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml deleted file mode 100644 index 5891df5b32..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml +++ /dev/null @@ -1,114 +0,0 @@ -name: "agg-gb300-tp4-c96-mtp-hicache-jid2195211" - -# Agentic-coding SGLang aggregated recipe for Qwen3.5-397B-A17B-NVFP4 on GB300 -# (single aggregated worker, TP4, NEXTN MTP + hierarchical cache). -# -# Ported from the manually-run srtctl base:/zip_override_conc: starter to the -# flat single-variant agentic schema. Pure sglang (no dynamo): frontend.type -# sglang serves directly. Concurrency comes from the GHA matrix (exported as -# CONC into agentic_srt.sh), not the recipe. Serving tuning preserved verbatim -# from the source run; only CI scaffolding is added. - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - nats_max_payload_mb: 8 - -frontend: - type: sglang - -backend: - type: sglang - aggregated_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - TORCH_CUDA_ARCH_LIST: '10.0' - PYTHONNOUSERSITE: '1' - NCCL_NVLS_ENABLE: '1' - SGL_ENABLE_JIT_DEEPGEMM: 'false' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - sglang_config: - aggregated: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-symm-mem: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - mamba-ssm-dtype: bfloat16 - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 8192 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - cuda-graph-max-bs: 256 - max-running-requests: 1024 - max-prefill-tokens: 16384 - chunked-prefill-size: 16384 - mem-fraction-static: 0.8 - max-mamba-cache-size: 1024 - allow-auto-truncate: true - stream-interval: 50 - scheduler-recv-interval: 10 - tokenizer-worker-num: 6 - page-size: 64 - enable-hierarchical-cache: true - hicache-ratio: 1.05 - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-write-policy: write_through - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user into the container as a non-root uid; - # agentic_srt.sh's apt-get install git step needs EUID 0. Remap to uid 0 inside - # the container. srt-slurm renders empty-string values as flag-only srun args. - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - # Aggregated: one worker serves both prefill and decode, so GPU accounting is - # the single-worker form num_gpus = TP (not the disagg prefill+decode sum). - # Force the single-node post-proc path; TP must match sglang_config tp-size. - IS_MULTINODE: "false" - TP: "4" - # Match the source run's dataset (256k cc-traces-with-subagents variant). - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - # Container-side path of the aiperf mmap dataset cache; host-side mount wired - # via launch_gb300-nv.sh srtslurm.yaml default_mounts. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also via default_mounts). - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml new file mode 100644 index 0000000000..626f1e21b3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml @@ -0,0 +1,206 @@ +name: disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + decode_nodes: 1 + decode_workers: 1 + gpu_type: gb300 + gpus_per_decode: 2 + gpus_per_node: 2 + gpus_per_prefill: 2 + prefill_nodes: 1 + prefill_workers: 1 +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +backend: + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 80 + max-running-requests: 80 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + decode-log-interval: 10 + mamba-max-states-per-path: -1 + prefill: + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 2 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + max-mamba-cache-size: 320 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + mamba-max-states-per-path: 1 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml similarity index 70% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml index 6449be871c..bf0b6dd7e2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml @@ -1,168 +1,123 @@ -name: "disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562" - -# Agentic-coding SGLang disaggregated 2P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300, ported to the flat single-variant agentic schema. Concurrency is -# NOT a recipe field: the GHA matrix fans out one job per concurrency and -# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, -# frontend, dynamo hash, topology) is preserved verbatim from the source run. - +name: disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417 model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 dynamo: install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a slurm: - time_limit: "8:00:00" - + time_limit: '8:00:00' health_check: max_attempts: 1440 interval_seconds: 10 - resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 2 decode_nodes: 1 - prefill_workers: 2 decode_workers: 1 - gpus_per_prefill: 4 + gpu_type: gb300 gpus_per_decode: 4 - + gpus_per_node: 4 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 infra: etcd_nats_dedicated_node: true nats_max_payload_mb: 8 - frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - num_additional_frontends: 0 + enable_multiple_frontends: true + num_additional_frontends: 4 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" + PIP_BREAK_SYSTEM_PACKAGES: '1' args: router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - PIP_BREAK_SYSTEM_PACKAGES: "1" - + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false prefill-round-robin-balance: true mamba-scheduler-strategy: no_buffer mamba-track-interval: 128 @@ -172,44 +127,75 @@ backend: disaggregation-bootstrap-port: 31000 page-size: 64 attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl + moe-runner-backend: flashinfer_trtllm linear-attn-decode-backend: flashinfer disable-shared-experts-fusion: true - moe-a2a-backend: deepep - deepep-mode: low_latency ep-dispatch-algorithm: static eplb-algorithm: deepseek speculative-algorithm: NEXTN speculative-num-steps: 3 speculative-eagle-topk: 1 speculative-num-draft-tokens: 4 - speculative-moe-runner-backend: deep_gemm - speculative-moe-a2a-backend: deepep chunked-prefill-size: 4096 mem-fraction-static: 0.75 max-mamba-cache-size: 1024 max-running-requests: 512 - cuda-graph-max-bs: 64 + cuda-graph-max-bs: 160 watchdog-timeout: 1000000 decode-log-interval: 10 - + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + type: sglang sbatch_directives: - mem: "0" - cpus-per-task: "144" - + mem: '0' + cpus-per-task: '144' srun_options: - mem: "0" - container-remap-root: "" - + mem: '0' + container-remap-root: '' benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" + PORT: '8000' + IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml similarity index 80% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml index cbf7157955..1657ffb134 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml @@ -1,161 +1,114 @@ -name: "disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783" - -# Agentic-coding SGLang disaggregated 1P1D recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300, ported to the flat single-variant agentic schema. Concurrency is -# NOT a recipe field: the GHA matrix fans out one job per concurrency and -# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, -# frontend, dynamo hash, topology) is preserved verbatim from the source run. - +name: disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027 model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 dynamo: install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a slurm: - time_limit: "8:00:00" - + time_limit: '8:00:00' health_check: max_attempts: 1440 interval_seconds: 10 - resources: + decode_nodes: 1 + decode_workers: 1 gpu_type: gb300 + gpus_per_decode: 4 gpus_per_node: 4 + gpus_per_prefill: 4 prefill_nodes: 1 - decode_nodes: 1 prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - infra: etcd_nats_dedicated_node: true nats_max_payload_mb: 8 - frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" + PIP_BREAK_SYSTEM_PACKAGES: '1' args: router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - PIP_BREAK_SYSTEM_PACKAGES: "1" - + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 @@ -190,24 +143,59 @@ backend: cuda-graph-max-bs: 64 watchdog-timeout: 1000000 decode-log-interval: 10 - + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + type: sglang sbatch_directives: - mem: "0" - cpus-per-task: "144" - + mem: '0' + cpus-per-task: '144' srun_options: - mem: "0" - container-remap-root: "" - + mem: '0' + container-remap-root: '' benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" + PORT: '8000' + IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml similarity index 79% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml index 4f093f5ec8..67bfe4f61c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml @@ -1,161 +1,114 @@ -name: "disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440" - -# Agentic-coding SGLang disaggregated 1P1D recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300, ported to the flat single-variant agentic schema. Concurrency is -# NOT a recipe field: the GHA matrix fans out one job per concurrency and -# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, -# frontend, dynamo hash, topology) is preserved verbatim from the source run. - +name: disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028 model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 dynamo: install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a slurm: - time_limit: "8:00:00" - + time_limit: '8:00:00' health_check: max_attempts: 1440 interval_seconds: 10 - resources: + decode_nodes: 1 + decode_workers: 1 gpu_type: gb300 + gpus_per_decode: 4 gpus_per_node: 4 + gpus_per_prefill: 4 prefill_nodes: 1 - decode_nodes: 1 prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - infra: etcd_nats_dedicated_node: true nats_max_payload_mb: 8 - frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" + PIP_BREAK_SYSTEM_PACKAGES: '1' args: router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - PIP_BREAK_SYSTEM_PACKAGES: "1" - + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: false - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 @@ -190,24 +143,59 @@ backend: cuda-graph-max-bs: 64 watchdog-timeout: 1000000 decode-log-interval: 10 - + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + type: sglang sbatch_directives: - mem: "0" - cpus-per-task: "144" - + mem: '0' + cpus-per-task: '144' srun_options: - mem: "0" - container-remap-root: "" - + mem: '0' + container-remap-root: '' benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" + PORT: '8000' + IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml deleted file mode 100644 index 52aabae1ef..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml +++ /dev/null @@ -1,237 +0,0 @@ -name: "disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439" - -# Agentic-coding SGLang disaggregated 1P1D recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300 (TP4 prefill / TP4 decode, NEXTN MTP + hierarchical cache). -# -# Ported from the manually-run srtctl base:/zip_override_conc: starter to the -# flat single-variant schema the agentic CI flow expects (modeled on the -# proven sglang/deepseek-v4/agentic recipes). Concurrency is NOT a recipe -# field here: the GHA matrix fans out one job per concurrency from the -# master-config conc-list, exporting CONC into agentic_srt.sh. Serving tuning -# (sglang_config, prefill/decode env, frontend, dynamo hash, topology) is -# preserved verbatim from the source run; only the CI scaffolding is added. - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 1 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -infra: - # Dedicated etcd/nats node — under the dynamo router + multiple frontends - # the infra services need their own node (matches the proven agentic recipes). - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: true - num_additional_frontends: 4 - env: - # The sglang image is PEP 668 externally-managed; the runtime dynamo - # install (dynamo.hash source build) runs pip and fails with - # "externally-managed-environment" without this. - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - SGLANG_DISAGG_STAGING_BUFFER: '1' - SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' - SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - # Lets the runtime dynamo install's pip bypass PEP 668 on the sglang image. - PIP_BREAK_SYSTEM_PACKAGES: "1" - - sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: false - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 4 - data-parallel-size: 1 - expert-parallel-size: 1 - enable-dp-attention: false - enable-dp-lm-head: false - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user into the container as a non-root - # uid; agentic_srt.sh's apt-get install git step needs EUID 0. Remap to - # uid 0 inside the container. srt-slurm renders empty-string values as - # flag-only srun args. - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Match the source run's dataset (256k cc-traces-with-subagents variant). - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - # Per-session affinity so multi-turn KV cache is reused across turns - # (recipe name: "convaware"); build_replay_cmd adds - # --use-dynamo-conv-aware-routing for dynamo-* frameworks. - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - # Container-side path of the aiperf mmap dataset cache; host-side mount - # wired via launch_gb300-nv.sh srtslurm.yaml default_mounts. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also via default_mounts) so the trace dataset - # isn't re-downloaded each run. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml new file mode 100644 index 0000000000..14012bd254 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml @@ -0,0 +1,201 @@ +name: disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + decode_nodes: 1 + decode_workers: 1 + gpu_type: gb300 + gpus_per_decode: 4 + gpus_per_node: 4 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +backend: + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 80 + watchdog-timeout: 1000000 + decode-log-interval: 10 + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml similarity index 72% rename from benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml rename to benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml index 75e0dddc08..f33326871d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml @@ -1,168 +1,123 @@ -name: "disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685" - -# Agentic-coding SGLang disaggregated 1P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300, ported to the flat single-variant agentic schema. Concurrency is -# NOT a recipe field: the GHA matrix fans out one job per concurrency and -# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, -# frontend, dynamo hash, topology) is preserved verbatim from the source run. - +name: disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030 model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 dynamo: install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a slurm: - time_limit: "8:00:00" - + time_limit: '8:00:00' health_check: max_attempts: 1440 interval_seconds: 10 - resources: + decode_nodes: 1 + decode_workers: 1 gpu_type: gb300 + gpus_per_decode: 4 gpus_per_node: 4 + gpus_per_prefill: 4 prefill_nodes: 1 - decode_nodes: 1 prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 4 - infra: etcd_nats_dedicated_node: true nats_max_payload_mb: 8 - frontend: type: dynamo nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID enable_multiple_frontends: true num_additional_frontends: 4 env: - PIP_BREAK_SYSTEM_PACKAGES: "1" + PIP_BREAK_SYSTEM_PACKAGES: '1' args: router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '512' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - PIP_BREAK_SYSTEM_PACKAGES: "1" - + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 decode: served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true model-path: /model/ trust-remote-code: true quantization: modelopt_fp4 kv-cache-dtype: fp8_e4m3 tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false prefill-round-robin-balance: true mamba-scheduler-strategy: no_buffer mamba-track-interval: 128 @@ -172,19 +127,15 @@ backend: disaggregation-bootstrap-port: 31000 page-size: 64 attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl + moe-runner-backend: flashinfer_trtllm linear-attn-decode-backend: flashinfer disable-shared-experts-fusion: true - moe-a2a-backend: deepep - deepep-mode: low_latency ep-dispatch-algorithm: static eplb-algorithm: deepseek speculative-algorithm: NEXTN speculative-num-steps: 3 speculative-eagle-topk: 1 speculative-num-draft-tokens: 4 - speculative-moe-runner-backend: deep_gemm - speculative-moe-a2a-backend: deepep chunked-prefill-size: 4096 mem-fraction-static: 0.75 max-mamba-cache-size: 1024 @@ -192,24 +143,59 @@ backend: cuda-graph-max-bs: 64 watchdog-timeout: 1000000 decode-log-interval: 10 - + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + type: sglang sbatch_directives: - mem: "0" - cpus-per-task: "144" - + mem: '0' + cpus-per-task: '144' srun_options: - mem: "0" - container-remap-root: "" - + mem: '0' + container-remap-root: '' benchmark: type: custom command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" + PORT: '8000' + IS_MULTINODE: 'true' WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml new file mode 100644 index 0000000000..0584b59017 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml @@ -0,0 +1,201 @@ +name: disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409 +model: + path: qwen3.5-fp4 + container: dynamo-sglang + precision: fp4 +dynamo: + install: true + hash: 5a638087d82c990d35c69cb8e41c2c2582e9ef9a +slurm: + time_limit: '8:00:00' +health_check: + max_attempts: 1440 + interval_seconds: 10 +resources: + decode_nodes: 1 + decode_workers: 1 + gpu_type: gb300 + gpus_per_decode: 4 + gpus_per_node: 4 + gpus_per_prefill: 4 + prefill_nodes: 1 + prefill_workers: 1 +infra: + etcd_nats_dedicated_node: true + nats_max_payload_mb: 8 +frontend: + type: dynamo + nginx_session_affinity: true + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: '1' + args: + router-mode: kv + router-session-affinity-ttl-secs: '3600' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None + nginx_session_affinity_header: X-Dynamo-Session-ID +backend: + decode_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + MC_TE_METRIC: 'true' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' + SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' + SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' + SGLANG_ENABLE_JIT_DEEPGEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' + SGLANG_HEALTH_CHECK_TIMEOUT: '1800' + SGLANG_HEALTH_STARTING_OK: '1' + SGLANG_MOE_NVFP4_DISPATCH: '1' + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + prefill_environment: + FLASHINFER_DISABLE_VERSION_CHECK: '1' + FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache + MC_FORCE_MNNVL: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NO_COLOR: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache + SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' + SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' + SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' + SGLANG_DISAGG_STAGING_BUFFER: '1' + SGLANG_DISAGG_STAGING_BUFFER_SIZE_MB: '1024' + SGLANG_DISAGG_STAGING_POOL_SIZE_MB: '8192' + SGLANG_ENABLE_FLASHINFER_GEMM: 'true' + SGLANG_ENABLE_SPEC_V2: '1' + SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass + SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' + SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' + TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' + SGLANG_SIMULATE_ACC_LEN: '3.39' + SGLANG_SIMULATE_ACC_METHOD: match-expected + SGLANG_SIMULATE_ACC_TOKEN_MODE: real-draft-token + sglang_config: + decode: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + trust-remote-code: true + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + prefill-round-robin-balance: true + mamba-scheduler-strategy: no_buffer + mamba-track-interval: 128 + mamba-ssm-dtype: bfloat16 + disaggregation-mode: decode + disable-radix-cache: true + disaggregation-bootstrap-port: 31000 + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + disable-shared-experts-fusion: true + ep-dispatch-algorithm: static + eplb-algorithm: deepseek + speculative-algorithm: NEXTN + speculative-num-steps: 3 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 4 + chunked-prefill-size: 4096 + mem-fraction-static: 0.75 + max-mamba-cache-size: 1024 + max-running-requests: 512 + cuda-graph-max-bs: 128 + watchdog-timeout: 1000000 + decode-log-interval: 10 + prefill: + served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 + enable-cache-report: true + enable-metrics: true + model-path: /model/ + quantization: modelopt_fp4 + kv-cache-dtype: fp8_e4m3 + trust-remote-code: true + tensor-parallel-size: 4 + data-parallel-size: 1 + expert-parallel-size: 1 + enable-dp-attention: false + enable-dp-lm-head: false + mamba-ssm-dtype: bfloat16 + disaggregation-mode: prefill + disaggregation-bootstrap-port: 31000 + load-balance-method: round_robin + watchdog-timeout: 1000000 + log-level: info + page-size: 64 + attention-backend: trtllm_mha + moe-runner-backend: flashinfer_trtllm + linear-attn-decode-backend: flashinfer + mem-fraction-static: 0.8 + chunked-prefill-size: 65536 + disable-cuda-graph: true + mamba-scheduler-strategy: extra_buffer + mamba-track-interval: 2048 + enable-hierarchical-cache: true + hicache-write-policy: write_back + mamba-max-states-per-path: 3 + max-mamba-cache-size: 1100 + hicache-io-backend: kernel + hicache-mem-layout: page_first_direct + hicache-ratio: 0.9 + type: sglang +sbatch_directives: + mem: '0' + cpus-per-task: '144' +srun_options: + mem: '0' + container-remap-root: '' +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml deleted file mode 100644 index 62f641193f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml +++ /dev/null @@ -1,215 +0,0 @@ -name: "disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613" - -# Agentic-coding SGLang disaggregated 3P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300, ported to the flat single-variant agentic schema. Concurrency is -# NOT a recipe field: the GHA matrix fans out one job per concurrency and -# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, -# frontend, dynamo hash, topology) is preserved verbatim from the source run. - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 3 - decode_nodes: 2 - prefill_workers: 3 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - num_additional_frontends: 0 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - PIP_BREAK_SYSTEM_PACKAGES: "1" - - sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - moe-a2a-backend: deepep - deepep-mode: low_latency - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - speculative-moe-runner-backend: deep_gemm - speculative-moe-a2a-backend: deepep - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml deleted file mode 100644 index 66a8a650b0..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml +++ /dev/null @@ -1,215 +0,0 @@ -name: "disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740" - -# Agentic-coding SGLang disaggregated 4P1D DEP recipe for Qwen3.5-397B-A17B-NVFP4 -# on GB300, ported to the flat single-variant agentic schema. Concurrency is -# NOT a recipe field: the GHA matrix fans out one job per concurrency and -# exports CONC into agentic_srt.sh. Serving tuning (sglang_config, env, -# frontend, dynamo hash, topology) is preserved verbatim from the source run. - -model: - path: "qwen3.5-fp4" - container: "dynamo-sglang" - precision: "fp4" - -dynamo: - install: true - hash: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: gb300 - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 4 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - nats_max_payload_mb: 8 - -frontend: - type: dynamo - nginx_session_affinity: true - nginx_session_affinity_header: X-Dynamo-Session-ID - enable_multiple_frontends: false - num_additional_frontends: 0 - env: - PIP_BREAK_SYSTEM_PACKAGES: "1" - args: - router-mode: kv - router-session-affinity-ttl-secs: "3600" - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - -backend: - type: sglang - - prefill_environment: - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - PIP_BREAK_SYSTEM_PACKAGES: "1" - decode_environment: - SGLANG_SIMULATE_ACC_LEN: '3.39' - SGLANG_SIMULATE_ACC_METHOD: match-expected - SGLANG_SIMULATE_ACC_TOKEN_MODE: "real-draft-token" - SGLANG_DISAGG_STAGING_BUFFER: '1' - NO_COLOR: '1' - TORCH_DISTRIBUTED_DEFAULT_TIMEOUT: '1800' - SGLANG_ENABLE_SPEC_V2: '1' - PYTHONUNBUFFERED: '1' - NCCL_MNNVL_ENABLE: '1' - NCCL_CUMEM_ENABLE: '1' - NCCL_NVLS_ENABLE: '1' - MC_FORCE_MNNVL: '1' - MC_TE_METRIC: 'true' - SGLANG_ENABLE_FLASHINFER_GEMM: 'true' - SGLANG_ENABLE_JIT_DEEPGEMM: 'true' - SGLANG_FLASHINFER_FP4_GEMM_BACKEND: cutlass - SGLANG_MOE_NVFP4_DISPATCH: '1' - SGLANG_CUTEDSL_MOE_NVFP4_DISPATCH: '1' - SGLANG_NVFP4_CKPT_FP8_NEXTN_MOE: '1' - SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' - FLASHINFER_DISABLE_VERSION_CHECK: '1' - SGLANG_DG_CACHE_DIR: /configs/deepgemm-cache - FLASHINFER_WORKSPACE_BASE: /configs/flashinfer-cache - SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE: '100000' - SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT: '100000' - SGLANG_DISAGGREGATION_WAITING_TIMEOUT: '100000' - SGLANG_DECODE_BOOTSTRAP_TIMEOUT: '1000' - SGLANG_HACK_SEQ_BOOTSTRAP_ROOM: '1' - SGLANG_MOONCAKE_CUSTOM_MEM_POOL: 'True' - SGLANG_USE_MESSAGE_QUEUE_BROADCASTER: '0' - SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK: '1' - SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK: '256' - SGLANG_HEALTH_CHECK_TIMEOUT: '1800' - SGLANG_HEALTH_STARTING_OK: '1' - SGLANG_ENABLE_HEALTH_ENDPOINT_GENERATION: '0' - PIP_BREAK_SYSTEM_PACKAGES: "1" - - sglang_config: - prefill: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - trust-remote-code: true - tensor-parallel-size: 4 - data-parallel-size: 4 - expert-parallel-size: 4 - enable-dp-attention: true - enable-dp-lm-head: true - moe-dense-tp-size: 1 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: prefill - disaggregation-bootstrap-port: 31000 - load-balance-method: round_robin - watchdog-timeout: 1000000 - log-level: info - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_trtllm - linear-attn-decode-backend: flashinfer - mem-fraction-static: 0.8 - chunked-prefill-size: 65536 - disable-cuda-graph: true - mamba-scheduler-strategy: extra_buffer - mamba-track-interval: 2048 - enable-hierarchical-cache: true - hicache-write-policy: write_back - hicache-io-backend: kernel - hicache-mem-layout: page_first_direct - hicache-ratio: 1.01 - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - decode: - served-model-name: nvidia/Qwen3.5-397B-A17B-NVFP4 - model-path: /model/ - trust-remote-code: true - quantization: modelopt_fp4 - kv-cache-dtype: fp8_e4m3 - tensor-parallel-size: 8 - data-parallel-size: 8 - expert-parallel-size: 8 - moe-dense-tp-size: 1 - enable-dp-attention: true - enable-dp-lm-head: true - prefill-round-robin-balance: true - mamba-scheduler-strategy: no_buffer - mamba-track-interval: 128 - mamba-ssm-dtype: bfloat16 - disaggregation-mode: decode - disable-radix-cache: true - disaggregation-bootstrap-port: 31000 - page-size: 64 - attention-backend: trtllm_mha - moe-runner-backend: flashinfer_cutedsl - linear-attn-decode-backend: flashinfer - disable-shared-experts-fusion: true - moe-a2a-backend: deepep - deepep-mode: low_latency - ep-dispatch-algorithm: static - eplb-algorithm: deepseek - speculative-algorithm: NEXTN - speculative-num-steps: 3 - speculative-eagle-topk: 1 - speculative-num-draft-tokens: 4 - speculative-moe-runner-backend: deep_gemm - speculative-moe-a2a-backend: deepep - chunked-prefill-size: 4096 - mem-fraction-static: 0.75 - max-mamba-cache-size: 1024 - max-running-requests: 512 - cuda-graph-max-bs: 64 - watchdog-timeout: 1000000 - decode-log-interval: 10 - -sbatch_directives: - mem: "0" - cpus-per-task: "144" - -srun_options: - mem: "0" - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126_256k - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 73a83024c9..52e896f93b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6925,6 +6925,10 @@ qwen3.5-fp4-gb300-dynamo-sglang: ep: 16 dp-attn: true +# GB300 SGLang AgentX, re-swept across low/mid/high concurrency after aiperf +# restored real inter-tool-call pacing. The seven aggregate frontier points use +# one TP2 worker with MTP/NEXTN + hierarchical KV cache. CONFIG_FILE carries the +# full srtctl recipe; the matrix topology stays in sync for naming/GPU accounting. qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1 model: nvidia/Qwen3.5-397B-A17B-NVFP4 @@ -6936,42 +6940,125 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: disagg: false scenarios: agentic-coding: - - dram-utilization: 0.80 + - dram-utilization: 0.90 search-space: - - spec-decoding: "mtp" + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } conc-list: [1] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c1-mtp-hicache-jid2530006.yaml" + decode: + num-worker: 0 + tp: 2 + ep: 1 + dp-attn: false + - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } + conc-list: [24] prefill: num-worker: 1 - tp: 4 + tp: 2 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c24-mtp-hicache-jid2530012.yaml" decode: num-worker: 0 - tp: 4 + tp: 2 ep: 1 dp-attn: false - - spec-decoding: "mtp" - conc-list: [96] + - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } + conc-list: [32] prefill: num-worker: 1 - tp: 4 + tp: 2 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c32-mtp-hicache-jid2530013.yaml" decode: num-worker: 0 - tp: 4 + tp: 2 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [40] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c40-mtp-hicache-jid2530015.yaml" + decode: + num-worker: 0 + tp: 2 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [48] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c48-mtp-hicache-jid2530017.yaml" + decode: + num-worker: 0 + tp: 2 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [52] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c52-mtp-hicache-jid2527406.yaml" + decode: + num-worker: 0 + tp: 2 + ep: 1 + dp-attn: false + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [64] + prefill: + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp2-c64-mtp-hicache-jid2527410.yaml" + decode: + num-worker: 0 + tp: 2 ep: 1 dp-attn: false +# The seven disaggregated frontier points use six TP4 shapes plus one TP2 +# shape. Stable X-Dynamo-Session-ID affinity replaces the removed conv-aware +# routing message path. qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1 model: nvidia/Qwen3.5-397B-A17B-NVFP4 @@ -6979,129 +7066,126 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: runner: cluster:gb300-nv precision: fp4 framework: dynamo-sglang - router: { name: dynamo-router, version: "8e4bfa69ba486f07f6083fbe8cd026f432f520bc" } + router: { name: dynamo-router, version: "5a638087d82c990d35c69cb8e41c2c2582e9ef9a" } kv-p2p-transfer: mooncake multinode: true disagg: true scenarios: agentic-coding: - - dram-utilization: 0.80 + - dram-utilization: 0.90 search-space: - # 1P1D STP (TP4 prefill / TP4 decode). 2 GB300 nodes. - - spec-decoding: "mtp" - conc-list: [32] + - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } + conc-list: [8] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-convaware-jid2212783.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c8-mtp-hicache-session-jid2530030.yaml" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false - - spec-decoding: "mtp" - conc-list: [64] - kv-offloading: none + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [16] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-convaware-jid2214439.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c16-mtp-hicache-session-jid2530027.yaml" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false - - spec-decoding: "mtp" - conc-list: [96] - kv-offloading: none + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: hicache } + conc-list: [32] prefill: num-worker: 1 tp: 4 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-convaware-jid2214440.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c32-mtp-hicache-session-jid2530028.yaml" decode: num-worker: 1 tp: 4 ep: 1 dp-attn: false - # 1P1D wide-EP (DEP4 prefill / DEP4 decode). 2 GB300 nodes. - - spec-decoding: "mtp" - conc-list: [128] + - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } + conc-list: [64] prefill: num-worker: 1 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-dep4-dep4-c128-mtp-hicache-convaware-jid2220685.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c64-mtp-hicache-session-jid2530029.yaml" decode: num-worker: 1 tp: 4 - ep: 4 - dp-attn: true - # 2P1D wide-EP (2x DEP4 prefill / DEP4 decode). 3 GB300 nodes. - - spec-decoding: "mtp" - conc-list: [192] + ep: 1 + dp-attn: false + - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } + conc-list: [96] prefill: - num-worker: 2 + num-worker: 1 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-2p1d-dep4-dep4-c192-mtp-hicache-convaware-jid2230562.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c96-mtp-hicache-session-jid2527409.yaml" decode: num-worker: 1 tp: 4 - ep: 4 - dp-attn: true - # 3P1D wide-EP (3x DEP4 prefill / DEP8 decode). 5 GB300 nodes. - - spec-decoding: "mtp" - conc-list: [256] + ep: 1 + dp-attn: false + - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } + conc-list: [128] prefill: - num-worker: 3 + num-worker: 1 tp: 4 - ep: 4 - dp-attn: true + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-3p1d-dep4-dep8-c256-mtp-hicache-convaware-jid2228613.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp4-tp4-c128-mtp-hicache-session-jid2527417.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # 4P1D wide-EP (4x DEP4 prefill / DEP8 decode). 6 GB300 nodes. - - spec-decoding: "mtp" - conc-list: [384] + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: hicache } + conc-list: [72] prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true + num-worker: 1 + tp: 2 + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-4p1d-dep4-dep8-c384-mtp-hicache-convaware-jid2239740.yaml" + - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/disagg-gb300-1p1d-tp2-tp2-c72-mtp-hicache-session-jid2527415.yaml" decode: num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true + tp: 2 + ep: 1 + dp-attn: false # ---------- 1k1k high-throughput (wide-EP decode, EAGLE MTP) ---------- qwen3.5-fp8-b300-sglang-agentic-hicache: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4dd819f46d..dc5481915b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5460,3 +5460,13 @@ - "Use vllm/vllm-openai:nightly-387189c42997b27e2c04b5d97ef8190ffa2bf909 with prefix caching enabled, default KV-cache dtype, FlashInfer TRT-LLM attention with FP8 indexer KV, and the EAGLE3 drafter on FLASH_ATTN." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2328 +- config-keys: + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg + - qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg + description: + - "Refresh the Qwen3.5 GB300 AgentX frontier after aiperf 381758a restored real conversation pacing; replace the stale nine-point sweep with a complete fourteen-point locally validated curve: disaggregate TP4/TP4 c8/c16/c32/c64/c96/c128, disaggregate TP2/TP2 c72, and aggregate TP2 c1/c24/c32/c40/c48/c52/c64." + - "The 30-minute local re-sweep covered low, mid, and high concurrency across aggregate TP2, disaggregate TP2/TP2, and disaggregate TP4/TP4. All 33 new-client steady-state samples completed with zero request errors and zero retract events; using the same historical output-token-throughput-per-user p50 metric as the old curve, the fourteen submitted points are the strict non-dominated set. The InferenceMAX sweep keeps the canonical 60-minute AgentX benchmark duration." + - "Align both configurations on the 2026-07-24 SGLang container, current aiperf be758d6, the merged Dynamo #12081 commit 5a638087, X-Dynamo-Session-ID affinity with the removed legacy conv-aware CLI path explicitly disabled, and NVIDIA/srt-slurm v1.0.38 with logical worker metric endpoint injection." + - "Enable SGLang backend metrics on every aggregate, prefill, and decode engine, and fail before publishing a partial trace artifact if required sglang: metrics are absent." + - "Record the recipes' active HiCache host-DRAM tier and, for disaggregated points, Dynamo router commit 5a638087 in nvidia-master metadata so generated artifacts no longer report kv_offloading=none, allocated_cpu_dram_gb=0, or a null router." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2477 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 1669b39dfc..f31714039b 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -185,8 +185,8 @@ SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then - # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.22: the two features the - # cquil11 fork was pinned for are merged upstream (present in v1.0.22) — + # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.38: the two features the + # cquil11 fork was pinned for are merged upstream (present in v1.0.36) — # - `srtctl apply --no-preflight` (skip the in-process model FS check): # model.path resolves to /scratch/models/Qwen3.5-397B-A17B-NVFP4 # (compute-node-only NVMe), which the GHA runner pod can't stat, so @@ -195,9 +195,13 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == # genuinely missing on the compute node. # - benchmark_stage propagates srun_options (container-remap-root must # reach the agentic_srt.sh srun). + # v1.0.38 additionally injects AIPERF_SERVER_METRICS_URLS for custom + # benchmarks using each logical SGLang worker leader. This is required for + # complete AgentX trace artifacts; the public frontend alone may expose no + # Prometheus endpoint or only Dynamo frontend metrics. git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout v1.0.22 + git checkout v1.0.38 mkdir -p recipes/sglang/qwen3.5 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" \ recipes/sglang/qwen3.5