diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-dep8-eagle3-c64-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-dep8-eagle3-c64-8k1k.yaml new file mode 100644 index 0000000000..0378818634 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-dep8-eagle3-c64-8k1k.yaml @@ -0,0 +1,115 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-dep8-mxfp8-8k1k-eagle3-c64" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "64" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c1-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c1-8k1k.yaml new file mode 100644 index 0000000000..b0eb8d8621 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c1-8k1k.yaml @@ -0,0 +1,113 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp4-mxfp8-8k1k-eagle3-c1" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 4 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c8-8k1k.yaml new file mode 100644 index 0000000000..e280dc3084 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c8-8k1k.yaml @@ -0,0 +1,113 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp4-mxfp8-8k1k-eagle3-c8" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 4 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c1-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c1-8k1k.yaml new file mode 100644 index 0000000000..290cce803e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c1-8k1k.yaml @@ -0,0 +1,113 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp8-mxfp8-8k1k-eagle3-c1" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 8 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c4-8k1k.yaml new file mode 100644 index 0000000000..9034801b32 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c4-8k1k.yaml @@ -0,0 +1,113 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp8-mxfp8-8k1k-eagle3-c4" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 8 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c8-8k1k.yaml new file mode 100644 index 0000000000..e893a70651 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c8-8k1k.yaml @@ -0,0 +1,113 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp8-mxfp8-8k1k-eagle3-c8" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 8 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/2p1d-dep2-dep8-eagle3-c512-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/2p1d-dep2-dep8-eagle3-c512-8k1k.yaml new file mode 100644 index 0000000000..4e7e02e386 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/2p1d-dep2-dep8-eagle3-c512-8k1k.yaml @@ -0,0 +1,115 @@ +name: "minimax-m3-vllm-disagg-gb300-2p1d-dep2-dep8-mxfp8-8k1k-eagle3-c512" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/3p1d-dep2-dep8-eagle3-c256-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/3p1d-dep2-dep8-eagle3-c256-8k1k.yaml new file mode 100644 index 0000000000..cef1e28a6b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/3p1d-dep2-dep8-eagle3-c256-8k1k.yaml @@ -0,0 +1,115 @@ +name: "minimax-m3-vllm-disagg-gb300-3p1d-dep2-dep8-mxfp8-8k1k-eagle3-c256" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c1024-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c1024-8k1k.yaml new file mode 100644 index 0000000000..27434e7693 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c1024-8k1k.yaml @@ -0,0 +1,115 @@ +name: "minimax-m3-vllm-disagg-gb300-4p1d-dep2-dep8-mxfp8-8k1k-eagle3-c1024" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 4 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1024" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c2048-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c2048-8k1k.yaml new file mode 100644 index 0000000000..b9c9b36c8a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c2048-8k1k.yaml @@ -0,0 +1,115 @@ +name: "minimax-m3-vllm-disagg-gb300-4p1d-dep2-dep8-mxfp8-8k1k-eagle3-c2048" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 4 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/6p1d-dep2-dep8-eagle3-c2048-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/6p1d-dep2-dep8-eagle3-c2048-8k1k.yaml new file mode 100644 index 0000000000..3fd038b09e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/6p1d-dep2-dep8-eagle3-c2048-8k1k.yaml @@ -0,0 +1,115 @@ +name: "minimax-m3-vllm-disagg-gb300-6p1d-dep2-dep8-mxfp8-8k1k-eagle3-c2048" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 3 + decode_nodes: 2 + prefill_workers: 6 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 90e54195b6..a9debc7c2a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8055,6 +8055,177 @@ minimaxm3-fp8-gb300-dynamo-vllm: ep: 8 dp-attn: true +minimaxm3-fp8-gb300-dynamo-vllm-mtp: + image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: gb300-nv + precision: fp8 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.4.0.dev20260730" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c1-8k1k.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [8] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c8-8k1k.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c1-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [4] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c4-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [8] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c8-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [64] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-dep8-eagle3-c64-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [512] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/2p1d-dep2-dep8-eagle3-c512-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [256] + prefill: + num-worker: 3 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/3p1d-dep2-dep8-eagle3-c256-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [1024] + prefill: + num-worker: 4 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c1024-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [2048] + prefill: + num-worker: 4 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c2048-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [2048] + prefill: + num-worker: 6 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/6p1d-dep2-dep8-eagle3-c2048-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + qwen3.5-fp4-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 model: nvidia/Qwen3.5-397B-A17B-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7a064f8090..10712ddad4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5373,4 +5373,19 @@ - "Use the shared AIPerf watchdog to cap whole-trajectory runtime idle gaps at 300 seconds" - "Image: lmsysorg/sglang:v0.5.16-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2420 - + +- config-keys: + - minimaxm3-fp8-gb300-dynamo-vllm-mtp + scenario-type: + - fixed-seq-len + description: + - "Add MiniMax-M3-MXFP8 GB300 8k1k disaggregated Dynamo-vLLM EAGLE3 MTP Pareto" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2478 + +- config-keys: + - minimaxm3-fp8-gb300-dynamo-vllm-mtp + scenario-type: + - fixed-seq-len + description: + - "Bump MiniMax-M3-MXFP8 GB300 EAGLE3 MTP recipes to vLLM nightly 5e35a6f4f and enable CUTLASS MSA sparse decode backend on decode servers" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2478 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 7232174d92..36fcad9e50 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -226,7 +226,11 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout sa-submission-q2-2026 + if [[ "${SPEC_DECODING:-}" == "mtp" ]]; then + git checkout v1.0.38 + else + git checkout sa-submission-q2-2026 + fi mkdir -p recipes/vllm/minimax-m3-gb300-fp8 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8" recipes/vllm/minimax-m3-gb300-fp8 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then