diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-dep8-eagle3-c64-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-dep8-eagle3-c64-8k1k.yaml new file mode 100644 index 0000000000..72fcd6cc2e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-dep8-eagle3-c64-8k1k.yaml @@ -0,0 +1,116 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-dep8-mxfp8-8k1k-eagle3-c64" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "64" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c1-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c1-8k1k.yaml new file mode 100644 index 0000000000..990e540075 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c1-8k1k.yaml @@ -0,0 +1,113 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp4-mxfp8-8k1k-eagle3-c1" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 4 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c8-8k1k.yaml new file mode 100644 index 0000000000..ff8f84cf47 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c8-8k1k.yaml @@ -0,0 +1,113 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp4-mxfp8-8k1k-eagle3-c8" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 4 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c1-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c1-8k1k.yaml new file mode 100644 index 0000000000..4fa642e3d9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c1-8k1k.yaml @@ -0,0 +1,113 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp8-mxfp8-8k1k-eagle3-c1" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 8 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c4-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c4-8k1k.yaml new file mode 100644 index 0000000000..280819ae5a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c4-8k1k.yaml @@ -0,0 +1,113 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp8-mxfp8-8k1k-eagle3-c4" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 8 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "4" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c8-8k1k.yaml new file mode 100644 index 0000000000..94322297f0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c8-8k1k.yaml @@ -0,0 +1,113 @@ +name: "minimax-m3-vllm-disagg-gb300-1p1d-dep2-tp8-mxfp8-8k1k-eagle3-c8" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 8 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "8" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/2p1d-dep2-dep8-eagle3-c512-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/2p1d-dep2-dep8-eagle3-c512-8k1k.yaml new file mode 100644 index 0000000000..e75a8eb06a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/2p1d-dep2-dep8-eagle3-c512-8k1k.yaml @@ -0,0 +1,116 @@ +name: "minimax-m3-vllm-disagg-gb300-2p1d-dep2-dep8-mxfp8-8k1k-eagle3-c512" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 1 + decode_nodes: 2 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "512" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/3p1d-dep2-dep8-eagle3-c256-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/3p1d-dep2-dep8-eagle3-c256-8k1k.yaml new file mode 100644 index 0000000000..3a7753467c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/3p1d-dep2-dep8-eagle3-c256-8k1k.yaml @@ -0,0 +1,116 @@ +name: "minimax-m3-vllm-disagg-gb300-3p1d-dep2-dep8-mxfp8-8k1k-eagle3-c256" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 3 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "256" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c1024-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c1024-8k1k.yaml new file mode 100644 index 0000000000..4b25736627 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c1024-8k1k.yaml @@ -0,0 +1,116 @@ +name: "minimax-m3-vllm-disagg-gb300-4p1d-dep2-dep8-mxfp8-8k1k-eagle3-c1024" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 4 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "1024" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c2048-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c2048-8k1k.yaml new file mode 100644 index 0000000000..c9537a9fcd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c2048-8k1k.yaml @@ -0,0 +1,116 @@ +name: "minimax-m3-vllm-disagg-gb300-4p1d-dep2-dep8-mxfp8-8k1k-eagle3-c2048" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 4 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/6p1d-dep2-dep8-eagle3-c2048-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/6p1d-dep2-dep8-eagle3-c2048-8k1k.yaml new file mode 100644 index 0000000000..601864149f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/6p1d-dep2-dep8-eagle3-c2048-8k1k.yaml @@ -0,0 +1,116 @@ +name: "minimax-m3-vllm-disagg-gb300-6p1d-dep2-dep8-mxfp8-8k1k-eagle3-c2048" + +model: + path: "minimax-m3-mxfp8" + container: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + precision: "fp8" + +identity: + model: + repo: "MiniMaxAI/MiniMax-M3-MXFP8" + revision: "c5454eb03678d8710e54a4e0fc681b9f3b4a3dba" + container: + image: "vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7" + frameworks: + dynamo: "1.4.0.dev20260730" + vllm: "0.26.1rc1.dev255+g5e35a6f4f" + +dynamo: + install: true + version: "1.4.0.dev20260730" + request_plane: "nats" + +health_check: + max_attempts: 720 + interval_seconds: 10 + +sbatch_directives: + mem: "0" + cpus-per-task: "72" + +srun_options: + mem: "0" + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + prefill_nodes: 3 + decode_nodes: 2 + prefill_workers: 6 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 8 + +frontend: + type: "dynamo" + enable_multiple_frontends: false + +backend: + type: "vllm" + connector: null + + prefill_environment: &worker-environment + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_FLOAT32_MATMUL_PRECISION: "high" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: "mnnvl" + UCX_CUDA_IPC_ENABLE_MNNVL: "y" + UCX_MODULE_DIR: "/usr/local/lib/python3.12/dist-packages/nixl_cu13.libs/ucx" + UCX_RNDV_PIPELINE_ERROR_HANDLING: "y" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + + decode_environment: *worker-environment + + vllm_config: + prefill: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + + decode: + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8", "minimax_m3_msa_decode_backend": "cutlass"}' + speculative-config: '{"method":"eagle3","model":"Inferact/MiniMax-M3-EAGLE3-GQA","num_speculative_tokens":3,"attention_backend":"FLASH_ATTN"}' + tensor-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + kv-cache-dtype: "fp8" + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + concurrencies: "2048" + req_rate: "inf" + num_warmup_mult: 2 + random_range_ratio: 0.8 + use_chat_template: true diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 39087de3ae..d6ea947df5 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -8008,6 +8008,178 @@ minimaxm3-fp4-b300-dynamo-vllm-mtp-legacy-dep4: # MiniMax-M3 GB300 disagg sweep — refreshed recipe set (no Marlin variants). # All prefill DEP2 (TP1 DP2 EP, 2 GPU/worker). Decode: DEP4, TEP8, DEP8, TEP4. # 4 GPU/node (GB300 NVL72). kv-cache-dtype=fp8. srun_options mem=0 required. +minimaxm3-fp8-gb300-dynamo-vllm-mtp: + image: vllm/vllm-openai:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: gb300-nv + precision: fp8 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.4.0.dev20260730" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c1-8k1k.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [8] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp4-eagle3-c8-8k1k.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [1] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c1-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [4] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c4-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [8] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-tp8-eagle3-c8-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: "mtp" + conc-list: [64] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/1p1d-dep2-dep8-eagle3-c64-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [512] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/2p1d-dep2-dep8-eagle3-c512-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [256] + prefill: + num-worker: 3 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/3p1d-dep2-dep8-eagle3-c256-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [1024] + prefill: + num-worker: 4 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c1024-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [2048] + prefill: + num-worker: 4 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/4p1d-dep2-dep8-eagle3-c2048-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: "mtp" + conc-list: [2048] + prefill: + num-worker: 6 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3-gb300-fp8/8k1k/mtp/6p1d-dep2-dep8-eagle3-c2048-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + + minimaxm3-fp8-gb300-dynamo-vllm: image: vllm/vllm-openai:nightly-4080263bb2c5d10deac17aaeb88e0823bc35bca9 model: MiniMaxAI/MiniMax-M3-MXFP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 43dc7fdb6f..41bd57494c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5431,3 +5431,12 @@ description: - "Add MiniMax M3 NVFP4 B200 disagg EAGLE3 MTP recipes, mirrored from B300" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2462 + +- config-keys: + - minimaxm3-fp8-gb300-dynamo-vllm-mtp + scenario-type: + - fixed-seq-len + description: + - "Add MiniMax-M3-MXFP8 GB300 8k1k disaggregated Dynamo-vLLM EAGLE3 MTP Pareto on vLLM nightly 5e35a6f4f" + - "The six dep8 decode configurations enable the CUTLASS MSA sparse decode backend and FULL_DECODE_ONLY cudagraph mode; the five low-concurrency c1/c4/c8 configurations use neither" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2486 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 7232174d92..36fcad9e50 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -226,7 +226,11 @@ elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout sa-submission-q2-2026 + if [[ "${SPEC_DECODING:-}" == "mtp" ]]; then + git checkout v1.0.38 + else + git checkout sa-submission-q2-2026 + fi mkdir -p recipes/vllm/minimax-m3-gb300-fp8 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3-gb300-fp8" recipes/vllm/minimax-m3-gb300-fp8 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then