From 457ccad3836b3414fea077fa3b8a8fbc2056a8da Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 2 Aug 2026 17:39:41 -0400 Subject: [PATCH 01/11] kimik2.5 fp4 b200: v0.26.0 image, TP/DEP sweep, tuning --- .../fixed_seq_len/kimik2.5_fp4_b200.sh | 28 ++++++++++++++++--- configs/nvidia-master.yaml | 7 +++-- perf-changelog.yaml | 6 ++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh index 59b55c90c6..130a861497 100644 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh @@ -12,6 +12,20 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) +GMU=0.90 +PREFILL_SCHEDULE_ARGS=() +if [ "${DP_ATTENTION:-false}" = "true" ]; then + PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") + GMU=0.80 + PREFILL_SCHEDULE_ARGS=(--prefill-schedule-interval 4) +fi + +EP_ARGS=() +if [ "${EP_SIZE:-1}" -gt 1 ]; then + EP_ARGS=(--enable-expert-parallel) +fi + if [[ -n "$SLURM_JOB_ID" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" fi @@ -22,6 +36,7 @@ nvidia-smi export TORCH_CUDA_ARCH_LIST="10.0" export PYTHONNOUSERSITE=1 +export VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS="" SERVER_LOG=/workspace/server.log @@ -41,17 +56,22 @@ export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 set -x vllm serve $MODEL --host 0.0.0.0 --port $PORT \ ---tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.90 \ +"${PARALLEL_ARGS[@]}" \ +"${EP_ARGS[@]}" \ +"${PREFILL_SCHEDULE_ARGS[@]}" \ +--gpu-memory-utilization "$GMU" \ --max-model-len $MAX_MODEL_LEN \ --max-num-seqs $CONC \ --reasoning-parser kimi_k2 \ --tool-call-parser kimi_k2 \ --compilation_config.pass_config.fuse_allreduce_rms true \ --kv-cache-dtype fp8 \ ---max-cudagraph-capture-size 2048 \ +--max-cudagraph-capture-size "$CONC" \ --max-num-batched-tokens "$((ISL * 2 ))" \ ---stream-interval 20 --no-enable-prefix-caching \ +--stream-interval 32 \ +--attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ +--linear-backend flashinfer_cutedsl \ +--no-enable-prefix-caching \ --trust-remote-code > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a30af59641..c3a343dfa3 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1508,7 +1508,7 @@ kimik2.5-int4-h200-vllm-agentic: # Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. kimik2.5-fp4-b200-vllm: - image: vllm/vllm-openai:v0.22.0 + image: vllm/vllm-openai:nightly-e2fa28594f7baad142a426b0b6a2cfe2c79201c7 model: nvidia/Kimi-K2.5-NVFP4 model-prefix: kimik2.5 runner: b200 @@ -1520,8 +1520,9 @@ kimik2.5-fp4-b200-vllm: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 1, conc-end: 128 } + - { tp: 8, ep: 1, conc-start: 1, conc-end: 512 } + - { tp: 4, ep: 1, conc-start: 1, conc-end: 512 } + - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 512 } # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html # does not have a B300-specific recipe, so this config reuses the existing diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..55e9502403 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,9 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - kimik2.5-fp4-b200-vllm + description: + - "Kimi K2.5 NVFP4 B200 vLLM: v0.26.0 image, TP/DEP sweep, DEP gmu 0.80, tuning" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2443 From 8e8113fd8179cc228c2917090c37f1148c4bedc3 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 2 Aug 2026 19:21:50 -0400 Subject: [PATCH 02/11] kimik2.5 fp4 b200: nightly image, extend sweep space with TP/DEP arms --- benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh | 1 - perf-changelog.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh index 130a861497..fe76dedf68 100644 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh @@ -70,7 +70,6 @@ vllm serve $MODEL --host 0.0.0.0 --port $PORT \ --max-num-batched-tokens "$((ISL * 2 ))" \ --stream-interval 32 \ --attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ ---linear-backend flashinfer_cutedsl \ --no-enable-prefix-caching \ --trust-remote-code > $SERVER_LOG 2>&1 & diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 55e9502403..e5015d557d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5359,5 +5359,5 @@ - config-keys: - kimik2.5-fp4-b200-vllm description: - - "Kimi K2.5 NVFP4 B200 vLLM: v0.26.0 image, TP/DEP sweep, DEP gmu 0.80, tuning" + - "Kimi K2.5 NVFP4 B200 vLLM: nightly image, extend sweep space with TP/DEP arms, DEP gmu 0.80" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2443 From ece97bed3c1d1467413b245f7f8512719b70bc46 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 2 Aug 2026 23:32:04 -0400 Subject: [PATCH 03/11] kimik2.5 fp4 b200: nightly image, extend sweep space with TP/DEP arms --- configs/nvidia-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c3a343dfa3..ddcded830b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1522,7 +1522,7 @@ kimik2.5-fp4-b200-vllm: search-space: - { tp: 8, ep: 1, conc-start: 1, conc-end: 512 } - { tp: 4, ep: 1, conc-start: 1, conc-end: 512 } - - { tp: 8, ep: 8, dp-attn: true, conc-start: 128, conc-end: 512 } + - { tp: 8, ep: 8, dp-attn: true, conc-list: [512, 1024] } # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html # does not have a B300-specific recipe, so this config reuses the existing From baae8fcb3005d31bb944c8269a53730a272ef389 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Sun, 2 Aug 2026 23:34:12 -0400 Subject: [PATCH 04/11] drop --attention-config from B200 --- benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh index fe76dedf68..a39c02629d 100644 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh @@ -69,7 +69,6 @@ vllm serve $MODEL --host 0.0.0.0 --port $PORT \ --max-cudagraph-capture-size "$CONC" \ --max-num-batched-tokens "$((ISL * 2 ))" \ --stream-interval 32 \ ---attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ --no-enable-prefix-caching \ --trust-remote-code > $SERVER_LOG 2>&1 & From d815c2ce23cb6f9adad2ca1e3880586227b93e0b Mon Sep 17 00:00:00 2001 From: Xin Li Date: Mon, 3 Aug 2026 00:42:55 -0400 Subject: [PATCH 05/11] add attention-config with prefill query quant and linear-backend flashinfer_cutlass --- benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh index a39c02629d..7d8b591977 100644 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh @@ -69,6 +69,8 @@ vllm serve $MODEL --host 0.0.0.0 --port $PORT \ --max-cudagraph-capture-size "$CONC" \ --max-num-batched-tokens "$((ISL * 2 ))" \ --stream-interval 32 \ +--attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ +--linear-backend flashinfer_cutlass \ --no-enable-prefix-caching \ --trust-remote-code > $SERVER_LOG 2>&1 & From e1233ac8e5c12cc1771c7e17e427107e39166adb Mon Sep 17 00:00:00 2001 From: Xin Li Date: Mon, 3 Aug 2026 00:45:53 -0400 Subject: [PATCH 06/11] set max-cudagraph-capture-size to 2x concurrency --- benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh index 7d8b591977..f6a3fc0aa5 100644 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh @@ -66,7 +66,7 @@ vllm serve $MODEL --host 0.0.0.0 --port $PORT \ --tool-call-parser kimi_k2 \ --compilation_config.pass_config.fuse_allreduce_rms true \ --kv-cache-dtype fp8 \ ---max-cudagraph-capture-size "$CONC" \ +--max-cudagraph-capture-size "$((CONC * 2))" \ --max-num-batched-tokens "$((ISL * 2 ))" \ --stream-interval 32 \ --attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ From 9354ca90eb41d55ce9bf79d6dd7435385fb18b6e Mon Sep 17 00:00:00 2001 From: Xin Li Date: Mon, 3 Aug 2026 01:50:29 -0400 Subject: [PATCH 07/11] set max-num-batched-tokens to 16k --- benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh index f6a3fc0aa5..ab30e63745 100644 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh @@ -67,7 +67,7 @@ vllm serve $MODEL --host 0.0.0.0 --port $PORT \ --compilation_config.pass_config.fuse_allreduce_rms true \ --kv-cache-dtype fp8 \ --max-cudagraph-capture-size "$((CONC * 2))" \ ---max-num-batched-tokens "$((ISL * 2 ))" \ +--max-num-batched-tokens 16384 \ --stream-interval 32 \ --attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ --linear-backend flashinfer_cutlass \ From 6f03918464a816c1764d0b42b84b5c4e150c3c52 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Mon, 3 Aug 2026 03:29:07 -0400 Subject: [PATCH 08/11] remove max-num-batched-tokens, set VLLM_USE_V2_MODEL_RUNNER=0 --- benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh index ab30e63745..5f0bfa4798 100644 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh @@ -36,6 +36,7 @@ nvidia-smi export TORCH_CUDA_ARCH_LIST="10.0" export PYTHONNOUSERSITE=1 +export VLLM_USE_V2_MODEL_RUNNER=0 export VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS="" SERVER_LOG=/workspace/server.log @@ -67,7 +68,6 @@ vllm serve $MODEL --host 0.0.0.0 --port $PORT \ --compilation_config.pass_config.fuse_allreduce_rms true \ --kv-cache-dtype fp8 \ --max-cudagraph-capture-size "$((CONC * 2))" \ ---max-num-batched-tokens 16384 \ --stream-interval 32 \ --attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ --linear-backend flashinfer_cutlass \ From cfabd6a34aca0b11a4a1c4b74d078077e11df809 Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 4 Aug 2026 19:20:13 -0400 Subject: [PATCH 09/11] add --no-enable-flashinfer-autotune --- benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh index 5f0bfa4798..8a24ed7cc6 100644 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh @@ -71,6 +71,7 @@ vllm serve $MODEL --host 0.0.0.0 --port $PORT \ --stream-interval 32 \ --attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ --linear-backend flashinfer_cutlass \ +--no-enable-flashinfer-autotune \ --no-enable-prefix-caching \ --trust-remote-code > $SERVER_LOG 2>&1 & From 207fde31520026e4890485b83b1ae5bd46b24a1e Mon Sep 17 00:00:00 2001 From: Xin Li Date: Tue, 4 Aug 2026 19:22:01 -0400 Subject: [PATCH 10/11] fix perf-changelog --- perf-changelog.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 01af97c83c..a892d7e6de 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5450,12 +5450,8 @@ - "Cap the 1P1D TP4 concurrency sweep at 256 (was 512); drop the 2P1D TP4 layout (128/256/512) as it is CI-flaky with negligible curve impact." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1943 - - config-keys: - - minimaxm3-fp4-mi355x-vllm-disagg + - kimik2.5-fp4-b200-vllm description: - - "Update the MiniMax-M3 MXFP4 MI355X vLLM disagg image to vllm/vllm-openai-rocm:nightly-2dfaae752b4db0d43cfc0715c780e33be030d0f1 (from rocm/vllm-dev:vllm-0.23.1-rocm723-mi35x-mori-0625) for AITER MoE and shared-expert fusion support." - - "Export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 for MiniMax-M3-MXFP4 (both prefill and decode)." - - "Enable prefill-only INT4 quick-reduce: set VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 and VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB=2048 on the prefill workers via a new prefill_env channel (mirrors the existing decode_env path in server_vllm.sh)." - - "Cap the 1P1D TP4 concurrency sweep at 256 (was 512); drop the 2P1D TP4 layout (128/256/512) as it is CI-flaky with negligible curve impact." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1943 + - "Kimi K2.5 NVFP4 B200 vLLM: nightly image, extend sweep space with TP/DEP arms, DEP gmu 0.80" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2443 From 583c165917122287b181d1f1a42be32f9033677b Mon Sep 17 00:00:00 2001 From: Xin Li Date: Wed, 5 Aug 2026 00:08:29 -0400 Subject: [PATCH 11/11] remove --no-enable-flashinfer-autotune, add TEP arms --- .../fixed_seq_len/kimik2.5_fp4_b200.sh | 1 - configs/nvidia-master.yaml | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh index 939ea0bd83..793bf2553c 100644 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b200.sh @@ -71,7 +71,6 @@ vllm serve $MODEL --host 0.0.0.0 --port $PORT \ --stream-interval 32 \ --attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ --linear-backend flashinfer_cutlass \ ---no-enable-flashinfer-autotune \ --no-enable-prefix-caching \ --trust-remote-code > $SERVER_LOG 2>&1 & diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 492b9c2d61..ac60822d51 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1416,6 +1416,28 @@ kimik2.5-int4-h200-vllm: # Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. kimik2.5-fp4-b200-vllm: + image: vllm/vllm-openai:nightly-e2fa28594f7baad142a426b0b6a2cfe2c79201c7 + model: nvidia/Kimi-K2.5-NVFP4 + model-prefix: kimik2.5 + runner: b200 + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-start: 1, conc-end: 512 } + - { tp: 4, ep: 1, conc-start: 1, conc-end: 512 } + - { tp: 8, ep: 8, dp-attn: false, conc-start: 1, conc-end: 512 } + - { tp: 4, ep: 4, dp-attn: false, conc-start: 1, conc-end: 512 } + - { tp: 8, ep: 8, dp-attn: true, conc-list: [512, 1024] } + +# NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/moonshotai/Kimi-K2.5.html +# does not have a B300-specific recipe, so this config reuses the existing +# Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. +m: image: vllm/vllm-openai:nightly-e2fa28594f7baad142a426b0b6a2cfe2c79201c7 model: nvidia/Kimi-K2.5-NVFP4 model-prefix: kimik2.5