diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b300.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b300.sh index db6d3fb0d..8f1906aaa 100755 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b300.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_b300.sh @@ -16,6 +16,21 @@ check_env_vars \ RANDOM_RANGE_RATIO \ RESULT_FILENAME +PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) +GMU=0.90 +PREFILL_SCHEDULE_ARGS=() +if [ "${DP_ATTENTION:-false}" = "true" ]; then + PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") + GMU=0.85 + PREFILL_SCHEDULE_ARGS=(--prefill-schedule-interval 4) +fi + +EP_ARGS=() +if [ "${EP_SIZE:-1}" -gt 1 ]; then + EP_ARGS=(--enable-expert-parallel) +fi + + # `hf download` creates the target dir if missing and is itself idempotent. # When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE # Either way, MODEL_PATH is what the server is launched with. @@ -37,6 +52,9 @@ nvidia-smi export TORCH_CUDA_ARCH_LIST="10.0" export PYTHONNOUSERSITE=1 +export VLLM_USE_V2_MODEL_RUNNER=0 +export VLLM_FLASHINFER_AUTOTUNE_SKIP_OPS="" +export VLLM_RPC_TIMEOUT=600000 SERVER_LOG=/workspace/server.log @@ -49,13 +67,20 @@ start_gpu_monitor set -x vllm serve $MODEL_PATH --served-model-name $MODEL --host 0.0.0.0 --port $PORT \ ---tensor-parallel-size $TP \ ---gpu-memory-utilization 0.90 \ +"${PARALLEL_ARGS[@]}" \ +"${EP_ARGS[@]}" \ +"${PREFILL_SCHEDULE_ARGS[@]}" \ +--gpu-memory-utilization "$GMU" \ --max-model-len $MAX_MODEL_LEN \ --max-num-seqs $CONC \ --reasoning-parser kimi_k2 \ --tool-call-parser kimi_k2 \ --compilation_config.pass_config.fuse_allreduce_rms true \ +--kv-cache-dtype fp8 \ +--max-cudagraph-capture-size "$((CONC * 2))" \ +--stream-interval 32 \ +--attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' \ +--linear-backend flashinfer_cutlass \ --no-enable-prefix-caching \ --trust-remote-code > $SERVER_LOG 2>&1 & diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 51a650397..1e9f93cbd 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1436,7 +1436,7 @@ kimik2.5-fp4-b200-vllm: # Kimi-K2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. kimik2.5-fp4-b300-vllm: - image: vllm/vllm-openai:v0.22.0 + image: vllm/vllm-openai:nightly-e2fa28594f7baad142a426b0b6a2cfe2c79201c7 model: nvidia/Kimi-K2.5-NVFP4 model-prefix: kimik2.5 runner: b300 @@ -1448,8 +1448,27 @@ kimik2.5-fp4-b300-vllm: - isl: 8192 osl: 1024 search-space: - - { tp: 8, ep: 1, conc-start: 1, conc-end: 4 } - - { tp: 4, ep: 1, conc-start: 1, conc-end: 128 } + - { tp: 8, ep: 1, conc-list: [1] } + - { tp: 4, ep: 1, conc-start: 1, conc-end: 512 } + - { tp: 8, ep: 8, dp-attn: false, conc-list: [1] } + - { tp: 4, ep: 4, dp-attn: false, conc-start: 1, conc-end: 512 } + - { tp: 4, ep: 4, dp-attn: true, conc-start: 128, conc-end: 512 } +-vllm: + image: vllm/vllm-openai:nightly-e2fa28594f7baad142a426b0b6a2cfe2c79201c7 + model: nvidia/Kimi-K2.5-NVFP4 + model-prefix: kimik2.5 + runner: b300 + precision: fp4 + framework: vllm + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, ep: 1, conc-list: [1] } + - { tp: 4, ep: 1, conc-start: 1, conc-end: 512 } + - { tp: 4, ep: 4, dp-attn: true, conc-start: 128, conc-end: 512 } dsr1-fp8-b200-sglang-mtp: image: lmsysorg/sglang:v0.5.12-cu130 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c855c972c..479597ad6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5449,3 +5449,9 @@ - "Enable prefill-only INT4 quick-reduce: set VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 and VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB=2048 on the prefill workers via a new prefill_env channel (mirrors the existing decode_env path in server_vllm.sh)." - "Cap the 1P1D TP4 concurrency sweep at 256 (was 512); drop the 2P1D TP4 layout (128/256/512) as it is CI-flaky with negligible curve impact." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1943 + +- config-keys: + - kimik2.5-fp4-b300-vllm + description: + - "Kimi K2.5 NVFP4 B300 vLLM: nightly image, extend sweep space with TP/DEP arms, TP8 conc-1 only, DEP gmu 0.85" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2442 diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index f39616801..fdf9df08c 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -406,7 +406,9 @@ else # MODEL stays as the HF id for the client (--served-model-name, tokenizer); # MODEL_PATH is what the server reads weights from. MODEL_BASENAME="${MODEL##*/}" - if [[ " ${STAGED_MODELS[*]} " == *" ${MODEL_BASENAME} "* ]]; then + if [[ $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then + export MODEL_PATH="${WRITABLE_MODELS_DIR%/}/${MODEL_BASENAME}" + elif [[ " ${STAGED_MODELS[*]} " == *" ${MODEL_BASENAME} "* ]]; then export MODEL_PATH="${HF_HUB_CACHE_MOUNT%/}/${MODEL_BASENAME}" else export MODEL_PATH="${WRITABLE_MODELS_DIR%/}/${MODEL_BASENAME}"