From 6ebbf3dc003cac3369a0db4f65f7761556a5be09 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 4 Aug 2026 13:04:14 -0500 Subject: [PATCH 1/2] perf(agentx): refresh MI355X MiniMax M3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align the MI355X MXFP4 server with the supported MiniMax-M3 ROCm and EAGLE3-GQA configuration: block-128 Triton attention for target and draft, FP8 KV cache, AITER MoE, INT4 QuickReduce, three speculative tokens, and the validated synthetic acceptance length. Keep TP4 concurrency 1-24 GPU-resident, add a validated native DRAM offload point at concurrency 28, and add TEP8 concurrency 20, 28, and 36 for the higher-throughput range. Rename the benchmark and configuration as the MTP variant. 中文：将 MI355X MXFP4 服务配置与受支持的 MiniMax-M3 ROCm 和 EAGLE3-GQA 方案对齐：目标模型与草稿模型均使用 block-128 Triton 注意力，并启用 FP8 KV 缓存、AITER MoE、INT4 QuickReduce、3 个投机 token 以及经验证的合成接受长度。 TP4 并发 1-24 保持 KV 缓存驻留 GPU，在并发 28 增加已验证的原生 DRAM KV 卸载点，并增加 TEP8 并发 20、28、36 以覆盖高吞吐区间。基准脚本和配置重命名为 MTP 版本。 --- ...5x.sh => minimaxm3_fp4_mi355x_vllm_mtp.sh} | 71 ++++++++++++++----- configs/amd-master.yaml | 10 +-- 2 files changed, 60 insertions(+), 21 deletions(-) rename benchmarks/single_node/agentic/{minimaxm3_fp4_mi355x.sh => minimaxm3_fp4_mi355x_vllm_mtp.sh} (68%) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_vllm_mtp.sh similarity index 68% rename from benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh rename to benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_vllm_mtp.sh index 7faaf69a51..457ab9aab1 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_vllm_mtp.sh @@ -2,10 +2,11 @@ set -euo pipefail set -x -# Agentic trace replay benchmark for Minimax-M3 FP4 on MI355X using vLLM. +# Agentic trace replay benchmark for MiniMax-M3 FP4 on MI355X using vLLM +# EAGLE3 speculative decoding. # # Required env vars: -# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND, +# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, # TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION source "$(dirname "$0")/../../benchmark_lib.sh" @@ -17,11 +18,15 @@ source "$(dirname "$0")/../../benchmark_lib.sh" # setting it here makes the effective framework always lm-eval, never swebench. export EVAL_FRAMEWORK="lm-eval" -check_env_vars MODEL TP CONC KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" -PORT=8888 +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" +NUM_SPEC_TOKENS=3 +# golden_al_distribution/minimaxm3_eagle3_gqa.yaml: +# minimax-m3.thinking_on[3] +SYNTHETIC_ACCEPT_LEN=2.78 if [[ -n "${SLURM_JOB_ID+x}" ]]; then echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" @@ -40,6 +45,7 @@ else hf download "$MODEL" export MODEL_PATH="$MODEL" fi +hf download "$DRAFT_MODEL" rocm-smi || true amd-smi || true @@ -49,16 +55,33 @@ install_agentic_deps # ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" -LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" -OFFLOAD_ARGS=(--no-enable-prefix-caching) - -case "$KV_OFFLOAD_BACKEND" in - vllm-simple) +SERVER_PID="" +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# AgentX replays growing multi-turn prefixes, so keep prefix caching enabled +# for both GPU-resident and native-offload configurations. +OFFLOAD_ARGS=() + +case "${KV_OFFLOAD_BACKEND:-}" in + "") + require_agentic_kv_offload_none + ;; + vllm-native) + require_agentic_kv_offload_backend vllm-native unset VLLM_USE_SIMPLE_KV_OFFLOAD # Use vLLM's regular native KV-offload path (OffloadingConnector), - # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to + # NOT the SimpleCPUOffloadConnector. The "vllm-native" backend resolves to # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 # would switch it to SimpleCPUOffloadConnector. We intentionally leave # that env var UNSET here so the regular OffloadingConnector path is @@ -73,28 +96,39 @@ case "$KV_OFFLOAD_BACKEND" in --kv_offloading_size "$TOTAL_CPU_DRAM_GB" ) ;; + *) + echo "Unsupported KV_OFFLOAD_BACKEND: ${KV_OFFLOAD_BACKEND:-}" >&2 + exit 1 + ;; esac # ---- LLM server config ---------------------------------------------------------- PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then +if [ "$EP_SIZE" -gt 1 ]; then PARALLEL_ARGS+=(--enable-expert-parallel) fi +# Synthetic acceptance standardizes throughput against the committed golden +# EAGLE3-GQA curve. Accuracy evals must use real target verification. +if [ "${EVAL_ONLY}" = "true" ]; then + SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" +else + SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" +fi + echo "Starting vllm server..." export PYTHONNOUSERSITE=1 export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 export VLLM_USE_BREAKABLE_CUDAGRAPH=0 export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_USE_AITER_MOE=1 export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 +# The official MiniMax-M3 MI355X EAGLE3 baseline uses the native page-128 +# layout. The shuffled AITER page-16 path is a separate high-concurrency mode +# and must only be enabled after that band is validated independently. +export VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0 # INT4 quantized all-reduce for the (~1.5 MB) decode all-reduces, which are the # single biggest decode kernel at high concurrency. The MIN_SIZE_KB override is # required: vLLM's default INT4 quick-reduce size gate for (bf16, TP4) is 16 MB, @@ -113,11 +147,13 @@ VLLM_CMD=( --block-size 128 --gpu-memory-utilization 0.85 --language-model-only + --enable-prefix-caching --attention-backend TRITON_ATTN --moe-backend aiter --kv-cache-dtype fp8 --tool-call-parser minimax_m3 --enable-auto-tool-choice + --default-chat-template-kwargs '{"thinking_mode":"enabled"}' # NOTE: --reasoning-parser minimax_m3 is intentionally OMITTED. # MiniMax-M3 is an interleaved-thinking model: its ... # block MUST be round-tripped back into the conversation history every turn @@ -129,6 +165,7 @@ VLLM_CMD=( # inline in message.content, so the client preserves it across turns. The # tool-call parser above still extracts tool calls from the full output. --max-num-seqs "$CONC" + --speculative-config "$SPEC_CONFIG" "${OFFLOAD_ARGS[@]}" ) printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0d46afe2e1..c6b86446f7 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2231,8 +2231,8 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96], router: { name: vllm-router, version: "0.1.14" } } -minimaxm3-fp4-mi355x-vllm-agentic: - image: vllm/vllm-openai-rocm:nightly-dcfebf93f4eccf30f71872283331eee757915daf +minimaxm3-fp4-mi355x-vllm-agentic-mtp: + image: vllm/vllm-openai-rocm:nightly-5e35a6f4f9bbc217c599692157ca985c894373f7 model: amd/MiniMax-M3-MXFP4 model-prefix: minimaxm3 runner: cluster:mi355x-amds @@ -2241,9 +2241,11 @@ minimaxm3-fp4-mi355x-vllm-agentic: multinode: false scenarios: agentic-coding: - - dram-utilization: 0.80 + - dram-utilization: 0.20 search-space: - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 8, 16] } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24] } + - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, conc-list: [28] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [20, 28, 36] } dsv4-fp4-mi355x-sglang-disagg-agentic-hicache: image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 From 73e2f4fcdb0a273a40098c34cdbd5e68dd752c2e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 4 Aug 2026 13:04:21 -0500 Subject: [PATCH 2/2] chore(perf): record MI355X MiniMax M3 refresh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Append the performance changelog entry for the corrected AgentX harness. 中文：追加使用修正后 AgentX 测试框架的性能变更日志条目。 --- perf-changelog.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b5aa7a3857..db8274c8f1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5425,3 +5425,9 @@ description: - "Extend the search space to include the TP2EP1 configuration for fixed seq len 8k1k" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2446 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-agentic-mtp + description: + - "Initial MI355X MiniMax M3 with new AgentX harness" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2458