From 7906bedb1ac62670756bdd8dc75b659a64e6a589 Mon Sep 17 00:00:00 2001 From: Sirra Date: Tue, 4 Aug 2026 16:26:14 +0530 Subject: [PATCH 1/5] [AMD] [WIP] [AGENTX] MiniMax-M3 Support on MI355X with MTP Signed-off-by: Sirra --- .../agentic/minimaxm3_fp4_mi355x_mtp.sh | 139 ++++++++++++++++++ configs/amd-master.yaml | 14 ++ perf-changelog.yaml | 7 + 3 files changed, 160 insertions(+) create mode 100644 benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh new file mode 100644 index 000000000..a00947844 --- /dev/null +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# Agentic trace replay benchmark for Minimax-M3 FP4 on MI355X using vLLM. +# +# Required env vars: +# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND, +# TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION + +source "$(dirname "$0")/../../benchmark_lib.sh" + +# Force the eval framework to lm-eval for this recipe. run_eval derives its +# default as swebench for agentic scenarios (scenario_default=swebench when +# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence +# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so +# setting it here makes the effective framework always lm-eval, never swebench. +export EVAL_FRAMEWORK="lm-eval" + +check_env_vars MODEL TP CONC KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION + +echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" + +PORT=8888 + +if [[ -n "${SLURM_JOB_ID+x}" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +# ROCR/HIP visibility for vLLM 0.14+ +if [[ -n "${ROCR_VISIBLE_DEVICES+x}" ]]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "${MODEL_PATH:-}" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi + +rocm-smi || true +amd-smi || true + +resolve_trace_source +install_agentic_deps + +# ---- Server config ---------------------------------------------------------- +SERVER_LOG="$RESULT_DIR/server.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" +mkdir -p "$RESULT_DIR" + +OFFLOAD_ARGS=(--no-enable-prefix-caching) + +case "$KV_OFFLOAD_BACKEND" in + vllm-simple) + unset VLLM_USE_SIMPLE_KV_OFFLOAD + # Use vLLM's regular native KV-offload path (OffloadingConnector), + # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to + # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 + # would switch it to SimpleCPUOffloadConnector. We intentionally leave + # that env var UNSET here so the regular OffloadingConnector path is + # used. The shortcut --kv_offloading_backend native + --kv_offloading_size + # form constructs the KVTransferConfig at engine startup + # (vllm/config/vllm.py:662). + + # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) + # This gives extra cache hit than disabling hybrid kv cache manager + OFFLOAD_ARGS=( + --kv_offloading_backend native + --kv_offloading_size "$TOTAL_CPU_DRAM_GB" + ) + ;; +esac + +# ---- LLM server config ---------------------------------------------------------- +PARALLEL_ARGS=(--tensor-parallel-size "$TP") +if [ "${DP_ATTENTION}" = "true" ]; then + PARALLEL_ARGS=( + --tensor-parallel-size 1 + --data-parallel-size "$TP" + --enable-expert-parallel + ) +elif [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS+=(--enable-expert-parallel) +fi + +echo "Starting vllm server..." +export PYTHONNOUSERSITE=1 + +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_USE_BREAKABLE_CUDAGRAPH=0 +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_USE_AITER_MOE=1 +export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 +# INT4 quantized all-reduce for the (~1.5 MB) decode all-reduces, which are the +# single biggest decode kernel at high concurrency. The MIN_SIZE_KB override is +# required: vLLM's default INT4 quick-reduce size gate for (bf16, TP4) is 16 MB, +# so it never fires for decode-sized tensors without it. +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 +export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 + +VLLM_CMD=( + vllm serve "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + "${PARALLEL_ARGS[@]}" + --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" + --trust-remote-code + --block-size 128 + --gpu-memory-utilization 0.80 + --language-model-only + --attention-backend TRITON_ATTN + --moe-backend aiter + --kv-cache-dtype fp8 + --tool-call-parser minimax_m3 + --enable-auto-tool-choice + --max-num-seqs "$CONC" + "${OFFLOAD_ARGS[@]}" +) +printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" +printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! +echo "Server PID: $SERVER_PID" + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +# ---- Run benchmark ---------------------------------------------------------- +if [ "${EVAL_ONLY}" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0d46afe2e..1607550e0 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2361,3 +2361,17 @@ dsv4-fp8-mi325x-vllm-mtp: # is 55.8%. conc128 passed cleanly on the memory-tightest SKU (MI300X); # cap 8k1k MTP at 128 (normal holds 256, 1k1k holds 512). - { tp: 8, conc-start: 4, conc-end: 128, spec-decoding: mtp } + +minimaxm3-fp4-mi355x-vllm-agentic-mtp: + image: vllm/vllm-openai-rocm:nightly-cb8104839c141609d99f1254459ef3a4f1bd4263 + model: amd/MiniMax-M3-MXFP4 + model-prefix: minimaxm3 + runner: cluster:mi355x-amds + precision: fp4 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [16], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 41bd57494..7f69dc3d7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5440,3 +5440,10 @@ - "Add MiniMax-M3-MXFP8 GB300 8k1k disaggregated Dynamo-vLLM EAGLE3 MTP Pareto on vLLM nightly 5e35a6f4f" - "The six dep8 decode configurations enable the CUTLASS MSA sparse decode backend and FULL_DECODE_ONLY cudagraph mode; the five low-concurrency c1/c4/c8 configurations use neither" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2486 + +- config-keys: + - minimaxm3-fp4-mi355x-vllm-agentic-mtp + description: + - "Add Minimax-M3 FP4 vLLM Single Node Agentic Support" + - "Image: vllm/vllm-openai-rocm:nightly-cb8104839c141609d99f1254459ef3a4f1bd4263" + pr-link: TO-BE ADDED - https://github.com/SemiAnalysisAI/InferenceX/pull/ From 4edbe0ee0b280aede7150e40e055391d0df0f894 Mon Sep 17 00:00:00 2001 From: Sirra Date: Tue, 4 Aug 2026 16:31:22 +0530 Subject: [PATCH 2/5] [AMD] [WIP] [AGENTX] MiniMax-M3 Support on MI355X with MTP - Modifying Search Space & KV Backend Signed-off-by: Sirra --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 1607550e0..ab1fcf64a 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -2374,4 +2374,4 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [16], spec-decoding: mtp } + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 8, 12, 16], spec-decoding: mtp } From 635a5d69788246ddee31281d8dbf1164ad79eb10 Mon Sep 17 00:00:00 2001 From: Sirra Date: Tue, 4 Aug 2026 16:32:28 +0530 Subject: [PATCH 3/5] [AMD] [WIP] [AGENTX] MiniMax-M3 Support on MI355X with MTP - Adding PR Id to Perf Change Log. Signed-off-by: Sirra --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7f69dc3d7..d4ae24e54 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5446,4 +5446,4 @@ description: - "Add Minimax-M3 FP4 vLLM Single Node Agentic Support" - "Image: vllm/vllm-openai-rocm:nightly-cb8104839c141609d99f1254459ef3a4f1bd4263" - pr-link: TO-BE ADDED - https://github.com/SemiAnalysisAI/InferenceX/pull/ + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2487 From f2404811b84a97df61f4d8b3b0c32b65e51cc6fc Mon Sep 17 00:00:00 2001 From: Sirra Date: Tue, 4 Aug 2026 16:51:19 +0530 Subject: [PATCH 4/5] [AMD] [WIP] [AGENTX] MiniMax-M3 Support on MI355X with MTP - Adding DRAFT Model & Details. Signed-off-by: Sirra --- benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh index a00947844..9309dfccf 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh @@ -47,6 +47,9 @@ amd-smi || true resolve_trace_source install_agentic_deps +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" +NUM_SPEC_TOKENS=5 + # ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" From cebeb274ea1e4d5e8ff96ca74b5b1f738e1dfab2 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:52:15 -0400 Subject: [PATCH 5/5] feat(minimaxm3-fp4-mi355x-vllm-agentic-mtp): pin synthetic EAGLE3 acceptance to the golden AL Throughput replay now runs with rejection_sample_method=synthetic and synthetic_acceptance_length=3.35, the committed minimax-m3 EAGLE3 golden AL for thinking_on at num_speculative_tokens=5 (golden_al_distribution/minimaxm3_eagle3.yaml), as required by the AgentX fairness guidelines for agentic speculative-decoding submissions. EVAL_ONLY accuracy runs keep real target verification -- synthetic acceptance bypasses verification and corrupts the eval score. Co-Authored-By: Claude Opus 5 (1M context) --- .../agentic/minimaxm3_fp4_mi355x_mtp.sh | 14 +++++++++++++- perf-changelog.yaml | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh index 9309dfccf..3239327e5 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x_mtp.sh @@ -49,6 +49,18 @@ install_agentic_deps DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3" NUM_SPEC_TOKENS=5 +# Throughput pins synthetic EAGLE3 acceptance to the minimax-m3 golden AL +# (thinking_on, num_speculative_tokens=5, golden_al_distribution/minimaxm3_eagle3.yaml). +# MiniMax-M3 is an interleaved-thinking model and the replay runs it with +# thinking active, so thinking_on is the matching curve. The EVAL_ONLY accuracy +# run uses real target verification instead -- synthetic acceptance bypasses +# verification and corrupts the eval score. +SYNTHETIC_ACCEPT_LEN=3.35 +if [ "${EVAL_ONLY:-false}" = "true" ]; then + SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" +else + SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" +fi # ---- Server config ---------------------------------------------------------- SERVER_LOG="$RESULT_DIR/server.log" @@ -112,7 +124,7 @@ VLLM_CMD=( --host 0.0.0.0 --port "$PORT" "${PARALLEL_ARGS[@]}" - --speculative-config "{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" + --speculative-config "$SPEC_CONFIG" --trust-remote-code --block-size 128 --gpu-memory-utilization 0.80 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d4ae24e54..6b50f2460 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5446,4 +5446,5 @@ description: - "Add Minimax-M3 FP4 vLLM Single Node Agentic Support" - "Image: vllm/vllm-openai-rocm:nightly-cb8104839c141609d99f1254459ef3a4f1bd4263" + - "Throughput runs use synthetic EAGLE3 acceptance pinned to the minimax-m3 golden AL (thinking_on, num_speculative_tokens=5, AL=3.35); EVAL_ONLY accuracy runs keep real target verification" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2487