From c26b73154b6872ae210f43ffed95302fd244f695 Mon Sep 17 00:00:00 2001 From: yuzho Date: Mon, 3 Aug 2026 05:42:26 -0500 Subject: [PATCH 1/2] update qwen3.5-397b-mxfp4 model --- benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh | 1 + configs/amd-master.yaml | 2 +- perf-changelog.yaml | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh index 87809f54b6..f31cb41b14 100644 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh @@ -21,6 +21,7 @@ export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 export SGLANG_MAMBA_SSM_DTYPE=bfloat16 +export SGLANG_USE_AITER_FP8_PER_TOKEN=1 SERVER_LOG=/workspace/server.log MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b8f6f4fc31..1d04939da8 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -367,7 +367,7 @@ qwen3.5-fp8-mi355x-sglang-disagg: qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 - model: amd/Qwen3.5-397B-A17B-MXFP4 + model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 model-prefix: qwen3.5 runner: mi355x precision: fp4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd97070..5f0ca8b8c9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,10 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + description: + - "Update the Qwen3.5 FP4 MI355X SGLang checkpoint to amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2" + - "Quantize attention to PTPC FP8 and shared experts to MXFP4, and enable SGLANG_USE_AITER_FP8_PER_TOKEN=1" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From b3fb4e3b11dda1850bb9391538a26748c8d333a0 Mon Sep 17 00:00:00 2001 From: yuzho Date: Mon, 3 Aug 2026 05:50:57 -0500 Subject: [PATCH 2/2] update PR --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5f0ca8b8c9..f5d454cdff 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5361,4 +5361,4 @@ description: - "Update the Qwen3.5 FP4 MI355X SGLang checkpoint to amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2" - "Quantize attention to PTPC FP8 and shared experts to MXFP4, and enable SGLANG_USE_AITER_FP8_PER_TOKEN=1" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2465