diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh index 87809f54b..f31cb41b1 100644 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh @@ -21,6 +21,7 @@ export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 export SGLANG_MAMBA_SSM_DTYPE=bfloat16 +export SGLANG_USE_AITER_FP8_PER_TOKEN=1 SERVER_LOG=/workspace/server.log MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b8f6f4fc3..1d04939da 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -367,7 +367,7 @@ qwen3.5-fp8-mi355x-sglang-disagg: qwen3.5-fp4-mi355x-sglang: image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 - model: amd/Qwen3.5-397B-A17B-MXFP4 + model: amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2 model-prefix: qwen3.5 runner: mi355x precision: fp4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6eebd9707..f5d454cdf 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5355,3 +5355,10 @@ - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang + description: + - "Update the Qwen3.5 FP4 MI355X SGLang checkpoint to amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2" + - "Quantize attention to PTPC FP8 and shared experts to MXFP4, and enable SGLANG_USE_AITER_FP8_PER_TOKEN=1" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2465