From 4af17643978d0b351a68c91c5e9cb600d53603f9 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 31 May 2026 14:33:25 +0900 Subject: [PATCH 1/3] feat(atom/dsr1-fp4-mi355x-mtp): update image + model, switch to TP=4 Co-Authored-By: Claude Opus 4.6 --- .github/configs/amd-master.yaml | 8 +++----- perf-changelog.yaml | 10 ++++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index c6d06c2e1..594ef29c3 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -67,8 +67,8 @@ dsr1-fp4-mi355x-atom: - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } dsr1-fp4-mi355x-atom-mtp: - image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 - model: amd/DeepSeek-R1-0528-MXFP4 + image: rocm/atom-dev:nightly_202605301523 + model: amd/DeepSeek-R1-0528-MXFP4-MTP-MoEFP4 model-prefix: dsr1 runner: mi355x precision: fp4 @@ -81,12 +81,10 @@ dsr1-fp4-mi355x-atom-mtp: osl: 1024 search-space: - { tp: 4, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } - isl: 8192 osl: 1024 search-space: - #- { tp: 4, conc-start: 32, conc-end: 256, spec-decoding: mtp } - - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 4, conc-start: 4, conc-end: 256, spec-decoding: mtp } dsr1-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.12-rocm700-mi30x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c6ddcbe0d..31a2c4ba6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3330,6 +3330,16 @@ - "Update vLLM ROCm image from nightly-4f940896a32c9e2a0eba7f50d521bf5f6b4de458 to v0.22.0" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1624 +- config-keys: + - dsr1-fp4-mi355x-atom-mtp + description: + - "Update ATOM image from rocm7.2.3_..._atom20260511 to nightly_202605301523" + - "Update model from amd/DeepSeek-R1-0528-MXFP4 to amd/DeepSeek-R1-0528-MXFP4-MTP-MoEFP4" + - "Switch to TP=4 only (drop TP=8 search-space); enable isl=8192 TP=4 sweep" + - "isl=1024/osl=1024 TP=4: +5.7% to +16.6% improvement at conc 4-64 vs prior InferenceX numbers" + - "isl=1024/osl=1024 TP=4: -0.6% to -1.9% at high concurrency (conc 128-256)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER + - config-keys: - kimik2.5-fp4-mi355x-vllm description: From e705ffcb26b5fff24d6a69956e505167181cc5d0 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 31 May 2026 14:34:03 +0900 Subject: [PATCH 2/3] chore: update perf-changelog pr-link to #1629 Co-Authored-By: Claude Opus 4.6 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 31a2c4ba6..6aa607343 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3338,7 +3338,7 @@ - "Switch to TP=4 only (drop TP=8 search-space); enable isl=8192 TP=4 sweep" - "isl=1024/osl=1024 TP=4: +5.7% to +16.6% improvement at conc 4-64 vs prior InferenceX numbers" - "isl=1024/osl=1024 TP=4: -0.6% to -1.9% at high concurrency (conc 128-256)" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1629 - config-keys: - kimik2.5-fp4-mi355x-vllm From 14da3ee6d85bcf81eb6d262b86a09b7e25eddbbb Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 31 May 2026 14:40:39 +0900 Subject: [PATCH 3/3] Update dsr1-fp4-mi355x-atom-mtp: stable image, wider concurrency, DP-attn support - Switch image from nightly_202605301523 to stable atom0.1.3 - Expand concurrency search space from 4-256 to 4-1024 - Refactor benchmark script to use PARALLEL_ARGS/SPEC_ARGS pattern - Remove ISL/OSL-based max-model-len calculation - Remove WIP comment from config - Update perf-changelog image reference Co-Authored-By: Claude Sonnet 4.6 --- .github/configs/amd-master.yaml | 7 ++--- .../single_node/dsr1_fp4_mi355x_atom_mtp.sh | 29 +++++++++---------- perf-changelog.yaml | 2 +- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 594ef29c3..098b41b45 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -67,12 +67,11 @@ dsr1-fp4-mi355x-atom: - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } dsr1-fp4-mi355x-atom-mtp: - image: rocm/atom-dev:nightly_202605301523 + image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 model: amd/DeepSeek-R1-0528-MXFP4-MTP-MoEFP4 model-prefix: dsr1 runner: mi355x precision: fp4 - # WIP framework (no customers yet) framework: atom multinode: false scenarios: @@ -80,11 +79,11 @@ dsr1-fp4-mi355x-atom-mtp: - isl: 1024 osl: 1024 search-space: - - { tp: 4, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 4, conc-start: 4, conc-end: 1024, spec-decoding: mtp } - isl: 8192 osl: 1024 search-space: - - { tp: 4, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 4, conc-start: 4, conc-end: 1024, spec-decoding: mtp } dsr1-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.12-rocm700-mi30x diff --git a/benchmarks/single_node/dsr1_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/dsr1_fp4_mi355x_atom_mtp.sh index 1d557684e..e5bd6579e 100644 --- a/benchmarks/single_node/dsr1_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/dsr1_fp4_mi355x_atom_mtp.sh @@ -24,23 +24,22 @@ PORT=${PORT:-8888} export OMP_NUM_THREADS=1 -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - +CALCULATED_MAX_MODEL_LEN="" if [ "${EVAL_ONLY}" = "true" ]; then setup_eval_context CALCULATED_MAX_MODEL_LEN=" --max-model-len $EVAL_MAX_MODEL_LEN " fi -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi +PARALLEL_ARGS=(-tp "$TP") #TP +if [ "$DP_ATTENTION" = "true" ]; then + if [ "$EP_SIZE" -gt 1 ]; then #DP+EP + PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) + else #DP+TP + PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) + fi +fi + +SPEC_ARGS=(--method mtp --num-speculative-tokens 3 ) # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -52,9 +51,9 @@ export AMDGCN_USE_BUFFER_OPS=1 python3 -m atom.entrypoints.openai_server \ --model $MODEL \ --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --method mtp \ + "${PARALLEL_ARGS[@]}" \ + "${SPEC_ARGS[@]}" \ + --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN \ > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6aa607343..de4e08a06 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3333,7 +3333,7 @@ - config-keys: - dsr1-fp4-mi355x-atom-mtp description: - - "Update ATOM image from rocm7.2.3_..._atom20260511 to nightly_202605301523" + - "Update ATOM image to rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3" - "Update model from amd/DeepSeek-R1-0528-MXFP4 to amd/DeepSeek-R1-0528-MXFP4-MTP-MoEFP4" - "Switch to TP=4 only (drop TP=8 search-space); enable isl=8192 TP=4 sweep" - "isl=1024/osl=1024 TP=4: +5.7% to +16.6% improvement at conc 4-64 vs prior InferenceX numbers"