From 76a60731c7ad0bdc695c46f69f4921dec7e2584f Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 31 May 2026 14:19:44 +0900 Subject: [PATCH 1/6] feat(atom/dsr1-fp8-mi355x-mtp): update ATOM image to nightly_202605301523 Co-Authored-By: Claude Opus 4.6 --- .github/configs/amd-master.yaml | 2 +- perf-changelog.yaml | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index c6d06c2e1..749b312e9 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -1197,7 +1197,7 @@ dsr1-fp8-mi355x-atom: - { tp: 8, conc-start: 4, conc-end: 128 } dsr1-fp8-mi355x-atom-mtp: - image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 + image: rocm/atom-dev:nightly_202605301523 model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 runner: mi355x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c6ddcbe0d..8bd2fca31 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3330,6 +3330,14 @@ - "Update vLLM ROCm image from nightly-4f940896a32c9e2a0eba7f50d521bf5f6b4de458 to v0.22.0" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1624 +- config-keys: + - dsr1-fp8-mi355x-atom-mtp + description: + - "Update ATOM image from rocm7.2.3_..._atom20260511 to nightly_202605301523" + - "isl=1024/osl=1024: +47% to +116% improvement across conc 4-256 vs prior InferenceX numbers" + - "isl=8192/osl=1024: +47% to +131% improvement across conc 4-256" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER + - config-keys: - kimik2.5-fp4-mi355x-vllm description: From 621f8970367f6664000eb70697566ac51df80352 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 31 May 2026 14:21:11 +0900 Subject: [PATCH 2/6] chore: update perf-changelog pr-link to #1628 Co-Authored-By: Claude Opus 4.6 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8bd2fca31..42f564ebe 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3336,7 +3336,7 @@ - "Update ATOM image from rocm7.2.3_..._atom20260511 to nightly_202605301523" - "isl=1024/osl=1024: +47% to +116% improvement across conc 4-256 vs prior InferenceX numbers" - "isl=8192/osl=1024: +47% to +131% improvement across conc 4-256" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/PLACEHOLDER + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1628 - config-keys: - kimik2.5-fp4-mi355x-vllm From a251a0dd4eb03f9ed34fef57f516104609926071 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 31 May 2026 14:30:35 +0900 Subject: [PATCH 3/6] Update dsr1-fp8-mi355x-atom-mtp: stable image, wider concurrency, DP-attn support - Switch image from nightly_202605301523 to stable atom0.1.3 - Expand concurrency search space from 4-256 to 4-1024 - Refactor benchmark script to use PARALLEL_ARGS/SPEC_ARGS pattern - Remove ISL/OSL-based max-model-len calculation - Update perf-changelog image reference Co-Authored-By: Claude Sonnet 4.6 --- .github/configs/amd-master.yaml | 6 ++-- .../single_node/dsr1_fp8_mi355x_atom_mtp.sh | 30 +++++++++---------- perf-changelog.yaml | 2 +- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 749b312e9..d8f3c5b65 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -1197,7 +1197,7 @@ dsr1-fp8-mi355x-atom: - { tp: 8, conc-start: 4, conc-end: 128 } dsr1-fp8-mi355x-atom-mtp: - image: rocm/atom-dev:nightly_202605301523 + image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 runner: mi355x @@ -1209,11 +1209,11 @@ dsr1-fp8-mi355x-atom-mtp: - isl: 1024 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 8, conc-start: 4, conc-end: 1024, spec-decoding: mtp } - isl: 8192 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 8, conc-start: 4, conc-end: 1024, spec-decoding: mtp } dsr1-fp8-mi355x-sglang-disagg: image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 diff --git a/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh b/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh index 69179cec0..fbdaf4758 100644 --- a/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh @@ -24,23 +24,22 @@ PORT=${PORT:-8888} export OMP_NUM_THREADS=1 -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - +CALCULATED_MAX_MODEL_LEN="" if [ "${EVAL_ONLY}" = "true" ]; then setup_eval_context CALCULATED_MAX_MODEL_LEN=" --max-model-len $EVAL_MAX_MODEL_LEN " fi -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi +PARALLEL_ARGS=(-tp "$TP") #TP +if [ "$DP_ATTENTION" = "true" ]; then + if [ "$EP_SIZE" -gt 1 ]; then #DP+EP + PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) + else #DP+TP + PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) + fi +fi + +SPEC_ARGS=(--method mtp --num-speculative-tokens 3 ) # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -50,10 +49,9 @@ set -x python3 -m atom.entrypoints.openai_server \ --model $MODEL \ --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --method mtp \ - --num-speculative-tokens 3 \ + "${PARALLEL_ARGS[@]}" \ + "${SPEC_ARGS[@]}" \ + --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN \ > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 42f564ebe..21f265088 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3333,7 +3333,7 @@ - config-keys: - dsr1-fp8-mi355x-atom-mtp description: - - "Update ATOM image from rocm7.2.3_..._atom20260511 to nightly_202605301523" + - "Update ATOM image to rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 - "isl=1024/osl=1024: +47% to +116% improvement across conc 4-256 vs prior InferenceX numbers" - "isl=8192/osl=1024: +47% to +131% improvement across conc 4-256" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1628 From 4a32700cc25890adc2f76720e53597a8ac71283f Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 31 May 2026 14:42:21 +0900 Subject: [PATCH 4/6] dsr1_fp8_mi355x_atom_mtp.sh: remove trailing whitespace Co-Authored-By: Claude Sonnet 4.6 --- benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh b/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh index fbdaf4758..3289a9912 100644 --- a/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh @@ -49,7 +49,7 @@ set -x python3 -m atom.entrypoints.openai_server \ --model $MODEL \ --server-port $PORT \ - "${PARALLEL_ARGS[@]}" \ + "${PARALLEL_ARGS[@]}" \ "${SPEC_ARGS[@]}" \ --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN \ > $SERVER_LOG 2>&1 & From c1e8e6fa7bd8b98d301bde7b6e70a06544cb6ba8 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Sun, 31 May 2026 14:45:28 +0900 Subject: [PATCH 5/6] perf-changelog: fix unclosed quote in dsr1-fp8-mi355x-atom-mtp entry Co-Authored-By: Claude Sonnet 4.6 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 21f265088..1bab4b605 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3333,7 +3333,7 @@ - config-keys: - dsr1-fp8-mi355x-atom-mtp description: - - "Update ATOM image to rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 + - "Update ATOM image to rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3" - "isl=1024/osl=1024: +47% to +116% improvement across conc 4-256 vs prior InferenceX numbers" - "isl=8192/osl=1024: +47% to +131% improvement across conc 4-256" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1628 From f94aab9169feeb320054e0f5091e47c639ed2351 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Mon, 1 Jun 2026 17:32:24 +0900 Subject: [PATCH 6/6] [AMD] fix dsr1-fp8-mi355x-atom-mtp: reduce concurrency limits and disable prefix caching Co-Authored-By: Claude Sonnet 4.6 --- .github/configs/amd-master.yaml | 4 ++-- benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 454aefedf..156910da7 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -1209,11 +1209,11 @@ dsr1-fp8-mi355x-atom-mtp: - isl: 1024 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 1024, spec-decoding: mtp } + - { tp: 8, conc-start: 4, conc-end: 512, spec-decoding: mtp } - isl: 8192 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 1024, spec-decoding: mtp } + - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } dsr1-fp8-mi355x-sglang-disagg: image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 diff --git a/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh b/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh index 3289a9912..5cee63e78 100644 --- a/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh @@ -52,6 +52,7 @@ python3 -m atom.entrypoints.openai_server \ "${PARALLEL_ARGS[@]}" \ "${SPEC_ARGS[@]}" \ --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN \ + --no-enable_prefix_caching \ > $SERVER_LOG 2>&1 & SERVER_PID=$!