diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index c6d06c2e1..d8f3c5b65 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -1197,7 +1197,7 @@ dsr1-fp8-mi355x-atom: - { tp: 8, conc-start: 4, conc-end: 128 } dsr1-fp8-mi355x-atom-mtp: - image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 + image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 model: deepseek-ai/DeepSeek-R1-0528 model-prefix: dsr1 runner: mi355x @@ -1209,11 +1209,11 @@ dsr1-fp8-mi355x-atom-mtp: - isl: 1024 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 8, conc-start: 4, conc-end: 1024, spec-decoding: mtp } - isl: 8192 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 8, conc-start: 4, conc-end: 1024, spec-decoding: mtp } dsr1-fp8-mi355x-sglang-disagg: image: rocm/sgl-dev:sglang-0.5.9-rocm720-mi35x-mori-0227-2 diff --git a/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh b/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh index 69179cec0..3289a9912 100644 --- a/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/dsr1_fp8_mi355x_atom_mtp.sh @@ -24,23 +24,22 @@ PORT=${PORT:-8888} export OMP_NUM_THREADS=1 -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - +CALCULATED_MAX_MODEL_LEN="" if [ "${EVAL_ONLY}" = "true" ]; then setup_eval_context CALCULATED_MAX_MODEL_LEN=" --max-model-len $EVAL_MAX_MODEL_LEN " fi -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi +PARALLEL_ARGS=(-tp "$TP") #TP +if [ "$DP_ATTENTION" = "true" ]; then + if [ "$EP_SIZE" -gt 1 ]; then #DP+EP + PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) + else #DP+TP + PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) + fi +fi + +SPEC_ARGS=(--method mtp --num-speculative-tokens 3 ) # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -50,10 +49,9 @@ set -x python3 -m atom.entrypoints.openai_server \ --model $MODEL \ --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --method mtp \ - --num-speculative-tokens 3 \ + "${PARALLEL_ARGS[@]}" \ + "${SPEC_ARGS[@]}" \ + --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN \ > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c6ddcbe0d..1bab4b605 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3330,6 +3330,14 @@ - "Update vLLM ROCm image from nightly-4f940896a32c9e2a0eba7f50d521bf5f6b4de458 to v0.22.0" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1624 +- config-keys: + - dsr1-fp8-mi355x-atom-mtp + description: + - "Update ATOM image to rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3" + - "isl=1024/osl=1024: +47% to +116% improvement across conc 4-256 vs prior InferenceX numbers" + - "isl=8192/osl=1024: +47% to +131% improvement across conc 4-256" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1628 + - config-keys: - kimik2.5-fp4-mi355x-vllm description: