diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index c6d06c2e1..098b41b45 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -67,12 +67,11 @@ dsr1-fp4-mi355x-atom: - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } dsr1-fp4-mi355x-atom-mtp: - image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 - model: amd/DeepSeek-R1-0528-MXFP4 + image: rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3 + model: amd/DeepSeek-R1-0528-MXFP4-MTP-MoEFP4 model-prefix: dsr1 runner: mi355x precision: fp4 - # WIP framework (no customers yet) framework: atom multinode: false scenarios: @@ -80,13 +79,11 @@ dsr1-fp4-mi355x-atom-mtp: - isl: 1024 osl: 1024 search-space: - - { tp: 4, conc-start: 4, conc-end: 256, spec-decoding: mtp } - - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 4, conc-start: 4, conc-end: 1024, spec-decoding: mtp } - isl: 8192 osl: 1024 search-space: - #- { tp: 4, conc-start: 32, conc-end: 256, spec-decoding: mtp } - - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 4, conc-start: 4, conc-end: 1024, spec-decoding: mtp } dsr1-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.12-rocm700-mi30x diff --git a/benchmarks/single_node/dsr1_fp4_mi355x_atom_mtp.sh b/benchmarks/single_node/dsr1_fp4_mi355x_atom_mtp.sh index 1d557684e..e5bd6579e 100644 --- a/benchmarks/single_node/dsr1_fp4_mi355x_atom_mtp.sh +++ b/benchmarks/single_node/dsr1_fp4_mi355x_atom_mtp.sh @@ -24,23 +24,22 @@ PORT=${PORT:-8888} export OMP_NUM_THREADS=1 -# Calculate max-model-len based on ISL and OSL -if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then - CALCULATED_MAX_MODEL_LEN="" -else - CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " -fi - +CALCULATED_MAX_MODEL_LEN="" if [ "${EVAL_ONLY}" = "true" ]; then setup_eval_context CALCULATED_MAX_MODEL_LEN=" --max-model-len $EVAL_MAX_MODEL_LEN " fi -if [ "$EP_SIZE" -gt 1 ]; then - EP=" --enable-expert-parallel" -else - EP=" " -fi +PARALLEL_ARGS=(-tp "$TP") #TP +if [ "$DP_ATTENTION" = "true" ]; then + if [ "$EP_SIZE" -gt 1 ]; then #DP+EP + PARALLEL_ARGS=(-tp "$TP" --enable-expert-parallel --enable-dp-attention ) + else #DP+TP + PARALLEL_ARGS=(-tp "$TP" --enable-dp-attention ) + fi +fi + +SPEC_ARGS=(--method mtp --num-speculative-tokens 3 ) # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -52,9 +51,9 @@ export AMDGCN_USE_BUFFER_OPS=1 python3 -m atom.entrypoints.openai_server \ --model $MODEL \ --server-port $PORT \ - -tp $TP \ - --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ - --method mtp \ + "${PARALLEL_ARGS[@]}" \ + "${SPEC_ARGS[@]}" \ + --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN \ > $SERVER_LOG 2>&1 & SERVER_PID=$! diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c6ddcbe0d..de4e08a06 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3330,6 +3330,16 @@ - "Update vLLM ROCm image from nightly-4f940896a32c9e2a0eba7f50d521bf5f6b4de458 to v0.22.0" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1624 +- config-keys: + - dsr1-fp4-mi355x-atom-mtp + description: + - "Update ATOM image to rocm/atom:rocm7.2.4_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.3" + - "Update model from amd/DeepSeek-R1-0528-MXFP4 to amd/DeepSeek-R1-0528-MXFP4-MTP-MoEFP4" + - "Switch to TP=4 only (drop TP=8 search-space); enable isl=8192 TP=4 sweep" + - "isl=1024/osl=1024 TP=4: +5.7% to +16.6% improvement at conc 4-64 vs prior InferenceX numbers" + - "isl=1024/osl=1024 TP=4: -0.6% to -1.9% at high concurrency (conc 128-256)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1629 + - config-keys: - kimik2.5-fp4-mi355x-vllm description: