diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml index 3c55839fa3..0776eb58f5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml @@ -65,31 +65,24 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' AGENTIC_WARMUP_GRACE_PERIOD: '3600' -zip_override_tp8: - roles: - agg: - args: - # Eager batches above the decode graphs OOM in the MoE with ~1 GB free. - max-running-requests: [2, 4, 8, 8, 8] - # Each graph slot reserves seven full-context int32 page tables, and - # bs=16 graphs leave too little headroom for the vision encoder. - cuda-graph-max-bs-decode: [2, 4, 8, 8, 8] - benchmark: - env: - CONC: ['1', '2', '4', '8', '16'] - KV_OFFLOADING: none - # The GPU KV pool is a few GB per rank, so agentic prefixes evict between turns; # HiCache keeps them in host DRAM instead of recomputing them. zip_override_tp8_hicache: roles: agg: args: - max-running-requests: [4, 8, 8] - cuda-graph-max-bs-decode: [4, 8, 8] + # Eager batches above the decode graphs OOM in the MoE with ~1 GB free. + max-running-requests: [2, 4, 8, 8] + # Each graph slot reserves seven full-context int32 page tables, and + # bs=16 graphs leave too little headroom for the vision encoder. + cuda-graph-max-bs-decode: [2, 4, 8, 8] enable-hierarchical-cache: true hicache-size: 160 + # 0.97 left ~1.3 GB free and HiCache's serving buffers OOMed it at c4; 0.95 + # shrank the GPU KV pool to ~98k tokens, below the trace's ~196k-token + # inputs. 0.965 keeps ~1.8 GB free and a ~250k-token pool. + mem-fraction-static: 0.965 benchmark: env: - CONC: ['2', '4', '8'] + CONC: ['1', '2', '4', '8'] KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 69dc0904bd..67649757e4 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9740,5 +9740,4 @@ musespark-fp4-h100-sglang-agentic-dflash: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: agentic.yaml } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [2, 4, 8], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [1, 2, 4, 8], srt-recipe: agentic.yaml } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index fefd7900d2..f52e89de38 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10014,6 +10014,15 @@ - "Keep the batch-8 cap on running requests and decode graphs from the GPU-resident sweep." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4181 +- config-keys: + - musespark-fp4-h100-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Run every point with HiCache host-DRAM KV offload at 160 GB per rank, sweeping concurrency 1, 2, 4 and 8; drop the GPU-resident arm and the concurrency 16 queueing point." + - "Keep the batch-8 cap on running requests and decode graphs, and lower mem-fraction-static from 0.97 to 0.965: with HiCache attached, 0.97 ran out of GPU memory in the MoE at concurrency 4, while 0.95 shrank the GPU KV pool to 98k tokens and rejected the trace's ~196k-token inputs." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4182 + - config-keys: - dsv41flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg scenario-type: