diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml index 52a90cf2cd..b95cb73761 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml @@ -58,12 +58,18 @@ base: AGENTIC_WARMUP_GRACE_PERIOD: '3600' -zip_override_tp8: +zip_override_tp8_hicache: roles: agg: args: max-running-requests: [2, 4, 8, 16, 32, 48, 64, 96, 128] cuda-graph-max-bs-decode: [2, 4, 8, 16, 32, 48, 64, 96, 128] + # The host pool keeps evicted agentic prefixes for reuse across turns. + # 160 GB per rank plus the ~30 GB DFlash draft pool exceeded the + # ~1.39 TiB cgroup across 8 ranks; 120 GB keeps the total near 1.2 TB. + enable-hierarchical-cache: true + hicache-size: 120 benchmark: env: CONC: ['1', '2', '4', '8', '16', '24', '32', '48', '64'] + KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 610f44b40f..13acb63261 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8376,8 +8376,9 @@ musespark-fp4-h200-sglang-agentic-dflash: srt-recipe-dir: musespark/sglang/h200-fp4-mtp scenarios: agentic-coding: - - search-space: - - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 24, 32, 48, 64], srt-recipe: agentic.yaml } + - dram-utilization: 0.80 + search-space: + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 24, 32, 48, 64], srt-recipe: agentic.yaml } dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 85b1badba4..033a7a46cf 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10043,3 +10043,12 @@ - "Enable SGLang HiCache CPU DRAM KV offloading above conc 8: the TP4/EP4 and TP8/EP8 conc 12 and 16 points add enable-hierarchical-cache with hicache-size 180 GB per rank and write_through (720 GB at TP4, 1.44 TB at TP8; dram-utilization 0.85), matching the GLM-5.2 MXFP4 MI355X SGLang sibling." - "Conc 1, 4 and 8 stay GPU-resident with no KV offloading. Image, serve flags, DFlash2 draft and golden AL are unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4201 + +- config-keys: + - musespark-fp4-h200-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Offload every Muse Spark Open H200 point to host DRAM with SGLang HiCache (120 GB per rank so the target and DFlash draft host pools fit the node cgroup, dram-utilization 0.80), keeping the concurrency 1-64 sweep." + - "Keep the GPU-resident settings unchanged: mem-fraction-static 0.85, matching running-request and decode-graph caps." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4199