From f316152ae75b00bc99bf9b5950248740e3035904 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:59:07 +0800 Subject: [PATCH 1/3] feat: offload every H200 Muse Spark point to HiCache --- .../musespark/sglang/h200-fp4-mtp/agentic.yaml | 6 +++++- inferencex-e2e/configs/nvidia-master.yaml | 5 +++-- inferencex-e2e/perf-changelog.yaml | 9 +++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml index 52a90cf2cd..68dd05af42 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml @@ -58,12 +58,16 @@ base: AGENTIC_WARMUP_GRACE_PERIOD: '3600' -zip_override_tp8: +zip_override_tp8_hicache: roles: agg: args: max-running-requests: [2, 4, 8, 16, 32, 48, 64, 96, 128] cuda-graph-max-bs-decode: [2, 4, 8, 16, 32, 48, 64, 96, 128] + # The host pool keeps evicted agentic prefixes for reuse across turns. + enable-hierarchical-cache: true + hicache-size: 160 benchmark: env: CONC: ['1', '2', '4', '8', '16', '24', '32', '48', '64'] + KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 610f44b40f..13acb63261 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8376,8 +8376,9 @@ musespark-fp4-h200-sglang-agentic-dflash: srt-recipe-dir: musespark/sglang/h200-fp4-mtp scenarios: agentic-coding: - - search-space: - - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 24, 32, 48, 64], srt-recipe: agentic.yaml } + - dram-utilization: 0.80 + search-space: + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 24, 32, 48, 64], srt-recipe: agentic.yaml } dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 32157d3f58..1da2eea2aa 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10034,3 +10034,12 @@ - "TP4 c1 admits one request at a time, enables decoder SWA bounded replay and disables the prefill CUDA graph." - "Add DEP2 c160 with the c128 settings, a prefill-decode interval of 8, mem-fraction-static 0.85, 256 max running requests and a 128 decode graph batch." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4033 + +- config-keys: + - musespark-fp4-h200-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Offload every Muse Spark Open H200 point to host DRAM with SGLang HiCache (160 GB per rank, dram-utilization 0.80), keeping the concurrency 1-64 sweep." + - "Keep the GPU-resident settings unchanged: mem-fraction-static 0.85, matching running-request and decode-graph caps." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX From 0dd41051537dc374a1bc5da581929e3f6405d10a Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:59:13 +0800 Subject: [PATCH 2/3] chore: link H200 HiCache entry to PR 4199 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 1da2eea2aa..c0e1d066fc 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10042,4 +10042,4 @@ description: - "Offload every Muse Spark Open H200 point to host DRAM with SGLang HiCache (160 GB per rank, dram-utilization 0.80), keeping the concurrency 1-64 sweep." - "Keep the GPU-resident settings unchanged: mem-fraction-static 0.85, matching running-request and decode-graph caps." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4199 From b5b61acbd577f2524f9a23fca05063e6812144ba Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Sat, 17 Oct 2026 01:39:56 +0800 Subject: [PATCH 3/3] fix: shrink H200 HiCache to 120 GB per rank to fit host RAM --- .../musespark/sglang/h200-fp4-mtp/agentic.yaml | 4 +++- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml index 68dd05af42..b95cb73761 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml @@ -65,8 +65,10 @@ zip_override_tp8_hicache: max-running-requests: [2, 4, 8, 16, 32, 48, 64, 96, 128] cuda-graph-max-bs-decode: [2, 4, 8, 16, 32, 48, 64, 96, 128] # The host pool keeps evicted agentic prefixes for reuse across turns. + # 160 GB per rank plus the ~30 GB DFlash draft pool exceeded the + # ~1.39 TiB cgroup across 8 ranks; 120 GB keeps the total near 1.2 TB. enable-hierarchical-cache: true - hicache-size: 160 + hicache-size: 120 benchmark: env: CONC: ['1', '2', '4', '8', '16', '24', '32', '48', '64'] diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index c0e1d066fc..87ccff5b0f 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10040,6 +10040,6 @@ scenario-type: - agentic-coding description: - - "Offload every Muse Spark Open H200 point to host DRAM with SGLang HiCache (160 GB per rank, dram-utilization 0.80), keeping the concurrency 1-64 sweep." + - "Offload every Muse Spark Open H200 point to host DRAM with SGLang HiCache (120 GB per rank so the target and DFlash draft host pools fit the node cgroup, dram-utilization 0.80), keeping the concurrency 1-64 sweep." - "Keep the GPU-resident settings unchanged: mem-fraction-static 0.85, matching running-request and decode-graph caps." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4199