From f43644be0ea2764b8c6c29a3beb8b2168c15fb62 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Fri, 16 Oct 2026 21:36:23 +0800 Subject: [PATCH 1/4] feat: offload every H100 Muse Spark point to HiCache, sweep c1-c8 --- .../sglang/h100-fp4-mtp/agentic.yaml | 23 +++++-------------- inferencex-e2e/configs/nvidia-master.yaml | 3 +-- inferencex-e2e/perf-changelog.yaml | 9 ++++++++ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml index 3c55839fa3..9c13bdc44f 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml @@ -65,31 +65,20 @@ base: AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' AGENTIC_WARMUP_GRACE_PERIOD: '3600' -zip_override_tp8: - roles: - agg: - args: - # Eager batches above the decode graphs OOM in the MoE with ~1 GB free. - max-running-requests: [2, 4, 8, 8, 8] - # Each graph slot reserves seven full-context int32 page tables, and - # bs=16 graphs leave too little headroom for the vision encoder. - cuda-graph-max-bs-decode: [2, 4, 8, 8, 8] - benchmark: - env: - CONC: ['1', '2', '4', '8', '16'] - KV_OFFLOADING: none - # The GPU KV pool is a few GB per rank, so agentic prefixes evict between turns; # HiCache keeps them in host DRAM instead of recomputing them. zip_override_tp8_hicache: roles: agg: args: - max-running-requests: [4, 8, 8] - cuda-graph-max-bs-decode: [4, 8, 8] + # Eager batches above the decode graphs OOM in the MoE with ~1 GB free. + max-running-requests: [2, 4, 8, 8] + # Each graph slot reserves seven full-context int32 page tables, and + # bs=16 graphs leave too little headroom for the vision encoder. + cuda-graph-max-bs-decode: [2, 4, 8, 8] enable-hierarchical-cache: true hicache-size: 160 benchmark: env: - CONC: ['2', '4', '8'] + CONC: ['1', '2', '4', '8'] KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 1fc173c6a4..4f0976cd96 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9710,5 +9710,4 @@ musespark-fp4-h100-sglang-agentic-dflash: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: agentic.yaml } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [2, 4, 8], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [1, 2, 4, 8], srt-recipe: agentic.yaml } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 8ec368892e..fc469237e2 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10013,3 +10013,12 @@ - "Add a HiCache host-DRAM KV offload arm at concurrency 2, 4 and 8 with 160 GB per rank, alongside the GPU-resident sweep; the few-GB GPU KV pool left by 67 GB of weights per H100 evicts agentic prefixes between turns." - "Keep the batch-8 cap on running requests and decode graphs from the GPU-resident sweep." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4181 + +- config-keys: + - musespark-fp4-h100-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Run every point with HiCache host-DRAM KV offload at 160 GB per rank, sweeping concurrency 1, 2, 4 and 8; drop the GPU-resident arm and the concurrency 16 queueing point." + - "Keep the batch-8 cap on running requests and decode graphs." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX From 5757b8fd607c8adfd6924d331af26002bbd61126 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Fri, 16 Oct 2026 21:36:30 +0800 Subject: [PATCH 2/4] chore: link H100 HiCache entry to PR 4182 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index fc469237e2..3213ac46ba 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10021,4 +10021,4 @@ description: - "Run every point with HiCache host-DRAM KV offload at 160 GB per rank, sweeping concurrency 1, 2, 4 and 8; drop the GPU-resident arm and the concurrency 16 queueing point." - "Keep the batch-8 cap on running requests and decode graphs." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4182 From c2e49c51e4f20faf4e212cb8a50063a8726394eb Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Fri, 16 Oct 2026 22:19:48 +0800 Subject: [PATCH 3/4] fix: lower H100 Muse Spark HiCache mem fraction to 0.95 after c4 OOM --- .../musespark/sglang/h100-fp4-mtp/agentic.yaml | 3 +++ inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml index 9c13bdc44f..1f69d6aff5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml @@ -78,6 +78,9 @@ zip_override_tp8_hicache: cuda-graph-max-bs-decode: [2, 4, 8, 8] enable-hierarchical-cache: true hicache-size: 160 + # HiCache consumed the ~1 GB of free memory left at 0.97, which OOMed at c4; + # the host pool backs the smaller GPU KV pool. + mem-fraction-static: 0.95 benchmark: env: CONC: ['1', '2', '4', '8'] diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3213ac46ba..fe2769e9a6 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10020,5 +10020,5 @@ - agentic-coding description: - "Run every point with HiCache host-DRAM KV offload at 160 GB per rank, sweeping concurrency 1, 2, 4 and 8; drop the GPU-resident arm and the concurrency 16 queueing point." - - "Keep the batch-8 cap on running requests and decode graphs." + - "Keep the batch-8 cap on running requests and decode graphs, and lower mem-fraction-static from 0.97 to 0.95: with HiCache attached, concurrency 4 ran out of GPU memory in the MoE during warmup once free memory fell from 0.88 GiB to 0.02 GiB." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4182 From 10784c73b780a7ae4f81f249290bad15286e9851 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:15:21 +0800 Subject: [PATCH 4/4] fix: set H100 Muse Spark HiCache mem fraction to 0.965 0.95 shrank the GPU KV pool to 98k tokens and rejected ~196k-token trace inputs. --- .../musespark/sglang/h100-fp4-mtp/agentic.yaml | 7 ++++--- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml index 1f69d6aff5..0776eb58f5 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml @@ -78,9 +78,10 @@ zip_override_tp8_hicache: cuda-graph-max-bs-decode: [2, 4, 8, 8] enable-hierarchical-cache: true hicache-size: 160 - # HiCache consumed the ~1 GB of free memory left at 0.97, which OOMed at c4; - # the host pool backs the smaller GPU KV pool. - mem-fraction-static: 0.95 + # 0.97 left ~1.3 GB free and HiCache's serving buffers OOMed it at c4; 0.95 + # shrank the GPU KV pool to ~98k tokens, below the trace's ~196k-token + # inputs. 0.965 keeps ~1.8 GB free and a ~250k-token pool. + mem-fraction-static: 0.965 benchmark: env: CONC: ['1', '2', '4', '8'] diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index fe2769e9a6..688a3ac4ab 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10020,5 +10020,5 @@ - agentic-coding description: - "Run every point with HiCache host-DRAM KV offload at 160 GB per rank, sweeping concurrency 1, 2, 4 and 8; drop the GPU-resident arm and the concurrency 16 queueing point." - - "Keep the batch-8 cap on running requests and decode graphs, and lower mem-fraction-static from 0.97 to 0.95: with HiCache attached, concurrency 4 ran out of GPU memory in the MoE during warmup once free memory fell from 0.88 GiB to 0.02 GiB." + - "Keep the batch-8 cap on running requests and decode graphs, and lower mem-fraction-static from 0.97 to 0.965: with HiCache attached, 0.97 ran out of GPU memory in the MoE at concurrency 4, while 0.95 shrank the GPU KV pool to 98k tokens and rejected the trace's ~196k-token inputs." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4182