From a37e51d776679c7e251d1adcb2a27a403ef7c22b Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 15 Oct 2026 17:01:43 -0700 Subject: [PATCH 1/8] feat: add Muse Spark Open H100 capacity check --- .../sglang/h100-fp4-mtp/agentic.yaml | 67 +++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 14 ++++ inferencex-e2e/configs/runners.yaml | 1 + inferencex-e2e/docs/MODELS.md | 1 + inferencex-e2e/infx/srt_slurm/single_node.py | 4 +- .../infx/srt_slurm/synthetic_acceptance.py | 10 +++ .../tests/srt_slurm/test_srt_single_node.py | 9 ++- .../srt_slurm/test_synthetic_acceptance.py | 33 +++++++++ inferencex-e2e/perf-changelog.yaml | 9 +++ 9 files changed, 143 insertions(+), 5 deletions(-) create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..8c95728b40 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml @@ -0,0 +1,67 @@ +base: + schema: 2 + name: musespark-fp4-h100-sglang-agentic + resources: + gpu_type: h100 + gpus_per_node: 8 + slurm: + time_limit: '4:00:00' + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 8 + quantization: modelopt_fp4 + moe-runner-backend: marlin + fp4-gemm-backend: marlin + attention-backend: fa3 + kv-cache-dtype: fp8_e4m3 + mem-fraction-static: 0.85 + # Reserve H100 memory for the target, draft, and required AgentX context. + disable-prefill-cuda-graph: true + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + # DFlash2 verifies 8 tokens per step: the current token plus 7 drafts. + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Muse-Spark-Open-DFlash2 + speculative-draft-attention-backend: fa4 + speculative-num-draft-tokens: 8 + mm-feature-transport: cpu + reasoning-parser: muse + tool-call-parser: muse + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + health_check: + max_attempts: 1440 + interval_seconds: 5 + benchmark: + env: + # The gated Hub repo is unreadable to the client; use the staged checkpoint. + TOKENIZER: /model + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +override_tp8_c1: + benchmark: + env: + CONC: '1' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 2ff820f5b9..1164470772 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9661,3 +9661,17 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-dspark-disagg: tp: 16 ep: 16 dp-attn: true + +musespark-fp4-h100-sglang-agentic-dflash: + image: lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a + model: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 + model-prefix: musespark + runner: cluster:h100-dsxe + precision: fp4 + framework: sglang + multinode: false + srt-recipe-dir: musespark/sglang/h100-fp4-mtp + scenarios: + agentic-coding: + - search-space: + - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1], srt-recipe: agentic.yaml } diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index 4d25914d7e..f99346db06 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -362,6 +362,7 @@ clusters: entries: DeepSeek-V4.1-Flash: {root: data-models, dir: DeepSeek-V4.1-Flash} MiniMax-M3-MXFP8: {root: data-models, dir: MiniMax-M3-MXFP8} + Muse-Spark-Open-816B-A42B-NVFP4: {root: data-models, dir: Muse-Spark-Open-816B-A42B-NVFP4} Qwen3.5-397B-A17B-FP8: {root: data-models, dir: Qwen3.5-397B-A17B-FP8} scheduler: slurm slurm: diff --git a/inferencex-e2e/docs/MODELS.md b/inferencex-e2e/docs/MODELS.md index 0733de44f4..bb04039310 100644 --- a/inferencex-e2e/docs/MODELS.md +++ b/inferencex-e2e/docs/MODELS.md @@ -110,6 +110,7 @@ Other offloading tiers, including NVMe KV cache offloading, are outside the init | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| +| Muse Spark Open 816B-A42B | `musespark` | 2026-10-15 | Agentic coding (SGLang with DFlash2 draft `RadixArk/Muse-Spark-Open-DFlash2`; golden AL curve `musespark_dflash.yaml`) | | | DeepSeek-V4.1-Flash | `dsv41flash` | 2026-09-10 | Agentic coding (vLLM: DSpark, Engram UVA offload; SGLang: DSpark arms added per SKU from 2026-09-17; ATOM: MI355X TP2/TP4 DSpark added 2026-09-23, removed in #3463, restored on srt-slurm 2026-09-30; GPU validation pending) | | | GLM-5.3 | `glm5.3` | 2026-09-22 ([#3366](https://github.com/SemiAnalysisAI/InferenceX/pull/3366)) | Agentic coding (MTP only, per the deprecation policy) | Single-turn 1k1k (deprecated for all models before this model was added; never run) | | Qwen3.8-Flash-Next | `qwen3.8next` | 2026-08-26 ([#2742](https://github.com/SemiAnalysisAI/InferenceX/pull/2742)) | Agentic coding | | diff --git a/inferencex-e2e/infx/srt_slurm/single_node.py b/inferencex-e2e/infx/srt_slurm/single_node.py index c9f8b6c8ac..3e7fc8c6c1 100644 --- a/inferencex-e2e/infx/srt_slurm/single_node.py +++ b/inferencex-e2e/infx/srt_slurm/single_node.py @@ -113,9 +113,9 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N if environment["FRAMEWORK"] not in {"sglang", "trt", "atom", "vllm"}: raise ValueError(f"Unsupported single-node framework: {environment['FRAMEWORK']!r}") spec = spec_parameters(role, engine) - if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark"}: + if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark", "dflash"}: raise ValueError( - "Single-node SRT supports only native MTP, EAGLE3, DSpark or no speculation" + "Single-node SRT supports only native MTP, EAGLE3, DSpark, DFlash or no speculation" ) speculation = "mtp" if spec else workload.get("SPEC_DECODING", "none") agentic = environment["IS_AGENTIC"] == "1" diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 2de8687c22..9884fe1a07 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -77,6 +77,16 @@ def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]: algorithm = next(iter(algorithms), "") if not algorithm: return {} + if algorithm == "dflash": + # DFlash verifies a block of the current token plus its drafts. + width = args.get("speculative-num-draft-tokens") + return { + "method": "dflash", + "num_speculative_tokens": width - 1 + if isinstance(width, int) and not isinstance(width, bool) + else None, + "model": args.get("speculative-draft-model-path", ""), + } return { "method": "dspark" if algorithm == "dspark" else algorithm, "num_speculative_tokens": args.get( diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py index 4d88d9c86f..0e1b6bbdad 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py @@ -114,15 +114,18 @@ def test_ambiguous_native_variants_are_rejected(point): runtime_arguments(str(path), env) -def test_mtp_binding_uses_real_verification_and_the_chat_template(point, tmp_path): +@pytest.mark.parametrize("algorithm,spec_decoding", [("EAGLE", "mtp"), ("DFLASH", "draft_model")]) +def test_speculative_binding_uses_real_verification_and_the_chat_template( + point, tmp_path, algorithm, spec_decoding +): path, recipe, env = point recipe["roles"]["agg"]["args"].update({ - "expert-parallel-size": 4, "speculative-algorithm": "EAGLE", + "expert-parallel-size": 4, "speculative-algorithm": algorithm, "speculative-num-steps": 2, "speculative-num-draft-tokens": 3, }) recipe["roles"]["agg"]["env"] = {"SGLANG_SIMULATE_ACC_LEN": "2.5"} path.write_text(yaml.safe_dump({"base": recipe})) - env = {**env, "EP_SIZE": "4", "SPEC_DECODING": "mtp"} + env = {**env, "EP_SIZE": "4", "SPEC_DECODING": spec_decoding} _, bound = select_recipe(f"{path}:base", env) assert bound["benchmark"]["env"]["USE_CHAT_TEMPLATE"] == "true" concrete = tmp_path / "bound.yaml" diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index b51e80bfd1..be7be00ed8 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -51,6 +51,7 @@ def golden_dir(tmp_path: Path) -> Path: ("minimaxm3_eagle3.yaml", "minimax-m3", 2.5), ("minimaxm3_eagle3_gqa.yaml", "minimax-m3", 2.6), ("glm5.3_mtp.yaml", "glm-5.3", 3.2), + ("musespark_dflash.yaml", "muse-spark-open", 3.4), ]: (directory / filename).write_text( yaml.safe_dump( @@ -177,6 +178,20 @@ def test_kimi_curve_requires_an_explicit_supported_sampler( "SGLANG_SIMULATE_ACC_TOKEN_MODE": "real-draft-token", }, ), + ( + "sglang", + { + "speculative-algorithm": "DFLASH", + "speculative-draft-model-path": "RadixArk/Muse-Spark-Open-DFlash2", + "speculative-num-draft-tokens": 4, + }, + {"MODEL_PREFIX": "musespark"}, + { + "SGLANG_SIMULATE_ACC_LEN": "3.4", + "SGLANG_SIMULATE_ACC_METHOD": "match-expected", + "SGLANG_SIMULATE_ACC_TOKEN_MODE": "real-draft-token", + }, + ), ( "dynamo-sglang", {"speculative-algorithm": "EAGLE", "speculative-num-steps": 3}, @@ -226,6 +241,24 @@ def test_engine_token_selection_and_environment( assert result["environment"] == {"KEEP_GLOBAL": "yes"} +@pytest.mark.parametrize("width", [None, True, 1, 4.0]) +def test_dflash_rejects_invalid_verification_block(golden_dir: Path, width: Any) -> None: + recipe = { + "roles": { + "agg": { + "args": { + "speculative-algorithm": "DFLASH", + "speculative-num-draft-tokens": width, + } + } + } + } + with pytest.raises(ValueError, match="positive integer draft length"): + build_overrides( + recipe, "sglang", {**ENV, "MODEL_PREFIX": "musespark"}, golden_dir=golden_dir + ) + + @pytest.mark.parametrize( "environment", [{"EVAL_ONLY": "true"}, {"IS_AGENTIC": "0"}, {"SPEC_DECODING": "none"}], diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f66d1177c7..186be3ec94 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9892,3 +9892,12 @@ - "The new image brings Kimi-K3 ROCm changes vllm-project/vllm#59591 (latent MoE up-proj stores only the local rank's shards), #59069 (AttnRes output fused with per-token FP8 input quantization), #60717 (vLLM's FP8 max in that fused quantization), #60446 (KDA autotune keys), #59523 (cuMem CUDA-graph pool offload on ROCm) and #59880 (speculative block reservations across KV cache groups with a KV connector)." - "No change to the Inferact/Kimi-K3-DSpark draft's precision: K3DSparkModel and get_draft_quant_config() are unchanged between the two images, so the draft keeps BF16 weights and an fp8 KV cache." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3951 + +- config-keys: + - musespark-fp4-h100-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Add Muse Spark Open NVFP4 AgentX on H100 at TP8/EP8 with upstream Marlin dense/MoE kernels, FA3 target attention, FA4 draft attention, and FP8 KV." + - "Start at concurrency 1 with DFlash2, GPU-resident KV, and prefill CUDA graphs disabled; use the committed golden acceptance curve for throughput and real verification for evals." + pr-link: "" From eb54f18ab5c49def1627b7d590593fd272158106 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 15 Oct 2026 17:02:46 -0700 Subject: [PATCH 2/8] chore: link H100 Muse Spark sweep to PR 4149 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 186be3ec94..34f1512ac1 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9900,4 +9900,4 @@ description: - "Add Muse Spark Open NVFP4 AgentX on H100 at TP8/EP8 with upstream Marlin dense/MoE kernels, FA3 target attention, FA4 draft attention, and FP8 KV." - "Start at concurrency 1 with DFlash2, GPU-resident KV, and prefill CUDA graphs disabled; use the committed golden acceptance curve for throughput and real verification for evals." - pr-link: "" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4149 From 43926e6f2908415baf8c68c85b09f1a66a184224 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 15 Oct 2026 17:24:00 -0700 Subject: [PATCH 3/8] fix: reserve H100 memory for Muse Spark KV cache --- .../musespark/sglang/h100-fp4-mtp/agentic.yaml | 3 ++- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml index 8c95728b40..eb38c77cb3 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml @@ -29,7 +29,8 @@ base: fp4-gemm-backend: marlin attention-backend: fa3 kv-cache-dtype: fp8_e4m3 - mem-fraction-static: 0.85 + # Loaded target and draft weights require a static fraction above 0.94. + mem-fraction-static: 0.97 # Reserve H100 memory for the target, draft, and required AgentX context. disable-prefill-cuda-graph: true max-running-requests: 2 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 34f1512ac1..747ffa10ab 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9899,5 +9899,5 @@ - agentic-coding description: - "Add Muse Spark Open NVFP4 AgentX on H100 at TP8/EP8 with upstream Marlin dense/MoE kernels, FA3 target attention, FA4 draft attention, and FP8 KV." - - "Start at concurrency 1 with DFlash2, GPU-resident KV, and prefill CUDA graphs disabled; use the committed golden acceptance curve for throughput and real verification for evals." + - "Start at concurrency 1 with DFlash2, GPU-resident KV, a 0.97 static memory fraction, and prefill CUDA graphs disabled; use the committed golden acceptance curve for throughput and real verification for evals." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4149 From e8e287469b9b8bf39d95187811d4a3796b8d9452 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 15 Oct 2026 18:10:01 -0700 Subject: [PATCH 4/8] fix: fit Muse Spark prefill and long-context KV on H100 --- .../musespark/sglang/h100-fp4-mtp/agentic.yaml | 5 ++++- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml index eb38c77cb3..5d1d68d3ca 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml @@ -31,7 +31,10 @@ base: kv-cache-dtype: fp8_e4m3 # Loaded target and draft weights require a static fraction above 0.94. mem-fraction-static: 0.97 - # Reserve H100 memory for the target, draft, and required AgentX context. + # Bound prefill activations while retaining the full request context. + chunked-prefill-size: 1024 + # Full-attention KV must fit 256k prompts; SWA retains its 2048-token window. + swa-full-tokens-ratio: 0.1 disable-prefill-cuda-graph: true max-running-requests: 2 cuda-graph-max-bs-decode: 2 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 747ffa10ab..fcdd8ae81b 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9899,5 +9899,5 @@ - agentic-coding description: - "Add Muse Spark Open NVFP4 AgentX on H100 at TP8/EP8 with upstream Marlin dense/MoE kernels, FA3 target attention, FA4 draft attention, and FP8 KV." - - "Start at concurrency 1 with DFlash2, GPU-resident KV, a 0.97 static memory fraction, and prefill CUDA graphs disabled; use the committed golden acceptance curve for throughput and real verification for evals." + - "Start at concurrency 1 with DFlash2, GPU-resident KV, a 0.97 static memory fraction, 1024-token prefill chunks, a 0.1 SWA/full KV token ratio, and prefill CUDA graphs disabled; use the committed golden acceptance curve for throughput and real verification for evals." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4149 From 7f352d519e4122e4c6a87e507972b05804a6595b Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:47:03 +0800 Subject: [PATCH 5/8] fix: cap H100 Muse Spark decode graphs at 8 for vision headroom c48 started with 0.29 GB free per GPU and OOMed in the vision encoder's all-gather on the first image prefill. --- .../musespark/sglang/h100-fp4-mtp/agentic.yaml | 5 +++-- inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml index 4a887483a6..267f57fcd1 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml @@ -68,8 +68,9 @@ zip_override_tp8: agg: args: max-running-requests: [2, 4, 8, 16, 32, 48, 64, 96, 128] - # Each graph slot reserves seven full-context int32 page tables. - cuda-graph-max-bs-decode: [2, 4, 8, 16, 16, 16, 16, 16, 16] + # Each graph slot reserves seven full-context int32 page tables, and + # bs=16 graphs leave too little headroom for the vision encoder. + cuda-graph-max-bs-decode: [2, 4, 8, 8, 8, 8, 8, 8, 8] benchmark: env: CONC: ['1', '2', '4', '8', '16', '24', '32', '48', '64'] diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 39cf7d2c3a..ac4b89e980 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9980,5 +9980,5 @@ - agentic-coding description: - "Add Muse Spark Open NVFP4 AgentX on H100 at TP8/EP8 with upstream Marlin dense/MoE kernels, FA3 target attention, FA4 draft attention, and FP8 KV." - - "Sweep concurrency 1, 2, 4, 8, 16, 24, 32, 48, and 64 with DFlash2, GPU-resident KV, a 0.97 static memory fraction, 1024-token prefill chunks, a 0.1 SWA/full KV token ratio, and prefill CUDA graphs disabled; cap decode graphs at batch 16 while retaining 2x concurrency request capacity. Use the committed golden acceptance curve for throughput and real verification for evals." + - "Sweep concurrency 1, 2, 4, 8, 16, 24, 32, 48, and 64 with DFlash2, GPU-resident KV, a 0.97 static memory fraction, 1024-token prefill chunks, a 0.1 SWA/full KV token ratio, and prefill CUDA graphs disabled; cap decode graphs at batch 8 to leave vision-encoder headroom while retaining 2x concurrency request capacity. Use the committed golden acceptance curve for throughput and real verification for evals." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4149 From 266c72c1fa3e153598284a414996a8a80122471a Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Fri, 16 Oct 2026 00:55:43 -0700 Subject: [PATCH 6/8] fix: keep H100 DFlash sampling valid above graph capacity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 启用 eager draft sampler，保留解码图批量上限 8 和完整并发扫描。 --- .../musespark/sglang/h100-fp4-mtp/agentic.yaml | 2 ++ inferencex-e2e/perf-changelog.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml index 267f57fcd1..2e03ea6db7 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml @@ -49,6 +49,8 @@ base: env: PYTHONNOUSERSITE: '1' PYTHONUNBUFFERED: '1' + # Eager sampling supports admission above the captured graph batch size. + SGLANG_DFLASH_EAGER_DRAFT_SAMPLER: '1' # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. SGLANG_TIMEOUT_KEEP_ALIVE: '900' health_check: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index ac4b89e980..1d1b02e8f2 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9980,5 +9980,5 @@ - agentic-coding description: - "Add Muse Spark Open NVFP4 AgentX on H100 at TP8/EP8 with upstream Marlin dense/MoE kernels, FA3 target attention, FA4 draft attention, and FP8 KV." - - "Sweep concurrency 1, 2, 4, 8, 16, 24, 32, 48, and 64 with DFlash2, GPU-resident KV, a 0.97 static memory fraction, 1024-token prefill chunks, a 0.1 SWA/full KV token ratio, and prefill CUDA graphs disabled; cap decode graphs at batch 8 to leave vision-encoder headroom while retaining 2x concurrency request capacity. Use the committed golden acceptance curve for throughput and real verification for evals." + - "Sweep concurrency 1, 2, 4, 8, 16, 24, 32, 48, and 64 with DFlash2, GPU-resident KV, a 0.97 static memory fraction, 1024-token prefill chunks, a 0.1 SWA/full KV token ratio, and prefill CUDA graphs disabled; cap decode graphs at batch 8 to leave vision-encoder headroom and use eager DFlash draft sampling above graph capacity while retaining 2x concurrency request capacity. Use the committed golden acceptance curve for throughput and real verification for evals." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4149 From 5f0c0b284811f8ecbd45aa22cfe0efc4cfcdca75 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Fri, 16 Oct 2026 17:10:15 +0800 Subject: [PATCH 7/8] fix: cap H100 Muse Spark at 8 running requests, sweep c1-c16 Batches above the batch-8 decode graphs run eager and OOM in fused MoE with ~1 GB free per GPU (c48, run 38870885984). Drop c24-c64. --- .../musespark/sglang/h100-fp4-mtp/agentic.yaml | 7 ++++--- inferencex-e2e/configs/nvidia-master.yaml | 2 +- inferencex-e2e/perf-changelog.yaml | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml index 2e03ea6db7..860efad7e9 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml @@ -69,10 +69,11 @@ zip_override_tp8: roles: agg: args: - max-running-requests: [2, 4, 8, 16, 32, 48, 64, 96, 128] + # Eager batches above the decode graphs OOM in the MoE with ~1 GB free. + max-running-requests: [2, 4, 8, 8, 8] # Each graph slot reserves seven full-context int32 page tables, and # bs=16 graphs leave too little headroom for the vision encoder. - cuda-graph-max-bs-decode: [2, 4, 8, 8, 8, 8, 8, 8, 8] + cuda-graph-max-bs-decode: [2, 4, 8, 8, 8] benchmark: env: - CONC: ['1', '2', '4', '8', '16', '24', '32', '48', '64'] + CONC: ['1', '2', '4', '8', '16'] diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index bdc557cc8b..03321c08f2 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9695,4 +9695,4 @@ musespark-fp4-h100-sglang-agentic-dflash: scenarios: agentic-coding: - search-space: - - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 24, 32, 48, 64], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: agentic.yaml } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 1d1b02e8f2..96431e93a2 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9980,5 +9980,5 @@ - agentic-coding description: - "Add Muse Spark Open NVFP4 AgentX on H100 at TP8/EP8 with upstream Marlin dense/MoE kernels, FA3 target attention, FA4 draft attention, and FP8 KV." - - "Sweep concurrency 1, 2, 4, 8, 16, 24, 32, 48, and 64 with DFlash2, GPU-resident KV, a 0.97 static memory fraction, 1024-token prefill chunks, a 0.1 SWA/full KV token ratio, and prefill CUDA graphs disabled; cap decode graphs at batch 8 to leave vision-encoder headroom and use eager DFlash draft sampling above graph capacity while retaining 2x concurrency request capacity. Use the committed golden acceptance curve for throughput and real verification for evals." + - "Sweep concurrency 1, 2, 4, 8, and 16 with DFlash2, GPU-resident KV, a 0.97 static memory fraction, 1024-token prefill chunks, a 0.1 SWA/full KV token ratio, and prefill CUDA graphs disabled; cap decode graphs and running requests at batch 8, since eager batches above graph capacity OOM with ~1 GB free per GPU; concurrency 16 measures queueing at that cap. Use the committed golden acceptance curve for throughput and real verification for evals." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4149 From 367f687d26934c46e4f79f02ef766f56a1192b69 Mon Sep 17 00:00:00 2001 From: Bryan Shan <58582368+Oseltamivir@users.noreply.github.com> Date: Fri, 16 Oct 2026 21:32:12 +0800 Subject: [PATCH 8/8] =?UTF-8?q?[AgentX]=20add=20HiCache=20offload=20to=20M?= =?UTF-8?q?use=20Spark=20Open=20H100=20/=20[AgentX]=20=E4=B8=BA=20Muse=20S?= =?UTF-8?q?park=20Open=20H100=20=E6=B7=BB=E5=8A=A0=20HiCache=20=E5=8D=B8?= =?UTF-8?q?=E8=BD=BD=20(#4181)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add HiCache offload arm to H100 Muse Spark at c2-c8 * chore: link H100 HiCache entry to PR 4181 --- .../musespark/sglang/h100-fp4-mtp/agentic.yaml | 16 ++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 4 +++- inferencex-e2e/perf-changelog.yaml | 9 +++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml index 860efad7e9..3c55839fa3 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml @@ -77,3 +77,19 @@ zip_override_tp8: benchmark: env: CONC: ['1', '2', '4', '8', '16'] + KV_OFFLOADING: none + +# The GPU KV pool is a few GB per rank, so agentic prefixes evict between turns; +# HiCache keeps them in host DRAM instead of recomputing them. +zip_override_tp8_hicache: + roles: + agg: + args: + max-running-requests: [4, 8, 8] + cuda-graph-max-bs-decode: [4, 8, 8] + enable-hierarchical-cache: true + hicache-size: 160 + benchmark: + env: + CONC: ['2', '4', '8'] + KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index bc36fab26d..1fc173c6a4 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9708,5 +9708,7 @@ musespark-fp4-h100-sglang-agentic-dflash: srt-recipe-dir: musespark/sglang/h100-fp4-mtp scenarios: agentic-coding: - - search-space: + - dram-utilization: 0.80 + search-space: - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [2, 4, 8], srt-recipe: agentic.yaml } diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 2ea1838d04..8ec368892e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10004,3 +10004,12 @@ - "Synthetic acceptance follows the committed musespark_dflash golden curve (#4138, #4139): AL 4.05 at 7 draft tokens, thinking_on." - "The server loads the checkpoint and draft from the pre-seeded shared HF cache with HF_HUB_OFFLINE=1, so concurrent points make no Hub API calls." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4127 + +- config-keys: + - musespark-fp4-h100-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Add a HiCache host-DRAM KV offload arm at concurrency 2, 4 and 8 with 160 GB per rank, alongside the GPU-resident sweep; the few-GB GPU KV pool left by 67 GB of weights per H100 evicts agentic prefixes between turns." + - "Keep the batch-8 cap on running requests and decode graphs from the GPU-resident sweep." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4181