From 3b73774ed2a59fbfb6b0806212a8746712c1b774 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 15 Oct 2026 16:50:46 -0700 Subject: [PATCH 1/2] feat: add Muse Spark Open H200 AgentX support --- .../sglang/h200-fp4-mtp/agentic.yaml | 69 +++++++++++++++++++ inferencex-e2e/configs/nvidia-master.yaml | 14 ++++ inferencex-e2e/configs/runners.yaml | 1 + inferencex-e2e/docs/MODELS.md | 1 + .../infx/launch/drivers/srt/models.py | 6 +- inferencex-e2e/infx/srt_slurm/single_node.py | 4 +- .../infx/srt_slurm/synthetic_acceptance.py | 10 +++ .../infx/tests/launch/test_srt_policy.py | 9 +++ .../tests/srt_slurm/test_srt_single_node.py | 9 ++- .../srt_slurm/test_synthetic_acceptance.py | 33 +++++++++ inferencex-e2e/perf-changelog.yaml | 10 +++ 11 files changed, 160 insertions(+), 6 deletions(-) create mode 100644 inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..52a90cf2cd --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml @@ -0,0 +1,69 @@ +base: + schema: 2 + name: musespark-fp4-h200-sglang-agentic + resources: + gpu_type: h200 + gpus_per_node: 8 + slurm: + time_limit: '4:00:00' + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 8 + quantization: modelopt_fp4 + moe-runner-backend: marlin + fp4-gemm-backend: marlin + attention-backend: fa3 + kv-cache-dtype: fp8_e4m3 + mem-fraction-static: 0.85 + # DFlash2 verifies 8 tokens per step: the current token plus 7 drafts. + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Muse-Spark-Open-DFlash2 + speculative-draft-attention-backend: fa4 + speculative-num-draft-tokens: 8 + mm-feature-transport: cpu + reasoning-parser: muse + tool-call-parser: muse + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + health_check: + max_attempts: 1440 + interval_seconds: 5 + benchmark: + env: + # The gated Hub repo is unreadable to the client; use the staged checkpoint. + TOKENIZER: /model + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + + +zip_override_tp8: + roles: + agg: + args: + max-running-requests: [2, 4, 8, 16, 32, 48, 64, 96, 128] + cuda-graph-max-bs-decode: [2, 4, 8, 16, 32, 48, 64, 96, 128] + benchmark: + env: + CONC: ['1', '2', '4', '8', '16', '24', '32', '48', '64'] diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 2ff820f5b9..c31cd5b1d7 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8365,6 +8365,20 @@ dsv41flash-fp4-gb200-sglang-agentic-dspark: - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16], srt-recipe: agentic.yaml } - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [32, 64, 128, 192], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } +musespark-fp4-h200-sglang-agentic-dflash: + image: lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a + model: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 + model-prefix: musespark + runner: cluster:h200-dgxc + precision: fp4 + framework: sglang + multinode: false + srt-recipe-dir: musespark/sglang/h200-fp4-mtp + scenarios: + agentic-coding: + - search-space: + - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 24, 32, 48, 64], srt-recipe: agentic.yaml } + dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7 model: deepseek-ai/DeepSeek-V4.1-Flash diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index 4d25914d7e..28443e2d67 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -396,6 +396,7 @@ clusters: GLM-5.2-FP8: {root: models, dir: GLM-5.2-FP8} DeepSeek-V4-Pro-0813: {root: hf-hub-cache, dir: DeepSeek-V4-Pro} Kimi-K3: {root: hf-hub-cache, dir: Kimi-K3} + Muse-Spark-Open-816B-A42B-NVFP4: {root: hf-hub-cache, dir: Muse-Spark-Open-816B-A42B-NVFP4} Qwen3.5-397B-A17B-FP8: {root: hf-hub-cache, dir: Qwen3.5-397B-A17B-FP8} scheduler: slurm slurm: diff --git a/inferencex-e2e/docs/MODELS.md b/inferencex-e2e/docs/MODELS.md index 0733de44f4..bb04039310 100644 --- a/inferencex-e2e/docs/MODELS.md +++ b/inferencex-e2e/docs/MODELS.md @@ -110,6 +110,7 @@ Other offloading tiers, including NVMe KV cache offloading, are outside the init | Model architecture class | Prefix | Date added | Active scenarios | Deprecated scenarios | |---|---|---|---|---| +| Muse Spark Open 816B-A42B | `musespark` | 2026-10-15 | Agentic coding (SGLang with DFlash2 draft `RadixArk/Muse-Spark-Open-DFlash2`; golden AL curve `musespark_dflash.yaml`) | | | DeepSeek-V4.1-Flash | `dsv41flash` | 2026-09-10 | Agentic coding (vLLM: DSpark, Engram UVA offload; SGLang: DSpark arms added per SKU from 2026-09-17; ATOM: MI355X TP2/TP4 DSpark added 2026-09-23, removed in #3463, restored on srt-slurm 2026-09-30; GPU validation pending) | | | GLM-5.3 | `glm5.3` | 2026-09-22 ([#3366](https://github.com/SemiAnalysisAI/InferenceX/pull/3366)) | Agentic coding (MTP only, per the deprecation policy) | Single-turn 1k1k (deprecated for all models before this model was added; never run) | | Qwen3.8-Flash-Next | `qwen3.8next` | 2026-08-26 ([#2742](https://github.com/SemiAnalysisAI/InferenceX/pull/2742)) | Agentic coding | | diff --git a/inferencex-e2e/infx/launch/drivers/srt/models.py b/inferencex-e2e/infx/launch/drivers/srt/models.py index 83685efad9..4c9fd68d87 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/models.py +++ b/inferencex-e2e/infx/launch/drivers/srt/models.py @@ -32,6 +32,7 @@ class Override: served_name: str | None = None require_config: bool = False hub: bool = False + single_node_staged: bool = False OVERRIDES: dict[str, tuple[Override, ...]] = { @@ -74,6 +75,7 @@ class Override: ), ), "h200-dgxc": ( + Override(Match(model_glob="*/Muse-Spark-Open-816B-A42B-NVFP4"), single_node_staged=True), Override(Match(any_of("dsv4")), require_config=True), Override( Match(any_of("dsr1"), any_of("fp8"), any_of("dynamo-trt")), @@ -182,7 +184,9 @@ def job_env(cluster: Cluster, request: LaunchRequest, served: str | None) -> dic def single_node_model_path(cluster: Cluster, request: LaunchRequest) -> str: """What the single-node recipe's ``hf:`` serves: the staged checkpoint, or the Hub.""" srt = slurm_settings(cluster).srt_slurm - staged = srt is not None and srt.single_node_models == "staged" + staged = (srt is not None and srt.single_node_models == "staged") or ( + _override(cluster, request, "single_node_staged") is True + ) model = checkpoint(cluster, request) if staged else None return str(host_path(cluster, model)) if model is not None else f"hf:{request.model}" diff --git a/inferencex-e2e/infx/srt_slurm/single_node.py b/inferencex-e2e/infx/srt_slurm/single_node.py index c9f8b6c8ac..3e7fc8c6c1 100644 --- a/inferencex-e2e/infx/srt_slurm/single_node.py +++ b/inferencex-e2e/infx/srt_slurm/single_node.py @@ -113,9 +113,9 @@ def validate_recipe(recipe: dict[str, Any], environment: Mapping[str, str]) -> N if environment["FRAMEWORK"] not in {"sglang", "trt", "atom", "vllm"}: raise ValueError(f"Unsupported single-node framework: {environment['FRAMEWORK']!r}") spec = spec_parameters(role, engine) - if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark"}: + if spec and spec["method"] not in {"eagle", "eagle3", "nextn", "mtp", "dspark", "dflash"}: raise ValueError( - "Single-node SRT supports only native MTP, EAGLE3, DSpark or no speculation" + "Single-node SRT supports only native MTP, EAGLE3, DSpark, DFlash or no speculation" ) speculation = "mtp" if spec else workload.get("SPEC_DECODING", "none") agentic = environment["IS_AGENTIC"] == "1" diff --git a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py index 2de8687c22..9884fe1a07 100644 --- a/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py +++ b/inferencex-e2e/infx/srt_slurm/synthetic_acceptance.py @@ -77,6 +77,16 @@ def spec_parameters(role: Mapping[str, Any], engine: str) -> dict[str, Any]: algorithm = next(iter(algorithms), "") if not algorithm: return {} + if algorithm == "dflash": + # DFlash verifies a block of the current token plus its drafts. + width = args.get("speculative-num-draft-tokens") + return { + "method": "dflash", + "num_speculative_tokens": width - 1 + if isinstance(width, int) and not isinstance(width, bool) + else None, + "model": args.get("speculative-draft-model-path", ""), + } return { "method": "dspark" if algorithm == "dspark" else algorithm, "num_speculative_tokens": args.get( diff --git a/inferencex-e2e/infx/tests/launch/test_srt_policy.py b/inferencex-e2e/infx/tests/launch/test_srt_policy.py index dc6f5eb360..a6e52eee69 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_policy.py @@ -96,6 +96,15 @@ def test_models_resolve_by_basename_with_overrides(tmp_path, monkeypatch, env, p assert single_node_model_path(cluster(tmp_path, "hub"), point) == f"hf:{env['MODEL']}" +def test_single_node_staging_can_be_selected_for_one_workload(tmp_path, monkeypatch): + monkeypatch.setitem(models.OVERRIDES, "c", ( + Override(Match(model_glob="org/M"), single_node_staged=True), + )) + c = cluster(tmp_path, "hub") + assert single_node_model_path(c, request(MODEL="org/M")) == str(tmp_path / "nvme/m") + assert single_node_model_path(c, request(MODEL="org/S")) == "hf:org/S" + + def test_a_checkpoint_that_must_be_readable_fails_before_submission(tmp_path, monkeypatch): monkeypatch.setitem(models.OVERRIDES, "c", OVERRIDES) with pytest.raises(LaunchError, match="no readable .*/s/config.json"): diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py index 4d88d9c86f..0e1b6bbdad 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_srt_single_node.py @@ -114,15 +114,18 @@ def test_ambiguous_native_variants_are_rejected(point): runtime_arguments(str(path), env) -def test_mtp_binding_uses_real_verification_and_the_chat_template(point, tmp_path): +@pytest.mark.parametrize("algorithm,spec_decoding", [("EAGLE", "mtp"), ("DFLASH", "draft_model")]) +def test_speculative_binding_uses_real_verification_and_the_chat_template( + point, tmp_path, algorithm, spec_decoding +): path, recipe, env = point recipe["roles"]["agg"]["args"].update({ - "expert-parallel-size": 4, "speculative-algorithm": "EAGLE", + "expert-parallel-size": 4, "speculative-algorithm": algorithm, "speculative-num-steps": 2, "speculative-num-draft-tokens": 3, }) recipe["roles"]["agg"]["env"] = {"SGLANG_SIMULATE_ACC_LEN": "2.5"} path.write_text(yaml.safe_dump({"base": recipe})) - env = {**env, "EP_SIZE": "4", "SPEC_DECODING": "mtp"} + env = {**env, "EP_SIZE": "4", "SPEC_DECODING": spec_decoding} _, bound = select_recipe(f"{path}:base", env) assert bound["benchmark"]["env"]["USE_CHAT_TEMPLATE"] == "true" concrete = tmp_path / "bound.yaml" diff --git a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py index b51e80bfd1..be7be00ed8 100644 --- a/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py +++ b/inferencex-e2e/infx/tests/srt_slurm/test_synthetic_acceptance.py @@ -51,6 +51,7 @@ def golden_dir(tmp_path: Path) -> Path: ("minimaxm3_eagle3.yaml", "minimax-m3", 2.5), ("minimaxm3_eagle3_gqa.yaml", "minimax-m3", 2.6), ("glm5.3_mtp.yaml", "glm-5.3", 3.2), + ("musespark_dflash.yaml", "muse-spark-open", 3.4), ]: (directory / filename).write_text( yaml.safe_dump( @@ -177,6 +178,20 @@ def test_kimi_curve_requires_an_explicit_supported_sampler( "SGLANG_SIMULATE_ACC_TOKEN_MODE": "real-draft-token", }, ), + ( + "sglang", + { + "speculative-algorithm": "DFLASH", + "speculative-draft-model-path": "RadixArk/Muse-Spark-Open-DFlash2", + "speculative-num-draft-tokens": 4, + }, + {"MODEL_PREFIX": "musespark"}, + { + "SGLANG_SIMULATE_ACC_LEN": "3.4", + "SGLANG_SIMULATE_ACC_METHOD": "match-expected", + "SGLANG_SIMULATE_ACC_TOKEN_MODE": "real-draft-token", + }, + ), ( "dynamo-sglang", {"speculative-algorithm": "EAGLE", "speculative-num-steps": 3}, @@ -226,6 +241,24 @@ def test_engine_token_selection_and_environment( assert result["environment"] == {"KEEP_GLOBAL": "yes"} +@pytest.mark.parametrize("width", [None, True, 1, 4.0]) +def test_dflash_rejects_invalid_verification_block(golden_dir: Path, width: Any) -> None: + recipe = { + "roles": { + "agg": { + "args": { + "speculative-algorithm": "DFLASH", + "speculative-num-draft-tokens": width, + } + } + } + } + with pytest.raises(ValueError, match="positive integer draft length"): + build_overrides( + recipe, "sglang", {**ENV, "MODEL_PREFIX": "musespark"}, golden_dir=golden_dir + ) + + @pytest.mark.parametrize( "environment", [{"EVAL_ONLY": "true"}, {"IS_AGENTIC": "0"}, {"SPEC_DECODING": "none"}], diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f66d1177c7..d57d9fd8d3 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9892,3 +9892,13 @@ - "The new image brings Kimi-K3 ROCm changes vllm-project/vllm#59591 (latent MoE up-proj stores only the local rank's shards), #59069 (AttnRes output fused with per-token FP8 input quantization), #60717 (vLLM's FP8 max in that fused quantization), #60446 (KDA autotune keys), #59523 (cuMem CUDA-graph pool offload on ROCm) and #59880 (speculative block reservations across KV cache groups with a KV connector)." - "No change to the Inferact/Kimi-K3-DSpark draft's precision: K3DSparkModel and get_draft_quant_config() are unchanged between the two images, so the draft keeps BF16 weights and an fp8 KV cache." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3951 + +- config-keys: + - musespark-fp4-h200-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Add Muse Spark Open NVFP4 AgentX on H200 with upstream SGLang and DFlash2 speculative decoding at TP8/EP8." + - "Use Marlin W4A16 dense and MoE kernels, FA3 target attention, FA4 draft attention, FP8 KV, and an 8-token DFlash verification block with acceptance selected from the committed golden curve." + - "Sweep concurrency 1, 2, 4, 8, 16, 24, 32, 48, and 64 with GPU-resident KV and no layernorm sequence parallelism or MegaMoE." + pr-link: "" From adc40a25d4fc5055cca5016e18ac08c9d5f2b9d4 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Thu, 15 Oct 2026 16:51:43 -0700 Subject: [PATCH 2/2] chore: link H200 Muse Spark sweep to PR 4148 --- inferencex-e2e/perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index d57d9fd8d3..070a1901f9 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9901,4 +9901,4 @@ - "Add Muse Spark Open NVFP4 AgentX on H200 with upstream SGLang and DFlash2 speculative decoding at TP8/EP8." - "Use Marlin W4A16 dense and MoE kernels, FA3 target attention, FA4 draft attention, FP8 KV, and an 8-token DFlash verification block with acceptance selected from the committed golden curve." - "Sweep concurrency 1, 2, 4, 8, 16, 24, 32, 48, and 64 with GPU-resident KV and no layernorm sequence parallelism or MegaMoE." - pr-link: "" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4148