diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..52a90cf2cd --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h200-fp4-mtp/agentic.yaml @@ -0,0 +1,69 @@ +base: + schema: 2 + name: musespark-fp4-h200-sglang-agentic + resources: + gpu_type: h200 + gpus_per_node: 8 + slurm: + time_limit: '4:00:00' + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 8 + quantization: modelopt_fp4 + moe-runner-backend: marlin + fp4-gemm-backend: marlin + attention-backend: fa3 + kv-cache-dtype: fp8_e4m3 + mem-fraction-static: 0.85 + # DFlash2 verifies 8 tokens per step: the current token plus 7 drafts. + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Muse-Spark-Open-DFlash2 + speculative-draft-attention-backend: fa4 + speculative-num-draft-tokens: 8 + mm-feature-transport: cpu + reasoning-parser: muse + tool-call-parser: muse + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + health_check: + max_attempts: 1440 + interval_seconds: 5 + benchmark: + env: + # The gated Hub repo is unreadable to the client; use the staged checkpoint. + TOKENIZER: /model + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + + +zip_override_tp8: + roles: + agg: + args: + max-running-requests: [2, 4, 8, 16, 32, 48, 64, 96, 128] + cuda-graph-max-bs-decode: [2, 4, 8, 16, 32, 48, 64, 96, 128] + benchmark: + env: + CONC: ['1', '2', '4', '8', '16', '24', '32', '48', '64'] diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 7304874bc4..ffd7a896bb 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8365,6 +8365,20 @@ musespark-fp4-b200-sglang-agentic-dflash: - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 12], srt-recipe: agentic.yaml } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [16, 24, 32, 48], srt-recipe: agentic.yaml } +musespark-fp4-h200-sglang-agentic-dflash: + image: lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a + model: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 + model-prefix: musespark + runner: cluster:h200-dgxc + precision: fp4 + framework: sglang + multinode: false + srt-recipe-dir: musespark/sglang/h200-fp4-mtp + scenarios: + agentic-coding: + - search-space: + - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 24, 32, 48, 64], srt-recipe: agentic.yaml } + dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7 model: deepseek-ai/DeepSeek-V4.1-Flash diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index f8828a785a..e5e775bd8b 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -396,6 +396,7 @@ clusters: GLM-5.2-FP8: {root: models, dir: GLM-5.2-FP8} DeepSeek-V4-Pro-0813: {root: hf-hub-cache, dir: DeepSeek-V4-Pro} Kimi-K3: {root: hf-hub-cache, dir: Kimi-K3} + Muse-Spark-Open-816B-A42B-NVFP4: {root: hf-hub-cache, dir: Muse-Spark-Open-816B-A42B-NVFP4} Qwen3.5-397B-A17B-FP8: {root: hf-hub-cache, dir: Qwen3.5-397B-A17B-FP8} scheduler: slurm slurm: diff --git a/inferencex-e2e/infx/launch/drivers/srt/models.py b/inferencex-e2e/infx/launch/drivers/srt/models.py index 83685efad9..4c9fd68d87 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/models.py +++ b/inferencex-e2e/infx/launch/drivers/srt/models.py @@ -32,6 +32,7 @@ class Override: served_name: str | None = None require_config: bool = False hub: bool = False + single_node_staged: bool = False OVERRIDES: dict[str, tuple[Override, ...]] = { @@ -74,6 +75,7 @@ class Override: ), ), "h200-dgxc": ( + Override(Match(model_glob="*/Muse-Spark-Open-816B-A42B-NVFP4"), single_node_staged=True), Override(Match(any_of("dsv4")), require_config=True), Override( Match(any_of("dsr1"), any_of("fp8"), any_of("dynamo-trt")), @@ -182,7 +184,9 @@ def job_env(cluster: Cluster, request: LaunchRequest, served: str | None) -> dic def single_node_model_path(cluster: Cluster, request: LaunchRequest) -> str: """What the single-node recipe's ``hf:`` serves: the staged checkpoint, or the Hub.""" srt = slurm_settings(cluster).srt_slurm - staged = srt is not None and srt.single_node_models == "staged" + staged = (srt is not None and srt.single_node_models == "staged") or ( + _override(cluster, request, "single_node_staged") is True + ) model = checkpoint(cluster, request) if staged else None return str(host_path(cluster, model)) if model is not None else f"hf:{request.model}" diff --git a/inferencex-e2e/infx/tests/launch/test_srt_policy.py b/inferencex-e2e/infx/tests/launch/test_srt_policy.py index dc6f5eb360..a6e52eee69 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_policy.py @@ -96,6 +96,15 @@ def test_models_resolve_by_basename_with_overrides(tmp_path, monkeypatch, env, p assert single_node_model_path(cluster(tmp_path, "hub"), point) == f"hf:{env['MODEL']}" +def test_single_node_staging_can_be_selected_for_one_workload(tmp_path, monkeypatch): + monkeypatch.setitem(models.OVERRIDES, "c", ( + Override(Match(model_glob="org/M"), single_node_staged=True), + )) + c = cluster(tmp_path, "hub") + assert single_node_model_path(c, request(MODEL="org/M")) == str(tmp_path / "nvme/m") + assert single_node_model_path(c, request(MODEL="org/S")) == "hf:org/S" + + def test_a_checkpoint_that_must_be_readable_fails_before_submission(tmp_path, monkeypatch): monkeypatch.setitem(models.OVERRIDES, "c", OVERRIDES) with pytest.raises(LaunchError, match="no readable .*/s/config.json"): diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 235a0d74e3..5030c27785 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9893,6 +9893,16 @@ - "No change to the Inferact/Kimi-K3-DSpark draft's precision: K3DSparkModel and get_draft_quant_config() are unchanged between the two images, so the draft keeps BF16 weights and an fp8 KV cache." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3951 +- config-keys: + - musespark-fp4-h200-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Add Muse Spark Open NVFP4 AgentX on H200 with upstream SGLang and DFlash2 speculative decoding at TP8/EP8." + - "Use Marlin W4A16 dense and MoE kernels, FA3 target attention, FA4 draft attention, FP8 KV, and an 8-token DFlash verification block with acceptance selected from the committed golden curve." + - "Sweep concurrency 1, 2, 4, 8, 16, 24, 32, 48, and 64 with GPU-resident KV and no layernorm sequence parallelism or MegaMoE." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4148 + - config-keys: - musespark-fp4-b200-sglang-agentic-dflash scenario-type: