diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml new file mode 100644 index 0000000000..3c55839fa3 --- /dev/null +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/musespark/sglang/h100-fp4-mtp/agentic.yaml @@ -0,0 +1,95 @@ +base: + schema: 2 + name: musespark-fp4-h100-sglang-agentic + resources: + gpu_type: h100 + gpus_per_node: 8 + slurm: + time_limit: '4:00:00' + frontend: + type: sglang + enable_multiple_frontends: false + observability: + enabled: false + tachometer: + enabled: false + engine: sglang + roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + args: + served-model-name: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 + trust-remote-code: true + tensor-parallel-size: 8 + expert-parallel-size: 8 + quantization: modelopt_fp4 + moe-runner-backend: marlin + fp4-gemm-backend: marlin + attention-backend: fa3 + kv-cache-dtype: fp8_e4m3 + # Loaded target and draft weights require a static fraction above 0.94. + mem-fraction-static: 0.97 + # Bound prefill activations while retaining the full request context. + chunked-prefill-size: 1024 + # Full-attention KV must fit 256k prompts; SWA retains its 2048-token window. + swa-full-tokens-ratio: 0.1 + disable-prefill-cuda-graph: true + # DFlash2 verifies 8 tokens per step: the current token plus 7 drafts. + speculative-algorithm: DFLASH + speculative-draft-model-path: RadixArk/Muse-Spark-Open-DFlash2 + speculative-draft-attention-backend: fa4 + speculative-num-draft-tokens: 8 + mm-feature-transport: cpu + reasoning-parser: muse + tool-call-parser: muse + watchdog-timeout: 3600 + enable-metrics: true + env: + PYTHONNOUSERSITE: '1' + PYTHONUNBUFFERED: '1' + # Eager sampling supports admission above the captured graph batch size. + SGLANG_DFLASH_EAGER_DRAFT_SAMPLER: '1' + # Outlast AIPerf's pooled connections past Uvicorn's keep-alive. + SGLANG_TIMEOUT_KEEP_ALIVE: '900' + health_check: + max_attempts: 1440 + interval_seconds: 5 + benchmark: + env: + # The gated Hub repo is unreadable to the client; use the staged checkpoint. + TOKENIZER: /model + # Warmup can leave bytes unacknowledged past AIPerf's 30 s default. + AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + +zip_override_tp8: + roles: + agg: + args: + # Eager batches above the decode graphs OOM in the MoE with ~1 GB free. + max-running-requests: [2, 4, 8, 8, 8] + # Each graph slot reserves seven full-context int32 page tables, and + # bs=16 graphs leave too little headroom for the vision encoder. + cuda-graph-max-bs-decode: [2, 4, 8, 8, 8] + benchmark: + env: + CONC: ['1', '2', '4', '8', '16'] + KV_OFFLOADING: none + +# The GPU KV pool is a few GB per rank, so agentic prefixes evict between turns; +# HiCache keeps them in host DRAM instead of recomputing them. +zip_override_tp8_hicache: + roles: + agg: + args: + max-running-requests: [4, 8, 8] + cuda-graph-max-bs-decode: [4, 8, 8] + enable-hierarchical-cache: true + hicache-size: 160 + benchmark: + env: + CONC: ['2', '4', '8'] + KV_OFFLOADING: dram diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index ffd7a896bb..1fc173c6a4 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9696,3 +9696,19 @@ musespark-fp4-b300-sglang-agentic-dflash: agentic-coding: - search-space: - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16, 24, 32, 48, 64, 128], srt-recipe: agentic.yaml } + +musespark-fp4-h100-sglang-agentic-dflash: + image: lmsysorg/sglang:dev-muse-spark-open@sha256:8cfc5a72cb296771f5f8a4a4688cb6d05685610a830cbb616fa4ab0eb7bb2b0a + model: meta-models/Muse-Spark-Open-816B-A42B-NVFP4 + model-prefix: musespark + runner: cluster:h100-dsxe + precision: fp4 + framework: sglang + multinode: false + srt-recipe-dir: musespark/sglang/h100-fp4-mtp + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 8, ep: 8, kv-offloading: none, spec-decoding: draft_model, conc-list: [1, 2, 4, 8, 16], srt-recipe: agentic.yaml } + - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: draft_model, conc-list: [2, 4, 8], srt-recipe: agentic.yaml } diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index e5e775bd8b..01dfc45c9a 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -362,6 +362,7 @@ clusters: entries: DeepSeek-V4.1-Flash: {root: data-models, dir: DeepSeek-V4.1-Flash} MiniMax-M3-MXFP8: {root: data-models, dir: MiniMax-M3-MXFP8} + Muse-Spark-Open-816B-A42B-NVFP4: {root: data-models, dir: Muse-Spark-Open-816B-A42B-NVFP4} Qwen3.5-397B-A17B-FP8: {root: data-models, dir: Qwen3.5-397B-A17B-FP8} scheduler: slurm slurm: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index c5da284820..8ec368892e 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9984,6 +9984,15 @@ - "DEP2 and DEP4 arms: MegaMoE all-to-all with flashinfer_mxfp4 runner, trtllm DSv4 attention, HiCache (ratio 4 on DEP2, 2 on DEP4), shortest-prefill-first, DP spec-prefill coordination, and sglang-router 0.3.2 consistent hashing with correlation-ID affinity." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4094 +- config-keys: + - musespark-fp4-h100-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Add Muse Spark Open NVFP4 AgentX on H100 at TP8/EP8 with upstream Marlin dense/MoE kernels, FA3 target attention, FA4 draft attention, and FP8 KV." + - "Sweep concurrency 1, 2, 4, 8, and 16 with DFlash2, GPU-resident KV, a 0.97 static memory fraction, 1024-token prefill chunks, a 0.1 SWA/full KV token ratio, and prefill CUDA graphs disabled; cap decode graphs and running requests at batch 8, since eager batches above graph capacity OOM with ~1 GB free per GPU; concurrency 16 measures queueing at that cap. Use the committed golden acceptance curve for throughput and real verification for evals." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4149 + - config-keys: - musespark-fp4-mi355x-sglang-agentic-dflash scenario-type: @@ -9995,3 +10004,12 @@ - "Synthetic acceptance follows the committed musespark_dflash golden curve (#4138, #4139): AL 4.05 at 7 draft tokens, thinking_on." - "The server loads the checkpoint and draft from the pre-seeded shared HF cache with HF_HUB_OFFLINE=1, so concurrent points make no Hub API calls." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4127 + +- config-keys: + - musespark-fp4-h100-sglang-agentic-dflash + scenario-type: + - agentic-coding + description: + - "Add a HiCache host-DRAM KV offload arm at concurrency 2, 4 and 8 with 160 GB per rank, alongside the GPU-resident sweep; the few-GB GPU KV pool left by 67 GB of weights per H100 evicts agentic prefixes between turns." + - "Keep the batch-8 cap on running requests and decode graphs from the GPU-resident sweep." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4181