From f94cabe247ec0cf044d60534319dfaab2d2e9c54 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 3 Aug 2026 11:27:57 -0500 Subject: [PATCH 01/10] perf(agentx): add B200 Kimi K3 DSpark refresh --- .../agg-b200-tp8pp2-dspark-agentic.yaml | 143 ++++++++++++++++++ configs/nvidia-master.yaml | 32 ++++ perf-changelog.yaml | 5 + 3 files changed, 180 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml new file mode 100644 index 0000000000..49f1f0b0dd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml @@ -0,0 +1,143 @@ +name: "kimik3-vllm-agg-b200-tp8pp2-dspark-agentic" + +# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). +# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not +# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the +# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is +# deliberately off, so the 896 routed experts are TP-sharded inside each +# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. +# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. +# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in +# the kimi-k3 bring-up image. +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the +# multinode extension on functionstackx/srt-slurm-nv branch +# klaud/direct-vllm-multinode): `vllm serve` owns the OpenAI port itself, so +# no Dynamo frontend/worker is involved and no dynamo install is needed. +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns + # the public port; rank-1 runs a headless engine worker (vLLM-native + # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, carried by the + # functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch). + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_SERVER_DEV_MODE: "1" + # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window + # generous, and let one long AgentX request hold a PP stage beyond vLLM's + # 300-second model-execution default. + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + # Prefix-cache retention (variant G): 0, on the otherwise-unchanged + # GPU-resident variant D config. Any positive value must be a multiple of + # Kimi-K3's KDA-hybrid scheduler_block_size (3145728; the GB recipes' + # 32768 is hard-rejected at engine init — verified on this PR family), so + # 0 is the only value below one 3.1M-token scheduler block. + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value + # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a + # scheduler_block_size of 3145728 and the interval must be a multiple of + # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative + # and a multiple of scheduler_block_size (3145728)"). Default retention + # served fine in earlier runs. + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a + # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first + # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first + # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 + # matches the GB200/GB300 agentic recipes. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — + # legitimate here because this recipe serves directly with vllm serve + # (frontend.type: vllm), not through the dynamo worker entrypoint that + # rejects them. + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + # DSpark level 7 with probabilistic drafting and the committed golden + # synthetic acceptance length measured for Kimi-K3 at this draft length. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + # No explicit max-model-len: let vLLM derive the native 1M window from + # the model config (agentic trajectories blow past any small cap, and + # K3's KDA layers keep per-token KV small — only the 24 gated-MLA + # layers hold cache). Prefix caching stays on (default) for trajectory + # reuse. Cap prefill chunks so a single long request cannot OOM a + # pipeline stage; let vLLM pick max-num-seqs. + max-num-batched-tokens: 8192 + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep the aggregate worker in the multinode result schema so ingestion + # uses the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + # aiperf's conv-aware routing emits nvext.session_control, a removed POC + # field this dynamo build 400-rejects at warmup (schema moved to + # router/routing_constraints/agent_hints). Same opt-out as the GB300 + # aggregate AgentX recipes — and with a single aggregate worker there is + # no P/D routing to bind anyway. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 90e54195b6..4fdafcea1f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9253,6 +9253,38 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false +# Kimi-K3 MXFP4 B200 DSpark variant of the aggregated TP8 x PP2 topology. +# The server recipe applies the DSpark draft configuration; `mtp` selects the +# repository's speculative-decoding benchmark path and result identity. +kimik3-fp4-b200-dynamo-vllm-agentic-dspark: + image: vllm/vllm-openai:kimi-k3 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b200-dgxc + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: mtp + conc-list: [1, 2, 4, 8, 16, 32] + prefill: + num-worker: 1 + tp: 8 + pp: 2 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + pp: 2 + ep: 1 + dp-attn: false + qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7a064f8090..2b7d6e3703 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5374,3 +5374,8 @@ - "Image: lmsysorg/sglang:v0.5.16-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2420 +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + description: + - "Initial B200 Kimi K3 DSpark submission with corrected AgentX harness" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/0 From 256df24ace82b7c009948369d4896c942902e795 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 3 Aug 2026 11:28:22 -0500 Subject: [PATCH 02/10] chore: link B200 DSpark refresh PR --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2b7d6e3703..9383252290 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5378,4 +5378,4 @@ - kimik3-fp4-b200-dynamo-vllm-agentic-dspark description: - "Initial B200 Kimi K3 DSpark submission with corrected AgentX harness" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/0 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 From 9515848a301da0173ce60dbeea7243af1df246c4 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:31:55 -0500 Subject: [PATCH 03/10] feat: add b200 kimi k3 agentx profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：添加 B200 Kimi K3 AgentX 延迟、均衡吞吐、GPU 常驻高并发和 CPU KV 卸载配置，并通过 srt-slurm 启动多节点聚合式推理。 --- .../configs/kimik3-dspark-config-compat.sh | 47 ++++++ ...-b200-dep16-throughput-dspark-agentic.yaml | 124 +++++++++++++++ ...ut-vllm-simple-offload-dspark-agentic.yaml | 125 +++++++++++++++ ...gg-b200-tep16-balanced-dspark-agentic.yaml | 103 +++++++++++++ .../agg-b200-tp16-latency-dspark-agentic.yaml | 105 +++++++++++++ .../agg-b200-tp8pp2-dspark-agentic.yaml | 143 ------------------ configs/nvidia-master.yaml | 70 +++++++-- perf-changelog.yaml | 4 +- runners/launch_b200-dgxc.sh | 22 ++- 9 files changed, 577 insertions(+), 166 deletions(-) create mode 100755 benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh new file mode 100755 index 0000000000..1822eadc9a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Dynamo serializes the draft config without Kimi K3's config-class alias. +# Prepare a local metadata view that exposes the equivalent parallel-draft token +# without changing the downloaded checkpoint or vLLM source. +python3 - <<'PY' +import json +import os +from pathlib import Path + +from huggingface_hub import snapshot_download + +repo_id = "Inferact/Kimi-K3-DSpark" +target = Path("/tmp/Kimi-K3-DSpark") +snapshot = Path(snapshot_download(repo_id=repo_id)) +target.mkdir(parents=True, exist_ok=True) + +for source in snapshot.iterdir(): + if source.name == "config.json": + continue + destination = target / source.name + if destination.is_symlink(): + if destination.resolve() == source.resolve(): + continue + destination.unlink() + elif destination.exists(): + raise RuntimeError(f"Refusing to replace non-symlink path: {destination}") + destination.symlink_to(source) + +config = json.loads((snapshot / "config.json").read_text()) +mask_token_id = config.get("mask_token_id") +if not isinstance(mask_token_id, int): + raise RuntimeError(f"{repo_id} config is missing integer mask_token_id") + +pard_token = config.get("pard_token") +if pard_token not in (None, mask_token_id): + raise RuntimeError( + f"{repo_id} pard_token={pard_token} disagrees with mask_token_id={mask_token_id}" + ) +config["pard_token"] = mask_token_id + +temporary = target / "config.json.tmp" +temporary.write_text(json.dumps(config, indent=2) + "\n") +os.replace(temporary, target / "config.json") +print(f"Prepared {repo_id} compatibility view at {target}") +PY diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml new file mode 100644 index 0000000000..e14c238acf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml @@ -0,0 +1,124 @@ +name: "kimik3-vllm-agg-b200-dep16-throughput-dspark-agentic" + +# Official throughput-oriented multi_node_dep profile for two 8xB200 nodes. +# One Dynamo vLLM process per node owns eight local DP ranks; srt-slurm derives +# start ranks 0 and 8 from the allocated topology and EP spans all 16 GPUs. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +setup_script: kimik3-dspark-config-compat.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "dynamo" + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 2 + kv-cache-block-size: 64 + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + data-parallel-size: 16 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: deep_gemm_mega_moe + all2all-backend: deepep_v2 + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + dyn-tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + dyn-reasoning-parser: kimi_k3 + language-model-only: true + block-size: 64 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AGENTIC_WARMUP_GRACE_PERIOD: "3600" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml new file mode 100644 index 0000000000..c1d8a6a651 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml @@ -0,0 +1,125 @@ +name: "kimik3-vllm-agg-b200-dep16-simple-offload-dspark-agentic" + +# Official multi_node_dep profile with SimpleCPUOffloadConnector. Each of the +# 16 DP ranks reserves the official 220 GiB host-KV pool. Eight ranks consume +# 1.72 TiB on each B200 node, below the runner's configured host-DRAM budget. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + frameworks: + dynamo: "1.3.0" + +dynamo: + hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" + install: true + +setup_script: kimik3-dspark-config-compat.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: "dynamo" + router-mode: "kv" + router-kv-events: true + router-temperature: "0" + router-min-initial-workers: 2 + kv-cache-block-size: 64 + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + data-parallel-size: 16 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: deep_gemm_mega_moe + all2all-backend: deepep_v2 + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + dyn-tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + dyn-reasoning-parser: kimi_k3 + language-model-only: true + block-size: 64 + max-num-seqs: 32 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" + AGENTIC_WARMUP_GRACE_PERIOD: "3600" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml new file mode 100644 index 0000000000..6641c4d1b0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml @@ -0,0 +1,103 @@ +name: "kimik3-vllm-agg-b200-tep16-balanced-dspark-agentic" + +# Official balanced multi_node_tep profile for two 8xB200 nodes. TP16 shards +# attention and dense layers; EP16 shards the routed experts. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tep +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + enable-expert-parallel: true + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + all2all-backend: deepep_v2 + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 32 + max-num-batched-tokens: 8192 + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml new file mode 100644 index 0000000000..6f7f2e8580 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -0,0 +1,105 @@ +name: "kimik3-vllm-agg-b200-tp16-latency-dspark-agentic" + +# Official latency-oriented multi_node_tp profile for two 8xB200 nodes. +# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_ALLREDUCE_USE_FLASHINFER: "1" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can + # OOM at the official 0.95 setting. Preserve the observed-safe headroom. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Performance runs use the committed K7 golden synthetic AL. The official + # recipe's real-verification form uses rejection_sample_method=block. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml deleted file mode 100644 index 49f1f0b0dd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-dspark-agentic" - -# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). -# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not -# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the -# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is -# deliberately off, so the 896 routed experts are TP-sharded inside each -# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. -# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. -# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in -# the kimi-k3 bring-up image. -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - -# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the -# multinode extension on functionstackx/srt-slurm-nv branch -# klaud/direct-vllm-multinode): `vllm serve` owns the OpenAI port itself, so -# no Dynamo frontend/worker is involved and no dynamo install is needed. -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 1440 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns - # the public port; rank-1 runs a headless engine worker (vLLM-native - # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, carried by the - # functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch). - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_SERVER_DEV_MODE: "1" - # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window - # generous, and let one long AgentX request hold a PP stage beyond vLLM's - # 300-second model-execution default. - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - # Prefix-cache retention (variant G): 0, on the otherwise-unchanged - # GPU-resident variant D config. Any positive value must be a multiple of - # Kimi-K3's KDA-hybrid scheduler_block_size (3145728; the GB recipes' - # 32768 is hard-rejected at engine init — verified on this PR family), so - # 0 is the only value below one 3.1M-token scheduler block. - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value - # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a - # scheduler_block_size of 3145728 and the interval must be a multiple of - # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative - # and a multiple of scheduler_block_size (3145728)"). Default retention - # served fine in earlier runs. - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a - # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first - # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first - # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 - # matches the GB200/GB300 agentic recipes. - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — - # legitimate here because this recipe serves directly with vllm serve - # (frontend.type: vllm), not through the dynamo worker entrypoint that - # rejects them. - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - # DSpark level 7 with probabilistic drafting and the committed golden - # synthetic acceptance length measured for Kimi-K3 at this draft length. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - # No explicit max-model-len: let vLLM derive the native 1M window from - # the model config (agentic trajectories blow past any small cap, and - # K3's KDA layers keep per-token KV small — only the 24 gated-MLA - # layers hold cache). Prefix caching stays on (default) for trajectory - # reuse. Cap prefill chunks so a single long request cannot OOM a - # pipeline stage; let vLLM pick max-num-seqs. - max-num-batched-tokens: 8192 - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep the aggregate worker in the multinode result schema so ingestion - # uses the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - # aiperf's conv-aware routing emits nvext.session_control, a removed POC - # field this dynamo build 400-rejects at warmup (schema moved to - # router/routing_constraints/agent_hints). Same opt-out as the GB300 - # aggregate AgentX recipes — and with a single aggregate worker there is - # no P/D routing to bind anyway. - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 4fdafcea1f..fa846bde98 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9253,9 +9253,12 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false -# Kimi-K3 MXFP4 B200 DSpark variant of the aggregated TP8 x PP2 topology. -# The server recipe applies the DSpark draft configuration; `mtp` selects the -# repository's speculative-decoding benchmark path and result identity. +# Kimi K3 B200 AgentX DSpark profiles derived from the official vLLM B200 +# strategies. TP16 and TEP16 use direct multi-node vLLM through srt-slurm; +# DEP16 uses srt-slurm's Dynamo aggregate path so node-local DP start ranks are +# derived from the allocation. All four recipes use the committed K7 synthetic +# acceptance length 3.84 for comparable performance runs. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200 kimik3-fp4-b200-dynamo-vllm-agentic-dspark: image: vllm/vllm-openai:kimi-k3 model: moonshotai/Kimi-K3 @@ -9267,23 +9270,70 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: disagg: false scenarios: agentic-coding: - - search-space: + - dram-utilization: 0.61 + search-space: + # Low-latency and high-interactivity profile. - spec-decoding: mtp - conc-list: [1, 2, 4, 8, 16, 32] + conc-list: [1, 2, 4, 8] prefill: num-worker: 1 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" decode: num-worker: 0 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false + # Balanced medium-concurrency TEP16 profile. + - spec-decoding: mtp + conc-list: [8, 16, 24, 32] + prefill: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 16 + ep: 16 + dp-attn: false + # High-concurrency GPU-resident DEP16 profile. + - spec-decoding: mtp + conc-list: [32, 64, 96, 128, 192, 256] + prefill: + num-worker: 1 + tp: 1 + ep: 16 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 1 + ep: 16 + dp-attn: true + # Maximum-concurrency DEP16 profile with 220 GiB of CPU KV per rank. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple, version: "13c59a3" } + conc-list: [128, 192, 256, 384] + prefill: + num-worker: 1 + tp: 1 + ep: 16 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 1 + ep: 16 + dp-attn: true qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9383252290..cd348e0b9e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5377,5 +5377,7 @@ - config-keys: - kimik3-fp4-b200-dynamo-vllm-agentic-dspark description: - - "Initial B200 Kimi K3 DSpark submission with corrected AgentX harness" + - "Add official Kimi K3 B200 AgentX profiles for TP16 latency, TEP16 balanced throughput, and DEP16 high concurrency through checked-in srt-slurm recipes" + - "Add a DEP16 SimpleCPUOffloadConnector profile with the official 220 GiB host-KV pool per rank" + - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 16d3338d09..bf759a010a 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -107,21 +107,19 @@ if [[ "$IS_MULTINODE" == "true" ]]; then rm -rf "$SRT_REPO_DIR" fi - # TODO(CJQ): make first class upon srt-slurm upstream refactor + # Kimi K3 aggregate profiles use the srt-slurm fork that supports direct + # multi-node vLLM for TP/TEP and topology-derived per-node DP ranks for + # Dynamo DEP. Pin the tested renderer so branch movement cannot change + # generated rank commands between sweep points. if [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then - # Direct-vLLM agentic experiment (Variant D): srt-slurm PR #278 - # (kylliang/direct-aggregate-vllm) adds frontend.type: vllm — `vllm - # serve` owns the OpenAI port itself, no Dynamo layer. The fork branch - # carries PR #278 plus the multi-node extension (vLLM-native - # --master-addr/--nnodes/--node-rank serve + headless non-leader - # ranks) so the 2-node TP8xPP2 topology can run. git clone --branch klaud/direct-vllm-multinode --single-branch https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 - if [[ $MODEL_PREFIX == "kimik3" ]]; then - mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic || exit 1 - fi + git checkout df5baa93f4caf5169dea2a4236ad2cc742fe40e7 || exit 1 + mkdir -p recipes/vllm/kimi-k3/agentic configs || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic || exit 1 + cp "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh" \ + configs/kimik3-dspark-config-compat.sh || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 From c5fa8dac0ef9785fc1c07968e2bb330251da7414 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:19:33 -0500 Subject: [PATCH 04/10] fix: bypass unsupported b200 custom collectives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：绕过 B200 上不受支持的自定义集合通信，避免 TP16 在权重加载前停滞。 --- .../kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 6f7f2e8580..e0ffba22cc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -69,6 +69,9 @@ backend: # OOM at the official 0.95 setting. Preserve the observed-safe headroom. gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true + # This B200 pool has no cross-node MNNVL multicast. Explicitly bypass + # custom collective initialization, which otherwise stalls before load. + disable-custom-all-reduce: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' From 88d3bb7262403137a5915e3bc11403f47accdc29 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:34:39 -0500 Subject: [PATCH 05/10] fix: correct b200 dep launch accounting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：修正 B200 DEP 启动参数、KV 事件发布、GPU 计数与 CPU DRAM 预算，并禁用跨节点不受支持的 FlashInfer 集合通信。 --- ...-b200-dep16-throughput-dspark-agentic.yaml | 3 +- ...ut-vllm-simple-offload-dspark-agentic.yaml | 3 +- ...gg-b200-tep16-balanced-dspark-agentic.yaml | 3 +- .../agg-b200-tp16-latency-dspark-agentic.yaml | 6 +-- .../aggregation/process_agentic_result.py | 37 ++++++++++++-- .../test_process_agentic_result.py | 29 +++++++++++ utils/matrix_logic/generate_sweep_configs.py | 39 ++++++++------- .../test_generate_sweep_configs.py | 50 +++++++++++++++++++ 8 files changed, 141 insertions(+), 29 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml index e14c238acf..a549b7d04e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml @@ -59,6 +59,8 @@ backend: type: vllm connector: null dp_launch_mode: per_node + kv_events_config: + aggregated: true aggregated_environment: VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" @@ -89,7 +91,6 @@ backend: kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' enable-prefix-caching: true - enable-auto-tool-choice: true dyn-tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 dyn-reasoning-parser: kimi_k3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml index c1d8a6a651..c1a0899688 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml @@ -59,6 +59,8 @@ backend: type: vllm connector: null dp_launch_mode: per_node + kv_events_config: + aggregated: true aggregated_environment: VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" VLLM_ALLREDUCE_USE_FLASHINFER: "1" @@ -89,7 +91,6 @@ backend: kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' enable-prefix-caching: true - enable-auto-tool-choice: true dyn-tool-call-parser: kimi_k3 reasoning-parser: kimi_k3 dyn-reasoning-parser: kimi_k3 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml index 6641c4d1b0..557ca3a9ce 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml @@ -44,7 +44,8 @@ backend: connector: null aggregated_environment: VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" + # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. + VLLM_ALLREDUCE_USE_FLASHINFER: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index e0ffba22cc..57f29a1d88 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -44,7 +44,8 @@ backend: connector: null aggregated_environment: VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" + # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. + VLLM_ALLREDUCE_USE_FLASHINFER: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" VLLM_USE_V2_MODEL_RUNNER: "1" @@ -69,9 +70,6 @@ backend: # OOM at the official 0.95 setting. Preserve the observed-safe headroom. gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true - # This B200 pool has no cross-node MNNVL multicast. Explicitly bypass - # custom collective initialization, which otherwise stalls before load. - disable-custom-all-reduce: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' diff --git a/utils/agentic/aggregation/process_agentic_result.py b/utils/agentic/aggregation/process_agentic_result.py index 5d13fa0f1b..4971a132ba 100644 --- a/utils/agentic/aggregation/process_agentic_result.py +++ b/utils/agentic/aggregation/process_agentic_result.py @@ -95,6 +95,18 @@ def _validate_kv_offload_env() -> tuple[str, dict[str, str] | None]: return kv_offloading, backend_metadata +def _parallel_gpu_count( + tp: int, + pp: int, + pcp_size: int, + ep: int, + dp_attention: bool, +) -> int: + """Return the physical GPU count for one worker replica.""" + tensor_gpu_count = tp * pp * pcp_size + return max(tensor_gpu_count, ep) if dp_attention else tensor_gpu_count + + def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: is_multinode = env_bool("IS_MULTINODE") tp = env_int("TP", 1) @@ -111,7 +123,10 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: "PP_SIZE, DCP_SIZE, and PCP_SIZE must be positive integers." ) fields.update({"pp": pp, "dcp_size": dcp_size, "pcp_size": pcp_size}) - return fields, tp * pp * pcp_size, tp, ep, dp_attention + num_gpus = _parallel_gpu_count( + tp, pp, pcp_size, ep, env_bool("DP_ATTENTION") + ) + return fields, num_gpus, tp, ep, dp_attention prefill_num_workers = env_int("PREFILL_NUM_WORKERS") prefill_tp = env_int("PREFILL_TP") @@ -120,6 +135,7 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: prefill_pcp_size = env_int("PREFILL_PCP_SIZE", 1) prefill_ep = env_int("PREFILL_EP", 1) prefill_dp_attention = os.environ.get("PREFILL_DP_ATTN", "false") + prefill_dp_attention_enabled = env_bool("PREFILL_DP_ATTN") decode_num_workers = env_int("DECODE_NUM_WORKERS") decode_tp = env_int("DECODE_TP") decode_pp = env_int("DECODE_PP_SIZE", 1) @@ -127,6 +143,7 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: decode_pcp_size = env_int("DECODE_PCP_SIZE", 1) decode_ep = env_int("DECODE_EP", 1) decode_dp_attention = os.environ.get("DECODE_DP_ATTN", "false") + decode_dp_attention_enabled = env_bool("DECODE_DP_ATTN") worker_parallelism = ( prefill_pp, prefill_dcp_size, @@ -145,8 +162,20 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: raise SystemExit( "PREFILL_HARDWARE and DECODE_HARDWARE must be specified together." ) - num_prefill_gpu = prefill_num_workers * prefill_tp * prefill_pp * prefill_pcp_size - num_decode_gpu = decode_num_workers * decode_tp * decode_pp * decode_pcp_size + num_prefill_gpu = prefill_num_workers * _parallel_gpu_count( + prefill_tp, + prefill_pp, + prefill_pcp_size, + prefill_ep, + prefill_dp_attention_enabled, + ) + num_decode_gpu = decode_num_workers * _parallel_gpu_count( + decode_tp, + decode_pp, + decode_pcp_size, + decode_ep, + decode_dp_attention_enabled, + ) num_gpus = num_prefill_gpu + num_decode_gpu # Aggregated configs set decode num-worker 0 (prefill+decode co-located on one # worker), so there are no separate decode GPUs. Mirror process_result.py and drop @@ -162,7 +191,7 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: ep = max(prefill_ep, decode_ep) dp_attention = ( "true" - if env_bool("PREFILL_DP_ATTN") or env_bool("DECODE_DP_ATTN") + if prefill_dp_attention_enabled or decode_dp_attention_enabled else "false" ) fields.update( diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index 21a7307630..8bb4d36b06 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -564,6 +564,35 @@ def test_multinode_processor_surfaces_heterogeneous_hardware(tmp_path: Path): agg["num_decode_gpu"], ) == (2, 4, 1, 32) +def test_multinode_processor_counts_attention_dp_ranks( + monkeypatch: pytest.MonkeyPatch, +): + env = { + "IS_MULTINODE": "true", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "1", + "PREFILL_PP_SIZE": "1", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "16", + "PREFILL_DP_ATTN": "true", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "1", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "1", + "DECODE_DP_ATTN": "false", + } + for name, value in env.items(): + monkeypatch.setenv(name, value) + + fields, num_gpus, tp, ep, dp_attention = _gpu_shape() + + assert fields["num_prefill_gpu"] == 16 + assert fields["num_decode_gpu"] == 0 + assert (num_gpus, tp, ep, dp_attention) == (16, 1, 16, "true") + def test_multinode_processor_omits_homogeneous_hardware(tmp_path: Path): result_dir = _write_fixture(tmp_path) diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 86f192b1c8..6ccd84b604 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -82,13 +82,21 @@ def runner_gpus_per_node(runner: str, runner_data: dict) -> int: return runner_hardware_int(runner, runner_data, Fields.GPUS_PER_NODE.value) -def effective_gpu_count(benchmark: dict) -> int: - """Return GPUs used by a single-node TP/PP/PCP topology.""" - return ( - benchmark[Fields.TP.value] - * benchmark.get(Fields.PP.value, 1) - * benchmark.get(Fields.PCP_SIZE.value, 1) +def _parallel_gpu_count(config: dict) -> int: + """Return the GPU footprint encoded by one parallelism configuration.""" + tensor_gpu_count = ( + config[Fields.TP.value] + * config.get(Fields.PP.value, 1) + * config.get(Fields.PCP_SIZE.value, 1) ) + if config.get(Fields.DP_ATTN.value, False): + return max(tensor_gpu_count, config.get(Fields.EP.value, 1)) + return tensor_gpu_count + + +def effective_gpu_count(benchmark: dict) -> int: + """Return GPUs used by a single-node parallel topology.""" + return _parallel_gpu_count(benchmark) def with_worker_parallelism_defaults(worker: dict) -> dict: """Return a worker config with explicit parallelism defaults.""" @@ -109,29 +117,24 @@ def worker_gpus_per_node(worker: dict, gpus_per_node: int) -> int: are rejected rather than silently truncated, keeping parity with the single-node "must fit the node" rule: - * A replica larger than one node (tp*pp*pcp > gpus-per-node) must fill whole - nodes, i.e. be an exact multiple of gpus-per-node; each of its nodes is - then fully occupied (fraction 1). + * A replica larger than one node must fill whole nodes, i.e. be an exact + multiple of gpus-per-node; each of its nodes is then fully occupied. * A replica within one node must divide it evenly so co-located replicas of the same role tile the node without overlap. + * Attention-DP topologies include the EP-sized independent rank pool. """ - gpus_per_replica = ( - worker[Fields.TP.value] - * worker.get(Fields.PP.value, 1) - * worker.get(Fields.PCP_SIZE.value, 1) - ) + gpus_per_replica = _parallel_gpu_count(worker) if gpus_per_replica > gpus_per_node: if gpus_per_replica % gpus_per_node != 0: raise ValueError( - f"worker {Fields.TP.value}*{Fields.PP.value}*{Fields.PCP_SIZE.value}" - f"={gpus_per_replica} spans multiple nodes but is not a multiple " + f"worker GPU footprint={gpus_per_replica} spans multiple nodes " + "but is not a multiple " f"of {Fields.GPUS_PER_NODE.value}={gpus_per_node}" ) return gpus_per_node if gpus_per_node % gpus_per_replica != 0: raise ValueError( - f"worker {Fields.TP.value}*{Fields.PP.value}*{Fields.PCP_SIZE.value}" - f"={gpus_per_replica} does not divide " + f"worker GPU footprint={gpus_per_replica} does not divide " f"{Fields.GPUS_PER_NODE.value}={gpus_per_node} evenly" ) return gpus_per_replica diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 7cbfea79af..47974eb268 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2274,6 +2274,56 @@ def test_multinode_agentic_preserves_kv_offload_fields(self, sample_runner_confi # DRAM: 2861022 MiB * 0.80. assert result[0]["total-cpu-dram-gb"] == 2399 + def test_multinode_agentic_dep_budget_counts_attention_dp_ranks( + self, sample_runner_config + ): + config = { + "kimik3-dep": { + "image": "vllm/vllm-openai:kimi-k3", + "model": "moonshotai/Kimi-K3", + "model-prefix": "kimik3", + "precision": "fp4", + "framework": "dynamo-vllm", + "runner": "cluster:b200-dgxc", + "multinode": True, + "scenarios": { + "agentic-coding": [{ + "dram-utilization": 0.61, + "search-space": [{ + "conc-list": [128], + "kv-offloading": "dram", + "kv-offload-backend": {"name": "vllm-simple"}, + "prefill": { + "num-worker": 1, + "tp": 1, + "ep": 16, + "dp-attn": True, + }, + "decode": { + "num-worker": 0, + "tp": 1, + "ep": 1, + "dp-attn": False, + }, + }], + }], + }, + }, + } + args = argparse.Namespace( + config_keys=["kimik3-dep"], + seq_lens=None, + conc=None, + scenario_type=["agentic-coding"], + runner_node_filter=None, + ) + + result = generate_test_config_sweep(args, config, sample_runner_config) + + assert len(result) == 1 + # DEP16 spans both nodes, so all eight ranks on each node share DRAM. + assert result[0]["total-cpu-dram-gb"] == 1829 + def test_multinode_agentic_budget_ignores_decode_topology( self, sample_runner_config ): From 3cb4971232d0b16bde3d3b511b652c4bc97249cd Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:54:01 -0500 Subject: [PATCH 06/10] fix: force pynccl for cross-node tp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：为跨节点 TP16 和 TEP16 强制使用 PyNCCL，跳过会在 B200 集群上停滞的 MNNVL 自定义集合通信初始化。 --- .../agentic/agg-b200-tep16-balanced-dspark-agentic.yaml | 3 +++ .../kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml index 557ca3a9ce..ec7516160a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml @@ -70,6 +70,9 @@ backend: all2all-backend: deepep_v2 gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true + # The generic custom communicator also probes MNNVL and stalls before + # model load on this pool. Use PyNCCL for the cross-node TP group. + disable-custom-all-reduce: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 57f29a1d88..81890ce47b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -70,6 +70,9 @@ backend: # OOM at the official 0.95 setting. Preserve the observed-safe headroom. gpu-memory-utilization: 0.90 no-enable-flashinfer-autotune: true + # The generic custom communicator also probes MNNVL and stalls before + # model load on this pool. Use PyNCCL for the cross-node TP group. + disable-custom-all-reduce: true max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' From 94d677aeb370a4b5033878cb1357f1849ff86ab6 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:03:02 -0500 Subject: [PATCH 07/10] fix: disable cross-node latent tail fusion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：为跨节点 TP16 和 TEP16 禁用 Kimi K3 LatentMoE tail fusion，避免 torch symmetric memory 无法跨节点传递文件描述符。 --- .../agentic/agg-b200-tep16-balanced-dspark-agentic.yaml | 4 +++- .../kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml index ec7516160a..15ef2a7386 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml @@ -43,7 +43,9 @@ backend: type: vllm connector: null aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + # K3's fused tail uses torch symmetric memory, which cannot exchange file + # descriptors across this two-node TP group. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. VLLM_ALLREDUCE_USE_FLASHINFER: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 81890ce47b..42a9b25371 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -43,7 +43,9 @@ backend: type: vllm connector: null aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + # K3's fused tail uses torch symmetric memory, which cannot exchange file + # descriptors across this two-node TP group. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. VLLM_ALLREDUCE_USE_FLASHINFER: "0" VLLM_ENGINE_READY_TIMEOUT_S: "3600" From 349ddfdfa0e3f307cb2eb3503fadfde24b52b479 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:45:10 -0500 Subject: [PATCH 08/10] fix: disable cross-node allreduce fusion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：为跨节点 TP16 和 TEP16 禁用 allreduce/RMS fusion，避免 profile_run 再次选择不受支持的 FlashInfer MNNVL 工作区。 --- .../agentic/agg-b200-tep16-balanced-dspark-agentic.yaml | 3 +++ .../kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml index 15ef2a7386..b7b0616811 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml @@ -75,6 +75,9 @@ backend: # The generic custom communicator also probes MNNVL and stalls before # model load on this pool. Use PyNCCL for the cross-node TP group. disable-custom-all-reduce: true + # The default allreduce/RMS fusion independently allocates a FlashInfer + # MNNVL workspace during profile_run, bypassing the communicator flags. + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml index 42a9b25371..7163e44a8e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -75,6 +75,9 @@ backend: # The generic custom communicator also probes MNNVL and stalls before # model load on this pool. Use PyNCCL for the cross-node TP group. disable-custom-all-reduce: true + # The default allreduce/RMS fusion independently allocates a FlashInfer + # MNNVL workspace during profile_run, bypassing the communicator flags. + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' max-model-len: 1048576 kv-cache-dtype: fp8 attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' From 48dffa17d08d9279877a4af1d30ad3b408c04a21 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:43:23 -0500 Subject: [PATCH 09/10] fix: use viable b200 tp8pp2 profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：改用可运行的 B200 TP8 x PP2 配置，并移除无法启动的跨节点 TP16、TEP16 和 DEP16 配置。 --- ...-b200-dep16-throughput-dspark-agentic.yaml | 125 --------------- ...ut-vllm-simple-offload-dspark-agentic.yaml | 126 --------------- ...gg-b200-tep16-balanced-dspark-agentic.yaml | 112 ------------- .../agg-b200-tp16-latency-dspark-agentic.yaml | 114 -------------- .../agg-b200-tp8pp2-dspark-agentic.yaml | 143 +++++++++++++++++ ...p2-vllm-simple-offload-dspark-agentic.yaml | 147 ++++++++++++++++++ configs/nvidia-master.yaml | 79 +++++----- perf-changelog.yaml | 4 +- 8 files changed, 335 insertions(+), 515 deletions(-) delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml deleted file mode 100644 index a549b7d04e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml +++ /dev/null @@ -1,125 +0,0 @@ -name: "kimik3-vllm-agg-b200-dep16-throughput-dspark-agentic" - -# Official throughput-oriented multi_node_dep profile for two 8xB200 nodes. -# One Dynamo vLLM process per node owns eight local DP ranks; srt-slurm derives -# start ranks 0 and 8 from the allocated topology and EP spans all 16 GPUs. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "dynamo" - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 2 - kv-cache-block-size: 64 - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: - aggregated: true - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - data-parallel-size: 16 - enable-expert-parallel: true - trust-remote-code: true - load-format: fastsafetensors - moe-backend: deep_gemm_mega_moe - all2all-backend: deepep_v2 - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' - enable-prefix-caching: true - dyn-tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - dyn-reasoning-parser: kimi_k3 - language-model-only: true - block-size: 64 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AGENTIC_WARMUP_GRACE_PERIOD: "3600" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml deleted file mode 100644 index c1a0899688..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml +++ /dev/null @@ -1,126 +0,0 @@ -name: "kimik3-vllm-agg-b200-dep16-simple-offload-dspark-agentic" - -# Official multi_node_dep profile with SimpleCPUOffloadConnector. Each of the -# 16 DP ranks reserves the official 220 GiB host-KV pool. Eight ranks consume -# 1.72 TiB on each B200 node, below the runner's configured host-DRAM budget. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_dep -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - frameworks: - dynamo: "1.3.0" - -dynamo: - hash: "ba83080ecd31c1ce918559e576d3c5bc9e092ff1" - install: true - -setup_script: kimik3-dspark-config-compat.sh - -environment: - ETCD_LEASE_TTL: "7200" - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - dyn-chat-processor: "dynamo" - router-mode: "kv" - router-kv-events: true - router-temperature: "0" - router-min-initial-workers: 2 - kv-cache-block-size: 64 - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - kv_events_config: - aggregated: true - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - data-parallel-size: 16 - enable-expert-parallel: true - trust-remote-code: true - load-format: fastsafetensors - moe-backend: deep_gemm_mega_moe - all2all-backend: deepep_v2 - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - max-model-len: 1048576 - kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' - enable-prefix-caching: true - dyn-tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - dyn-reasoning-parser: kimi_k3 - language-model-only: true - block-size: 64 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AGENTIC_WARMUP_GRACE_PERIOD: "3600" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml deleted file mode 100644 index b7b0616811..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml +++ /dev/null @@ -1,112 +0,0 @@ -name: "kimik3-vllm-agg-b200-tep16-balanced-dspark-agentic" - -# Official balanced multi_node_tep profile for two 8xB200 nodes. TP16 shards -# attention and dense layers; EP16 shards the routed experts. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tep -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - # K3's fused tail uses torch symmetric memory, which cannot exchange file - # descriptors across this two-node TP group. - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" - # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. - VLLM_ALLREDUCE_USE_FLASHINFER: "0" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - enable-expert-parallel: true - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - all2all-backend: deepep_v2 - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - # The generic custom communicator also probes MNNVL and stalls before - # model load on this pool. Use PyNCCL for the cross-node TP group. - disable-custom-all-reduce: true - # The default allreduce/RMS fusion independently allocates a FlashInfer - # MNNVL workspace during profile_run, bypassing the communicator flags. - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' - max-model-len: 1048576 - kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' - enable-prefix-caching: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 32 - max-num-batched-tokens: 8192 - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml deleted file mode 100644 index 7163e44a8e..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml +++ /dev/null @@ -1,114 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp16-latency-dspark-agentic" - -# Official latency-oriented multi_node_tp profile for two 8xB200 nodes. -# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 720 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - # K3's fused tail uses torch symmetric memory, which cannot exchange file - # descriptors across this two-node TP group. - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" - # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. - VLLM_ALLREDUCE_USE_FLASHINFER: "0" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_LOG_STATS_INTERVAL: "1" - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 16 - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can - # OOM at the official 0.95 setting. Preserve the observed-safe headroom. - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - # The generic custom communicator also probes MNNVL and stalls before - # model load on this pool. Use PyNCCL for the cross-node TP group. - disable-custom-all-reduce: true - # The default allreduce/RMS fusion independently allocates a FlashInfer - # MNNVL workspace during profile_run, bypassing the communicator flags. - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' - max-model-len: 1048576 - kv-cache-dtype: fp8 - attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' - enable-prefix-caching: true - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - language-model-only: true - max-num-seqs: 8 - max-num-batched-tokens: 8192 - # Performance runs use the committed K7 golden synthetic AL. The official - # recipe's real-verification form uses rejection_sample_method=block. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" - AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml new file mode 100644 index 0000000000..49f1f0b0dd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml @@ -0,0 +1,143 @@ +name: "kimik3-vllm-agg-b200-tp8pp2-dspark-agentic" + +# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). +# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not +# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the +# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is +# deliberately off, so the 896 routed experts are TP-sharded inside each +# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. +# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. +# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in +# the kimi-k3 bring-up image. +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the +# multinode extension on functionstackx/srt-slurm-nv branch +# klaud/direct-vllm-multinode): `vllm serve` owns the OpenAI port itself, so +# no Dynamo frontend/worker is involved and no dynamo install is needed. +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns + # the public port; rank-1 runs a headless engine worker (vLLM-native + # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, carried by the + # functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch). + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_SERVER_DEV_MODE: "1" + # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window + # generous, and let one long AgentX request hold a PP stage beyond vLLM's + # 300-second model-execution default. + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + # Prefix-cache retention (variant G): 0, on the otherwise-unchanged + # GPU-resident variant D config. Any positive value must be a multiple of + # Kimi-K3's KDA-hybrid scheduler_block_size (3145728; the GB recipes' + # 32768 is hard-rejected at engine init — verified on this PR family), so + # 0 is the only value below one 3.1M-token scheduler block. + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value + # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a + # scheduler_block_size of 3145728 and the interval must be a multiple of + # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative + # and a multiple of scheduler_block_size (3145728)"). Default retention + # served fine in earlier runs. + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a + # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first + # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first + # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 + # matches the GB200/GB300 agentic recipes. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — + # legitimate here because this recipe serves directly with vllm serve + # (frontend.type: vllm), not through the dynamo worker entrypoint that + # rejects them. + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + # DSpark level 7 with probabilistic drafting and the committed golden + # synthetic acceptance length measured for Kimi-K3 at this draft length. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + # No explicit max-model-len: let vLLM derive the native 1M window from + # the model config (agentic trajectories blow past any small cap, and + # K3's KDA layers keep per-token KV small — only the 24 gated-MLA + # layers hold cache). Prefix caching stays on (default) for trajectory + # reuse. Cap prefill chunks so a single long request cannot OOM a + # pipeline stage; let vLLM pick max-num-seqs. + max-num-batched-tokens: 8192 + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep the aggregate worker in the multinode result schema so ingestion + # uses the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + # aiperf's conv-aware routing emits nvext.session_control, a removed POC + # field this dynamo build 400-rejects at warmup (schema moved to + # router/routing_constraints/agent_hints). Same opt-out as the GB300 + # aggregate AgentX recipes — and with a single aggregate worker there is + # no P/D routing to bind anyway. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml new file mode 100644 index 0000000000..f179c13146 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml @@ -0,0 +1,147 @@ +name: "kimik3-vllm-agg-b200-tp8pp2-simple-offload-dspark-agentic" + +# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). +# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not +# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the +# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is +# deliberately off, so the 896 routed experts are TP-sharded inside each +# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. +# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. +# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in +# the kimi-k3 bring-up image. +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the +# multinode extension on functionstackx/srt-slurm-nv branch +# klaud/direct-vllm-multinode): `vllm serve` owns the OpenAI port itself, so +# no Dynamo frontend/worker is involved and no dynamo install is needed. +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns + # the public port; rank-1 runs a headless engine worker (vLLM-native + # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, carried by the + # functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch). + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" + VLLM_SERVER_DEV_MODE: "1" + # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window + # generous, and let one long AgentX request hold a PP stage beyond vLLM's + # 300-second model-execution default. + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + # Prefix-cache retention (variant G): 0, on the otherwise-unchanged + # GPU-resident variant D config. Any positive value must be a multiple of + # Kimi-K3's KDA-hybrid scheduler_block_size (3145728; the GB recipes' + # 32768 is hard-rejected at engine init — verified on this PR family), so + # 0 is the only value below one 3.1M-token scheduler block. + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" + # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value + # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a + # scheduler_block_size of 3145728 and the interval must be a multiple of + # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative + # and a multiple of scheduler_block_size (3145728)"). Default retention + # served fine in earlier runs. + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a + # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first + # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first + # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 + # matches the GB200/GB300 agentic recipes. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — + # legitimate here because this recipe serves directly with vllm serve + # (frontend.type: vllm), not through the dynamo worker entrypoint that + # rejects them. + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + # DSpark level 7 with probabilistic drafting and the committed golden + # synthetic acceptance length measured for Kimi-K3 at this draft length. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + # 220 GiB of host DRAM per TP rank extends the KV working set beyond + # the observed GPU-resident c16-c32 cliff. Eager offload avoids a first-hit + # allocation spike during long AgentX trajectories. + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + # No explicit max-model-len: let vLLM derive the native 1M window from + # the model config (agentic trajectories blow past any small cap, and + # K3's KDA layers keep per-token KV small — only the 24 gated-MLA + # layers hold cache). Prefix caching stays on (default) for trajectory + # reuse. Cap prefill chunks so a single long request cannot OOM a + # pipeline stage; let vLLM pick max-num-seqs. + max-num-batched-tokens: 8192 + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + # Keep the aggregate worker in the multinode result schema so ingestion + # uses the zero decode-worker count instead of duplicating TP into P and D. + IS_MULTINODE: "true" + # aiperf's conv-aware routing emits nvext.session_control, a removed POC + # field this dynamo build 400-rejects at warmup (schema moved to + # router/routing_constraints/agent_hints). Same opt-out as the GB300 + # aggregate AgentX recipes — and with a single aggregate worker there is + # no P/D routing to bind anyway. + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index fa846bde98..594b713e89 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9253,12 +9253,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false -# Kimi K3 B200 AgentX DSpark profiles derived from the official vLLM B200 -# strategies. TP16 and TEP16 use direct multi-node vLLM through srt-slurm; -# DEP16 uses srt-slurm's Dynamo aggregate path so node-local DP start ranks are -# derived from the allocation. All four recipes use the committed K7 synthetic -# acceptance length 3.84 for comparable performance runs. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200 +# Kimi K3's 2.8T MXFP4 weights require TP8 x PP2 across all 16 B200s. +# Cross-node TP16/TEP16 is unsupported on this pool because it lacks MNNVL +# multicast, while DEP16 replicates the full model and OOMs before weight load. +# Split the proven TP8 x PP2 topology into GPU-resident latency, balanced, and +# throughput tiers, plus a CPU KV-offloaded capacity tier. kimik3-fp4-b200-dynamo-vllm-agentic-dspark: image: vllm/vllm-openai:kimi-k3 model: moonshotai/Kimi-K3 @@ -9272,68 +9271,76 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: agentic-coding: - dram-utilization: 0.61 search-space: - # Low-latency and high-interactivity profile. + # Low-latency and high-interactivity points. - spec-decoding: mtp - conc-list: [1, 2, 4, 8] + conc-list: [1, 2, 4] prefill: num-worker: 1 - tp: 16 + tp: 8 + pp: 2 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" decode: num-worker: 0 - tp: 16 + tp: 8 + pp: 2 ep: 1 dp-attn: false - # Balanced medium-concurrency TEP16 profile. + # Balanced medium-concurrency points. - spec-decoding: mtp - conc-list: [8, 16, 24, 32] + conc-list: [8, 12, 16] prefill: num-worker: 1 - tp: 16 - ep: 16 + tp: 8 + pp: 2 + ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tep16-balanced-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" decode: num-worker: 0 - tp: 16 - ep: 16 + tp: 8 + pp: 2 + ep: 1 dp-attn: false - # High-concurrency GPU-resident DEP16 profile. + # GPU-resident throughput points around the observed c16-c32 KV cliff. - spec-decoding: mtp - conc-list: [32, 64, 96, 128, 192, 256] + conc-list: [20, 24, 28, 32] prefill: num-worker: 1 - tp: 1 - ep: 16 - dp-attn: true + tp: 8 + pp: 2 + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" decode: num-worker: 0 - tp: 1 - ep: 16 - dp-attn: true - # Maximum-concurrency DEP16 profile with 220 GiB of CPU KV per rank. + tp: 8 + pp: 2 + ep: 1 + dp-attn: false + # CPU KV-offloaded capacity points beyond the GPU-resident cliff. - spec-decoding: mtp kv-offloading: dram kv-offload-backend: { name: vllm-simple, version: "13c59a3" } - conc-list: [128, 192, 256, 384] + conc-list: [32, 48, 64] prefill: num-worker: 1 - tp: 1 - ep: 16 - dp-attn: true + tp: 8 + pp: 2 + ep: 1 + dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-dep16-throughput-vllm-simple-offload-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml" decode: num-worker: 0 - tp: 1 - ep: 16 - dp-attn: true + tp: 8 + pp: 2 + ep: 1 + dp-attn: false qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cd348e0b9e..ea815e0a30 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5377,7 +5377,7 @@ - config-keys: - kimik3-fp4-b200-dynamo-vllm-agentic-dspark description: - - "Add official Kimi K3 B200 AgentX profiles for TP16 latency, TEP16 balanced throughput, and DEP16 high concurrency through checked-in srt-slurm recipes" - - "Add a DEP16 SimpleCPUOffloadConnector profile with the official 220 GiB host-KV pool per rank" + - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on the proven TP8 x PP2 topology" + - "Add a TP8 x PP2 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 From 4508808a9776006e137865b753a64f959bad6be2 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:59:56 -0500 Subject: [PATCH 10/10] fix: keep dspark on pp-free tp16 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：DSpark 草稿模型不支持流水线并行，因此改回无 PP 的 TP16 配置，并保留 GPU 与 CPU KV 容量档位。 --- .../agg-b200-tp16-latency-dspark-agentic.yaml | 114 ++++++++++++++ ...16-vllm-simple-offload-dspark-agentic.yaml | 115 ++++++++++++++ .../agg-b200-tp8pp2-dspark-agentic.yaml | 143 ----------------- ...p2-vllm-simple-offload-dspark-agentic.yaml | 147 ------------------ configs/nvidia-master.yaml | 44 +++--- perf-changelog.yaml | 4 +- 6 files changed, 249 insertions(+), 318 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml delete mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml new file mode 100644 index 0000000000..7163e44a8e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -0,0 +1,114 @@ +name: "kimik3-vllm-agg-b200-tp16-latency-dspark-agentic" + +# Official latency-oriented multi_node_tp profile for two 8xB200 nodes. +# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + # K3's fused tail uses torch symmetric memory, which cannot exchange file + # descriptors across this two-node TP group. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" + # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. + VLLM_ALLREDUCE_USE_FLASHINFER: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can + # OOM at the official 0.95 setting. Preserve the observed-safe headroom. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # The generic custom communicator also probes MNNVL and stalls before + # model load on this pool. Use PyNCCL for the cross-node TP group. + disable-custom-all-reduce: true + # The default allreduce/RMS fusion independently allocates a FlashInfer + # MNNVL workspace during profile_run, bypassing the communicator flags. + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Performance runs use the committed K7 golden synthetic AL. The official + # recipe's real-verification form uses rejection_sample_method=block. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml new file mode 100644 index 0000000000..391aef0a44 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml @@ -0,0 +1,115 @@ +name: "kimik3-vllm-agg-b200-tp16-simple-offload-dspark-agentic" + +# Official latency-oriented multi_node_tp profile for two 8xB200 nodes. +# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + # K3's fused tail uses torch symmetric memory, which cannot exchange file + # descriptors across this two-node TP group. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" + # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. + VLLM_ALLREDUCE_USE_FLASHINFER: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can + # OOM at the official 0.95 setting. Preserve the observed-safe headroom. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # The generic custom communicator also probes MNNVL and stalls before + # model load on this pool. Use PyNCCL for the cross-node TP group. + disable-custom-all-reduce: true + # The default allreduce/RMS fusion independently allocates a FlashInfer + # MNNVL workspace during profile_run, bypassing the communicator flags. + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Performance runs use the committed K7 golden synthetic AL. The official + # recipe's real-verification form uses rejection_sample_method=block. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml deleted file mode 100644 index 49f1f0b0dd..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml +++ /dev/null @@ -1,143 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-dspark-agentic" - -# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). -# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not -# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the -# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is -# deliberately off, so the 896 routed experts are TP-sharded inside each -# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. -# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. -# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in -# the kimi-k3 bring-up image. -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - -# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the -# multinode extension on functionstackx/srt-slurm-nv branch -# klaud/direct-vllm-multinode): `vllm serve` owns the OpenAI port itself, so -# no Dynamo frontend/worker is involved and no dynamo install is needed. -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 1440 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns - # the public port; rank-1 runs a headless engine worker (vLLM-native - # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, carried by the - # functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch). - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_SERVER_DEV_MODE: "1" - # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window - # generous, and let one long AgentX request hold a PP stage beyond vLLM's - # 300-second model-execution default. - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - # Prefix-cache retention (variant G): 0, on the otherwise-unchanged - # GPU-resident variant D config. Any positive value must be a multiple of - # Kimi-K3's KDA-hybrid scheduler_block_size (3145728; the GB recipes' - # 32768 is hard-rejected at engine init — verified on this PR family), so - # 0 is the only value below one 3.1M-token scheduler block. - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value - # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a - # scheduler_block_size of 3145728 and the interval must be a multiple of - # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative - # and a multiple of scheduler_block_size (3145728)"). Default retention - # served fine in earlier runs. - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a - # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first - # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first - # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 - # matches the GB200/GB300 agentic recipes. - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — - # legitimate here because this recipe serves directly with vllm serve - # (frontend.type: vllm), not through the dynamo worker entrypoint that - # rejects them. - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - # DSpark level 7 with probabilistic drafting and the committed golden - # synthetic acceptance length measured for Kimi-K3 at this draft length. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - # No explicit max-model-len: let vLLM derive the native 1M window from - # the model config (agentic trajectories blow past any small cap, and - # K3's KDA layers keep per-token KV small — only the 24 gated-MLA - # layers hold cache). Prefix caching stays on (default) for trajectory - # reuse. Cap prefill chunks so a single long request cannot OOM a - # pipeline stage; let vLLM pick max-num-seqs. - max-num-batched-tokens: 8192 - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep the aggregate worker in the multinode result schema so ingestion - # uses the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - # aiperf's conv-aware routing emits nvext.session_control, a removed POC - # field this dynamo build 400-rejects at warmup (schema moved to - # router/routing_constraints/agent_hints). Same opt-out as the GB300 - # aggregate AgentX recipes — and with a single aggregate worker there is - # no P/D routing to bind anyway. - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml deleted file mode 100644 index f179c13146..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml +++ /dev/null @@ -1,147 +0,0 @@ -name: "kimik3-vllm-agg-b200-tp8pp2-simple-offload-dspark-agentic" - -# Kimi-K3 MXFP4 B200 AGGREGATED TP8 x PP2 agentic recipe (2 nodes / 16 GPUs). -# The native MXFP4 checkpoint (2.8T total params, ~1.4TB of weights) does not -# fit one 8xB200 node, so TP8 shards attention/dense (/8) and PP2 splits the -# 93 layers (/2) across 16 GPUs. Plain TP (NOT TEP): expert parallelism is -# deliberately off, so the 896 routed experts are TP-sharded inside each -# pipeline stage. Node allocation = tp*pp/gpus_per_node = 8*2/8 = 2 nodes. -# Aggregated (single worker, decode num-worker 0) — no P/D split, no NIXL. -# VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION fuses the K3 LatentMoE tail path in -# the kimi-k3 bring-up image. -model: - path: "kimik3" - container: "vllm/vllm-openai:kimi-k3" - precision: "fp4" - -identity: - model: - repo: "moonshotai/Kimi-K3" - container: - image: "vllm/vllm-openai:kimi-k3" - -# Direct vLLM serving (frontend.type: vllm, srt-slurm PR #278 + the -# multinode extension on functionstackx/srt-slurm-nv branch -# klaud/direct-vllm-multinode): `vllm serve` owns the OpenAI port itself, so -# no Dynamo frontend/worker is involved and no dynamo install is needed. -dynamo: - install: false - -slurm: - time_limit: "8:00:00" - -health_check: - interval_seconds: 10 - max_attempts: 1440 - -resources: - gpu_type: "b200" - gpus_per_node: 8 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 16 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - # Direct vLLM OpenAI server (srt-slurm PR #278): the vllm serve leader owns - # the public port; rank-1 runs a headless engine worker (vLLM-native - # multi-node TP8xPP2 via --master-addr/--nnodes/--node-rank, carried by the - # functionstackx/srt-slurm-nv klaud/direct-vllm-multinode branch). - type: vllm - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - aggregated_environment: - VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "1" - VLLM_SERVER_DEV_MODE: "1" - # ~1.4TB of MXFP4 weights off shared Lustre: keep the engine-ready window - # generous, and let one long AgentX request hold a PP stage beyond vLLM's - # 300-second model-execution default. - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - # Prefix-cache retention (variant G): 0, on the otherwise-unchanged - # GPU-resident variant D config. Any positive value must be a multiple of - # Kimi-K3's KDA-hybrid scheduler_block_size (3145728; the GB recipes' - # 32768 is hard-rejected at engine init — verified on this PR family), so - # 0 is the only value below one 3.1M-token scheduler block. - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "0" - # No VLLM_PREFIX_CACHE_RETENTION_INTERVAL: the GB200/GB300 AgentX value - # (32768) hard-fails engine init on Kimi-K3 — the KDA hybrid gives it a - # scheduler_block_size of 3145728 and the interval must be a multiple of - # it ("VLLM_PREFIX_CACHE_RETENTION_INTERVAL (32768) must be non-negative - # and a multiple of scheduler_block_size (3145728)"). Default retention - # served fine in earlier runs. - NCCL_CUMEM_ENABLE: "1" - TILELANG_CLEANUP_TEMP_FILES: "1" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - vllm_config: - aggregated: - served-model-name: "moonshotai/Kimi-K3" - tensor-parallel-size: 8 - pipeline-parallel-size: 2 - trust-remote-code: true - load-format: fastsafetensors - moe-backend: auto - # 0.90, not 0.95: the flashinfer trtllm MXFP4 MoE kernel allocates a - # ~1.6 GiB runtime workspace OUTSIDE vLLM's memory pool on the first - # forward; at 0.95 a 178 GiB B200 has only ~1.35 GiB free and the first - # warmup request OOMs (seen on the dynamo-frontend variants). 0.90 - # matches the GB200/GB300 agentic recipes. - gpu-memory-utilization: 0.90 - no-enable-flashinfer-autotune: true - # kimi_k3 parsers via the native vllm serve OpenAI-frontend flags — - # legitimate here because this recipe serves directly with vllm serve - # (frontend.type: vllm), not through the dynamo worker entrypoint that - # rejects them. - enable-auto-tool-choice: true - tool-call-parser: kimi_k3 - reasoning-parser: kimi_k3 - # DSpark level 7 with probabilistic drafting and the committed golden - # synthetic acceptance length measured for Kimi-K3 at this draft length. - speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' - # 220 GiB of host DRAM per TP rank extends the KV working set beyond - # the observed GPU-resident c16-c32 cliff. Eager offload avoids a first-hit - # allocation spike during long AgentX trajectories. - kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' - # No explicit max-model-len: let vLLM derive the native 1M window from - # the model config (agentic trajectories blow past any small cap, and - # K3's KDA layers keep per-token KV small — only the 24 gated-MLA - # layers hold cache). Prefix caching stays on (default) for trajectory - # reuse. Cap prefill chunks so a single long request cannot OOM a - # pipeline stage; let vLLM pick max-num-seqs. - max-num-batched-tokens: 8192 - -sbatch_directives: - segment: "1" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep the aggregate worker in the multinode result schema so ingestion - # uses the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - # aiperf's conv-aware routing emits nvext.session_control, a removed POC - # field this dynamo build 400-rejects at warmup (schema moved to - # router/routing_constraints/agent_hints). Same opt-out as the GB300 - # aggregate AgentX recipes — and with a single aggregate worker there is - # no P/D routing to bind anyway. - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 594b713e89..f93560d511 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9253,11 +9253,11 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false -# Kimi K3's 2.8T MXFP4 weights require TP8 x PP2 across all 16 B200s. -# Cross-node TP16/TEP16 is unsupported on this pool because it lacks MNNVL -# multicast, while DEP16 replicates the full model and OOMs before weight load. -# Split the proven TP8 x PP2 topology into GPU-resident latency, balanced, and -# throughput tiers, plus a CPU KV-offloaded capacity tier. +# DSpark's K3DSparkModel does not implement pipeline parallelism, so Kimi K3 +# DSpark must use the official PP1 TP16 layout across all 16 B200s. Keep plain +# TP because the latency and capacity objectives do not require expert +# parallelism. Split the same serving topology into latency, balanced, +# throughput, and CPU KV-offloaded capacity tiers. kimik3-fp4-b200-dynamo-vllm-agentic-dspark: image: vllm/vllm-openai:kimi-k3 model: moonshotai/Kimi-K3 @@ -9276,16 +9276,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [1, 2, 4] prefill: num-worker: 1 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" decode: num-worker: 0 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false # Balanced medium-concurrency points. @@ -9293,33 +9291,29 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [8, 12, 16] prefill: num-worker: 1 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" decode: num-worker: 0 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false - # GPU-resident throughput points around the observed c16-c32 KV cliff. + # GPU-resident throughput points around the prior c16-c32 KV cliff. - spec-decoding: mtp conc-list: [20, 24, 28, 32] prefill: num-worker: 1 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" decode: num-worker: 0 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false # CPU KV-offloaded capacity points beyond the GPU-resident cliff. @@ -9329,16 +9323,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: conc-list: [32, 48, 64] prefill: num-worker: 1 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-vllm-simple-offload-dspark-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml" decode: num-worker: 0 - tp: 8 - pp: 2 + tp: 16 ep: 1 dp-attn: false diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ea815e0a30..3f57ad75e2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5377,7 +5377,7 @@ - config-keys: - kimik3-fp4-b200-dynamo-vllm-agentic-dspark description: - - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on the proven TP8 x PP2 topology" - - "Add a TP8 x PP2 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" + - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on the PP-free TP16 DSpark topology" + - "Add a TP16 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475