diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh new file mode 100755 index 0000000000..1822eadc9a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Dynamo serializes the draft config without Kimi K3's config-class alias. +# Prepare a local metadata view that exposes the equivalent parallel-draft token +# without changing the downloaded checkpoint or vLLM source. +python3 - <<'PY' +import json +import os +from pathlib import Path + +from huggingface_hub import snapshot_download + +repo_id = "Inferact/Kimi-K3-DSpark" +target = Path("/tmp/Kimi-K3-DSpark") +snapshot = Path(snapshot_download(repo_id=repo_id)) +target.mkdir(parents=True, exist_ok=True) + +for source in snapshot.iterdir(): + if source.name == "config.json": + continue + destination = target / source.name + if destination.is_symlink(): + if destination.resolve() == source.resolve(): + continue + destination.unlink() + elif destination.exists(): + raise RuntimeError(f"Refusing to replace non-symlink path: {destination}") + destination.symlink_to(source) + +config = json.loads((snapshot / "config.json").read_text()) +mask_token_id = config.get("mask_token_id") +if not isinstance(mask_token_id, int): + raise RuntimeError(f"{repo_id} config is missing integer mask_token_id") + +pard_token = config.get("pard_token") +if pard_token not in (None, mask_token_id): + raise RuntimeError( + f"{repo_id} pard_token={pard_token} disagrees with mask_token_id={mask_token_id}" + ) +config["pard_token"] = mask_token_id + +temporary = target / "config.json.tmp" +temporary.write_text(json.dumps(config, indent=2) + "\n") +os.replace(temporary, target / "config.json") +print(f"Prepared {repo_id} compatibility view at {target}") +PY diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml new file mode 100644 index 0000000000..7163e44a8e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml @@ -0,0 +1,114 @@ +name: "kimik3-vllm-agg-b200-tp16-latency-dspark-agentic" + +# Official latency-oriented multi_node_tp profile for two 8xB200 nodes. +# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + # K3's fused tail uses torch symmetric memory, which cannot exchange file + # descriptors across this two-node TP group. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" + # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. + VLLM_ALLREDUCE_USE_FLASHINFER: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can + # OOM at the official 0.95 setting. Preserve the observed-safe headroom. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # The generic custom communicator also probes MNNVL and stalls before + # model load on this pool. Use PyNCCL for the cross-node TP group. + disable-custom-all-reduce: true + # The default allreduce/RMS fusion independently allocates a FlashInfer + # MNNVL workspace during profile_run, bypassing the communicator flags. + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Performance runs use the committed K7 golden synthetic AL. The official + # recipe's real-verification form uses rejection_sample_method=block. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml new file mode 100644 index 0000000000..391aef0a44 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml @@ -0,0 +1,115 @@ +name: "kimik3-vllm-agg-b200-tp16-simple-offload-dspark-agentic" + +# Official latency-oriented multi_node_tp profile for two 8xB200 nodes. +# srt-slurm owns both vLLM ranks; the leader serves OpenAI and rank 1 is headless. +# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=b200&nodes=2&strategy=multi_node_tp +model: + path: "kimik3" + container: "vllm/vllm-openai:kimi-k3" + precision: "fp4" + +identity: + model: + repo: "moonshotai/Kimi-K3" + container: + image: "vllm/vllm-openai:kimi-k3" + +dynamo: + install: false + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 720 + +resources: + gpu_type: "b200" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: vllm + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + # K3's fused tail uses torch symmetric memory, which cannot exchange file + # descriptors across this two-node TP group. + VLLM_ENABLE_K3_LATENT_MOE_TAIL_FUSION: "0" + # Cross-node B200 groups lack the MNNVL multicast FlashInfer requires. + VLLM_ALLREDUCE_USE_FLASHINFER: "0" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_USE_RUST_FRONTEND: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_LOG_STATS_INTERVAL: "1" + NCCL_CUMEM_ENABLE: "1" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + vllm_config: + aggregated: + served-model-name: "moonshotai/Kimi-K3" + tensor-parallel-size: 16 + trust-remote-code: true + load-format: fastsafetensors + moe-backend: auto + # Target-B200 bring-up showed the first FlashInfer MXFP4 MoE workspace can + # OOM at the official 0.95 setting. Preserve the observed-safe headroom. + gpu-memory-utilization: 0.90 + no-enable-flashinfer-autotune: true + # The generic custom communicator also probes MNNVL and stalls before + # model load on this pool. Use PyNCCL for the cross-node TP group. + disable-custom-all-reduce: true + # The default allreduce/RMS fusion independently allocates a FlashInfer + # MNNVL workspace during profile_run, bypassing the communicator flags. + compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' + max-model-len: 1048576 + kv-cache-dtype: fp8 + attention-config: '{"use_prefill_query_quantization":true,"mla_prefill_backend":"TRTLLM_RAGGED"}' + enable-prefix-caching: true + enable-auto-tool-choice: true + tool-call-parser: kimi_k3 + reasoning-parser: kimi_k3 + language-model-only: true + max-num-seqs: 8 + max-num-batched-tokens: 8192 + # Performance runs use the committed K7 golden synthetic AL. The official + # recipe's real-verification form uses rejection_sample_method=block. + speculative-config: '{"model":"Inferact/Kimi-K3-DSpark","num_speculative_tokens":7,"method":"dspark","attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.84}' + kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use_per_rank":236223201280,"lazy_offload":false}}' + +sbatch_directives: + segment: "1" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" + AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 90e54195b6..f93560d511 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9253,6 +9253,87 @@ kimik3-fp4-b200-dynamo-vllm-agentic: ep: 1 dp-attn: false +# DSpark's K3DSparkModel does not implement pipeline parallelism, so Kimi K3 +# DSpark must use the official PP1 TP16 layout across all 16 B200s. Keep plain +# TP because the latency and capacity objectives do not require expert +# parallelism. Split the same serving topology into latency, balanced, +# throughput, and CPU KV-offloaded capacity tiers. +kimik3-fp4-b200-dynamo-vllm-agentic-dspark: + image: vllm/vllm-openai:kimi-k3 + model: moonshotai/Kimi-K3 + model-prefix: kimik3 + runner: cluster:b200-dgxc + precision: fp4 + framework: dynamo-vllm + multinode: true + disagg: false + scenarios: + agentic-coding: + - dram-utilization: 0.61 + search-space: + # Low-latency and high-interactivity points. + - spec-decoding: mtp + conc-list: [1, 2, 4] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 16 + ep: 1 + dp-attn: false + # Balanced medium-concurrency points. + - spec-decoding: mtp + conc-list: [8, 12, 16] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 16 + ep: 1 + dp-attn: false + # GPU-resident throughput points around the prior c16-c32 KV cliff. + - spec-decoding: mtp + conc-list: [20, 24, 28, 32] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-latency-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 16 + ep: 1 + dp-attn: false + # CPU KV-offloaded capacity points beyond the GPU-resident cliff. + - spec-decoding: mtp + kv-offloading: dram + kv-offload-backend: { name: vllm-simple, version: "13c59a3" } + conc-list: [32, 48, 64] + prefill: + num-worker: 1 + tp: 16 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp16-vllm-simple-offload-dspark-agentic.yaml" + decode: + num-worker: 0 + tp: 16 + ep: 1 + dp-attn: false + qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7a064f8090..3f57ad75e2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5374,3 +5374,10 @@ - "Image: lmsysorg/sglang:v0.5.16-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2420 +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + description: + - "Add Kimi K3 B200 AgentX latency, balanced, and throughput tiers on the PP-free TP16 DSpark topology" + - "Add a TP16 SimpleCPUOffloadConnector capacity tier with 220 GiB of host KV per rank" + - "Run DSpark level 7 with probabilistic drafting and the committed synthetic acceptance length 3.84" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2475 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 16d3338d09..bf759a010a 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -107,21 +107,19 @@ if [[ "$IS_MULTINODE" == "true" ]]; then rm -rf "$SRT_REPO_DIR" fi - # TODO(CJQ): make first class upon srt-slurm upstream refactor + # Kimi K3 aggregate profiles use the srt-slurm fork that supports direct + # multi-node vLLM for TP/TEP and topology-derived per-node DP ranks for + # Dynamo DEP. Pin the tested renderer so branch movement cannot change + # generated rank commands between sweep points. if [[ "$IS_AGENTIC" == "1" && $MODEL_PREFIX == "kimik3" ]]; then - # Direct-vLLM agentic experiment (Variant D): srt-slurm PR #278 - # (kylliang/direct-aggregate-vllm) adds frontend.type: vllm — `vllm - # serve` owns the OpenAI port itself, no Dynamo layer. The fork branch - # carries PR #278 plus the multi-node extension (vLLM-native - # --master-addr/--nnodes/--node-rank serve + headless non-leader - # ranks) so the 2-node TP8xPP2 topology can run. git clone --branch klaud/direct-vllm-multinode --single-branch https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 - if [[ $MODEL_PREFIX == "kimik3" ]]; then - mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ - recipes/vllm/kimi-k3/agentic || exit 1 - fi + git checkout df5baa93f4caf5169dea2a4236ad2cc742fe40e7 || exit 1 + mkdir -p recipes/vllm/kimi-k3/agentic configs || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic || exit 1 + cp "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/kimik3-dspark-config-compat.sh" \ + configs/kimik3-dspark-config-compat.sh || exit 1 elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1 diff --git a/utils/agentic/aggregation/process_agentic_result.py b/utils/agentic/aggregation/process_agentic_result.py index 5d13fa0f1b..4971a132ba 100644 --- a/utils/agentic/aggregation/process_agentic_result.py +++ b/utils/agentic/aggregation/process_agentic_result.py @@ -95,6 +95,18 @@ def _validate_kv_offload_env() -> tuple[str, dict[str, str] | None]: return kv_offloading, backend_metadata +def _parallel_gpu_count( + tp: int, + pp: int, + pcp_size: int, + ep: int, + dp_attention: bool, +) -> int: + """Return the physical GPU count for one worker replica.""" + tensor_gpu_count = tp * pp * pcp_size + return max(tensor_gpu_count, ep) if dp_attention else tensor_gpu_count + + def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: is_multinode = env_bool("IS_MULTINODE") tp = env_int("TP", 1) @@ -111,7 +123,10 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: "PP_SIZE, DCP_SIZE, and PCP_SIZE must be positive integers." ) fields.update({"pp": pp, "dcp_size": dcp_size, "pcp_size": pcp_size}) - return fields, tp * pp * pcp_size, tp, ep, dp_attention + num_gpus = _parallel_gpu_count( + tp, pp, pcp_size, ep, env_bool("DP_ATTENTION") + ) + return fields, num_gpus, tp, ep, dp_attention prefill_num_workers = env_int("PREFILL_NUM_WORKERS") prefill_tp = env_int("PREFILL_TP") @@ -120,6 +135,7 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: prefill_pcp_size = env_int("PREFILL_PCP_SIZE", 1) prefill_ep = env_int("PREFILL_EP", 1) prefill_dp_attention = os.environ.get("PREFILL_DP_ATTN", "false") + prefill_dp_attention_enabled = env_bool("PREFILL_DP_ATTN") decode_num_workers = env_int("DECODE_NUM_WORKERS") decode_tp = env_int("DECODE_TP") decode_pp = env_int("DECODE_PP_SIZE", 1) @@ -127,6 +143,7 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: decode_pcp_size = env_int("DECODE_PCP_SIZE", 1) decode_ep = env_int("DECODE_EP", 1) decode_dp_attention = os.environ.get("DECODE_DP_ATTN", "false") + decode_dp_attention_enabled = env_bool("DECODE_DP_ATTN") worker_parallelism = ( prefill_pp, prefill_dcp_size, @@ -145,8 +162,20 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: raise SystemExit( "PREFILL_HARDWARE and DECODE_HARDWARE must be specified together." ) - num_prefill_gpu = prefill_num_workers * prefill_tp * prefill_pp * prefill_pcp_size - num_decode_gpu = decode_num_workers * decode_tp * decode_pp * decode_pcp_size + num_prefill_gpu = prefill_num_workers * _parallel_gpu_count( + prefill_tp, + prefill_pp, + prefill_pcp_size, + prefill_ep, + prefill_dp_attention_enabled, + ) + num_decode_gpu = decode_num_workers * _parallel_gpu_count( + decode_tp, + decode_pp, + decode_pcp_size, + decode_ep, + decode_dp_attention_enabled, + ) num_gpus = num_prefill_gpu + num_decode_gpu # Aggregated configs set decode num-worker 0 (prefill+decode co-located on one # worker), so there are no separate decode GPUs. Mirror process_result.py and drop @@ -162,7 +191,7 @@ def _gpu_shape() -> tuple[dict[str, Any], int, int, int, str]: ep = max(prefill_ep, decode_ep) dp_attention = ( "true" - if env_bool("PREFILL_DP_ATTN") or env_bool("DECODE_DP_ATTN") + if prefill_dp_attention_enabled or decode_dp_attention_enabled else "false" ) fields.update( diff --git a/utils/agentic/aggregation/test_process_agentic_result.py b/utils/agentic/aggregation/test_process_agentic_result.py index 21a7307630..8bb4d36b06 100644 --- a/utils/agentic/aggregation/test_process_agentic_result.py +++ b/utils/agentic/aggregation/test_process_agentic_result.py @@ -564,6 +564,35 @@ def test_multinode_processor_surfaces_heterogeneous_hardware(tmp_path: Path): agg["num_decode_gpu"], ) == (2, 4, 1, 32) +def test_multinode_processor_counts_attention_dp_ranks( + monkeypatch: pytest.MonkeyPatch, +): + env = { + "IS_MULTINODE": "true", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "1", + "PREFILL_PP_SIZE": "1", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "16", + "PREFILL_DP_ATTN": "true", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "1", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "1", + "DECODE_DP_ATTN": "false", + } + for name, value in env.items(): + monkeypatch.setenv(name, value) + + fields, num_gpus, tp, ep, dp_attention = _gpu_shape() + + assert fields["num_prefill_gpu"] == 16 + assert fields["num_decode_gpu"] == 0 + assert (num_gpus, tp, ep, dp_attention) == (16, 1, 16, "true") + def test_multinode_processor_omits_homogeneous_hardware(tmp_path: Path): result_dir = _write_fixture(tmp_path) diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 86f192b1c8..6ccd84b604 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -82,13 +82,21 @@ def runner_gpus_per_node(runner: str, runner_data: dict) -> int: return runner_hardware_int(runner, runner_data, Fields.GPUS_PER_NODE.value) -def effective_gpu_count(benchmark: dict) -> int: - """Return GPUs used by a single-node TP/PP/PCP topology.""" - return ( - benchmark[Fields.TP.value] - * benchmark.get(Fields.PP.value, 1) - * benchmark.get(Fields.PCP_SIZE.value, 1) +def _parallel_gpu_count(config: dict) -> int: + """Return the GPU footprint encoded by one parallelism configuration.""" + tensor_gpu_count = ( + config[Fields.TP.value] + * config.get(Fields.PP.value, 1) + * config.get(Fields.PCP_SIZE.value, 1) ) + if config.get(Fields.DP_ATTN.value, False): + return max(tensor_gpu_count, config.get(Fields.EP.value, 1)) + return tensor_gpu_count + + +def effective_gpu_count(benchmark: dict) -> int: + """Return GPUs used by a single-node parallel topology.""" + return _parallel_gpu_count(benchmark) def with_worker_parallelism_defaults(worker: dict) -> dict: """Return a worker config with explicit parallelism defaults.""" @@ -109,29 +117,24 @@ def worker_gpus_per_node(worker: dict, gpus_per_node: int) -> int: are rejected rather than silently truncated, keeping parity with the single-node "must fit the node" rule: - * A replica larger than one node (tp*pp*pcp > gpus-per-node) must fill whole - nodes, i.e. be an exact multiple of gpus-per-node; each of its nodes is - then fully occupied (fraction 1). + * A replica larger than one node must fill whole nodes, i.e. be an exact + multiple of gpus-per-node; each of its nodes is then fully occupied. * A replica within one node must divide it evenly so co-located replicas of the same role tile the node without overlap. + * Attention-DP topologies include the EP-sized independent rank pool. """ - gpus_per_replica = ( - worker[Fields.TP.value] - * worker.get(Fields.PP.value, 1) - * worker.get(Fields.PCP_SIZE.value, 1) - ) + gpus_per_replica = _parallel_gpu_count(worker) if gpus_per_replica > gpus_per_node: if gpus_per_replica % gpus_per_node != 0: raise ValueError( - f"worker {Fields.TP.value}*{Fields.PP.value}*{Fields.PCP_SIZE.value}" - f"={gpus_per_replica} spans multiple nodes but is not a multiple " + f"worker GPU footprint={gpus_per_replica} spans multiple nodes " + "but is not a multiple " f"of {Fields.GPUS_PER_NODE.value}={gpus_per_node}" ) return gpus_per_node if gpus_per_node % gpus_per_replica != 0: raise ValueError( - f"worker {Fields.TP.value}*{Fields.PP.value}*{Fields.PCP_SIZE.value}" - f"={gpus_per_replica} does not divide " + f"worker GPU footprint={gpus_per_replica} does not divide " f"{Fields.GPUS_PER_NODE.value}={gpus_per_node} evenly" ) return gpus_per_replica diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 7cbfea79af..47974eb268 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -2274,6 +2274,56 @@ def test_multinode_agentic_preserves_kv_offload_fields(self, sample_runner_confi # DRAM: 2861022 MiB * 0.80. assert result[0]["total-cpu-dram-gb"] == 2399 + def test_multinode_agentic_dep_budget_counts_attention_dp_ranks( + self, sample_runner_config + ): + config = { + "kimik3-dep": { + "image": "vllm/vllm-openai:kimi-k3", + "model": "moonshotai/Kimi-K3", + "model-prefix": "kimik3", + "precision": "fp4", + "framework": "dynamo-vllm", + "runner": "cluster:b200-dgxc", + "multinode": True, + "scenarios": { + "agentic-coding": [{ + "dram-utilization": 0.61, + "search-space": [{ + "conc-list": [128], + "kv-offloading": "dram", + "kv-offload-backend": {"name": "vllm-simple"}, + "prefill": { + "num-worker": 1, + "tp": 1, + "ep": 16, + "dp-attn": True, + }, + "decode": { + "num-worker": 0, + "tp": 1, + "ep": 1, + "dp-attn": False, + }, + }], + }], + }, + }, + } + args = argparse.Namespace( + config_keys=["kimik3-dep"], + seq_lens=None, + conc=None, + scenario_type=["agentic-coding"], + runner_node_filter=None, + ) + + result = generate_test_config_sweep(args, config, sample_runner_config) + + assert len(result) == 1 + # DEP16 spans both nodes, so all eight ranks on each node share DRAM. + assert result[0]["total-cpu-dram-gb"] == 1829 + def test_multinode_agentic_budget_ignores_decode_topology( self, sample_runner_config ):