From f1b6df064ebe8fcdeaa0b29f3bc8da427194bac8 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 3 Aug 2026 16:08:29 -0500 Subject: [PATCH] config: make aggregate multinode topology explicit Require aggregate search-space entries to declare one worker and num-nodes, while reserving separate prefill and decode workers for disaggregated serving.\n\nBackfill every existing aggregate entry and translate the new master shape to the legacy runtime matrix representation. --- configs/nvidia-master.yaml | 64 ++------- utils/matrix_logic/generate_sweep_configs.py | 46 ++++--- .../test_generate_sweep_configs.py | 43 ++++++ utils/matrix_logic/test_validation.py | 78 +++++++++-- utils/matrix_logic/validation.py | 130 ++++++++++++++---- utils/runner_setup/RUNNER_SETUP.md | 14 +- 6 files changed, 262 insertions(+), 113 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 90e54195b..368a9fddb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7104,37 +7104,27 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: - dram-utilization: 0.80 search-space: - spec-decoding: "mtp" + num-nodes: 1 conc-list: [1] kv-offloading: dram kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 + worker: tp: 4 ep: 1 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c1-mtp-hicache-jid2191933.yaml" - decode: - num-worker: 0 - tp: 4 - ep: 1 - dp-attn: false - spec-decoding: "mtp" + num-nodes: 1 conc-list: [96] kv-offloading: dram kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 + worker: tp: 4 ep: 1 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/sglang/qwen3.5/gb300-fp4/agentic/agg-gb300-tp4-c96-mtp-hicache-jid2195211.yaml" - decode: - num-worker: 0 - tp: 4 - ep: 1 - dp-attn: false qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: @@ -8793,37 +8783,23 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: agentic-coding: - search-space: - spec-decoding: mtp + num-nodes: 2 conc-list: [1] - prefill: - num-worker: 1 + worker: tp: 8 ep: 1 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" - # The aggregate worker also performs decode; keep the decode worker - # count at zero so result aggregation counts eight GPUs only once. - decode: - num-worker: 0 - tp: 8 - ep: 1 - dp-attn: false - spec-decoding: mtp + num-nodes: 1 conc-list: [4] - prefill: - num-worker: 1 + worker: tp: 4 ep: 1 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" - # The aggregate worker also performs decode; keep the decode worker - # count at zero so result aggregation counts four GPUs only once. - decode: - num-worker: 0 - tp: 4 - ep: 1 - dp-attn: false dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f @@ -8883,24 +8859,16 @@ dsv4-fp4-gb300-dynamo-sglang-agentic-agg: - dram-utilization: 0.80 search-space: - spec-decoding: mtp + num-nodes: 1 conc-list: [1, 4, 8, 16] kv-offloading: dram kv-offload-backend: { name: hicache } - prefill: - num-worker: 1 + worker: tp: 4 ep: 1 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/sglang/deepseek-v4/agentic/agg-gb300-tp4-mtp-kvoffload.yaml" - # Aggregated worker: prefill's GPUs also serve decode, so decode has no - # separate allocation (num-worker: 0). Deployment is recipe-driven - # (agg_workers: 1); this only makes GPU accounting count 4, not 4+4. - decode: - num-worker: 0 - tp: 4 - ep: 1 - dp-attn: false dsv4-fp4-gb300-dynamo-sglang-agentic-disagg: image: lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 @@ -9235,23 +9203,15 @@ kimik3-fp4-b200-dynamo-vllm-agentic: agentic-coding: - search-space: - spec-decoding: none + num-nodes: 2 conc-list: [1, 2, 4, 8, 16, 32] - prefill: - num-worker: 1 + worker: tp: 8 pp: 2 ep: 1 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml" - # The aggregate worker also performs decode; keep the decode worker - # count at zero so result aggregation counts the 16 GPUs only once. - decode: - num-worker: 0 - tp: 8 - pp: 2 - ep: 1 - dp-attn: false qwen3.5-fp8-gb300-dynamo-sglang-mtp: image: lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3 diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index bd32edb7b..57bc5e93f 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -248,6 +248,27 @@ def with_worker_parallelism_defaults(worker: dict) -> dict: } +def multinode_worker_pair(benchmark: dict, disagg: bool) -> tuple[dict, dict]: + """Return the legacy prefill/decode matrix pair for a master entry.""" + if disagg: + return ( + with_worker_parallelism_defaults(benchmark[Fields.PREFILL.value]), + with_worker_parallelism_defaults(benchmark[Fields.DECODE.value]), + ) + + worker = with_worker_parallelism_defaults(benchmark[Fields.WORKER.value]) + prefill = {Fields.NUM_WORKER.value: 1, **worker} + decode = { + Fields.NUM_WORKER.value: 0, + **{ + key: value + for key, value in worker.items() + if key != Fields.ADDITIONAL_SETTINGS.value + }, + } + return prefill, decode + + def worker_gpus_per_node(worker: dict, gpus_per_node: int) -> int: """Return GPUs a single worker replica occupies on one node. @@ -309,7 +330,10 @@ def agentic_dram_offload_gb( utilization = Decimal(str(agentic_config[Fields.DRAM_UTILIZATION.value])) gpus_per_node = runner_gpus_per_node(runner, runner_data) - if Fields.PREFILL.value in benchmark: + if Fields.WORKER.value in benchmark: + gpu_count = worker_gpus_per_node( + benchmark[Fields.WORKER.value], gpus_per_node) + elif Fields.PREFILL.value in benchmark: gpu_count = worker_gpus_per_node( benchmark[Fields.PREFILL.value], gpus_per_node) else: @@ -649,10 +673,7 @@ def generate_full_sweep(args, all_config_data, runner_data): # spec_decoding defaults to "none" if not specified spec_decoding = bmk.get(Fields.SPEC_DECODING.value, "none") - prefill = with_worker_parallelism_defaults( - bmk[Fields.PREFILL.value]) - decode = with_worker_parallelism_defaults( - bmk[Fields.DECODE.value]) + prefill, decode = multinode_worker_pair(bmk, disagg) # Get concurrency values (can be list or range) conc_list = bmk.get(Fields.CONC_LIST.value) @@ -858,10 +879,7 @@ def generate_full_sweep(args, all_config_data, runner_data): for bmk in bmk_space: if is_multinode: - prefill = with_worker_parallelism_defaults( - bmk[Fields.PREFILL.value]) - decode = with_worker_parallelism_defaults( - bmk[Fields.DECODE.value]) + prefill, decode = multinode_worker_pair(bmk, disagg) spec_decoding = bmk.get(Fields.SPEC_DECODING.value, "none") kv_offloading = bmk.get(Fields.KV_OFFLOADING.value, "none") kv_offload_backend = bmk.get(Fields.KV_OFFLOAD_BACKEND.value) @@ -1050,10 +1068,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): if is_multinode: # Multinode config spec_decoding = bmk.get(Fields.SPEC_DECODING.value, "none") - prefill = with_worker_parallelism_defaults( - bmk[Fields.PREFILL.value]) - decode = with_worker_parallelism_defaults( - bmk[Fields.DECODE.value]) + prefill, decode = multinode_worker_pair(bmk, disagg) # Get concurrency values if Fields.CONC_LIST.value in bmk: @@ -1172,10 +1187,7 @@ def generate_test_config_sweep(args, all_config_data, runner_data=None): for bmk in bmk_space: if is_multinode: - prefill = with_worker_parallelism_defaults( - bmk[Fields.PREFILL.value]) - decode = with_worker_parallelism_defaults( - bmk[Fields.DECODE.value]) + prefill, decode = multinode_worker_pair(bmk, disagg) spec_decoding = bmk.get(Fields.SPEC_DECODING.value, "none") kv_offloading = bmk.get(Fields.KV_OFFLOADING.value, "none") kv_offload_backend = bmk.get(Fields.KV_OFFLOAD_BACKEND.value) diff --git a/utils/matrix_logic/test_generate_sweep_configs.py b/utils/matrix_logic/test_generate_sweep_configs.py index 9e1daa084..dfb36ca47 100644 --- a/utils/matrix_logic/test_generate_sweep_configs.py +++ b/utils/matrix_logic/test_generate_sweep_configs.py @@ -15,6 +15,7 @@ mark_all_eval_entries, mark_eval_entries, multinode_node_count, + multinode_worker_pair, seq_len_itos, seq_len_stoi, seq_len_to_str, @@ -45,6 +46,40 @@ def test_disaggregated_multinode_node_count_rejects_num_nodes(): with pytest.raises(ValueError, match="num-nodes.*disaggregated"): add_multinode_node_count(entry, {}, num_nodes=3) + +def test_aggregated_worker_expands_to_legacy_matrix_pair(): + benchmark = { + "worker": { + "tp": 8, + "pp": 2, + "ep": 1, + "dp-attn": False, + "additional-settings": ["CONFIG_FILE=recipes/aggregate.yaml"], + } + } + + prefill, decode = multinode_worker_pair(benchmark, disagg=False) + + assert prefill == { + "num-worker": 1, + "tp": 8, + "pp": 2, + "dcp-size": 1, + "pcp-size": 1, + "ep": 1, + "dp-attn": False, + "additional-settings": ["CONFIG_FILE=recipes/aggregate.yaml"], + } + assert decode == { + "num-worker": 0, + "tp": 8, + "pp": 2, + "dcp-size": 1, + "pcp-size": 1, + "ep": 1, + "dp-attn": False, + } + def test_multinode_node_count_uses_role_gpu_footprints(sample_runner_config): prefill = {"num-worker": 3, "tp": 2, "pp": 1, "pcp-size": 1} decode = {"num-worker": 2, "tp": 8, "pp": 1, "pcp-size": 1} @@ -1193,6 +1228,8 @@ def test_runner_node_filter_multinode(self, sample_runner_config, full_sweep_arg "framework": "dynamo-trt", "runner": "h200", "multinode": True, + "disagg": True, + "kv-p2p-transfer": "nixl", "scenarios": { "fixed-seq-len": [ @@ -1456,6 +1493,8 @@ def test_multinode_conc_range_expansion(self, sample_runner_config, full_sweep_a "framework": "dynamo-trt", "runner": "gb200", "multinode": True, + "disagg": True, + "kv-p2p-transfer": "nixl", "scenarios": { "fixed-seq-len": [ @@ -1574,6 +1613,8 @@ def test_multinode_max_conc_zero_or_negative_skips(self, sample_runner_config, f "framework": "dynamo-trt", "runner": "gb200", "multinode": True, + "disagg": True, + "kv-p2p-transfer": "nixl", "scenarios": { "fixed-seq-len": [ @@ -1622,6 +1663,8 @@ def test_multinode_max_conc_creates_config_when_below_min(self, sample_runner_co "framework": "dynamo-trt", "runner": "gb200", "multinode": True, + "disagg": True, + "kv-p2p-transfer": "nixl", "scenarios": { "fixed-seq-len": [ diff --git a/utils/matrix_logic/test_validation.py b/utils/matrix_logic/test_validation.py index 453285232..85662775d 100644 --- a/utils/matrix_logic/test_validation.py +++ b/utils/matrix_logic/test_validation.py @@ -830,6 +830,23 @@ def test_with_spec_decoding_mtp(self): class TestMultiNodeSearchSpaceEntry: """Tests for MultiNodeSearchSpaceEntry model.""" + def test_valid_aggregate_worker(self): + """An aggregate entry has one worker rather than serving roles.""" + entry = MultiNodeSearchSpaceEntry(**{ + "worker": { + "tp": 8, + "pp": 2, + "ep": 1, + "dp-attn": False, + }, + "num-nodes": 2, + "conc-list": [1, 2, 4], + }) + assert entry.worker.tp == 8 + assert entry.worker.pp == 2 + assert entry.prefill is None + assert entry.decode is None + def test_valid_with_conc_list(self): """Valid multinode search space with list (like gb200 config).""" entry = MultiNodeSearchSpaceEntry(**{ @@ -976,6 +993,20 @@ def test_multinode_seq_len_config(self): # Test MasterConfigEntry models # ============================================================================= +def make_aggregated_multinode_master_config(config, num_nodes=3): + """Convert the disaggregated fixture to one aggregate worker.""" + config["disagg"] = False + search_entry = config[ + "scenarios" + ]["fixed-seq-len"][0]["search-space"][0] + worker = search_entry.pop("prefill") + search_entry.pop("decode") + worker.pop("num-worker") + search_entry["worker"] = worker + search_entry["num-nodes"] = num_nodes + return search_entry + + class TestMasterConfigEntries: """Tests for master config entry models.""" @@ -1047,7 +1078,7 @@ def test_aggregated_multinode_allows_kv_p2p_transfer( valid_multinode_master_config, ): """P2P transfer is not restricted to disaggregated multinode serving.""" - valid_multinode_master_config["disagg"] = False + make_aggregated_multinode_master_config(valid_multinode_master_config) config = MultiNodeMasterConfigEntry(**valid_multinode_master_config) @@ -1057,17 +1088,39 @@ def test_aggregated_multinode_allows_explicit_num_nodes( self, valid_multinode_master_config, ): - """Aggregated search-space entries may set their Slurm node count.""" - valid_multinode_master_config["disagg"] = False - search_entry = valid_multinode_master_config[ - "scenarios" - ]["fixed-seq-len"][0]["search-space"][0] - search_entry["num-nodes"] = 3 + """Aggregated entries require one worker and a Slurm node count.""" + make_aggregated_multinode_master_config(valid_multinode_master_config) config = MultiNodeMasterConfigEntry(**valid_multinode_master_config) validated_entry = config.scenarios.fixed_seq_len[0].search_space[0] assert validated_entry.num_nodes == 3 + assert validated_entry.worker.tp == 4 + assert validated_entry.prefill is None + assert validated_entry.decode is None + + def test_aggregated_multinode_requires_num_nodes( + self, + valid_multinode_master_config, + ): + """Every aggregate multi-node entry must declare its allocation.""" + search_entry = make_aggregated_multinode_master_config( + valid_multinode_master_config + ) + search_entry.pop("num-nodes") + + with pytest.raises(Exception, match="disagg=false requires num-nodes"): + MultiNodeMasterConfigEntry(**valid_multinode_master_config) + + def test_aggregated_multinode_rejects_prefill_decode( + self, + valid_multinode_master_config, + ): + """Aggregate master entries cannot model separate serving roles.""" + valid_multinode_master_config["disagg"] = False + + with pytest.raises(Exception, match="disagg=false requires one worker"): + MultiNodeMasterConfigEntry(**valid_multinode_master_config) def test_disaggregated_multinode_rejects_num_nodes( self, @@ -1079,7 +1132,7 @@ def test_disaggregated_multinode_rejects_num_nodes( ]["fixed-seq-len"][0]["search-space"][0] search_entry["num-nodes"] = 3 - with pytest.raises(Exception, match="num-nodes is only valid"): + with pytest.raises(Exception, match="disagg=true.*num-nodes"): MultiNodeMasterConfigEntry(**valid_multinode_master_config) @pytest.mark.parametrize("num_nodes", [0, -1, True]) @@ -1089,11 +1142,10 @@ def test_aggregated_multinode_rejects_invalid_num_nodes( num_nodes, ): """Explicit aggregate node counts must be strict positive integers.""" - valid_multinode_master_config["disagg"] = False - search_entry = valid_multinode_master_config[ - "scenarios" - ]["fixed-seq-len"][0]["search-space"][0] - search_entry["num-nodes"] = num_nodes + make_aggregated_multinode_master_config( + valid_multinode_master_config, + num_nodes=num_nodes, + ) with pytest.raises(Exception, match="num-nodes"): MultiNodeMasterConfigEntry(**valid_multinode_master_config) diff --git a/utils/matrix_logic/validation.py b/utils/matrix_logic/validation.py index 53ea84099..b061d984e 100644 --- a/utils/matrix_logic/validation.py +++ b/utils/matrix_logic/validation.py @@ -56,6 +56,7 @@ class Fields(Enum): # Multinode-specific fields (when MULTINODE = true) SPEC_DECODING = 'spec-decoding' + WORKER = 'worker' PREFILL = 'prefill' DECODE = 'decode' NUM_WORKER = 'num-worker' @@ -188,6 +189,27 @@ class WorkerConfig(BaseModel): model_config = ConfigDict(extra='forbid', populate_by_name=True) num_worker: int = Field(alias=Fields.NUM_WORKER.value) + tp: int + pp: int = Field(default=1, gt=0, strict=True) + dcp_size: int = Field( + default=1, alias=Fields.DCP_SIZE.value, gt=0, strict=True) + pcp_size: int = Field( + default=1, alias=Fields.PCP_SIZE.value, gt=0, strict=True) + ep: int + dp_attn: bool = Field(alias=Fields.DP_ATTN.value) + hardware: Optional[str] = Field(default=None, min_length=1) + additional_settings: Optional[List[str]] = Field( + default_factory=list, alias=Fields.ADDITIONAL_SETTINGS.value) + + @model_validator(mode='after') + def validate_worker_topology(self): + return _validate_tp_context_topology(self) + + +class AggregateWorkerConfig(BaseModel): + """Topology for one worker serving both prefill and decode.""" + model_config = ConfigDict(extra='forbid', populate_by_name=True) + tp: int pp: int = Field(default=1, gt=0, strict=True) dcp_size: int = Field( @@ -518,8 +540,9 @@ class MultiNodeSearchSpaceEntry(BaseModel): spec_decoding: Literal["mtp", "draft_model", "none"] = Field( default="none", alias=Fields.SPEC_DECODING.value) - prefill: WorkerConfig - decode: WorkerConfig + worker: Optional[AggregateWorkerConfig] = None + prefill: Optional[WorkerConfig] = None + decode: Optional[WorkerConfig] = None num_nodes: Optional[int] = Field( default=None, alias=Fields.NUM_NODES.value, gt=0, strict=True) router: Optional[ComponentMetadata] = None @@ -539,7 +562,21 @@ def validate_conc_fields(self): @model_validator(mode='after') def validate_worker_hardware_pair(self): - return _validate_worker_hardware_pair(self) + has_worker = self.worker is not None + has_any_disagg_worker = self.prefill is not None or self.decode is not None + has_complete_disagg_workers = ( + self.prefill is not None and self.decode is not None + ) + if has_worker == has_any_disagg_worker or ( + has_any_disagg_worker and not has_complete_disagg_workers + ): + raise ValueError( + "Multinode search-space entries must specify either worker " + "or both prefill and decode" + ) + if has_complete_disagg_workers: + _validate_worker_hardware_pair(self) + return self class SingleNodeSeqLenConfig(BaseModel): @@ -576,6 +613,7 @@ class AgenticCodingSearchSpaceEntry(BaseModel): dp_attn: Optional[bool] = Field(default=None, alias=Fields.DP_ATTN.value) spec_decoding: Literal["mtp", "draft_model", "none"] = Field( default="none", alias=Fields.SPEC_DECODING.value) + worker: Optional[AggregateWorkerConfig] = None prefill: Optional[WorkerConfig] = None decode: Optional[WorkerConfig] = None num_nodes: Optional[int] = Field( @@ -605,14 +643,21 @@ def validate_kv_offload_fields(self): @model_validator(mode='after') def validate_topology_fields(self): has_single_node = self.tp is not None + has_aggregate_worker = self.worker is not None has_any_multinode_field = self.prefill is not None or self.decode is not None has_complete_multinode = self.prefill is not None and self.decode is not None - if has_single_node: - valid = not has_any_multinode_field - else: - valid = has_complete_multinode - if not valid: - raise ValueError("Agentic search-space entries must specify either tp or both prefill and decode") + topology_count = sum(( + has_single_node, + has_aggregate_worker, + has_complete_multinode, + )) + if topology_count != 1 or ( + has_any_multinode_field and not has_complete_multinode + ): + raise ValueError( + "Agentic search-space entries must specify exactly one of tp, " + "worker, or both prefill and decode" + ) if has_single_node: if self.kv_offloading is None: raise ValueError( @@ -620,7 +665,7 @@ def validate_topology_fields(self): f"{Fields.KV_OFFLOADING.value}" ) _validate_tp_context_topology(self) - if has_complete_multinode: + if has_aggregate_worker or has_complete_multinode: explicitly_single_node_fields = { "pp", "dcp_size", @@ -639,7 +684,8 @@ def validate_topology_fields(self): "Multinode agentic search-space entries cannot specify " f"{field_names}" ) - _validate_worker_hardware_pair(self) + if has_complete_multinode: + _validate_worker_hardware_pair(self) return self class AgenticCodingConfig(BaseModel): @@ -750,19 +796,51 @@ def _validate_component_metadata_scope(self: BaseModel) -> BaseModel: return self -def _validate_num_nodes_scope(self: BaseModel) -> BaseModel: - """Allow explicit node counts only for aggregated multinode entries.""" +def _validate_multinode_entry_scope(self: BaseModel) -> BaseModel: + """Match each search-space topology to its master serving mode.""" search_space_entries = _master_search_space_entries(self) - entries_with_num_nodes = [ - entry for entry in search_space_entries - if getattr(entry, "num_nodes", None) is not None - ] + for entry in search_space_entries: + worker = getattr(entry, "worker", None) + prefill = getattr(entry, "prefill", None) + decode = getattr(entry, "decode", None) + num_nodes = getattr(entry, "num_nodes", None) + + if not self.multinode: + if ( + worker is not None + or prefill is not None + or decode is not None + or num_nodes is not None + ): + raise ValueError( + "Single-node search-space entries must specify tp topology " + "and cannot declare worker, prefill, decode, or num-nodes" + ) + continue - if (not self.multinode or self.disagg) and entries_with_num_nodes: - raise ValueError( - f"{Fields.NUM_NODES.value} is only valid when " - f"{Fields.MULTINODE.value}=true and {Fields.DISAGG.value}=false" - ) + if self.disagg: + if worker is not None or num_nodes is not None: + raise ValueError( + f"{Fields.DISAGG.value}=true requires prefill and decode " + f"and rejects {Fields.WORKER.value} and " + f"{Fields.NUM_NODES.value}" + ) + if prefill is None or decode is None: + raise ValueError( + f"{Fields.DISAGG.value}=true requires prefill and decode" + ) + continue + + if worker is None or prefill is not None or decode is not None: + raise ValueError( + f"{Fields.DISAGG.value}=false requires one " + f"{Fields.WORKER.value} and rejects prefill and decode" + ) + if num_nodes is None: + raise ValueError( + f"{Fields.DISAGG.value}=false requires " + f"{Fields.NUM_NODES.value} in every search-space entry" + ) return self @@ -791,8 +869,8 @@ def validate_component_metadata_scope(self): return _validate_component_metadata_scope(self) @model_validator(mode='after') - def validate_num_nodes_scope(self): - return _validate_num_nodes_scope(self) + def validate_multinode_entry_scope(self): + return _validate_multinode_entry_scope(self) class MultiNodeMasterConfigEntry(BaseModel): @@ -823,8 +901,8 @@ def validate_component_metadata_scope(self): return _validate_component_metadata_scope(self) @model_validator(mode='after') - def validate_num_nodes_scope(self): - return _validate_num_nodes_scope(self) + def validate_multinode_entry_scope(self): + return _validate_multinode_entry_scope(self) def validate_master_config(master_configs: dict) -> List[dict]: diff --git a/utils/runner_setup/RUNNER_SETUP.md b/utils/runner_setup/RUNNER_SETUP.md index 11793645a..e92f7800c 100644 --- a/utils/runner_setup/RUNNER_SETUP.md +++ b/utils/runner_setup/RUNNER_SETUP.md @@ -213,11 +213,15 @@ must also remain enabled. If either variable is disabled, workflows omit subtracts `0.001` per additional node so otherwise equal work prefers smaller allocations without overriding the existing business-priority signals. -Aggregated multi-node search-space entries can declare `num-nodes`; that value -becomes the generated `node-count` directly. `num-nodes` is rejected for -disaggregated entries because their independent prefill and decode allocations -determine the total. During migration, entries without `num-nodes` derive -`node-count` from, in precedence order: +Aggregated multi-node search-space entries must declare one `worker` and +`num-nodes`; that value becomes the generated `node-count` directly. The +generator expands the aggregate worker into the legacy internal prefill/decode +matrix shape expected by the launcher. Aggregate master entries cannot declare +separate `prefill` or `decode` roles. + +Disaggregated entries must declare `prefill` and `decode`, and reject both +`worker` and `num-nodes`. Their generated `node-count` is derived from, in +precedence order: 1. checked-in srt-slurm recipe `resources`; 2. explicit `PREFILL_NODES` and `DECODE_NODES` settings; or