diff --git a/.github/scripts/ci/run_colossalai_batch.sh b/.github/scripts/ci/run_colossalai_batch.sh index aaa078a8a707..a613a0098c23 100644 --- a/.github/scripts/ci/run_colossalai_batch.sh +++ b/.github/scripts/ci/run_colossalai_batch.sh @@ -81,7 +81,10 @@ case "${BATCH}" in tests/test_benchmark/test_benchmark_search.py tests/test_benchmark/test_benchmark_scaling.py tests/test_lazy/test_chatglm_cpu.py + tests/test_moe/test_ep_moe_layer_cpu.py tests/test_numerics/test_numerics_acceptance.py + tests/test_moe/test_ep_moe_layer_vs_hf.py + tests/test_moe/test_grad_sync_plan.py tests/test_numerics/test_numerics_core.py tests/test_model_zoo/test_gemma4.py tests/test_numerics/test_numerics_data_parallel.py @@ -92,6 +95,7 @@ case "${BATCH}" in tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py tests/test_shardformer/test_layer/test_attention_backends.py + tests/test_shardformer/test_qwen3_moe_policy_cpu.py tests/test_tensor/test_dtensor/test_dtensor_sharding_spec.py tests/test_tensor/test_shape_consistency.py tests/test_tensor/test_sharding_spec.py diff --git a/.github/scripts/ci/run_cpu_unit_tests.sh b/.github/scripts/ci/run_cpu_unit_tests.sh index 4c4febd9b15f..9620ae744d09 100644 --- a/.github/scripts/ci/run_cpu_unit_tests.sh +++ b/.github/scripts/ci/run_cpu_unit_tests.sh @@ -14,7 +14,10 @@ TESTS=( tests/test_config/test_load_config.py tests/test_infer/test_async_engine/test_request_tracer.py tests/test_lazy/test_chatglm_cpu.py + tests/test_moe/test_ep_moe_layer_cpu.py tests/test_numerics/test_numerics_acceptance.py + tests/test_moe/test_ep_moe_layer_vs_hf.py + tests/test_moe/test_grad_sync_plan.py tests/test_numerics/test_numerics_core.py tests/test_model_zoo/test_gemma4.py tests/test_numerics/test_numerics_data_parallel.py @@ -25,6 +28,7 @@ TESTS=( tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py tests/test_shardformer/test_layer/test_attention_backends.py + tests/test_shardformer/test_qwen3_moe_policy_cpu.py tests/test_tensor/test_dtensor/test_dtensor_sharding_spec.py tests/test_tensor/test_shape_consistency.py tests/test_tensor/test_sharding_spec.py diff --git a/colossalai/booster/plugin/hybrid_parallel_plugin.py b/colossalai/booster/plugin/hybrid_parallel_plugin.py index a4806bf7e55e..b1763581d457 100644 --- a/colossalai/booster/plugin/hybrid_parallel_plugin.py +++ b/colossalai/booster/plugin/hybrid_parallel_plugin.py @@ -41,6 +41,7 @@ from colossalai.shardformer.shard.shard_config import SUPPORT_SP_MODE from colossalai.tensor.colo_parameter import ColoParameter from colossalai.tensor.d_tensor.api import is_customized_distributed_tensor, is_distributed_tensor +from colossalai.tensor.moe_tensor.grad_sync import GradSyncSpec from colossalai.tensor.param_op_hook import ColoParamOpHookManager from colossalai.zero.low_level import LowLevelZeroOptimizer from colossalai.zero.low_level.zero_hook import ZeroOpHook, wait_all_gather_handle @@ -698,6 +699,7 @@ def __init__( overlap_allgather: bool = False, fp8_communication: bool = False, grad_norm_process_group: Optional[ProcessGroup] = None, + grad_sync_plan: Optional[Dict[torch.nn.Parameter, GradSyncSpec]] = None, ): self.model = model self.param_info = param_info @@ -730,6 +732,7 @@ def __init__( fp8_communication=fp8_communication, backward_context=model._hook_context, grad_norm_process_group=grad_norm_process_group, + grad_sync_plan=grad_sync_plan, ) def sync_dp_grads(self): diff --git a/colossalai/booster/plugin/moe_hybrid_parallel_plugin.py b/colossalai/booster/plugin/moe_hybrid_parallel_plugin.py index 61a52a6b7cc6..ceb0b61beb63 100644 --- a/colossalai/booster/plugin/moe_hybrid_parallel_plugin.py +++ b/colossalai/booster/plugin/moe_hybrid_parallel_plugin.py @@ -1,6 +1,6 @@ from collections import defaultdict from types import MethodType -from typing import Callable, List, Optional, OrderedDict, Tuple +from typing import Callable, Dict, List, Optional, OrderedDict, Tuple import torch import torch.distributed as dist @@ -33,6 +33,7 @@ from colossalai.shardformer.shard.grad_ckpt_config import GradientCheckpointConfig from colossalai.shardformer.shard.shard_config import SUPPORT_SP_MODE, ShardConfig from colossalai.tensor.moe_tensor.api import is_moe_tensor +from colossalai.tensor.moe_tensor.grad_sync import GradSyncSpec, build_grad_sync_plan, has_reduction_scaled_experts class MoeHybridParallelZeroOptimizer(HybridParallelZeroOptimizer): @@ -62,6 +63,7 @@ def __init__( cpu_offload: bool = False, # cpu offload forced_dtype: Optional[torch.dtype] = None, overlap_allgather: bool = False, + grad_sync_plan: Optional[Dict[torch.nn.Parameter, GradSyncSpec]] = None, ): if dp_process_group is moe_dp_group: pg_param_list = { @@ -102,6 +104,7 @@ def __init__( overlap_allgather=overlap_allgather, # the dp group spans the moe_dp and ep axes, so it covers every shard of dense and expert grads grad_norm_process_group=dp_process_group, + grad_sync_plan=grad_sync_plan, ) @@ -472,6 +475,11 @@ def configure( ) if optimizer is not None and not isinstance(optimizer, OptimizerWrapper): if self.zero_stage == 0: + if has_reduction_scaled_experts(model.parameters()): + raise NotImplementedError( + "Experts with ExpertGradScale.IN_REDUCTION need the gradient sync plan, which only the ZeRO " + "optimizer reads so far; use zero_stage >= 1 or ExpertGradScale.IN_FORWARD." + ) is_zero = False if self.precision in ["fp16", "bf16"]: optimizer = HybridParallelAMPOptimizer( @@ -507,6 +515,9 @@ def configure( model, use_pipeline=self.enable_pipeline_parallelism or self.ep_size > 1, param_info=param_info, + grad_sync_plan=build_grad_sync_plan( + model.parameters(), self.mixed_dp_group, self.mixed_moe_dp_group + ), dp_process_group=self.mixed_dp_group, tp_process_group=self.tp_group, pp_process_group=self.pp_group, diff --git a/colossalai/moe/__init__.py b/colossalai/moe/__init__.py index f6ced3fedc5a..6b47b77e5915 100644 --- a/colossalai/moe/__init__.py +++ b/colossalai/moe/__init__.py @@ -1,26 +1,34 @@ -"""MoE operations, loaded on demand to keep package imports lightweight.""" +"""MoE operations and the expert-parallel layer, loaded on demand to keep package imports lightweight.""" from importlib import import_module -__all__ = [ - "AllGather", - "AllToAll", - "EPGradScalerIn", - "EPGradScalerOut", - "HierarchicalAllToAll", - "MoeCombine", - "MoeDispatch", - "ReduceScatter", - "all_to_all_uneven", - "moe_cumsum", -] +_SUBMODULE = { + "AllGather": "_operation", + "AllToAll": "_operation", + "EPGradScalerIn": "_operation", + "EPGradScalerOut": "_operation", + "HierarchicalAllToAll": "_operation", + "MoeCombine": "_operation", + "MoeDispatch": "_operation", + "ReduceScatter": "_operation", + "all_to_all_uneven": "_operation", + "moe_cumsum": "_operation", + "DispatchOutput": "dispatcher", + "Dispatcher": "dispatcher", + "NCCLAllToAllDispatcher": "dispatcher", + "Experts": "experts", + "GroupedMMExperts": "experts", + "ReferenceExperts": "experts", + "EPMoELayer": "layer", +} +__all__ = list(_SUBMODULE) def __getattr__(name): - if name in __all__: - operation = getattr(import_module("._operation", __name__), name) - globals()[name] = operation - return operation + if name in _SUBMODULE: + value = getattr(import_module(f".{_SUBMODULE[name]}", __name__), name) + globals()[name] = value + return value raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/colossalai/moe/dispatcher.py b/colossalai/moe/dispatcher.py new file mode 100644 index 000000000000..f49800ad6bfd --- /dev/null +++ b/colossalai/moe/dispatcher.py @@ -0,0 +1,116 @@ +"""Token dispatchers for expert parallelism and the data contract they hand to the experts. + +A dispatcher moves each token's copies to the ranks that hold its top-k experts (``dispatch``) and brings the expert +outputs back, weighted by the routing weights (``combine``). Experts only see a :class:`DispatchOutput`, so any +dispatcher can be paired with any experts backend. +""" + +from dataclasses import dataclass +from typing import Any, List, Optional, Protocol + +import torch +import torch.distributed as dist +from torch import Tensor +from torch.distributed import ProcessGroup + +from ._operation import all_to_all_uneven + +__all__ = ["DispatchOutput", "Dispatcher", "NCCLAllToAllDispatcher"] + + +@dataclass +class DispatchOutput: + """What a dispatcher gives the experts. + + Attributes: + tokens: ``[S, H]`` rows for the local experts, grouped by local expert (all rows of local expert 0 first). + Within an expert, rows keep the order of their tokens in the batch of the whole EP group (source rank + first, then token position), so expert grads can be compared bitwise with a single-GPU reference. + tokens_per_expert: ``[E_local]`` int64 row count per local expert, on the same device as ``tokens``. + tokens_per_expert_cpu: the same counts as a list, when the dispatcher already has them on the host. + probs: ``[S]`` fp32 routing weight of each row, for dispatchers that let the experts apply the weights. ``None`` + when ``combine`` applies them on the source rank. Experts never apply them otherwise. + handle: dispatcher-private state needed by ``combine``. + """ + + tokens: Tensor + tokens_per_expert: Tensor + tokens_per_expert_cpu: Optional[List[int]] = None + probs: Optional[Tensor] = None + handle: Any = None + + +class Dispatcher(Protocol): + def dispatch(self, hidden_states: Tensor, topk_idx: Tensor, topk_weights: Tensor) -> DispatchOutput: + """Send ``hidden_states`` ``[T, H]`` to the ranks of the experts in ``topk_idx`` ``[T, K]`` (global ids).""" + + def combine(self, expert_output: Tensor, dispatched: DispatchOutput) -> Tensor: + """Return ``[T, H]``: for every token, the sum of its expert outputs weighted by ``topk_weights``.""" + + +class NCCLAllToAllDispatcher: + """Dispatcher built on NCCL all-to-all, with one host sync per layer (the split sizes NCCL needs on the host). + + The weighted sum is done on the source rank in fp32 with the routing weights cast to fp32 and cast once, which + matches the grouped_mm path of HF transformers v5 bitwise in the forward pass when the router returns fp32 weights. + A bf16 model's HF router rounds the weights to bf16 first, and the results then agree up to bf16 rounding. + + Args: + num_experts: total number of experts in the layer. + ep_group: expert parallel group; experts are split evenly and contiguously over its ranks. + """ + + def __init__(self, num_experts: int, ep_group: Optional[ProcessGroup] = None): + self.ep_group = ep_group + self.ep_size = dist.get_world_size(ep_group) if dist.is_initialized() else 1 + if num_experts % self.ep_size != 0: + raise ValueError(f"num_experts ({num_experts}) must be divisible by the ep size ({self.ep_size})") + self.num_experts = num_experts + self.num_local_experts = num_experts // self.ep_size + + def dispatch(self, hidden_states: Tensor, topk_idx: Tensor, topk_weights: Tensor) -> DispatchOutput: + num_tokens, top_k = topk_idx.shape + flat_idx = topk_idx.reshape(-1) + # (token, slot) pairs sorted by global expert id, i.e. by destination rank then local expert + order = torch.sort(flat_idx, stable=True)[1] + send = hidden_states[order // top_k] + send_counts = torch.bincount(flat_idx, minlength=self.num_experts).view(self.ep_size, self.num_local_experts) + if self.ep_size > 1: + recv_counts = torch.empty_like(send_counts) + dist.all_to_all_single(recv_counts, send_counts, group=self.ep_group) # [source rank, local expert] + else: + recv_counts = send_counts + # the one host sync: split sizes for NCCL and the per-expert row counts + sizes = torch.cat([send_counts.sum(1), recv_counts.sum(1), recv_counts.sum(0)]).tolist() + send_splits = sizes[: self.ep_size] + recv_splits = sizes[self.ep_size : 2 * self.ep_size] + tokens_per_expert_cpu = sizes[2 * self.ep_size :] + if self.ep_size > 1: + recv, _ = all_to_all_uneven(send, send_splits, recv_splits, self.ep_group) + else: + recv = send + # rows arrive grouped by (source rank, local expert); regroup them by local expert on the GPU + local_expert = torch.arange(self.num_local_experts, device=recv.device).repeat(self.ep_size) + row_expert = local_expert.repeat_interleave(recv_counts.reshape(-1), output_size=recv.size(0)) + perm = torch.sort(row_expert, stable=True)[1] + return DispatchOutput( + tokens=recv[perm], + tokens_per_expert=recv_counts.sum(0), + tokens_per_expert_cpu=tokens_per_expert_cpu, + handle=(order, perm, send_splits, recv_splits, num_tokens, top_k, topk_weights), + ) + + def combine(self, expert_output: Tensor, dispatched: DispatchOutput) -> Tensor: + order, perm, send_splits, recv_splits, num_tokens, top_k, topk_weights = dispatched.handle + out = expert_output[_inverse_permutation(perm)] + if self.ep_size > 1: + out, _ = all_to_all_uneven(out, recv_splits, send_splits, self.ep_group) + weights = topk_weights.reshape(-1)[order].float() + weighted = (out.float() * weights[:, None])[_inverse_permutation(order)] + return weighted.view(num_tokens, top_k, out.size(-1)).sum(1).to(expert_output.dtype) + + +def _inverse_permutation(perm: Tensor) -> Tensor: + inv = torch.empty_like(perm) + inv[perm] = torch.arange(perm.numel(), device=perm.device) + return inv diff --git a/colossalai/moe/experts.py b/colossalai/moe/experts.py new file mode 100644 index 000000000000..6f95d6151b45 --- /dev/null +++ b/colossalai/moe/experts.py @@ -0,0 +1,78 @@ +"""Experts backends: the SwiGLU MLPs of the local experts, run on the rows of a :class:`DispatchOutput`. + +Backends are stateless; the weights belong to the layer and use the transformers v5 layout: +``gate_up_proj`` ``[E_local, 2 * I, H]`` (gate rows first) and ``down_proj`` ``[E_local, H, I]``. +Outputs are not multiplied by the routing weights: the dispatcher's ``combine`` does that. +""" + +from typing import Callable, Protocol + +import torch +import torch.nn.functional as F +from torch import Tensor + +from .dispatcher import DispatchOutput + +__all__ = ["Experts", "GroupedMMExperts", "ReferenceExperts"] + +ActFn = Callable[[Tensor], Tensor] + + +class Experts(Protocol): + def __call__(self, dispatched: DispatchOutput, gate_up_proj: Tensor, down_proj: Tensor, act_fn: ActFn) -> Tensor: + """Return ``[S, H]``, one output row per row of ``dispatched.tokens``, in the same order.""" + + +def _torch_version() -> tuple: + return tuple(int(v) for v in torch.__version__.split("+")[0].split(".")[:2]) + + +class GroupedMMExperts: + """All local experts in two ``torch._grouped_mm`` calls (torch >= 2.8). + + Before torch 2.11, ``_grouped_mm`` needs every group's row count to be a multiple of 8 (16 bytes in bf16), so the + rows are padded per expert and the padding is dropped afterwards. + """ + + def __init__(self): + if _torch_version() < (2, 8) or not hasattr(torch, "_grouped_mm"): + raise RuntimeError(f"GroupedMMExperts needs torch >= 2.8, got {torch.__version__}; use ReferenceExperts") + self.align = 1 if _torch_version() >= (2, 11) else 8 + + def __call__(self, dispatched: DispatchOutput, gate_up_proj: Tensor, down_proj: Tensor, act_fn: ActFn) -> Tensor: + tokens, counts = dispatched.tokens, dispatched.tokens_per_expert + if self.align == 1: + return self._mlp(tokens, counts, gate_up_proj, down_proj, act_fn) + num_experts = counts.numel() + padded_counts = (counts + self.align - 1) // self.align * self.align + start = torch.cumsum(counts, 0) - counts + padded_start = torch.cumsum(padded_counts, 0) - padded_counts + row_expert = torch.arange(num_experts, device=tokens.device).repeat_interleave( + counts, output_size=tokens.size(0) + ) + dest = torch.arange(tokens.size(0), device=tokens.device) - start[row_expert] + padded_start[row_expert] + padded = tokens.new_zeros(tokens.size(0) + (self.align - 1) * num_experts, tokens.size(1)) + padded = padded.index_copy(0, dest, tokens) + return self._mlp(padded, padded_counts, gate_up_proj, down_proj, act_fn)[dest] + + @staticmethod + def _mlp(tokens: Tensor, counts: Tensor, gate_up_proj: Tensor, down_proj: Tensor, act_fn: ActFn) -> Tensor: + offs = torch.cumsum(counts, 0).to(torch.int32) + gate, up = torch._grouped_mm(tokens, gate_up_proj.transpose(1, 2), offs=offs).chunk(2, dim=-1) + return torch._grouped_mm(act_fn(gate) * up, down_proj.transpose(1, 2), offs=offs) + + +class ReferenceExperts: + """One ``F.linear`` pair per expert. Slow; for debugging and as the reference in tests.""" + + def __call__(self, dispatched: DispatchOutput, gate_up_proj: Tensor, down_proj: Tensor, act_fn: ActFn) -> Tensor: + counts = dispatched.tokens_per_expert_cpu + if counts is None: + counts = dispatched.tokens_per_expert.tolist() + outputs = [] + for expert, rows in enumerate(dispatched.tokens.split(counts)): + gate, up = F.linear(rows, gate_up_proj[expert]).chunk(2, dim=-1) + outputs.append(F.linear(act_fn(gate) * up, down_proj[expert])) + if not outputs: + return dispatched.tokens.new_zeros(0, down_proj.size(1)) + return torch.cat(outputs) diff --git a/colossalai/moe/layer.py b/colossalai/moe/layer.py new file mode 100644 index 000000000000..3b517878b5b0 --- /dev/null +++ b/colossalai/moe/layer.py @@ -0,0 +1,179 @@ +"""EPMoELayer: the expert-parallel part of an MoE block, shared by all models. + +An MoE block is split into Router -> dispatch -> experts -> combine. Models keep their own router (top-k selection, +grouped routing, ...) and hand its result to this layer, which runs the other three steps with a pluggable +:class:`~colossalai.moe.dispatcher.Dispatcher` and :class:`~colossalai.moe.experts.Experts` backend. +""" + +import math +from dataclasses import replace +from typing import Optional + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor +from torch.distributed import ProcessGroup + +from colossalai.tensor.moe_tensor.api import ( + ExpertGradScale, + set_expert_grad_scale, + set_moe_tensor_ep_fused, + set_moe_tensor_ep_group, +) + +from ._operation import EPGradScalerIn, EPGradScalerOut +from .dispatcher import Dispatcher, NCCLAllToAllDispatcher +from .experts import ActFn, Experts, GroupedMMExperts + +__all__ = ["EPMoELayer"] + + +class EPMoELayer(nn.Module): + """Routed experts of an MoE block with expert parallelism. + + Each rank of ``ep_group`` holds ``num_experts / ep_size`` consecutive experts as two parameters in the + transformers v5 layout: ``gate_up_proj`` ``[E_local, 2 * intermediate_size, hidden_size]`` and ``down_proj`` + ``[E_local, hidden_size, intermediate_size]``. + + Gradient contract: the gradient of an expert weight on one rank sums the contributions of all tokens the EP group + sent to it. Where its ``1 / ep_size`` factor is applied is recorded on both parameters + (:class:`~colossalai.tensor.moe_tensor.api.ExpertGradScale`): + + - ``IN_FORWARD`` (default): this layer applies it in backward, like the existing EP models, so the current + plugins average expert grads correctly. + - ``IN_REDUCTION``: this layer does not; the gradient reduction must divide expert grads by + ``replica_group_size * ep_size``. Only use it with a plugin that reads this tag, or expert grads are ``ep_size`` + times too large. + + Checkpoints: both parameters are marked ``ep_fused``, so ``MoECheckpointIO`` gathers the experts of all EP ranks + into the full ``[num_experts, ...]`` tensors when saving, and loading such a tensor keeps this rank's experts only. + + Args: + num_experts: total number of routed experts. + hidden_size: model hidden size. + intermediate_size: hidden size of each expert MLP. + ep_group: expert parallel group; ``None`` means the default group. + act_fn: activation on the gate projection (SwiGLU: ``act_fn(gate) * up``). + dispatcher: defaults to :class:`~colossalai.moe.dispatcher.NCCLAllToAllDispatcher`. + experts: defaults to :class:`~colossalai.moe.experts.GroupedMMExperts`. + expert_grad_scale: see the gradient contract above. + """ + + def __init__( + self, + num_experts: int, + hidden_size: int, + intermediate_size: int, + ep_group: Optional[ProcessGroup] = None, + act_fn: ActFn = F.silu, + dispatcher: Optional[Dispatcher] = None, + experts: Optional[Experts] = None, + expert_grad_scale: ExpertGradScale = ExpertGradScale.IN_FORWARD, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__() + if ep_group is None and dist.is_initialized(): + ep_group = dist.group.WORLD + self.ep_group = ep_group + self.ep_size = dist.get_world_size(ep_group) if dist.is_initialized() else 1 + self.ep_rank = dist.get_rank(ep_group) if dist.is_initialized() else 0 + if num_experts % self.ep_size != 0: + raise ValueError(f"num_experts ({num_experts}) must be divisible by the ep size ({self.ep_size})") + self.num_experts = num_experts + self.num_local_experts = num_experts // self.ep_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.act_fn = act_fn + self.expert_grad_scale = ExpertGradScale(expert_grad_scale) + self.dispatcher = dispatcher if dispatcher is not None else NCCLAllToAllDispatcher(num_experts, ep_group) + self.experts = experts if experts is not None else GroupedMMExperts() + + factory = dict(device=device, dtype=dtype) + self.gate_up_proj = nn.Parameter( + torch.empty(self.num_local_experts, 2 * intermediate_size, hidden_size, **factory) + ) + self.down_proj = nn.Parameter(torch.empty(self.num_local_experts, hidden_size, intermediate_size, **factory)) + self.reset_parameters() + for param in (self.gate_up_proj, self.down_proj): + set_moe_tensor_ep_group(param, ep_group) + set_moe_tensor_ep_fused(param) + set_expert_grad_scale(param, self.expert_grad_scale) + self.register_load_state_dict_pre_hook(self._slice_full_experts) + + @property + def local_expert_range(self) -> range: + """Global ids of the experts held by this rank.""" + start = self.ep_rank * self.num_local_experts + return range(start, start + self.num_local_experts) + + def reset_parameters(self) -> None: + # each expert initialised like nn.Linear + for weight in (self.gate_up_proj, self.down_proj): + bound = 1 / math.sqrt(weight.size(-1)) + nn.init.uniform_(weight, -bound, bound) + + @torch.no_grad() + def load_full_weights(self, gate_up_proj: Tensor, down_proj: Tensor) -> None: + """Copy this rank's experts out of the weights of all experts (``[E, 2I, H]`` and ``[E, H, I]``). + + Only ``.data`` is written, so the parameter objects that optimizers and ZeRO already hold stay valid. + """ + local = slice(self.local_expert_range.start, self.local_expert_range.stop) + self.gate_up_proj.data.copy_(gate_up_proj[local]) + self.down_proj.data.copy_(down_proj[local]) + + def _slice_full_experts( + self, module, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ) -> None: + """Slice a full ``[num_experts, ...]`` checkpoint tensor down to this rank's experts before loading. + + A tensor that already has the local shape is loaded as is; any other shape is left for ``load_state_dict`` to + report. + """ + experts = self.local_expert_range + for name in ("gate_up_proj", "down_proj"): + key = prefix + name + checkpoint_param = state_dict.get(key) + local_param = getattr(self, name) + if checkpoint_param is None or checkpoint_param.shape == local_param.shape: + continue + if checkpoint_param.ndim == 0: + continue + if checkpoint_param.shape[0] == self.num_experts and checkpoint_param.shape[1:] == local_param.shape[1:]: + state_dict[key] = checkpoint_param[experts.start : experts.stop].contiguous() + + def forward(self, hidden_states: Tensor, topk_idx: Tensor, topk_weights: Tensor) -> Tensor: + """Run the routed experts. + + Args: + hidden_states: ``[..., hidden_size]`` input of the MoE block. + topk_idx: ``[..., top_k]`` global expert ids chosen by the router for each token. + topk_weights: ``[..., top_k]`` routing weights; they are combined in fp32, so weights that the router + already rounded to the model dtype keep that rounding. Gradients flow back through them. + + Returns: + Tensor of the shape and dtype of ``hidden_states``: per token, the weighted sum of its experts' outputs. + """ + shape = hidden_states.shape + top_k = topk_idx.size(-1) + dispatched = self.dispatcher.dispatch( + hidden_states.reshape(-1, self.hidden_size), topk_idx.reshape(-1, top_k), topk_weights.reshape(-1, top_k) + ) + scale_in_forward = self.expert_grad_scale == ExpertGradScale.IN_FORWARD and self.ep_size > 1 + if scale_in_forward: + # backward: divide the expert weight grads by ep_size, leave the input grads unchanged + dispatched = replace(dispatched, tokens=EPGradScalerIn.apply(dispatched.tokens, self.ep_size)) + output = self.experts(dispatched, self.gate_up_proj, self.down_proj, self.act_fn) + if scale_in_forward: + output = EPGradScalerOut.apply(output, self.ep_size) + return self.dispatcher.combine(output, dispatched).view(shape) + + def extra_repr(self) -> str: + return ( + f"num_experts={self.num_experts}, num_local_experts={self.num_local_experts}, " + f"hidden_size={self.hidden_size}, intermediate_size={self.intermediate_size}, ep_size={self.ep_size}, " + f"expert_grad_scale={self.expert_grad_scale.value}" + ) diff --git a/colossalai/shardformer/modeling/_qwen3_moe_tf4.py b/colossalai/shardformer/modeling/_qwen3_moe_tf4.py new file mode 100644 index 000000000000..e0ccd0540085 --- /dev/null +++ b/colossalai/shardformer/modeling/_qwen3_moe_tf4.py @@ -0,0 +1,12 @@ +__all__ = ["Qwen3MoeEPExperts"] + +_MESSAGE = "Qwen3-MoE sharding needs transformers>=5.17 (the v4 layout has no fused experts)." + + +class Qwen3MoeEPExperts: + def __init__(self, *args, **kwargs): + raise NotImplementedError(_MESSAGE) + + @staticmethod + def from_native_module(*args, **kwargs): + raise NotImplementedError(_MESSAGE) diff --git a/colossalai/shardformer/modeling/_qwen3_moe_tf5.py b/colossalai/shardformer/modeling/_qwen3_moe_tf5.py new file mode 100644 index 000000000000..e1cc40eeea65 --- /dev/null +++ b/colossalai/shardformer/modeling/_qwen3_moe_tf5.py @@ -0,0 +1,82 @@ +r"""Qwen3-MoE sharding helpers for transformers v5. + +``Qwen3MoeEPExperts`` turns the fused ``Qwen3MoeExperts`` of a sparse block into the shared +:class:`~colossalai.moe.EPMoELayer`. The router and the rest of the block stay the native HF modules: v5 calls +``self.experts(hidden_states, top_k_index, top_k_weights)``, which is exactly the signature of the layer, and the layer +keeps the parameter names ``gate_up_proj`` / ``down_proj``. +""" + +import warnings + +import torch +from torch.distributed import ProcessGroup +from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeExperts + +from colossalai.lazy import LazyInitContext +from colossalai.moe import EPMoELayer, GroupedMMExperts, ReferenceExperts +from colossalai.tensor.moe_tensor.api import set_expert_grad_scale, set_moe_tensor_ep_fused, set_moe_tensor_ep_group + +__all__ = ["Qwen3MoeEPExperts"] + + +def _shard_param_in_place(param: torch.nn.Parameter, local: torch.Tensor) -> None: + """Swap a parameter's storage for its local shard while keeping the Parameter object. + + Optimizers are built from the unsharded model and the plugins only prune (never re-add) optimizer params, so + replacing the Parameter object would silently drop the experts from training. + """ + param.data = local + + +class Qwen3MoeEPExperts: + r"""Factory used by the :class:`Sharder`: ``from_native_module`` returns an ``EPMoELayer``. + + The returned layer holds the *original* ``gate_up_proj`` / ``down_proj`` Parameter objects, whose storage is swapped + for this rank's experts in place (see :func:`_shard_param_in_place`). + """ + + def __init__(self, *args, **kwargs): + raise RuntimeError(f"Please use `from_native_module` to create an instance of {self.__class__.__name__}") + + @staticmethod + def from_native_module( + module: Qwen3MoeExperts, + process_group: ProcessGroup = None, + ep_group: ProcessGroup = None, + **kwargs, + ) -> EPMoELayer: + del process_group # the tensor parallel group: the experts are not tensor parallel + assert ep_group is not None, "Qwen3MoeEPExperts needs the expert parallel group" + LazyInitContext.materialize(module) + + try: + experts = GroupedMMExperts() + except RuntimeError: # torch < 2.8 has no torch._grouped_mm + warnings.warn("torch._grouped_mm is not available, falling back to the slower per-expert loop.") + experts = ReferenceExperts() + # built on the meta device only to get the dispatcher, the backend and the bookkeeping (which experts this rank + # holds); its parameters are replaced by the (sharded) original ones below + layer = EPMoELayer( + module.num_experts, + module.hidden_dim, + module.intermediate_dim, + ep_group=ep_group, + act_fn=module.act_fn, + experts=experts, + device="meta", + ) + + gate_up_proj, down_proj = module.gate_up_proj, module.down_proj + local = layer.local_expert_range + _shard_param_in_place(gate_up_proj, gate_up_proj.data[local.start : local.stop].clone()) + _shard_param_in_place(down_proj, down_proj.data[local.start : local.stop].clone()) + layer.gate_up_proj = gate_up_proj + layer.down_proj = down_proj + for param in (gate_up_proj, down_proj): + set_moe_tensor_ep_group(param, ep_group) + set_expert_grad_scale(param, layer.expert_grad_scale) + # checkpoints: gather the experts of all EP ranks when saving (MoECheckpointIO); the layer's own load hook + # slices a full ``[num_experts, ...]`` tensor down to this rank's experts and reads the parameters at call + # time, so it also serves the original Parameter objects assigned above + set_moe_tensor_ep_fused(param) + return layer diff --git a/colossalai/shardformer/modeling/qwen3_moe.py b/colossalai/shardformer/modeling/qwen3_moe.py new file mode 100644 index 000000000000..ca762b89929b --- /dev/null +++ b/colossalai/shardformer/modeling/qwen3_moe.py @@ -0,0 +1,9 @@ +r"""Qwen3-MoE sharding helpers, selected by the installed transformers major version. + +Only ``_qwen3_moe_tf5`` exists for transformers v5; ``_qwen3_moe_tf4`` is a stub that fails with a clear message. See +:func:`colossalai._compat.reexport_transformers_impl`. +""" + +from colossalai._compat import reexport_transformers_impl + +reexport_transformers_impl(globals(), __package__, "_qwen3_moe") diff --git a/colossalai/shardformer/policies/_qwen3_moe_tf4.py b/colossalai/shardformer/policies/_qwen3_moe_tf4.py new file mode 100644 index 000000000000..4100f05382b1 --- /dev/null +++ b/colossalai/shardformer/policies/_qwen3_moe_tf4.py @@ -0,0 +1,16 @@ +__all__ = ["Qwen3MoePolicy", "Qwen3MoeModelPolicy", "Qwen3MoeForCausalLMPolicy"] + +_MESSAGE = "Qwen3-MoE sharding needs transformers>=5.17 (the v4 layout has no fused experts)." + + +class Qwen3MoePolicy: + def __init__(self, *args, **kwargs): + raise NotImplementedError(_MESSAGE) + + +class Qwen3MoeModelPolicy(Qwen3MoePolicy): + pass + + +class Qwen3MoeForCausalLMPolicy(Qwen3MoePolicy): + pass diff --git a/colossalai/shardformer/policies/_qwen3_moe_tf5.py b/colossalai/shardformer/policies/_qwen3_moe_tf5.py new file mode 100644 index 000000000000..0741f89ac705 --- /dev/null +++ b/colossalai/shardformer/policies/_qwen3_moe_tf5.py @@ -0,0 +1,228 @@ +import warnings +from typing import Dict, List, Union + +import torch.distributed as dist +import torch.nn as nn +from torch import Tensor +from torch.nn import Module +from transformers.models.qwen3_moe.modeling_qwen3_moe import ( + Qwen3MoeAttention, + Qwen3MoeDecoderLayer, + Qwen3MoeForCausalLM, + Qwen3MoeModel, +) + +from colossalai.shardformer.layer import ( + FusedRMSNorm, + Linear1D_Col, + Linear1D_Row, + PaddingEmbedding, + VocabParallelEmbedding1D, +) + +from ..modeling.qwen3_moe import Qwen3MoeEPExperts +from .base_policy import ModulePolicyDescription, Policy, SubModuleReplacementDescription + +__all__ = ["Qwen3MoePolicy", "Qwen3MoeModelPolicy", "Qwen3MoeForCausalLMPolicy"] + + +class Qwen3MoePolicy(Policy): + r"""Sharding policy of the transformers v5 Qwen3-MoE. + + - Attention, the dense ``Qwen3MoeMLP`` layers (``mlp_only_layers``), the embedding and the LM head follow the dense + Qwen3 policy. + - The router (``mlp.gate``) and the sparse block itself stay the native HF modules, so that v5's + ``OutputRecorder(Qwen3MoeTopKRouter)`` still records the router logits. + - With ``shard_config.ep_group`` the fused experts of every sparse block become a :class:`~colossalai.moe.EPMoELayer`. + + Not supported yet: pipeline parallelism, sequence parallelism, and expert parallelism together with tensor + parallelism (the shared layer has no tensor-parallel experts). + """ + + def __init__(self) -> None: + super().__init__() + import transformers + from packaging.version import Version + + assert Version(transformers.__version__) >= Version( + "5.17.0" + ), "The Qwen3-MoE model should run on a transformers version of 5.17.0 or higher." + + def config_sanity_check(self): + pass + + def preprocess(self): + self.tie_weight = self.tie_weight_check() + self.origin_attn_implement = self.model.config._attn_implementation + return self.model + + def module_policy(self) -> Dict[Union[str, nn.Module], ModulePolicyDescription]: + shard_config = self.shard_config + if self.pipeline_stage_manager is not None: + raise NotImplementedError("Pipeline parallelism is not supported for Qwen3-MoE yet.") + if shard_config.enable_sequence_parallelism: + raise NotImplementedError("Sequence parallelism is not supported for Qwen3-MoE yet.") + tp_size = shard_config.tensor_parallel_size if shard_config.enable_tensor_parallelism else 1 + if shard_config.ep_group and tp_size > 1: + raise NotImplementedError( + "Expert parallelism together with tensor parallelism is not supported for Qwen3-MoE yet: " + "the shared EPMoELayer has no tensor-parallel experts." + ) + + policy = {} + config = self.model.config + + embedding_cls = None + if shard_config.enable_tensor_parallelism: + embedding_cls = VocabParallelEmbedding1D + elif self.tie_weight: + embedding_cls = PaddingEmbedding + + if shard_config.enable_tensor_parallelism: + assert ( + config.num_attention_heads % tp_size == 0 + ), "The number of attention heads must be divisible by tensor parallel size." + assert ( + config.num_key_value_heads % tp_size == 0 + ), "The number of key_value heads must be divisible by tensor parallel size." + fp8 = shard_config.fp8_communication + # no attribute replacement: transformers v5 attention infers the local number of heads from the shape of + # its projections (`view(..., -1, head_dim)`) and does not read `num_heads` / `hidden_size` + policy[Qwen3MoeDecoderLayer] = ModulePolicyDescription( + sub_module_replacement=[ + SubModuleReplacementDescription( + suffix="self_attn.q_proj", target_module=Linear1D_Col, kwargs=dict(fp8_communication=fp8) + ), + SubModuleReplacementDescription( + suffix="self_attn.k_proj", target_module=Linear1D_Col, kwargs=dict(fp8_communication=fp8) + ), + SubModuleReplacementDescription( + suffix="self_attn.v_proj", target_module=Linear1D_Col, kwargs=dict(fp8_communication=fp8) + ), + SubModuleReplacementDescription( + suffix="self_attn.o_proj", target_module=Linear1D_Row, kwargs=dict(fp8_communication=fp8) + ), + # The dense Qwen3MoeMLP of `mlp_only_layers`. Sparse layers have no such submodules, so skip them + # there; their experts are replicated over the tensor parallel group. + SubModuleReplacementDescription( + suffix="mlp.gate_proj", + target_module=Linear1D_Col, + kwargs=dict(fp8_communication=fp8), + ignore_if_not_exist=True, + ), + SubModuleReplacementDescription( + suffix="mlp.up_proj", + target_module=Linear1D_Col, + kwargs=dict(fp8_communication=fp8), + ignore_if_not_exist=True, + ), + SubModuleReplacementDescription( + suffix="mlp.down_proj", + target_module=Linear1D_Row, + kwargs=dict(fp8_communication=fp8), + ignore_if_not_exist=True, + ), + ], + ) + + if embedding_cls is not None: + self.append_or_create_submodule_replacement( + description=SubModuleReplacementDescription( + suffix="embed_tokens", + target_module=embedding_cls, + kwargs=( + { + "make_vocab_size_divisible_by": shard_config.make_vocab_size_divisible_by, + "fp8_communication": shard_config.fp8_communication, + } + if shard_config.enable_tensor_parallelism + else {"make_vocab_size_divisible_by": shard_config.make_vocab_size_divisible_by} + ), + ), + policy=policy, + target_key=Qwen3MoeModel, + ) + + if shard_config.ep_group: + # Only the experts are replaced: dense layers have no `mlp.experts` and are skipped, and the router keeps + # its native class for the router-logits hook of transformers v5. + self.append_or_create_submodule_replacement( + description=SubModuleReplacementDescription( + suffix="mlp.experts", + target_module=Qwen3MoeEPExperts, + kwargs={"ep_group": shard_config.ep_group}, + ignore_if_not_exist=True, + ), + policy=policy, + target_key=Qwen3MoeDecoderLayer, + ) + + if shard_config.enable_fused_normalization: + self.append_or_create_submodule_replacement( + description=[ + SubModuleReplacementDescription(suffix="input_layernorm", target_module=FusedRMSNorm), + SubModuleReplacementDescription(suffix="post_attention_layernorm", target_module=FusedRMSNorm), + ], + policy=policy, + target_key=Qwen3MoeDecoderLayer, + ) + self.append_or_create_submodule_replacement( + description=SubModuleReplacementDescription(suffix="norm", target_module=FusedRMSNorm), + policy=policy, + target_key=Qwen3MoeModel, + ) + + if shard_config.enable_flash_attention: + warnings.warn("Flash attention is natively supported in transformers, will ignore the flag.") + shard_config.enable_flash_attention = False + + return policy + + def postprocess(self): + if self.shard_config.enable_tensor_parallelism and self.shard_config.tensor_parallel_size > 1: + tp_group = self.shard_config.tensor_parallel_process_group + + def sync_head_grad(grad): + # Q/K norm weights are shared by the locally partitioned heads. + grad = grad.clone() + dist.all_reduce(grad, group=tp_group) + return grad + + for module in self.model.modules(): + if isinstance(module, Qwen3MoeAttention): + for norm in (module.q_norm, module.k_norm): + weight = norm.weight + if ( + weight is not None + and weight.requires_grad + and not getattr(weight, "_qwen3_moe_tp_grad_hook", False) + ): + weight.register_hook(sync_head_grad) + weight._qwen3_moe_tp_grad_hook = True + return self.model + + def get_held_layers(self) -> List[Module]: + raise NotImplementedError("Pipeline parallelism is not supported for Qwen3-MoE yet.") + + def get_shared_params(self) -> List[Dict[int, Tensor]]: + return [] + + +class Qwen3MoeModelPolicy(Qwen3MoePolicy): + pass + + +class Qwen3MoeForCausalLMPolicy(Qwen3MoePolicy): + def module_policy(self): + policy = super().module_policy() + if self.shard_config.enable_tensor_parallelism: + policy[Qwen3MoeForCausalLM] = ModulePolicyDescription( + sub_module_replacement=[ + SubModuleReplacementDescription( + suffix="lm_head", + target_module=Linear1D_Col, + kwargs=dict(gather_output=True, fp8_communication=self.shard_config.fp8_communication), + ) + ], + ) + return policy diff --git a/colossalai/shardformer/policies/auto_policy.py b/colossalai/shardformer/policies/auto_policy.py index 80c3ab5f05f9..51d31dca90f2 100644 --- a/colossalai/shardformer/policies/auto_policy.py +++ b/colossalai/shardformer/policies/auto_policy.py @@ -256,6 +256,13 @@ class PolicyLocation: "transformers.models.qwen3.modeling_qwen3.Qwen3ForSequenceClassification": PolicyLocation( file_name="qwen3", class_name="Qwen3ForSequenceClassificationPolicy" ), + # Qwen3-MoE + "transformers.models.qwen3_moe.modeling_qwen3_moe.Qwen3MoeModel": PolicyLocation( + file_name="qwen3_moe", class_name="Qwen3MoeModelPolicy" + ), + "transformers.models.qwen3_moe.modeling_qwen3_moe.Qwen3MoeForCausalLM": PolicyLocation( + file_name="qwen3_moe", class_name="Qwen3MoeForCausalLMPolicy" + ), # command "transformers.models.cohere.modeling_cohere.CohereModel": PolicyLocation( file_name="command", class_name="CommandModelPolicy" diff --git a/colossalai/shardformer/policies/qwen3_moe.py b/colossalai/shardformer/policies/qwen3_moe.py new file mode 100644 index 000000000000..cb3875100898 --- /dev/null +++ b/colossalai/shardformer/policies/qwen3_moe.py @@ -0,0 +1,9 @@ +r"""Qwen3-MoE sharding policies, selected by the installed transformers major version. + +Only ``_qwen3_moe_tf5`` exists for transformers v5; ``_qwen3_moe_tf4`` is a stub that fails with a clear message. See +:func:`colossalai._compat.reexport_transformers_impl`. +""" + +from colossalai._compat import reexport_transformers_impl + +reexport_transformers_impl(globals(), __package__, "_qwen3_moe") diff --git a/colossalai/tensor/moe_tensor/api.py b/colossalai/tensor/moe_tensor/api.py index a565ecf34eb1..dc010fb5cd3c 100644 --- a/colossalai/tensor/moe_tensor/api.py +++ b/colossalai/tensor/moe_tensor/api.py @@ -1,4 +1,5 @@ -from typing import List +from enum import Enum +from typing import List, Optional import torch import torch.distributed as dist @@ -57,6 +58,46 @@ def is_moe_tensor_ep_fused(tensor: torch.Tensor) -> bool: return getattr(tensor, "ep_fused", False) +class ExpertGradScale(str, Enum): + """Where the ``1 / ep_size`` factor of an expert parameter's gradient is applied. + + The gradient of an expert weight on one rank sums the contributions of every token the EP group routed to it, so + averaging it over the data-parallel replicas needs one more division by ``ep_size``. + """ + + # inside the layer's autograd (EPGradScaler); gradient reduction divides by the replica group size only + IN_FORWARD = "in_forward" + # not in the layer; gradient reduction divides by replica group size * ep_size + IN_REDUCTION = "in_reduction" + + +def set_expert_grad_scale(tensor: torch.Tensor, scale: ExpertGradScale) -> None: + """Record where the ``1 / ep_size`` factor of this expert parameter's gradient is applied. + + Args: + tensor (torch.Tensor): An expert parameter, already marked with :func:`set_moe_tensor_ep_group`. + scale (ExpertGradScale): Where the factor is applied. + """ + assert is_moe_tensor(tensor), "call set_moe_tensor_ep_group first" + tensor.__setattr__("expert_grad_scale", ExpertGradScale(scale)) + + +def get_expert_grad_scale(tensor: torch.Tensor) -> Optional[ExpertGradScale]: + """ + Get where the ``1 / ep_size`` factor of the given parameter's gradient is applied. + + Args: + tensor (torch.Tensor): The tensor to be checked. + + Returns: + Optional[ExpertGradScale]: ``None`` for non-expert tensors. Expert tensors without a tag are treated as + ``IN_FORWARD``, which is what the existing EP models do. + """ + if not is_moe_tensor(tensor): + return None + return getattr(tensor, "expert_grad_scale", ExpertGradScale.IN_FORWARD) + + def get_moe_info(ep_size: int, dp_size: int, pp_size: int, ep_inside: bool) -> MoeParallelInfo: """ Get moe info for the given tensor. diff --git a/colossalai/tensor/moe_tensor/grad_sync.py b/colossalai/tensor/moe_tensor/grad_sync.py new file mode 100644 index 000000000000..b240f9ca7898 --- /dev/null +++ b/colossalai/tensor/moe_tensor/grad_sync.py @@ -0,0 +1,68 @@ +"""Gradient synchronization plan of the MoE plugin. + +The model adaptation layer only tags what a parameter is: an expert or not and, for an expert, where its ``1 / ep_size`` +factor is applied (:class:`~colossalai.tensor.moe_tensor.api.ExpertGradScale`). The plugin combines those tags with the +process groups of the parallel topology into one :class:`GradSyncSpec` per parameter, and every reducer reads that +plan instead of classifying parameters on its own. + +The contract is ``final grad = SUM(grad over replica_group) / sum_divisor``. The size of the group and the divisor are +different numbers: an expert whose layer leaves ``1 / ep_size`` to the reduction is summed over the ``moe_dp`` group but +divided by ``moe_dp_size * ep_size``. +""" + +from dataclasses import dataclass +from typing import Dict, Iterable + +import torch.distributed as dist +import torch.nn as nn +from torch.distributed import ProcessGroup + +from .api import ExpertGradScale, get_ep_size, get_expert_grad_scale, is_moe_tensor + +__all__ = ["GradSyncSpec", "build_grad_sync_plan", "has_reduction_scaled_experts"] + + +@dataclass(frozen=True) +class GradSyncSpec: + """How the gradient of one parameter is reduced. + + Args: + replica_group: the ranks that hold replicas of the parameter, i.e. where the gradients are summed. + sum_divisor: the number the sum is divided by. + """ + + replica_group: ProcessGroup + sum_divisor: float + + @property + def extra_divisor(self) -> float: + """What is left to divide by once the gradient has been averaged over ``replica_group``.""" + return self.sum_divisor / dist.get_world_size(self.replica_group) + + +def build_grad_sync_plan( + params: Iterable[nn.Parameter], dp_group: ProcessGroup, moe_dp_group: ProcessGroup +) -> Dict[nn.Parameter, GradSyncSpec]: + """Map each parameter to its :class:`GradSyncSpec`. + + Args: + params: the parameters of the model, after the model has been sharded and moved to its device. + dp_group: replica group of the dense parameters, the router and the shared experts (``mixed_dp_group``). + moe_dp_group: replica group of the routed experts (``mixed_moe_dp_group``). + """ + dp_size = dist.get_world_size(dp_group) + moe_dp_size = dist.get_world_size(moe_dp_group) + plan = {} + for param in params: + if not is_moe_tensor(param): + plan[param] = GradSyncSpec(dp_group, dp_size) + elif get_expert_grad_scale(param) == ExpertGradScale.IN_REDUCTION: + plan[param] = GradSyncSpec(moe_dp_group, moe_dp_size * get_ep_size(param)) + else: # IN_FORWARD (also the default of untagged experts): the layer already applied 1 / ep_size + plan[param] = GradSyncSpec(moe_dp_group, moe_dp_size) + return plan + + +def has_reduction_scaled_experts(params: Iterable[nn.Parameter]) -> bool: + """Whether some parameter leaves the ``1 / ep_size`` of its gradient to the gradient reduction.""" + return any(is_moe_tensor(p) and get_expert_grad_scale(p) == ExpertGradScale.IN_REDUCTION for p in params) diff --git a/colossalai/zero/low_level/low_level_optim.py b/colossalai/zero/low_level/low_level_optim.py index 9f0d00afd925..bd4c0165cda1 100644 --- a/colossalai/zero/low_level/low_level_optim.py +++ b/colossalai/zero/low_level/low_level_optim.py @@ -23,6 +23,7 @@ from colossalai.logging import get_dist_logger from colossalai.quantization.fp8 import all_gather_fp8, all_reduce_fp8, reduce_scatter_fp8 from colossalai.tensor.moe_tensor.api import is_moe_tensor +from colossalai.tensor.moe_tensor.grad_sync import GradSyncSpec from ._utils import ( all_gather_into_flat_tensor_nd, @@ -102,6 +103,7 @@ def __init__( fp8_communication: bool = False, backward_context=None, grad_norm_process_group: Optional[Union[ProcessGroup, Tuple[ProcessGroup, ...]]] = None, + grad_sync_plan: Optional[Dict[nn.Parameter, GradSyncSpec]] = None, ): super(LowLevelZeroOptimizer, self).__init__(optim=optimizer) @@ -204,6 +206,17 @@ def __init__( self.pg_to_bucket_store = {pg: BucketStore(pg, reduce_bucket_size) for pg in self.pg_to_param_list} # param id to bucket store, have to use id(param) as key since it is used in stores self.pid_to_bucket_store = {id(param): self.pg_to_bucket_store[param_to_pg[param]] for param in param_to_pg} + # Gradients are averaged over the store's group; a parameter whose sync plan divides by more (an expert that + # leaves 1 / ep_size to the reduction) is divided by the rest when its gradient enters the bucket. + self._grad_pre_divisor = {} + for param, spec in (grad_sync_plan or {}).items(): + if param not in param_to_pg: + continue + if spec.replica_group is not param_to_pg[param]: + raise ValueError("The replica group of the gradient sync plan differs from the ZeRO group of a param.") + extra = spec.sum_divisor / self.pg_to_bucket_store[param_to_pg[param]].world_size + if extra != 1: + self._grad_pre_divisor[id(param)] = extra # iterate over the param group in the optimizer # partition these param groups for data parallel training @@ -453,6 +466,10 @@ def _add_to_bucket(self, param, group_id): ): self._run_reduction() + pre_divisor = self._grad_pre_divisor.get(id(param)) + if pre_divisor is not None: + param.grad.div_(pre_divisor) + padding_size = self.get_param_padding_size(param) self.pid_to_bucket_store[id(param)].add_param_grad(group_id, param, padding_size) diff --git a/tests/kit/model_zoo/transformers/__init__.py b/tests/kit/model_zoo/transformers/__init__.py index a8459cc0af2d..4a92aa70e463 100644 --- a/tests/kit/model_zoo/transformers/__init__.py +++ b/tests/kit/model_zoo/transformers/__init__.py @@ -15,6 +15,7 @@ from .opt import * from .qwen2 import * from .qwen3 import * +from .qwen3_moe import * from .sam import * from .t5 import * from .vit import * diff --git a/tests/kit/model_zoo/transformers/qwen3_moe.py b/tests/kit/model_zoo/transformers/qwen3_moe.py new file mode 100644 index 000000000000..cd19aca02b0a --- /dev/null +++ b/tests/kit/model_zoo/transformers/qwen3_moe.py @@ -0,0 +1,79 @@ +import torch +import transformers + +from ..registry import ModelAttribute, model_zoo + +try: + from transformers import Qwen3MoeConfig + + HAS_QWEN3_MOE = True +except ImportError: + HAS_QWEN3_MOE = False + + +if HAS_QWEN3_MOE: + + def data_gen(): + input_ids = torch.tensor([[1, 22, 55, 77, 53, 34, 43, 22]], dtype=torch.long) + return dict(input_ids=input_ids, attention_mask=torch.ones_like(input_ids)) + + def data_gen_for_causal_lm(): + data = data_gen() + data["labels"] = data["input_ids"].clone() + return data + + output_transform_fn = lambda x: x + loss_fn = lambda output: output["last_hidden_state"].mean() + loss_fn_for_causal_lm = lambda output: output["loss"] + + config = Qwen3MoeConfig( + vocab_size=128, + hidden_size=32, + intermediate_size=64, + moe_intermediate_size=16, + num_hidden_layers=2, + num_attention_heads=2, + num_key_value_heads=1, + head_dim=16, + num_experts=8, + num_experts_per_tok=2, + norm_topk_prob=True, + mlp_only_layers=[1], + max_position_embeddings=64, + use_cache=False, + attention_dropout=0.0, + tie_word_embeddings=False, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + experts_implementation="eager", + attn_implementation="eager", + output_router_logits=True, + ) + config.pad_token_id = 0 + + def is_router(name, module): + return name.endswith("mlp.gate") + + attributes = ModelAttribute(has_control_flow=True) + if hasattr(attributes, "numerics"): + from tests.kit.numerics.spec import NumericsSpec, Routers + + attributes.numerics = NumericsSpec(routers=Routers(select=is_router, top_k=config.num_experts_per_tok)) + + model_zoo.register( + name="transformers_qwen3_moe", + model_fn=lambda: transformers.Qwen3MoeModel(config), + data_gen_fn=data_gen, + output_transform_fn=output_transform_fn, + loss_fn=loss_fn, + model_attribute=attributes, + ) + model_zoo.register( + name="transformers_qwen3_moe_for_causal_lm", + model_fn=lambda: transformers.Qwen3MoeForCausalLM(config), + data_gen_fn=data_gen_for_causal_lm, + output_transform_fn=output_transform_fn, + loss_fn=loss_fn_for_causal_lm, + model_attribute=attributes, + ) diff --git a/tests/test_moe/test_ep_moe_layer.py b/tests/test_moe/test_ep_moe_layer.py new file mode 100644 index 000000000000..a1c35b1fcf07 --- /dev/null +++ b/tests/test_moe/test_ep_moe_layer.py @@ -0,0 +1,204 @@ +import pytest +import torch +import torch.distributed as dist +import torch.nn.functional as F + +import colossalai +from colossalai.moe import EPMoELayer, GroupedMMExperts, NCCLAllToAllDispatcher, ReferenceExperts +from colossalai.tensor.moe_tensor.api import ( + ExpertGradScale, + get_ep_group, + get_ep_size, + get_expert_grad_scale, + is_moe_tensor, +) +from colossalai.testing import parameterize, rerun_if_address_is_in_use, spawn + +# Relative L2 error allowed between the bf16 layer and an fp32 single-GPU reference. The largest error over all checks +# here is 6e-3; dropping the EP grad scaler, for example, gives 0.9. +RTOL = 2e-2 + +NUM_EXPERTS, TOP_K, HIDDEN_SIZE, INTERMEDIATE_SIZE, NUM_TOKENS = 8, 2, 64, 32, 64 + + +def ep_groups(ep_size: int): + """Split the world into consecutive groups of ep_size ranks; return this rank's group.""" + world, rank = dist.get_world_size(), dist.get_rank() + mine = None + for start in range(0, world, ep_size): + group = dist.new_group(list(range(start, start + ep_size))) + if start <= rank < start + ep_size: + mine = group + return mine + + +def route(x: torch.Tensor, gate: torch.Tensor, bias: torch.Tensor): + """Softmax top-k router with renormalised fp32 weights (Mixtral / Qwen style).""" + probs = (F.linear(x.float(), gate) + bias).softmax(-1) + weights, idx = torch.topk(probs, TOP_K, dim=-1) + return idx, weights / weights.sum(-1, keepdim=True) + + +def rel(a: torch.Tensor, b: torch.Tensor) -> float: + return ((a.float() - b.float()).norm() / b.float().norm().clamp_min(1e-12)).item() + + +@parameterize("ep_size", [1, 2, 4]) +def run_dispatch_contract(ep_size: int): + """Rows reach the right experts, grouped by expert and in global token order; combine undoes dispatch.""" + group = ep_groups(ep_size) + ep_rank = dist.get_rank(group) + dispatcher = NCCLAllToAllDispatcher(NUM_EXPERTS, group) + num_local = NUM_EXPERTS // ep_size + + # ep rank 1 sends nothing, and expert 1 is never chosen + num_tokens = 0 if ep_rank == 1 else NUM_TOKENS + generator = torch.Generator(device="cuda").manual_seed(dist.get_rank()) + choices = torch.tensor([e for e in range(NUM_EXPERTS) if e != 1], device="cuda") + scores = torch.rand(num_tokens, choices.numel(), device="cuda", generator=generator) + topk_idx = choices[scores.argsort(-1)[:, :TOP_K]] + topk_weights = torch.rand(num_tokens, TOP_K, device="cuda", generator=generator) + topk_weights = topk_weights / topk_weights.sum(-1, keepdim=True) + x = torch.randn(num_tokens, HIDDEN_SIZE, device="cuda", generator=generator) + x[:, 0] = torch.arange(num_tokens, device="cuda") + ep_rank * NUM_TOKENS # global token id within the ep group + + dispatched = dispatcher.dispatch(x, topk_idx, topk_weights) + counts = dispatched.tokens_per_expert.tolist() + assert dispatched.tokens_per_expert_cpu == counts + assert dispatched.tokens.size(0) == sum(counts) + + # expected rows of each local expert: ids of the tokens that chose it, in ep-group token order + gathered = [None] * ep_size + dist.all_gather_object(gathered, (x[:, 0].long().tolist(), topk_idx.tolist()), group=group) + for local, rows in enumerate(dispatched.tokens.split(counts)): + expert = ep_rank * num_local + local + expected = [tid for ids, idx in gathered for tid, chosen in zip(ids, idx) if expert in chosen] + assert rows[:, 0].long().tolist() == expected, f"expert {expert}: rows out of place" + + # identity experts: combine must give back sum_j w_j * x = x + y = dispatcher.combine(dispatched.tokens, dispatched) + assert y.shape == x.shape + torch.testing.assert_close(y, x, rtol=1e-6, atol=1e-6) + + +@parameterize("ep_size", [1, 2, 4]) +@parameterize("backend", ["grouped_mm", "reference"]) +@parameterize("grad_scale", [ExpertGradScale.IN_FORWARD, ExpertGradScale.IN_REDUCTION]) +@parameterize("skewed", [False, True]) +def run_layer_vs_reference(ep_size: int, backend: str, grad_scale: ExpertGradScale, skewed: bool): + """Output, input grad and expert weight grads against an fp32 single-GPU computation.""" + group = ep_groups(ep_size) + experts = GroupedMMExperts() if backend == "grouped_mm" else ReferenceExperts() + layer = EPMoELayer( + NUM_EXPERTS, + HIDDEN_SIZE, + INTERMEDIATE_SIZE, + ep_group=group, + experts=experts, + expert_grad_scale=grad_scale, + device="cuda", + dtype=torch.bfloat16, + ) + + generator = torch.Generator(device="cuda").manual_seed(0) # same weights on every rank + gate = torch.randn(NUM_EXPERTS, HIDDEN_SIZE, device="cuda", generator=generator) * HIDDEN_SIZE**-0.5 + full_gate_up = torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE_SIZE, HIDDEN_SIZE, device="cuda", generator=generator) * ( + HIDDEN_SIZE**-0.5 + ) + full_down = torch.randn(NUM_EXPERTS, HIDDEN_SIZE, INTERMEDIATE_SIZE, device="cuda", generator=generator) * ( + INTERMEDIATE_SIZE**-0.5 + ) + # skewed: the second half of the experts receives no token at all + bias = torch.zeros(NUM_EXPERTS, device="cuda") + if skewed: + bias[NUM_EXPERTS // 2 :] = -1e4 + layer.load_full_weights(full_gate_up, full_down) + + generator = torch.Generator(device="cuda").manual_seed(1 + dist.get_rank()) + x = torch.randn(NUM_TOKENS, HIDDEN_SIZE, device="cuda", generator=generator) + upstream = torch.randn(NUM_TOKENS, HIDDEN_SIZE, device="cuda", generator=generator) + + # layer (bf16), router in fp32 as models do + x_layer = x.to(torch.bfloat16).requires_grad_() + topk_idx, topk_weights = route(x_layer, gate, bias) + y = layer(x_layer, topk_idx, topk_weights) + (y.float() * upstream).sum().backward() + + # fp32 reference over this rank's tokens; expert grads summed over the ep group, as the layer computes them + x_ref = x.to(torch.bfloat16).float().requires_grad_() + gate_up_ref = full_gate_up.to(torch.bfloat16).float().requires_grad_() + down_ref = full_down.to(torch.bfloat16).float().requires_grad_() + ref_idx, ref_weights = route(x_ref, gate, bias) + y_ref = torch.zeros_like(x_ref) + for expert in range(NUM_EXPERTS): + token, slot = torch.where(ref_idx == expert) + g, u = F.linear(x_ref[token], gate_up_ref[expert]).chunk(2, dim=-1) + out = F.linear(F.silu(g) * u, down_ref[expert]) * ref_weights[token, slot, None] + y_ref = y_ref.index_add(0, token, out) + (y_ref * upstream).sum().backward() + dist.all_reduce(gate_up_ref.grad, group=group) + dist.all_reduce(down_ref.grad, group=group) + local = slice(layer.local_expert_range.start, layer.local_expert_range.stop) + + # IN_FORWARD divides expert grads by ep_size inside the layer; IN_REDUCTION leaves that to the reduction + factor = ep_size if grad_scale == ExpertGradScale.IN_FORWARD else 1 + config = f"ep={ep_size} {backend} {grad_scale.value} skewed={skewed}" + assert rel(y, y_ref) < RTOL, f"{config}: output {rel(y, y_ref):.3e}" + assert rel(x_layer.grad, x_ref.grad) < RTOL, f"{config}: input grad {rel(x_layer.grad, x_ref.grad):.3e}" + for name, grad, ref in [ + ("gate_up_proj", layer.gate_up_proj.grad, gate_up_ref.grad[local]), + ("down_proj", layer.down_proj.grad, down_ref.grad[local]), + ]: + assert grad is not None, f"{config}: {name} has no grad" + for expert, (g, r) in enumerate(zip(grad, ref)): + if r.abs().max() == 0: # expert received no token anywhere in the ep group + assert g.abs().max() == 0, f"{config}: {name}[{expert}] should be zero" + else: + err = rel(g.float() * factor, r) + assert err < RTOL, f"{config}: {name}[{expert}] grad {err:.3e}" + + +@parameterize("use_default_group", [False, True]) +def run_param_contract(use_default_group: bool): + """Expert params carry their EP tags, and loading weights keeps the parameter objects.""" + group = None if use_default_group else ep_groups(2) + expected_group = dist.group.WORLD if use_default_group else group + ep_size = dist.get_world_size(expected_group) + ep_rank = dist.get_rank(expected_group) + layer = EPMoELayer(NUM_EXPERTS, HIDDEN_SIZE, INTERMEDIATE_SIZE, ep_group=group, device="cuda") + assert layer.ep_group is expected_group + assert layer.dispatcher.ep_group is expected_group + assert layer.num_local_experts == NUM_EXPERTS // ep_size + start = ep_rank * layer.num_local_experts + assert layer.local_expert_range == range(start, start + layer.num_local_experts) + params = (layer.gate_up_proj, layer.down_proj) + for param in params: + assert is_moe_tensor(param) and get_ep_group(param) is expected_group + assert get_ep_size(param) == ep_size + assert get_expert_grad_scale(param) == ExpertGradScale.IN_FORWARD + ids = [id(p) for p in params] + layer.load_full_weights( + torch.ones(NUM_EXPERTS, 2 * INTERMEDIATE_SIZE, HIDDEN_SIZE, device="cuda"), + torch.ones(NUM_EXPERTS, HIDDEN_SIZE, INTERMEDIATE_SIZE, device="cuda"), + ) + assert [id(p) for p in (layer.gate_up_proj, layer.down_proj)] == ids + assert get_expert_grad_scale(torch.zeros(1)) is None + + +def run_dist(rank, world_size, port): + colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port, backend="nccl") + torch.cuda.set_device(rank) + run_dispatch_contract() + run_layer_vs_reference() + run_param_contract() + + +@pytest.mark.dist +@pytest.mark.parametrize("world_size", [4]) +@rerun_if_address_is_in_use() +def test_ep_moe_layer(world_size): + spawn(run_dist, world_size) + + +if __name__ == "__main__": + test_ep_moe_layer(world_size=4) diff --git a/tests/test_moe/test_ep_moe_layer_cpu.py b/tests/test_moe/test_ep_moe_layer_cpu.py new file mode 100644 index 000000000000..7eb83215b5bd --- /dev/null +++ b/tests/test_moe/test_ep_moe_layer_cpu.py @@ -0,0 +1,129 @@ +import pytest +import torch +import torch.distributed as dist +import torch.nn as nn + +from colossalai.moe import EPMoELayer, ReferenceExperts +from colossalai.tensor.moe_tensor.api import ( + ExpertGradScale, + get_ep_group, + get_ep_size, + get_expert_grad_scale, + is_moe_tensor_ep_fused, +) +from colossalai.testing import rerun_if_address_is_in_use, spawn + +NUM_EXPERTS, HIDDEN, INTERMEDIATE = 4, 8, 4 + + +@pytest.mark.parametrize("initialized", [False, True]) +def test_ep_moe_layer_default_process_group(initialized: bool, tmp_path): + """Default-group expert tags work with the parameter API without changing local construction.""" + assert not dist.is_initialized() + if initialized: + dist.init_process_group("gloo", init_method=(tmp_path / "rendezvous").as_uri(), rank=0, world_size=1) + try: + layer = EPMoELayer(4, 8, 4, experts=ReferenceExperts(), device="cpu") + expected_group = dist.group.WORLD if initialized else None + assert layer.ep_group is expected_group + assert layer.dispatcher.ep_group is expected_group + assert layer.ep_size == 1 + assert layer.ep_rank == 0 + assert layer.num_local_experts == 4 + for param in layer.parameters(): + assert get_ep_group(param) is expected_group + assert get_expert_grad_scale(param) == ExpertGradScale.IN_FORWARD + if initialized: + assert get_ep_size(param) == 1 + finally: + if initialized: + dist.destroy_process_group() + + +def _full_weights(): + generator = torch.Generator().manual_seed(0) + return ( + torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN, generator=generator), + torch.randn(NUM_EXPERTS, HIDDEN, INTERMEDIATE, generator=generator), + ) + + +def _init_gloo(rank, world_size, port): + dist.init_process_group("gloo", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=world_size) + return dist.new_group(list(range(world_size))) + + +def _ep_layer(ep_group): + return EPMoELayer(NUM_EXPERTS, HIDDEN, INTERMEDIATE, ep_group=ep_group, experts=ReferenceExperts(), device="cpu") + + +def test_ep_moe_layer_marks_experts_ep_fused(): + """``MoECheckpointIO`` gathers the experts of all EP ranks only for parameters marked ``ep_fused``.""" + layer = _ep_layer(None) + assert all(is_moe_tensor_ep_fused(param) for param in layer.parameters()) + + +def _run_load_full_experts(rank, world_size, port): + ep_group = _init_gloo(rank, world_size, port) + try: + layer = _ep_layer(ep_group) + gate_up, down = _full_weights() + local = layer.local_expert_range + local_gate_up, local_down = gate_up[local.start : local.stop], down[local.start : local.stop] + + # the full [num_experts, ...] tensors of a checkpoint: this rank keeps its own experts + layer.load_state_dict({"gate_up_proj": gate_up, "down_proj": down}) + assert torch.equal(layer.gate_up_proj, local_gate_up) and torch.equal(layer.down_proj, local_down) + + # a tensor that already has the local shape is loaded as is + layer.load_state_dict({"gate_up_proj": 2 * local_gate_up, "down_proj": 2 * local_down}) + assert torch.equal(layer.gate_up_proj, 2 * local_gate_up) and torch.equal(layer.down_proj, 2 * local_down) + + # any other shape is still reported by load_state_dict + with pytest.raises(RuntimeError, match="size mismatch"): + layer.load_state_dict({"gate_up_proj": gate_up[:3], "down_proj": local_down}) + with pytest.raises(RuntimeError, match="size mismatch"): + layer.load_state_dict({"gate_up_proj": torch.tensor(1.0), "down_proj": local_down}) + finally: + dist.destroy_process_group() + + +@rerun_if_address_is_in_use() +def test_ep_moe_layer_loads_full_experts(): + spawn(_run_load_full_experts, 2) + + +def _run_checkpoint_roundtrip(rank, world_size, port): + from colossalai.checkpoint_io.moe_checkpoint import MoECheckpointIO + + ep_group = _init_gloo(rank, world_size, port) + try: + model = nn.Module() + model.experts = _ep_layer(ep_group) + gate_up, down = _full_weights() + model.experts.load_full_weights(gate_up, down) + + saved = {} + for shard, _ in MoECheckpointIO._model_sharder(model): + saved.update(shard) + if rank == 0: + # the root rank writes the experts of all EP ranks, i.e. the full [num_experts, ...] tensors + assert torch.equal(saved["experts.gate_up_proj"], gate_up) + assert torch.equal(saved["experts.down_proj"], down) + payload = [saved if rank == 0 else None] + dist.broadcast_object_list(payload, src=0) + + # every rank reads the same files back, into a layer that starts from other weights + restored = nn.Module() + restored.experts = _ep_layer(ep_group) + restored.load_state_dict(payload[0]) + local = restored.experts.local_expert_range + assert torch.equal(restored.experts.gate_up_proj, gate_up[local.start : local.stop]) + assert torch.equal(restored.experts.down_proj, down[local.start : local.stop]) + finally: + dist.destroy_process_group() + + +@rerun_if_address_is_in_use() +def test_ep_moe_layer_checkpoint_roundtrip(): + spawn(_run_checkpoint_roundtrip, 2) diff --git a/tests/test_moe/test_ep_moe_layer_vs_hf.py b/tests/test_moe/test_ep_moe_layer_vs_hf.py new file mode 100644 index 000000000000..1ff911a730b6 --- /dev/null +++ b/tests/test_moe/test_ep_moe_layer_vs_hf.py @@ -0,0 +1,221 @@ +"""EPMoELayer against the native HF Qwen3-MoE sparse block (CPU only). + +The layer is a drop-in for ``Qwen3MoeSparseMoeBlock.experts``: HF calls ``experts(hidden_states, top_k_index, +top_k_weights)`` and the layer takes the same three tensors, with the same parameter names. The router stays HF's. +Everything is judged with the baseline-error rule of ``tests.kit.numerics``: float64 HF is the truth, HF at the +tested precision is the reference whose error sets the allowance, and the swapped model is the test. +""" + +import copy +import os +from unittest.mock import patch + +import pytest +import torch +import torch.distributed as dist +import torch.nn.functional as F + +from colossalai._compat import is_transformers_v5 +from colossalai.testing import rerun_if_address_is_in_use, spawn +from tests.kit.numerics import Trace, assert_aligned, run_fwd_bwd + +pytestmark = pytest.mark.skipif(not is_transformers_v5(), reason="needs the transformers v5 fused-experts layout") + +# the shapes can be overridden to approach a real model (Qwen3-30B-A3B: 128 experts, top-8), e.g. +# MOE_TEST_NUM_EXPERTS=64 MOE_TEST_TOP_K=8; spawned workers inherit the environment +NUM_EXPERTS = int(os.environ.get("MOE_TEST_NUM_EXPERTS", 8)) +TOP_K = int(os.environ.get("MOE_TEST_TOP_K", 2)) +HIDDEN, INTER = 32, 16 +BACKENDS = [ + pytest.param( + "grouped_mm", + marks=pytest.mark.skipif(not hasattr(torch, "_grouped_mm"), reason="needs torch._grouped_mm (torch >= 2.8)"), + ), + "reference", +] + + +def _tiny_model(seed: int = 0): + from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM + + config = Qwen3MoeConfig( + vocab_size=64, + hidden_size=HIDDEN, + intermediate_size=64, + moe_intermediate_size=INTER, + num_hidden_layers=2, + num_attention_heads=2, + num_key_value_heads=1, + head_dim=16, + num_experts=NUM_EXPERTS, + num_experts_per_tok=TOP_K, + norm_topk_prob=True, + mlp_only_layers=[], + max_position_embeddings=64, + use_cache=False, + attention_dropout=0.0, + tie_word_embeddings=False, + experts_implementation="eager", + attn_implementation="eager", + ) + torch.manual_seed(seed) + model = Qwen3MoeForCausalLM(config).float() + # HF initialises the router to a tiny scale and the experts to 0.02: use sizeable values so that routing is + # not near-uniform and the experts matter + for name, param in model.named_parameters(): + if name.endswith("mlp.gate.weight"): + param.data.normal_() + elif name.endswith("experts.gate_up_proj"): + param.data.normal_(std=HIDDEN**-0.5) + elif name.endswith("experts.down_proj"): + param.data.normal_(std=INTER**-0.5) + return model + + +def _swap_experts(module, backend, ep_group=None): + """Replace every HF experts module below ``module`` by an EPMoELayer holding the same weights.""" + from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeExperts + + from colossalai.moe import EPMoELayer, GroupedMMExperts, ReferenceExperts + + for parent in list(module.modules()): + for name, child in list(parent.named_children()): + if isinstance(child, Qwen3MoeExperts): + layer = EPMoELayer( + child.num_experts, + child.hidden_dim, + child.intermediate_dim, + ep_group=ep_group, + act_fn=F.silu, + experts=GroupedMMExperts() if backend == "grouped_mm" else ReferenceExperts(), + ) + layer.load_full_weights(child.gate_up_proj.data, child.down_proj.data) + setattr(parent, name, layer) + + +def _block_inputs(batch: int = 4): + generator = torch.Generator().manual_seed(1) + return {"hidden_states": torch.randn(batch, 16, HIDDEN, generator=generator, dtype=torch.float64)} + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("backend", BACKENDS) +def test_block_matches_hf(dtype, backend): + """Layer level: the router sees the same input in all three runs, so there are no routing flips to explain.""" + block = _tiny_model().model.layers[0].mlp + new = copy.deepcopy(block) + _swap_experts(new, backend) + inputs = _block_inputs() + truth = run_fwd_bwd(block, inputs, torch.float64) + ref = run_fwd_bwd(block, inputs, dtype) + if backend == "grouped_mm": + with patch.object(torch, "_grouped_mm", wraps=torch._grouped_mm) as grouped_mm: + test = run_fwd_bwd(new, inputs, dtype) + assert grouped_mm.call_count > 0, "the test never executed torch._grouped_mm" + else: + test = run_fwd_bwd(new, inputs, dtype) + assert_aligned(test, ref, truth) + + +def _init_gloo(rank, world_size, port): + dist.init_process_group("gloo", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=world_size) + + +def _gather(tensor, group=None): + parts = [torch.empty_like(tensor) for _ in range(dist.get_world_size(group))] + dist.all_gather(parts, tensor.contiguous(), group=group) + return torch.cat(parts, 0) + + +def _run_ep_block(rank, world_size, port, backend): + _init_gloo(rank, world_size, port) + try: + ep_group = dist.new_group(list(range(world_size))) + block = _tiny_model().model.layers[0].mlp + local = 2 + inputs = _block_inputs(batch=world_size * local) + truth = run_fwd_bwd(block, inputs, torch.float64) + ref = run_fwd_bwd(block, inputs, torch.float32) + + new = copy.deepcopy(block) + _swap_experts(new, backend, ep_group) + x = inputs["hidden_states"] + mine = x[rank * local : (rank + 1) * local].float().requires_grad_() + out = new(mine) + # the same upstream gradient as run_fwd_bwd draws for the full output + upstream = torch.randn(x.shape, generator=torch.Generator().manual_seed(0), dtype=torch.float64) + (out * upstream[rank * local : (rank + 1) * local].float()).sum().backward() + + router_grad = new.gate.weight.grad.clone() + dist.all_reduce(router_grad) # the router is replicated: its gradient is summed over the ranks' tokens + # the layer divides expert gradients by ep_size in backward (ExpertGradScale.IN_FORWARD), so undo that to + # compare with the full-batch gradient + test = Trace( + { + "out": _gather(out.detach()), + "grad/hidden_states": _gather(mine.grad), + "grad/param/experts.gate_up_proj": _gather(new.experts.gate_up_proj.grad) * world_size, + "grad/param/experts.down_proj": _gather(new.experts.down_proj.grad) * world_size, + "grad/param/gate.weight": router_grad, + } + ) + assert_aligned(test, ref, truth) + finally: + dist.destroy_process_group() + + +@pytest.mark.parametrize("backend", BACKENDS) +@pytest.mark.parametrize("ep_size", [2, 4]) +@rerun_if_address_is_in_use() +def test_ep_block_matches_hf(backend, ep_size): + spawn(_run_ep_block, ep_size, backend=backend) + + +def _run_dp_ep_grad_scale(rank, world_size, port): + """ep 2 x moe_dp 2, reduced the way the current ZeRO code does: dense gradients are averaged over all dp ranks, + expert gradients only over the moe_dp group (zero/low_level/low_level_optim.py divides by that group's size).""" + _init_gloo(rank, world_size, port) + try: + ep = 2 + ep_groups = [dist.new_group([0, 1]), dist.new_group([2, 3])] + moe_dp_groups = [dist.new_group([0, 2]), dist.new_group([1, 3])] + ep_group, moe_dp_group = ep_groups[rank // ep], moe_dp_groups[rank % ep] + + block = _tiny_model().model.layers[0].mlp + local = 4 + x = _block_inputs(batch=world_size * local)["hidden_states"].float() + upstream = torch.randn(x.shape, generator=torch.Generator().manual_seed(7)) + + exact = copy.deepcopy(block) + (exact(x.clone()) * upstream).sum().backward() + target = { + "gate_up": exact.experts.gate_up_proj.grad / world_size, + "down": exact.experts.down_proj.grad / world_size, + "router": exact.gate.weight.grad / world_size, + } + + new = copy.deepcopy(block) + _swap_experts(new, "reference", ep_group) + mine = x[rank * local : (rank + 1) * local] + (new(mine) * upstream[rank * local : (rank + 1) * local]).sum().backward() + + got = {} + router = new.gate.weight.grad.clone() + dist.all_reduce(router) + got["router"] = router / world_size + for key, param in (("gate_up", new.experts.gate_up_proj), ("down", new.experts.down_proj)): + grad = param.grad.clone() + dist.all_reduce(grad, group=moe_dp_group) + grad /= dist.get_world_size(moe_dp_group) + got[key] = _gather(grad, ep_group) + for key in target: + rel = ((got[key] - target[key]).norm() / target[key].norm()).item() + assert rel < 1e-4, f"{key}: relative error {rel:.2e} against the full-batch gradient" + finally: + dist.destroy_process_group() + + +@rerun_if_address_is_in_use() +def test_dp_ep_expert_gradient_scale(): + """With the default ExpertGradScale.IN_FORWARD the current reduction rule gives the exact gradient.""" + spawn(_run_dp_ep_grad_scale, 4) diff --git a/tests/test_moe/test_ep_moe_model_vs_hf.py b/tests/test_moe/test_ep_moe_model_vs_hf.py new file mode 100644 index 000000000000..5155614a30aa --- /dev/null +++ b/tests/test_moe/test_ep_moe_model_vs_hf.py @@ -0,0 +1,45 @@ +"""Whole-model Qwen3-MoE checks kept outside the CPU CI layer. + +The layer, EP and DP-contract tests live in ``test_ep_moe_layer_vs_hf.py`` and are collected by the CPU batch. These +checks exercise the complete HF model and remain available for local L2 reports without making the CPU CI job a model +level acceptance gate. +""" + +import copy + +import pytest +import torch + +from colossalai._compat import is_transformers_v5 +from tests.kit.numerics import assert_aligned, run_fwd_bwd +from tests.test_moe.test_ep_moe_layer_vs_hf import BACKENDS, _swap_experts, _tiny_model + +pytestmark = pytest.mark.skipif(not is_transformers_v5(), reason="needs the transformers v5 fused-experts layout") + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_model_matches_hf_fp32(backend): + """Whole model in fp32: logits, loss, aux loss, router logits and every parameter gradient.""" + model = _tiny_model() + new = copy.deepcopy(model) + _swap_experts(new, backend) + ids = torch.randint(1, 64, (2, 16), generator=torch.Generator().manual_seed(2)) + inputs = {"input_ids": ids, "labels": ids, "output_router_logits": True, "use_cache": False} + truth = run_fwd_bwd(model, inputs, torch.float64) + ref = run_fwd_bwd(model, inputs, torch.float32) + test = run_fwd_bwd(new, inputs, torch.float32) + assert_aligned(test, ref, truth) + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_greedy_generation_matches_hf(backend): + """Prefill plus KV-cache decoding, including single-token steps where most experts receive no row.""" + model = _tiny_model().eval() + new = copy.deepcopy(model) + _swap_experts(new, backend) + ids = torch.randint(1, 64, (2, 8), generator=torch.Generator().manual_seed(3)) + kwargs = dict(max_new_tokens=8, do_sample=False, use_cache=True, pad_token_id=0) + with torch.no_grad(): + expected = model.generate(ids, attention_mask=torch.ones_like(ids), **kwargs) + actual = new.generate(ids, attention_mask=torch.ones_like(ids), **kwargs) + assert torch.equal(actual, expected) diff --git a/tests/test_moe/test_grad_sync_plan.py b/tests/test_moe/test_grad_sync_plan.py new file mode 100644 index 000000000000..da1877313fbd --- /dev/null +++ b/tests/test_moe/test_grad_sync_plan.py @@ -0,0 +1,132 @@ +"""The gradient sync plan of the MoE plugin, on CPU with gloo (4 ranks: ep 2 x moe_dp 2). + +Rank layout: ``ep_group`` = {0,1} / {2,3}, ``moe_dp_group`` = {0,2} / {1,3}, ``dp_group`` = all ranks. +""" + +import pytest +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F + +from colossalai.moe import EPMoELayer, ReferenceExperts +from colossalai.tensor.moe_tensor.api import ExpertGradScale, set_moe_tensor_ep_group +from colossalai.tensor.moe_tensor.grad_sync import GradSyncSpec, build_grad_sync_plan, has_reduction_scaled_experts +from colossalai.testing import rerun_if_address_is_in_use, spawn + +NUM_EXPERTS, TOP_K, HIDDEN, INTERMEDIATE, LOCAL_TOKENS = 4, 2, 8, 4, 6 +EP = 2 + + +def _init_gloo(rank, world_size, port): + dist.init_process_group("gloo", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=world_size) + # every rank creates every group, in the same order + ep_groups = [dist.new_group([0, 1]), dist.new_group([2, 3])] + moe_dp_groups = [dist.new_group([0, 2]), dist.new_group([1, 3])] + return dist.group.WORLD, ep_groups[rank // EP], moe_dp_groups[rank % EP] + + +def _run_plan_structure(rank, world_size, port): + dp_group, ep_group, moe_dp_group = _init_gloo(rank, world_size, port) + try: + dense = nn.Parameter(torch.zeros(2)) + # an expert of the existing EP models: only the ep group is tagged, the layer applies 1 / ep_size itself + legacy_expert = nn.Parameter(torch.zeros(2)) + set_moe_tensor_ep_group(legacy_expert, ep_group) + in_forward = EPMoELayer(NUM_EXPERTS, HIDDEN, INTERMEDIATE, ep_group=ep_group, experts=ReferenceExperts()) + in_reduction = EPMoELayer( + NUM_EXPERTS, + HIDDEN, + INTERMEDIATE, + ep_group=ep_group, + experts=ReferenceExperts(), + expert_grad_scale=ExpertGradScale.IN_REDUCTION, + ) + + plan = build_grad_sync_plan( + [dense, legacy_expert, *in_forward.parameters(), *in_reduction.parameters()], dp_group, moe_dp_group + ) + assert plan[dense] == GradSyncSpec(dp_group, 4) + assert plan[legacy_expert] == GradSyncSpec(moe_dp_group, 2) + for param in in_forward.parameters(): + assert plan[param] == GradSyncSpec(moe_dp_group, 2) and plan[param].extra_divisor == 1 + for param in in_reduction.parameters(): + # summed over the 2 replicas, divided by moe_dp_size * ep_size + assert plan[param] == GradSyncSpec(moe_dp_group, 4) and plan[param].extra_divisor == 2 + + assert not has_reduction_scaled_experts([dense, legacy_expert, *in_forward.parameters()]) + assert has_reduction_scaled_experts(in_reduction.parameters()) + finally: + dist.destroy_process_group() + + +@rerun_if_address_is_in_use() +def test_grad_sync_plan_structure(): + spawn(_run_plan_structure, 4) + + +def _route(tokens, router): + weights, idx = torch.topk((tokens @ router.t()).softmax(-1), TOP_K, dim=-1) + return idx, weights / weights.sum(-1, keepdim=True) + + +def _reference(tokens, router, gate_up, down): + """SwiGLU experts, one expert at a time, on a single process.""" + idx, weights = _route(tokens, router) + out = torch.zeros_like(tokens) + for expert in range(NUM_EXPERTS): + token, slot = torch.where(idx == expert) + gate, up = F.linear(tokens[token], gate_up[expert]).chunk(2, dim=-1) + out = out.index_add(0, token, F.linear(F.silu(gate) * up, down[expert]) * weights[token, slot, None]) + return out + + +def _run_plan_gradient(rank, world_size, port, grad_scale): + """Reducing by the plan (sum over the replica group, then divide) gives the gradient of the mean-over-ranks loss.""" + dp_group, ep_group, moe_dp_group = _init_gloo(rank, world_size, port) + try: + generator = torch.Generator().manual_seed(0) # same weights and data on every rank + gate_up = torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN, generator=generator) + down = torch.randn(NUM_EXPERTS, HIDDEN, INTERMEDIATE, generator=generator) + router = torch.randn(NUM_EXPERTS, HIDDEN, generator=generator) + x = torch.randn(world_size * LOCAL_TOKENS, HIDDEN, generator=generator) + upstream = torch.randn(world_size * LOCAL_TOKENS, HIDDEN, generator=generator) + + # reference: all tokens on one process, loss = mean over the ranks' sums + full = {k: v.clone().requires_grad_() for k, v in {"gate_up": gate_up, "down": down, "router": router}.items()} + (_reference(x, full["router"], full["gate_up"], full["down"]) * upstream).sum().backward() + + layer = EPMoELayer( + NUM_EXPERTS, + HIDDEN, + INTERMEDIATE, + ep_group=ep_group, + experts=ReferenceExperts(), + expert_grad_scale=grad_scale, + ) + layer.load_full_weights(gate_up, down) + router_param = nn.Parameter(router.clone()) + mine = slice(rank * LOCAL_TOKENS, (rank + 1) * LOCAL_TOKENS) + idx, weights = _route(x[mine], router_param) + (layer(x[mine], idx, weights) * upstream[mine]).sum().backward() + + plan = build_grad_sync_plan([router_param, *layer.parameters()], dp_group, moe_dp_group) + local = layer.local_expert_range + for name, param, expected in ( + ("router", router_param, full["router"].grad), + ("gate_up_proj", layer.gate_up_proj, full["gate_up"].grad[local.start : local.stop]), + ("down_proj", layer.down_proj, full["down"].grad[local.start : local.stop]), + ): + spec = plan[param] + grad = param.grad.clone() + dist.all_reduce(grad, group=spec.replica_group) + grad /= spec.sum_divisor + assert torch.allclose(grad, expected / world_size, rtol=1e-4, atol=1e-6), f"{grad_scale.value} {name}" + finally: + dist.destroy_process_group() + + +@pytest.mark.parametrize("grad_scale", [ExpertGradScale.IN_FORWARD, ExpertGradScale.IN_REDUCTION]) +@rerun_if_address_is_in_use() +def test_grad_sync_plan_gives_exact_gradient(grad_scale): + spawn(_run_plan_gradient, 4, grad_scale=grad_scale) diff --git a/tests/test_moe/test_grad_sync_zero.py b/tests/test_moe/test_grad_sync_zero.py new file mode 100644 index 000000000000..7490504f7067 --- /dev/null +++ b/tests/test_moe/test_grad_sync_zero.py @@ -0,0 +1,194 @@ +"""Exercise the gradient sync plan through the real MoE plugin and ZeRO optimizer on CUDA/NCCL.""" + +import faulthandler +from contextlib import nullcontext + +import pytest +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F + +import colossalai +from colossalai.accelerator import set_accelerator +from colossalai.booster import Booster +from colossalai.booster.plugin import MoeHybridParallelPlugin +from colossalai.moe import EPMoELayer +from colossalai.shardformer.policies.base_policy import Policy +from colossalai.tensor.moe_tensor.api import ExpertGradScale +from colossalai.testing import rerun_if_address_is_in_use, spawn + +WORLD_SIZE = 4 +EP_SIZE = 2 +NUM_EXPERTS = 4 +TOP_K = 2 +HIDDEN = 8 +INTERMEDIATE = 8 +LOCAL_TOKENS = 6 +GRAD_RTOL = 5e-2 + + +class NoOpPolicy(Policy): + def config_sanity_check(self): + pass + + def preprocess(self): + return self.model + + def module_policy(self): + return {} + + def postprocess(self): + return self.model + + +class TinyMoeModel(nn.Module): + def __init__(self, ep_group, grad_scale, empty_experts=False): + super().__init__() + generator = torch.Generator().manual_seed(17) + self.router_weight = nn.Parameter(torch.randn(NUM_EXPERTS, HIDDEN, generator=generator)) + self.router_bias = nn.Parameter(torch.zeros(NUM_EXPERTS)) + self.experts = EPMoELayer( + NUM_EXPERTS, + HIDDEN, + INTERMEDIATE, + ep_group=ep_group, + expert_grad_scale=grad_scale, + ) + gate_up = torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN, generator=generator) + down = torch.randn(NUM_EXPERTS, HIDDEN, INTERMEDIATE, generator=generator) + self.experts.load_full_weights(gate_up, down) + self.top_k = 1 if empty_experts else TOP_K + if empty_experts: + with torch.no_grad(): + self.router_weight.zero_() + self.router_bias.copy_(torch.tensor([8.0, -8.0, -8.0, -8.0])) + + def forward(self, tokens): + logits = F.linear(tokens, self.router_weight, self.router_bias) + weights, idx = torch.topk(logits.softmax(-1), self.top_k, dim=-1) + weights = weights / weights.sum(-1, keepdim=True) + return self.experts(tokens, idx, weights) + + +def _reference(tokens, router_weight, router_bias, gate_up, down, top_k): + logits = F.linear(tokens, router_weight, router_bias) + weights, idx = torch.topk(logits.softmax(-1), top_k, dim=-1) + weights = weights / weights.sum(-1, keepdim=True) + out = torch.zeros_like(tokens) + for expert in range(NUM_EXPERTS): + token, slot = torch.where(idx == expert) + gate, up = F.linear(tokens[token], gate_up[expert]).chunk(2, dim=-1) + out = out.index_add(0, token, F.linear(F.silu(gate) * up, down[expert]) * weights[token, slot, None]) + return out + + +def _full_reference(model, inputs, upstreams): + params = {name: param.detach().clone().float().requires_grad_() for name, param in model.named_parameters()} + for name in ("experts.gate_up_proj", "experts.down_proj"): + local = params[name] + shards = [torch.empty_like(local) for _ in range(EP_SIZE)] + dist.all_gather(shards, local, group=model.experts.ep_group) + params[name] = torch.cat(shards).detach().requires_grad_() + for tokens, upstream in zip(inputs, upstreams): + device = params["router_weight"].device + tokens, upstream = tokens.to(device), upstream.to(device) + out = _reference( + tokens, + params["router_weight"], + params["router_bias"], + params["experts.gate_up_proj"], + params["experts.down_proj"], + model.top_k, + ) + (out * upstream).sum().backward() + return params + + +def _run_case(stage, grad_scale, overlap, empty_experts): + rank = dist.get_rank() + plugin = MoeHybridParallelPlugin( + tp_size=1, + pp_size=1, + ep_size=EP_SIZE, + zero_stage=stage, + precision="bf16", + overlap_communication=overlap, + custom_policy=NoOpPolicy(), + ) + if stage == 2 or overlap: + assert plugin.zero_stage == 1 + assert plugin.zero_config["overlap_communication"] is False + model = TinyMoeModel(plugin.ep_group, grad_scale, empty_experts=empty_experts) + booster = Booster(plugin=plugin) + optimizer = torch.optim.SGD(model.parameters(), lr=0.1) + model, optimizer, _, _, _ = booster.boost(model, optimizer) + + generator = torch.Generator().manual_seed(23) + inputs = [torch.randn(WORLD_SIZE * LOCAL_TOKENS, HIDDEN, generator=generator) for _ in range(2)] + upstreams = [torch.randn(WORLD_SIZE * LOCAL_TOKENS, HIDDEN, generator=generator) for _ in range(2)] + microbatches = 1 if stage == 2 else 2 + reference = _full_reference(model.unwrap(), inputs[:microbatches], upstreams[:microbatches]) + + for tokens, upstream in zip(inputs[:microbatches], upstreams[:microbatches]): + mine = slice(rank * LOCAL_TOKENS, (rank + 1) * LOCAL_TOKENS) + sync_context = optimizer.no_sync() if microbatches == 2 and tokens is inputs[0] else nullcontext() + with sync_context: + out = model(tokens[mine].cuda()) + optimizer.backward((out * upstream[mine].cuda()).sum()) + + errors = [] + local = model.unwrap().experts.local_expert_range + for name, param in model.unwrap().named_parameters(): + got = optimizer.get_param_grad(param) + expected = reference[name].grad.detach().cpu() / WORLD_SIZE + if name.startswith("experts."): + expected = expected[local.start : local.stop] + got = torch.zeros_like(expected) if got is None else got.detach().cpu().float() + error = (got - expected).norm() + allowance = GRAD_RTOL * expected.norm() + 1e-6 + if error > allowance: + errors.append( + f"{name}: relative L2={error.item() / max(expected.norm().item(), 1e-12):.4f}, " + f"got/reference norm={got.norm().item() / max(expected.norm().item(), 1e-12):.4f}" + ) + assert ( + not errors + ), f"stage={stage}, scale={grad_scale.value}, overlap={overlap}, empty={empty_experts}: " + "; ".join(errors) + + +CASES = [ + (1, ExpertGradScale.IN_FORWARD, False, False), + (1, ExpertGradScale.IN_REDUCTION, False, False), + (1, ExpertGradScale.IN_REDUCTION, True, False), + (2, ExpertGradScale.IN_REDUCTION, False, False), + (1, ExpertGradScale.IN_REDUCTION, False, True), +] + + +def _run_dist(rank, world_size, port, case): + set_accelerator("cuda") + torch.cuda.set_device(rank) + torch.empty(1, device=f"cuda:{rank}") + faulthandler.dump_traceback_later(120, repeat=True) + try: + colossalai.launch( + rank=rank, world_size=world_size, host="localhost", port=port, backend="nccl", local_rank=rank + ) + print(f"START rank={rank} case={case}", flush=True) + _run_case(*case) + print(f"END rank={rank} case={case}", flush=True) + finally: + faulthandler.cancel_dump_traceback_later() + + +@pytest.mark.dist +@pytest.mark.parametrize("case", CASES) +@rerun_if_address_is_in_use() +def test_grad_sync_zero(case): + spawn(_run_dist, WORLD_SIZE, case=case) + + +if __name__ == "__main__": + for case in CASES: + test_grad_sync_zero(case=case) diff --git a/tests/test_shardformer/test_model/test_shard_qwen3.py b/tests/test_shardformer/test_model/test_shard_qwen3.py index 0118eb7c888a..d2eb400da1c3 100644 --- a/tests/test_shardformer/test_model/test_shard_qwen3.py +++ b/tests/test_shardformer/test_model/test_shard_qwen3.py @@ -236,7 +236,7 @@ def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn, ], ) def run_qwen3_test(test_config): - sub_model_zoo = model_zoo.get_sub_registry("transformers_qwen3") + sub_model_zoo = model_zoo.get_sub_registry("transformers_qwen3", exclude="transformers_qwen3_moe") for name, (model_fn, data_gen_fn, output_transform_fn, loss_fn, _) in sub_model_zoo.items(): try: @@ -285,7 +285,7 @@ def run_qwen3_test(test_config): ], ) def run_qwen3_3d_test(test_config): - sub_model_zoo = model_zoo.get_sub_registry("transformers_qwen3") + sub_model_zoo = model_zoo.get_sub_registry("transformers_qwen3", exclude="transformers_qwen3_moe") for name, (model_fn, data_gen_fn, output_transform_fn, loss_fn, _) in sub_model_zoo.items(): try: diff --git a/tests/test_shardformer/test_model/test_shard_qwen3_moe.py b/tests/test_shardformer/test_model/test_shard_qwen3_moe.py new file mode 100644 index 000000000000..96a2f1e29935 --- /dev/null +++ b/tests/test_shardformer/test_model/test_shard_qwen3_moe.py @@ -0,0 +1,212 @@ +import copy +import faulthandler + +import pytest +import torch +import torch.distributed as dist + +import colossalai +from colossalai._compat import is_transformers_v5 +from colossalai.accelerator import set_accelerator +from colossalai.booster.plugin import MoeHybridParallelPlugin +from colossalai.testing import parameterize, rerun_if_address_is_in_use, spawn +from tests.kit.model_zoo import model_zoo +from tests.test_checkpoint_io.utils import shared_tempdir +from tests.test_shardformer.test_model._utils import ( + build_model_from_hybrid_plugin, + run_forward_backward_with_hybrid_plugin, +) + + +def _tolerances(precision): + return (1e-4, 1e-4) if precision == "fp32" else (2e-2, 2e-2) + + +def _reference_grad(name, reference, module): + if reference is None: + return None + if ".mlp.experts." in name: + local = module.local_expert_range + return reference[local.start : local.stop] + return reference + + +def _check_parameters(reference_model, actual_model, actual_optimizer, zero_stage, precision): + reference_params = dict(reference_model.named_parameters()) + actual_modules = dict(actual_model.unwrap().named_modules()) + atol, rtol = _tolerances(precision) + errors = [] + for name, param in actual_model.unwrap().named_parameters(): + reference_param = reference_params[name] + owner_name = name.rsplit(".", 1)[0] + owner = actual_modules.get(owner_name) + expected = _reference_grad(name, reference_param.grad, owner) + if zero_stage == 0: + actual = param.grad + else: + actual = actual_optimizer.get_param_grad(param) + if expected is None: + if actual is not None and torch.count_nonzero(actual) != 0: + errors.append(f"{name}: expected no gradient") + continue + if actual is None: + errors.append(f"{name}: missing gradient") + continue + try: + torch.testing.assert_close(actual.float(), expected.float(), atol=atol, rtol=rtol) + except AssertionError as exc: + errors.append(f"{name}: {exc}") + assert not errors, "\n".join(errors) + + +def _check_weights(reference_model, actual_model, precision): + reference_params = dict(reference_model.named_parameters()) + actual_modules = dict(actual_model.unwrap().named_modules()) + atol, rtol = _tolerances(precision) + for name, param in actual_model.unwrap().named_parameters(): + expected = reference_params[name].detach() + owner = actual_modules.get(name.rsplit(".", 1)[0]) + if ".mlp.experts." in name: + local = owner.local_expert_range + expected = expected[local.start : local.stop] + torch.testing.assert_close(param.detach().float(), expected.float(), atol=atol, rtol=rtol) + + +def _check_checkpoint(model_fn, reference_model, model, optimizer, booster, precision): + with shared_tempdir() as tempdir: + checkpoint = f"{tempdir}/model" + booster.save_model(model, checkpoint, shard=True, use_safetensors=True) + dist.barrier() + + restored_hf = type(reference_model).from_pretrained(checkpoint).cuda() + if precision != "fp32": + restored_hf = restored_hf.to(torch.bfloat16) + reference_params = dict(reference_model.named_parameters()) + for name, param in restored_hf.named_parameters(): + torch.testing.assert_close(param.float(), reference_params[name].float(), atol=2e-2, rtol=2e-2) + + restored = model_fn().cuda() + if precision != "fp32": + restored = restored.to(torch.bfloat16) + restored_optimizer = torch.optim.Adam(restored.parameters(), lr=1e-3) + restored, restored_optimizer, _, _, _ = booster.boost(restored, restored_optimizer) + booster.load_model(restored, checkpoint, strict=True) + saved_params = dict(model.named_parameters()) + for name, param in restored.named_parameters(): + torch.testing.assert_close(param, saved_params[name], atol=0, rtol=0) + dist.barrier() + + +def check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn, test_config): + enable_gradient_checkpointing = test_config.pop("enable_gradient_checkpointing", False) + org_model, org_optimizer, sharded_model, sharded_optimizer, criterion, booster = build_model_from_hybrid_plugin( + model_fn, loss_fn, test_config, pluggin_cls=MoeHybridParallelPlugin + ) + if enable_gradient_checkpointing: + kwargs = {"use_reentrant": False} + org_model.gradient_checkpointing_enable(gradient_checkpointing_kwargs=kwargs) + sharded_model.unwrap().gradient_checkpointing_enable(gradient_checkpointing_kwargs=kwargs) + + precision = booster.plugin.precision + if precision == "bf16": + org_model = org_model.to(torch.bfloat16) + org_loss, org_output, sharded_loss, sharded_output = run_forward_backward_with_hybrid_plugin( + org_model, sharded_model, sharded_optimizer, data_gen_fn, output_transform_fn, criterion, booster + ) + + atol, rtol = _tolerances(precision) + torch.testing.assert_close(sharded_loss.float(), org_loss.float(), atol=atol, rtol=rtol) + output_key = "last_hidden_state" if hasattr(org_output, "last_hidden_state") else "logits" + torch.testing.assert_close( + getattr(sharded_output, output_key).float(), getattr(org_output, output_key).float(), atol=atol, rtol=rtol + ) + _check_weights(org_model, sharded_model, precision) + _check_parameters(org_model, sharded_model, sharded_optimizer, booster.plugin.zero_stage, precision) + + org_optimizer.step() + sharded_optimizer.step() + _check_weights(org_model, sharded_model, precision) + _check_checkpoint(model_fn, org_model, sharded_model, sharded_optimizer, booster, precision) + + dist.barrier() + booster.plugin.pg_mesh.destroy_mesh_process_groups() + dist.barrier() + torch.cuda.empty_cache() + + +@parameterize( + "test_config", + [ + {"tp_size": 1, "pp_size": 1, "ep_size": 1, "zero_stage": 0, "precision": "fp32"}, + { + "tp_size": 1, + "pp_size": 1, + "ep_size": 1, + "zero_stage": 1, + "precision": "bf16", + "use_lazy_init": True, + "enable_gradient_checkpointing": True, + "overlap_communication": False, + }, + { + "tp_size": 1, + "pp_size": 1, + "ep_size": 1, + "zero_stage": 2, + "precision": "bf16", + "overlap_communication": False, + }, + { + "tp_size": 1, + "pp_size": 1, + "ep_size": 2, + "zero_stage": 1, + "precision": "bf16", + "overlap_communication": False, + }, + { + "tp_size": 1, + "pp_size": 1, + "ep_size": 2, + "zero_stage": 2, + "precision": "bf16", + "use_lazy_init": True, + "overlap_communication": False, + }, + ], +) +def run_qwen3_moe_test(test_config): + entries = model_zoo.get_sub_registry("transformers_qwen3_moe") + for name, (model_fn, data_gen_fn, output_transform_fn, loss_fn, _) in entries.items(): + print(f"START model={name}, config={test_config}", flush=True) + try: + check_forward_backward(model_fn, data_gen_fn, output_transform_fn, loss_fn, copy.deepcopy(test_config)) + except Exception: + print(f"Failed model={name}, config={test_config}", flush=True) + raise + print(f"END model={name}, config={test_config}", flush=True) + + +def _run_dist(rank, world_size, port): + set_accelerator("cuda") + torch.cuda.set_device(rank) + torch.empty(1, device=f"cuda:{rank}") + faulthandler.dump_traceback_later(120, repeat=True) + try: + colossalai.launch( + rank=rank, world_size=world_size, host="localhost", port=port, backend="nccl", local_rank=rank + ) + run_qwen3_moe_test() + finally: + faulthandler.cancel_dump_traceback_later() + + +@pytest.mark.dist +@pytest.mark.skipif(not is_transformers_v5(), reason="Qwen3-MoE sharding needs transformers v5") +@rerun_if_address_is_in_use() +def test_qwen3_moe(): + spawn(_run_dist, 4) + + +if __name__ == "__main__": + test_qwen3_moe() diff --git a/tests/test_shardformer/test_qwen3_moe_policy_cpu.py b/tests/test_shardformer/test_qwen3_moe_policy_cpu.py new file mode 100644 index 000000000000..f638cb56ed05 --- /dev/null +++ b/tests/test_shardformer/test_qwen3_moe_policy_cpu.py @@ -0,0 +1,162 @@ +"""Qwen3-MoE ShardFormer policy on CPU: dispatch, the native fallback without EP, and the EP path with gloo. + +The tiny model has one sparse layer and one dense ``Qwen3MoeMLP`` layer (``mlp_only_layers=[1]``), because the policy has +to handle both kinds of ``mlp`` in the same model. +""" + +import copy + +import pytest +import torch +import torch.distributed as dist + +from colossalai._compat import is_transformers_v5 +from colossalai.shardformer import ShardConfig, ShardFormer +from colossalai.shardformer.policies.auto_policy import get_autopolicy +from colossalai.testing import rerun_if_address_is_in_use, spawn + +pytestmark = pytest.mark.skipif(not is_transformers_v5(), reason="Qwen3-MoE sharding needs transformers v5") + +NUM_EXPERTS = 8 +SPARSE = "model.layers.0.mlp.experts" + + +def _tiny_model(seed: int = 0): + from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM + + config = Qwen3MoeConfig( + vocab_size=64, + hidden_size=32, + intermediate_size=64, + moe_intermediate_size=16, + num_hidden_layers=2, + num_attention_heads=2, + num_key_value_heads=1, + head_dim=16, + num_experts=NUM_EXPERTS, + num_experts_per_tok=2, + norm_topk_prob=True, + mlp_only_layers=[1], + max_position_embeddings=64, + use_cache=False, + attention_dropout=0.0, + tie_word_embeddings=False, + experts_implementation="eager", + attn_implementation="eager", + ) + torch.manual_seed(seed) + model = Qwen3MoeForCausalLM(config).float() + for name, param in model.named_parameters(): + if name.endswith("mlp.gate.weight"): + param.data.normal_() + elif name.endswith("experts.gate_up_proj"): + param.data.normal_(std=32**-0.5) + elif name.endswith("experts.down_proj"): + param.data.normal_(std=16**-0.5) + return model + + +def test_policy_dispatch(): + from transformers import Qwen3MoeModel + + model = _tiny_model() + assert type(get_autopolicy(model)).__name__ == "Qwen3MoeForCausalLMPolicy" + assert type(get_autopolicy(Qwen3MoeModel(model.config))).__name__ == "Qwen3MoeModelPolicy" + + +def test_model_zoo_entries_run_forward_backward(): + from tests.kit.model_zoo import model_zoo + + entries = model_zoo.get_sub_registry("transformers_qwen3_moe") + assert set(entries) == {"transformers_qwen3_moe", "transformers_qwen3_moe_for_causal_lm"} + dense_entries = model_zoo.get_sub_registry("transformers_qwen3", exclude="transformers_qwen3_moe") + assert all("qwen3_moe" not in name for name in dense_entries) + + for name, (model_fn, data_gen_fn, output_transform_fn, loss_fn, _) in entries.items(): + model = model_fn() + output = output_transform_fn(model(**data_gen_fn())) + loss_fn(output).backward() + layers = model.model.layers if hasattr(model, "model") else model.layers + assert layers[0].mlp.experts.gate_up_proj.grad is not None, name + + +def test_without_ep_the_model_stays_native(): + from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeExperts, Qwen3MoeTopKRouter + + model = _tiny_model() + expected = copy.deepcopy(model) + model, _ = ShardFormer(ShardConfig(enable_tensor_parallelism=False)).optimize(model) + assert isinstance(model.model.layers[0].mlp.experts, Qwen3MoeExperts) + assert isinstance(model.model.layers[0].mlp.gate, Qwen3MoeTopKRouter) + ids = torch.randint(1, 64, (2, 8), generator=torch.Generator().manual_seed(1)) + with torch.no_grad(): + assert torch.equal(model(input_ids=ids).logits, expected(input_ids=ids).logits) + + +def _init_gloo(rank, world_size, port): + dist.init_process_group("gloo", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=world_size) + + +def _run_ep(rank, world_size, port): + from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeTopKRouter + + from colossalai.moe import EPMoELayer + from colossalai.tensor.moe_tensor.api import get_ep_group, is_moe_tensor, is_moe_tensor_ep_fused + + _init_gloo(rank, world_size, port) + try: + ep_group = dist.new_group(list(range(world_size))) + model = _tiny_model() + reference = copy.deepcopy(model) + full_state = copy.deepcopy(model.state_dict()) + original = {name: param for name, param in model.named_parameters()} + + model, _ = ShardFormer(ShardConfig(enable_tensor_parallelism=False, ep_group=ep_group)).optimize(model) + + sparse = model.model.layers[0].mlp + assert isinstance(sparse.experts, EPMoELayer) + assert isinstance(sparse.gate, Qwen3MoeTopKRouter) # the router keeps its class: the logits hook needs it + assert not hasattr(model.model.layers[1].mlp, "experts") # the dense layer is untouched + local = NUM_EXPERTS // world_size + + # the Parameter objects are the original ones, now holding this rank's experts and tagged for EP and checkpoints + for key in ("gate_up_proj", "down_proj"): + param = getattr(sparse.experts, key) + assert param is original[f"{SPARSE}.{key}"] + assert param.shape[0] == local + assert is_moe_tensor(param) and get_ep_group(param) is ep_group and is_moe_tensor_ep_fused(param) + torch.testing.assert_close( + param.data, full_state[f"{SPARSE}.{key}"][rank * local : (rank + 1) * local], rtol=0, atol=0 + ) + + # forward on this rank's batch equals the native model on the same batch; the gradient reaches the original + # Parameter objects (what optimizers and ZeRO hold) + ids = torch.randint(1, 64, (world_size * 2, 8), generator=torch.Generator().manual_seed(1)) + mine = ids[rank * 2 : (rank + 1) * 2] + out = model(input_ids=mine, labels=mine) + out.loss.backward() + with torch.no_grad(): + expected = reference(input_ids=mine).logits + torch.testing.assert_close(out.logits.detach(), expected, rtol=1e-4, atol=1e-4) + assert original[f"{SPARSE}.gate_up_proj"].grad is not None + assert original[f"{SPARSE}.gate_up_proj"].grad.shape[0] == local + + # a checkpoint in the HF layout (all experts) loads into the sharded model through the layer's load hook + for param in model.parameters(): + param.data.zero_() + model.load_state_dict(full_state, strict=True) + for key in ("gate_up_proj", "down_proj"): + torch.testing.assert_close( + getattr(sparse.experts, key).data, + full_state[f"{SPARSE}.{key}"][rank * local : (rank + 1) * local], + rtol=0, + atol=0, + ) + finally: + dist.destroy_process_group() + + +@pytest.mark.parametrize("ep_size", [2, 4]) +@rerun_if_address_is_in_use() +def test_ep_replaces_only_the_experts(ep_size): + spawn(_run_ep, ep_size)