diff --git a/.github/scripts/ci/run_colossalai_batch.sh b/.github/scripts/ci/run_colossalai_batch.sh index a613a0098c23..2b426a7d4f7c 100644 --- a/.github/scripts/ci/run_colossalai_batch.sh +++ b/.github/scripts/ci/run_colossalai_batch.sh @@ -87,6 +87,7 @@ case "${BATCH}" in tests/test_moe/test_grad_sync_plan.py tests/test_numerics/test_numerics_core.py tests/test_model_zoo/test_gemma4.py + tests/test_model_zoo/test_gemma4_moe.py tests/test_numerics/test_numerics_data_parallel.py tests/test_numerics/test_numerics_independent_impl.py tests/test_numerics/test_numerics_resume.py @@ -204,6 +205,7 @@ case "${BATCH}" in tests/test_booster/test_plugin/test_torch_fsdp_plugin.py tests/test_booster/test_plugin/test_low_level_zero_plugin.py tests/test_booster/test_plugin/test_gemma4.py + tests/test_booster/test_plugin/test_gemma4_moe.py tests/test_booster/test_plugin/test_gemma4_alignment.py tests/test_booster/test_mixed_precision/test_bf16.py tests/test_checkpoint_io/test_low_level_zero_master_resume.py @@ -275,6 +277,7 @@ case "${BATCH}" in --ignore=tests/test_booster/test_plugin/test_torch_fsdp_plugin.py --ignore=tests/test_booster/test_plugin/test_low_level_zero_plugin.py --ignore=tests/test_booster/test_plugin/test_gemma4.py + --ignore=tests/test_booster/test_plugin/test_gemma4_moe.py --ignore=tests/test_booster/test_plugin/test_gemma4_alignment.py --ignore=tests/test_booster/test_mixed_precision/test_bf16.py --ignore=tests/test_checkpoint_io/test_low_level_zero_master_resume.py diff --git a/.github/scripts/ci/run_cpu_unit_tests.sh b/.github/scripts/ci/run_cpu_unit_tests.sh index 9620ae744d09..96e7fe9323ec 100644 --- a/.github/scripts/ci/run_cpu_unit_tests.sh +++ b/.github/scripts/ci/run_cpu_unit_tests.sh @@ -20,6 +20,7 @@ TESTS=( tests/test_moe/test_grad_sync_plan.py tests/test_numerics/test_numerics_core.py tests/test_model_zoo/test_gemma4.py + tests/test_model_zoo/test_gemma4_moe.py tests/test_numerics/test_numerics_data_parallel.py tests/test_numerics/test_numerics_independent_impl.py tests/test_numerics/test_numerics_resume.py diff --git a/colossalai/moe/dispatcher.py b/colossalai/moe/dispatcher.py index f49800ad6bfd..1dbf42dbac75 100644 --- a/colossalai/moe/dispatcher.py +++ b/colossalai/moe/dispatcher.py @@ -58,11 +58,20 @@ class NCCLAllToAllDispatcher: Args: num_experts: total number of experts in the layer. ep_group: expert parallel group; experts are split evenly and contiguously over its ranks. + local_only: keep every expert on this rank, even inside a data-parallel process group. No EP collectives. """ - def __init__(self, num_experts: int, ep_group: Optional[ProcessGroup] = None): + def __init__( + self, + num_experts: int, + ep_group: Optional[ProcessGroup] = None, + *, + local_only: bool = False, + ): + if local_only and ep_group is not None: + raise ValueError("local_only dispatch cannot use an expert parallel group") self.ep_group = ep_group - self.ep_size = dist.get_world_size(ep_group) if dist.is_initialized() else 1 + self.ep_size = dist.get_world_size(ep_group) if dist.is_initialized() and not local_only else 1 if num_experts % self.ep_size != 0: raise ValueError(f"num_experts ({num_experts}) must be divisible by the ep size ({self.ep_size})") self.num_experts = num_experts diff --git a/colossalai/shardformer/modeling/gemma4.py b/colossalai/shardformer/modeling/gemma4.py new file mode 100644 index 000000000000..029411989771 --- /dev/null +++ b/colossalai/shardformer/modeling/gemma4.py @@ -0,0 +1,83 @@ +"""Gemma 4 replicated experts: shared dispatch/GEMM, native router and decoder semantics. + +This adapter is DP-only. Expert weights retain their HF names, layout and Parameter identities; they have no EP +tags or gradient scaling. CPU uses the explicit reference backend; CUDA requires the shared grouped GEMM backend. +""" + +from dataclasses import replace + +import torch +import torch.nn as nn + +from colossalai.moe import GroupedMMExperts, NCCLAllToAllDispatcher, ReferenceExperts + + +class _Gemma4LocalDispatcher(NCCLAllToAllDispatcher): + """Shared dispatch with Gemma's native cast-per-contribution, increasing-expert-id reduction order. + + The generic dispatcher's FP32 sum/cast-once changes that recipe. With local experts, restore token order and + add the top-k contributions in expert-id order in just top_k additions (no per-expert GEMM loop). + """ + + def dispatch(self, hidden_states, topk_idx, topk_weights): + dispatched = super().dispatch(hidden_states, topk_idx, topk_weights) + return replace(dispatched, handle=(dispatched.handle, topk_idx)) + + def combine(self, expert_output, dispatched): + handle, topk_idx = dispatched.handle + order, perm, _, _, num_tokens, top_k, weights = handle + output = expert_output[perm.argsort()][order.argsort()].view(num_tokens, top_k, -1) + contributions = (output * weights.unsqueeze(-1)).to(expert_output.dtype) + by_expert = topk_idx.argsort(dim=-1).unsqueeze(-1).expand_as(contributions) + contributions = contributions.gather(1, by_expert) + result = expert_output.new_zeros(num_tokens, expert_output.size(-1)) + for slot in range(top_k): + result = result + contributions[:, slot] + return result + + +class Gemma4GroupedExperts(nn.Module): + @staticmethod + def from_native_module(module, process_group=None, **kwargs): + # process_group belongs to TP. The policy checks TP=1 and rejects EP; do not infer EP from WORLD. + layer = Gemma4GroupedExperts() + layer.num_experts = module.num_experts + layer.hidden_dim = module.hidden_dim + layer.intermediate_dim = module.intermediate_dim + layer.act_fn = module.act_fn + layer.gate_up_proj = module.gate_up_proj + layer.down_proj = module.down_proj + layer.dispatcher = _Gemma4LocalDispatcher(module.num_experts, local_only=True) + layer.train(module.training) + return layer + + def forward(self, hidden_states, top_k_index, top_k_weights): + dispatched = self.dispatcher.dispatch(hidden_states, top_k_index, top_k_weights) + gate_up, down = self.gate_up_proj, self.down_proj + if hidden_states.device.type == "cpu": + backend = ReferenceExperts() + elif hidden_states.device.type == "cuda": + backend = GroupedMMExperts() + # torch._grouped_mm is not registered for CUDA autocast like HF's F.linear. Cast explicitly so DDP/FSDP + # autocast uses the same GEMM/activation precision while gradients still reach the original parameters. + if torch.is_autocast_enabled("cuda") and hidden_states.dtype != torch.float64: + dtype = torch.get_autocast_dtype("cuda") + dispatched = replace(dispatched, tokens=dispatched.tokens.to(dtype)) + gate_up, down = gate_up.to(dtype), down.to(dtype) + else: + raise NotImplementedError("Gemma 4 shared experts currently support CPU reference and CUDA grouped GEMM") + output = backend(dispatched, gate_up, down, self.act_fn) + # Under autocast GEMM returns BF16/FP16 even when the residual is FP32. Native HF combines into + # zeros_like(hidden_states), so promote before combining rather than round the combined result early. + return self.dispatcher.combine(output.to(hidden_states.dtype), dispatched) + + +def replace_gemma4_experts(model): + """Adapt a Gemma text/multimodal model in place, before constructing a DP plugin, without materializing weights.""" + from transformers.models.gemma4.modeling_gemma4 import Gemma4TextExperts + + for parent in list(model.modules()): + for name, module in list(parent.named_children()): + if isinstance(module, Gemma4TextExperts): + setattr(parent, name, Gemma4GroupedExperts.from_native_module(module)) + return model diff --git a/colossalai/shardformer/policies/gemma4.py b/colossalai/shardformer/policies/gemma4.py index 24cbe4a023dd..728911c69698 100644 --- a/colossalai/shardformer/policies/gemma4.py +++ b/colossalai/shardformer/policies/gemma4.py @@ -1,7 +1,7 @@ -"""Native Gemma 4 dense policy for data parallel training (TP=1). +"""Gemma 4 policy for data parallel training (TP=1), with shared replicated MoE experts. Keeping the native forward preserves per-layer embeddings and cross-layer KV autograd. -No fused kernels, tensor/sequence parallelism or pipeline parallelism are implemented here. +Attention and normalization stay native; MoE uses shared dispatch/GEMM. TP/SP/PP and EP are not implemented here. """ from .base_policy import Policy @@ -27,9 +27,13 @@ def config_sanity_check(self): raise NotImplementedError("Use native Gemma 4 kernels and HF non-reentrant gradient checkpointing") def preprocess(self): + if self.shard_config.ep_group is not None: + raise NotImplementedError("Gemma 4 currently supports replicated experts (EP=1) only") text_config = getattr(self.model.config, "text_config", self.model.config) if text_config.enable_moe_block: - raise NotImplementedError("This policy covers Gemma 4 dense models only") + from ..modeling.gemma4 import replace_gemma4_experts + + replace_gemma4_experts(self.model) return self.model def module_policy(self): diff --git a/docs/GEMMA4-MOE.md b/docs/GEMMA4-MOE.md new file mode 100644 index 000000000000..fbaf5b22b942 --- /dev/null +++ b/docs/GEMMA4-MOE.md @@ -0,0 +1,69 @@ +# Gemma 4 26B-A4B: replicated shared experts + +The TP=1 Gemma policy replaces only the native HF expert module with +`Gemma4GroupedExperts`. The router, its RMSNorm/scale/FP32 softmax/top-k and learned +per-expert scale, and the decoder's parallel dense and routed branches remain native. +The shared dispatcher runs with `local_only=True`, so a DP process group never becomes +an EP group. Expert parameters are replicated, without EP gradient tags or scaling. + +The adapter keeps the original `gate_up_proj` and `down_proj` Parameter objects and +HF checkpoint keys. Optimizers created before adaptation retain both expert tensors. +Gemma's combine casts each weighted contribution to the residual dtype and adds them +in increasing expert-id order. The adapter preserves that native order with top-k +additions, while sharing token dispatch and both grouped GEMMs. The generic FP32 +sum/cast-once combine otherwise changes low-precision multi-step training. +CPU execution explicitly uses `ReferenceExperts`; CUDA uses `GroupedMMExperts` and +requires `torch._grouped_mm` (PyTorch >=2.8). No CUDA fallback is selected silently. +The development environment is Transformers 5.17.0 / PyTorch 2.13.0+cu130. + +```python +from colossalai.shardformer import ShardConfig, ShardFormer + +# For a native Gemma4ForCausalLM or Gemma4ForConditionalGeneration: +model, _ = ShardFormer(ShardConfig(enable_tensor_parallelism=False)).optimize(model) +``` + +For DDP, ZeRO, Gemini or FSDP without ShardFormer, adapt the model before constructing +the plugin/optimizer: + +```python +from colossalai.shardformer.modeling.gemma4 import replace_gemma4_experts + +replace_gemma4_experts(model) +``` + +This path supports replicated experts with TP=1, EP=1 and no PP/SP or optional fused +attention/normalization. It does not add expert-parallel training. + +## Numerical validation entry + +`transformers_gemma4_26b_a4b_for_causal_lm` is a structural **tiny text model**, not the +full 26B checkpoint: six layers (five sliding, one full), hidden32, dense64/expert32, +eight experts/top2, local/global KV heads2/1 and head dimensions16/32, global K=V, +window4 with sequence8, no PLE or cross-layer KV sharing, GELU and tied embeddings. + +Its candidate uses the shared experts; `reference_fn` builds unmodified eager HF. +Both enable non-reentrant gradient checkpointing. The spec records tied parameters, +causal valid-token loss normalization and router choices. No expert parameter is +excluded from gradient checking. + +The explicit FP16 initial loss scale is 1024 for both candidate and reference (65536 +overflowed in the initial trial). The blank control uses native HF on both sides to +validate the reference recipe; the independent-implementation check still compares +the grouped candidate against native HF with the unchanged judge. BF16 autocast is +applied explicitly around grouped GEMM's inputs/weights because `torch._grouped_mm` +does not perform the `F.linear` autocast conversion itself. + +```bash +python -m pytest -q tests/test_model_zoo/test_gemma4_moe.py +# Requires an allocated CUDA device; asserts actual grouped GEMM execution: +python -m pytest -q tests/test_booster/test_plugin/test_gemma4_moe.py +# Requires two allocated CUDA devices for the full DP matrix: +python -m tests.kit.numerics.acceptance \ + --entry transformers_gemma4_26b_a4b_for_causal_lm --out gemma4-moe.json +``` + +The result file, including failures and not_run/unsupported checks, determines what +was validated. CPU reference checks and fixed-routing layer checks alone do not +establish whole-model CUDA/DP numerical acceptance. Full-size 26B training, long +contexts, multimodal behavior and performance are outside this tiny validation. diff --git a/tests/kit/model_zoo/transformers/gemma4.py b/tests/kit/model_zoo/transformers/gemma4.py index 288759d2dd87..08a71aae7abc 100644 --- a/tests/kit/model_zoo/transformers/gemma4.py +++ b/tests/kit/model_zoo/transformers/gemma4.py @@ -1,4 +1,4 @@ -"""Small native Gemma 4 dense models retaining PLE and shared KV layers.""" +"""Structural Gemma 4 tiny models: dense PLE/shared KV and 26B-A4B replicated shared experts.""" from functools import partial @@ -6,6 +6,7 @@ import torch.nn.functional as F from tests.kit.numerics import NumericsSpec, TokenLoss +from tests.kit.numerics.spec import Routers from ..registry import ModelAttribute, model_zoo @@ -16,6 +17,37 @@ def tiny_gemma4_config(variant="e2b"): + if variant == "26b_a4b": + return Gemma4TextConfig( + vocab_size=128, + hidden_size=32, + intermediate_size=64, + moe_intermediate_size=32, + num_experts=8, + top_k_experts=2, + enable_moe_block=True, + hidden_activation="gelu_pytorch_tanh", + num_hidden_layers=6, + num_attention_heads=4, + num_key_value_heads=2, + num_global_key_value_heads=1, + head_dim=16, + global_head_dim=32, + attention_k_eq_v=True, + hidden_size_per_layer_input=0, + num_kv_shared_layers=0, + use_double_wide_mlp=False, + layer_types=["sliding_attention"] * 5 + ["full_attention"], + sliding_window=4, + max_position_embeddings=64, + attention_dropout=0.0, + final_logit_softcapping=30.0, + tie_word_embeddings=True, + use_cache=False, + attn_implementation="eager", + experts_implementation="eager", + pad_token_id=0, + ) if variant not in ("e2b", "e4b"): raise ValueError(f"Unknown dense variant: {variant}") cycle = 5 if variant == "e2b" else 6 @@ -47,10 +79,24 @@ def tiny_gemma4(variant="e2b"): return Gemma4ForCausalLM(tiny_gemma4_config(variant)) +def tiny_gemma4_moe(): + from colossalai.shardformer.modeling.gemma4 import replace_gemma4_experts + + return replace_gemma4_experts(tiny_gemma4("26b_a4b")) + + +def gemma4_router(name, module): + return name.endswith(".router") + + def gemma4_data(): ids = torch.tensor([[2, 5, 9, 12, 21, 34, 55, 3], [2, 7, 11, 23, 31, 47, 0, 0]]) mask = ids.ne(0).long() - return {"input_ids": ids, "attention_mask": mask, "labels": ids.masked_fill(mask == 0, -100)} + return { + "input_ids": ids, + "attention_mask": mask, + "labels": ids.masked_fill(mask == 0, -100), + } def gemma4_numerics_data(): @@ -74,7 +120,12 @@ def gemma4_summed_token_loss(output, batch): # Native HF causal LM loss upcasts logits to float32 and predicts each token from the preceding position. logits = output.logits[:, :-1].float() labels = batch["labels"][:, 1:] - return F.cross_entropy(logits.reshape(-1, logits.shape[-1]), labels.reshape(-1), ignore_index=-100, reduction="sum") + return F.cross_entropy( + logits.reshape(-1, logits.shape[-1]), + labels.reshape(-1), + ignore_index=-100, + reduction="sum", + ) def gemma4_valid_tokens(batch): @@ -90,6 +141,26 @@ def gemma4_loss(output): if Gemma4ForCausalLM is not None: + model_zoo.register( + "transformers_gemma4_26b_a4b_for_causal_lm", + tiny_gemma4_moe, + gemma4_numerics_data, + gemma4_output, + gemma4_loss, + ModelAttribute( + has_control_flow=True, + numerics=NumericsSpec( + reference_fn=partial(tiny_gemma4, "26b_a4b"), + blank_control_fn=partial(tiny_gemma4, "26b_a4b"), + prepare_fn=gemma4_numerics_prepare, + token_loss=TokenLoss(loss=gemma4_summed_token_loss, count=gemma4_valid_tokens), + tied_params=(("model.embed_tokens.weight", "lm_head.weight"),), + routers=Routers(select=gemma4_router, top_k=2), + # Independently observed overflow at 65536 in native/reference FP16. Applied equally to both sides. + fp16_loss_scale=1024.0, + ), + ), + ) for variant in ("e2b", "e4b"): model_zoo.register( f"transformers_gemma4_{variant}_for_causal_lm", diff --git a/tests/kit/numerics/README.md b/tests/kit/numerics/README.md index 995cf8a2300a..6885b4651578 100644 --- a/tests/kit/numerics/README.md +++ b/tests/kit/numerics/README.md @@ -325,6 +325,16 @@ not cover the full model. ### Real weights: caching the reference +An independent candidate can deliberately change arithmetic (for example, shared grouped GEMM versus native HF). +Its FP32/BF16 error need not be identical to HF's, so the blank control's stricter near-identical-error requirement +must check the reference recipe on the reference model itself. Such an entry may declare +`NumericsSpec(blank_control_fn=)`. This factory is used on **both** sides of the blank +control only; the candidate still runs against `reference_fn` in every independent-implementation check with the +unchanged numerical judge. The factory must expose the same parameter names (`param_map=None`); the existing +prepare/tied/unused/token/router contracts still apply. The JSON records `blank_control_factory`. Without this +explicit setting the existing blank-control behavior is unchanged. Never use this setting to bypass a failed +independent-implementation comparison. + Small random configs are cheap in float64. With real weights (`model_fn` loading them with `from_pretrained`, offline from the shared cache) the float64 truth is the slow part, so `training_traces(..., cache_dir=...)` and `acceptance --cache-dir` keep the reference runs and reuse them (`capture.cache`). A cached run is reused only if diff --git a/tests/kit/numerics/acceptance.py b/tests/kit/numerics/acceptance.py index fa6cdc547c22..7d3a460c0d5b 100644 --- a/tests/kit/numerics/acceptance.py +++ b/tests/kit/numerics/acceptance.py @@ -166,6 +166,9 @@ def run_acceptance( "environment": _environment(), "steps": steps, "recipe_overrides": {"fp16_initial_loss_scale": spec.fp16_loss_scale}, + "blank_control_factory": ( + repr(spec.blank_control_fn) if spec.blank_control_fn is not None else "entry candidate (default)" + ), "results": results, "summary": {s: sum(r["status"] == s for r in results) for s in (PASSED, FAILED, UNSUPPORTED, NOT_RUN)}, } diff --git a/tests/kit/numerics/scenarios/independent_impl.py b/tests/kit/numerics/scenarios/independent_impl.py index 1ce5f5e97609..5f85d6519756 100644 --- a/tests/kit/numerics/scenarios/independent_impl.py +++ b/tests/kit/numerics/scenarios/independent_impl.py @@ -22,6 +22,8 @@ runs come from nondeterministic kernels, and the worse of the two runs is used as ``ref`` for them. """ +import copy +from dataclasses import replace from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union import torch @@ -182,6 +184,18 @@ def blank_control( precision="bf16")`` (``TorchDDPPlugin()`` in float32 on CPU, where LowLevelZero cannot run), the same AdamW on both sides. Every tensor's error must then be within ``slack`` times the baseline error. """ + if isinstance(entry, str): + from tests.kit.model_zoo import model_zoo + + entry = model_zoo[entry] + model_fn, data_fn, output_fn, loss_fn, attribute = entry + spec = attribute.numerics + if spec.blank_control_fn is not None: + if spec.param_map is not None: + raise ValueError("blank_control_fn requires identical parameter names (param_map=None)") + attribute = copy.copy(attribute) + attribute.numerics = replace(spec, reference_fn=None, blank_control_fn=None) + entry = (spec.blank_control_fn, data_fn, output_fn, loss_fn, attribute) if get_accelerator().name == "cpu": plugin = TorchDDPPlugin() else: diff --git a/tests/kit/numerics/spec.py b/tests/kit/numerics/spec.py index 071d24d58124..d8027182f343 100644 --- a/tests/kit/numerics/spec.py +++ b/tests/kit/numerics/spec.py @@ -95,6 +95,10 @@ class NumericsSpec: fp16_loss_scale: Optional initial FP16 loss scale for acceptance recipes, applied equally to reference and plugin and recorded in the report. Models can need a smaller scale than the default 65536. Overflow and skipped optimizer updates still fail; this does not change the numerical judge or its tolerance. + blank_control_fn: Optional same-name model factory for the blank control only, when the candidate deliberately + changes arithmetic (e.g. grouped GEMM versus native HF). It runs on both sides of that control to validate + the reference precision recipe itself. Independent-implementation checks still run the original candidate + against reference_fn, with the normal judge. Requires param_map=None; no parameter renaming is inferred. """ reference_fn: Optional[Callable[[], nn.Module]] = None @@ -107,3 +111,4 @@ class NumericsSpec: tied_params: Tuple[Tuple[str, ...], ...] = () # Initial scale for both FP16 acceptance runs (reference and plugin). Overflow remains a failure. fp16_loss_scale: Optional[float] = None + blank_control_fn: Optional[Callable[[], nn.Module]] = None diff --git a/tests/test_booster/test_plugin/test_gemma4.py b/tests/test_booster/test_plugin/test_gemma4.py index eb99b446d656..8fd8b6386988 100644 --- a/tests/test_booster/test_plugin/test_gemma4.py +++ b/tests/test_booster/test_plugin/test_gemma4.py @@ -86,14 +86,18 @@ def _run(rank, world_size, port, variant, plugin_name, checkpoint, lazy): assert all(torch.isfinite(p).all() for p in after.values()) changed = [n for n, p in after.items() if not torch.equal(before[n], p)] assert changed, "No parameter changed" - assert any("embed_tokens_per_layer" in n for n in changed), "No PLE update" + if variant == "26b_a4b": + for pattern in ("router.proj", "experts.gate_up_proj", "experts.down_proj", "mlp."): + assert any(pattern in n for n in changed), f"No {pattern} update" + else: + assert any("embed_tokens_per_layer" in n for n in changed), "No PLE update" finally: accelerator.empty_cache() dist.destroy_process_group() @pytest.mark.parametrize("world_size", [1, 2]) -@pytest.mark.parametrize("variant", ["e2b", "e4b"]) +@pytest.mark.parametrize("variant", ["e2b", "e4b", "26b_a4b"]) @pytest.mark.parametrize("plugin", ["ddp", "ddp_bf16", "zero1", "zero2", "gemini", "fsdp"]) @pytest.mark.parametrize("checkpoint,lazy", [(False, False), (True, True)]) def test_gemma4_training(world_size, variant, plugin, checkpoint, lazy): diff --git a/tests/test_booster/test_plugin/test_gemma4_moe.py b/tests/test_booster/test_plugin/test_gemma4_moe.py new file mode 100644 index 000000000000..599c4dbe2a65 --- /dev/null +++ b/tests/test_booster/test_plugin/test_gemma4_moe.py @@ -0,0 +1,63 @@ +"""Gemma shared grouped GEMM validation on allocated CUDA devices.""" + +import copy +from unittest.mock import patch + +import pytest +import torch + +from colossalai.shardformer.modeling.gemma4 import Gemma4GroupedExperts +from tests.kit.model_zoo.transformers.gemma4 import tiny_gemma4 +from tests.kit.numerics.scenarios.independent_impl import check_fwd_bwd + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("empty_experts", [False, True]) +def test_gemma4_grouped_experts(dtype, empty_experts): + assert torch.cuda.is_available(), "requires an allocated CUDA device" + torch.manual_seed(42) + reference = tiny_gemma4("26b_a4b").model.layers[0].experts.cuda() + candidate = Gemma4GroupedExperts.from_native_module(copy.deepcopy(reference)) + tokens = 19 + # Skewed routes, odd per-expert row counts and experts receiving no tokens. + ids = torch.arange(tokens, device="cuda") % (2 if empty_experts else 8) + indices = torch.stack((ids, (ids + 1) % (2 if empty_experts else 8)), -1) + weights = torch.rand(tokens, 2, device="cuda") + weights /= weights.sum(-1, keepdim=True) + inputs = dict( + hidden_states=torch.randn(tokens, 32, device="cuda"), + top_k_index=indices, + top_k_weights=weights, + ) + report = check_fwd_bwd(candidate, reference, inputs, dtype, expect_calls=((torch, "_grouped_mm"),)) + assert report.passed, report + + +def test_gemma4_grouped_model_checkpointed(): + assert torch.cuda.is_available(), "requires an allocated CUDA device" + from colossalai.shardformer.modeling.gemma4 import replace_gemma4_experts + from tests.kit.model_zoo.transformers.gemma4 import gemma4_data + + model = replace_gemma4_experts(tiny_gemma4("26b_a4b")).cuda() + model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + opt = torch.optim.AdamW(model.parameters(), lr=1e-3) + before = {n: p.detach().clone() for n, p in model.named_parameters()} + with patch.object(torch, "_grouped_mm", wraps=torch._grouped_mm) as grouped: + for _ in range(3): + opt.zero_grad() + with torch.autocast("cuda", dtype=torch.bfloat16): + loss = model(**{n: v.cuda() for n, v in gemma4_data().items()}).loss + assert torch.isfinite(loss) + loss.backward() + assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters()) + opt.step() + assert grouped.call_count >= 3 * 6 * 2 * 2 # two GEMMs, every layer, forward plus recomputation + for pattern in ( + "router.proj", + "router.scale", + "router.per_expert_scale", + "experts.gate_up_proj", + "experts.down_proj", + "mlp.", + ): + assert any(pattern in n and not torch.equal(p, before[n]) for n, p in model.named_parameters()), pattern diff --git a/tests/test_model_zoo/test_gemma4.py b/tests/test_model_zoo/test_gemma4.py index 6ee6a7578ba1..68e459ed0db3 100644 --- a/tests/test_model_zoo/test_gemma4.py +++ b/tests/test_model_zoo/test_gemma4.py @@ -66,7 +66,7 @@ def test_gemma4_hf_roundtrip(variant, tmp_path): def test_gemma4_registered(): - assert len(model_zoo.get_sub_registry("transformers_gemma4_")) == 2 + assert len(model_zoo.get_sub_registry("transformers_gemma4_")) == 3 @pytest.mark.parametrize("variant", ["e2b", "e4b"]) diff --git a/tests/test_model_zoo/test_gemma4_moe.py b/tests/test_model_zoo/test_gemma4_moe.py new file mode 100644 index 000000000000..b8c6a7420388 --- /dev/null +++ b/tests/test_model_zoo/test_gemma4_moe.py @@ -0,0 +1,150 @@ +"""CPU contracts for the Gemma 26B-A4B structural tiny model; not grouped CUDA acceptance.""" + +import copy + +import pytest +import torch +import torch.distributed as dist + +from colossalai.lazy import LazyInitContext +from colossalai.shardformer import ShardConfig, ShardFormer +from colossalai.shardformer.modeling.gemma4 import Gemma4GroupedExperts, replace_gemma4_experts +from colossalai.testing import rerun_if_address_is_in_use, spawn +from tests.kit.model_zoo import model_zoo +from tests.kit.model_zoo.transformers.gemma4 import gemma4_data, tiny_gemma4 +from tests.kit.numerics.capture.training import prepare + +ENTRY = "transformers_gemma4_26b_a4b_for_causal_lm" + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +def test_moe_native_low_precision_reduction(dtype): + torch.manual_seed(17) + reference = tiny_gemma4("26b_a4b").model.layers[0].experts.to(dtype) + candidate = Gemma4GroupedExperts.from_native_module(copy.deepcopy(reference)) + # Expert-id order differs from router slot order; four contributions expose cast-once versus cast-per-expert. + indices = (torch.arange(11)[:, None] + torch.tensor([5, 1, 7, 3])) % 8 + weights = torch.rand(11, 4) + weights /= weights.sum(-1, keepdim=True) + inputs = torch.randn(11, 32).to(dtype) + torch.testing.assert_close(candidate(inputs, indices, weights), reference(inputs, indices, weights), rtol=0, atol=0) + + +def test_moe_structure_and_independent_reference(): + s = prepare(ENTRY, steps=2) + from transformers.models.gemma4.modeling_gemma4 import Gemma4TextExperts, Gemma4TextRouter + + config = s.model.config + assert config.enable_moe_block and config.num_experts == 8 and config.top_k_experts == 2 + assert config.hidden_size_per_layer_input == config.num_kv_shared_layers == 0 + assert config.attention_k_eq_v + assert config.per_layer_config[5].num_key_value_heads == 1 + assert config.per_layer_config[0].num_key_value_heads == 2 + assert s.model.model.layers[5].self_attn.v_proj is None + assert s.model.model.layers[0].self_attn.v_proj is not None + assert config.layer_types == ["sliding_attention"] * 5 + ["full_attention"] + assert config.hidden_activation == "gelu_pytorch_tanh" + assert s.model.is_gradient_checkpointing and s.reference.is_gradient_checkpointing + for model, cls in ( + (s.model, Gemma4GroupedExperts), + (s.reference, Gemma4TextExperts), + ): + assert model.model.embed_tokens.weight is model.lm_head.weight + for layer in model.model.layers: + assert isinstance(layer.experts, cls) + assert isinstance(layer.router, Gemma4TextRouter) + assert layer.experts.gate_up_proj.shape == (8, 64, 32) + assert layer.experts.down_proj.shape == (8, 32, 32) + for name, p in s.reference.state_dict().items(): + torch.testing.assert_close(p, s.model.state_dict()[name], rtol=0, atol=0) + assert model_zoo[ENTRY][4].numerics.unused_params == () + + +@pytest.mark.parametrize("lazy", [False, True]) +def test_moe_policy_parameter_identity(lazy): + if lazy: + with LazyInitContext(default_device="cpu"): + model = tiny_gemma4("26b_a4b") + else: + model = tiny_gemma4("26b_a4b") + original = dict(model.named_parameters()) + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3) + model, _ = ShardFormer(ShardConfig(enable_tensor_parallelism=False)).optimize(model) + if lazy: + LazyInitContext.materialize(model) + assert original.keys() == dict(model.named_parameters()).keys() + assert all(p is original[n] for n, p in model.named_parameters()) + assert {id(p) for group in optimizer.param_groups for p in group["params"]} == {id(p) for p in model.parameters()} + assert all(isinstance(l.experts, Gemma4GroupedExperts) for l in model.model.layers) + assert all(l.experts.dispatcher.ep_size == 1 for l in model.model.layers) + loss = model(**gemma4_data()).loss + loss.backward() + assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters()) + + +@pytest.mark.parametrize("checkpoint", [False, True]) +def test_moe_cpu_multistep_and_hf_roundtrip(checkpoint, tmp_path): + from transformers import Gemma4ForCausalLM + + torch.manual_seed(17) + reference = tiny_gemma4("26b_a4b").double() + candidate = replace_gemma4_experts(copy.deepcopy(reference)) + if checkpoint: + candidate.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + opts = [torch.optim.AdamW(m.parameters(), lr=1e-3) for m in (reference, candidate)] + before = {n: p.detach().clone() for n, p in candidate.named_parameters()} + for _ in range(3): + for model, opt in zip((reference, candidate), opts): + opt.zero_grad() + loss = model(**gemma4_data()).loss + assert torch.isfinite(loss) + loss.backward() + assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters()) + opt.step() + # FP32 router softmax and shared FP32 combine remain intentional even in a double model. + torch.testing.assert_close(candidate(**gemma4_data()).loss, reference(**gemma4_data()).loss) + changed = [n for n, p in candidate.named_parameters() if not torch.equal(p, before[n])] + for pattern in ( + "router.proj", + "router.scale", + "router.per_expert_scale", + "experts.gate_up_proj", + "experts.down_proj", + "mlp.", + ): + assert any(pattern in n for n in changed), pattern + candidate.save_pretrained(tmp_path) + restored = Gemma4ForCausalLM.from_pretrained( + tmp_path, + dtype=torch.float64, + attn_implementation="eager", + experts_implementation="eager", + ) + assert restored.model.embed_tokens.weight is restored.lm_head.weight + for n, value in candidate.state_dict().items(): + torch.testing.assert_close(restored.state_dict()[n], value, rtol=0, atol=0) + torch.testing.assert_close(restored(**gemma4_data()).loss, candidate(**gemma4_data()).loss) + + +def _local_dispatch_in_dp(rank, world_size, port): + dist.init_process_group("gloo", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=world_size) + try: + from colossalai.moe import NCCLAllToAllDispatcher + from colossalai.tensor.moe_tensor.api import is_moe_tensor + + model = replace_gemma4_experts(tiny_gemma4("26b_a4b")) + assert all(l.experts.dispatcher.ep_size == 1 for l in model.model.layers) + assert all(not is_moe_tensor(p) for p in model.parameters()) + # Different ranks need not have equal batches or participate in an EP collective. + batch = {n: v[: rank + 1] for n, v in gemma4_data().items()} + model(**batch).loss.backward() + assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters()) + with pytest.raises(ValueError, match="local_only"): + NCCLAllToAllDispatcher(8, dist.group.WORLD, local_only=True) + finally: + dist.destroy_process_group() + + +@rerun_if_address_is_in_use() +def test_moe_local_dispatch_inside_two_rank_dp(): + spawn(_local_dispatch_in_dp, 2) diff --git a/tests/test_numerics/test_numerics_independent_impl.py b/tests/test_numerics/test_numerics_independent_impl.py index 9eb67f884289..b5b8ad102db7 100644 --- a/tests/test_numerics/test_numerics_independent_impl.py +++ b/tests/test_numerics/test_numerics_independent_impl.py @@ -63,6 +63,11 @@ def forward(self, x): return {"y": self.out(h)} +class WrongTinyMLP(TinyMLP): + def forward(self, x): + return {"y": super().forward(x)["y"] * 2} + + PARAMS = [f"fc{i}.{p}" for i in (1, 2) for p in ("weight", "bias")] TRACE_NAMES = ( {"out/y", "grad/x"} @@ -112,6 +117,14 @@ def _check_blank_control(rank, world_size, port): blank_control(_entry(), steps=STEPS) with pytest.raises(AssertionError, match="blank control"): blank_control(_entry(BUGGY)) + # A declared self-control validates the reference recipe. It must not turn the candidate comparison into + # a self-comparison: this deliberately broken candidate still fails the independent check. + spec = NumericsSpec(reference_fn=TinyMLP, blank_control_fn=TinyMLP) + entry = (WrongTinyMLP, _data, lambda out: out, None, _attribute(spec)) + blank_control(entry, steps=STEPS).assert_pass() + assert not check_training(entry, TorchDDPPlugin(), steps=STEPS).passed + with pytest.raises(ValueError, match="param_map=None"): + blank_control(_entry(NumericsSpec(blank_control_fn=TinyMLP, param_map=_rename))) @rerun_if_address_is_in_use()