From 04321e8a4a56d248e1c6b68bacbfaf8e60ae5757 Mon Sep 17 00:00:00 2001 From: richardoo-707 Date: Thu, 19 Nov 2026 10:00:53 +0800 Subject: [PATCH 1/4] feat(gemma4): integrate replicated shared experts for 26B-A4B validation --- .github/scripts/ci/run_colossalai_batch.sh | 3 + .github/scripts/ci/run_cpu_unit_tests.sh | 1 + colossalai/moe/dispatcher.py | 33 ++++- colossalai/shardformer/modeling/gemma4.py | 47 +++++++ colossalai/shardformer/policies/gemma4.py | 8 +- tests/kit/model_zoo/transformers/gemma4.py | 72 +++++++++- .../test_plugin/test_gemma4_moe.py | 62 ++++++++ tests/test_model_zoo/test_gemma4.py | 7 +- tests/test_model_zoo/test_gemma4_moe.py | 133 ++++++++++++++++++ 9 files changed, 356 insertions(+), 10 deletions(-) create mode 100644 colossalai/shardformer/modeling/gemma4.py create mode 100644 tests/test_booster/test_plugin/test_gemma4_moe.py create mode 100644 tests/test_model_zoo/test_gemma4_moe.py diff --git a/.github/scripts/ci/run_colossalai_batch.sh b/.github/scripts/ci/run_colossalai_batch.sh index a613a0098c23..2b426a7d4f7c 100644 --- a/.github/scripts/ci/run_colossalai_batch.sh +++ b/.github/scripts/ci/run_colossalai_batch.sh @@ -87,6 +87,7 @@ case "${BATCH}" in tests/test_moe/test_grad_sync_plan.py tests/test_numerics/test_numerics_core.py tests/test_model_zoo/test_gemma4.py + tests/test_model_zoo/test_gemma4_moe.py tests/test_numerics/test_numerics_data_parallel.py tests/test_numerics/test_numerics_independent_impl.py tests/test_numerics/test_numerics_resume.py @@ -204,6 +205,7 @@ case "${BATCH}" in tests/test_booster/test_plugin/test_torch_fsdp_plugin.py tests/test_booster/test_plugin/test_low_level_zero_plugin.py tests/test_booster/test_plugin/test_gemma4.py + tests/test_booster/test_plugin/test_gemma4_moe.py tests/test_booster/test_plugin/test_gemma4_alignment.py tests/test_booster/test_mixed_precision/test_bf16.py tests/test_checkpoint_io/test_low_level_zero_master_resume.py @@ -275,6 +277,7 @@ case "${BATCH}" in --ignore=tests/test_booster/test_plugin/test_torch_fsdp_plugin.py --ignore=tests/test_booster/test_plugin/test_low_level_zero_plugin.py --ignore=tests/test_booster/test_plugin/test_gemma4.py + --ignore=tests/test_booster/test_plugin/test_gemma4_moe.py --ignore=tests/test_booster/test_plugin/test_gemma4_alignment.py --ignore=tests/test_booster/test_mixed_precision/test_bf16.py --ignore=tests/test_checkpoint_io/test_low_level_zero_master_resume.py diff --git a/.github/scripts/ci/run_cpu_unit_tests.sh b/.github/scripts/ci/run_cpu_unit_tests.sh index 9620ae744d09..96e7fe9323ec 100644 --- a/.github/scripts/ci/run_cpu_unit_tests.sh +++ b/.github/scripts/ci/run_cpu_unit_tests.sh @@ -20,6 +20,7 @@ TESTS=( tests/test_moe/test_grad_sync_plan.py tests/test_numerics/test_numerics_core.py tests/test_model_zoo/test_gemma4.py + tests/test_model_zoo/test_gemma4_moe.py tests/test_numerics/test_numerics_data_parallel.py tests/test_numerics/test_numerics_independent_impl.py tests/test_numerics/test_numerics_resume.py diff --git a/colossalai/moe/dispatcher.py b/colossalai/moe/dispatcher.py index f49800ad6bfd..9e2f7bfb8ae4 100644 --- a/colossalai/moe/dispatcher.py +++ b/colossalai/moe/dispatcher.py @@ -58,11 +58,20 @@ class NCCLAllToAllDispatcher: Args: num_experts: total number of experts in the layer. ep_group: expert parallel group; experts are split evenly and contiguously over its ranks. + local_only: keep every expert on this rank, even inside a data-parallel process group. No EP collectives. """ - def __init__(self, num_experts: int, ep_group: Optional[ProcessGroup] = None): + def __init__( + self, + num_experts: int, + ep_group: Optional[ProcessGroup] = None, + *, + local_only: bool = False, + ): + if local_only and ep_group is not None: + raise ValueError("local_only dispatch cannot use an expert parallel group") self.ep_group = ep_group - self.ep_size = dist.get_world_size(ep_group) if dist.is_initialized() else 1 + self.ep_size = dist.get_world_size(ep_group) if dist.is_initialized() and not local_only else 1 if num_experts % self.ep_size != 0: raise ValueError(f"num_experts ({num_experts}) must be divisible by the ep size ({self.ep_size})") self.num_experts = num_experts @@ -97,11 +106,27 @@ def dispatch(self, hidden_states: Tensor, topk_idx: Tensor, topk_weights: Tensor tokens=recv[perm], tokens_per_expert=recv_counts.sum(0), tokens_per_expert_cpu=tokens_per_expert_cpu, - handle=(order, perm, send_splits, recv_splits, num_tokens, top_k, topk_weights), + handle=( + order, + perm, + send_splits, + recv_splits, + num_tokens, + top_k, + topk_weights, + ), ) def combine(self, expert_output: Tensor, dispatched: DispatchOutput) -> Tensor: - order, perm, send_splits, recv_splits, num_tokens, top_k, topk_weights = dispatched.handle + ( + order, + perm, + send_splits, + recv_splits, + num_tokens, + top_k, + topk_weights, + ) = dispatched.handle out = expert_output[_inverse_permutation(perm)] if self.ep_size > 1: out, _ = all_to_all_uneven(out, recv_splits, send_splits, self.ep_group) diff --git a/colossalai/shardformer/modeling/gemma4.py b/colossalai/shardformer/modeling/gemma4.py new file mode 100644 index 000000000000..6195d7ea5461 --- /dev/null +++ b/colossalai/shardformer/modeling/gemma4.py @@ -0,0 +1,47 @@ +"""Gemma 4 replicated experts: shared dispatch/GEMM, native router and decoder semantics. + +This adapter is DP-only. Expert weights retain their HF names, layout and Parameter identities; they have no EP +tags or gradient scaling. CPU uses the explicit reference backend; CUDA requires the shared grouped GEMM backend. +""" + +import torch.nn as nn + +from colossalai.moe import GroupedMMExperts, NCCLAllToAllDispatcher, ReferenceExperts + + +class Gemma4GroupedExperts(nn.Module): + @staticmethod + def from_native_module(module, process_group=None, **kwargs): + # process_group belongs to TP. The policy checks TP=1 and rejects EP; do not infer EP from WORLD. + layer = Gemma4GroupedExperts() + layer.num_experts = module.num_experts + layer.hidden_dim = module.hidden_dim + layer.intermediate_dim = module.intermediate_dim + layer.act_fn = module.act_fn + layer.gate_up_proj = module.gate_up_proj + layer.down_proj = module.down_proj + layer.dispatcher = NCCLAllToAllDispatcher(module.num_experts, local_only=True) + layer.train(module.training) + return layer + + def forward(self, hidden_states, top_k_index, top_k_weights): + dispatched = self.dispatcher.dispatch(hidden_states, top_k_index, top_k_weights) + if hidden_states.device.type == "cpu": + backend = ReferenceExperts() + elif hidden_states.device.type == "cuda": + backend = GroupedMMExperts() + else: + raise NotImplementedError("Gemma 4 shared experts currently support CPU reference and CUDA grouped GEMM") + output = backend(dispatched, self.gate_up_proj, self.down_proj, self.act_fn) + return self.dispatcher.combine(output, dispatched) + + +def replace_gemma4_experts(model): + """Adapt a Gemma text/multimodal model in place, before constructing a DP plugin, without materializing weights.""" + from transformers.models.gemma4.modeling_gemma4 import Gemma4TextExperts + + for parent in list(model.modules()): + for name, module in list(parent.named_children()): + if isinstance(module, Gemma4TextExperts): + setattr(parent, name, Gemma4GroupedExperts.from_native_module(module)) + return model diff --git a/colossalai/shardformer/policies/gemma4.py b/colossalai/shardformer/policies/gemma4.py index 24cbe4a023dd..f105f1ca57ca 100644 --- a/colossalai/shardformer/policies/gemma4.py +++ b/colossalai/shardformer/policies/gemma4.py @@ -1,4 +1,4 @@ -"""Native Gemma 4 dense policy for data parallel training (TP=1). +"""Gemma 4 policy for data parallel training (TP=1), with shared replicated MoE experts. Keeping the native forward preserves per-layer embeddings and cross-layer KV autograd. No fused kernels, tensor/sequence parallelism or pipeline parallelism are implemented here. @@ -27,9 +27,13 @@ def config_sanity_check(self): raise NotImplementedError("Use native Gemma 4 kernels and HF non-reentrant gradient checkpointing") def preprocess(self): + if self.shard_config.ep_group is not None: + raise NotImplementedError("Gemma 4 currently supports replicated experts (EP=1) only") text_config = getattr(self.model.config, "text_config", self.model.config) if text_config.enable_moe_block: - raise NotImplementedError("This policy covers Gemma 4 dense models only") + from ..modeling.gemma4 import replace_gemma4_experts + + replace_gemma4_experts(self.model) return self.model def module_policy(self): diff --git a/tests/kit/model_zoo/transformers/gemma4.py b/tests/kit/model_zoo/transformers/gemma4.py index 288759d2dd87..c56c75803d11 100644 --- a/tests/kit/model_zoo/transformers/gemma4.py +++ b/tests/kit/model_zoo/transformers/gemma4.py @@ -6,6 +6,7 @@ import torch.nn.functional as F from tests.kit.numerics import NumericsSpec, TokenLoss +from tests.kit.numerics.spec import Routers from ..registry import ModelAttribute, model_zoo @@ -16,6 +17,37 @@ def tiny_gemma4_config(variant="e2b"): + if variant == "26b_a4b": + return Gemma4TextConfig( + vocab_size=128, + hidden_size=32, + intermediate_size=64, + moe_intermediate_size=32, + num_experts=8, + top_k_experts=2, + enable_moe_block=True, + hidden_activation="gelu_pytorch_tanh", + num_hidden_layers=6, + num_attention_heads=4, + num_key_value_heads=2, + num_global_key_value_heads=1, + head_dim=16, + global_head_dim=32, + attention_k_eq_v=True, + hidden_size_per_layer_input=0, + num_kv_shared_layers=0, + use_double_wide_mlp=False, + layer_types=["sliding_attention"] * 5 + ["full_attention"], + sliding_window=4, + max_position_embeddings=64, + attention_dropout=0.0, + final_logit_softcapping=30.0, + tie_word_embeddings=True, + use_cache=False, + attn_implementation="eager", + experts_implementation="eager", + pad_token_id=0, + ) if variant not in ("e2b", "e4b"): raise ValueError(f"Unknown dense variant: {variant}") cycle = 5 if variant == "e2b" else 6 @@ -47,10 +79,24 @@ def tiny_gemma4(variant="e2b"): return Gemma4ForCausalLM(tiny_gemma4_config(variant)) +def tiny_gemma4_moe(): + from colossalai.shardformer.modeling.gemma4 import replace_gemma4_experts + + return replace_gemma4_experts(tiny_gemma4("26b_a4b")) + + +def gemma4_router(name, module): + return name.endswith(".router") + + def gemma4_data(): ids = torch.tensor([[2, 5, 9, 12, 21, 34, 55, 3], [2, 7, 11, 23, 31, 47, 0, 0]]) mask = ids.ne(0).long() - return {"input_ids": ids, "attention_mask": mask, "labels": ids.masked_fill(mask == 0, -100)} + return { + "input_ids": ids, + "attention_mask": mask, + "labels": ids.masked_fill(mask == 0, -100), + } def gemma4_numerics_data(): @@ -74,7 +120,12 @@ def gemma4_summed_token_loss(output, batch): # Native HF causal LM loss upcasts logits to float32 and predicts each token from the preceding position. logits = output.logits[:, :-1].float() labels = batch["labels"][:, 1:] - return F.cross_entropy(logits.reshape(-1, logits.shape[-1]), labels.reshape(-1), ignore_index=-100, reduction="sum") + return F.cross_entropy( + logits.reshape(-1, logits.shape[-1]), + labels.reshape(-1), + ignore_index=-100, + reduction="sum", + ) def gemma4_valid_tokens(batch): @@ -90,6 +141,23 @@ def gemma4_loss(output): if Gemma4ForCausalLM is not None: + model_zoo.register( + "transformers_gemma4_26b_a4b_for_causal_lm", + tiny_gemma4_moe, + gemma4_numerics_data, + gemma4_output, + gemma4_loss, + ModelAttribute( + has_control_flow=True, + numerics=NumericsSpec( + reference_fn=partial(tiny_gemma4, "26b_a4b"), + prepare_fn=gemma4_numerics_prepare, + token_loss=TokenLoss(loss=gemma4_summed_token_loss, count=gemma4_valid_tokens), + tied_params=(("model.embed_tokens.weight", "lm_head.weight"),), + routers=Routers(select=gemma4_router, top_k=2), + ), + ), + ) for variant in ("e2b", "e4b"): model_zoo.register( f"transformers_gemma4_{variant}_for_causal_lm", diff --git a/tests/test_booster/test_plugin/test_gemma4_moe.py b/tests/test_booster/test_plugin/test_gemma4_moe.py new file mode 100644 index 000000000000..03b6e350abc4 --- /dev/null +++ b/tests/test_booster/test_plugin/test_gemma4_moe.py @@ -0,0 +1,62 @@ +"""Gemma shared grouped GEMM validation on allocated CUDA devices.""" + +import copy +from unittest.mock import patch + +import pytest +import torch + +from colossalai.shardformer.modeling.gemma4 import Gemma4GroupedExperts +from tests.kit.model_zoo.transformers.gemma4 import tiny_gemma4 +from tests.kit.numerics.scenarios.independent_impl import check_fwd_bwd + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("empty_experts", [False, True]) +def test_gemma4_grouped_experts(dtype, empty_experts): + assert torch.cuda.is_available(), "requires an allocated CUDA device" + torch.manual_seed(42) + reference = tiny_gemma4("26b_a4b").model.layers[0].experts.cuda() + candidate = Gemma4GroupedExperts.from_native_module(copy.deepcopy(reference)) + tokens = 19 + # Skewed routes, odd per-expert row counts and experts receiving no tokens. + ids = torch.arange(tokens, device="cuda") % (2 if empty_experts else 8) + indices = torch.stack((ids, (ids + 1) % (2 if empty_experts else 8)), -1) + weights = torch.rand(tokens, 2, device="cuda") + weights /= weights.sum(-1, keepdim=True) + inputs = dict( + hidden_states=torch.randn(tokens, 32, device="cuda"), + top_k_index=indices, + top_k_weights=weights, + ) + report = check_fwd_bwd(candidate, reference, inputs, dtype, expect_calls=((torch, "_grouped_mm"),)) + assert report.passed, report + + +def test_gemma4_grouped_model_checkpointed(): + assert torch.cuda.is_available(), "requires an allocated CUDA device" + from colossalai.shardformer.modeling.gemma4 import replace_gemma4_experts + from tests.kit.model_zoo.transformers.gemma4 import gemma4_data + + model = replace_gemma4_experts(tiny_gemma4("26b_a4b")).cuda().bfloat16() + model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + opt = torch.optim.AdamW(model.parameters(), lr=1e-3) + before = {n: p.detach().clone() for n, p in model.named_parameters()} + with patch.object(torch, "_grouped_mm", wraps=torch._grouped_mm) as grouped: + for _ in range(3): + opt.zero_grad() + loss = model(**{n: v.cuda() for n, v in gemma4_data().items()}).loss + assert torch.isfinite(loss) + loss.backward() + assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters()) + opt.step() + assert grouped.call_count >= 3 * 6 * 2 * 2 # two GEMMs, every layer, forward plus recomputation + for pattern in ( + "router.proj", + "router.scale", + "router.per_expert_scale", + "experts.gate_up_proj", + "experts.down_proj", + "mlp.", + ): + assert any(pattern in n and not torch.equal(p, before[n]) for n, p in model.named_parameters()), pattern diff --git a/tests/test_model_zoo/test_gemma4.py b/tests/test_model_zoo/test_gemma4.py index 6ee6a7578ba1..1e20a67e6694 100644 --- a/tests/test_model_zoo/test_gemma4.py +++ b/tests/test_model_zoo/test_gemma4.py @@ -66,7 +66,7 @@ def test_gemma4_hf_roundtrip(variant, tmp_path): def test_gemma4_registered(): - assert len(model_zoo.get_sub_registry("transformers_gemma4_")) == 2 + assert len(model_zoo.get_sub_registry("transformers_gemma4_")) == 3 @pytest.mark.parametrize("variant", ["e2b", "e4b"]) @@ -221,7 +221,10 @@ def _ddp_equivalence(rank, world_size, port, variant): local_batch = {k: v[rank::world_size] for k, v in batch.items()} local_tokens = local_batch["labels"][:, 1:].ne(-100).sum() # DDP averages rank gradients; compensate using the global valid-token count. - booster.backward(candidate(**local_batch).loss * (world_size * local_tokens / total_tokens), optimizer) + booster.backward( + candidate(**local_batch).loss * (world_size * local_tokens / total_tokens), + optimizer, + ) grads = lambda m: {n: p.grad.detach() for n, p in m.named_parameters()} assert_aligned(grads(accumulated), grads(reference), grads(truth)) assert_aligned(grads(candidate.unwrap()), grads(accumulated), grads(truth)) diff --git a/tests/test_model_zoo/test_gemma4_moe.py b/tests/test_model_zoo/test_gemma4_moe.py new file mode 100644 index 000000000000..c045d2cd2e5a --- /dev/null +++ b/tests/test_model_zoo/test_gemma4_moe.py @@ -0,0 +1,133 @@ +"""CPU contracts for the Gemma 26B-A4B structural tiny model; not grouped CUDA acceptance.""" + +import copy + +import pytest +import torch +import torch.distributed as dist + +from colossalai.lazy import LazyInitContext +from colossalai.shardformer import ShardConfig, ShardFormer +from colossalai.shardformer.modeling.gemma4 import Gemma4GroupedExperts, replace_gemma4_experts +from colossalai.testing import rerun_if_address_is_in_use, spawn +from tests.kit.model_zoo import model_zoo +from tests.kit.model_zoo.transformers.gemma4 import gemma4_data, tiny_gemma4 +from tests.kit.numerics.capture.training import prepare + +ENTRY = "transformers_gemma4_26b_a4b_for_causal_lm" + + +def test_moe_structure_and_independent_reference(): + s = prepare(ENTRY, steps=2) + from transformers.models.gemma4.modeling_gemma4 import Gemma4TextExperts, Gemma4TextRouter + + config = s.model.config + assert config.enable_moe_block and config.num_experts == 8 and config.top_k_experts == 2 + assert config.hidden_size_per_layer_input == config.num_kv_shared_layers == 0 + assert config.attention_k_eq_v and config.num_global_key_value_heads < config.num_key_value_heads + assert config.layer_types == ["sliding_attention"] * 5 + ["full_attention"] + assert config.hidden_activation == "gelu_pytorch_tanh" + assert s.model.is_gradient_checkpointing and s.reference.is_gradient_checkpointing + for model, cls in ( + (s.model, Gemma4GroupedExperts), + (s.reference, Gemma4TextExperts), + ): + assert model.model.embed_tokens.weight is model.lm_head.weight + for layer in model.model.layers: + assert isinstance(layer.experts, cls) + assert isinstance(layer.router, Gemma4TextRouter) + assert layer.experts.gate_up_proj.shape == (8, 64, 32) + assert layer.experts.down_proj.shape == (8, 32, 32) + for name, p in s.reference.state_dict().items(): + torch.testing.assert_close(p, s.model.state_dict()[name], rtol=0, atol=0) + assert model_zoo[ENTRY][4].numerics.unused_params == () + + +@pytest.mark.parametrize("lazy", [False, True]) +def test_moe_policy_parameter_identity(lazy): + if lazy: + with LazyInitContext(default_device="cpu"): + model = tiny_gemma4("26b_a4b") + else: + model = tiny_gemma4("26b_a4b") + original = dict(model.named_parameters()) + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3) + model, _ = ShardFormer(ShardConfig(enable_tensor_parallelism=False)).optimize(model) + if lazy: + LazyInitContext.materialize(model) + assert original.keys() == dict(model.named_parameters()).keys() + assert all(p is original[n] for n, p in model.named_parameters()) + assert {id(p) for group in optimizer.param_groups for p in group["params"]} == {id(p) for p in model.parameters()} + assert all(isinstance(l.experts, Gemma4GroupedExperts) for l in model.model.layers) + assert all(l.experts.dispatcher.ep_size == 1 for l in model.model.layers) + loss = model(**gemma4_data()).loss + loss.backward() + assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters()) + + +@pytest.mark.parametrize("checkpoint", [False, True]) +def test_moe_cpu_multistep_and_hf_roundtrip(checkpoint, tmp_path): + from transformers import Gemma4ForCausalLM + + torch.manual_seed(17) + reference = tiny_gemma4("26b_a4b").double() + candidate = replace_gemma4_experts(copy.deepcopy(reference)) + if checkpoint: + candidate.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) + opts = [torch.optim.AdamW(m.parameters(), lr=1e-3) for m in (reference, candidate)] + before = {n: p.detach().clone() for n, p in candidate.named_parameters()} + for _ in range(3): + for model, opt in zip((reference, candidate), opts): + opt.zero_grad() + loss = model(**gemma4_data()).loss + assert torch.isfinite(loss) + loss.backward() + assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters()) + opt.step() + # FP32 router softmax and shared FP32 combine remain intentional even in a double model. + torch.testing.assert_close(candidate(**gemma4_data()).loss, reference(**gemma4_data()).loss) + changed = [n for n, p in candidate.named_parameters() if not torch.equal(p, before[n])] + for pattern in ( + "router.proj", + "router.scale", + "router.per_expert_scale", + "experts.gate_up_proj", + "experts.down_proj", + "mlp.", + ): + assert any(pattern in n for n in changed), pattern + candidate.save_pretrained(tmp_path) + restored = Gemma4ForCausalLM.from_pretrained( + tmp_path, + dtype=torch.float64, + attn_implementation="eager", + experts_implementation="eager", + ) + assert restored.model.embed_tokens.weight is restored.lm_head.weight + for n, value in candidate.state_dict().items(): + torch.testing.assert_close(restored.state_dict()[n], value, rtol=0, atol=0) + torch.testing.assert_close(restored(**gemma4_data()).loss, candidate(**gemma4_data()).loss) + + +def _local_dispatch_in_dp(rank, world_size, port): + dist.init_process_group("gloo", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=world_size) + try: + from colossalai.moe import NCCLAllToAllDispatcher + from colossalai.tensor.moe_tensor.api import is_moe_tensor + + model = replace_gemma4_experts(tiny_gemma4("26b_a4b")) + assert all(l.experts.dispatcher.ep_size == 1 for l in model.model.layers) + assert all(not is_moe_tensor(p) for p in model.parameters()) + # Different ranks need not have equal batches or participate in an EP collective. + batch = {n: v[: rank + 1] for n, v in gemma4_data().items()} + model(**batch).loss.backward() + assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters()) + with pytest.raises(ValueError, match="local_only"): + NCCLAllToAllDispatcher(8, dist.group.WORLD, local_only=True) + finally: + dist.destroy_process_group() + + +@rerun_if_address_is_in_use() +def test_moe_local_dispatch_inside_two_rank_dp(): + spawn(_local_dispatch_in_dp, 2) From c67c568254475b701ae27818ef5a5725abed7e8d Mon Sep 17 00:00:00 2001 From: richardoo-707 Date: Thu, 19 Nov 2026 10:05:08 +0800 Subject: [PATCH 2/4] fix(gemma4): preserve residual precision under autocast and validate layer config --- colossalai/shardformer/modeling/gemma4.py | 4 +++- tests/test_booster/test_plugin/test_gemma4_moe.py | 5 +++-- tests/test_model_zoo/test_gemma4_moe.py | 6 +++++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/colossalai/shardformer/modeling/gemma4.py b/colossalai/shardformer/modeling/gemma4.py index 6195d7ea5461..44cfaf6c273a 100644 --- a/colossalai/shardformer/modeling/gemma4.py +++ b/colossalai/shardformer/modeling/gemma4.py @@ -33,7 +33,9 @@ def forward(self, hidden_states, top_k_index, top_k_weights): else: raise NotImplementedError("Gemma 4 shared experts currently support CPU reference and CUDA grouped GEMM") output = backend(dispatched, self.gate_up_proj, self.down_proj, self.act_fn) - return self.dispatcher.combine(output, dispatched) + # Under autocast GEMM returns BF16/FP16 even when the residual is FP32. Native HF combines into + # zeros_like(hidden_states), so promote before combining rather than round the combined result early. + return self.dispatcher.combine(output.to(hidden_states.dtype), dispatched) def replace_gemma4_experts(model): diff --git a/tests/test_booster/test_plugin/test_gemma4_moe.py b/tests/test_booster/test_plugin/test_gemma4_moe.py index 03b6e350abc4..599c4dbe2a65 100644 --- a/tests/test_booster/test_plugin/test_gemma4_moe.py +++ b/tests/test_booster/test_plugin/test_gemma4_moe.py @@ -38,14 +38,15 @@ def test_gemma4_grouped_model_checkpointed(): from colossalai.shardformer.modeling.gemma4 import replace_gemma4_experts from tests.kit.model_zoo.transformers.gemma4 import gemma4_data - model = replace_gemma4_experts(tiny_gemma4("26b_a4b")).cuda().bfloat16() + model = replace_gemma4_experts(tiny_gemma4("26b_a4b")).cuda() model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False}) opt = torch.optim.AdamW(model.parameters(), lr=1e-3) before = {n: p.detach().clone() for n, p in model.named_parameters()} with patch.object(torch, "_grouped_mm", wraps=torch._grouped_mm) as grouped: for _ in range(3): opt.zero_grad() - loss = model(**{n: v.cuda() for n, v in gemma4_data().items()}).loss + with torch.autocast("cuda", dtype=torch.bfloat16): + loss = model(**{n: v.cuda() for n, v in gemma4_data().items()}).loss assert torch.isfinite(loss) loss.backward() assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters()) diff --git a/tests/test_model_zoo/test_gemma4_moe.py b/tests/test_model_zoo/test_gemma4_moe.py index c045d2cd2e5a..db8c56f8089f 100644 --- a/tests/test_model_zoo/test_gemma4_moe.py +++ b/tests/test_model_zoo/test_gemma4_moe.py @@ -24,7 +24,11 @@ def test_moe_structure_and_independent_reference(): config = s.model.config assert config.enable_moe_block and config.num_experts == 8 and config.top_k_experts == 2 assert config.hidden_size_per_layer_input == config.num_kv_shared_layers == 0 - assert config.attention_k_eq_v and config.num_global_key_value_heads < config.num_key_value_heads + assert config.attention_k_eq_v + assert config.per_layer_config[5].num_key_value_heads == 1 + assert config.per_layer_config[0].num_key_value_heads == 2 + assert s.model.model.layers[5].self_attn.v_proj is None + assert s.model.model.layers[0].self_attn.v_proj is not None assert config.layer_types == ["sliding_attention"] * 5 + ["full_attention"] assert config.hidden_activation == "gelu_pytorch_tanh" assert s.model.is_gradient_checkpointing and s.reference.is_gradient_checkpointing From e3b858da85847ffa8523197ebc7232d3b81aaffb Mon Sep 17 00:00:00 2001 From: richardoo-707 Date: Thu, 19 Nov 2026 10:09:41 +0800 Subject: [PATCH 3/4] fix(gemma4): match autocast recipe and add explicit reference blank control --- colossalai/moe/dispatcher.py | 20 +----- colossalai/shardformer/modeling/gemma4.py | 12 +++- colossalai/shardformer/policies/gemma4.py | 2 +- docs/GEMMA4-MOE.md | 65 +++++++++++++++++++ tests/kit/model_zoo/transformers/gemma4.py | 5 +- tests/kit/numerics/README.md | 10 +++ tests/kit/numerics/acceptance.py | 3 + .../numerics/scenarios/independent_impl.py | 14 ++++ tests/kit/numerics/spec.py | 5 ++ tests/test_booster/test_plugin/test_gemma4.py | 8 ++- tests/test_model_zoo/test_gemma4.py | 5 +- .../test_numerics_independent_impl.py | 13 ++++ 12 files changed, 135 insertions(+), 27 deletions(-) create mode 100644 docs/GEMMA4-MOE.md diff --git a/colossalai/moe/dispatcher.py b/colossalai/moe/dispatcher.py index 9e2f7bfb8ae4..1dbf42dbac75 100644 --- a/colossalai/moe/dispatcher.py +++ b/colossalai/moe/dispatcher.py @@ -106,27 +106,11 @@ def dispatch(self, hidden_states: Tensor, topk_idx: Tensor, topk_weights: Tensor tokens=recv[perm], tokens_per_expert=recv_counts.sum(0), tokens_per_expert_cpu=tokens_per_expert_cpu, - handle=( - order, - perm, - send_splits, - recv_splits, - num_tokens, - top_k, - topk_weights, - ), + handle=(order, perm, send_splits, recv_splits, num_tokens, top_k, topk_weights), ) def combine(self, expert_output: Tensor, dispatched: DispatchOutput) -> Tensor: - ( - order, - perm, - send_splits, - recv_splits, - num_tokens, - top_k, - topk_weights, - ) = dispatched.handle + order, perm, send_splits, recv_splits, num_tokens, top_k, topk_weights = dispatched.handle out = expert_output[_inverse_permutation(perm)] if self.ep_size > 1: out, _ = all_to_all_uneven(out, recv_splits, send_splits, self.ep_group) diff --git a/colossalai/shardformer/modeling/gemma4.py b/colossalai/shardformer/modeling/gemma4.py index 44cfaf6c273a..80b41e5c8dcb 100644 --- a/colossalai/shardformer/modeling/gemma4.py +++ b/colossalai/shardformer/modeling/gemma4.py @@ -4,6 +4,9 @@ tags or gradient scaling. CPU uses the explicit reference backend; CUDA requires the shared grouped GEMM backend. """ +from dataclasses import replace + +import torch import torch.nn as nn from colossalai.moe import GroupedMMExperts, NCCLAllToAllDispatcher, ReferenceExperts @@ -26,13 +29,20 @@ def from_native_module(module, process_group=None, **kwargs): def forward(self, hidden_states, top_k_index, top_k_weights): dispatched = self.dispatcher.dispatch(hidden_states, top_k_index, top_k_weights) + gate_up, down = self.gate_up_proj, self.down_proj if hidden_states.device.type == "cpu": backend = ReferenceExperts() elif hidden_states.device.type == "cuda": backend = GroupedMMExperts() + # torch._grouped_mm is not registered for CUDA autocast like HF's F.linear. Cast explicitly so DDP/FSDP + # autocast uses the same GEMM/activation precision while gradients still reach the original parameters. + if torch.is_autocast_enabled("cuda") and hidden_states.dtype != torch.float64: + dtype = torch.get_autocast_dtype("cuda") + dispatched = replace(dispatched, tokens=dispatched.tokens.to(dtype)) + gate_up, down = gate_up.to(dtype), down.to(dtype) else: raise NotImplementedError("Gemma 4 shared experts currently support CPU reference and CUDA grouped GEMM") - output = backend(dispatched, self.gate_up_proj, self.down_proj, self.act_fn) + output = backend(dispatched, gate_up, down, self.act_fn) # Under autocast GEMM returns BF16/FP16 even when the residual is FP32. Native HF combines into # zeros_like(hidden_states), so promote before combining rather than round the combined result early. return self.dispatcher.combine(output.to(hidden_states.dtype), dispatched) diff --git a/colossalai/shardformer/policies/gemma4.py b/colossalai/shardformer/policies/gemma4.py index f105f1ca57ca..728911c69698 100644 --- a/colossalai/shardformer/policies/gemma4.py +++ b/colossalai/shardformer/policies/gemma4.py @@ -1,7 +1,7 @@ """Gemma 4 policy for data parallel training (TP=1), with shared replicated MoE experts. Keeping the native forward preserves per-layer embeddings and cross-layer KV autograd. -No fused kernels, tensor/sequence parallelism or pipeline parallelism are implemented here. +Attention and normalization stay native; MoE uses shared dispatch/GEMM. TP/SP/PP and EP are not implemented here. """ from .base_policy import Policy diff --git a/docs/GEMMA4-MOE.md b/docs/GEMMA4-MOE.md new file mode 100644 index 000000000000..b076cbcb96cc --- /dev/null +++ b/docs/GEMMA4-MOE.md @@ -0,0 +1,65 @@ +# Gemma 4 26B-A4B: replicated shared experts + +The TP=1 Gemma policy replaces only the native HF expert module with +`Gemma4GroupedExperts`. The router, its RMSNorm/scale/FP32 softmax/top-k and learned +per-expert scale, and the decoder's parallel dense and routed branches remain native. +The shared dispatcher runs with `local_only=True`, so a DP process group never becomes +an EP group. Expert parameters are replicated, without EP gradient tags or scaling. + +The adapter keeps the original `gate_up_proj` and `down_proj` Parameter objects and +HF checkpoint keys. Optimizers created before adaptation retain both expert tensors. +CPU execution explicitly uses `ReferenceExperts`; CUDA uses `GroupedMMExperts` and +requires `torch._grouped_mm` (PyTorch >=2.8). No CUDA fallback is selected silently. +The development environment is Transformers 5.17.0 / PyTorch 2.13.0+cu130. + +```python +from colossalai.shardformer import ShardConfig, ShardFormer + +# For a native Gemma4ForCausalLM or Gemma4ForConditionalGeneration: +model, _ = ShardFormer(ShardConfig(enable_tensor_parallelism=False)).optimize(model) +``` + +For DDP, ZeRO, Gemini or FSDP without ShardFormer, adapt the model before constructing +the plugin/optimizer: + +```python +from colossalai.shardformer.modeling.gemma4 import replace_gemma4_experts + +replace_gemma4_experts(model) +``` + +This path supports replicated experts with TP=1, EP=1 and no PP/SP or optional fused +attention/normalization. It does not add expert-parallel training. + +## Numerical validation entry + +`transformers_gemma4_26b_a4b_for_causal_lm` is a structural **tiny text model**, not the +full 26B checkpoint: six layers (five sliding, one full), hidden32, dense64/expert32, +eight experts/top2, local/global KV heads2/1 and head dimensions16/32, global K=V, +window4 with sequence8, no PLE or cross-layer KV sharing, GELU and tied embeddings. + +Its candidate uses the shared experts; `reference_fn` builds unmodified eager HF. +Both enable non-reentrant gradient checkpointing. The spec records tied parameters, +causal valid-token loss normalization and router choices. No expert parameter is +excluded from gradient checking. + +The explicit FP16 initial loss scale is 1024 for both candidate and reference (65536 +overflowed in the initial trial). The blank control uses native HF on both sides to +validate the reference recipe; the independent-implementation check still compares +the grouped candidate against native HF with the unchanged judge. BF16 autocast is +applied explicitly around grouped GEMM's inputs/weights because `torch._grouped_mm` +does not perform the `F.linear` autocast conversion itself. + +```bash +python -m pytest -q tests/test_model_zoo/test_gemma4_moe.py +# Requires an allocated CUDA device; asserts actual grouped GEMM execution: +python -m pytest -q tests/test_booster/test_plugin/test_gemma4_moe.py +# Requires two allocated CUDA devices for the full DP matrix: +python -m tests.kit.numerics.acceptance \ + --entry transformers_gemma4_26b_a4b_for_causal_lm --out gemma4-moe.json +``` + +The result file, including failures and not_run/unsupported checks, determines what +was validated. CPU reference checks and fixed-routing layer checks alone do not +establish whole-model CUDA/DP numerical acceptance. Full-size 26B training, long +contexts, multimodal behavior and performance are outside this tiny validation. diff --git a/tests/kit/model_zoo/transformers/gemma4.py b/tests/kit/model_zoo/transformers/gemma4.py index c56c75803d11..08a71aae7abc 100644 --- a/tests/kit/model_zoo/transformers/gemma4.py +++ b/tests/kit/model_zoo/transformers/gemma4.py @@ -1,4 +1,4 @@ -"""Small native Gemma 4 dense models retaining PLE and shared KV layers.""" +"""Structural Gemma 4 tiny models: dense PLE/shared KV and 26B-A4B replicated shared experts.""" from functools import partial @@ -151,10 +151,13 @@ def gemma4_loss(output): has_control_flow=True, numerics=NumericsSpec( reference_fn=partial(tiny_gemma4, "26b_a4b"), + blank_control_fn=partial(tiny_gemma4, "26b_a4b"), prepare_fn=gemma4_numerics_prepare, token_loss=TokenLoss(loss=gemma4_summed_token_loss, count=gemma4_valid_tokens), tied_params=(("model.embed_tokens.weight", "lm_head.weight"),), routers=Routers(select=gemma4_router, top_k=2), + # Independently observed overflow at 65536 in native/reference FP16. Applied equally to both sides. + fp16_loss_scale=1024.0, ), ), ) diff --git a/tests/kit/numerics/README.md b/tests/kit/numerics/README.md index 995cf8a2300a..6885b4651578 100644 --- a/tests/kit/numerics/README.md +++ b/tests/kit/numerics/README.md @@ -325,6 +325,16 @@ not cover the full model. ### Real weights: caching the reference +An independent candidate can deliberately change arithmetic (for example, shared grouped GEMM versus native HF). +Its FP32/BF16 error need not be identical to HF's, so the blank control's stricter near-identical-error requirement +must check the reference recipe on the reference model itself. Such an entry may declare +`NumericsSpec(blank_control_fn=)`. This factory is used on **both** sides of the blank +control only; the candidate still runs against `reference_fn` in every independent-implementation check with the +unchanged numerical judge. The factory must expose the same parameter names (`param_map=None`); the existing +prepare/tied/unused/token/router contracts still apply. The JSON records `blank_control_factory`. Without this +explicit setting the existing blank-control behavior is unchanged. Never use this setting to bypass a failed +independent-implementation comparison. + Small random configs are cheap in float64. With real weights (`model_fn` loading them with `from_pretrained`, offline from the shared cache) the float64 truth is the slow part, so `training_traces(..., cache_dir=...)` and `acceptance --cache-dir` keep the reference runs and reuse them (`capture.cache`). A cached run is reused only if diff --git a/tests/kit/numerics/acceptance.py b/tests/kit/numerics/acceptance.py index fa6cdc547c22..7d3a460c0d5b 100644 --- a/tests/kit/numerics/acceptance.py +++ b/tests/kit/numerics/acceptance.py @@ -166,6 +166,9 @@ def run_acceptance( "environment": _environment(), "steps": steps, "recipe_overrides": {"fp16_initial_loss_scale": spec.fp16_loss_scale}, + "blank_control_factory": ( + repr(spec.blank_control_fn) if spec.blank_control_fn is not None else "entry candidate (default)" + ), "results": results, "summary": {s: sum(r["status"] == s for r in results) for s in (PASSED, FAILED, UNSUPPORTED, NOT_RUN)}, } diff --git a/tests/kit/numerics/scenarios/independent_impl.py b/tests/kit/numerics/scenarios/independent_impl.py index 1ce5f5e97609..5f85d6519756 100644 --- a/tests/kit/numerics/scenarios/independent_impl.py +++ b/tests/kit/numerics/scenarios/independent_impl.py @@ -22,6 +22,8 @@ runs come from nondeterministic kernels, and the worse of the two runs is used as ``ref`` for them. """ +import copy +from dataclasses import replace from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union import torch @@ -182,6 +184,18 @@ def blank_control( precision="bf16")`` (``TorchDDPPlugin()`` in float32 on CPU, where LowLevelZero cannot run), the same AdamW on both sides. Every tensor's error must then be within ``slack`` times the baseline error. """ + if isinstance(entry, str): + from tests.kit.model_zoo import model_zoo + + entry = model_zoo[entry] + model_fn, data_fn, output_fn, loss_fn, attribute = entry + spec = attribute.numerics + if spec.blank_control_fn is not None: + if spec.param_map is not None: + raise ValueError("blank_control_fn requires identical parameter names (param_map=None)") + attribute = copy.copy(attribute) + attribute.numerics = replace(spec, reference_fn=None, blank_control_fn=None) + entry = (spec.blank_control_fn, data_fn, output_fn, loss_fn, attribute) if get_accelerator().name == "cpu": plugin = TorchDDPPlugin() else: diff --git a/tests/kit/numerics/spec.py b/tests/kit/numerics/spec.py index 071d24d58124..d8027182f343 100644 --- a/tests/kit/numerics/spec.py +++ b/tests/kit/numerics/spec.py @@ -95,6 +95,10 @@ class NumericsSpec: fp16_loss_scale: Optional initial FP16 loss scale for acceptance recipes, applied equally to reference and plugin and recorded in the report. Models can need a smaller scale than the default 65536. Overflow and skipped optimizer updates still fail; this does not change the numerical judge or its tolerance. + blank_control_fn: Optional same-name model factory for the blank control only, when the candidate deliberately + changes arithmetic (e.g. grouped GEMM versus native HF). It runs on both sides of that control to validate + the reference precision recipe itself. Independent-implementation checks still run the original candidate + against reference_fn, with the normal judge. Requires param_map=None; no parameter renaming is inferred. """ reference_fn: Optional[Callable[[], nn.Module]] = None @@ -107,3 +111,4 @@ class NumericsSpec: tied_params: Tuple[Tuple[str, ...], ...] = () # Initial scale for both FP16 acceptance runs (reference and plugin). Overflow remains a failure. fp16_loss_scale: Optional[float] = None + blank_control_fn: Optional[Callable[[], nn.Module]] = None diff --git a/tests/test_booster/test_plugin/test_gemma4.py b/tests/test_booster/test_plugin/test_gemma4.py index eb99b446d656..8fd8b6386988 100644 --- a/tests/test_booster/test_plugin/test_gemma4.py +++ b/tests/test_booster/test_plugin/test_gemma4.py @@ -86,14 +86,18 @@ def _run(rank, world_size, port, variant, plugin_name, checkpoint, lazy): assert all(torch.isfinite(p).all() for p in after.values()) changed = [n for n, p in after.items() if not torch.equal(before[n], p)] assert changed, "No parameter changed" - assert any("embed_tokens_per_layer" in n for n in changed), "No PLE update" + if variant == "26b_a4b": + for pattern in ("router.proj", "experts.gate_up_proj", "experts.down_proj", "mlp."): + assert any(pattern in n for n in changed), f"No {pattern} update" + else: + assert any("embed_tokens_per_layer" in n for n in changed), "No PLE update" finally: accelerator.empty_cache() dist.destroy_process_group() @pytest.mark.parametrize("world_size", [1, 2]) -@pytest.mark.parametrize("variant", ["e2b", "e4b"]) +@pytest.mark.parametrize("variant", ["e2b", "e4b", "26b_a4b"]) @pytest.mark.parametrize("plugin", ["ddp", "ddp_bf16", "zero1", "zero2", "gemini", "fsdp"]) @pytest.mark.parametrize("checkpoint,lazy", [(False, False), (True, True)]) def test_gemma4_training(world_size, variant, plugin, checkpoint, lazy): diff --git a/tests/test_model_zoo/test_gemma4.py b/tests/test_model_zoo/test_gemma4.py index 1e20a67e6694..68e459ed0db3 100644 --- a/tests/test_model_zoo/test_gemma4.py +++ b/tests/test_model_zoo/test_gemma4.py @@ -221,10 +221,7 @@ def _ddp_equivalence(rank, world_size, port, variant): local_batch = {k: v[rank::world_size] for k, v in batch.items()} local_tokens = local_batch["labels"][:, 1:].ne(-100).sum() # DDP averages rank gradients; compensate using the global valid-token count. - booster.backward( - candidate(**local_batch).loss * (world_size * local_tokens / total_tokens), - optimizer, - ) + booster.backward(candidate(**local_batch).loss * (world_size * local_tokens / total_tokens), optimizer) grads = lambda m: {n: p.grad.detach() for n, p in m.named_parameters()} assert_aligned(grads(accumulated), grads(reference), grads(truth)) assert_aligned(grads(candidate.unwrap()), grads(accumulated), grads(truth)) diff --git a/tests/test_numerics/test_numerics_independent_impl.py b/tests/test_numerics/test_numerics_independent_impl.py index 9eb67f884289..b5b8ad102db7 100644 --- a/tests/test_numerics/test_numerics_independent_impl.py +++ b/tests/test_numerics/test_numerics_independent_impl.py @@ -63,6 +63,11 @@ def forward(self, x): return {"y": self.out(h)} +class WrongTinyMLP(TinyMLP): + def forward(self, x): + return {"y": super().forward(x)["y"] * 2} + + PARAMS = [f"fc{i}.{p}" for i in (1, 2) for p in ("weight", "bias")] TRACE_NAMES = ( {"out/y", "grad/x"} @@ -112,6 +117,14 @@ def _check_blank_control(rank, world_size, port): blank_control(_entry(), steps=STEPS) with pytest.raises(AssertionError, match="blank control"): blank_control(_entry(BUGGY)) + # A declared self-control validates the reference recipe. It must not turn the candidate comparison into + # a self-comparison: this deliberately broken candidate still fails the independent check. + spec = NumericsSpec(reference_fn=TinyMLP, blank_control_fn=TinyMLP) + entry = (WrongTinyMLP, _data, lambda out: out, None, _attribute(spec)) + blank_control(entry, steps=STEPS).assert_pass() + assert not check_training(entry, TorchDDPPlugin(), steps=STEPS).passed + with pytest.raises(ValueError, match="param_map=None"): + blank_control(_entry(NumericsSpec(blank_control_fn=TinyMLP, param_map=_rename))) @rerun_if_address_is_in_use() From c0578194496dce463fd51b31fefe766a3472a169 Mon Sep 17 00:00:00 2001 From: richardoo-707 Date: Thu, 19 Nov 2026 11:23:51 +0800 Subject: [PATCH 4/4] fix(gemma4): preserve native expert contribution rounding and reduction order --- colossalai/shardformer/modeling/gemma4.py | 26 ++++++++++++++++++++++- docs/GEMMA4-MOE.md | 4 ++++ tests/test_model_zoo/test_gemma4_moe.py | 13 ++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/colossalai/shardformer/modeling/gemma4.py b/colossalai/shardformer/modeling/gemma4.py index 80b41e5c8dcb..029411989771 100644 --- a/colossalai/shardformer/modeling/gemma4.py +++ b/colossalai/shardformer/modeling/gemma4.py @@ -12,6 +12,30 @@ from colossalai.moe import GroupedMMExperts, NCCLAllToAllDispatcher, ReferenceExperts +class _Gemma4LocalDispatcher(NCCLAllToAllDispatcher): + """Shared dispatch with Gemma's native cast-per-contribution, increasing-expert-id reduction order. + + The generic dispatcher's FP32 sum/cast-once changes that recipe. With local experts, restore token order and + add the top-k contributions in expert-id order in just top_k additions (no per-expert GEMM loop). + """ + + def dispatch(self, hidden_states, topk_idx, topk_weights): + dispatched = super().dispatch(hidden_states, topk_idx, topk_weights) + return replace(dispatched, handle=(dispatched.handle, topk_idx)) + + def combine(self, expert_output, dispatched): + handle, topk_idx = dispatched.handle + order, perm, _, _, num_tokens, top_k, weights = handle + output = expert_output[perm.argsort()][order.argsort()].view(num_tokens, top_k, -1) + contributions = (output * weights.unsqueeze(-1)).to(expert_output.dtype) + by_expert = topk_idx.argsort(dim=-1).unsqueeze(-1).expand_as(contributions) + contributions = contributions.gather(1, by_expert) + result = expert_output.new_zeros(num_tokens, expert_output.size(-1)) + for slot in range(top_k): + result = result + contributions[:, slot] + return result + + class Gemma4GroupedExperts(nn.Module): @staticmethod def from_native_module(module, process_group=None, **kwargs): @@ -23,7 +47,7 @@ def from_native_module(module, process_group=None, **kwargs): layer.act_fn = module.act_fn layer.gate_up_proj = module.gate_up_proj layer.down_proj = module.down_proj - layer.dispatcher = NCCLAllToAllDispatcher(module.num_experts, local_only=True) + layer.dispatcher = _Gemma4LocalDispatcher(module.num_experts, local_only=True) layer.train(module.training) return layer diff --git a/docs/GEMMA4-MOE.md b/docs/GEMMA4-MOE.md index b076cbcb96cc..fbaf5b22b942 100644 --- a/docs/GEMMA4-MOE.md +++ b/docs/GEMMA4-MOE.md @@ -8,6 +8,10 @@ an EP group. Expert parameters are replicated, without EP gradient tags or scali The adapter keeps the original `gate_up_proj` and `down_proj` Parameter objects and HF checkpoint keys. Optimizers created before adaptation retain both expert tensors. +Gemma's combine casts each weighted contribution to the residual dtype and adds them +in increasing expert-id order. The adapter preserves that native order with top-k +additions, while sharing token dispatch and both grouped GEMMs. The generic FP32 +sum/cast-once combine otherwise changes low-precision multi-step training. CPU execution explicitly uses `ReferenceExperts`; CUDA uses `GroupedMMExperts` and requires `torch._grouped_mm` (PyTorch >=2.8). No CUDA fallback is selected silently. The development environment is Transformers 5.17.0 / PyTorch 2.13.0+cu130. diff --git a/tests/test_model_zoo/test_gemma4_moe.py b/tests/test_model_zoo/test_gemma4_moe.py index db8c56f8089f..b8c6a7420388 100644 --- a/tests/test_model_zoo/test_gemma4_moe.py +++ b/tests/test_model_zoo/test_gemma4_moe.py @@ -17,6 +17,19 @@ ENTRY = "transformers_gemma4_26b_a4b_for_causal_lm" +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +def test_moe_native_low_precision_reduction(dtype): + torch.manual_seed(17) + reference = tiny_gemma4("26b_a4b").model.layers[0].experts.to(dtype) + candidate = Gemma4GroupedExperts.from_native_module(copy.deepcopy(reference)) + # Expert-id order differs from router slot order; four contributions expose cast-once versus cast-per-expert. + indices = (torch.arange(11)[:, None] + torch.tensor([5, 1, 7, 3])) % 8 + weights = torch.rand(11, 4) + weights /= weights.sum(-1, keepdim=True) + inputs = torch.randn(11, 32).to(dtype) + torch.testing.assert_close(candidate(inputs, indices, weights), reference(inputs, indices, weights), rtol=0, atol=0) + + def test_moe_structure_and_independent_reference(): s = prepare(ENTRY, steps=2) from transformers.models.gemma4.modeling_gemma4 import Gemma4TextExperts, Gemma4TextRouter