Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/scripts/ci/run_colossalai_batch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ case "${BATCH}" in
tests/test_moe/test_grad_sync_plan.py
tests/test_numerics/test_numerics_core.py
tests/test_model_zoo/test_gemma4.py
tests/test_model_zoo/test_gemma4_moe.py
tests/test_numerics/test_numerics_data_parallel.py
tests/test_numerics/test_numerics_independent_impl.py
tests/test_numerics/test_numerics_resume.py
Expand Down Expand Up @@ -204,6 +205,7 @@ case "${BATCH}" in
tests/test_booster/test_plugin/test_torch_fsdp_plugin.py
tests/test_booster/test_plugin/test_low_level_zero_plugin.py
tests/test_booster/test_plugin/test_gemma4.py
tests/test_booster/test_plugin/test_gemma4_moe.py
tests/test_booster/test_plugin/test_gemma4_alignment.py
tests/test_booster/test_mixed_precision/test_bf16.py
tests/test_checkpoint_io/test_low_level_zero_master_resume.py
Expand Down Expand Up @@ -275,6 +277,7 @@ case "${BATCH}" in
--ignore=tests/test_booster/test_plugin/test_torch_fsdp_plugin.py
--ignore=tests/test_booster/test_plugin/test_low_level_zero_plugin.py
--ignore=tests/test_booster/test_plugin/test_gemma4.py
--ignore=tests/test_booster/test_plugin/test_gemma4_moe.py
--ignore=tests/test_booster/test_plugin/test_gemma4_alignment.py
--ignore=tests/test_booster/test_mixed_precision/test_bf16.py
--ignore=tests/test_checkpoint_io/test_low_level_zero_master_resume.py
Expand Down
1 change: 1 addition & 0 deletions .github/scripts/ci/run_cpu_unit_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ TESTS=(
tests/test_moe/test_grad_sync_plan.py
tests/test_numerics/test_numerics_core.py
tests/test_model_zoo/test_gemma4.py
tests/test_model_zoo/test_gemma4_moe.py
tests/test_numerics/test_numerics_data_parallel.py
tests/test_numerics/test_numerics_independent_impl.py
tests/test_numerics/test_numerics_resume.py
Expand Down
13 changes: 11 additions & 2 deletions colossalai/moe/dispatcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,20 @@ class NCCLAllToAllDispatcher:
Args:
num_experts: total number of experts in the layer.
ep_group: expert parallel group; experts are split evenly and contiguously over its ranks.
local_only: keep every expert on this rank, even inside a data-parallel process group. No EP collectives.
"""

def __init__(self, num_experts: int, ep_group: Optional[ProcessGroup] = None):
def __init__(
self,
num_experts: int,
ep_group: Optional[ProcessGroup] = None,
*,
local_only: bool = False,
):
if local_only and ep_group is not None:
raise ValueError("local_only dispatch cannot use an expert parallel group")
self.ep_group = ep_group
self.ep_size = dist.get_world_size(ep_group) if dist.is_initialized() else 1
self.ep_size = dist.get_world_size(ep_group) if dist.is_initialized() and not local_only else 1
if num_experts % self.ep_size != 0:
raise ValueError(f"num_experts ({num_experts}) must be divisible by the ep size ({self.ep_size})")
self.num_experts = num_experts
Expand Down
83 changes: 83 additions & 0 deletions colossalai/shardformer/modeling/gemma4.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
"""Gemma 4 replicated experts: shared dispatch/GEMM, native router and decoder semantics.

This adapter is DP-only. Expert weights retain their HF names, layout and Parameter identities; they have no EP
tags or gradient scaling. CPU uses the explicit reference backend; CUDA requires the shared grouped GEMM backend.
"""

from dataclasses import replace

import torch
import torch.nn as nn

from colossalai.moe import GroupedMMExperts, NCCLAllToAllDispatcher, ReferenceExperts


class _Gemma4LocalDispatcher(NCCLAllToAllDispatcher):
"""Shared dispatch with Gemma's native cast-per-contribution, increasing-expert-id reduction order.

The generic dispatcher's FP32 sum/cast-once changes that recipe. With local experts, restore token order and
add the top-k contributions in expert-id order in just top_k additions (no per-expert GEMM loop).
"""

def dispatch(self, hidden_states, topk_idx, topk_weights):
dispatched = super().dispatch(hidden_states, topk_idx, topk_weights)
return replace(dispatched, handle=(dispatched.handle, topk_idx))

def combine(self, expert_output, dispatched):
handle, topk_idx = dispatched.handle
order, perm, _, _, num_tokens, top_k, weights = handle
output = expert_output[perm.argsort()][order.argsort()].view(num_tokens, top_k, -1)
contributions = (output * weights.unsqueeze(-1)).to(expert_output.dtype)
by_expert = topk_idx.argsort(dim=-1).unsqueeze(-1).expand_as(contributions)
contributions = contributions.gather(1, by_expert)
result = expert_output.new_zeros(num_tokens, expert_output.size(-1))
for slot in range(top_k):
result = result + contributions[:, slot]
return result


class Gemma4GroupedExperts(nn.Module):
@staticmethod
def from_native_module(module, process_group=None, **kwargs):
# process_group belongs to TP. The policy checks TP=1 and rejects EP; do not infer EP from WORLD.
layer = Gemma4GroupedExperts()
layer.num_experts = module.num_experts
layer.hidden_dim = module.hidden_dim
layer.intermediate_dim = module.intermediate_dim
layer.act_fn = module.act_fn
layer.gate_up_proj = module.gate_up_proj
layer.down_proj = module.down_proj
layer.dispatcher = _Gemma4LocalDispatcher(module.num_experts, local_only=True)
layer.train(module.training)
return layer

def forward(self, hidden_states, top_k_index, top_k_weights):
dispatched = self.dispatcher.dispatch(hidden_states, top_k_index, top_k_weights)
gate_up, down = self.gate_up_proj, self.down_proj
if hidden_states.device.type == "cpu":
backend = ReferenceExperts()
elif hidden_states.device.type == "cuda":
backend = GroupedMMExperts()
# torch._grouped_mm is not registered for CUDA autocast like HF's F.linear. Cast explicitly so DDP/FSDP
# autocast uses the same GEMM/activation precision while gradients still reach the original parameters.
if torch.is_autocast_enabled("cuda") and hidden_states.dtype != torch.float64:
dtype = torch.get_autocast_dtype("cuda")
dispatched = replace(dispatched, tokens=dispatched.tokens.to(dtype))
gate_up, down = gate_up.to(dtype), down.to(dtype)
else:
raise NotImplementedError("Gemma 4 shared experts currently support CPU reference and CUDA grouped GEMM")
output = backend(dispatched, gate_up, down, self.act_fn)
# Under autocast GEMM returns BF16/FP16 even when the residual is FP32. Native HF combines into
# zeros_like(hidden_states), so promote before combining rather than round the combined result early.
return self.dispatcher.combine(output.to(hidden_states.dtype), dispatched)


def replace_gemma4_experts(model):
"""Adapt a Gemma text/multimodal model in place, before constructing a DP plugin, without materializing weights."""
from transformers.models.gemma4.modeling_gemma4 import Gemma4TextExperts

for parent in list(model.modules()):
for name, module in list(parent.named_children()):
if isinstance(module, Gemma4TextExperts):
setattr(parent, name, Gemma4GroupedExperts.from_native_module(module))
return model
10 changes: 7 additions & 3 deletions colossalai/shardformer/policies/gemma4.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Native Gemma 4 dense policy for data parallel training (TP=1).
"""Gemma 4 policy for data parallel training (TP=1), with shared replicated MoE experts.

Keeping the native forward preserves per-layer embeddings and cross-layer KV autograd.
No fused kernels, tensor/sequence parallelism or pipeline parallelism are implemented here.
Attention and normalization stay native; MoE uses shared dispatch/GEMM. TP/SP/PP and EP are not implemented here.
"""

from .base_policy import Policy
Expand All @@ -27,9 +27,13 @@ def config_sanity_check(self):
raise NotImplementedError("Use native Gemma 4 kernels and HF non-reentrant gradient checkpointing")

def preprocess(self):
if self.shard_config.ep_group is not None:
raise NotImplementedError("Gemma 4 currently supports replicated experts (EP=1) only")
text_config = getattr(self.model.config, "text_config", self.model.config)
if text_config.enable_moe_block:
raise NotImplementedError("This policy covers Gemma 4 dense models only")
from ..modeling.gemma4 import replace_gemma4_experts

replace_gemma4_experts(self.model)
return self.model

def module_policy(self):
Expand Down
69 changes: 69 additions & 0 deletions docs/GEMMA4-MOE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# Gemma 4 26B-A4B: replicated shared experts

The TP=1 Gemma policy replaces only the native HF expert module with
`Gemma4GroupedExperts`. The router, its RMSNorm/scale/FP32 softmax/top-k and learned
per-expert scale, and the decoder's parallel dense and routed branches remain native.
The shared dispatcher runs with `local_only=True`, so a DP process group never becomes
an EP group. Expert parameters are replicated, without EP gradient tags or scaling.

The adapter keeps the original `gate_up_proj` and `down_proj` Parameter objects and
HF checkpoint keys. Optimizers created before adaptation retain both expert tensors.
Gemma's combine casts each weighted contribution to the residual dtype and adds them
in increasing expert-id order. The adapter preserves that native order with top-k
additions, while sharing token dispatch and both grouped GEMMs. The generic FP32
sum/cast-once combine otherwise changes low-precision multi-step training.
CPU execution explicitly uses `ReferenceExperts`; CUDA uses `GroupedMMExperts` and
requires `torch._grouped_mm` (PyTorch >=2.8). No CUDA fallback is selected silently.
The development environment is Transformers 5.17.0 / PyTorch 2.13.0+cu130.

```python
from colossalai.shardformer import ShardConfig, ShardFormer

# For a native Gemma4ForCausalLM or Gemma4ForConditionalGeneration:
model, _ = ShardFormer(ShardConfig(enable_tensor_parallelism=False)).optimize(model)
```

For DDP, ZeRO, Gemini or FSDP without ShardFormer, adapt the model before constructing
the plugin/optimizer:

```python
from colossalai.shardformer.modeling.gemma4 import replace_gemma4_experts

replace_gemma4_experts(model)
```

This path supports replicated experts with TP=1, EP=1 and no PP/SP or optional fused
attention/normalization. It does not add expert-parallel training.

## Numerical validation entry

`transformers_gemma4_26b_a4b_for_causal_lm` is a structural **tiny text model**, not the
full 26B checkpoint: six layers (five sliding, one full), hidden32, dense64/expert32,
eight experts/top2, local/global KV heads2/1 and head dimensions16/32, global K=V,
window4 with sequence8, no PLE or cross-layer KV sharing, GELU and tied embeddings.

Its candidate uses the shared experts; `reference_fn` builds unmodified eager HF.
Both enable non-reentrant gradient checkpointing. The spec records tied parameters,
causal valid-token loss normalization and router choices. No expert parameter is
excluded from gradient checking.

The explicit FP16 initial loss scale is 1024 for both candidate and reference (65536
overflowed in the initial trial). The blank control uses native HF on both sides to
validate the reference recipe; the independent-implementation check still compares
the grouped candidate against native HF with the unchanged judge. BF16 autocast is
applied explicitly around grouped GEMM's inputs/weights because `torch._grouped_mm`
does not perform the `F.linear` autocast conversion itself.

```bash
python -m pytest -q tests/test_model_zoo/test_gemma4_moe.py
# Requires an allocated CUDA device; asserts actual grouped GEMM execution:
python -m pytest -q tests/test_booster/test_plugin/test_gemma4_moe.py
# Requires two allocated CUDA devices for the full DP matrix:
python -m tests.kit.numerics.acceptance \
--entry transformers_gemma4_26b_a4b_for_causal_lm --out gemma4-moe.json
```

The result file, including failures and not_run/unsupported checks, determines what
was validated. CPU reference checks and fixed-routing layer checks alone do not
establish whole-model CUDA/DP numerical acceptance. Full-size 26B training, long
contexts, multimodal behavior and performance are outside this tiny validation.
77 changes: 74 additions & 3 deletions tests/kit/model_zoo/transformers/gemma4.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
"""Small native Gemma 4 dense models retaining PLE and shared KV layers."""
"""Structural Gemma 4 tiny models: dense PLE/shared KV and 26B-A4B replicated shared experts."""

from functools import partial

import torch
import torch.nn.functional as F

from tests.kit.numerics import NumericsSpec, TokenLoss
from tests.kit.numerics.spec import Routers

from ..registry import ModelAttribute, model_zoo

Expand All @@ -16,6 +17,37 @@


def tiny_gemma4_config(variant="e2b"):
if variant == "26b_a4b":
return Gemma4TextConfig(
vocab_size=128,
hidden_size=32,
intermediate_size=64,
moe_intermediate_size=32,
num_experts=8,
top_k_experts=2,
enable_moe_block=True,
hidden_activation="gelu_pytorch_tanh",
num_hidden_layers=6,
num_attention_heads=4,
num_key_value_heads=2,
num_global_key_value_heads=1,
head_dim=16,
global_head_dim=32,
attention_k_eq_v=True,
hidden_size_per_layer_input=0,
num_kv_shared_layers=0,
use_double_wide_mlp=False,
layer_types=["sliding_attention"] * 5 + ["full_attention"],
sliding_window=4,
max_position_embeddings=64,
attention_dropout=0.0,
final_logit_softcapping=30.0,
tie_word_embeddings=True,
use_cache=False,
attn_implementation="eager",
experts_implementation="eager",
pad_token_id=0,
)
if variant not in ("e2b", "e4b"):
raise ValueError(f"Unknown dense variant: {variant}")
cycle = 5 if variant == "e2b" else 6
Expand Down Expand Up @@ -47,10 +79,24 @@ def tiny_gemma4(variant="e2b"):
return Gemma4ForCausalLM(tiny_gemma4_config(variant))


def tiny_gemma4_moe():
from colossalai.shardformer.modeling.gemma4 import replace_gemma4_experts

return replace_gemma4_experts(tiny_gemma4("26b_a4b"))


def gemma4_router(name, module):
return name.endswith(".router")


def gemma4_data():
ids = torch.tensor([[2, 5, 9, 12, 21, 34, 55, 3], [2, 7, 11, 23, 31, 47, 0, 0]])
mask = ids.ne(0).long()
return {"input_ids": ids, "attention_mask": mask, "labels": ids.masked_fill(mask == 0, -100)}
return {
"input_ids": ids,
"attention_mask": mask,
"labels": ids.masked_fill(mask == 0, -100),
}


def gemma4_numerics_data():
Expand All @@ -74,7 +120,12 @@ def gemma4_summed_token_loss(output, batch):
# Native HF causal LM loss upcasts logits to float32 and predicts each token from the preceding position.
logits = output.logits[:, :-1].float()
labels = batch["labels"][:, 1:]
return F.cross_entropy(logits.reshape(-1, logits.shape[-1]), labels.reshape(-1), ignore_index=-100, reduction="sum")
return F.cross_entropy(
logits.reshape(-1, logits.shape[-1]),
labels.reshape(-1),
ignore_index=-100,
reduction="sum",
)


def gemma4_valid_tokens(batch):
Expand All @@ -90,6 +141,26 @@ def gemma4_loss(output):


if Gemma4ForCausalLM is not None:
model_zoo.register(
"transformers_gemma4_26b_a4b_for_causal_lm",
tiny_gemma4_moe,
gemma4_numerics_data,
gemma4_output,
gemma4_loss,
ModelAttribute(
has_control_flow=True,
numerics=NumericsSpec(
reference_fn=partial(tiny_gemma4, "26b_a4b"),
blank_control_fn=partial(tiny_gemma4, "26b_a4b"),
prepare_fn=gemma4_numerics_prepare,
token_loss=TokenLoss(loss=gemma4_summed_token_loss, count=gemma4_valid_tokens),
tied_params=(("model.embed_tokens.weight", "lm_head.weight"),),
routers=Routers(select=gemma4_router, top_k=2),
# Independently observed overflow at 65536 in native/reference FP16. Applied equally to both sides.
fp16_loss_scale=1024.0,
),
),
)
for variant in ("e2b", "e4b"):
model_zoo.register(
f"transformers_gemma4_{variant}_for_causal_lm",
Expand Down
10 changes: 10 additions & 0 deletions tests/kit/numerics/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,16 @@ not cover the full model.

### Real weights: caching the reference

An independent candidate can deliberately change arithmetic (for example, shared grouped GEMM versus native HF).
Its FP32/BF16 error need not be identical to HF's, so the blank control's stricter near-identical-error requirement
must check the reference recipe on the reference model itself. Such an entry may declare
`NumericsSpec(blank_control_fn=<native reference factory>)`. This factory is used on **both** sides of the blank
control only; the candidate still runs against `reference_fn` in every independent-implementation check with the
unchanged numerical judge. The factory must expose the same parameter names (`param_map=None`); the existing
prepare/tied/unused/token/router contracts still apply. The JSON records `blank_control_factory`. Without this
explicit setting the existing blank-control behavior is unchanged. Never use this setting to bypass a failed
independent-implementation comparison.

Small random configs are cheap in float64. With real weights (`model_fn` loading them with `from_pretrained`, offline
from the shared cache) the float64 truth is the slow part, so `training_traces(..., cache_dir=...)` and
`acceptance --cache-dir` keep the reference runs and reuse them (`capture.cache`). A cached run is reused only if
Expand Down
3 changes: 3 additions & 0 deletions tests/kit/numerics/acceptance.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,9 @@ def run_acceptance(
"environment": _environment(),
"steps": steps,
"recipe_overrides": {"fp16_initial_loss_scale": spec.fp16_loss_scale},
"blank_control_factory": (
repr(spec.blank_control_fn) if spec.blank_control_fn is not None else "entry candidate (default)"
),
"results": results,
"summary": {s: sum(r["status"] == s for r in results) for s in (PASSED, FAILED, UNSUPPORTED, NOT_RUN)},
}
Expand Down
Loading
Loading