Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/scripts/ci/run_cpu_unit_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ TESTS=(
tests/test_numerics/test_numerics_data_parallel.py
tests/test_numerics/test_numerics_independent_impl.py
tests/test_numerics/test_numerics_resume.py
tests/test_optimizer/test_dist_muon.py
tests/test_optimizer/test_lr_scheduler.py
tests/test_optimizer/test_muon.py
tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py
Expand Down
6 changes: 6 additions & 0 deletions colossalai/booster/plugin/gemini_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from colossalai.cluster import DistCoordinator, ProcessGroupMesh
from colossalai.interface import ModelWrapper, OptimizerWrapper
from colossalai.logging import get_dist_logger
from colossalai.nn.optimizer import Muon
from colossalai.shardformer import ShardConfig, ShardFormer
from colossalai.zero import GeminiDDP, GeminiOptimizer
from colossalai.zero.gemini.memory_tracer import MemStats
Expand Down Expand Up @@ -665,6 +666,11 @@ def configure(
dataloader: Optional[DataLoader] = None,
lr_scheduler: Optional[LRScheduler] = None,
) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]:
if isinstance(optimizer, Muon):
raise NotImplementedError(
"GeminiPlugin does not support Muon: its chunks split matrices at arbitrary boundaries. "
"Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin."
)
params_info = get_param_info(optimizer)
if not isinstance(model, ModelWrapper):
# convert model to sync bn
Expand Down
6 changes: 6 additions & 0 deletions colossalai/booster/plugin/torch_fsdp_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
from colossalai.cluster import DistCoordinator
from colossalai.interface import ModelWrapper, OptimizerWrapper
from colossalai.logging import get_dist_logger
from colossalai.nn.optimizer import Muon
from colossalai.utils.safetensors import load_flat

from .dp_plugin_base import DPPluginBase
Expand Down Expand Up @@ -536,6 +537,11 @@ def configure(
dataloader: Optional[DataLoader] = None,
lr_scheduler: Optional[LRScheduler] = None,
) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]:
if isinstance(optimizer, Muon):
raise NotImplementedError(
"TorchFSDPPlugin does not support Muon: FSDP hands the optimizer flattened parameters in one group. "
"Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin."
)
# wrap the model with PyTorch FSDP
fsdp_model = TorchFSDPModel(model, device_id=torch.cuda.current_device(), **self.fsdp_kwargs)

Expand Down
155 changes: 136 additions & 19 deletions colossalai/nn/optimizer/muon.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@
norms, biases) go to the AdamW branch of the same optimizer, as in KellerJordan/Muon's ``MuonWithAuxAdam`` and
Moonlight, so a Booster still receives a single optimizer.

Under LowLevelZeRO (stage 1 and 2) each rank holds a flat 1/N slice of every parameter, while Newton-Schulz needs whole
matrices. As in Moonlight's distributed Muon, each rank updates the momentum of its own slices, the bf16
Newton-Schulz inputs are all-gathered over the data-parallel group, and every rank orthogonalizes the whole matrices
and keeps the slice of each update it owns. The gathered input is the bf16 cast the unsharded iteration makes anyway,
so a sharded step computes the unsharded one, at the cost of one bf16 all-gather of the Muon parameters per step and
every rank orthogonalizing all of them.

References:
- https://kellerjordan.github.io/posts/muon/
- Muon is Scalable for LLM Training, https://arxiv.org/abs/2502.16982
Expand All @@ -14,9 +21,13 @@
from typing import Dict, List, Optional, Tuple

import torch
import torch.distributed as dist
import torch.nn as nn
import torch.nn.functional as F
from torch import Tensor
from torch.optim import Optimizer

from colossalai.accelerator import get_accelerator
from colossalai.interface.optimizer import DistributedOptim

__all__ = ["Muon", "muon_param_groups", "zeropower_via_newtonschulz"]

Expand Down Expand Up @@ -63,7 +74,7 @@ def _adjust_lr(lr: float, adjust_lr_fn: Optional[str], shape: torch.Size) -> flo
return lr * 0.2 * math.sqrt(max(rows, cols))


class Muon(Optimizer):
class Muon(DistributedOptim):
"""Muon for 2D hidden-layer weights plus AdamW for every other parameter, in one optimizer.

Each parameter group sets ``use_muon``; ``muon_param_groups(model)`` builds both groups from a model. Groups
Expand All @@ -72,6 +83,11 @@ class Muon(Optimizer):
``betas`` and ``eps``. Missing keys fall back to the constructor arguments, with ``adamw_eps`` as the AdamW groups'
``eps``.

``LowLevelZeroPlugin`` and ``HybridParallelPlugin`` call ``setup_distributed`` on this very object (it is not
replaced, so an LR scheduler built on it before ``boost`` keeps driving it), after which ZeRO's parameter shards are
orthogonalized as whole matrices (see the module docstring). The AdamW branch is elementwise and runs on the
shards as they are. Tensor parallelism, Gemini and FSDP are not supported.

Args:
params: parameters or parameter groups.
lr (float): learning rate shared by both branches unless a group overrides it (default: 1e-3).
Expand All @@ -88,6 +104,10 @@ class Muon(Optimizer):
adamw_eps (float): AdamW epsilon (default: 1e-8).
"""

# Under ZeRO at most this many elements of whole matrices are gathered at once (512 MiB in bf16), which bounds the
# extra memory of a step.
bucket_numel = 1 << 28

def __init__(
self,
params,
Expand Down Expand Up @@ -122,6 +142,8 @@ def __init__(
)
self._adamw_defaults = dict(betas=tuple(betas), eps=adamw_eps)
super().__init__(params, dict(lr=lr, weight_decay=weight_decay))
self.dp_group: Optional[dist.ProcessGroup] = None
self.shard_to_working_param: Dict[int, Tensor] = {}

def add_param_group(self, param_group: Dict) -> None:
use_muon = param_group.setdefault("use_muon", True)
Expand All @@ -142,6 +164,45 @@ def add_param_group(self, param_group: Dict) -> None:
)
super().add_param_group(param_group)

def setup_distributed(
self,
tp_group: Optional[dist.ProcessGroup] = None,
dp_group: Optional[dist.ProcessGroup] = None,
shard_to_working_param: Optional[Dict] = None,
padding_map: Optional[Dict] = None,
is_zero: Optional[bool] = False,
):
"""Record the data-parallel group and, for ZeRO, which whole parameter each shard belongs to.

Arguments:
tp_group (dist.ProcessGroup): tensor-parallel group; only a group of size 1 is supported.
dp_group (dist.ProcessGroup): the group ZeRO shards over (default: the whole world).
shard_to_working_param (Dict): id of each shard to the parameter used in forward and backward.
padding_map (Dict): unused; the padding follows from the parameter's size and the group size.
is_zero (bool): unused; shards are recognized by their shape.
"""
if tp_group is not None and dist.get_world_size(tp_group) > 1:
raise NotImplementedError("Muon does not support tensor parallelism yet; use tp_size=1")
self.dp_group = dp_group
self.shard_to_working_param = dict(shard_to_working_param or {})
world_size = dist.get_world_size(dp_group)
for group in self.param_groups:
if not group["use_muon"]:
continue
for p in group["params"]:
working = self._working_param(p)
if working.shape == p.shape:
continue
if working.ndim != 2 or p.numel() != -(-working.numel() // world_size):
raise ValueError(
f"a {tuple(working.shape)} Muon parameter arrives as a shard of {p.numel()} elements, not one "
f"of {world_size} equal slices: its shards do not span the data-parallel group (e.g. expert "
"parameters sharded over their own group), which Muon does not support"
)

def _working_param(self, p: Tensor) -> Tensor:
return self.shard_to_working_param.get(id(p), p)

@torch.no_grad()
def step(self, closure=None):
loss = None
Expand All @@ -157,26 +218,82 @@ def step(self, closure=None):
return loss

def _muon_step(self, group: Dict) -> None:
lr, weight_decay, momentum = group["lr"], group["weight_decay"], group["momentum"]
shards = []
for p in group["params"]:
if p.grad is None:
continue
grad = p.grad
if grad.ndim != 2:
# A sharded 1D view, e.g. from LowLevelZeRO's master parameter slices.
raise RuntimeError(
f"Muon needs the full 2D gradient but got shape {tuple(grad.shape)}; "
"ZeRO / Gemini / FSDP sharding is not supported yet"
)
state = self.state[p]
if "momentum_buffer" not in state:
state["momentum_buffer"] = torch.zeros_like(grad, memory_format=torch.preserve_format)
buf = state["momentum_buffer"]
buf.lerp_(grad, 1 - momentum)
update = grad.lerp(buf, momentum) if group["nesterov"] else buf
update = zeropower_via_newtonschulz(update, group["ns_coefficients"], group["ns_steps"], group["eps"])
p.mul_(1 - lr * weight_decay)
p.add_(update, alpha=-_adjust_lr(lr, group["adjust_lr_fn"], p.shape))
if self._working_param(p).shape == p.shape:
self._step_matrix(p, group)
else:
shards.append(p)
# every rank holds the same shards in the same order, so the gathers line up
for bucket in self._buckets(shards):
inputs = [self._momentum_update(p, group) for p in bucket]
for p, update in zip(bucket, self._orthogonalize_shards(bucket, inputs, group)):
self._apply_update(p, update, self._working_param(p).shape, group)

def _step_matrix(self, p: Tensor, group: Dict) -> None:
if p.grad.ndim != 2:
# A sharded 1D view that setup_distributed was not told about.
raise RuntimeError(
f"Muon needs the full 2D gradient but got shape {tuple(p.grad.shape)}; under ZeRO the plugin has to "
"call setup_distributed (LowLevelZeroPlugin does), and Gemini / FSDP are not supported"
)
update = zeropower_via_newtonschulz(
self._momentum_update(p, group), group["ns_coefficients"], group["ns_steps"], group["eps"]
)
self._apply_update(p, update, p.shape, group)

def _momentum_update(self, p: Tensor, group: Dict) -> Tensor:
"""Update ``p``'s momentum buffer and return the Newton-Schulz input; elementwise, so shards work too."""
grad, momentum = p.grad, group["momentum"]
state = self.state[p]
if "momentum_buffer" not in state:
state["momentum_buffer"] = torch.zeros_like(grad, memory_format=torch.preserve_format)
buf = state["momentum_buffer"]
buf.lerp_(grad, 1 - momentum)
return grad.lerp(buf, momentum) if group["nesterov"] else buf

@staticmethod
def _apply_update(p: Tensor, update: Tensor, shape: torch.Size, group: Dict) -> None:
"""Apply weight decay and the orthogonalized ``update``; ``shape`` is the whole matrix's, for the lr scale."""
lr = group["lr"]
p.mul_(1 - lr * group["weight_decay"])
p.add_(update, alpha=-_adjust_lr(lr, group["adjust_lr_fn"], shape))

def _buckets(self, shards: List[Tensor]) -> List[List[Tensor]]:
buckets, numel = [], 0
for p in shards:
size = self._working_param(p).numel()
if not buckets or numel + size > self.bucket_numel:
buckets.append([])
numel = 0
buckets[-1].append(p)
numel += size
return buckets

def _orthogonalize_shards(self, shards: List[Tensor], inputs: List[Tensor], group: Dict) -> List[Tensor]:
"""Gather the whole Newton-Schulz inputs of ``shards``, orthogonalize them and return each rank's slices."""
world_size, rank = dist.get_world_size(self.dp_group), dist.get_rank(self.dp_group)
# gloo gathers CPU tensors; NCCL (and Newton-Schulz) want the accelerator
if dist.get_backend(self.dp_group) == dist.Backend.GLOO:
device = torch.device("cpu")
else:
device = get_accelerator().get_current_device()
local = torch.cat([x.to(device=device, dtype=torch.bfloat16).flatten() for x in inputs])
gathered = torch.empty(world_size * local.numel(), dtype=local.dtype, device=device)
dist.all_gather_into_tensor(gathered, local, group=self.dp_group)
# row r holds rank r's slices of every matrix, one after another
gathered = gathered.view(world_size, local.numel())
updates, offset = [], 0
for p in shards:
working, n = self._working_param(p), p.numel()
matrix = gathered[:, offset : offset + n].reshape(-1)[: working.numel()].view(working.shape)
offset += n
update = zeropower_via_newtonschulz(matrix, group["ns_coefficients"], group["ns_steps"], group["eps"])
update = F.pad(update.flatten(), (0, world_size * n - working.numel())).view(world_size, n)[rank]
updates.append(update.to(p.device))
return updates

def _adamw_step(self, group: Dict) -> None:
lr, weight_decay, eps = group["lr"], group["weight_decay"], group["eps"]
Expand Down
Loading