Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/scripts/ci/run_cpu_unit_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ TESTS=(
tests/test_optimizer/test_dist_muon.py
tests/test_optimizer/test_lr_scheduler.py
tests/test_optimizer/test_muon.py
tests/test_optimizer/test_orscale.py
tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py
tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py
tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py
Expand Down
4 changes: 2 additions & 2 deletions colossalai/booster/plugin/gemini_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -668,8 +668,8 @@ def configure(
) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]:
if isinstance(optimizer, Muon):
raise NotImplementedError(
"GeminiPlugin does not support Muon: its chunks split matrices at arbitrary boundaries. "
"Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin."
f"GeminiPlugin does not support {type(optimizer).__name__}: its chunks split matrices at arbitrary "
"boundaries. Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin."
)
params_info = get_param_info(optimizer)
if not isinstance(model, ModelWrapper):
Expand Down
4 changes: 2 additions & 2 deletions colossalai/booster/plugin/torch_fsdp_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -539,8 +539,8 @@ def configure(
) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]:
if isinstance(optimizer, Muon):
raise NotImplementedError(
"TorchFSDPPlugin does not support Muon: FSDP hands the optimizer flattened parameters in one group. "
"Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin."
f"TorchFSDPPlugin does not support {type(optimizer).__name__}: FSDP hands the optimizer flattened "
"parameters in one group. Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin."
)
# wrap the model with PyTorch FSDP
fsdp_model = TorchFSDPModel(model, device_id=torch.cuda.current_device(), **self.fsdp_kwargs)
Expand Down
2 changes: 2 additions & 0 deletions colossalai/nn/optimizer/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from .lamb import Lamb
from .lars import Lars
from .muon import Muon, muon_param_groups
from .orscale import OrScale

from .adafactor import Adafactor # noqa

Expand All @@ -39,6 +40,7 @@
"DistributedAdaFactor",
"Muon",
"muon_param_groups",
"OrScale",
]

optim2DistOptim = {
Expand Down
63 changes: 40 additions & 23 deletions colossalai/nn/optimizer/muon.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,15 +132,20 @@ def __init__(
raise ValueError(f"ns_steps must be in [1, 99], got {ns_steps}")
if adjust_lr_fn is not None and adjust_lr_fn not in ADJUST_LR_FNS:
raise ValueError(f"adjust_lr_fn must be one of {ADJUST_LR_FNS} or None, got {adjust_lr_fn!r}")
self._muon_defaults = dict(
muon_defaults = dict(
momentum=momentum,
nesterov=nesterov,
ns_coefficients=tuple(ns_coefficients),
ns_steps=ns_steps,
eps=eps,
adjust_lr_fn=adjust_lr_fn,
)
self._adamw_defaults = dict(betas=tuple(betas), eps=adamw_eps)
self._init_groups(params, lr, weight_decay, muon_defaults, dict(betas=tuple(betas), eps=adamw_eps))

def _init_groups(self, params, lr: float, weight_decay: float, muon_defaults: Dict, adamw_defaults: Dict) -> None:
"""Build the parameter groups; ``muon_defaults`` and ``adamw_defaults`` fill the keys a group leaves out."""
self._muon_defaults = muon_defaults
self._adamw_defaults = adamw_defaults
super().__init__(params, dict(lr=lr, weight_decay=weight_decay))
self.dp_group: Optional[dist.ProcessGroup] = None
self.shard_to_working_param: Dict[int, Tensor] = {}
Expand All @@ -159,7 +164,7 @@ def add_param_group(self, param_group: Dict) -> None:
for p in params:
if p.ndim != 2:
raise ValueError(
f"Muon only supports 2D parameters, got shape {tuple(p.shape)}; "
f"{type(self).__name__} only supports 2D parameters, got shape {tuple(p.shape)}; "
"put it in a group with use_muon=False (see muon_param_groups)"
)
super().add_param_group(param_group)
Expand All @@ -182,7 +187,7 @@ def setup_distributed(
is_zero (bool): unused; shards are recognized by their shape.
"""
if tp_group is not None and dist.get_world_size(tp_group) > 1:
raise NotImplementedError("Muon does not support tensor parallelism yet; use tp_size=1")
raise NotImplementedError(f"{type(self).__name__} does not support tensor parallelism yet; use tp_size=1")
self.dp_group = dp_group
self.shard_to_working_param = dict(shard_to_working_param or {})
world_size = dist.get_world_size(dp_group)
Expand All @@ -195,9 +200,9 @@ def setup_distributed(
continue
if working.ndim != 2 or p.numel() != -(-working.numel() // world_size):
raise ValueError(
f"a {tuple(working.shape)} Muon parameter arrives as a shard of {p.numel()} elements, not one "
f"a {tuple(working.shape)} matrix arrives as a shard of {p.numel()} elements, not one "
f"of {world_size} equal slices: its shards do not span the data-parallel group (e.g. expert "
"parameters sharded over their own group), which Muon does not support"
f"parameters sharded over their own group), which {type(self).__name__} does not support"
)

def _working_param(self, p: Tensor) -> Tensor:
Expand Down Expand Up @@ -226,23 +231,20 @@ def _muon_step(self, group: Dict) -> None:
self._step_matrix(p, group)
else:
shards.append(p)
# every rank holds the same shards in the same order, so the gathers line up
# every rank holds the same shards in the same order, so the collectives line up
for bucket in self._buckets(shards):
inputs = [self._momentum_update(p, group) for p in bucket]
for p, update in zip(bucket, self._orthogonalize_shards(bucket, inputs, group)):
self._apply_update(p, update, self._working_param(p).shape, group)
self._apply_shard_updates(bucket, self._orthogonalize_shards(bucket, inputs, group), group)

def _step_matrix(self, p: Tensor, group: Dict) -> None:
if p.grad.ndim != 2:
# A sharded 1D view that setup_distributed was not told about.
raise RuntimeError(
f"Muon needs the full 2D gradient but got shape {tuple(p.grad.shape)}; under ZeRO the plugin has to "
"call setup_distributed (LowLevelZeroPlugin does), and Gemini / FSDP are not supported"
f"{type(self).__name__} needs the full 2D gradient but got shape {tuple(p.grad.shape)}; under ZeRO "
"the plugin has to call setup_distributed (LowLevelZeroPlugin does), and Gemini / FSDP are not "
"supported"
)
update = zeropower_via_newtonschulz(
self._momentum_update(p, group), group["ns_coefficients"], group["ns_steps"], group["eps"]
)
self._apply_update(p, update, p.shape, group)
self._apply_update(p, self._orthogonalize(self._momentum_update(p, group), group), group)

def _momentum_update(self, p: Tensor, group: Dict) -> Tensor:
"""Update ``p``'s momentum buffer and return the Newton-Schulz input; elementwise, so shards work too."""
Expand All @@ -254,9 +256,22 @@ def _momentum_update(self, p: Tensor, group: Dict) -> Tensor:
buf.lerp_(grad, 1 - momentum)
return grad.lerp(buf, momentum) if group["nesterov"] else buf

def _orthogonalize(self, matrix: Tensor, group: Dict) -> Tensor:
"""Orthogonalize a whole matrix; the result is bf16."""
return zeropower_via_newtonschulz(matrix, group["ns_coefficients"], group["ns_steps"], group["eps"])

def _apply_update(self, p: Tensor, update: Tensor, group: Dict) -> None:
"""Apply weight decay and the orthogonalized ``update`` to a whole matrix."""
self._decay_and_add(p, update, p.shape, group)

def _apply_shard_updates(self, shards: List[Tensor], updates: List[Tensor], group: Dict) -> None:
"""Apply to each of this rank's ``shards`` its slice of the whole matrix's orthogonalized update."""
for p, update in zip(shards, updates):
self._decay_and_add(p, update, self._working_param(p).shape, group)

@staticmethod
def _apply_update(p: Tensor, update: Tensor, shape: torch.Size, group: Dict) -> None:
"""Apply weight decay and the orthogonalized ``update``; ``shape`` is the whole matrix's, for the lr scale."""
def _decay_and_add(p: Tensor, update: Tensor, shape: torch.Size, group: Dict) -> None:
# elementwise, so the same for a matrix and a shard; ``shape`` is the whole matrix's, for the lr scale
lr = group["lr"]
p.mul_(1 - lr * group["weight_decay"])
p.add_(update, alpha=-_adjust_lr(lr, group["adjust_lr_fn"], shape))
Expand All @@ -275,11 +290,7 @@ def _buckets(self, shards: List[Tensor]) -> List[List[Tensor]]:
def _orthogonalize_shards(self, shards: List[Tensor], inputs: List[Tensor], group: Dict) -> List[Tensor]:
"""Gather the whole Newton-Schulz inputs of ``shards``, orthogonalize them and return each rank's slices."""
world_size, rank = dist.get_world_size(self.dp_group), dist.get_rank(self.dp_group)
# gloo gathers CPU tensors; NCCL (and Newton-Schulz) want the accelerator
if dist.get_backend(self.dp_group) == dist.Backend.GLOO:
device = torch.device("cpu")
else:
device = get_accelerator().get_current_device()
device = self._collective_device()
local = torch.cat([x.to(device=device, dtype=torch.bfloat16).flatten() for x in inputs])
gathered = torch.empty(world_size * local.numel(), dtype=local.dtype, device=device)
dist.all_gather_into_tensor(gathered, local, group=self.dp_group)
Expand All @@ -290,11 +301,17 @@ def _orthogonalize_shards(self, shards: List[Tensor], inputs: List[Tensor], grou
working, n = self._working_param(p), p.numel()
matrix = gathered[:, offset : offset + n].reshape(-1)[: working.numel()].view(working.shape)
offset += n
update = zeropower_via_newtonschulz(matrix, group["ns_coefficients"], group["ns_steps"], group["eps"])
update = self._orthogonalize(matrix, group)
update = F.pad(update.flatten(), (0, world_size * n - working.numel())).view(world_size, n)[rank]
updates.append(update.to(p.device))
return updates

def _collective_device(self) -> torch.device:
# gloo communicates CPU tensors; NCCL (and Newton-Schulz) want the accelerator
if dist.get_backend(self.dp_group) == dist.Backend.GLOO:
return torch.device("cpu")
return get_accelerator().get_current_device()

def _adamw_step(self, group: Dict) -> None:
lr, weight_decay, eps = group["lr"], group["weight_decay"], group["eps"]
beta1, beta2 = group["betas"]
Expand Down
Loading