diff --git a/.github/scripts/ci/run_cpu_unit_tests.sh b/.github/scripts/ci/run_cpu_unit_tests.sh index 4f8944c839fa..68bbeaed221b 100644 --- a/.github/scripts/ci/run_cpu_unit_tests.sh +++ b/.github/scripts/ci/run_cpu_unit_tests.sh @@ -19,7 +19,9 @@ TESTS=( tests/test_numerics/test_numerics_data_parallel.py tests/test_numerics/test_numerics_independent_impl.py tests/test_numerics/test_numerics_resume.py + tests/test_optimizer/test_dist_muon.py tests/test_optimizer/test_lr_scheduler.py + tests/test_optimizer/test_muon.py tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py diff --git a/colossalai/booster/plugin/gemini_plugin.py b/colossalai/booster/plugin/gemini_plugin.py index 0d8846e07b30..7a189e17f494 100644 --- a/colossalai/booster/plugin/gemini_plugin.py +++ b/colossalai/booster/plugin/gemini_plugin.py @@ -31,6 +31,7 @@ from colossalai.cluster import DistCoordinator, ProcessGroupMesh from colossalai.interface import ModelWrapper, OptimizerWrapper from colossalai.logging import get_dist_logger +from colossalai.nn.optimizer import Muon from colossalai.shardformer import ShardConfig, ShardFormer from colossalai.zero import GeminiDDP, GeminiOptimizer from colossalai.zero.gemini.memory_tracer import MemStats @@ -665,6 +666,11 @@ def configure( dataloader: Optional[DataLoader] = None, lr_scheduler: Optional[LRScheduler] = None, ) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]: + if isinstance(optimizer, Muon): + raise NotImplementedError( + "GeminiPlugin does not support Muon: its chunks split matrices at arbitrary boundaries. " + "Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." + ) params_info = get_param_info(optimizer) if not isinstance(model, ModelWrapper): # convert model to sync bn diff --git a/colossalai/booster/plugin/torch_fsdp_plugin.py b/colossalai/booster/plugin/torch_fsdp_plugin.py index 05cc432a673d..5c9390539972 100644 --- a/colossalai/booster/plugin/torch_fsdp_plugin.py +++ b/colossalai/booster/plugin/torch_fsdp_plugin.py @@ -34,6 +34,7 @@ from colossalai.cluster import DistCoordinator from colossalai.interface import ModelWrapper, OptimizerWrapper from colossalai.logging import get_dist_logger +from colossalai.nn.optimizer import Muon from colossalai.utils.safetensors import load_flat from .dp_plugin_base import DPPluginBase @@ -536,6 +537,11 @@ def configure( dataloader: Optional[DataLoader] = None, lr_scheduler: Optional[LRScheduler] = None, ) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]: + if isinstance(optimizer, Muon): + raise NotImplementedError( + "TorchFSDPPlugin does not support Muon: FSDP hands the optimizer flattened parameters in one group. " + "Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." + ) # wrap the model with PyTorch FSDP fsdp_model = TorchFSDPModel(model, device_id=torch.cuda.current_device(), **self.fsdp_kwargs) diff --git a/colossalai/nn/optimizer/__init__.py b/colossalai/nn/optimizer/__init__.py index 37500f95624b..ebdd5dc3eb6d 100644 --- a/colossalai/nn/optimizer/__init__.py +++ b/colossalai/nn/optimizer/__init__.py @@ -15,6 +15,7 @@ from .hybrid_adam import HybridAdam from .lamb import Lamb from .lars import Lars +from .muon import Muon, muon_param_groups from .adafactor import Adafactor # noqa @@ -36,6 +37,8 @@ "DistributedCAME", "Adafactor", "DistributedAdaFactor", + "Muon", + "muon_param_groups", ] optim2DistOptim = { diff --git a/colossalai/nn/optimizer/muon.py b/colossalai/nn/optimizer/muon.py new file mode 100644 index 000000000000..bd6de205de84 --- /dev/null +++ b/colossalai/nn/optimizer/muon.py @@ -0,0 +1,352 @@ +"""Muon: momentum orthogonalized by Newton-Schulz, with a built-in AdamW for non-matrix parameters. + +The Muon update follows ``torch.optim.Muon`` (torch>=2.9) so the two are numerically interchangeable; it is +reimplemented here because ColossalAI still supports older torch. Non-matrix parameters (embeddings, output head, +norms, biases) go to the AdamW branch of the same optimizer, as in KellerJordan/Muon's ``MuonWithAuxAdam`` and +Moonlight, so a Booster still receives a single optimizer. + +Under LowLevelZeRO (stage 1 and 2) each rank holds a flat 1/N slice of every parameter, while Newton-Schulz needs whole +matrices. As in Moonlight's distributed Muon, each rank updates the momentum of its own slices, the bf16 +Newton-Schulz inputs are all-gathered over the data-parallel group, and every rank orthogonalizes the whole matrices +and keeps the slice of each update it owns. The gathered input is the bf16 cast the unsharded iteration makes anyway, +so a sharded step computes the unsharded one, at the cost of one bf16 all-gather of the Muon parameters per step and +every rank orthogonalizing all of them. + +References: + - https://kellerjordan.github.io/posts/muon/ + - Muon is Scalable for LLM Training, https://arxiv.org/abs/2502.16982 +""" + +import math +from typing import Dict, List, Optional, Tuple + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor + +from colossalai.accelerator import get_accelerator +from colossalai.interface.optimizer import DistributedOptim + +__all__ = ["Muon", "muon_param_groups", "zeropower_via_newtonschulz"] + +# Quintic Newton-Schulz coefficients from KellerJordan/Muon, also torch.optim.Muon's defaults. +DEFAULT_NS_COEFFICIENTS = (3.4445, -4.7750, 2.0315) +ADJUST_LR_FNS = ("original", "match_rms_adamw") + + +def zeropower_via_newtonschulz( + grad: Tensor, + ns_coefficients: Tuple[float, float, float] = DEFAULT_NS_COEFFICIENTS, + ns_steps: int = 5, + eps: float = 1e-7, +) -> Tensor: + """Approximately orthogonalize a 2D matrix with a quintic Newton-Schulz iteration. + + With the default coefficients the result is U S' V^T with S' roughly in [0.5, 1.5] rather than the exact polar + factor U V^T, which works as well in training. The iteration runs in bf16, as in every reference implementation. + """ + if grad.ndim != 2: + raise ValueError(f"Newton-Schulz expects a 2D matrix, got shape {tuple(grad.shape)}") + a, b, c = ns_coefficients + x = grad.bfloat16() + transposed = grad.size(0) > grad.size(1) + if transposed: + x = x.T + # Scale the spectral norm to at most 1; out of place so a low-precision input buffer is never modified. + x = x / x.norm().clamp(min=eps) + for _ in range(ns_steps): + gram = x @ x.T + poly = torch.addmm(gram, gram, gram, beta=b, alpha=c) + x = torch.addmm(x, poly, x, beta=a) + if transposed: + x = x.T + return x + + +def _adjust_lr(lr: float, adjust_lr_fn: Optional[str], shape: torch.Size) -> float: + rows, cols = shape[:2] + if adjust_lr_fn is None or adjust_lr_fn == "original": + # KellerJordan/Muon: keep the update RMS consistent across rectangular matrices. + return lr * math.sqrt(max(1.0, rows / cols)) + # Moonlight: match the update RMS of AdamW so AdamW's lr and weight decay carry over. + return lr * 0.2 * math.sqrt(max(rows, cols)) + + +class Muon(DistributedOptim): + """Muon for 2D hidden-layer weights plus AdamW for every other parameter, in one optimizer. + + Each parameter group sets ``use_muon``; ``muon_param_groups(model)`` builds both groups from a model. Groups + without the flag are Muon groups. Muon groups take ``lr``, ``weight_decay``, ``momentum``, ``nesterov``, + ``ns_coefficients``, ``ns_steps``, ``eps`` and ``adjust_lr_fn``; AdamW groups take ``lr``, ``weight_decay``, + ``betas`` and ``eps``. Missing keys fall back to the constructor arguments, with ``adamw_eps`` as the AdamW groups' + ``eps``. + + ``LowLevelZeroPlugin`` and ``HybridParallelPlugin`` call ``setup_distributed`` on this very object (it is not + replaced, so an LR scheduler built on it before ``boost`` keeps driving it), after which ZeRO's parameter shards are + orthogonalized as whole matrices (see the module docstring). The AdamW branch is elementwise and runs on the + shards as they are. Tensor parallelism, Gemini and FSDP are not supported. + + Args: + params: parameters or parameter groups. + lr (float): learning rate shared by both branches unless a group overrides it (default: 1e-3). + weight_decay (float): decoupled weight decay, applied as ``p *= 1 - lr * weight_decay`` (default: 0.1). + momentum (float): Muon momentum, kept as an exponential moving average like torch.optim.Muon (default: 0.95). + nesterov (bool): use Nesterov momentum for Muon (default: True). + ns_coefficients (tuple): Newton-Schulz coefficients (a, b, c) (default: KellerJordan's). + ns_steps (int): Newton-Schulz iterations (default: 5). + eps (float): lower bound of the norm used to scale the Newton-Schulz input (default: 1e-7). + adjust_lr_fn (str, optional): ``"original"`` scales the Muon lr by sqrt(max(1, rows / cols)); + ``"match_rms_adamw"`` by 0.2 * sqrt(max(rows, cols)) so AdamW hyperparameters carry over. + None means ``"original"`` (default: None). + betas (tuple): AdamW betas (default: (0.9, 0.95)). + adamw_eps (float): AdamW epsilon (default: 1e-8). + """ + + # Under ZeRO at most this many elements of whole matrices are gathered at once (512 MiB in bf16), which bounds the + # extra memory of a step. + bucket_numel = 1 << 28 + + def __init__( + self, + params, + lr: float = 1e-3, + weight_decay: float = 0.1, + momentum: float = 0.95, + nesterov: bool = True, + ns_coefficients: Tuple[float, float, float] = DEFAULT_NS_COEFFICIENTS, + ns_steps: int = 5, + eps: float = 1e-7, + adjust_lr_fn: Optional[str] = None, + betas: Tuple[float, float] = (0.9, 0.95), + adamw_eps: float = 1e-8, + ): + if lr < 0.0: + raise ValueError(f"Invalid learning rate: {lr}") + if weight_decay < 0.0: + raise ValueError(f"Invalid weight_decay: {weight_decay}") + if not 0.0 <= momentum < 1.0: + raise ValueError(f"Invalid momentum: {momentum}") + if not 0 < ns_steps < 100: + raise ValueError(f"ns_steps must be in [1, 99], got {ns_steps}") + if adjust_lr_fn is not None and adjust_lr_fn not in ADJUST_LR_FNS: + raise ValueError(f"adjust_lr_fn must be one of {ADJUST_LR_FNS} or None, got {adjust_lr_fn!r}") + self._muon_defaults = dict( + momentum=momentum, + nesterov=nesterov, + ns_coefficients=tuple(ns_coefficients), + ns_steps=ns_steps, + eps=eps, + adjust_lr_fn=adjust_lr_fn, + ) + self._adamw_defaults = dict(betas=tuple(betas), eps=adamw_eps) + super().__init__(params, dict(lr=lr, weight_decay=weight_decay)) + self.dp_group: Optional[dist.ProcessGroup] = None + self.shard_to_working_param: Dict[int, Tensor] = {} + + def add_param_group(self, param_group: Dict) -> None: + use_muon = param_group.setdefault("use_muon", True) + for key, value in (self._muon_defaults if use_muon else self._adamw_defaults).items(): + param_group.setdefault(key, value) + params = param_group["params"] + if isinstance(params, Tensor): + params = [params] + elif not isinstance(params, set): # leave sets for torch to reject: their order is not deterministic + params = list(params) + param_group["params"] = params + if use_muon: + for p in params: + if p.ndim != 2: + raise ValueError( + f"Muon only supports 2D parameters, got shape {tuple(p.shape)}; " + "put it in a group with use_muon=False (see muon_param_groups)" + ) + super().add_param_group(param_group) + + def setup_distributed( + self, + tp_group: Optional[dist.ProcessGroup] = None, + dp_group: Optional[dist.ProcessGroup] = None, + shard_to_working_param: Optional[Dict] = None, + padding_map: Optional[Dict] = None, + is_zero: Optional[bool] = False, + ): + """Record the data-parallel group and, for ZeRO, which whole parameter each shard belongs to. + + Arguments: + tp_group (dist.ProcessGroup): tensor-parallel group; only a group of size 1 is supported. + dp_group (dist.ProcessGroup): the group ZeRO shards over (default: the whole world). + shard_to_working_param (Dict): id of each shard to the parameter used in forward and backward. + padding_map (Dict): unused; the padding follows from the parameter's size and the group size. + is_zero (bool): unused; shards are recognized by their shape. + """ + if tp_group is not None and dist.get_world_size(tp_group) > 1: + raise NotImplementedError("Muon does not support tensor parallelism yet; use tp_size=1") + self.dp_group = dp_group + self.shard_to_working_param = dict(shard_to_working_param or {}) + world_size = dist.get_world_size(dp_group) + for group in self.param_groups: + if not group["use_muon"]: + continue + for p in group["params"]: + working = self._working_param(p) + if working.shape == p.shape: + continue + if working.ndim != 2 or p.numel() != -(-working.numel() // world_size): + raise ValueError( + f"a {tuple(working.shape)} Muon parameter arrives as a shard of {p.numel()} elements, not one " + f"of {world_size} equal slices: its shards do not span the data-parallel group (e.g. expert " + "parameters sharded over their own group), which Muon does not support" + ) + + def _working_param(self, p: Tensor) -> Tensor: + return self.shard_to_working_param.get(id(p), p) + + @torch.no_grad() + def step(self, closure=None): + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + if group["use_muon"]: + self._muon_step(group) + else: + self._adamw_step(group) + return loss + + def _muon_step(self, group: Dict) -> None: + shards = [] + for p in group["params"]: + if p.grad is None: + continue + if self._working_param(p).shape == p.shape: + self._step_matrix(p, group) + else: + shards.append(p) + # every rank holds the same shards in the same order, so the gathers line up + for bucket in self._buckets(shards): + inputs = [self._momentum_update(p, group) for p in bucket] + for p, update in zip(bucket, self._orthogonalize_shards(bucket, inputs, group)): + self._apply_update(p, update, self._working_param(p).shape, group) + + def _step_matrix(self, p: Tensor, group: Dict) -> None: + if p.grad.ndim != 2: + # A sharded 1D view that setup_distributed was not told about. + raise RuntimeError( + f"Muon needs the full 2D gradient but got shape {tuple(p.grad.shape)}; under ZeRO the plugin has to " + "call setup_distributed (LowLevelZeroPlugin does), and Gemini / FSDP are not supported" + ) + update = zeropower_via_newtonschulz( + self._momentum_update(p, group), group["ns_coefficients"], group["ns_steps"], group["eps"] + ) + self._apply_update(p, update, p.shape, group) + + def _momentum_update(self, p: Tensor, group: Dict) -> Tensor: + """Update ``p``'s momentum buffer and return the Newton-Schulz input; elementwise, so shards work too.""" + grad, momentum = p.grad, group["momentum"] + state = self.state[p] + if "momentum_buffer" not in state: + state["momentum_buffer"] = torch.zeros_like(grad, memory_format=torch.preserve_format) + buf = state["momentum_buffer"] + buf.lerp_(grad, 1 - momentum) + return grad.lerp(buf, momentum) if group["nesterov"] else buf + + @staticmethod + def _apply_update(p: Tensor, update: Tensor, shape: torch.Size, group: Dict) -> None: + """Apply weight decay and the orthogonalized ``update``; ``shape`` is the whole matrix's, for the lr scale.""" + lr = group["lr"] + p.mul_(1 - lr * group["weight_decay"]) + p.add_(update, alpha=-_adjust_lr(lr, group["adjust_lr_fn"], shape)) + + def _buckets(self, shards: List[Tensor]) -> List[List[Tensor]]: + buckets, numel = [], 0 + for p in shards: + size = self._working_param(p).numel() + if not buckets or numel + size > self.bucket_numel: + buckets.append([]) + numel = 0 + buckets[-1].append(p) + numel += size + return buckets + + def _orthogonalize_shards(self, shards: List[Tensor], inputs: List[Tensor], group: Dict) -> List[Tensor]: + """Gather the whole Newton-Schulz inputs of ``shards``, orthogonalize them and return each rank's slices.""" + world_size, rank = dist.get_world_size(self.dp_group), dist.get_rank(self.dp_group) + # gloo gathers CPU tensors; NCCL (and Newton-Schulz) want the accelerator + if dist.get_backend(self.dp_group) == dist.Backend.GLOO: + device = torch.device("cpu") + else: + device = get_accelerator().get_current_device() + local = torch.cat([x.to(device=device, dtype=torch.bfloat16).flatten() for x in inputs]) + gathered = torch.empty(world_size * local.numel(), dtype=local.dtype, device=device) + dist.all_gather_into_tensor(gathered, local, group=self.dp_group) + # row r holds rank r's slices of every matrix, one after another + gathered = gathered.view(world_size, local.numel()) + updates, offset = [], 0 + for p in shards: + working, n = self._working_param(p), p.numel() + matrix = gathered[:, offset : offset + n].reshape(-1)[: working.numel()].view(working.shape) + offset += n + update = zeropower_via_newtonschulz(matrix, group["ns_coefficients"], group["ns_steps"], group["eps"]) + update = F.pad(update.flatten(), (0, world_size * n - working.numel())).view(world_size, n)[rank] + updates.append(update.to(p.device)) + return updates + + def _adamw_step(self, group: Dict) -> None: + lr, weight_decay, eps = group["lr"], group["weight_decay"], group["eps"] + beta1, beta2 = group["betas"] + for p in group["params"]: + if p.grad is None: + continue + grad = p.grad + state = self.state[p] + if "step" not in state: + state["step"] = 0 + state["exp_avg"] = torch.zeros_like(grad, memory_format=torch.preserve_format) + state["exp_avg_sq"] = torch.zeros_like(grad, memory_format=torch.preserve_format) + state["step"] += 1 + exp_avg, exp_avg_sq = state["exp_avg"], state["exp_avg_sq"] + exp_avg.lerp_(grad, 1 - beta1) + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) + # Same arithmetic as torch.optim.AdamW's single-tensor path. + bias_correction1 = 1 - beta1 ** state["step"] + bias_correction2 = 1 - beta2 ** state["step"] + denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps) + p.mul_(1 - lr * weight_decay) + p.addcdiv_(exp_avg, denom, value=-lr / bias_correction1) + + +def muon_param_groups(model: nn.Module) -> List[Dict]: + """Split a model's trainable parameters into a Muon group and an AdamW group. + + Muon takes 2D weights of hidden layers. Embeddings, the output head returned by ``get_output_embeddings()`` + (tied or not) and every non-2D parameter go to AdamW. Setting ``param.use_muon`` to True or False overrides + the rule for that parameter. Empty groups are dropped, so pass the result straight to ``Muon``. + """ + excluded = set() + for module in model.modules(): + if isinstance(module, nn.Embedding): + excluded.add(id(module.weight)) + get_output_embeddings = getattr(model, "get_output_embeddings", None) + head = get_output_embeddings() if callable(get_output_embeddings) else None + if head is not None and getattr(head, "weight", None) is not None: + excluded.add(id(head.weight)) + + muon_params, adamw_params = [], [] + for p in model.parameters(): + if not p.requires_grad: + continue + use_muon = getattr(p, "use_muon", None) + if use_muon is None: + use_muon = p.ndim == 2 and id(p) not in excluded + (muon_params if use_muon else adamw_params).append(p) + + groups = [] + if muon_params: + groups.append({"params": muon_params, "use_muon": True}) + if adamw_params: + groups.append({"params": adamw_params, "use_muon": False}) + return groups diff --git a/tests/test_optimizer/test_dist_muon.py b/tests/test_optimizer/test_dist_muon.py new file mode 100644 index 000000000000..4b88001084c3 --- /dev/null +++ b/tests/test_optimizer/test_dist_muon.py @@ -0,0 +1,291 @@ +import os +import tempfile +from functools import partial +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F + +import colossalai +from colossalai.booster import Booster +from colossalai.booster.plugin import GeminiPlugin, LowLevelZeroPlugin, TorchFSDPPlugin +from colossalai.logging import disable_existing_loggers +from colossalai.nn.optimizer import Muon +from colossalai.nn.optimizer.muon import zeropower_via_newtonschulz +from colossalai.testing import free_port, rerun_if_address_is_in_use, spawn +from tests.kit.numerics import NumericsSpec, Trace, judge_equal, judge_relative_bound +from tests.kit.numerics.capture.training import prepare, run_steps +from tests.kit.numerics.scenarios import resume + +STEPS, LR = 3, 0.02 +# odd sizes, so that ZeRO pads every matrix and the padding sits inside the last rank's slice +DIM_IN, HIDDEN, DIM_OUT = 31, 63, 17 + + +def requires_gpus(n): + return pytest.mark.skipif(torch.cuda.device_count() < n, reason=f"needs {n} GPUs") + + +def muon_optimizer(params, cls=Muon): + params = list(params) + groups = [ + {"params": [p for p in params if p.ndim == 2], "use_muon": True}, + {"params": [p for p in params if p.ndim != 2], "use_muon": False}, + ] + return cls(groups, lr=LR, weight_decay=0.1) + + +class ShardLocalMuon(Muon): + """A typical sharding bug: each rank orthogonalizes its own slice as a 1 x n matrix.""" + + def _orthogonalize_shards(self, shards, inputs, group): + return [ + zeropower_via_newtonschulz(x.view(1, -1), group["ns_coefficients"], group["ns_steps"], group["eps"]) + .flatten() + .to(p.device) + for p, x in zip(shards, inputs) + ] + + +# --------------------------------------------------------------------------- +# Muon on hand-made ZeRO shards, over gloo: runs without GPUs +# --------------------------------------------------------------------------- + + +def shard_of(t, world_size, rank): + """``rank``'s slice of ``t``, padded and split the way LowLevelZeRO does.""" + flat = t.flatten() + flat = F.pad(flat, (0, (-flat.numel()) % world_size)) + return flat.view(world_size, -1)[rank] + + +def run_shards(rank, world_size, port, optimizer_cls): + # colossalai.launch would pick NCCL on a GPU machine; these shards live on the CPU + os.environ.update(MASTER_ADDR="127.0.0.1", MASTER_PORT=str(port)) + dist.init_process_group("gloo", rank=rank, world_size=world_size) + gen = torch.Generator().manual_seed(0) + shapes = {"tall": (HIDDEN, DIM_OUT), "wide": (DIM_IN, HIDDEN), "square": (48, 48), "bias": (HIDDEN,)} + init = {n: torch.randn(s, generator=gen) for n, s in shapes.items()} + grads = [{n: torch.randn(s, generator=gen) for n, s in shapes.items()} for _ in range(STEPS)] + matrices = [n for n, s in shapes.items() if len(s) == 2] + + def groups(params): + return [ + {"params": [params[n] for n in matrices], "use_muon": True}, + {"params": [params["bias"]], "use_muon": False}, + ] + + whole = {n: nn.Parameter(t.clone()) for n, t in init.items()} + ref = Muon(groups(whole), lr=LR, weight_decay=0.1) + # as with LowLevelZeRO: built on the whole parameters, then handed the shards and told what they belong to + working = {n: nn.Parameter(t.clone()) for n, t in init.items()} + shards = {n: nn.Parameter(shard_of(t, world_size, rank).clone()) for n, t in init.items()} + test = optimizer_cls(groups(working), lr=LR, weight_decay=0.1) + for group, sharded in zip(test.param_groups, groups(shards)): + group["params"] = sharded["params"] + test.setup_distributed(None, None, {id(shards[n]): working[n] for n in shapes}, None, True) + + for step_grads in grads: + for n in shapes: + whole[n].grad = step_grads[n].clone() + shards[n].grad = shard_of(step_grads[n], world_size, rank).clone() + ref.step() + test.step() + + def gathered(t, shape): + parts = [torch.empty_like(t) for _ in range(world_size)] + dist.all_gather(parts, t.detach().contiguous()) + return torch.cat(parts)[: torch.Size(shape).numel()].view(shape) + + # updates, not weights: next to unit-variance weights a wrong update would hide inside any relative bound + result = {f"update/{n}": gathered(shards[n], shapes[n]) - init[n] for n in shapes} + result.update({f"momentum/{n}": gathered(test.state[shards[n]]["momentum_buffer"], shapes[n]) for n in matrices}) + expected = {f"update/{n}": whole[n].detach() - init[n] for n in shapes} + expected.update({f"momentum/{n}": ref.state[whole[n]]["momentum_buffer"] for n in matrices}) + if optimizer_cls is Muon: + # the gathered input is the bf16 cast the whole-matrix step makes, so the results are the same bits + judge_equal(result, expected).assert_pass() + else: + report = judge_relative_bound(result, expected, torch.bfloat16) + assert {e.name for e in report.failures} >= {f"update/{n}" for n in matrices}, report.format() + dist.destroy_process_group() + + +@pytest.mark.parametrize("world_size", [2, 3]) +def test_shards_match_whole_matrices(world_size): + torch.multiprocessing.spawn(run_shards, args=(world_size, free_port(), Muon), nprocs=world_size) + + +def test_shard_local_orthogonalization_detected(): + torch.multiprocessing.spawn(run_shards, args=(2, free_port(), ShardLocalMuon), nprocs=2) + + +def run_setup_checks(rank, world_size, port): + os.environ.update(MASTER_ADDR="127.0.0.1", MASTER_PORT=str(port)) + dist.init_process_group("gloo", rank=rank, world_size=world_size) + weight = nn.Parameter(torch.randn(4, 6)) + optimizer = Muon([weight]) + with pytest.raises(NotImplementedError, match="tensor parallelism"): + optimizer.setup_distributed(tp_group=dist.group.WORLD) + # not one of world_size equal slices, as with expert parameters ZeRO shards over their own group + shard = nn.Parameter(torch.randn(5)) + optimizer.param_groups[0]["params"] = [shard] + with pytest.raises(ValueError, match="do not span the data-parallel group"): + optimizer.setup_distributed(None, None, {id(shard): weight}, None, True) + dist.destroy_process_group() + + +def test_setup_rejects_tensor_parallel_and_foreign_shards(): + torch.multiprocessing.spawn(run_setup_checks, args=(2, free_port()), nprocs=2) + + +# --------------------------------------------------------------------------- +# LowLevelZeRO on GPUs: N ranks against one rank of the same plugin +# --------------------------------------------------------------------------- + + +class MLP(nn.Module): + def __init__(self): + super().__init__() + self.fc1 = nn.Linear(DIM_IN, HIDDEN) + self.norm = nn.LayerNorm(HIDDEN) + self.fc2 = nn.Linear(HIDDEN, DIM_OUT) + + def forward(self, x): + return {"y": self.fc2(self.norm(F.gelu(self.fc1(x))))} + + +def _data(): + return {"x": torch.randn(8, DIM_IN)} + + +def _identity(output): + return output + + +# module-level parts only: the entry is sent to spawned processes +ENTRY = (MLP, _data, _identity, None, SimpleNamespace(numerics=NumericsSpec())) + + +def run_zero(rank, world_size, port, plugin_fn, optimizer_fn, path): + disable_existing_loggers() + colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) + # every rank trains on the same batch: averaging N equal gradients is exact for N = 2, so only sharding differs + trace = run_steps(prepare(ENTRY, STEPS), plugin_fn(), 0, STEPS, optimizer_fn=optimizer_fn) + if rank == 0: + trace.save(path) + dist.destroy_process_group() + + +def zero_trace(plugin_fn, world_size, optimizer_fn=muon_optimizer): + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "trace.safetensors") + spawn(run_zero, world_size, plugin_fn=plugin_fn, optimizer_fn=optimizer_fn, path=path) + return Trace.load(path) + + +def comparable(test, world_size, ref): + """Losses, weights and the optimizer's state (float32 master weights, momentum, AdamW moments) of both runs. + + The optimizer's tensors are recorded per rank in ZeRO's flat layout; the slices of ``test`` are joined and the + padding cut to the length of ``ref``'s, which one rank holds unpadded.""" + keep = lambda k: k.startswith(("loss/", "param/")) # noqa: E731 + t, r = {k: v for k, v in test.items() if keep(k)}, {k: v for k, v in ref.items() if keep(k)} + for key, value in ref.items(): + if key.startswith("optim/"): + parts = [test[f"rank{i}/{key}"].flatten() for i in range(world_size)] + r[key] = value.flatten() + t[key] = torch.cat(parts)[: r[key].numel()] + return t, r + + +def updates(state, precision): + """The master weights' change over the run instead of the weights, plus the optimizer's other state. + + For a relative bound: three steps at lr = 0.02 change these weights by a few percent, so a wrong update would stay + within the bound if the weights themselves were compared.""" + params = [p for p in prepare(ENTRY, STEPS).model.parameters() if p.requires_grad] + groups = [[p for p in params if p.ndim == 2], [p for p in params if p.ndim != 2]] # as muon_optimizer builds them + # the plugin casts the model to its precision before ZeRO copies the float32 master weights + dtype = torch.bfloat16 if precision == "bf16" else torch.float32 + start = { + f"optim/{g}.{i}/param": p.detach().to(dtype).float().flatten() + for g, ps in enumerate(groups) + for i, p in enumerate(ps) + } + out = {} + for key, value in state.items(): + if key in start: + out[f"update/{key}"] = value - start[key] + elif key.startswith("optim/"): + out[key] = value + return out + + +ZERO_CONFIGS = [(1, "fp32", False), (2, "fp32", False), (1, "bf16", False), (2, "bf16", False), (2, "bf16", True)] + + +@pytest.mark.dist +@requires_gpus(2) +@pytest.mark.parametrize("stage, precision, cpu_offload", ZERO_CONFIGS) +@rerun_if_address_is_in_use() +def test_zero_two_ranks_equal_one(stage, precision, cpu_offload): + plugin_fn = partial(LowLevelZeroPlugin, stage=stage, precision=precision, cpu_offload=cpu_offload) + test, ref = comparable(zero_trace(plugin_fn, 2), 2, zero_trace(plugin_fn, 1)) + judge_equal(test, ref).assert_pass() + + +@pytest.mark.dist +@requires_gpus(4) +@rerun_if_address_is_in_use() +def test_zero_four_ranks_within_bound(): + # averaging 4 equal gradients rounds, so the runs differ in the last bits: Megatron-LM's relative bound + plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="bf16") + test, ref = comparable(zero_trace(plugin_fn, 4), 4, zero_trace(plugin_fn, 1)) + judge_relative_bound(updates(test, "bf16"), updates(ref, "bf16"), torch.bfloat16).assert_pass() + + +@pytest.mark.dist +@requires_gpus(2) +@rerun_if_address_is_in_use() +def test_zero_shard_local_orthogonalization_detected(): + plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="bf16") + shard_local = partial(muon_optimizer, cls=ShardLocalMuon) + test, ref = comparable(zero_trace(plugin_fn, 2, shard_local), 2, zero_trace(plugin_fn, 1)) + report = judge_relative_bound(updates(test, "bf16"), updates(ref, "bf16"), torch.bfloat16) + assert {e.name for e in report.failures} >= {"update/optim/0.0/param", "update/optim/0.1/param"}, report.format() + + +@pytest.mark.dist +@requires_gpus(2) +@rerun_if_address_is_in_use() +def test_zero_resume_is_exact(): + # fp32: in bf16 no optimizer resumes exactly, as LowLevelZero rebuilds the float32 master weights from the bf16 + # ones on load (tests/kit/numerics/README.md, "Resuming from a checkpoint") + plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="fp32") + resume.check(ENTRY, plugin_fn, world_size=2, optimizer_fn=muon_optimizer).assert_pass() + + +def run_plugins(rank, world_size, port): + disable_existing_loggers() + colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) + for plugin in (GeminiPlugin(), TorchFSDPPlugin()): + with pytest.raises(NotImplementedError, match="does not support Muon"): + model = MLP().cuda() + Booster(plugin=plugin).boost(model, muon_optimizer(model.parameters())) + model = MLP().cuda() + muon = muon_optimizer(model.parameters()) + _, optimizer, *_ = Booster(plugin=LowLevelZeroPlugin(stage=1)).boost(model, muon) + # set up in place, not replaced: an LR scheduler built on `muon` before boost keeps driving it + assert optimizer.optim is muon and muon.shard_to_working_param + dist.destroy_process_group() + + +@pytest.mark.dist +@requires_gpus(1) +@rerun_if_address_is_in_use() +def test_plugins_with_muon(): + spawn(run_plugins, 1) diff --git a/tests/test_optimizer/test_muon.py b/tests/test_optimizer/test_muon.py new file mode 100644 index 000000000000..340d12924e02 --- /dev/null +++ b/tests/test_optimizer/test_muon.py @@ -0,0 +1,333 @@ +import copy +import json + +import pytest +import torch +import torch.nn as nn + +import colossalai +from colossalai.booster import Booster +from colossalai.booster.plugin import TorchDDPPlugin +from colossalai.logging import disable_existing_loggers +from colossalai.nn.optimizer import Muon, muon_param_groups +from colossalai.nn.optimizer.muon import zeropower_via_newtonschulz +from colossalai.testing import rerun_if_address_is_in_use, spawn +from tests.kit.numerics import assert_aligned, judge, judge_relative_bound +from tests.kit.numerics.scenarios.independent_impl import check_optimizer + +requires_torch_muon = pytest.mark.skipif(not hasattr(torch.optim, "Muon"), reason="torch.optim.Muon needs torch>=2.9") +DEVICES = ["cpu"] + (["cuda"] if torch.cuda.is_available() else []) + +MATRIX_SHAPES = [(64, 32), (32, 64), (48, 48)] +OTHER_SHAPES = [(64,), (100, 32)] # a bias and an embedding: both go to the AdamW branch +STEPS = 3 +LR, ADAMW_LR, BETAS, ADAMW_EPS = 0.02, 3e-3, (0.9, 0.95), 1e-8 + + +# --------------------------------------------------------------------------- +# float64 ground truth, transcribed from torch/optim/_muon.py (torch 2.13) without its bfloat16 cast +# --------------------------------------------------------------------------- + + +def reference_newtonschulz(grad, steps=5): + a, b, c = 3.4445, -4.7750, 2.0315 + x = grad.double() + if grad.size(0) > grad.size(1): + x = x.T + x = x / x.norm().clamp(min=1e-7) + for _ in range(steps): + gram = x @ x.T + x = a * x + (b * gram + c * gram @ gram) @ x + if grad.size(0) > grad.size(1): + x = x.T + return x + + +def reference_muon_step(param, grad, buf, lr, weight_decay, momentum, nesterov, adjust_lr_fn): + buf.lerp_(grad, 1 - momentum) + update = grad.lerp(buf, momentum) if nesterov else buf + update = reference_newtonschulz(update) + rows, cols = param.shape + if adjust_lr_fn == "match_rms_adamw": + adjusted_lr = lr * 0.2 * max(rows, cols) ** 0.5 + else: + adjusted_lr = lr * max(1, rows / cols) ** 0.5 + param.mul_(1 - lr * weight_decay) + param.add_(update, alpha=-adjusted_lr) + + +# --------------------------------------------------------------------------- +# single-step and multi-step alignment against torch.optim.Muon / torch.optim.AdamW +# --------------------------------------------------------------------------- + + +def make_problem(): + gen = torch.Generator().manual_seed(0) + shapes = MATRIX_SHAPES + OTHER_SHAPES + init = [torch.randn(s, generator=gen, dtype=torch.float64) for s in shapes] + grads = [[torch.randn(s, generator=gen, dtype=torch.float64) for s in shapes] for _ in range(STEPS)] + return init, grads + + +def run_steps(init, grads, dtype, device, step_fn): + """Apply ``step_fn`` for every step; return the parameter deltas and the Muon momentum buffers.""" + # copy=True: without it the float64 run would update the shared ``init`` tensors in place. + params = [nn.Parameter(t.to(device=device, dtype=dtype, copy=True)) for t in init] + start = [p.detach().clone() for p in params] + n = len(MATRIX_SHAPES) + bufs = step_fn(params[:n], params[n:], grads, dtype, device) + out = {f"delta/{i}": (p.detach().double() - s.double()).cpu() for i, (p, s) in enumerate(zip(params, start))} + out.update({f"momentum/{i}": b.detach().cpu() for i, b in enumerate(bufs)}) + return out + + +def colossal_step_fn(**muon_kwargs): + def step_fn(matrices, others, grads, dtype, device): + groups = [ + {"params": matrices, "use_muon": True}, + {"params": others, "use_muon": False, "lr": ADAMW_LR}, + ] + opt = Muon(groups, lr=LR, betas=BETAS, adamw_eps=ADAMW_EPS, **muon_kwargs) + for step_grads in grads: + for p, g in zip(matrices + others, step_grads): + p.grad = g.to(device=device, dtype=dtype, copy=True) + opt.step() + return [opt.state[p]["momentum_buffer"] for p in matrices] + + return step_fn + + +def torch_step_fn(weight_decay, nesterov, adjust_lr_fn, reference_muon): + def step_fn(matrices, others, grads, dtype, device): + adamw = torch.optim.AdamW( + others, lr=ADAMW_LR, betas=BETAS, eps=ADAMW_EPS, weight_decay=weight_decay, foreach=False + ) + if reference_muon: + bufs = [torch.zeros_like(p) for p in matrices] + else: + muon = torch.optim.Muon( + matrices, lr=LR, weight_decay=weight_decay, nesterov=nesterov, adjust_lr_fn=adjust_lr_fn + ) + for step_grads in grads: + for p, g in zip(matrices + others, step_grads): + p.grad = g.to(device=device, dtype=dtype, copy=True) + with torch.no_grad(): + if reference_muon: + for p, buf in zip(matrices, bufs): + reference_muon_step(p, p.grad, buf, LR, weight_decay, 0.95, nesterov, adjust_lr_fn) + else: + muon.step() + adamw.step() + return bufs if reference_muon else [muon.state[p]["momentum_buffer"] for p in matrices] + + return step_fn + + +@requires_torch_muon +@pytest.mark.parametrize("device", DEVICES) +@pytest.mark.parametrize("shape", MATRIX_SHAPES) +def test_newton_schulz_matches_torch(device, shape): + from torch.optim._muon import _zeropower_via_newtonschulz + + grad = torch.randn(shape, generator=torch.Generator().manual_seed(1), dtype=torch.float64).to(device) + truth = reference_newtonschulz(grad) + ref = _zeropower_via_newtonschulz(grad.float(), (3.4445, -4.7750, 2.0315), 5, 1e-7) + test = zeropower_via_newtonschulz(grad.float()) + assert_aligned(test, ref, truth) + + +@requires_torch_muon +@pytest.mark.parametrize("device", DEVICES) +@pytest.mark.parametrize("weight_decay", [0.0, 0.1]) +@pytest.mark.parametrize("nesterov", [True, False]) +@pytest.mark.parametrize("adjust_lr_fn", [None, "match_rms_adamw"]) +def test_muon_matches_torch(device, weight_decay, nesterov, adjust_lr_fn): + init, grads = make_problem() + hyper = dict(weight_decay=weight_decay, nesterov=nesterov, adjust_lr_fn=adjust_lr_fn) + truth = run_steps(init, grads, torch.float64, device, torch_step_fn(**hyper, reference_muon=True)) + ref = run_steps(init, grads, torch.float32, device, torch_step_fn(**hyper, reference_muon=False)) + test = run_steps(init, grads, torch.float32, device, colossal_step_fn(**hyper)) + assert_aligned(test, ref, truth) + + +@requires_torch_muon +@pytest.mark.parametrize( + "wrong", + [ + {"adjust_lr_fn": "match_rms_adamw"}, + {"nesterov": False}, + {"momentum": 0.9}, + {"momentum": 0.949}, + {"weight_decay": 0.09}, + ], +) +def test_alignment_check_catches_wrong_hyperparameters(wrong): + # The baseline-error rule must not let a wrong update through just because bf16 Newton-Schulz is noisy. + init, grads = make_problem() + hyper = dict(weight_decay=0.1, nesterov=True, adjust_lr_fn=None) + truth = run_steps(init, grads, torch.float64, "cpu", torch_step_fn(**hyper, reference_muon=True)) + ref = run_steps(init, grads, torch.float32, "cpu", torch_step_fn(**hyper, reference_muon=False)) + test = run_steps(init, grads, torch.float32, "cpu", colossal_step_fn(**{**hyper, **wrong})) + assert not judge(test, ref, truth).passed + + +# check_optimizer judges the weights and judge scales tol by their RMS. At lr=1e-3 the updates are far smaller than +# these unit-variance weights, so the default tol would let lr * 1.05 or momentum 0.9 through its allclose fast path. +WEIGHT_TOL = 1e-7 + + +@requires_torch_muon +@pytest.mark.parametrize("adjust_lr_fn", [None, "match_rms_adamw"]) +def test_check_optimizer_against_torch(adjust_lr_fn): + gen = torch.Generator().manual_seed(3) + params = {f"matrix{i}": torch.randn(s, generator=gen, dtype=torch.float64) for i, s in enumerate(MATRIX_SHAPES)} + lr = 1e-3 + + def check(**muon_kwargs): + return check_optimizer( + lambda ps: Muon(ps, adjust_lr_fn=adjust_lr_fn, **muon_kwargs), + lambda ps: torch.optim.Muon(ps, lr=lr, adjust_lr_fn=adjust_lr_fn), + params, + torch.float32, + steps=STEPS, + tol=WEIGHT_TOL, + ) + + check(lr=lr).assert_pass() + assert not check(lr=lr * 1.05).passed + assert not check(lr=lr, momentum=0.9).passed + + +# --------------------------------------------------------------------------- +# parameter grouping and error handling +# --------------------------------------------------------------------------- + + +class TinyLM(nn.Module): + def __init__(self, tie: bool): + super().__init__() + self.embed = nn.Embedding(50, 16) + self.proj = nn.Linear(16, 16) + self.norm = nn.LayerNorm(16) + self.lm_head = nn.Linear(16, 50, bias=False) + if tie: + self.lm_head.weight = self.embed.weight + + def get_output_embeddings(self): + return self.lm_head + + +def group_ids(groups): + return {g["use_muon"]: {id(p) for p in g["params"]} for g in groups} + + +@pytest.mark.parametrize("tie", [True, False]) +def test_muon_param_groups(tie): + model = TinyLM(tie) + groups = group_ids(muon_param_groups(model)) + assert groups[True] == {id(model.proj.weight)} + expected_adamw = {model.embed.weight, model.proj.bias, model.norm.weight, model.norm.bias, model.lm_head.weight} + assert groups[False] == {id(p) for p in expected_adamw} + + +def test_muon_param_groups_override(): + model = TinyLM(tie=False) + model.proj.weight.use_muon = False + model.lm_head.weight.use_muon = True + groups = group_ids(muon_param_groups(model)) + assert groups[True] == {id(model.lm_head.weight)} + assert id(model.proj.weight) in groups[False] + + +def test_muon_rejects_non_matrix_params(): + with pytest.raises(ValueError, match="only supports 2D"): + Muon([nn.Parameter(torch.randn(8))]) + # A group without use_muon defaults to Muon, so a bias passed by mistake is caught. + with pytest.raises(ValueError, match="only supports 2D"): + Muon(nn.Linear(4, 4).parameters()) + + +def test_muon_rejects_sharded_gradients(): + # LowLevelZeRO hands the inner optimizer flat 1D slices of each parameter. + opt = Muon([nn.Parameter(torch.randn(4, 4))]) + shard = nn.Parameter(torch.randn(8)) + shard.grad = torch.randn(8) + opt.param_groups[0]["params"] = [shard] + with pytest.raises(RuntimeError, match="full 2D gradient"): + opt.step() + + +def test_muon_state_dict_round_trip(): + model = TinyLM(tie=False) + opt = Muon(muon_param_groups(model), lr=LR) + for p in model.parameters(): + p.grad = torch.randn_like(p) + opt.step() + state = opt.state_dict() + # Hyperparameters stay plain data so JSON-based checkpoint formats can store them. + json.dumps([{k: v for k, v in g.items() if k != "params"} for g in state["param_groups"]]) + + restored = Muon(muon_param_groups(model), lr=LR) + restored.load_state_dict(copy.deepcopy(state)) + for p in model.parameters(): + for key, value in opt.state[p].items(): + torch.testing.assert_close(restored.state[p][key], value, rtol=0, atol=0) + + +# --------------------------------------------------------------------------- +# DDP: two ranks on half batches must match one device on the full batch +# --------------------------------------------------------------------------- + +DIM, GLOBAL_BATCH = 32, 8 + + +def make_mlp(): + torch.manual_seed(0) + return nn.Sequential(nn.Linear(DIM, 64), nn.GELU(), nn.LayerNorm(64), nn.Linear(64, DIM)) + + +def train(model, batches, booster=None, rank=0, world_size=1): + device = torch.device("cuda", torch.cuda.current_device()) + model = model.to(device) + start = {n: p.detach().double().clone() for n, p in model.named_parameters()} + opt = Muon(muon_param_groups(model), lr=LR, weight_decay=0.1) + if booster is not None: + model, opt, *_ = booster.boost(model, opt) + local = GLOBAL_BATCH // world_size + for batch in batches: + x = batch[rank * local : (rank + 1) * local].to(device) + loss = model(x).pow(2).mean() + if booster is not None: + booster.backward(loss, opt) + else: + loss.backward() + opt.step() + opt.zero_grad() + module = model.unwrap() if hasattr(model, "unwrap") else model + return {n: (p.detach().double() - start[n]).cpu() for n, p in module.named_parameters()} + + +def run_ddp(rank, world_size, port): + disable_existing_loggers() + colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port, backend="nccl") + batches = torch.randn(STEPS, GLOBAL_BATCH, DIM, generator=torch.Generator().manual_seed(2)) + booster = Booster(plugin=TorchDDPPlugin()) + + test = train(make_mlp(), batches, booster, rank, world_size) + ref = train(make_mlp(), batches) + # Same code at the same precision, so there is no baseline error to measure: the tool's relative bound applies. + judge_relative_bound(test, ref, torch.bfloat16).assert_pass() + # Each rank stepping on its own half batch (no gradient all-reduce) must be caught. + unsynced = train(make_mlp(), batches, rank=rank, world_size=world_size) + assert not judge_relative_bound(unsynced, ref, torch.bfloat16).passed + + +@pytest.mark.dist +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="needs 2 GPUs") +@rerun_if_address_is_in_use() +def test_muon_ddp(): + spawn(run_ddp, nprocs=2) + + +if __name__ == "__main__": + test_muon_ddp()