diff --git a/.github/scripts/ci/run_cpu_unit_tests.sh b/.github/scripts/ci/run_cpu_unit_tests.sh index 68bbeaed221b..e981c7f33183 100644 --- a/.github/scripts/ci/run_cpu_unit_tests.sh +++ b/.github/scripts/ci/run_cpu_unit_tests.sh @@ -22,6 +22,7 @@ TESTS=( tests/test_optimizer/test_dist_muon.py tests/test_optimizer/test_lr_scheduler.py tests/test_optimizer/test_muon.py + tests/test_optimizer/test_orscale.py tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py diff --git a/colossalai/booster/plugin/gemini_plugin.py b/colossalai/booster/plugin/gemini_plugin.py index 7a189e17f494..a4a6bf610e05 100644 --- a/colossalai/booster/plugin/gemini_plugin.py +++ b/colossalai/booster/plugin/gemini_plugin.py @@ -668,8 +668,8 @@ def configure( ) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]: if isinstance(optimizer, Muon): raise NotImplementedError( - "GeminiPlugin does not support Muon: its chunks split matrices at arbitrary boundaries. " - "Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." + f"GeminiPlugin does not support {type(optimizer).__name__}: its chunks split matrices at arbitrary " + "boundaries. Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." ) params_info = get_param_info(optimizer) if not isinstance(model, ModelWrapper): diff --git a/colossalai/booster/plugin/torch_fsdp_plugin.py b/colossalai/booster/plugin/torch_fsdp_plugin.py index 5c9390539972..2e88bb3a5362 100644 --- a/colossalai/booster/plugin/torch_fsdp_plugin.py +++ b/colossalai/booster/plugin/torch_fsdp_plugin.py @@ -539,8 +539,8 @@ def configure( ) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]: if isinstance(optimizer, Muon): raise NotImplementedError( - "TorchFSDPPlugin does not support Muon: FSDP hands the optimizer flattened parameters in one group. " - "Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." + f"TorchFSDPPlugin does not support {type(optimizer).__name__}: FSDP hands the optimizer flattened " + "parameters in one group. Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." ) # wrap the model with PyTorch FSDP fsdp_model = TorchFSDPModel(model, device_id=torch.cuda.current_device(), **self.fsdp_kwargs) diff --git a/colossalai/nn/optimizer/__init__.py b/colossalai/nn/optimizer/__init__.py index ebdd5dc3eb6d..453ba250ca8f 100644 --- a/colossalai/nn/optimizer/__init__.py +++ b/colossalai/nn/optimizer/__init__.py @@ -16,6 +16,7 @@ from .lamb import Lamb from .lars import Lars from .muon import Muon, muon_param_groups +from .orscale import OrScale from .adafactor import Adafactor # noqa @@ -39,6 +40,7 @@ "DistributedAdaFactor", "Muon", "muon_param_groups", + "OrScale", ] optim2DistOptim = { diff --git a/colossalai/nn/optimizer/muon.py b/colossalai/nn/optimizer/muon.py index bd6de205de84..1f77b1e2ff69 100644 --- a/colossalai/nn/optimizer/muon.py +++ b/colossalai/nn/optimizer/muon.py @@ -132,7 +132,7 @@ def __init__( raise ValueError(f"ns_steps must be in [1, 99], got {ns_steps}") if adjust_lr_fn is not None and adjust_lr_fn not in ADJUST_LR_FNS: raise ValueError(f"adjust_lr_fn must be one of {ADJUST_LR_FNS} or None, got {adjust_lr_fn!r}") - self._muon_defaults = dict( + muon_defaults = dict( momentum=momentum, nesterov=nesterov, ns_coefficients=tuple(ns_coefficients), @@ -140,7 +140,12 @@ def __init__( eps=eps, adjust_lr_fn=adjust_lr_fn, ) - self._adamw_defaults = dict(betas=tuple(betas), eps=adamw_eps) + self._init_groups(params, lr, weight_decay, muon_defaults, dict(betas=tuple(betas), eps=adamw_eps)) + + def _init_groups(self, params, lr: float, weight_decay: float, muon_defaults: Dict, adamw_defaults: Dict) -> None: + """Build the parameter groups; ``muon_defaults`` and ``adamw_defaults`` fill the keys a group leaves out.""" + self._muon_defaults = muon_defaults + self._adamw_defaults = adamw_defaults super().__init__(params, dict(lr=lr, weight_decay=weight_decay)) self.dp_group: Optional[dist.ProcessGroup] = None self.shard_to_working_param: Dict[int, Tensor] = {} @@ -159,7 +164,7 @@ def add_param_group(self, param_group: Dict) -> None: for p in params: if p.ndim != 2: raise ValueError( - f"Muon only supports 2D parameters, got shape {tuple(p.shape)}; " + f"{type(self).__name__} only supports 2D parameters, got shape {tuple(p.shape)}; " "put it in a group with use_muon=False (see muon_param_groups)" ) super().add_param_group(param_group) @@ -182,7 +187,7 @@ def setup_distributed( is_zero (bool): unused; shards are recognized by their shape. """ if tp_group is not None and dist.get_world_size(tp_group) > 1: - raise NotImplementedError("Muon does not support tensor parallelism yet; use tp_size=1") + raise NotImplementedError(f"{type(self).__name__} does not support tensor parallelism yet; use tp_size=1") self.dp_group = dp_group self.shard_to_working_param = dict(shard_to_working_param or {}) world_size = dist.get_world_size(dp_group) @@ -195,9 +200,9 @@ def setup_distributed( continue if working.ndim != 2 or p.numel() != -(-working.numel() // world_size): raise ValueError( - f"a {tuple(working.shape)} Muon parameter arrives as a shard of {p.numel()} elements, not one " + f"a {tuple(working.shape)} matrix arrives as a shard of {p.numel()} elements, not one " f"of {world_size} equal slices: its shards do not span the data-parallel group (e.g. expert " - "parameters sharded over their own group), which Muon does not support" + f"parameters sharded over their own group), which {type(self).__name__} does not support" ) def _working_param(self, p: Tensor) -> Tensor: @@ -226,23 +231,20 @@ def _muon_step(self, group: Dict) -> None: self._step_matrix(p, group) else: shards.append(p) - # every rank holds the same shards in the same order, so the gathers line up + # every rank holds the same shards in the same order, so the collectives line up for bucket in self._buckets(shards): inputs = [self._momentum_update(p, group) for p in bucket] - for p, update in zip(bucket, self._orthogonalize_shards(bucket, inputs, group)): - self._apply_update(p, update, self._working_param(p).shape, group) + self._apply_shard_updates(bucket, self._orthogonalize_shards(bucket, inputs, group), group) def _step_matrix(self, p: Tensor, group: Dict) -> None: if p.grad.ndim != 2: # A sharded 1D view that setup_distributed was not told about. raise RuntimeError( - f"Muon needs the full 2D gradient but got shape {tuple(p.grad.shape)}; under ZeRO the plugin has to " - "call setup_distributed (LowLevelZeroPlugin does), and Gemini / FSDP are not supported" + f"{type(self).__name__} needs the full 2D gradient but got shape {tuple(p.grad.shape)}; under ZeRO " + "the plugin has to call setup_distributed (LowLevelZeroPlugin does), and Gemini / FSDP are not " + "supported" ) - update = zeropower_via_newtonschulz( - self._momentum_update(p, group), group["ns_coefficients"], group["ns_steps"], group["eps"] - ) - self._apply_update(p, update, p.shape, group) + self._apply_update(p, self._orthogonalize(self._momentum_update(p, group), group), group) def _momentum_update(self, p: Tensor, group: Dict) -> Tensor: """Update ``p``'s momentum buffer and return the Newton-Schulz input; elementwise, so shards work too.""" @@ -254,9 +256,22 @@ def _momentum_update(self, p: Tensor, group: Dict) -> Tensor: buf.lerp_(grad, 1 - momentum) return grad.lerp(buf, momentum) if group["nesterov"] else buf + def _orthogonalize(self, matrix: Tensor, group: Dict) -> Tensor: + """Orthogonalize a whole matrix; the result is bf16.""" + return zeropower_via_newtonschulz(matrix, group["ns_coefficients"], group["ns_steps"], group["eps"]) + + def _apply_update(self, p: Tensor, update: Tensor, group: Dict) -> None: + """Apply weight decay and the orthogonalized ``update`` to a whole matrix.""" + self._decay_and_add(p, update, p.shape, group) + + def _apply_shard_updates(self, shards: List[Tensor], updates: List[Tensor], group: Dict) -> None: + """Apply to each of this rank's ``shards`` its slice of the whole matrix's orthogonalized update.""" + for p, update in zip(shards, updates): + self._decay_and_add(p, update, self._working_param(p).shape, group) + @staticmethod - def _apply_update(p: Tensor, update: Tensor, shape: torch.Size, group: Dict) -> None: - """Apply weight decay and the orthogonalized ``update``; ``shape`` is the whole matrix's, for the lr scale.""" + def _decay_and_add(p: Tensor, update: Tensor, shape: torch.Size, group: Dict) -> None: + # elementwise, so the same for a matrix and a shard; ``shape`` is the whole matrix's, for the lr scale lr = group["lr"] p.mul_(1 - lr * group["weight_decay"]) p.add_(update, alpha=-_adjust_lr(lr, group["adjust_lr_fn"], shape)) @@ -275,11 +290,7 @@ def _buckets(self, shards: List[Tensor]) -> List[List[Tensor]]: def _orthogonalize_shards(self, shards: List[Tensor], inputs: List[Tensor], group: Dict) -> List[Tensor]: """Gather the whole Newton-Schulz inputs of ``shards``, orthogonalize them and return each rank's slices.""" world_size, rank = dist.get_world_size(self.dp_group), dist.get_rank(self.dp_group) - # gloo gathers CPU tensors; NCCL (and Newton-Schulz) want the accelerator - if dist.get_backend(self.dp_group) == dist.Backend.GLOO: - device = torch.device("cpu") - else: - device = get_accelerator().get_current_device() + device = self._collective_device() local = torch.cat([x.to(device=device, dtype=torch.bfloat16).flatten() for x in inputs]) gathered = torch.empty(world_size * local.numel(), dtype=local.dtype, device=device) dist.all_gather_into_tensor(gathered, local, group=self.dp_group) @@ -290,11 +301,17 @@ def _orthogonalize_shards(self, shards: List[Tensor], inputs: List[Tensor], grou working, n = self._working_param(p), p.numel() matrix = gathered[:, offset : offset + n].reshape(-1)[: working.numel()].view(working.shape) offset += n - update = zeropower_via_newtonschulz(matrix, group["ns_coefficients"], group["ns_steps"], group["eps"]) + update = self._orthogonalize(matrix, group) update = F.pad(update.flatten(), (0, world_size * n - working.numel())).view(world_size, n)[rank] updates.append(update.to(p.device)) return updates + def _collective_device(self) -> torch.device: + # gloo communicates CPU tensors; NCCL (and Newton-Schulz) want the accelerator + if dist.get_backend(self.dp_group) == dist.Backend.GLOO: + return torch.device("cpu") + return get_accelerator().get_current_device() + def _adamw_step(self, group: Dict) -> None: lr, weight_decay, eps = group["lr"], group["weight_decay"], group["eps"] beta1, beta2 = group["betas"] diff --git a/colossalai/nn/optimizer/orscale.py b/colossalai/nn/optimizer/orscale.py new file mode 100644 index 000000000000..4bab469ad713 --- /dev/null +++ b/colossalai/nn/optimizer/orscale.py @@ -0,0 +1,219 @@ +"""OrScale: Muon's orthogonalized update scaled by a layer-wise trust ratio, with a built-in AdamW. + +For a matrix W with Nesterov momentum M, the orthogonalized direction Q = PolarExpress(M) and shape scale s: + + r = clip(||W||_F / (c * ||wd * W + s * Q||_F + eps), r_min, r_max) + W <- W - lr * r * (wd * W + s * Q) + +so the weight decay is coupled into the trust-ratio-scaled step. The two variants of the paper: + + OrScale s = 1, c = 1 + OrScale-LM s = 0.2 * sqrt(max(rows, cols)) (Moonlight's), and c set per layer at its first step so that r = 1 there + +The arithmetic follows the official implementation (https://github.com/NUS-HPC-AI-Lab/OrScale, commit a45fc2b) +statement by statement: accumulated float32 momentum (not ``Muon``'s moving average), bf16 Polar Express +orthogonalization, float32 norms. Run eagerly, the two give the same bits. The official code compiles its +orthogonalization with ``torch.compile`` on CUDA, which this one does not; that alone moves the result by 1-3%, as much +as bf16 moves it from a float64 iteration, and so does any other change of the bf16 rounding, such as Muon's moving +average in place of the accumulated momentum. Its two optimizers (OrScale for matrices, AdamW for the rest) are one +here, as in ``Muon``, so a Booster still receives a single optimizer; its ablation variants and its flattening of +convolution weights are left out. + +Under LowLevelZeRO the matrices are orthogonalized whole, as ``Muon`` does. The trust ratio needs two norms of the +whole matrix: each rank sums the squares over its slice and one all-reduce of two numbers per matrix adds them up. +Sums in a different order round differently, so a sharded step equals the unsharded one up to float32 rounding of +the trust ratio (about 1e-7), not bit for bit. + +Reference: + - OrScale: Orthogonalised Optimization with Layer-Wise Trust-Ratio Scaling, https://arxiv.org/abs/2605.07815 + - The Polar Express, https://arxiv.org/abs/2505.16932 +""" + +import math +from typing import Dict, List, Optional, Tuple + +import torch +import torch.distributed as dist +from torch import Tensor + +from .muon import Muon + +__all__ = ["OrScale", "polar_express"] + +# One (a, b, c) per iteration, from the official OrScale implementation (orscale/optim/newton_schulz.py). +POLAR_EXPRESS_COEFFICIENTS = ( + (8.156554524902461, -22.48329292557795, 15.878769915207462), + (4.042929935166739, -2.808917465908714, 0.5000178451051316), + (3.8916678022926607, -2.772484153217685, 0.5060648178503393), + (3.285753657755655, -2.3681294933425376, 0.46449024233003106), + (2.3465413258596377, -1.7097828382687081, 0.42323551169305323), +) +VARIANTS = ("orscale", "orscale_lm") +MOONLIGHT_RMS_CONSTANT = 0.2 + + +def polar_express(grad: Tensor, ns_steps: int = 5) -> Tensor: + """Approximate the polar factor U V^T of a 2D matrix with the Polar Express iteration, in bf16. + + Unlike ``zeropower_via_newtonschulz`` each iteration has its own coefficients, which converge to U V^T itself, and + the input is scaled by 1 / (1.02 * ||grad||_F + 1e-6). + """ + if grad.ndim != 2: + raise ValueError(f"Polar Express expects a 2D matrix, got shape {tuple(grad.shape)}") + if not 0 < ns_steps <= len(POLAR_EXPRESS_COEFFICIENTS): + raise ValueError(f"ns_steps must be in [1, {len(POLAR_EXPRESS_COEFFICIENTS)}], got {ns_steps}") + x = grad.bfloat16() + transposed = grad.size(0) > grad.size(1) + if transposed: + x = x.T + x = (x / (x.norm(dim=(-2, -1), keepdim=True) * 1.02 + 1e-6)).contiguous() + for a, b, c in POLAR_EXPRESS_COEFFICIENTS[:ns_steps]: + gram = x @ x.T + poly = torch.addmm(gram, gram, gram, beta=b, alpha=c) + x = torch.addmm(x, poly, x, beta=a) + if transposed: + x = x.T + return x + + +class OrScale(Muon): + """OrScale or OrScale-LM for 2D hidden-layer weights plus AdamW for every other parameter, in one optimizer. + + Parameter groups work as in ``Muon``: each sets ``use_muon`` (``muon_param_groups(model)`` builds both groups); + groups without the flag are OrScale groups. OrScale groups take ``lr``, ``weight_decay``, ``momentum``, + ``variant``, ``r_min``, ``r_max``, ``eps``, ``ns_steps`` and ``c_denom``; AdamW groups take ``lr``, + ``weight_decay``, ``betas`` and ``eps``. Missing keys fall back to the constructor arguments, with ``adamw_eps`` as + the AdamW groups' ``eps``. The defaults are the official ones for the matrices, where ``lr`` is far larger than + AdamW's, so give the AdamW group its own ``lr`` (the official default is a tenth of it). + + Like ``Muon``, it runs under ``TorchDDPPlugin`` and ``LowLevelZeroPlugin`` (ZeRO stage 1 and 2, see the module + docstring); tensor parallelism, Gemini and FSDP are not supported. + + Args: + params: parameters or parameter groups. + lr (float): learning rate shared by both branches unless a group overrides it (default: 0.02). + weight_decay (float): weight decay; for the matrices it is coupled into the trust-ratio-scaled step, for the + AdamW groups it is decoupled (default: 0.0). + momentum (float): Nesterov momentum of the matrices, accumulated as ``M = momentum * M + grad`` in float32 + (default: 0.95). + variant (str): ``"orscale"`` or ``"orscale_lm"`` (default: ``"orscale"``). + r_min (float): lower clipping bound of the trust ratio (default: 0.5). + r_max (float): upper clipping bound of the trust ratio (default: 1.5). The official recommendation is + [0.5, 1.5] for ``"orscale"`` and [0.1, 5.0] for ``"orscale_lm"``. + eps (float): added to the denominator of the trust ratio (default: 1e-6). + ns_steps (int): Polar Express iterations, at most 5 (default: 5). + c_denom (float, optional): ``"orscale_lm"`` only. The constant c of every layer; None calibrates it per layer + at the layer's first step as ``||W|| / ||wd * W + s * Q||`` and keeps it in the optimizer state + (default: None). + betas (tuple): AdamW betas (default: (0.9, 0.95)). + adamw_eps (float): AdamW epsilon (default: 1e-8). + """ + + def __init__( + self, + params, + lr: float = 0.02, + weight_decay: float = 0.0, + momentum: float = 0.95, + variant: str = "orscale", + r_min: float = 0.5, + r_max: float = 1.5, + eps: float = 1e-6, + ns_steps: int = 5, + c_denom: Optional[float] = None, + betas: Tuple[float, float] = (0.9, 0.95), + adamw_eps: float = 1e-8, + ): + if lr < 0.0: + raise ValueError(f"Invalid learning rate: {lr}") + if weight_decay < 0.0: + raise ValueError(f"Invalid weight_decay: {weight_decay}") + orscale_defaults = dict( + momentum=momentum, variant=variant, r_min=r_min, r_max=r_max, eps=eps, ns_steps=ns_steps, c_denom=c_denom + ) + self._init_groups(params, lr, weight_decay, orscale_defaults, dict(betas=tuple(betas), eps=adamw_eps)) + + def add_param_group(self, param_group: Dict) -> None: + if param_group.setdefault("use_muon", True): + group = {**self._muon_defaults, **param_group} + if not 0.0 <= group["momentum"] < 1.0: + raise ValueError(f"Invalid momentum: {group['momentum']}") + if group["variant"] not in VARIANTS: + raise ValueError(f"variant must be one of {VARIANTS}, got {group['variant']!r}") + if not 0.0 <= group["r_min"] <= group["r_max"]: + raise ValueError(f"need 0 <= r_min <= r_max, got r_min={group['r_min']}, r_max={group['r_max']}") + if not 0 < group["ns_steps"] <= len(POLAR_EXPRESS_COEFFICIENTS): + raise ValueError(f"ns_steps must be in [1, {len(POLAR_EXPRESS_COEFFICIENTS)}], got {group['ns_steps']}") + if group["c_denom"] is not None and group["c_denom"] <= 0.0: + raise ValueError(f"c_denom must be positive or None, got {group['c_denom']}") + super().add_param_group(param_group) + + def _momentum_update(self, p: Tensor, group: Dict) -> Tensor: + """Update ``p``'s momentum buffer and return the Polar Express input; elementwise, so shards work too.""" + # accumulated in float32 whatever the parameter's dtype + grad, momentum = p.grad.float(), group["momentum"] + state = self.state[p] + if "momentum_buffer" not in state: + state["momentum_buffer"] = torch.zeros_like(grad, memory_format=torch.preserve_format) + buf = state["momentum_buffer"] + buf.mul_(momentum).add_(grad) + return buf.mul(momentum).add_(grad) + + def _orthogonalize(self, matrix: Tensor, group: Dict) -> Tensor: + return polar_express(matrix, group["ns_steps"]) + + @staticmethod + def _shape_scale(shape: torch.Size, group: Dict) -> float: + if group["variant"] == "orscale_lm": + return MOONLIGHT_RMS_CONSTANT * math.sqrt(max(shape)) + return 1.0 + + def _apply_update(self, p: Tensor, update: Tensor, group: Dict) -> None: + scale, weight_decay = self._shape_scale(p.shape, group), group["weight_decay"] + weight_norm = p.norm().float() + step_norm = (weight_decay * p.float() + scale * update.float()).norm() + self._scaled_step(p, update, self._trust_ratio(p, weight_norm, step_norm, p.shape, group), scale, group) + + def _apply_shard_updates(self, shards: List[Tensor], updates: List[Tensor], group: Dict) -> None: + weight_decay = group["weight_decay"] + scales = [self._shape_scale(self._working_param(p).shape, group) for p in shards] + # the padding of a matrix's last slice is zero in the weights and in the update, and stays zero + squares = [] + for p, update, scale in zip(shards, updates, scales): + weight = p.float() + squares.append(weight.pow(2).sum()) + squares.append((weight_decay * weight + scale * update.float()).pow(2).sum()) + squares = torch.stack(squares).to(self._collective_device()) + dist.all_reduce(squares, group=self.dp_group) + norms = squares.sqrt().cpu().view(-1, 2) + for p, update, scale, (weight_norm, step_norm) in zip(shards, updates, scales, norms): + ratio = self._trust_ratio(p, weight_norm, step_norm, self._working_param(p).shape, group) + self._scaled_step(p, update, ratio, scale, group) + + def _trust_ratio(self, p: Tensor, weight_norm: Tensor, step_norm: Tensor, shape: torch.Size, group: Dict) -> float: + """The clipped trust ratio from the float32 norms of the whole matrix and of ``wd * W + s * Q``.""" + eps = group["eps"] + if group["variant"] == "orscale_lm": + state = self.state[p] + if "c_denom" not in state: + c_denom = group["c_denom"] + if c_denom is None: + weight_init, step_init = weight_norm.item(), step_norm.item() + if weight_init < eps or step_init < eps: + # a layer initialized to (almost) zero: fall back to Moonlight's scale instead of a runaway c + c_denom = MOONLIGHT_RMS_CONSTANT * math.sqrt(max(shape)) + else: + c_denom = weight_init / step_init + # a float, not a 0-dim tensor: LowLevelZeRO's checkpoints treat every tensor in the state as sharded + state["c_denom"] = float(c_denom) + step_norm = state["c_denom"] * step_norm + ratio = (weight_norm / (step_norm + eps)).item() + return max(group["r_min"], min(group["r_max"], ratio)) + + @staticmethod + def _scaled_step(p: Tensor, update: Tensor, ratio: float, scale: float, group: Dict) -> None: + # elementwise, so the same for a matrix and a shard + lr, weight_decay = group["lr"], group["weight_decay"] + if weight_decay != 0.0: + p.add_(p, alpha=-lr * ratio * weight_decay) + p.add_(update.to(p.dtype), alpha=-lr * ratio * scale) diff --git a/tests/test_optimizer/orscale_reference.py b/tests/test_optimizer/orscale_reference.py new file mode 100644 index 000000000000..acd875a42de0 --- /dev/null +++ b/tests/test_optimizer/orscale_reference.py @@ -0,0 +1,233 @@ +"""The official OrScale, as the reference of the alignment tests of ``colossalai.nn.optimizer.OrScale``. + +An excerpt of https://github.com/NUS-HPC-AI-Lab/OrScale at commit a45fc2ba19d072ca73dbbce88822631d5c1dd1c7 +(``orscale/optim/newton_schulz.py`` and ``orscale/optim/orscale_optimizer.py``), cut down to the two variants of the +paper: the ablation variants, the diagnostics, ``torch.compile`` and the batched orthogonalization are removed and the +branches on the variant simplified accordingly; the arithmetic is the official statements. When it was added, the +excerpt gave the same bits as the full official code run eagerly (``ORSCALE_DISABLE_TORCH_COMPILE=1``) on CPU and +CUDA, for float32, float64 and bfloat16 parameters. ``polar_express_float64`` is not part of the official code: it is +its iteration without the bfloat16 cast, the ground truth for the orthogonalization alone. + +MIT License + +Copyright (c) 2026 NUS-HPC-AI-Lab + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import math +from typing import Optional + +import torch +from torch import Tensor +from torch.optim.optimizer import Optimizer + +MOONLIGHT_RMS_CONSTANT = 0.2 + +POLAR_EXPRESS_COEFFS = [ + (8.156554524902461, -22.48329292557795, 15.878769915207462), + (4.042929935166739, -2.808917465908714, 0.5000178451051316), + (3.8916678022926607, -2.772484153217685, 0.5060648178503393), + (3.285753657755655, -2.3681294933425376, 0.46449024233003106), + (2.3465413258596377, -1.7097828382687081, 0.42323551169305323), +] + + +def orthogonalize(M: Tensor, num_iters: int = 5) -> Tensor: + transposed = M.size(-2) > M.size(-1) + X = M.bfloat16() + if transposed: + X = X.mT + + # Scale so spectral norm <= 1 (required for convergence) + X = X / (X.norm(dim=(-2, -1), keepdim=True) * 1.02 + 1e-6) + X = X.contiguous() + + # Allocate work buffers + A = torch.empty((*X.shape[:-1], X.size(-2)), device=X.device, dtype=X.dtype) + B = torch.empty_like(A) + C = torch.empty_like(X) + + for a, b, c in POLAR_EXPRESS_COEFFS[:num_iters]: + # A = X @ X^T (small square matrix) + torch.mm(X, X.mT, out=A) + # B = b*A + c*(A@A) + torch.addmm(A, A, A, beta=b, alpha=c, out=B) + # C = a*X + B @ X + torch.addmm(X, B, X, beta=a, out=C) + X, C = C, X + + if transposed: + X = X.mT + return X + + +def polar_express_float64(M: Tensor, num_iters: int = 5) -> Tensor: + transposed = M.size(-2) > M.size(-1) + X = M.double() + if transposed: + X = X.mT + + X = X / (X.norm(dim=(-2, -1), keepdim=True) * 1.02 + 1e-6) + + for a, b, c in POLAR_EXPRESS_COEFFS[:num_iters]: + A = X @ X.mT + B = b * A + c * (A @ A) + X = a * X + B @ X + + if transposed: + X = X.mT + return X + + +class OrScaleOptimizer(Optimizer): + def __init__( + self, + params, + lr: float = 0.02, + momentum: float = 0.95, + weight_decay: float = 0.0, + variant: str = "orscale_lm", + r_min: float = 0.5, + r_max: float = 1.5, + eps: float = 1e-6, + ns_iters: int = 5, + c_denom: Optional[float] = None, + ): + if lr < 0.0: + raise ValueError(f"Invalid learning rate: {lr}") + if momentum < 0.0 or momentum >= 1.0: + raise ValueError(f"Invalid momentum: {momentum}") + if c_denom is not None and c_denom <= 0.0: + raise ValueError(f"Invalid c_denom: {c_denom} (must be positive or None)") + if variant not in ("orscale", "orscale_lm"): + raise ValueError(f"this excerpt has the variants 'orscale' and 'orscale_lm', got {variant!r}") + + defaults = dict( + lr=lr, + momentum=momentum, + weight_decay=weight_decay, + variant=variant, + r_min=r_min, + r_max=r_max, + eps=eps, + ns_iters=ns_iters, + c_denom=c_denom, + ) + super().__init__(params, defaults) + + @torch.no_grad() + def step(self, closure=None): + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + lr = group["lr"] + mu = group["momentum"] + wd = group["weight_decay"] + variant = group["variant"] + r_min = group["r_min"] + r_max = group["r_max"] + eps = group["eps"] + ns_iters = group["ns_iters"] + c_denom_user = group.get("c_denom", None) + + use_shape_norm = variant == "orscale_lm" + use_grad_matrix_denom = variant == "orscale" + effective_alpha = 1.0 + + for p in group["params"]: + if p.grad is None: + continue + + grad = p.grad + if grad.ndim < 2: + continue + + orig_shape = p.shape + state = self.state[p] + + if len(state) == 0: + state["momentum_buffer"] = torch.zeros_like(p, dtype=torch.float32) + + buf = state["momentum_buffer"] + + # Flatten >=3D Conv weights to 2D [C_out, C_in*kH*kW] + g = grad.float().reshape(orig_shape[0], -1) + buf_view = buf.view(orig_shape[0], -1) + p_view = p.view(orig_shape[0], -1) + p_float = p_view.float() + m, n = p_view.shape + + # --- Momentum --- + # Accumulating momentum: M_t = mu * M_{t-1} + G_t + buf_view.mul_(mu).add_(g) + # Nesterov lookahead: M_hat = mu * M_t + G_t + m_hat = buf_view.mul(mu).add(g) + + # --- Orthogonalization --- + Q = orthogonalize(m_hat, num_iters=ns_iters) + + # --- Shape normalization --- + shape_scale = MOONLIGHT_RMS_CONSTANT * math.sqrt(max(m, n)) if use_shape_norm else 1.0 + + # --- Trust ratio --- + w_norm = p_view.norm().float() # ||W||_F (float32) + + if use_grad_matrix_denom: + w_stat = w_norm + # p_float is already float32; promote Q for an explicit + # float32 sum. shape_scale and wd are Python floats. + grad_matrix = wd * p_float + shape_scale * Q.float() + m_stat = grad_matrix.norm() # ||λW + s·Q||_F + else: + # Coupled denominator c_denom_ℓ * ||λW + s·Q||_F. + grad_matrix = wd * p_float + shape_scale * Q.float() + grad_matrix_norm = grad_matrix.norm() # ||λW + s·Q||_F + if "c_denom" not in state: + if c_denom_user is not None: + state["c_denom"] = float(c_denom_user) + else: + w_init = w_norm.item() + gm_init = grad_matrix_norm.item() + if w_init < eps or gm_init < eps: + # Fallback: pathological tiny init. Use the + # Moonlight shape constant so the optimizer + # degenerates to scaled Moonlight rather than + # NaNing or producing a runaway ratio. + state["c_denom"] = MOONLIGHT_RMS_CONSTANT * math.sqrt(max(m, n)) + else: + state["c_denom"] = w_init / gm_init + c_denom_layer = state["c_denom"] + w_stat = w_norm + m_stat = c_denom_layer * grad_matrix_norm # c_denom_ℓ · ||λW + s·Q||_F + + ratio_raw = (w_stat / (m_stat + eps)) ** effective_alpha + ratio_raw_val = ratio_raw.item() if isinstance(ratio_raw, Tensor) else float(ratio_raw) + ratio_clipped = max(r_min, min(r_max, ratio_raw_val)) + + # --- Update --- + q_update = Q.to(p.dtype) + if wd != 0.0: + p_view.add_(p_view, alpha=-lr * ratio_clipped * wd) + p_view.add_(q_update, alpha=-lr * ratio_clipped * shape_scale) + + return loss diff --git a/tests/test_optimizer/test_dist_muon.py b/tests/test_optimizer/test_dist_muon.py index 4b88001084c3..591b4ac75c5c 100644 --- a/tests/test_optimizer/test_dist_muon.py +++ b/tests/test_optimizer/test_dist_muon.py @@ -13,7 +13,7 @@ from colossalai.booster import Booster from colossalai.booster.plugin import GeminiPlugin, LowLevelZeroPlugin, TorchFSDPPlugin from colossalai.logging import disable_existing_loggers -from colossalai.nn.optimizer import Muon +from colossalai.nn.optimizer import Muon, OrScale from colossalai.nn.optimizer.muon import zeropower_via_newtonschulz from colossalai.testing import free_port, rerun_if_address_is_in_use, spawn from tests.kit.numerics import NumericsSpec, Trace, judge_equal, judge_relative_bound @@ -29,13 +29,16 @@ def requires_gpus(n): return pytest.mark.skipif(torch.cuda.device_count() < n, reason=f"needs {n} GPUs") -def muon_optimizer(params, cls=Muon): +def muon_optimizer(params, cls=Muon, **kwargs): params = list(params) groups = [ {"params": [p for p in params if p.ndim == 2], "use_muon": True}, {"params": [p for p in params if p.ndim != 2], "use_muon": False}, ] - return cls(groups, lr=LR, weight_decay=0.1) + return cls(groups, lr=LR, weight_decay=0.1, **kwargs) + + +ORSCALE = {"orscale": dict(variant="orscale"), "orscale_lm": dict(variant="orscale_lm", r_min=0.1, r_max=5.0)} class ShardLocalMuon(Muon): @@ -50,10 +53,37 @@ def _orthogonalize_shards(self, shards, inputs, group): ] +class ShardLocalTrustRatio(OrScale): + """A typical sharding bug: each rank takes the trust ratio from the norms of its own slice.""" + + def _apply_shard_updates(self, shards, updates, group): + for p, update in zip(shards, updates): + shape = self._working_param(p).shape + scale = self._shape_scale(shape, group) + step = group["weight_decay"] * p.float() + scale * update.float() + ratio = self._trust_ratio(p, p.norm().float(), step.norm(), shape, group) + self._scaled_step(p, update, ratio, scale, group) + + # --------------------------------------------------------------------------- -# Muon on hand-made ZeRO shards, over gloo: runs without GPUs +# Muon and OrScale on hand-made ZeRO shards, over gloo: runs without GPUs # --------------------------------------------------------------------------- +# name: (optimizer on the shards, optimizer on the whole matrices it must match, their arguments, bound). +# Muon's sharded step gathers the bf16 cast the whole-matrix step makes, so the results are the same bits (no bound). +# OrScale's trust ratio comes from norms summed slice by slice, which round differently from the norm of the whole +# matrix: its weights agree within the float32 relative bound. The gradients are given, so nothing amplifies that. +SHARD_CASES = { + "muon": (Muon, Muon, {}, None), + "orscale": (OrScale, OrScale, ORSCALE["orscale"], torch.float32), + "orscale_lm": (OrScale, OrScale, ORSCALE["orscale_lm"], torch.float32), +} +BUGGY_SHARD_CASES = { + "muon": (ShardLocalMuon, Muon, {}, torch.bfloat16), + "orscale": (ShardLocalTrustRatio, OrScale, ORSCALE["orscale"], torch.float32), + "orscale_lm": (ShardLocalTrustRatio, OrScale, ORSCALE["orscale_lm"], torch.float32), +} + def shard_of(t, world_size, rank): """``rank``'s slice of ``t``, padded and split the way LowLevelZeRO does.""" @@ -62,13 +92,14 @@ def shard_of(t, world_size, rank): return flat.view(world_size, -1)[rank] -def run_shards(rank, world_size, port, optimizer_cls): +def run_shards(rank, world_size, port, test_cls, ref_cls, kwargs, bound, buggy): # colossalai.launch would pick NCCL on a GPU machine; these shards live on the CPU os.environ.update(MASTER_ADDR="127.0.0.1", MASTER_PORT=str(port)) dist.init_process_group("gloo", rank=rank, world_size=world_size) gen = torch.Generator().manual_seed(0) shapes = {"tall": (HIDDEN, DIM_OUT), "wide": (DIM_IN, HIDDEN), "square": (48, 48), "bias": (HIDDEN,)} - init = {n: torch.randn(s, generator=gen) for n, s in shapes.items()} + # weights about as large as their orthogonalized updates, so that OrScale's trust ratios are near 1, not clipped + init = {n: 0.125 * torch.randn(s, generator=gen) for n, s in shapes.items()} grads = [{n: torch.randn(s, generator=gen) for n, s in shapes.items()} for _ in range(STEPS)] matrices = [n for n, s in shapes.items() if len(s) == 2] @@ -79,11 +110,11 @@ def groups(params): ] whole = {n: nn.Parameter(t.clone()) for n, t in init.items()} - ref = Muon(groups(whole), lr=LR, weight_decay=0.1) + ref = ref_cls(groups(whole), lr=LR, weight_decay=0.1, **kwargs) # as with LowLevelZeRO: built on the whole parameters, then handed the shards and told what they belong to working = {n: nn.Parameter(t.clone()) for n, t in init.items()} shards = {n: nn.Parameter(shard_of(t, world_size, rank).clone()) for n, t in init.items()} - test = optimizer_cls(groups(working), lr=LR, weight_decay=0.1) + test = test_cls(groups(working), lr=LR, weight_decay=0.1, **kwargs) for group, sharded in zip(test.param_groups, groups(shards)): group["params"] = sharded["params"] test.setup_distributed(None, None, {id(shards[n]): working[n] for n in shapes}, None, True) @@ -100,27 +131,45 @@ def gathered(t, shape): dist.all_gather(parts, t.detach().contiguous()) return torch.cat(parts)[: torch.Size(shape).numel()].view(shape) - # updates, not weights: next to unit-variance weights a wrong update would hide inside any relative bound - result = {f"update/{n}": gathered(shards[n], shapes[n]) - init[n] for n in shapes} - result.update({f"momentum/{n}": gathered(test.state[shards[n]]["momentum_buffer"], shapes[n]) for n in matrices}) - expected = {f"update/{n}": whole[n].detach() - init[n] for n in shapes} - expected.update({f"momentum/{n}": ref.state[whole[n]]["momentum_buffer"] for n in matrices}) - if optimizer_cls is Muon: - # the gathered input is the bf16 cast the whole-matrix step makes, so the results are the same bits - judge_equal(result, expected).assert_pass() + final = {n: gathered(shards[n], shapes[n]) for n in shapes} + # elementwise on the shards, so the same bits whatever the optimizer does with whole matrices + exact = {f"momentum/{n}": gathered(test.state[shards[n]]["momentum_buffer"], shapes[n]) for n in matrices} + exact["weight/bias"] = final["bias"] + exact_expected = {f"momentum/{n}": ref.state[whole[n]]["momentum_buffer"] for n in matrices} + exact_expected["weight/bias"] = whole["bias"].detach() + judge_equal(exact, exact_expected).assert_pass() + + if bound == torch.float32: + # The weights: the difference of two float32 weights does not resolve a step to 1e-6 of itself, while a wrong + # trust ratio moves the weights by far more than 1e-6 of themselves. + result = {f"matrix/{n}": final[n] for n in matrices} + expected = {f"matrix/{n}": whole[n].detach() for n in matrices} + if "c_denom" in ref.state[whole[matrices[0]]]: + result.update({f"c_denom/{n}": torch.tensor(test.state[shards[n]]["c_denom"]) for n in matrices}) + expected.update({f"c_denom/{n}": torch.tensor(ref.state[whole[n]]["c_denom"]) for n in matrices}) + report = judge_relative_bound(result, expected, bound) else: - report = judge_relative_bound(result, expected, torch.bfloat16) - assert {e.name for e in report.failures} >= {f"update/{n}" for n in matrices}, report.format() + # The updates: next to the weights a wrong update would hide inside the bf16 bound. + result = {f"matrix/{n}": final[n] - init[n] for n in matrices} + expected = {f"matrix/{n}": whole[n].detach() - init[n] for n in matrices} + report = judge_equal(result, expected) if bound is None else judge_relative_bound(result, expected, bound) + if buggy: + assert {e.name for e in report.failures} >= {f"matrix/{n}" for n in matrices}, report.format() + else: + report.assert_pass() dist.destroy_process_group() @pytest.mark.parametrize("world_size", [2, 3]) -def test_shards_match_whole_matrices(world_size): - torch.multiprocessing.spawn(run_shards, args=(world_size, free_port(), Muon), nprocs=world_size) +@pytest.mark.parametrize("case", SHARD_CASES) +def test_shards_match_whole_matrices(case, world_size): + args = (world_size, free_port(), *SHARD_CASES[case], False) + torch.multiprocessing.spawn(run_shards, args=args, nprocs=world_size) -def test_shard_local_orthogonalization_detected(): - torch.multiprocessing.spawn(run_shards, args=(2, free_port(), ShardLocalMuon), nprocs=2) +@pytest.mark.parametrize("case", BUGGY_SHARD_CASES) +def test_shard_local_computation_detected(case): + torch.multiprocessing.spawn(run_shards, args=(2, free_port(), *BUGGY_SHARD_CASES[case], True), nprocs=2) def run_setup_checks(rank, world_size, port): @@ -170,25 +219,26 @@ def _identity(output): ENTRY = (MLP, _data, _identity, None, SimpleNamespace(numerics=NumericsSpec())) -def run_zero(rank, world_size, port, plugin_fn, optimizer_fn, path): +def run_zero(rank, world_size, port, plugin_fn, optimizer_fn, steps, path): disable_existing_loggers() colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) # every rank trains on the same batch: averaging N equal gradients is exact for N = 2, so only sharding differs - trace = run_steps(prepare(ENTRY, STEPS), plugin_fn(), 0, STEPS, optimizer_fn=optimizer_fn) + trace = run_steps(prepare(ENTRY, steps), plugin_fn(), 0, steps, optimizer_fn=optimizer_fn) if rank == 0: trace.save(path) dist.destroy_process_group() -def zero_trace(plugin_fn, world_size, optimizer_fn=muon_optimizer): +def zero_trace(plugin_fn, world_size, optimizer_fn=muon_optimizer, steps=STEPS): with tempfile.TemporaryDirectory() as directory: path = os.path.join(directory, "trace.safetensors") - spawn(run_zero, world_size, plugin_fn=plugin_fn, optimizer_fn=optimizer_fn, path=path) + spawn(run_zero, world_size, plugin_fn=plugin_fn, optimizer_fn=optimizer_fn, steps=steps, path=path) return Trace.load(path) def comparable(test, world_size, ref): - """Losses, weights and the optimizer's state (float32 master weights, momentum, AdamW moments) of both runs. + """Losses, weights and the optimizer's state (float32 master weights, momentum, AdamW moments, OrScale-LM's + calibrated constants) of both runs. The optimizer's tensors are recorded per rank in ZeRO's flat layout; the slices of ``test`` are joined and the padding cut to the length of ``ref``'s, which one rank holds unpadded.""" @@ -225,6 +275,11 @@ def updates(state, precision): return out +def optimizer_state(state): + """What the optimizer updates and keeps: the float32 master weights, the momentum, OrScale-LM's constants.""" + return {key: value for key, value in state.items() if key.startswith("optim/")} + + ZERO_CONFIGS = [(1, "fp32", False), (2, "fp32", False), (1, "bf16", False), (2, "bf16", False), (2, "bf16", True)] @@ -238,16 +293,41 @@ def test_zero_two_ranks_equal_one(stage, precision, cpu_offload): judge_equal(test, ref).assert_pass() +ZERO_OPTIMIZERS = { + "muon": muon_optimizer, + **{n: partial(muon_optimizer, cls=OrScale, **kw) for n, kw in ORSCALE.items()}, +} + + @pytest.mark.dist @requires_gpus(4) +@pytest.mark.parametrize("optimizer", ZERO_OPTIMIZERS) @rerun_if_address_is_in_use() -def test_zero_four_ranks_within_bound(): +def test_zero_four_ranks_within_bound(optimizer): # averaging 4 equal gradients rounds, so the runs differ in the last bits: Megatron-LM's relative bound plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="bf16") - test, ref = comparable(zero_trace(plugin_fn, 4), 4, zero_trace(plugin_fn, 1)) + optimizer_fn = ZERO_OPTIMIZERS[optimizer] + test, ref = comparable(zero_trace(plugin_fn, 4, optimizer_fn), 4, zero_trace(plugin_fn, 1, optimizer_fn)) judge_relative_bound(updates(test, "bf16"), updates(ref, "bf16"), torch.bfloat16).assert_pass() +@pytest.mark.dist +@requires_gpus(2) +@pytest.mark.parametrize("variant", ORSCALE) +@pytest.mark.parametrize("stage, precision, cpu_offload", [(1, "fp32", False), (2, "bf16", False), (2, "bf16", True)]) +@rerun_if_address_is_in_use() +def test_zero_orscale_two_ranks_match_one(variant, stage, precision, cpu_offload): + # One step: the gradients, the momentum and the orthogonalized updates of the two runs are the same bits, and the + # trust ratios differ by float32 rounding, so the float32 bound holds for the master weights (see run_shards). + # Over more steps the bf16 orthogonalization turns that rounding into bf16 rounding, which only the bf16 bound + # covers (test_zero_four_ranks_within_bound). + plugin_fn = partial(LowLevelZeroPlugin, stage=stage, precision=precision, cpu_offload=cpu_offload) + optimizer_fn = ZERO_OPTIMIZERS[variant] + two, one = zero_trace(plugin_fn, 2, optimizer_fn, steps=1), zero_trace(plugin_fn, 1, optimizer_fn, steps=1) + test, ref = comparable(two, 2, one) + judge_relative_bound(optimizer_state(test), optimizer_state(ref), torch.float32).assert_pass() + + @pytest.mark.dist @requires_gpus(2) @rerun_if_address_is_in_use() @@ -262,30 +342,49 @@ def test_zero_shard_local_orthogonalization_detected(): @pytest.mark.dist @requires_gpus(2) @rerun_if_address_is_in_use() -def test_zero_resume_is_exact(): +def test_zero_shard_local_trust_ratio_detected(): + plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="bf16") + shard_local = partial(muon_optimizer, cls=ShardLocalTrustRatio, **ORSCALE["orscale"]) + two = zero_trace(plugin_fn, 2, shard_local, steps=1) + test, ref = comparable(two, 2, zero_trace(plugin_fn, 1, ZERO_OPTIMIZERS["orscale"], steps=1)) + report = judge_relative_bound(optimizer_state(test), optimizer_state(ref), torch.float32) + # the master weights of the matrices and nothing else; after one warm-up step the smaller matrix is only a few + # times outside the bound, so it is not required to fail + failed = {e.name for e in report.failures} + assert "optim/0.0/param" in failed and failed <= {"optim/0.0/param", "optim/0.1/param"}, report.format() + + +@pytest.mark.dist +@requires_gpus(2) +@pytest.mark.parametrize("optimizer, shard", [("muon", True), ("orscale_lm", True), ("orscale_lm", False)]) +@rerun_if_address_is_in_use() +def test_zero_resume_is_exact(optimizer, shard): # fp32: in bf16 no optimizer resumes exactly, as LowLevelZero rebuilds the float32 master weights from the bf16 - # ones on load (tests/kit/numerics/README.md, "Resuming from a checkpoint") + # ones on load (tests/kit/numerics/README.md, "Resuming from a checkpoint"). OrScale-LM keeps a float per matrix + # in its state, which both checkpoint layouts have to carry. plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="fp32") - resume.check(ENTRY, plugin_fn, world_size=2, optimizer_fn=muon_optimizer).assert_pass() + optimizer_fn = ZERO_OPTIMIZERS[optimizer] + resume.check(ENTRY, plugin_fn, world_size=2, optimizer_fn=optimizer_fn, shard_optimizer=shard).assert_pass() def run_plugins(rank, world_size, port): disable_existing_loggers() colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) - for plugin in (GeminiPlugin(), TorchFSDPPlugin()): - with pytest.raises(NotImplementedError, match="does not support Muon"): - model = MLP().cuda() - Booster(plugin=plugin).boost(model, muon_optimizer(model.parameters())) - model = MLP().cuda() - muon = muon_optimizer(model.parameters()) - _, optimizer, *_ = Booster(plugin=LowLevelZeroPlugin(stage=1)).boost(model, muon) - # set up in place, not replaced: an LR scheduler built on `muon` before boost keeps driving it - assert optimizer.optim is muon and muon.shard_to_working_param + for cls in (Muon, OrScale): + for plugin in (GeminiPlugin(), TorchFSDPPlugin()): + with pytest.raises(NotImplementedError, match=f"does not support {cls.__name__}"): + model = MLP().cuda() + Booster(plugin=plugin).boost(model, muon_optimizer(model.parameters(), cls)) + model = MLP().cuda() + inner = muon_optimizer(model.parameters(), cls) + _, optimizer, *_ = Booster(plugin=LowLevelZeroPlugin(stage=1)).boost(model, inner) + # set up in place, not replaced: an LR scheduler built on `inner` before boost keeps driving it + assert optimizer.optim is inner and inner.shard_to_working_param dist.destroy_process_group() @pytest.mark.dist @requires_gpus(1) @rerun_if_address_is_in_use() -def test_plugins_with_muon(): +def test_plugins_with_muon_and_orscale(): spawn(run_plugins, 1) diff --git a/tests/test_optimizer/test_orscale.py b/tests/test_optimizer/test_orscale.py new file mode 100644 index 000000000000..7b113432a9d5 --- /dev/null +++ b/tests/test_optimizer/test_orscale.py @@ -0,0 +1,358 @@ +import copy +import json +import math + +import pytest +import torch +import torch.nn as nn + +import colossalai +from colossalai.booster import Booster +from colossalai.booster.plugin import TorchDDPPlugin +from colossalai.logging import disable_existing_loggers +from colossalai.nn.optimizer import OrScale, muon_param_groups +from colossalai.nn.optimizer.muon import zeropower_via_newtonschulz +from colossalai.nn.optimizer.orscale import polar_express +from colossalai.testing import rerun_if_address_is_in_use, spawn +from tests.kit.numerics import assert_aligned, judge, judge_relative_bound +from tests.kit.numerics.scenarios.independent_impl import check_optimizer +from tests.test_optimizer.orscale_reference import OrScaleOptimizer, orthogonalize, polar_express_float64 + +DEVICES = ["cpu"] + (["cuda"] if torch.cuda.is_available() else []) +STEPS, LR, ADAMW_LR = 3, 0.02, 2e-3 + +# name: (shape, std of the initial weights). With these stds the trust ratio of "orscale", about ||W|| / ||Q||, starts +# at 0, near 0.05, 0.8, 1.2 and 3.5: clipped from below (twice), inside [0.5, 1.5] (twice) and clipped from above. +MATRICES = { + "zero": ((24, 16), 0.0), + "below": ((16, 24), 0.01), + "tall": ((64, 32), 0.1), + "wide": ((32, 64), 0.15), + "above": ((48, 48), 0.5), +} +OTHERS = {"bias": (64,), "gain": (32,)} # go to the AdamW branch + +# check_optimizer judges the weights, and judge's default tol (1e-4 of their RMS) would let through an error of a few +# percent in a step that changes them by a fraction of a percent. +WEIGHT_TOL = 1e-7 + +CONFIGS = { + "orscale": dict(variant="orscale", weight_decay=0.1), + "orscale-no-decay": dict(variant="orscale"), + "orscale-lm": dict(variant="orscale_lm", weight_decay=0.1, r_min=0.1, r_max=5.0), + "orscale-lm-no-decay": dict(variant="orscale_lm", r_min=0.1, r_max=5.0), + "orscale-lm-given-c": dict(variant="orscale_lm", weight_decay=0.1, r_min=0.1, r_max=5.0, c_denom=0.05), +} + + +def make_params(device="cpu", others=False): + gen = torch.Generator().manual_seed(0) + params = {n: std * torch.randn(shape, generator=gen, dtype=torch.float64) for n, (shape, std) in MATRICES.items()} + if others: + params.update({n: torch.randn(shape, generator=gen, dtype=torch.float64) for n, shape in OTHERS.items()}) + return {n: p.to(device) for n, p in params.items()} + + +def ours(cls=OrScale, **kwargs): + return lambda params: cls(params, **{"lr": LR, **kwargs}) + + +def official(**kwargs): + return lambda params: OrScaleOptimizer(params, lr=LR, **kwargs) + + +def check(test_fn, config, device="cpu"): + return check_optimizer(test_fn, official(**config), make_params(device), torch.float32, steps=STEPS, tol=WEIGHT_TOL) + + +# --------------------------------------------------------------------------- +# Polar Express +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("device", DEVICES) +@pytest.mark.parametrize("shape", [(64, 32), (32, 64), (48, 48)]) +def test_polar_express_matches_official(device, shape): + grad = torch.randn(shape, generator=torch.Generator().manual_seed(1), dtype=torch.float64).to(device) + truth = polar_express_float64(grad) + ref = orthogonalize(grad.float()) + assert_aligned(polar_express(grad.float()), ref, truth) + # Muon's Newton-Schulz coefficients leave the singular values in about [0.5, 1.5]: not the same function + assert not judge(zeropower_via_newtonschulz(grad.float()), ref, truth).passed + + +@pytest.mark.parametrize("shape", [(64, 32), (32, 64), (48, 48)]) +def test_polar_express_approximates_the_polar_factor(shape): + # independent of the official code: guards the coefficient table itself + grad = torch.randn(shape, generator=torch.Generator().manual_seed(1), dtype=torch.float64) + u, _, vh = torch.linalg.svd(grad, full_matrices=False) + polar = u @ vh + error = (polar_express(grad.float()).double() - polar).norm() / polar.norm() + assert error < 0.1, f"relative error {error:.3f} against U V^T" + + +# --------------------------------------------------------------------------- +# the update rule against the official implementation +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("device", DEVICES) +@pytest.mark.parametrize("config", CONFIGS.values(), ids=CONFIGS.keys()) +def test_orscale_matches_official(device, config): + check(ours(**config), config, device).assert_pass() + + +@pytest.mark.parametrize( + "config, wrong", + [ + ("orscale", {"lr": LR * 1.01}), + ("orscale", {"momentum": 0.94}), + ("orscale", {"weight_decay": 0.09}), + ("orscale", {"r_min": 0.6}), + ("orscale", {"r_max": 1.4}), + ("orscale", {"eps": 1e-3}), + ("orscale", {"ns_steps": 4}), + ("orscale", {"variant": "orscale_lm"}), + ("orscale-lm", {"weight_decay": 0.09}), + ("orscale-lm", {"c_denom": 0.05}), + ("orscale-lm-given-c", {"c_denom": 0.051}), + ], +) +def test_alignment_check_catches_wrong_hyperparameters(config, wrong): + config = CONFIGS[config] + assert not check(ours(**{**config, **wrong}), config).passed + + +class DecoupledDecay(OrScale): + """Weight decay outside the trust ratio, as Muon applies it.""" + + @staticmethod + def _scaled_step(p, update, ratio, scale, group): + lr = group["lr"] + p.mul_(1 - lr * group["weight_decay"]) + p.add_(update.to(p.dtype), alpha=-lr * ratio * scale) + + +class AveragedMomentum(OrScale): + """Muon's exponential moving average: the same direction, but its bf16 cast before Polar Express rounds differently + from the accumulated momentum's, which already exceeds the float32 baseline error.""" + + def _momentum_update(self, p, group): + grad, momentum = p.grad.float(), group["momentum"] + buf = self.state[p].setdefault("momentum_buffer", torch.zeros_like(grad)) + buf.lerp_(grad, 1 - momentum) + return grad.lerp(buf, momentum) + + +@pytest.mark.parametrize("cls", [DecoupledDecay, AveragedMomentum]) +@pytest.mark.parametrize("config", ["orscale", "orscale-lm"]) +def test_alignment_check_catches_other_update_rules(cls, config): + config = CONFIGS[config] + assert not check(ours(cls, **config), config).passed + + +def grouped(params, cls, **kwargs): + params = list(params) + matrices, others = [p for p in params if p.ndim == 2], [p for p in params if p.ndim != 2] + return cls(matrices, others, **kwargs) + + +class OfficialPair: + """The official setup: OrScale for the matrices and torch's AdamW for everything else, stepped together.""" + + def __init__(self, matrices, others, **kwargs): + self.optimizers = [ + OrScaleOptimizer(matrices, lr=LR, **kwargs), + torch.optim.AdamW(others, lr=ADAMW_LR, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.01, foreach=False), + ] + + def step(self): + for optimizer in self.optimizers: + optimizer.step() + + def zero_grad(self, set_to_none=True): + for optimizer in self.optimizers: + optimizer.zero_grad(set_to_none=set_to_none) + + +def one_optimizer(matrices, others, **kwargs): + groups = [ + {"params": matrices, "use_muon": True}, + {"params": others, "use_muon": False, "lr": ADAMW_LR, "weight_decay": 0.01}, + ] + return OrScale(groups, lr=LR, betas=(0.9, 0.999), **kwargs) + + +@pytest.mark.parametrize("device", DEVICES) +@pytest.mark.parametrize("config", ["orscale", "orscale-lm"]) +def test_orscale_with_adamw_matches_official_pair(device, config): + config = CONFIGS[config] + check_optimizer( + lambda params: grouped(params, one_optimizer, **config), + lambda params: grouped(params, OfficialPair, **config), + make_params(device, others=True), + torch.float32, + steps=STEPS, + tol=WEIGHT_TOL, + ).assert_pass() + + +def test_orscale_lm_first_step_has_trust_ratio_one(): + # the calibration makes the first step of every layer Moonlight's, whatever the scale of its weights + params = {n: nn.Parameter(p.float()) for n, p in make_params().items() if n != "zero"} + start = {n: p.detach().clone() for n, p in params.items()} + optimizer = OrScale(params.values(), lr=LR, variant="orscale_lm", r_min=0.1, r_max=5.0) + for p in params.values(): + p.grad = torch.randn_like(p) + optimizer.step() + for name, p in params.items(): + # momentum starts at 0, so the Nesterov direction is (1 + momentum) * grad + direction = polar_express(p.grad * 1.95).float() + expected = start[name] - LR * 0.2 * math.sqrt(max(p.shape)) * direction + torch.testing.assert_close(p.detach(), expected, rtol=1e-5, atol=1e-7) + assert isinstance(optimizer.state[p]["c_denom"], float) + + +# --------------------------------------------------------------------------- +# parameter grouping, state and error handling +# --------------------------------------------------------------------------- + + +class TinyLM(nn.Module): + def __init__(self): + super().__init__() + self.embed = nn.Embedding(50, 16) + self.proj = nn.Linear(16, 16) + self.norm = nn.LayerNorm(16) + self.lm_head = nn.Linear(16, 50, bias=False) + + def get_output_embeddings(self): + return self.lm_head + + +def test_orscale_takes_muon_param_groups(): + model = TinyLM() + optimizer = OrScale(muon_param_groups(model), variant="orscale_lm") + groups = {g["use_muon"]: g for g in optimizer.param_groups} + assert [id(p) for p in groups[True]["params"]] == [id(model.proj.weight)] + assert groups[True]["variant"] == "orscale_lm" and "variant" not in groups[False] + start = {n: p.detach().clone() for n, p in model.named_parameters()} + for p in model.parameters(): + p.grad = torch.randn_like(p) + optimizer.step() + assert all(not torch.equal(p, start[n]) for n, p in model.named_parameters()) + + +def test_orscale_state_dict_round_trip(): + torch.manual_seed(0) + model = TinyLM() + kwargs = dict(lr=LR, weight_decay=0.1, variant="orscale_lm", r_min=0.1, r_max=5.0) + optimizer = OrScale(muon_param_groups(model), **kwargs) + grads = [[torch.randn_like(p) for p in model.parameters()] for _ in range(3)] + + def step(module, opt, step_grads): + for p, g in zip(module.parameters(), step_grads): + p.grad = g.clone() + opt.step() + + step(model, optimizer, grads[0]) + step(model, optimizer, grads[1]) + state = optimizer.state_dict() + # Hyperparameters and the calibrated constants stay plain data so JSON-based checkpoint formats can store them. + json.dumps([{k: v for k, v in g.items() if k != "params"} for g in state["param_groups"]]) + assert isinstance(optimizer.state[model.proj.weight]["c_denom"], float) + + resumed_model = copy.deepcopy(model) + resumed = OrScale(muon_param_groups(resumed_model), **kwargs) + resumed.load_state_dict(copy.deepcopy(state)) + step(model, optimizer, grads[2]) + step(resumed_model, resumed, grads[2]) + for p, q in zip(model.parameters(), resumed_model.parameters()): + torch.testing.assert_close(q, p, rtol=0, atol=0) + + +@pytest.mark.parametrize( + "kwargs, match", + [ + ({"variant": "muscale"}, "variant"), + ({"momentum": 1.0}, "momentum"), + ({"r_min": 2.0}, "r_min"), + ({"ns_steps": 6}, "ns_steps"), + ({"c_denom": 0.0}, "c_denom"), + ], +) +def test_orscale_rejects_invalid_arguments(kwargs, match): + with pytest.raises(ValueError, match=match): + OrScale([nn.Parameter(torch.randn(4, 4))], **kwargs) + optimizer = OrScale([nn.Parameter(torch.randn(4, 4))]) + with pytest.raises(ValueError, match=match): + optimizer.add_param_group({"params": [nn.Parameter(torch.randn(4, 4))], **kwargs}) + assert len(optimizer.param_groups) == 1 + + +def test_orscale_rejects_non_matrix_params(): + with pytest.raises(ValueError, match="OrScale only supports 2D"): + OrScale(nn.Linear(4, 4).parameters()) + + +# --------------------------------------------------------------------------- +# DDP: two ranks on half batches must match one device on the full batch +# --------------------------------------------------------------------------- + +DIM, GLOBAL_BATCH = 32, 8 + + +def make_mlp(): + torch.manual_seed(0) + return nn.Sequential(nn.Linear(DIM, 64), nn.GELU(), nn.LayerNorm(64), nn.Linear(64, DIM)) + + +def train(model, batches, variant, booster=None, rank=0, world_size=1): + device = torch.device("cuda", torch.cuda.current_device()) + model = model.to(device) + start = {n: p.detach().double().clone() for n, p in model.named_parameters()} + groups = muon_param_groups(model) + groups[1]["lr"] = ADAMW_LR + opt = OrScale(groups, lr=LR, weight_decay=0.1, variant=variant) + if booster is not None: + model, opt, *_ = booster.boost(model, opt) + local = GLOBAL_BATCH // world_size + for batch in batches: + x = batch[rank * local : (rank + 1) * local].to(device) + loss = model(x).pow(2).mean() + if booster is not None: + booster.backward(loss, opt) + else: + loss.backward() + opt.step() + opt.zero_grad() + module = model.unwrap() if hasattr(model, "unwrap") else model + return {n: (p.detach().double() - start[n]).cpu() for n, p in module.named_parameters()} + + +def run_ddp(rank, world_size, port, variant): + disable_existing_loggers() + colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port, backend="nccl") + batches = torch.randn(STEPS, GLOBAL_BATCH, DIM, generator=torch.Generator().manual_seed(2)) + booster = Booster(plugin=TorchDDPPlugin()) + + test = train(make_mlp(), batches, variant, booster, rank, world_size) + ref = train(make_mlp(), batches, variant) + # Same code at the same precision, so there is no baseline error to measure: the tool's relative bound applies, + # at the bf16 of the orthogonalization, which turns float32 rounding of the averaged gradient into bf16 rounding. + judge_relative_bound(test, ref, torch.bfloat16).assert_pass() + # Each rank stepping on its own half batch (no gradient all-reduce) must be caught. + unsynced = train(make_mlp(), batches, variant, rank=rank, world_size=world_size) + assert not judge_relative_bound(unsynced, ref, torch.bfloat16).passed + + +@pytest.mark.dist +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="needs 2 GPUs") +@pytest.mark.parametrize("variant", ["orscale", "orscale_lm"]) +@rerun_if_address_is_in_use() +def test_orscale_ddp(variant): + spawn(run_ddp, nprocs=2, variant=variant) + + +if __name__ == "__main__": + test_orscale_ddp("orscale")