From 569cbdfa2432e1207587cd3ed4ea037a2507a261 Mon Sep 17 00:00:00 2001 From: yuxuan-lou Date: Wed, 18 Nov 2026 18:36:59 +0800 Subject: [PATCH] [optimizer] add OrScale and OrScale-LM Add colossalai.nn.optimizer.OrScale: Muon's orthogonalized update scaled by a layer-wise trust ratio, with the weight decay coupled into the scaled step (arXiv:2605.07815). It subclasses Muon, so it has the same built-in AdamW branch for non-matrix parameters and runs under TorchDDP and LowLevelZeRO. The arithmetic follows the official implementation (NUS-HPC-AI-Lab/OrScale @ a45fc2b) statement by statement: accumulated float32 momentum, bf16 Polar Express orthogonalization, float32 norms. Run eagerly, the official code and this one give the same bits on CPU and CUDA. The official code compiles its orthogonalization with torch.compile on CUDA, which alone moves the result by 1-3%, as much as bf16 moves it from a float64 iteration. - orscale.py: OrScale with variant "orscale" or "orscale_lm", and polar_express. OrScale-LM's per-layer constant is calibrated at the first step and kept in the state as a float, which LowLevelZeRO's checkpoints carry. Under ZeRO each rank sums the squares of its slice and one all-reduce per bucket gives the two norms of the trust ratio, so a sharded step equals the unsharded one up to float32 rounding, not bit for bit. The ablation variants and the flattening of convolution weights of the official code are left out. - muon.py: the orthogonalization and the application of the update are methods a subclass overrides; no change in behavior. - gemini_plugin.py, torch_fsdp_plugin.py: the error names the optimizer class. - orscale_reference.py: excerpt of the official code (MIT) as the reference of the tests. - test_orscale.py: Polar Express and 3 steps of both variants against the reference under check_optimizer, with clipped and unclipped trust ratios and a zero-initialized layer; one optimizer against the official pair (OrScale + AdamW); wrong hyperparameters, decoupled weight decay and Muon's averaged momentum must fail. Grouping, state_dict round trip, error cases, 2-GPU TorchDDP. - test_dist_muon.py: OrScale on hand-made shards for 2 and 3 ranks and under LowLevelZeRO (2 ranks against 1 after one step within the float32 bound, 4 ranks within the bf16 bound, resume bit for bit with sharded and unsharded optimizer checkpoints). A trust ratio taken from the local slice must fail. - run_cpu_unit_tests.sh: run test_orscale.py in the CPU job. Co-Authored-By: Claude Fable 5.2 --- .github/scripts/ci/run_cpu_unit_tests.sh | 1 + colossalai/booster/plugin/gemini_plugin.py | 4 +- .../booster/plugin/torch_fsdp_plugin.py | 4 +- colossalai/nn/optimizer/__init__.py | 2 + colossalai/nn/optimizer/muon.py | 63 +-- colossalai/nn/optimizer/orscale.py | 219 +++++++++++ tests/test_optimizer/orscale_reference.py | 233 ++++++++++++ tests/test_optimizer/test_dist_muon.py | 183 +++++++-- tests/test_optimizer/test_orscale.py | 358 ++++++++++++++++++ 9 files changed, 998 insertions(+), 69 deletions(-) create mode 100644 colossalai/nn/optimizer/orscale.py create mode 100644 tests/test_optimizer/orscale_reference.py create mode 100644 tests/test_optimizer/test_orscale.py diff --git a/.github/scripts/ci/run_cpu_unit_tests.sh b/.github/scripts/ci/run_cpu_unit_tests.sh index 68bbeaed221b..e981c7f33183 100644 --- a/.github/scripts/ci/run_cpu_unit_tests.sh +++ b/.github/scripts/ci/run_cpu_unit_tests.sh @@ -22,6 +22,7 @@ TESTS=( tests/test_optimizer/test_dist_muon.py tests/test_optimizer/test_lr_scheduler.py tests/test_optimizer/test_muon.py + tests/test_optimizer/test_orscale.py tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py diff --git a/colossalai/booster/plugin/gemini_plugin.py b/colossalai/booster/plugin/gemini_plugin.py index 7a189e17f494..a4a6bf610e05 100644 --- a/colossalai/booster/plugin/gemini_plugin.py +++ b/colossalai/booster/plugin/gemini_plugin.py @@ -668,8 +668,8 @@ def configure( ) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]: if isinstance(optimizer, Muon): raise NotImplementedError( - "GeminiPlugin does not support Muon: its chunks split matrices at arbitrary boundaries. " - "Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." + f"GeminiPlugin does not support {type(optimizer).__name__}: its chunks split matrices at arbitrary " + "boundaries. Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." ) params_info = get_param_info(optimizer) if not isinstance(model, ModelWrapper): diff --git a/colossalai/booster/plugin/torch_fsdp_plugin.py b/colossalai/booster/plugin/torch_fsdp_plugin.py index 5c9390539972..2e88bb3a5362 100644 --- a/colossalai/booster/plugin/torch_fsdp_plugin.py +++ b/colossalai/booster/plugin/torch_fsdp_plugin.py @@ -539,8 +539,8 @@ def configure( ) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]: if isinstance(optimizer, Muon): raise NotImplementedError( - "TorchFSDPPlugin does not support Muon: FSDP hands the optimizer flattened parameters in one group. " - "Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." + f"TorchFSDPPlugin does not support {type(optimizer).__name__}: FSDP hands the optimizer flattened " + "parameters in one group. Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." ) # wrap the model with PyTorch FSDP fsdp_model = TorchFSDPModel(model, device_id=torch.cuda.current_device(), **self.fsdp_kwargs) diff --git a/colossalai/nn/optimizer/__init__.py b/colossalai/nn/optimizer/__init__.py index ebdd5dc3eb6d..453ba250ca8f 100644 --- a/colossalai/nn/optimizer/__init__.py +++ b/colossalai/nn/optimizer/__init__.py @@ -16,6 +16,7 @@ from .lamb import Lamb from .lars import Lars from .muon import Muon, muon_param_groups +from .orscale import OrScale from .adafactor import Adafactor # noqa @@ -39,6 +40,7 @@ "DistributedAdaFactor", "Muon", "muon_param_groups", + "OrScale", ] optim2DistOptim = { diff --git a/colossalai/nn/optimizer/muon.py b/colossalai/nn/optimizer/muon.py index bd6de205de84..1f77b1e2ff69 100644 --- a/colossalai/nn/optimizer/muon.py +++ b/colossalai/nn/optimizer/muon.py @@ -132,7 +132,7 @@ def __init__( raise ValueError(f"ns_steps must be in [1, 99], got {ns_steps}") if adjust_lr_fn is not None and adjust_lr_fn not in ADJUST_LR_FNS: raise ValueError(f"adjust_lr_fn must be one of {ADJUST_LR_FNS} or None, got {adjust_lr_fn!r}") - self._muon_defaults = dict( + muon_defaults = dict( momentum=momentum, nesterov=nesterov, ns_coefficients=tuple(ns_coefficients), @@ -140,7 +140,12 @@ def __init__( eps=eps, adjust_lr_fn=adjust_lr_fn, ) - self._adamw_defaults = dict(betas=tuple(betas), eps=adamw_eps) + self._init_groups(params, lr, weight_decay, muon_defaults, dict(betas=tuple(betas), eps=adamw_eps)) + + def _init_groups(self, params, lr: float, weight_decay: float, muon_defaults: Dict, adamw_defaults: Dict) -> None: + """Build the parameter groups; ``muon_defaults`` and ``adamw_defaults`` fill the keys a group leaves out.""" + self._muon_defaults = muon_defaults + self._adamw_defaults = adamw_defaults super().__init__(params, dict(lr=lr, weight_decay=weight_decay)) self.dp_group: Optional[dist.ProcessGroup] = None self.shard_to_working_param: Dict[int, Tensor] = {} @@ -159,7 +164,7 @@ def add_param_group(self, param_group: Dict) -> None: for p in params: if p.ndim != 2: raise ValueError( - f"Muon only supports 2D parameters, got shape {tuple(p.shape)}; " + f"{type(self).__name__} only supports 2D parameters, got shape {tuple(p.shape)}; " "put it in a group with use_muon=False (see muon_param_groups)" ) super().add_param_group(param_group) @@ -182,7 +187,7 @@ def setup_distributed( is_zero (bool): unused; shards are recognized by their shape. """ if tp_group is not None and dist.get_world_size(tp_group) > 1: - raise NotImplementedError("Muon does not support tensor parallelism yet; use tp_size=1") + raise NotImplementedError(f"{type(self).__name__} does not support tensor parallelism yet; use tp_size=1") self.dp_group = dp_group self.shard_to_working_param = dict(shard_to_working_param or {}) world_size = dist.get_world_size(dp_group) @@ -195,9 +200,9 @@ def setup_distributed( continue if working.ndim != 2 or p.numel() != -(-working.numel() // world_size): raise ValueError( - f"a {tuple(working.shape)} Muon parameter arrives as a shard of {p.numel()} elements, not one " + f"a {tuple(working.shape)} matrix arrives as a shard of {p.numel()} elements, not one " f"of {world_size} equal slices: its shards do not span the data-parallel group (e.g. expert " - "parameters sharded over their own group), which Muon does not support" + f"parameters sharded over their own group), which {type(self).__name__} does not support" ) def _working_param(self, p: Tensor) -> Tensor: @@ -226,23 +231,20 @@ def _muon_step(self, group: Dict) -> None: self._step_matrix(p, group) else: shards.append(p) - # every rank holds the same shards in the same order, so the gathers line up + # every rank holds the same shards in the same order, so the collectives line up for bucket in self._buckets(shards): inputs = [self._momentum_update(p, group) for p in bucket] - for p, update in zip(bucket, self._orthogonalize_shards(bucket, inputs, group)): - self._apply_update(p, update, self._working_param(p).shape, group) + self._apply_shard_updates(bucket, self._orthogonalize_shards(bucket, inputs, group), group) def _step_matrix(self, p: Tensor, group: Dict) -> None: if p.grad.ndim != 2: # A sharded 1D view that setup_distributed was not told about. raise RuntimeError( - f"Muon needs the full 2D gradient but got shape {tuple(p.grad.shape)}; under ZeRO the plugin has to " - "call setup_distributed (LowLevelZeroPlugin does), and Gemini / FSDP are not supported" + f"{type(self).__name__} needs the full 2D gradient but got shape {tuple(p.grad.shape)}; under ZeRO " + "the plugin has to call setup_distributed (LowLevelZeroPlugin does), and Gemini / FSDP are not " + "supported" ) - update = zeropower_via_newtonschulz( - self._momentum_update(p, group), group["ns_coefficients"], group["ns_steps"], group["eps"] - ) - self._apply_update(p, update, p.shape, group) + self._apply_update(p, self._orthogonalize(self._momentum_update(p, group), group), group) def _momentum_update(self, p: Tensor, group: Dict) -> Tensor: """Update ``p``'s momentum buffer and return the Newton-Schulz input; elementwise, so shards work too.""" @@ -254,9 +256,22 @@ def _momentum_update(self, p: Tensor, group: Dict) -> Tensor: buf.lerp_(grad, 1 - momentum) return grad.lerp(buf, momentum) if group["nesterov"] else buf + def _orthogonalize(self, matrix: Tensor, group: Dict) -> Tensor: + """Orthogonalize a whole matrix; the result is bf16.""" + return zeropower_via_newtonschulz(matrix, group["ns_coefficients"], group["ns_steps"], group["eps"]) + + def _apply_update(self, p: Tensor, update: Tensor, group: Dict) -> None: + """Apply weight decay and the orthogonalized ``update`` to a whole matrix.""" + self._decay_and_add(p, update, p.shape, group) + + def _apply_shard_updates(self, shards: List[Tensor], updates: List[Tensor], group: Dict) -> None: + """Apply to each of this rank's ``shards`` its slice of the whole matrix's orthogonalized update.""" + for p, update in zip(shards, updates): + self._decay_and_add(p, update, self._working_param(p).shape, group) + @staticmethod - def _apply_update(p: Tensor, update: Tensor, shape: torch.Size, group: Dict) -> None: - """Apply weight decay and the orthogonalized ``update``; ``shape`` is the whole matrix's, for the lr scale.""" + def _decay_and_add(p: Tensor, update: Tensor, shape: torch.Size, group: Dict) -> None: + # elementwise, so the same for a matrix and a shard; ``shape`` is the whole matrix's, for the lr scale lr = group["lr"] p.mul_(1 - lr * group["weight_decay"]) p.add_(update, alpha=-_adjust_lr(lr, group["adjust_lr_fn"], shape)) @@ -275,11 +290,7 @@ def _buckets(self, shards: List[Tensor]) -> List[List[Tensor]]: def _orthogonalize_shards(self, shards: List[Tensor], inputs: List[Tensor], group: Dict) -> List[Tensor]: """Gather the whole Newton-Schulz inputs of ``shards``, orthogonalize them and return each rank's slices.""" world_size, rank = dist.get_world_size(self.dp_group), dist.get_rank(self.dp_group) - # gloo gathers CPU tensors; NCCL (and Newton-Schulz) want the accelerator - if dist.get_backend(self.dp_group) == dist.Backend.GLOO: - device = torch.device("cpu") - else: - device = get_accelerator().get_current_device() + device = self._collective_device() local = torch.cat([x.to(device=device, dtype=torch.bfloat16).flatten() for x in inputs]) gathered = torch.empty(world_size * local.numel(), dtype=local.dtype, device=device) dist.all_gather_into_tensor(gathered, local, group=self.dp_group) @@ -290,11 +301,17 @@ def _orthogonalize_shards(self, shards: List[Tensor], inputs: List[Tensor], grou working, n = self._working_param(p), p.numel() matrix = gathered[:, offset : offset + n].reshape(-1)[: working.numel()].view(working.shape) offset += n - update = zeropower_via_newtonschulz(matrix, group["ns_coefficients"], group["ns_steps"], group["eps"]) + update = self._orthogonalize(matrix, group) update = F.pad(update.flatten(), (0, world_size * n - working.numel())).view(world_size, n)[rank] updates.append(update.to(p.device)) return updates + def _collective_device(self) -> torch.device: + # gloo communicates CPU tensors; NCCL (and Newton-Schulz) want the accelerator + if dist.get_backend(self.dp_group) == dist.Backend.GLOO: + return torch.device("cpu") + return get_accelerator().get_current_device() + def _adamw_step(self, group: Dict) -> None: lr, weight_decay, eps = group["lr"], group["weight_decay"], group["eps"] beta1, beta2 = group["betas"] diff --git a/colossalai/nn/optimizer/orscale.py b/colossalai/nn/optimizer/orscale.py new file mode 100644 index 000000000000..4bab469ad713 --- /dev/null +++ b/colossalai/nn/optimizer/orscale.py @@ -0,0 +1,219 @@ +"""OrScale: Muon's orthogonalized update scaled by a layer-wise trust ratio, with a built-in AdamW. + +For a matrix W with Nesterov momentum M, the orthogonalized direction Q = PolarExpress(M) and shape scale s: + + r = clip(||W||_F / (c * ||wd * W + s * Q||_F + eps), r_min, r_max) + W <- W - lr * r * (wd * W + s * Q) + +so the weight decay is coupled into the trust-ratio-scaled step. The two variants of the paper: + + OrScale s = 1, c = 1 + OrScale-LM s = 0.2 * sqrt(max(rows, cols)) (Moonlight's), and c set per layer at its first step so that r = 1 there + +The arithmetic follows the official implementation (https://github.com/NUS-HPC-AI-Lab/OrScale, commit a45fc2b) +statement by statement: accumulated float32 momentum (not ``Muon``'s moving average), bf16 Polar Express +orthogonalization, float32 norms. Run eagerly, the two give the same bits. The official code compiles its +orthogonalization with ``torch.compile`` on CUDA, which this one does not; that alone moves the result by 1-3%, as much +as bf16 moves it from a float64 iteration, and so does any other change of the bf16 rounding, such as Muon's moving +average in place of the accumulated momentum. Its two optimizers (OrScale for matrices, AdamW for the rest) are one +here, as in ``Muon``, so a Booster still receives a single optimizer; its ablation variants and its flattening of +convolution weights are left out. + +Under LowLevelZeRO the matrices are orthogonalized whole, as ``Muon`` does. The trust ratio needs two norms of the +whole matrix: each rank sums the squares over its slice and one all-reduce of two numbers per matrix adds them up. +Sums in a different order round differently, so a sharded step equals the unsharded one up to float32 rounding of +the trust ratio (about 1e-7), not bit for bit. + +Reference: + - OrScale: Orthogonalised Optimization with Layer-Wise Trust-Ratio Scaling, https://arxiv.org/abs/2605.07815 + - The Polar Express, https://arxiv.org/abs/2505.16932 +""" + +import math +from typing import Dict, List, Optional, Tuple + +import torch +import torch.distributed as dist +from torch import Tensor + +from .muon import Muon + +__all__ = ["OrScale", "polar_express"] + +# One (a, b, c) per iteration, from the official OrScale implementation (orscale/optim/newton_schulz.py). +POLAR_EXPRESS_COEFFICIENTS = ( + (8.156554524902461, -22.48329292557795, 15.878769915207462), + (4.042929935166739, -2.808917465908714, 0.5000178451051316), + (3.8916678022926607, -2.772484153217685, 0.5060648178503393), + (3.285753657755655, -2.3681294933425376, 0.46449024233003106), + (2.3465413258596377, -1.7097828382687081, 0.42323551169305323), +) +VARIANTS = ("orscale", "orscale_lm") +MOONLIGHT_RMS_CONSTANT = 0.2 + + +def polar_express(grad: Tensor, ns_steps: int = 5) -> Tensor: + """Approximate the polar factor U V^T of a 2D matrix with the Polar Express iteration, in bf16. + + Unlike ``zeropower_via_newtonschulz`` each iteration has its own coefficients, which converge to U V^T itself, and + the input is scaled by 1 / (1.02 * ||grad||_F + 1e-6). + """ + if grad.ndim != 2: + raise ValueError(f"Polar Express expects a 2D matrix, got shape {tuple(grad.shape)}") + if not 0 < ns_steps <= len(POLAR_EXPRESS_COEFFICIENTS): + raise ValueError(f"ns_steps must be in [1, {len(POLAR_EXPRESS_COEFFICIENTS)}], got {ns_steps}") + x = grad.bfloat16() + transposed = grad.size(0) > grad.size(1) + if transposed: + x = x.T + x = (x / (x.norm(dim=(-2, -1), keepdim=True) * 1.02 + 1e-6)).contiguous() + for a, b, c in POLAR_EXPRESS_COEFFICIENTS[:ns_steps]: + gram = x @ x.T + poly = torch.addmm(gram, gram, gram, beta=b, alpha=c) + x = torch.addmm(x, poly, x, beta=a) + if transposed: + x = x.T + return x + + +class OrScale(Muon): + """OrScale or OrScale-LM for 2D hidden-layer weights plus AdamW for every other parameter, in one optimizer. + + Parameter groups work as in ``Muon``: each sets ``use_muon`` (``muon_param_groups(model)`` builds both groups); + groups without the flag are OrScale groups. OrScale groups take ``lr``, ``weight_decay``, ``momentum``, + ``variant``, ``r_min``, ``r_max``, ``eps``, ``ns_steps`` and ``c_denom``; AdamW groups take ``lr``, + ``weight_decay``, ``betas`` and ``eps``. Missing keys fall back to the constructor arguments, with ``adamw_eps`` as + the AdamW groups' ``eps``. The defaults are the official ones for the matrices, where ``lr`` is far larger than + AdamW's, so give the AdamW group its own ``lr`` (the official default is a tenth of it). + + Like ``Muon``, it runs under ``TorchDDPPlugin`` and ``LowLevelZeroPlugin`` (ZeRO stage 1 and 2, see the module + docstring); tensor parallelism, Gemini and FSDP are not supported. + + Args: + params: parameters or parameter groups. + lr (float): learning rate shared by both branches unless a group overrides it (default: 0.02). + weight_decay (float): weight decay; for the matrices it is coupled into the trust-ratio-scaled step, for the + AdamW groups it is decoupled (default: 0.0). + momentum (float): Nesterov momentum of the matrices, accumulated as ``M = momentum * M + grad`` in float32 + (default: 0.95). + variant (str): ``"orscale"`` or ``"orscale_lm"`` (default: ``"orscale"``). + r_min (float): lower clipping bound of the trust ratio (default: 0.5). + r_max (float): upper clipping bound of the trust ratio (default: 1.5). The official recommendation is + [0.5, 1.5] for ``"orscale"`` and [0.1, 5.0] for ``"orscale_lm"``. + eps (float): added to the denominator of the trust ratio (default: 1e-6). + ns_steps (int): Polar Express iterations, at most 5 (default: 5). + c_denom (float, optional): ``"orscale_lm"`` only. The constant c of every layer; None calibrates it per layer + at the layer's first step as ``||W|| / ||wd * W + s * Q||`` and keeps it in the optimizer state + (default: None). + betas (tuple): AdamW betas (default: (0.9, 0.95)). + adamw_eps (float): AdamW epsilon (default: 1e-8). + """ + + def __init__( + self, + params, + lr: float = 0.02, + weight_decay: float = 0.0, + momentum: float = 0.95, + variant: str = "orscale", + r_min: float = 0.5, + r_max: float = 1.5, + eps: float = 1e-6, + ns_steps: int = 5, + c_denom: Optional[float] = None, + betas: Tuple[float, float] = (0.9, 0.95), + adamw_eps: float = 1e-8, + ): + if lr < 0.0: + raise ValueError(f"Invalid learning rate: {lr}") + if weight_decay < 0.0: + raise ValueError(f"Invalid weight_decay: {weight_decay}") + orscale_defaults = dict( + momentum=momentum, variant=variant, r_min=r_min, r_max=r_max, eps=eps, ns_steps=ns_steps, c_denom=c_denom + ) + self._init_groups(params, lr, weight_decay, orscale_defaults, dict(betas=tuple(betas), eps=adamw_eps)) + + def add_param_group(self, param_group: Dict) -> None: + if param_group.setdefault("use_muon", True): + group = {**self._muon_defaults, **param_group} + if not 0.0 <= group["momentum"] < 1.0: + raise ValueError(f"Invalid momentum: {group['momentum']}") + if group["variant"] not in VARIANTS: + raise ValueError(f"variant must be one of {VARIANTS}, got {group['variant']!r}") + if not 0.0 <= group["r_min"] <= group["r_max"]: + raise ValueError(f"need 0 <= r_min <= r_max, got r_min={group['r_min']}, r_max={group['r_max']}") + if not 0 < group["ns_steps"] <= len(POLAR_EXPRESS_COEFFICIENTS): + raise ValueError(f"ns_steps must be in [1, {len(POLAR_EXPRESS_COEFFICIENTS)}], got {group['ns_steps']}") + if group["c_denom"] is not None and group["c_denom"] <= 0.0: + raise ValueError(f"c_denom must be positive or None, got {group['c_denom']}") + super().add_param_group(param_group) + + def _momentum_update(self, p: Tensor, group: Dict) -> Tensor: + """Update ``p``'s momentum buffer and return the Polar Express input; elementwise, so shards work too.""" + # accumulated in float32 whatever the parameter's dtype + grad, momentum = p.grad.float(), group["momentum"] + state = self.state[p] + if "momentum_buffer" not in state: + state["momentum_buffer"] = torch.zeros_like(grad, memory_format=torch.preserve_format) + buf = state["momentum_buffer"] + buf.mul_(momentum).add_(grad) + return buf.mul(momentum).add_(grad) + + def _orthogonalize(self, matrix: Tensor, group: Dict) -> Tensor: + return polar_express(matrix, group["ns_steps"]) + + @staticmethod + def _shape_scale(shape: torch.Size, group: Dict) -> float: + if group["variant"] == "orscale_lm": + return MOONLIGHT_RMS_CONSTANT * math.sqrt(max(shape)) + return 1.0 + + def _apply_update(self, p: Tensor, update: Tensor, group: Dict) -> None: + scale, weight_decay = self._shape_scale(p.shape, group), group["weight_decay"] + weight_norm = p.norm().float() + step_norm = (weight_decay * p.float() + scale * update.float()).norm() + self._scaled_step(p, update, self._trust_ratio(p, weight_norm, step_norm, p.shape, group), scale, group) + + def _apply_shard_updates(self, shards: List[Tensor], updates: List[Tensor], group: Dict) -> None: + weight_decay = group["weight_decay"] + scales = [self._shape_scale(self._working_param(p).shape, group) for p in shards] + # the padding of a matrix's last slice is zero in the weights and in the update, and stays zero + squares = [] + for p, update, scale in zip(shards, updates, scales): + weight = p.float() + squares.append(weight.pow(2).sum()) + squares.append((weight_decay * weight + scale * update.float()).pow(2).sum()) + squares = torch.stack(squares).to(self._collective_device()) + dist.all_reduce(squares, group=self.dp_group) + norms = squares.sqrt().cpu().view(-1, 2) + for p, update, scale, (weight_norm, step_norm) in zip(shards, updates, scales, norms): + ratio = self._trust_ratio(p, weight_norm, step_norm, self._working_param(p).shape, group) + self._scaled_step(p, update, ratio, scale, group) + + def _trust_ratio(self, p: Tensor, weight_norm: Tensor, step_norm: Tensor, shape: torch.Size, group: Dict) -> float: + """The clipped trust ratio from the float32 norms of the whole matrix and of ``wd * W + s * Q``.""" + eps = group["eps"] + if group["variant"] == "orscale_lm": + state = self.state[p] + if "c_denom" not in state: + c_denom = group["c_denom"] + if c_denom is None: + weight_init, step_init = weight_norm.item(), step_norm.item() + if weight_init < eps or step_init < eps: + # a layer initialized to (almost) zero: fall back to Moonlight's scale instead of a runaway c + c_denom = MOONLIGHT_RMS_CONSTANT * math.sqrt(max(shape)) + else: + c_denom = weight_init / step_init + # a float, not a 0-dim tensor: LowLevelZeRO's checkpoints treat every tensor in the state as sharded + state["c_denom"] = float(c_denom) + step_norm = state["c_denom"] * step_norm + ratio = (weight_norm / (step_norm + eps)).item() + return max(group["r_min"], min(group["r_max"], ratio)) + + @staticmethod + def _scaled_step(p: Tensor, update: Tensor, ratio: float, scale: float, group: Dict) -> None: + # elementwise, so the same for a matrix and a shard + lr, weight_decay = group["lr"], group["weight_decay"] + if weight_decay != 0.0: + p.add_(p, alpha=-lr * ratio * weight_decay) + p.add_(update.to(p.dtype), alpha=-lr * ratio * scale) diff --git a/tests/test_optimizer/orscale_reference.py b/tests/test_optimizer/orscale_reference.py new file mode 100644 index 000000000000..acd875a42de0 --- /dev/null +++ b/tests/test_optimizer/orscale_reference.py @@ -0,0 +1,233 @@ +"""The official OrScale, as the reference of the alignment tests of ``colossalai.nn.optimizer.OrScale``. + +An excerpt of https://github.com/NUS-HPC-AI-Lab/OrScale at commit a45fc2ba19d072ca73dbbce88822631d5c1dd1c7 +(``orscale/optim/newton_schulz.py`` and ``orscale/optim/orscale_optimizer.py``), cut down to the two variants of the +paper: the ablation variants, the diagnostics, ``torch.compile`` and the batched orthogonalization are removed and the +branches on the variant simplified accordingly; the arithmetic is the official statements. When it was added, the +excerpt gave the same bits as the full official code run eagerly (``ORSCALE_DISABLE_TORCH_COMPILE=1``) on CPU and +CUDA, for float32, float64 and bfloat16 parameters. ``polar_express_float64`` is not part of the official code: it is +its iteration without the bfloat16 cast, the ground truth for the orthogonalization alone. + +MIT License + +Copyright (c) 2026 NUS-HPC-AI-Lab + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import math +from typing import Optional + +import torch +from torch import Tensor +from torch.optim.optimizer import Optimizer + +MOONLIGHT_RMS_CONSTANT = 0.2 + +POLAR_EXPRESS_COEFFS = [ + (8.156554524902461, -22.48329292557795, 15.878769915207462), + (4.042929935166739, -2.808917465908714, 0.5000178451051316), + (3.8916678022926607, -2.772484153217685, 0.5060648178503393), + (3.285753657755655, -2.3681294933425376, 0.46449024233003106), + (2.3465413258596377, -1.7097828382687081, 0.42323551169305323), +] + + +def orthogonalize(M: Tensor, num_iters: int = 5) -> Tensor: + transposed = M.size(-2) > M.size(-1) + X = M.bfloat16() + if transposed: + X = X.mT + + # Scale so spectral norm <= 1 (required for convergence) + X = X / (X.norm(dim=(-2, -1), keepdim=True) * 1.02 + 1e-6) + X = X.contiguous() + + # Allocate work buffers + A = torch.empty((*X.shape[:-1], X.size(-2)), device=X.device, dtype=X.dtype) + B = torch.empty_like(A) + C = torch.empty_like(X) + + for a, b, c in POLAR_EXPRESS_COEFFS[:num_iters]: + # A = X @ X^T (small square matrix) + torch.mm(X, X.mT, out=A) + # B = b*A + c*(A@A) + torch.addmm(A, A, A, beta=b, alpha=c, out=B) + # C = a*X + B @ X + torch.addmm(X, B, X, beta=a, out=C) + X, C = C, X + + if transposed: + X = X.mT + return X + + +def polar_express_float64(M: Tensor, num_iters: int = 5) -> Tensor: + transposed = M.size(-2) > M.size(-1) + X = M.double() + if transposed: + X = X.mT + + X = X / (X.norm(dim=(-2, -1), keepdim=True) * 1.02 + 1e-6) + + for a, b, c in POLAR_EXPRESS_COEFFS[:num_iters]: + A = X @ X.mT + B = b * A + c * (A @ A) + X = a * X + B @ X + + if transposed: + X = X.mT + return X + + +class OrScaleOptimizer(Optimizer): + def __init__( + self, + params, + lr: float = 0.02, + momentum: float = 0.95, + weight_decay: float = 0.0, + variant: str = "orscale_lm", + r_min: float = 0.5, + r_max: float = 1.5, + eps: float = 1e-6, + ns_iters: int = 5, + c_denom: Optional[float] = None, + ): + if lr < 0.0: + raise ValueError(f"Invalid learning rate: {lr}") + if momentum < 0.0 or momentum >= 1.0: + raise ValueError(f"Invalid momentum: {momentum}") + if c_denom is not None and c_denom <= 0.0: + raise ValueError(f"Invalid c_denom: {c_denom} (must be positive or None)") + if variant not in ("orscale", "orscale_lm"): + raise ValueError(f"this excerpt has the variants 'orscale' and 'orscale_lm', got {variant!r}") + + defaults = dict( + lr=lr, + momentum=momentum, + weight_decay=weight_decay, + variant=variant, + r_min=r_min, + r_max=r_max, + eps=eps, + ns_iters=ns_iters, + c_denom=c_denom, + ) + super().__init__(params, defaults) + + @torch.no_grad() + def step(self, closure=None): + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + lr = group["lr"] + mu = group["momentum"] + wd = group["weight_decay"] + variant = group["variant"] + r_min = group["r_min"] + r_max = group["r_max"] + eps = group["eps"] + ns_iters = group["ns_iters"] + c_denom_user = group.get("c_denom", None) + + use_shape_norm = variant == "orscale_lm" + use_grad_matrix_denom = variant == "orscale" + effective_alpha = 1.0 + + for p in group["params"]: + if p.grad is None: + continue + + grad = p.grad + if grad.ndim < 2: + continue + + orig_shape = p.shape + state = self.state[p] + + if len(state) == 0: + state["momentum_buffer"] = torch.zeros_like(p, dtype=torch.float32) + + buf = state["momentum_buffer"] + + # Flatten >=3D Conv weights to 2D [C_out, C_in*kH*kW] + g = grad.float().reshape(orig_shape[0], -1) + buf_view = buf.view(orig_shape[0], -1) + p_view = p.view(orig_shape[0], -1) + p_float = p_view.float() + m, n = p_view.shape + + # --- Momentum --- + # Accumulating momentum: M_t = mu * M_{t-1} + G_t + buf_view.mul_(mu).add_(g) + # Nesterov lookahead: M_hat = mu * M_t + G_t + m_hat = buf_view.mul(mu).add(g) + + # --- Orthogonalization --- + Q = orthogonalize(m_hat, num_iters=ns_iters) + + # --- Shape normalization --- + shape_scale = MOONLIGHT_RMS_CONSTANT * math.sqrt(max(m, n)) if use_shape_norm else 1.0 + + # --- Trust ratio --- + w_norm = p_view.norm().float() # ||W||_F (float32) + + if use_grad_matrix_denom: + w_stat = w_norm + # p_float is already float32; promote Q for an explicit + # float32 sum. shape_scale and wd are Python floats. + grad_matrix = wd * p_float + shape_scale * Q.float() + m_stat = grad_matrix.norm() # ||λW + s·Q||_F + else: + # Coupled denominator c_denom_ℓ * ||λW + s·Q||_F. + grad_matrix = wd * p_float + shape_scale * Q.float() + grad_matrix_norm = grad_matrix.norm() # ||λW + s·Q||_F + if "c_denom" not in state: + if c_denom_user is not None: + state["c_denom"] = float(c_denom_user) + else: + w_init = w_norm.item() + gm_init = grad_matrix_norm.item() + if w_init < eps or gm_init < eps: + # Fallback: pathological tiny init. Use the + # Moonlight shape constant so the optimizer + # degenerates to scaled Moonlight rather than + # NaNing or producing a runaway ratio. + state["c_denom"] = MOONLIGHT_RMS_CONSTANT * math.sqrt(max(m, n)) + else: + state["c_denom"] = w_init / gm_init + c_denom_layer = state["c_denom"] + w_stat = w_norm + m_stat = c_denom_layer * grad_matrix_norm # c_denom_ℓ · ||λW + s·Q||_F + + ratio_raw = (w_stat / (m_stat + eps)) ** effective_alpha + ratio_raw_val = ratio_raw.item() if isinstance(ratio_raw, Tensor) else float(ratio_raw) + ratio_clipped = max(r_min, min(r_max, ratio_raw_val)) + + # --- Update --- + q_update = Q.to(p.dtype) + if wd != 0.0: + p_view.add_(p_view, alpha=-lr * ratio_clipped * wd) + p_view.add_(q_update, alpha=-lr * ratio_clipped * shape_scale) + + return loss diff --git a/tests/test_optimizer/test_dist_muon.py b/tests/test_optimizer/test_dist_muon.py index 4b88001084c3..591b4ac75c5c 100644 --- a/tests/test_optimizer/test_dist_muon.py +++ b/tests/test_optimizer/test_dist_muon.py @@ -13,7 +13,7 @@ from colossalai.booster import Booster from colossalai.booster.plugin import GeminiPlugin, LowLevelZeroPlugin, TorchFSDPPlugin from colossalai.logging import disable_existing_loggers -from colossalai.nn.optimizer import Muon +from colossalai.nn.optimizer import Muon, OrScale from colossalai.nn.optimizer.muon import zeropower_via_newtonschulz from colossalai.testing import free_port, rerun_if_address_is_in_use, spawn from tests.kit.numerics import NumericsSpec, Trace, judge_equal, judge_relative_bound @@ -29,13 +29,16 @@ def requires_gpus(n): return pytest.mark.skipif(torch.cuda.device_count() < n, reason=f"needs {n} GPUs") -def muon_optimizer(params, cls=Muon): +def muon_optimizer(params, cls=Muon, **kwargs): params = list(params) groups = [ {"params": [p for p in params if p.ndim == 2], "use_muon": True}, {"params": [p for p in params if p.ndim != 2], "use_muon": False}, ] - return cls(groups, lr=LR, weight_decay=0.1) + return cls(groups, lr=LR, weight_decay=0.1, **kwargs) + + +ORSCALE = {"orscale": dict(variant="orscale"), "orscale_lm": dict(variant="orscale_lm", r_min=0.1, r_max=5.0)} class ShardLocalMuon(Muon): @@ -50,10 +53,37 @@ def _orthogonalize_shards(self, shards, inputs, group): ] +class ShardLocalTrustRatio(OrScale): + """A typical sharding bug: each rank takes the trust ratio from the norms of its own slice.""" + + def _apply_shard_updates(self, shards, updates, group): + for p, update in zip(shards, updates): + shape = self._working_param(p).shape + scale = self._shape_scale(shape, group) + step = group["weight_decay"] * p.float() + scale * update.float() + ratio = self._trust_ratio(p, p.norm().float(), step.norm(), shape, group) + self._scaled_step(p, update, ratio, scale, group) + + # --------------------------------------------------------------------------- -# Muon on hand-made ZeRO shards, over gloo: runs without GPUs +# Muon and OrScale on hand-made ZeRO shards, over gloo: runs without GPUs # --------------------------------------------------------------------------- +# name: (optimizer on the shards, optimizer on the whole matrices it must match, their arguments, bound). +# Muon's sharded step gathers the bf16 cast the whole-matrix step makes, so the results are the same bits (no bound). +# OrScale's trust ratio comes from norms summed slice by slice, which round differently from the norm of the whole +# matrix: its weights agree within the float32 relative bound. The gradients are given, so nothing amplifies that. +SHARD_CASES = { + "muon": (Muon, Muon, {}, None), + "orscale": (OrScale, OrScale, ORSCALE["orscale"], torch.float32), + "orscale_lm": (OrScale, OrScale, ORSCALE["orscale_lm"], torch.float32), +} +BUGGY_SHARD_CASES = { + "muon": (ShardLocalMuon, Muon, {}, torch.bfloat16), + "orscale": (ShardLocalTrustRatio, OrScale, ORSCALE["orscale"], torch.float32), + "orscale_lm": (ShardLocalTrustRatio, OrScale, ORSCALE["orscale_lm"], torch.float32), +} + def shard_of(t, world_size, rank): """``rank``'s slice of ``t``, padded and split the way LowLevelZeRO does.""" @@ -62,13 +92,14 @@ def shard_of(t, world_size, rank): return flat.view(world_size, -1)[rank] -def run_shards(rank, world_size, port, optimizer_cls): +def run_shards(rank, world_size, port, test_cls, ref_cls, kwargs, bound, buggy): # colossalai.launch would pick NCCL on a GPU machine; these shards live on the CPU os.environ.update(MASTER_ADDR="127.0.0.1", MASTER_PORT=str(port)) dist.init_process_group("gloo", rank=rank, world_size=world_size) gen = torch.Generator().manual_seed(0) shapes = {"tall": (HIDDEN, DIM_OUT), "wide": (DIM_IN, HIDDEN), "square": (48, 48), "bias": (HIDDEN,)} - init = {n: torch.randn(s, generator=gen) for n, s in shapes.items()} + # weights about as large as their orthogonalized updates, so that OrScale's trust ratios are near 1, not clipped + init = {n: 0.125 * torch.randn(s, generator=gen) for n, s in shapes.items()} grads = [{n: torch.randn(s, generator=gen) for n, s in shapes.items()} for _ in range(STEPS)] matrices = [n for n, s in shapes.items() if len(s) == 2] @@ -79,11 +110,11 @@ def groups(params): ] whole = {n: nn.Parameter(t.clone()) for n, t in init.items()} - ref = Muon(groups(whole), lr=LR, weight_decay=0.1) + ref = ref_cls(groups(whole), lr=LR, weight_decay=0.1, **kwargs) # as with LowLevelZeRO: built on the whole parameters, then handed the shards and told what they belong to working = {n: nn.Parameter(t.clone()) for n, t in init.items()} shards = {n: nn.Parameter(shard_of(t, world_size, rank).clone()) for n, t in init.items()} - test = optimizer_cls(groups(working), lr=LR, weight_decay=0.1) + test = test_cls(groups(working), lr=LR, weight_decay=0.1, **kwargs) for group, sharded in zip(test.param_groups, groups(shards)): group["params"] = sharded["params"] test.setup_distributed(None, None, {id(shards[n]): working[n] for n in shapes}, None, True) @@ -100,27 +131,45 @@ def gathered(t, shape): dist.all_gather(parts, t.detach().contiguous()) return torch.cat(parts)[: torch.Size(shape).numel()].view(shape) - # updates, not weights: next to unit-variance weights a wrong update would hide inside any relative bound - result = {f"update/{n}": gathered(shards[n], shapes[n]) - init[n] for n in shapes} - result.update({f"momentum/{n}": gathered(test.state[shards[n]]["momentum_buffer"], shapes[n]) for n in matrices}) - expected = {f"update/{n}": whole[n].detach() - init[n] for n in shapes} - expected.update({f"momentum/{n}": ref.state[whole[n]]["momentum_buffer"] for n in matrices}) - if optimizer_cls is Muon: - # the gathered input is the bf16 cast the whole-matrix step makes, so the results are the same bits - judge_equal(result, expected).assert_pass() + final = {n: gathered(shards[n], shapes[n]) for n in shapes} + # elementwise on the shards, so the same bits whatever the optimizer does with whole matrices + exact = {f"momentum/{n}": gathered(test.state[shards[n]]["momentum_buffer"], shapes[n]) for n in matrices} + exact["weight/bias"] = final["bias"] + exact_expected = {f"momentum/{n}": ref.state[whole[n]]["momentum_buffer"] for n in matrices} + exact_expected["weight/bias"] = whole["bias"].detach() + judge_equal(exact, exact_expected).assert_pass() + + if bound == torch.float32: + # The weights: the difference of two float32 weights does not resolve a step to 1e-6 of itself, while a wrong + # trust ratio moves the weights by far more than 1e-6 of themselves. + result = {f"matrix/{n}": final[n] for n in matrices} + expected = {f"matrix/{n}": whole[n].detach() for n in matrices} + if "c_denom" in ref.state[whole[matrices[0]]]: + result.update({f"c_denom/{n}": torch.tensor(test.state[shards[n]]["c_denom"]) for n in matrices}) + expected.update({f"c_denom/{n}": torch.tensor(ref.state[whole[n]]["c_denom"]) for n in matrices}) + report = judge_relative_bound(result, expected, bound) else: - report = judge_relative_bound(result, expected, torch.bfloat16) - assert {e.name for e in report.failures} >= {f"update/{n}" for n in matrices}, report.format() + # The updates: next to the weights a wrong update would hide inside the bf16 bound. + result = {f"matrix/{n}": final[n] - init[n] for n in matrices} + expected = {f"matrix/{n}": whole[n].detach() - init[n] for n in matrices} + report = judge_equal(result, expected) if bound is None else judge_relative_bound(result, expected, bound) + if buggy: + assert {e.name for e in report.failures} >= {f"matrix/{n}" for n in matrices}, report.format() + else: + report.assert_pass() dist.destroy_process_group() @pytest.mark.parametrize("world_size", [2, 3]) -def test_shards_match_whole_matrices(world_size): - torch.multiprocessing.spawn(run_shards, args=(world_size, free_port(), Muon), nprocs=world_size) +@pytest.mark.parametrize("case", SHARD_CASES) +def test_shards_match_whole_matrices(case, world_size): + args = (world_size, free_port(), *SHARD_CASES[case], False) + torch.multiprocessing.spawn(run_shards, args=args, nprocs=world_size) -def test_shard_local_orthogonalization_detected(): - torch.multiprocessing.spawn(run_shards, args=(2, free_port(), ShardLocalMuon), nprocs=2) +@pytest.mark.parametrize("case", BUGGY_SHARD_CASES) +def test_shard_local_computation_detected(case): + torch.multiprocessing.spawn(run_shards, args=(2, free_port(), *BUGGY_SHARD_CASES[case], True), nprocs=2) def run_setup_checks(rank, world_size, port): @@ -170,25 +219,26 @@ def _identity(output): ENTRY = (MLP, _data, _identity, None, SimpleNamespace(numerics=NumericsSpec())) -def run_zero(rank, world_size, port, plugin_fn, optimizer_fn, path): +def run_zero(rank, world_size, port, plugin_fn, optimizer_fn, steps, path): disable_existing_loggers() colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) # every rank trains on the same batch: averaging N equal gradients is exact for N = 2, so only sharding differs - trace = run_steps(prepare(ENTRY, STEPS), plugin_fn(), 0, STEPS, optimizer_fn=optimizer_fn) + trace = run_steps(prepare(ENTRY, steps), plugin_fn(), 0, steps, optimizer_fn=optimizer_fn) if rank == 0: trace.save(path) dist.destroy_process_group() -def zero_trace(plugin_fn, world_size, optimizer_fn=muon_optimizer): +def zero_trace(plugin_fn, world_size, optimizer_fn=muon_optimizer, steps=STEPS): with tempfile.TemporaryDirectory() as directory: path = os.path.join(directory, "trace.safetensors") - spawn(run_zero, world_size, plugin_fn=plugin_fn, optimizer_fn=optimizer_fn, path=path) + spawn(run_zero, world_size, plugin_fn=plugin_fn, optimizer_fn=optimizer_fn, steps=steps, path=path) return Trace.load(path) def comparable(test, world_size, ref): - """Losses, weights and the optimizer's state (float32 master weights, momentum, AdamW moments) of both runs. + """Losses, weights and the optimizer's state (float32 master weights, momentum, AdamW moments, OrScale-LM's + calibrated constants) of both runs. The optimizer's tensors are recorded per rank in ZeRO's flat layout; the slices of ``test`` are joined and the padding cut to the length of ``ref``'s, which one rank holds unpadded.""" @@ -225,6 +275,11 @@ def updates(state, precision): return out +def optimizer_state(state): + """What the optimizer updates and keeps: the float32 master weights, the momentum, OrScale-LM's constants.""" + return {key: value for key, value in state.items() if key.startswith("optim/")} + + ZERO_CONFIGS = [(1, "fp32", False), (2, "fp32", False), (1, "bf16", False), (2, "bf16", False), (2, "bf16", True)] @@ -238,16 +293,41 @@ def test_zero_two_ranks_equal_one(stage, precision, cpu_offload): judge_equal(test, ref).assert_pass() +ZERO_OPTIMIZERS = { + "muon": muon_optimizer, + **{n: partial(muon_optimizer, cls=OrScale, **kw) for n, kw in ORSCALE.items()}, +} + + @pytest.mark.dist @requires_gpus(4) +@pytest.mark.parametrize("optimizer", ZERO_OPTIMIZERS) @rerun_if_address_is_in_use() -def test_zero_four_ranks_within_bound(): +def test_zero_four_ranks_within_bound(optimizer): # averaging 4 equal gradients rounds, so the runs differ in the last bits: Megatron-LM's relative bound plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="bf16") - test, ref = comparable(zero_trace(plugin_fn, 4), 4, zero_trace(plugin_fn, 1)) + optimizer_fn = ZERO_OPTIMIZERS[optimizer] + test, ref = comparable(zero_trace(plugin_fn, 4, optimizer_fn), 4, zero_trace(plugin_fn, 1, optimizer_fn)) judge_relative_bound(updates(test, "bf16"), updates(ref, "bf16"), torch.bfloat16).assert_pass() +@pytest.mark.dist +@requires_gpus(2) +@pytest.mark.parametrize("variant", ORSCALE) +@pytest.mark.parametrize("stage, precision, cpu_offload", [(1, "fp32", False), (2, "bf16", False), (2, "bf16", True)]) +@rerun_if_address_is_in_use() +def test_zero_orscale_two_ranks_match_one(variant, stage, precision, cpu_offload): + # One step: the gradients, the momentum and the orthogonalized updates of the two runs are the same bits, and the + # trust ratios differ by float32 rounding, so the float32 bound holds for the master weights (see run_shards). + # Over more steps the bf16 orthogonalization turns that rounding into bf16 rounding, which only the bf16 bound + # covers (test_zero_four_ranks_within_bound). + plugin_fn = partial(LowLevelZeroPlugin, stage=stage, precision=precision, cpu_offload=cpu_offload) + optimizer_fn = ZERO_OPTIMIZERS[variant] + two, one = zero_trace(plugin_fn, 2, optimizer_fn, steps=1), zero_trace(plugin_fn, 1, optimizer_fn, steps=1) + test, ref = comparable(two, 2, one) + judge_relative_bound(optimizer_state(test), optimizer_state(ref), torch.float32).assert_pass() + + @pytest.mark.dist @requires_gpus(2) @rerun_if_address_is_in_use() @@ -262,30 +342,49 @@ def test_zero_shard_local_orthogonalization_detected(): @pytest.mark.dist @requires_gpus(2) @rerun_if_address_is_in_use() -def test_zero_resume_is_exact(): +def test_zero_shard_local_trust_ratio_detected(): + plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="bf16") + shard_local = partial(muon_optimizer, cls=ShardLocalTrustRatio, **ORSCALE["orscale"]) + two = zero_trace(plugin_fn, 2, shard_local, steps=1) + test, ref = comparable(two, 2, zero_trace(plugin_fn, 1, ZERO_OPTIMIZERS["orscale"], steps=1)) + report = judge_relative_bound(optimizer_state(test), optimizer_state(ref), torch.float32) + # the master weights of the matrices and nothing else; after one warm-up step the smaller matrix is only a few + # times outside the bound, so it is not required to fail + failed = {e.name for e in report.failures} + assert "optim/0.0/param" in failed and failed <= {"optim/0.0/param", "optim/0.1/param"}, report.format() + + +@pytest.mark.dist +@requires_gpus(2) +@pytest.mark.parametrize("optimizer, shard", [("muon", True), ("orscale_lm", True), ("orscale_lm", False)]) +@rerun_if_address_is_in_use() +def test_zero_resume_is_exact(optimizer, shard): # fp32: in bf16 no optimizer resumes exactly, as LowLevelZero rebuilds the float32 master weights from the bf16 - # ones on load (tests/kit/numerics/README.md, "Resuming from a checkpoint") + # ones on load (tests/kit/numerics/README.md, "Resuming from a checkpoint"). OrScale-LM keeps a float per matrix + # in its state, which both checkpoint layouts have to carry. plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="fp32") - resume.check(ENTRY, plugin_fn, world_size=2, optimizer_fn=muon_optimizer).assert_pass() + optimizer_fn = ZERO_OPTIMIZERS[optimizer] + resume.check(ENTRY, plugin_fn, world_size=2, optimizer_fn=optimizer_fn, shard_optimizer=shard).assert_pass() def run_plugins(rank, world_size, port): disable_existing_loggers() colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) - for plugin in (GeminiPlugin(), TorchFSDPPlugin()): - with pytest.raises(NotImplementedError, match="does not support Muon"): - model = MLP().cuda() - Booster(plugin=plugin).boost(model, muon_optimizer(model.parameters())) - model = MLP().cuda() - muon = muon_optimizer(model.parameters()) - _, optimizer, *_ = Booster(plugin=LowLevelZeroPlugin(stage=1)).boost(model, muon) - # set up in place, not replaced: an LR scheduler built on `muon` before boost keeps driving it - assert optimizer.optim is muon and muon.shard_to_working_param + for cls in (Muon, OrScale): + for plugin in (GeminiPlugin(), TorchFSDPPlugin()): + with pytest.raises(NotImplementedError, match=f"does not support {cls.__name__}"): + model = MLP().cuda() + Booster(plugin=plugin).boost(model, muon_optimizer(model.parameters(), cls)) + model = MLP().cuda() + inner = muon_optimizer(model.parameters(), cls) + _, optimizer, *_ = Booster(plugin=LowLevelZeroPlugin(stage=1)).boost(model, inner) + # set up in place, not replaced: an LR scheduler built on `inner` before boost keeps driving it + assert optimizer.optim is inner and inner.shard_to_working_param dist.destroy_process_group() @pytest.mark.dist @requires_gpus(1) @rerun_if_address_is_in_use() -def test_plugins_with_muon(): +def test_plugins_with_muon_and_orscale(): spawn(run_plugins, 1) diff --git a/tests/test_optimizer/test_orscale.py b/tests/test_optimizer/test_orscale.py new file mode 100644 index 000000000000..7b113432a9d5 --- /dev/null +++ b/tests/test_optimizer/test_orscale.py @@ -0,0 +1,358 @@ +import copy +import json +import math + +import pytest +import torch +import torch.nn as nn + +import colossalai +from colossalai.booster import Booster +from colossalai.booster.plugin import TorchDDPPlugin +from colossalai.logging import disable_existing_loggers +from colossalai.nn.optimizer import OrScale, muon_param_groups +from colossalai.nn.optimizer.muon import zeropower_via_newtonschulz +from colossalai.nn.optimizer.orscale import polar_express +from colossalai.testing import rerun_if_address_is_in_use, spawn +from tests.kit.numerics import assert_aligned, judge, judge_relative_bound +from tests.kit.numerics.scenarios.independent_impl import check_optimizer +from tests.test_optimizer.orscale_reference import OrScaleOptimizer, orthogonalize, polar_express_float64 + +DEVICES = ["cpu"] + (["cuda"] if torch.cuda.is_available() else []) +STEPS, LR, ADAMW_LR = 3, 0.02, 2e-3 + +# name: (shape, std of the initial weights). With these stds the trust ratio of "orscale", about ||W|| / ||Q||, starts +# at 0, near 0.05, 0.8, 1.2 and 3.5: clipped from below (twice), inside [0.5, 1.5] (twice) and clipped from above. +MATRICES = { + "zero": ((24, 16), 0.0), + "below": ((16, 24), 0.01), + "tall": ((64, 32), 0.1), + "wide": ((32, 64), 0.15), + "above": ((48, 48), 0.5), +} +OTHERS = {"bias": (64,), "gain": (32,)} # go to the AdamW branch + +# check_optimizer judges the weights, and judge's default tol (1e-4 of their RMS) would let through an error of a few +# percent in a step that changes them by a fraction of a percent. +WEIGHT_TOL = 1e-7 + +CONFIGS = { + "orscale": dict(variant="orscale", weight_decay=0.1), + "orscale-no-decay": dict(variant="orscale"), + "orscale-lm": dict(variant="orscale_lm", weight_decay=0.1, r_min=0.1, r_max=5.0), + "orscale-lm-no-decay": dict(variant="orscale_lm", r_min=0.1, r_max=5.0), + "orscale-lm-given-c": dict(variant="orscale_lm", weight_decay=0.1, r_min=0.1, r_max=5.0, c_denom=0.05), +} + + +def make_params(device="cpu", others=False): + gen = torch.Generator().manual_seed(0) + params = {n: std * torch.randn(shape, generator=gen, dtype=torch.float64) for n, (shape, std) in MATRICES.items()} + if others: + params.update({n: torch.randn(shape, generator=gen, dtype=torch.float64) for n, shape in OTHERS.items()}) + return {n: p.to(device) for n, p in params.items()} + + +def ours(cls=OrScale, **kwargs): + return lambda params: cls(params, **{"lr": LR, **kwargs}) + + +def official(**kwargs): + return lambda params: OrScaleOptimizer(params, lr=LR, **kwargs) + + +def check(test_fn, config, device="cpu"): + return check_optimizer(test_fn, official(**config), make_params(device), torch.float32, steps=STEPS, tol=WEIGHT_TOL) + + +# --------------------------------------------------------------------------- +# Polar Express +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("device", DEVICES) +@pytest.mark.parametrize("shape", [(64, 32), (32, 64), (48, 48)]) +def test_polar_express_matches_official(device, shape): + grad = torch.randn(shape, generator=torch.Generator().manual_seed(1), dtype=torch.float64).to(device) + truth = polar_express_float64(grad) + ref = orthogonalize(grad.float()) + assert_aligned(polar_express(grad.float()), ref, truth) + # Muon's Newton-Schulz coefficients leave the singular values in about [0.5, 1.5]: not the same function + assert not judge(zeropower_via_newtonschulz(grad.float()), ref, truth).passed + + +@pytest.mark.parametrize("shape", [(64, 32), (32, 64), (48, 48)]) +def test_polar_express_approximates_the_polar_factor(shape): + # independent of the official code: guards the coefficient table itself + grad = torch.randn(shape, generator=torch.Generator().manual_seed(1), dtype=torch.float64) + u, _, vh = torch.linalg.svd(grad, full_matrices=False) + polar = u @ vh + error = (polar_express(grad.float()).double() - polar).norm() / polar.norm() + assert error < 0.1, f"relative error {error:.3f} against U V^T" + + +# --------------------------------------------------------------------------- +# the update rule against the official implementation +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("device", DEVICES) +@pytest.mark.parametrize("config", CONFIGS.values(), ids=CONFIGS.keys()) +def test_orscale_matches_official(device, config): + check(ours(**config), config, device).assert_pass() + + +@pytest.mark.parametrize( + "config, wrong", + [ + ("orscale", {"lr": LR * 1.01}), + ("orscale", {"momentum": 0.94}), + ("orscale", {"weight_decay": 0.09}), + ("orscale", {"r_min": 0.6}), + ("orscale", {"r_max": 1.4}), + ("orscale", {"eps": 1e-3}), + ("orscale", {"ns_steps": 4}), + ("orscale", {"variant": "orscale_lm"}), + ("orscale-lm", {"weight_decay": 0.09}), + ("orscale-lm", {"c_denom": 0.05}), + ("orscale-lm-given-c", {"c_denom": 0.051}), + ], +) +def test_alignment_check_catches_wrong_hyperparameters(config, wrong): + config = CONFIGS[config] + assert not check(ours(**{**config, **wrong}), config).passed + + +class DecoupledDecay(OrScale): + """Weight decay outside the trust ratio, as Muon applies it.""" + + @staticmethod + def _scaled_step(p, update, ratio, scale, group): + lr = group["lr"] + p.mul_(1 - lr * group["weight_decay"]) + p.add_(update.to(p.dtype), alpha=-lr * ratio * scale) + + +class AveragedMomentum(OrScale): + """Muon's exponential moving average: the same direction, but its bf16 cast before Polar Express rounds differently + from the accumulated momentum's, which already exceeds the float32 baseline error.""" + + def _momentum_update(self, p, group): + grad, momentum = p.grad.float(), group["momentum"] + buf = self.state[p].setdefault("momentum_buffer", torch.zeros_like(grad)) + buf.lerp_(grad, 1 - momentum) + return grad.lerp(buf, momentum) + + +@pytest.mark.parametrize("cls", [DecoupledDecay, AveragedMomentum]) +@pytest.mark.parametrize("config", ["orscale", "orscale-lm"]) +def test_alignment_check_catches_other_update_rules(cls, config): + config = CONFIGS[config] + assert not check(ours(cls, **config), config).passed + + +def grouped(params, cls, **kwargs): + params = list(params) + matrices, others = [p for p in params if p.ndim == 2], [p for p in params if p.ndim != 2] + return cls(matrices, others, **kwargs) + + +class OfficialPair: + """The official setup: OrScale for the matrices and torch's AdamW for everything else, stepped together.""" + + def __init__(self, matrices, others, **kwargs): + self.optimizers = [ + OrScaleOptimizer(matrices, lr=LR, **kwargs), + torch.optim.AdamW(others, lr=ADAMW_LR, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.01, foreach=False), + ] + + def step(self): + for optimizer in self.optimizers: + optimizer.step() + + def zero_grad(self, set_to_none=True): + for optimizer in self.optimizers: + optimizer.zero_grad(set_to_none=set_to_none) + + +def one_optimizer(matrices, others, **kwargs): + groups = [ + {"params": matrices, "use_muon": True}, + {"params": others, "use_muon": False, "lr": ADAMW_LR, "weight_decay": 0.01}, + ] + return OrScale(groups, lr=LR, betas=(0.9, 0.999), **kwargs) + + +@pytest.mark.parametrize("device", DEVICES) +@pytest.mark.parametrize("config", ["orscale", "orscale-lm"]) +def test_orscale_with_adamw_matches_official_pair(device, config): + config = CONFIGS[config] + check_optimizer( + lambda params: grouped(params, one_optimizer, **config), + lambda params: grouped(params, OfficialPair, **config), + make_params(device, others=True), + torch.float32, + steps=STEPS, + tol=WEIGHT_TOL, + ).assert_pass() + + +def test_orscale_lm_first_step_has_trust_ratio_one(): + # the calibration makes the first step of every layer Moonlight's, whatever the scale of its weights + params = {n: nn.Parameter(p.float()) for n, p in make_params().items() if n != "zero"} + start = {n: p.detach().clone() for n, p in params.items()} + optimizer = OrScale(params.values(), lr=LR, variant="orscale_lm", r_min=0.1, r_max=5.0) + for p in params.values(): + p.grad = torch.randn_like(p) + optimizer.step() + for name, p in params.items(): + # momentum starts at 0, so the Nesterov direction is (1 + momentum) * grad + direction = polar_express(p.grad * 1.95).float() + expected = start[name] - LR * 0.2 * math.sqrt(max(p.shape)) * direction + torch.testing.assert_close(p.detach(), expected, rtol=1e-5, atol=1e-7) + assert isinstance(optimizer.state[p]["c_denom"], float) + + +# --------------------------------------------------------------------------- +# parameter grouping, state and error handling +# --------------------------------------------------------------------------- + + +class TinyLM(nn.Module): + def __init__(self): + super().__init__() + self.embed = nn.Embedding(50, 16) + self.proj = nn.Linear(16, 16) + self.norm = nn.LayerNorm(16) + self.lm_head = nn.Linear(16, 50, bias=False) + + def get_output_embeddings(self): + return self.lm_head + + +def test_orscale_takes_muon_param_groups(): + model = TinyLM() + optimizer = OrScale(muon_param_groups(model), variant="orscale_lm") + groups = {g["use_muon"]: g for g in optimizer.param_groups} + assert [id(p) for p in groups[True]["params"]] == [id(model.proj.weight)] + assert groups[True]["variant"] == "orscale_lm" and "variant" not in groups[False] + start = {n: p.detach().clone() for n, p in model.named_parameters()} + for p in model.parameters(): + p.grad = torch.randn_like(p) + optimizer.step() + assert all(not torch.equal(p, start[n]) for n, p in model.named_parameters()) + + +def test_orscale_state_dict_round_trip(): + torch.manual_seed(0) + model = TinyLM() + kwargs = dict(lr=LR, weight_decay=0.1, variant="orscale_lm", r_min=0.1, r_max=5.0) + optimizer = OrScale(muon_param_groups(model), **kwargs) + grads = [[torch.randn_like(p) for p in model.parameters()] for _ in range(3)] + + def step(module, opt, step_grads): + for p, g in zip(module.parameters(), step_grads): + p.grad = g.clone() + opt.step() + + step(model, optimizer, grads[0]) + step(model, optimizer, grads[1]) + state = optimizer.state_dict() + # Hyperparameters and the calibrated constants stay plain data so JSON-based checkpoint formats can store them. + json.dumps([{k: v for k, v in g.items() if k != "params"} for g in state["param_groups"]]) + assert isinstance(optimizer.state[model.proj.weight]["c_denom"], float) + + resumed_model = copy.deepcopy(model) + resumed = OrScale(muon_param_groups(resumed_model), **kwargs) + resumed.load_state_dict(copy.deepcopy(state)) + step(model, optimizer, grads[2]) + step(resumed_model, resumed, grads[2]) + for p, q in zip(model.parameters(), resumed_model.parameters()): + torch.testing.assert_close(q, p, rtol=0, atol=0) + + +@pytest.mark.parametrize( + "kwargs, match", + [ + ({"variant": "muscale"}, "variant"), + ({"momentum": 1.0}, "momentum"), + ({"r_min": 2.0}, "r_min"), + ({"ns_steps": 6}, "ns_steps"), + ({"c_denom": 0.0}, "c_denom"), + ], +) +def test_orscale_rejects_invalid_arguments(kwargs, match): + with pytest.raises(ValueError, match=match): + OrScale([nn.Parameter(torch.randn(4, 4))], **kwargs) + optimizer = OrScale([nn.Parameter(torch.randn(4, 4))]) + with pytest.raises(ValueError, match=match): + optimizer.add_param_group({"params": [nn.Parameter(torch.randn(4, 4))], **kwargs}) + assert len(optimizer.param_groups) == 1 + + +def test_orscale_rejects_non_matrix_params(): + with pytest.raises(ValueError, match="OrScale only supports 2D"): + OrScale(nn.Linear(4, 4).parameters()) + + +# --------------------------------------------------------------------------- +# DDP: two ranks on half batches must match one device on the full batch +# --------------------------------------------------------------------------- + +DIM, GLOBAL_BATCH = 32, 8 + + +def make_mlp(): + torch.manual_seed(0) + return nn.Sequential(nn.Linear(DIM, 64), nn.GELU(), nn.LayerNorm(64), nn.Linear(64, DIM)) + + +def train(model, batches, variant, booster=None, rank=0, world_size=1): + device = torch.device("cuda", torch.cuda.current_device()) + model = model.to(device) + start = {n: p.detach().double().clone() for n, p in model.named_parameters()} + groups = muon_param_groups(model) + groups[1]["lr"] = ADAMW_LR + opt = OrScale(groups, lr=LR, weight_decay=0.1, variant=variant) + if booster is not None: + model, opt, *_ = booster.boost(model, opt) + local = GLOBAL_BATCH // world_size + for batch in batches: + x = batch[rank * local : (rank + 1) * local].to(device) + loss = model(x).pow(2).mean() + if booster is not None: + booster.backward(loss, opt) + else: + loss.backward() + opt.step() + opt.zero_grad() + module = model.unwrap() if hasattr(model, "unwrap") else model + return {n: (p.detach().double() - start[n]).cpu() for n, p in module.named_parameters()} + + +def run_ddp(rank, world_size, port, variant): + disable_existing_loggers() + colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port, backend="nccl") + batches = torch.randn(STEPS, GLOBAL_BATCH, DIM, generator=torch.Generator().manual_seed(2)) + booster = Booster(plugin=TorchDDPPlugin()) + + test = train(make_mlp(), batches, variant, booster, rank, world_size) + ref = train(make_mlp(), batches, variant) + # Same code at the same precision, so there is no baseline error to measure: the tool's relative bound applies, + # at the bf16 of the orthogonalization, which turns float32 rounding of the averaged gradient into bf16 rounding. + judge_relative_bound(test, ref, torch.bfloat16).assert_pass() + # Each rank stepping on its own half batch (no gradient all-reduce) must be caught. + unsynced = train(make_mlp(), batches, variant, rank=rank, world_size=world_size) + assert not judge_relative_bound(unsynced, ref, torch.bfloat16).passed + + +@pytest.mark.dist +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="needs 2 GPUs") +@pytest.mark.parametrize("variant", ["orscale", "orscale_lm"]) +@rerun_if_address_is_in_use() +def test_orscale_ddp(variant): + spawn(run_ddp, nprocs=2, variant=variant) + + +if __name__ == "__main__": + test_orscale_ddp("orscale")