From 307baa155217928f02b2ba82c9b1cc0b2029d46e Mon Sep 17 00:00:00 2001 From: yuxuan-lou Date: Wed, 18 Nov 2026 18:36:58 +0800 Subject: [PATCH] [optimizer] run Muon under LowLevelZeRO on whole matrices LowLevelZeRO (stage 1 and 2) hands the inner optimizer a flat 1/N slice of every parameter, while Newton-Schulz needs whole matrices, so Muon raised there. As in Moonlight's distributed Muon, each rank now updates the momentum of its own slices, the bf16 Newton-Schulz inputs are all-gathered over the data-parallel group, and every rank orthogonalizes the whole matrices and keeps the slice of each update it owns. The gathered input is the bf16 cast the unsharded iteration makes anyway, so a sharded step gives the same bits as the unsharded one. The cost is one bf16 all-gather of the Muon parameters per step (in buckets of at most 512 MiB) and every rank orthogonalizing all of them. - muon.py: Muon is a DistributedOptim. LowLevelZeroPlugin and HybridParallelPlugin call setup_distributed on the object itself; it is not replaced through cast_to_distributed, so an LR scheduler built on it before boost keeps driving it. Tensor parallelism and shards that do not span the data-parallel group (expert parameters) raise. - gemini_plugin.py, torch_fsdp_plugin.py: Muon raises NotImplementedError at boost. Gemini's chunks split matrices at arbitrary boundaries and FSDP hands over flattened parameters in one group. - test_dist_muon.py: hand-made shards over gloo for 2 and 3 ranks against whole matrices, bit for bit (runs in the CPU job); LowLevelZeRO with 2 ranks against 1 rank bit for bit (stage 1 and 2, fp32 and bf16, CPU offload), 4 ranks within judge_relative_bound, resume from a checkpoint bit for bit in fp32. A subclass that orthogonalizes each slice on its own must fail in both settings. - run_cpu_unit_tests.sh: run test_dist_muon.py in the CPU job. Co-Authored-By: Claude Fable 5.2 --- .github/scripts/ci/run_cpu_unit_tests.sh | 1 + colossalai/booster/plugin/gemini_plugin.py | 6 + .../booster/plugin/torch_fsdp_plugin.py | 6 + colossalai/nn/optimizer/muon.py | 155 ++++++++-- tests/test_optimizer/test_dist_muon.py | 291 ++++++++++++++++++ 5 files changed, 440 insertions(+), 19 deletions(-) create mode 100644 tests/test_optimizer/test_dist_muon.py diff --git a/.github/scripts/ci/run_cpu_unit_tests.sh b/.github/scripts/ci/run_cpu_unit_tests.sh index 8916af3d83f4..68bbeaed221b 100644 --- a/.github/scripts/ci/run_cpu_unit_tests.sh +++ b/.github/scripts/ci/run_cpu_unit_tests.sh @@ -19,6 +19,7 @@ TESTS=( tests/test_numerics/test_numerics_data_parallel.py tests/test_numerics/test_numerics_independent_impl.py tests/test_numerics/test_numerics_resume.py + tests/test_optimizer/test_dist_muon.py tests/test_optimizer/test_lr_scheduler.py tests/test_optimizer/test_muon.py tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py diff --git a/colossalai/booster/plugin/gemini_plugin.py b/colossalai/booster/plugin/gemini_plugin.py index 0d8846e07b30..7a189e17f494 100644 --- a/colossalai/booster/plugin/gemini_plugin.py +++ b/colossalai/booster/plugin/gemini_plugin.py @@ -31,6 +31,7 @@ from colossalai.cluster import DistCoordinator, ProcessGroupMesh from colossalai.interface import ModelWrapper, OptimizerWrapper from colossalai.logging import get_dist_logger +from colossalai.nn.optimizer import Muon from colossalai.shardformer import ShardConfig, ShardFormer from colossalai.zero import GeminiDDP, GeminiOptimizer from colossalai.zero.gemini.memory_tracer import MemStats @@ -665,6 +666,11 @@ def configure( dataloader: Optional[DataLoader] = None, lr_scheduler: Optional[LRScheduler] = None, ) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]: + if isinstance(optimizer, Muon): + raise NotImplementedError( + "GeminiPlugin does not support Muon: its chunks split matrices at arbitrary boundaries. " + "Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." + ) params_info = get_param_info(optimizer) if not isinstance(model, ModelWrapper): # convert model to sync bn diff --git a/colossalai/booster/plugin/torch_fsdp_plugin.py b/colossalai/booster/plugin/torch_fsdp_plugin.py index 05cc432a673d..5c9390539972 100644 --- a/colossalai/booster/plugin/torch_fsdp_plugin.py +++ b/colossalai/booster/plugin/torch_fsdp_plugin.py @@ -34,6 +34,7 @@ from colossalai.cluster import DistCoordinator from colossalai.interface import ModelWrapper, OptimizerWrapper from colossalai.logging import get_dist_logger +from colossalai.nn.optimizer import Muon from colossalai.utils.safetensors import load_flat from .dp_plugin_base import DPPluginBase @@ -536,6 +537,11 @@ def configure( dataloader: Optional[DataLoader] = None, lr_scheduler: Optional[LRScheduler] = None, ) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]: + if isinstance(optimizer, Muon): + raise NotImplementedError( + "TorchFSDPPlugin does not support Muon: FSDP hands the optimizer flattened parameters in one group. " + "Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." + ) # wrap the model with PyTorch FSDP fsdp_model = TorchFSDPModel(model, device_id=torch.cuda.current_device(), **self.fsdp_kwargs) diff --git a/colossalai/nn/optimizer/muon.py b/colossalai/nn/optimizer/muon.py index 67dfeebb4536..bd6de205de84 100644 --- a/colossalai/nn/optimizer/muon.py +++ b/colossalai/nn/optimizer/muon.py @@ -5,6 +5,13 @@ norms, biases) go to the AdamW branch of the same optimizer, as in KellerJordan/Muon's ``MuonWithAuxAdam`` and Moonlight, so a Booster still receives a single optimizer. +Under LowLevelZeRO (stage 1 and 2) each rank holds a flat 1/N slice of every parameter, while Newton-Schulz needs whole +matrices. As in Moonlight's distributed Muon, each rank updates the momentum of its own slices, the bf16 +Newton-Schulz inputs are all-gathered over the data-parallel group, and every rank orthogonalizes the whole matrices +and keeps the slice of each update it owns. The gathered input is the bf16 cast the unsharded iteration makes anyway, +so a sharded step computes the unsharded one, at the cost of one bf16 all-gather of the Muon parameters per step and +every rank orthogonalizing all of them. + References: - https://kellerjordan.github.io/posts/muon/ - Muon is Scalable for LLM Training, https://arxiv.org/abs/2502.16982 @@ -14,9 +21,13 @@ from typing import Dict, List, Optional, Tuple import torch +import torch.distributed as dist import torch.nn as nn +import torch.nn.functional as F from torch import Tensor -from torch.optim import Optimizer + +from colossalai.accelerator import get_accelerator +from colossalai.interface.optimizer import DistributedOptim __all__ = ["Muon", "muon_param_groups", "zeropower_via_newtonschulz"] @@ -63,7 +74,7 @@ def _adjust_lr(lr: float, adjust_lr_fn: Optional[str], shape: torch.Size) -> flo return lr * 0.2 * math.sqrt(max(rows, cols)) -class Muon(Optimizer): +class Muon(DistributedOptim): """Muon for 2D hidden-layer weights plus AdamW for every other parameter, in one optimizer. Each parameter group sets ``use_muon``; ``muon_param_groups(model)`` builds both groups from a model. Groups @@ -72,6 +83,11 @@ class Muon(Optimizer): ``betas`` and ``eps``. Missing keys fall back to the constructor arguments, with ``adamw_eps`` as the AdamW groups' ``eps``. + ``LowLevelZeroPlugin`` and ``HybridParallelPlugin`` call ``setup_distributed`` on this very object (it is not + replaced, so an LR scheduler built on it before ``boost`` keeps driving it), after which ZeRO's parameter shards are + orthogonalized as whole matrices (see the module docstring). The AdamW branch is elementwise and runs on the + shards as they are. Tensor parallelism, Gemini and FSDP are not supported. + Args: params: parameters or parameter groups. lr (float): learning rate shared by both branches unless a group overrides it (default: 1e-3). @@ -88,6 +104,10 @@ class Muon(Optimizer): adamw_eps (float): AdamW epsilon (default: 1e-8). """ + # Under ZeRO at most this many elements of whole matrices are gathered at once (512 MiB in bf16), which bounds the + # extra memory of a step. + bucket_numel = 1 << 28 + def __init__( self, params, @@ -122,6 +142,8 @@ def __init__( ) self._adamw_defaults = dict(betas=tuple(betas), eps=adamw_eps) super().__init__(params, dict(lr=lr, weight_decay=weight_decay)) + self.dp_group: Optional[dist.ProcessGroup] = None + self.shard_to_working_param: Dict[int, Tensor] = {} def add_param_group(self, param_group: Dict) -> None: use_muon = param_group.setdefault("use_muon", True) @@ -142,6 +164,45 @@ def add_param_group(self, param_group: Dict) -> None: ) super().add_param_group(param_group) + def setup_distributed( + self, + tp_group: Optional[dist.ProcessGroup] = None, + dp_group: Optional[dist.ProcessGroup] = None, + shard_to_working_param: Optional[Dict] = None, + padding_map: Optional[Dict] = None, + is_zero: Optional[bool] = False, + ): + """Record the data-parallel group and, for ZeRO, which whole parameter each shard belongs to. + + Arguments: + tp_group (dist.ProcessGroup): tensor-parallel group; only a group of size 1 is supported. + dp_group (dist.ProcessGroup): the group ZeRO shards over (default: the whole world). + shard_to_working_param (Dict): id of each shard to the parameter used in forward and backward. + padding_map (Dict): unused; the padding follows from the parameter's size and the group size. + is_zero (bool): unused; shards are recognized by their shape. + """ + if tp_group is not None and dist.get_world_size(tp_group) > 1: + raise NotImplementedError("Muon does not support tensor parallelism yet; use tp_size=1") + self.dp_group = dp_group + self.shard_to_working_param = dict(shard_to_working_param or {}) + world_size = dist.get_world_size(dp_group) + for group in self.param_groups: + if not group["use_muon"]: + continue + for p in group["params"]: + working = self._working_param(p) + if working.shape == p.shape: + continue + if working.ndim != 2 or p.numel() != -(-working.numel() // world_size): + raise ValueError( + f"a {tuple(working.shape)} Muon parameter arrives as a shard of {p.numel()} elements, not one " + f"of {world_size} equal slices: its shards do not span the data-parallel group (e.g. expert " + "parameters sharded over their own group), which Muon does not support" + ) + + def _working_param(self, p: Tensor) -> Tensor: + return self.shard_to_working_param.get(id(p), p) + @torch.no_grad() def step(self, closure=None): loss = None @@ -157,26 +218,82 @@ def step(self, closure=None): return loss def _muon_step(self, group: Dict) -> None: - lr, weight_decay, momentum = group["lr"], group["weight_decay"], group["momentum"] + shards = [] for p in group["params"]: if p.grad is None: continue - grad = p.grad - if grad.ndim != 2: - # A sharded 1D view, e.g. from LowLevelZeRO's master parameter slices. - raise RuntimeError( - f"Muon needs the full 2D gradient but got shape {tuple(grad.shape)}; " - "ZeRO / Gemini / FSDP sharding is not supported yet" - ) - state = self.state[p] - if "momentum_buffer" not in state: - state["momentum_buffer"] = torch.zeros_like(grad, memory_format=torch.preserve_format) - buf = state["momentum_buffer"] - buf.lerp_(grad, 1 - momentum) - update = grad.lerp(buf, momentum) if group["nesterov"] else buf - update = zeropower_via_newtonschulz(update, group["ns_coefficients"], group["ns_steps"], group["eps"]) - p.mul_(1 - lr * weight_decay) - p.add_(update, alpha=-_adjust_lr(lr, group["adjust_lr_fn"], p.shape)) + if self._working_param(p).shape == p.shape: + self._step_matrix(p, group) + else: + shards.append(p) + # every rank holds the same shards in the same order, so the gathers line up + for bucket in self._buckets(shards): + inputs = [self._momentum_update(p, group) for p in bucket] + for p, update in zip(bucket, self._orthogonalize_shards(bucket, inputs, group)): + self._apply_update(p, update, self._working_param(p).shape, group) + + def _step_matrix(self, p: Tensor, group: Dict) -> None: + if p.grad.ndim != 2: + # A sharded 1D view that setup_distributed was not told about. + raise RuntimeError( + f"Muon needs the full 2D gradient but got shape {tuple(p.grad.shape)}; under ZeRO the plugin has to " + "call setup_distributed (LowLevelZeroPlugin does), and Gemini / FSDP are not supported" + ) + update = zeropower_via_newtonschulz( + self._momentum_update(p, group), group["ns_coefficients"], group["ns_steps"], group["eps"] + ) + self._apply_update(p, update, p.shape, group) + + def _momentum_update(self, p: Tensor, group: Dict) -> Tensor: + """Update ``p``'s momentum buffer and return the Newton-Schulz input; elementwise, so shards work too.""" + grad, momentum = p.grad, group["momentum"] + state = self.state[p] + if "momentum_buffer" not in state: + state["momentum_buffer"] = torch.zeros_like(grad, memory_format=torch.preserve_format) + buf = state["momentum_buffer"] + buf.lerp_(grad, 1 - momentum) + return grad.lerp(buf, momentum) if group["nesterov"] else buf + + @staticmethod + def _apply_update(p: Tensor, update: Tensor, shape: torch.Size, group: Dict) -> None: + """Apply weight decay and the orthogonalized ``update``; ``shape`` is the whole matrix's, for the lr scale.""" + lr = group["lr"] + p.mul_(1 - lr * group["weight_decay"]) + p.add_(update, alpha=-_adjust_lr(lr, group["adjust_lr_fn"], shape)) + + def _buckets(self, shards: List[Tensor]) -> List[List[Tensor]]: + buckets, numel = [], 0 + for p in shards: + size = self._working_param(p).numel() + if not buckets or numel + size > self.bucket_numel: + buckets.append([]) + numel = 0 + buckets[-1].append(p) + numel += size + return buckets + + def _orthogonalize_shards(self, shards: List[Tensor], inputs: List[Tensor], group: Dict) -> List[Tensor]: + """Gather the whole Newton-Schulz inputs of ``shards``, orthogonalize them and return each rank's slices.""" + world_size, rank = dist.get_world_size(self.dp_group), dist.get_rank(self.dp_group) + # gloo gathers CPU tensors; NCCL (and Newton-Schulz) want the accelerator + if dist.get_backend(self.dp_group) == dist.Backend.GLOO: + device = torch.device("cpu") + else: + device = get_accelerator().get_current_device() + local = torch.cat([x.to(device=device, dtype=torch.bfloat16).flatten() for x in inputs]) + gathered = torch.empty(world_size * local.numel(), dtype=local.dtype, device=device) + dist.all_gather_into_tensor(gathered, local, group=self.dp_group) + # row r holds rank r's slices of every matrix, one after another + gathered = gathered.view(world_size, local.numel()) + updates, offset = [], 0 + for p in shards: + working, n = self._working_param(p), p.numel() + matrix = gathered[:, offset : offset + n].reshape(-1)[: working.numel()].view(working.shape) + offset += n + update = zeropower_via_newtonschulz(matrix, group["ns_coefficients"], group["ns_steps"], group["eps"]) + update = F.pad(update.flatten(), (0, world_size * n - working.numel())).view(world_size, n)[rank] + updates.append(update.to(p.device)) + return updates def _adamw_step(self, group: Dict) -> None: lr, weight_decay, eps = group["lr"], group["weight_decay"], group["eps"] diff --git a/tests/test_optimizer/test_dist_muon.py b/tests/test_optimizer/test_dist_muon.py new file mode 100644 index 000000000000..4b88001084c3 --- /dev/null +++ b/tests/test_optimizer/test_dist_muon.py @@ -0,0 +1,291 @@ +import os +import tempfile +from functools import partial +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F + +import colossalai +from colossalai.booster import Booster +from colossalai.booster.plugin import GeminiPlugin, LowLevelZeroPlugin, TorchFSDPPlugin +from colossalai.logging import disable_existing_loggers +from colossalai.nn.optimizer import Muon +from colossalai.nn.optimizer.muon import zeropower_via_newtonschulz +from colossalai.testing import free_port, rerun_if_address_is_in_use, spawn +from tests.kit.numerics import NumericsSpec, Trace, judge_equal, judge_relative_bound +from tests.kit.numerics.capture.training import prepare, run_steps +from tests.kit.numerics.scenarios import resume + +STEPS, LR = 3, 0.02 +# odd sizes, so that ZeRO pads every matrix and the padding sits inside the last rank's slice +DIM_IN, HIDDEN, DIM_OUT = 31, 63, 17 + + +def requires_gpus(n): + return pytest.mark.skipif(torch.cuda.device_count() < n, reason=f"needs {n} GPUs") + + +def muon_optimizer(params, cls=Muon): + params = list(params) + groups = [ + {"params": [p for p in params if p.ndim == 2], "use_muon": True}, + {"params": [p for p in params if p.ndim != 2], "use_muon": False}, + ] + return cls(groups, lr=LR, weight_decay=0.1) + + +class ShardLocalMuon(Muon): + """A typical sharding bug: each rank orthogonalizes its own slice as a 1 x n matrix.""" + + def _orthogonalize_shards(self, shards, inputs, group): + return [ + zeropower_via_newtonschulz(x.view(1, -1), group["ns_coefficients"], group["ns_steps"], group["eps"]) + .flatten() + .to(p.device) + for p, x in zip(shards, inputs) + ] + + +# --------------------------------------------------------------------------- +# Muon on hand-made ZeRO shards, over gloo: runs without GPUs +# --------------------------------------------------------------------------- + + +def shard_of(t, world_size, rank): + """``rank``'s slice of ``t``, padded and split the way LowLevelZeRO does.""" + flat = t.flatten() + flat = F.pad(flat, (0, (-flat.numel()) % world_size)) + return flat.view(world_size, -1)[rank] + + +def run_shards(rank, world_size, port, optimizer_cls): + # colossalai.launch would pick NCCL on a GPU machine; these shards live on the CPU + os.environ.update(MASTER_ADDR="127.0.0.1", MASTER_PORT=str(port)) + dist.init_process_group("gloo", rank=rank, world_size=world_size) + gen = torch.Generator().manual_seed(0) + shapes = {"tall": (HIDDEN, DIM_OUT), "wide": (DIM_IN, HIDDEN), "square": (48, 48), "bias": (HIDDEN,)} + init = {n: torch.randn(s, generator=gen) for n, s in shapes.items()} + grads = [{n: torch.randn(s, generator=gen) for n, s in shapes.items()} for _ in range(STEPS)] + matrices = [n for n, s in shapes.items() if len(s) == 2] + + def groups(params): + return [ + {"params": [params[n] for n in matrices], "use_muon": True}, + {"params": [params["bias"]], "use_muon": False}, + ] + + whole = {n: nn.Parameter(t.clone()) for n, t in init.items()} + ref = Muon(groups(whole), lr=LR, weight_decay=0.1) + # as with LowLevelZeRO: built on the whole parameters, then handed the shards and told what they belong to + working = {n: nn.Parameter(t.clone()) for n, t in init.items()} + shards = {n: nn.Parameter(shard_of(t, world_size, rank).clone()) for n, t in init.items()} + test = optimizer_cls(groups(working), lr=LR, weight_decay=0.1) + for group, sharded in zip(test.param_groups, groups(shards)): + group["params"] = sharded["params"] + test.setup_distributed(None, None, {id(shards[n]): working[n] for n in shapes}, None, True) + + for step_grads in grads: + for n in shapes: + whole[n].grad = step_grads[n].clone() + shards[n].grad = shard_of(step_grads[n], world_size, rank).clone() + ref.step() + test.step() + + def gathered(t, shape): + parts = [torch.empty_like(t) for _ in range(world_size)] + dist.all_gather(parts, t.detach().contiguous()) + return torch.cat(parts)[: torch.Size(shape).numel()].view(shape) + + # updates, not weights: next to unit-variance weights a wrong update would hide inside any relative bound + result = {f"update/{n}": gathered(shards[n], shapes[n]) - init[n] for n in shapes} + result.update({f"momentum/{n}": gathered(test.state[shards[n]]["momentum_buffer"], shapes[n]) for n in matrices}) + expected = {f"update/{n}": whole[n].detach() - init[n] for n in shapes} + expected.update({f"momentum/{n}": ref.state[whole[n]]["momentum_buffer"] for n in matrices}) + if optimizer_cls is Muon: + # the gathered input is the bf16 cast the whole-matrix step makes, so the results are the same bits + judge_equal(result, expected).assert_pass() + else: + report = judge_relative_bound(result, expected, torch.bfloat16) + assert {e.name for e in report.failures} >= {f"update/{n}" for n in matrices}, report.format() + dist.destroy_process_group() + + +@pytest.mark.parametrize("world_size", [2, 3]) +def test_shards_match_whole_matrices(world_size): + torch.multiprocessing.spawn(run_shards, args=(world_size, free_port(), Muon), nprocs=world_size) + + +def test_shard_local_orthogonalization_detected(): + torch.multiprocessing.spawn(run_shards, args=(2, free_port(), ShardLocalMuon), nprocs=2) + + +def run_setup_checks(rank, world_size, port): + os.environ.update(MASTER_ADDR="127.0.0.1", MASTER_PORT=str(port)) + dist.init_process_group("gloo", rank=rank, world_size=world_size) + weight = nn.Parameter(torch.randn(4, 6)) + optimizer = Muon([weight]) + with pytest.raises(NotImplementedError, match="tensor parallelism"): + optimizer.setup_distributed(tp_group=dist.group.WORLD) + # not one of world_size equal slices, as with expert parameters ZeRO shards over their own group + shard = nn.Parameter(torch.randn(5)) + optimizer.param_groups[0]["params"] = [shard] + with pytest.raises(ValueError, match="do not span the data-parallel group"): + optimizer.setup_distributed(None, None, {id(shard): weight}, None, True) + dist.destroy_process_group() + + +def test_setup_rejects_tensor_parallel_and_foreign_shards(): + torch.multiprocessing.spawn(run_setup_checks, args=(2, free_port()), nprocs=2) + + +# --------------------------------------------------------------------------- +# LowLevelZeRO on GPUs: N ranks against one rank of the same plugin +# --------------------------------------------------------------------------- + + +class MLP(nn.Module): + def __init__(self): + super().__init__() + self.fc1 = nn.Linear(DIM_IN, HIDDEN) + self.norm = nn.LayerNorm(HIDDEN) + self.fc2 = nn.Linear(HIDDEN, DIM_OUT) + + def forward(self, x): + return {"y": self.fc2(self.norm(F.gelu(self.fc1(x))))} + + +def _data(): + return {"x": torch.randn(8, DIM_IN)} + + +def _identity(output): + return output + + +# module-level parts only: the entry is sent to spawned processes +ENTRY = (MLP, _data, _identity, None, SimpleNamespace(numerics=NumericsSpec())) + + +def run_zero(rank, world_size, port, plugin_fn, optimizer_fn, path): + disable_existing_loggers() + colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) + # every rank trains on the same batch: averaging N equal gradients is exact for N = 2, so only sharding differs + trace = run_steps(prepare(ENTRY, STEPS), plugin_fn(), 0, STEPS, optimizer_fn=optimizer_fn) + if rank == 0: + trace.save(path) + dist.destroy_process_group() + + +def zero_trace(plugin_fn, world_size, optimizer_fn=muon_optimizer): + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "trace.safetensors") + spawn(run_zero, world_size, plugin_fn=plugin_fn, optimizer_fn=optimizer_fn, path=path) + return Trace.load(path) + + +def comparable(test, world_size, ref): + """Losses, weights and the optimizer's state (float32 master weights, momentum, AdamW moments) of both runs. + + The optimizer's tensors are recorded per rank in ZeRO's flat layout; the slices of ``test`` are joined and the + padding cut to the length of ``ref``'s, which one rank holds unpadded.""" + keep = lambda k: k.startswith(("loss/", "param/")) # noqa: E731 + t, r = {k: v for k, v in test.items() if keep(k)}, {k: v for k, v in ref.items() if keep(k)} + for key, value in ref.items(): + if key.startswith("optim/"): + parts = [test[f"rank{i}/{key}"].flatten() for i in range(world_size)] + r[key] = value.flatten() + t[key] = torch.cat(parts)[: r[key].numel()] + return t, r + + +def updates(state, precision): + """The master weights' change over the run instead of the weights, plus the optimizer's other state. + + For a relative bound: three steps at lr = 0.02 change these weights by a few percent, so a wrong update would stay + within the bound if the weights themselves were compared.""" + params = [p for p in prepare(ENTRY, STEPS).model.parameters() if p.requires_grad] + groups = [[p for p in params if p.ndim == 2], [p for p in params if p.ndim != 2]] # as muon_optimizer builds them + # the plugin casts the model to its precision before ZeRO copies the float32 master weights + dtype = torch.bfloat16 if precision == "bf16" else torch.float32 + start = { + f"optim/{g}.{i}/param": p.detach().to(dtype).float().flatten() + for g, ps in enumerate(groups) + for i, p in enumerate(ps) + } + out = {} + for key, value in state.items(): + if key in start: + out[f"update/{key}"] = value - start[key] + elif key.startswith("optim/"): + out[key] = value + return out + + +ZERO_CONFIGS = [(1, "fp32", False), (2, "fp32", False), (1, "bf16", False), (2, "bf16", False), (2, "bf16", True)] + + +@pytest.mark.dist +@requires_gpus(2) +@pytest.mark.parametrize("stage, precision, cpu_offload", ZERO_CONFIGS) +@rerun_if_address_is_in_use() +def test_zero_two_ranks_equal_one(stage, precision, cpu_offload): + plugin_fn = partial(LowLevelZeroPlugin, stage=stage, precision=precision, cpu_offload=cpu_offload) + test, ref = comparable(zero_trace(plugin_fn, 2), 2, zero_trace(plugin_fn, 1)) + judge_equal(test, ref).assert_pass() + + +@pytest.mark.dist +@requires_gpus(4) +@rerun_if_address_is_in_use() +def test_zero_four_ranks_within_bound(): + # averaging 4 equal gradients rounds, so the runs differ in the last bits: Megatron-LM's relative bound + plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="bf16") + test, ref = comparable(zero_trace(plugin_fn, 4), 4, zero_trace(plugin_fn, 1)) + judge_relative_bound(updates(test, "bf16"), updates(ref, "bf16"), torch.bfloat16).assert_pass() + + +@pytest.mark.dist +@requires_gpus(2) +@rerun_if_address_is_in_use() +def test_zero_shard_local_orthogonalization_detected(): + plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="bf16") + shard_local = partial(muon_optimizer, cls=ShardLocalMuon) + test, ref = comparable(zero_trace(plugin_fn, 2, shard_local), 2, zero_trace(plugin_fn, 1)) + report = judge_relative_bound(updates(test, "bf16"), updates(ref, "bf16"), torch.bfloat16) + assert {e.name for e in report.failures} >= {"update/optim/0.0/param", "update/optim/0.1/param"}, report.format() + + +@pytest.mark.dist +@requires_gpus(2) +@rerun_if_address_is_in_use() +def test_zero_resume_is_exact(): + # fp32: in bf16 no optimizer resumes exactly, as LowLevelZero rebuilds the float32 master weights from the bf16 + # ones on load (tests/kit/numerics/README.md, "Resuming from a checkpoint") + plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="fp32") + resume.check(ENTRY, plugin_fn, world_size=2, optimizer_fn=muon_optimizer).assert_pass() + + +def run_plugins(rank, world_size, port): + disable_existing_loggers() + colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) + for plugin in (GeminiPlugin(), TorchFSDPPlugin()): + with pytest.raises(NotImplementedError, match="does not support Muon"): + model = MLP().cuda() + Booster(plugin=plugin).boost(model, muon_optimizer(model.parameters())) + model = MLP().cuda() + muon = muon_optimizer(model.parameters()) + _, optimizer, *_ = Booster(plugin=LowLevelZeroPlugin(stage=1)).boost(model, muon) + # set up in place, not replaced: an LR scheduler built on `muon` before boost keeps driving it + assert optimizer.optim is muon and muon.shard_to_working_param + dist.destroy_process_group() + + +@pytest.mark.dist +@requires_gpus(1) +@rerun_if_address_is_in_use() +def test_plugins_with_muon(): + spawn(run_plugins, 1)