diff --git a/.github/scripts/ci/run_cpu_unit_tests.sh b/.github/scripts/ci/run_cpu_unit_tests.sh index 8916af3d83f4..68bbeaed221b 100644 --- a/.github/scripts/ci/run_cpu_unit_tests.sh +++ b/.github/scripts/ci/run_cpu_unit_tests.sh @@ -19,6 +19,7 @@ TESTS=( tests/test_numerics/test_numerics_data_parallel.py tests/test_numerics/test_numerics_independent_impl.py tests/test_numerics/test_numerics_resume.py + tests/test_optimizer/test_dist_muon.py tests/test_optimizer/test_lr_scheduler.py tests/test_optimizer/test_muon.py tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py diff --git a/colossalai/booster/plugin/gemini_plugin.py b/colossalai/booster/plugin/gemini_plugin.py index 0d8846e07b30..7a189e17f494 100644 --- a/colossalai/booster/plugin/gemini_plugin.py +++ b/colossalai/booster/plugin/gemini_plugin.py @@ -31,6 +31,7 @@ from colossalai.cluster import DistCoordinator, ProcessGroupMesh from colossalai.interface import ModelWrapper, OptimizerWrapper from colossalai.logging import get_dist_logger +from colossalai.nn.optimizer import Muon from colossalai.shardformer import ShardConfig, ShardFormer from colossalai.zero import GeminiDDP, GeminiOptimizer from colossalai.zero.gemini.memory_tracer import MemStats @@ -665,6 +666,11 @@ def configure( dataloader: Optional[DataLoader] = None, lr_scheduler: Optional[LRScheduler] = None, ) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]: + if isinstance(optimizer, Muon): + raise NotImplementedError( + "GeminiPlugin does not support Muon: its chunks split matrices at arbitrary boundaries. " + "Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." + ) params_info = get_param_info(optimizer) if not isinstance(model, ModelWrapper): # convert model to sync bn diff --git a/colossalai/booster/plugin/torch_fsdp_plugin.py b/colossalai/booster/plugin/torch_fsdp_plugin.py index 05cc432a673d..5c9390539972 100644 --- a/colossalai/booster/plugin/torch_fsdp_plugin.py +++ b/colossalai/booster/plugin/torch_fsdp_plugin.py @@ -34,6 +34,7 @@ from colossalai.cluster import DistCoordinator from colossalai.interface import ModelWrapper, OptimizerWrapper from colossalai.logging import get_dist_logger +from colossalai.nn.optimizer import Muon from colossalai.utils.safetensors import load_flat from .dp_plugin_base import DPPluginBase @@ -536,6 +537,11 @@ def configure( dataloader: Optional[DataLoader] = None, lr_scheduler: Optional[LRScheduler] = None, ) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]: + if isinstance(optimizer, Muon): + raise NotImplementedError( + "TorchFSDPPlugin does not support Muon: FSDP hands the optimizer flattened parameters in one group. " + "Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin." + ) # wrap the model with PyTorch FSDP fsdp_model = TorchFSDPModel(model, device_id=torch.cuda.current_device(), **self.fsdp_kwargs) diff --git a/colossalai/nn/optimizer/muon.py b/colossalai/nn/optimizer/muon.py index 67dfeebb4536..bd6de205de84 100644 --- a/colossalai/nn/optimizer/muon.py +++ b/colossalai/nn/optimizer/muon.py @@ -5,6 +5,13 @@ norms, biases) go to the AdamW branch of the same optimizer, as in KellerJordan/Muon's ``MuonWithAuxAdam`` and Moonlight, so a Booster still receives a single optimizer. +Under LowLevelZeRO (stage 1 and 2) each rank holds a flat 1/N slice of every parameter, while Newton-Schulz needs whole +matrices. As in Moonlight's distributed Muon, each rank updates the momentum of its own slices, the bf16 +Newton-Schulz inputs are all-gathered over the data-parallel group, and every rank orthogonalizes the whole matrices +and keeps the slice of each update it owns. The gathered input is the bf16 cast the unsharded iteration makes anyway, +so a sharded step computes the unsharded one, at the cost of one bf16 all-gather of the Muon parameters per step and +every rank orthogonalizing all of them. + References: - https://kellerjordan.github.io/posts/muon/ - Muon is Scalable for LLM Training, https://arxiv.org/abs/2502.16982 @@ -14,9 +21,13 @@ from typing import Dict, List, Optional, Tuple import torch +import torch.distributed as dist import torch.nn as nn +import torch.nn.functional as F from torch import Tensor -from torch.optim import Optimizer + +from colossalai.accelerator import get_accelerator +from colossalai.interface.optimizer import DistributedOptim __all__ = ["Muon", "muon_param_groups", "zeropower_via_newtonschulz"] @@ -63,7 +74,7 @@ def _adjust_lr(lr: float, adjust_lr_fn: Optional[str], shape: torch.Size) -> flo return lr * 0.2 * math.sqrt(max(rows, cols)) -class Muon(Optimizer): +class Muon(DistributedOptim): """Muon for 2D hidden-layer weights plus AdamW for every other parameter, in one optimizer. Each parameter group sets ``use_muon``; ``muon_param_groups(model)`` builds both groups from a model. Groups @@ -72,6 +83,11 @@ class Muon(Optimizer): ``betas`` and ``eps``. Missing keys fall back to the constructor arguments, with ``adamw_eps`` as the AdamW groups' ``eps``. + ``LowLevelZeroPlugin`` and ``HybridParallelPlugin`` call ``setup_distributed`` on this very object (it is not + replaced, so an LR scheduler built on it before ``boost`` keeps driving it), after which ZeRO's parameter shards are + orthogonalized as whole matrices (see the module docstring). The AdamW branch is elementwise and runs on the + shards as they are. Tensor parallelism, Gemini and FSDP are not supported. + Args: params: parameters or parameter groups. lr (float): learning rate shared by both branches unless a group overrides it (default: 1e-3). @@ -88,6 +104,10 @@ class Muon(Optimizer): adamw_eps (float): AdamW epsilon (default: 1e-8). """ + # Under ZeRO at most this many elements of whole matrices are gathered at once (512 MiB in bf16), which bounds the + # extra memory of a step. + bucket_numel = 1 << 28 + def __init__( self, params, @@ -122,6 +142,8 @@ def __init__( ) self._adamw_defaults = dict(betas=tuple(betas), eps=adamw_eps) super().__init__(params, dict(lr=lr, weight_decay=weight_decay)) + self.dp_group: Optional[dist.ProcessGroup] = None + self.shard_to_working_param: Dict[int, Tensor] = {} def add_param_group(self, param_group: Dict) -> None: use_muon = param_group.setdefault("use_muon", True) @@ -142,6 +164,45 @@ def add_param_group(self, param_group: Dict) -> None: ) super().add_param_group(param_group) + def setup_distributed( + self, + tp_group: Optional[dist.ProcessGroup] = None, + dp_group: Optional[dist.ProcessGroup] = None, + shard_to_working_param: Optional[Dict] = None, + padding_map: Optional[Dict] = None, + is_zero: Optional[bool] = False, + ): + """Record the data-parallel group and, for ZeRO, which whole parameter each shard belongs to. + + Arguments: + tp_group (dist.ProcessGroup): tensor-parallel group; only a group of size 1 is supported. + dp_group (dist.ProcessGroup): the group ZeRO shards over (default: the whole world). + shard_to_working_param (Dict): id of each shard to the parameter used in forward and backward. + padding_map (Dict): unused; the padding follows from the parameter's size and the group size. + is_zero (bool): unused; shards are recognized by their shape. + """ + if tp_group is not None and dist.get_world_size(tp_group) > 1: + raise NotImplementedError("Muon does not support tensor parallelism yet; use tp_size=1") + self.dp_group = dp_group + self.shard_to_working_param = dict(shard_to_working_param or {}) + world_size = dist.get_world_size(dp_group) + for group in self.param_groups: + if not group["use_muon"]: + continue + for p in group["params"]: + working = self._working_param(p) + if working.shape == p.shape: + continue + if working.ndim != 2 or p.numel() != -(-working.numel() // world_size): + raise ValueError( + f"a {tuple(working.shape)} Muon parameter arrives as a shard of {p.numel()} elements, not one " + f"of {world_size} equal slices: its shards do not span the data-parallel group (e.g. expert " + "parameters sharded over their own group), which Muon does not support" + ) + + def _working_param(self, p: Tensor) -> Tensor: + return self.shard_to_working_param.get(id(p), p) + @torch.no_grad() def step(self, closure=None): loss = None @@ -157,26 +218,82 @@ def step(self, closure=None): return loss def _muon_step(self, group: Dict) -> None: - lr, weight_decay, momentum = group["lr"], group["weight_decay"], group["momentum"] + shards = [] for p in group["params"]: if p.grad is None: continue - grad = p.grad - if grad.ndim != 2: - # A sharded 1D view, e.g. from LowLevelZeRO's master parameter slices. - raise RuntimeError( - f"Muon needs the full 2D gradient but got shape {tuple(grad.shape)}; " - "ZeRO / Gemini / FSDP sharding is not supported yet" - ) - state = self.state[p] - if "momentum_buffer" not in state: - state["momentum_buffer"] = torch.zeros_like(grad, memory_format=torch.preserve_format) - buf = state["momentum_buffer"] - buf.lerp_(grad, 1 - momentum) - update = grad.lerp(buf, momentum) if group["nesterov"] else buf - update = zeropower_via_newtonschulz(update, group["ns_coefficients"], group["ns_steps"], group["eps"]) - p.mul_(1 - lr * weight_decay) - p.add_(update, alpha=-_adjust_lr(lr, group["adjust_lr_fn"], p.shape)) + if self._working_param(p).shape == p.shape: + self._step_matrix(p, group) + else: + shards.append(p) + # every rank holds the same shards in the same order, so the gathers line up + for bucket in self._buckets(shards): + inputs = [self._momentum_update(p, group) for p in bucket] + for p, update in zip(bucket, self._orthogonalize_shards(bucket, inputs, group)): + self._apply_update(p, update, self._working_param(p).shape, group) + + def _step_matrix(self, p: Tensor, group: Dict) -> None: + if p.grad.ndim != 2: + # A sharded 1D view that setup_distributed was not told about. + raise RuntimeError( + f"Muon needs the full 2D gradient but got shape {tuple(p.grad.shape)}; under ZeRO the plugin has to " + "call setup_distributed (LowLevelZeroPlugin does), and Gemini / FSDP are not supported" + ) + update = zeropower_via_newtonschulz( + self._momentum_update(p, group), group["ns_coefficients"], group["ns_steps"], group["eps"] + ) + self._apply_update(p, update, p.shape, group) + + def _momentum_update(self, p: Tensor, group: Dict) -> Tensor: + """Update ``p``'s momentum buffer and return the Newton-Schulz input; elementwise, so shards work too.""" + grad, momentum = p.grad, group["momentum"] + state = self.state[p] + if "momentum_buffer" not in state: + state["momentum_buffer"] = torch.zeros_like(grad, memory_format=torch.preserve_format) + buf = state["momentum_buffer"] + buf.lerp_(grad, 1 - momentum) + return grad.lerp(buf, momentum) if group["nesterov"] else buf + + @staticmethod + def _apply_update(p: Tensor, update: Tensor, shape: torch.Size, group: Dict) -> None: + """Apply weight decay and the orthogonalized ``update``; ``shape`` is the whole matrix's, for the lr scale.""" + lr = group["lr"] + p.mul_(1 - lr * group["weight_decay"]) + p.add_(update, alpha=-_adjust_lr(lr, group["adjust_lr_fn"], shape)) + + def _buckets(self, shards: List[Tensor]) -> List[List[Tensor]]: + buckets, numel = [], 0 + for p in shards: + size = self._working_param(p).numel() + if not buckets or numel + size > self.bucket_numel: + buckets.append([]) + numel = 0 + buckets[-1].append(p) + numel += size + return buckets + + def _orthogonalize_shards(self, shards: List[Tensor], inputs: List[Tensor], group: Dict) -> List[Tensor]: + """Gather the whole Newton-Schulz inputs of ``shards``, orthogonalize them and return each rank's slices.""" + world_size, rank = dist.get_world_size(self.dp_group), dist.get_rank(self.dp_group) + # gloo gathers CPU tensors; NCCL (and Newton-Schulz) want the accelerator + if dist.get_backend(self.dp_group) == dist.Backend.GLOO: + device = torch.device("cpu") + else: + device = get_accelerator().get_current_device() + local = torch.cat([x.to(device=device, dtype=torch.bfloat16).flatten() for x in inputs]) + gathered = torch.empty(world_size * local.numel(), dtype=local.dtype, device=device) + dist.all_gather_into_tensor(gathered, local, group=self.dp_group) + # row r holds rank r's slices of every matrix, one after another + gathered = gathered.view(world_size, local.numel()) + updates, offset = [], 0 + for p in shards: + working, n = self._working_param(p), p.numel() + matrix = gathered[:, offset : offset + n].reshape(-1)[: working.numel()].view(working.shape) + offset += n + update = zeropower_via_newtonschulz(matrix, group["ns_coefficients"], group["ns_steps"], group["eps"]) + update = F.pad(update.flatten(), (0, world_size * n - working.numel())).view(world_size, n)[rank] + updates.append(update.to(p.device)) + return updates def _adamw_step(self, group: Dict) -> None: lr, weight_decay, eps = group["lr"], group["weight_decay"], group["eps"] diff --git a/tests/test_optimizer/test_dist_muon.py b/tests/test_optimizer/test_dist_muon.py new file mode 100644 index 000000000000..4b88001084c3 --- /dev/null +++ b/tests/test_optimizer/test_dist_muon.py @@ -0,0 +1,291 @@ +import os +import tempfile +from functools import partial +from types import SimpleNamespace + +import pytest +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F + +import colossalai +from colossalai.booster import Booster +from colossalai.booster.plugin import GeminiPlugin, LowLevelZeroPlugin, TorchFSDPPlugin +from colossalai.logging import disable_existing_loggers +from colossalai.nn.optimizer import Muon +from colossalai.nn.optimizer.muon import zeropower_via_newtonschulz +from colossalai.testing import free_port, rerun_if_address_is_in_use, spawn +from tests.kit.numerics import NumericsSpec, Trace, judge_equal, judge_relative_bound +from tests.kit.numerics.capture.training import prepare, run_steps +from tests.kit.numerics.scenarios import resume + +STEPS, LR = 3, 0.02 +# odd sizes, so that ZeRO pads every matrix and the padding sits inside the last rank's slice +DIM_IN, HIDDEN, DIM_OUT = 31, 63, 17 + + +def requires_gpus(n): + return pytest.mark.skipif(torch.cuda.device_count() < n, reason=f"needs {n} GPUs") + + +def muon_optimizer(params, cls=Muon): + params = list(params) + groups = [ + {"params": [p for p in params if p.ndim == 2], "use_muon": True}, + {"params": [p for p in params if p.ndim != 2], "use_muon": False}, + ] + return cls(groups, lr=LR, weight_decay=0.1) + + +class ShardLocalMuon(Muon): + """A typical sharding bug: each rank orthogonalizes its own slice as a 1 x n matrix.""" + + def _orthogonalize_shards(self, shards, inputs, group): + return [ + zeropower_via_newtonschulz(x.view(1, -1), group["ns_coefficients"], group["ns_steps"], group["eps"]) + .flatten() + .to(p.device) + for p, x in zip(shards, inputs) + ] + + +# --------------------------------------------------------------------------- +# Muon on hand-made ZeRO shards, over gloo: runs without GPUs +# --------------------------------------------------------------------------- + + +def shard_of(t, world_size, rank): + """``rank``'s slice of ``t``, padded and split the way LowLevelZeRO does.""" + flat = t.flatten() + flat = F.pad(flat, (0, (-flat.numel()) % world_size)) + return flat.view(world_size, -1)[rank] + + +def run_shards(rank, world_size, port, optimizer_cls): + # colossalai.launch would pick NCCL on a GPU machine; these shards live on the CPU + os.environ.update(MASTER_ADDR="127.0.0.1", MASTER_PORT=str(port)) + dist.init_process_group("gloo", rank=rank, world_size=world_size) + gen = torch.Generator().manual_seed(0) + shapes = {"tall": (HIDDEN, DIM_OUT), "wide": (DIM_IN, HIDDEN), "square": (48, 48), "bias": (HIDDEN,)} + init = {n: torch.randn(s, generator=gen) for n, s in shapes.items()} + grads = [{n: torch.randn(s, generator=gen) for n, s in shapes.items()} for _ in range(STEPS)] + matrices = [n for n, s in shapes.items() if len(s) == 2] + + def groups(params): + return [ + {"params": [params[n] for n in matrices], "use_muon": True}, + {"params": [params["bias"]], "use_muon": False}, + ] + + whole = {n: nn.Parameter(t.clone()) for n, t in init.items()} + ref = Muon(groups(whole), lr=LR, weight_decay=0.1) + # as with LowLevelZeRO: built on the whole parameters, then handed the shards and told what they belong to + working = {n: nn.Parameter(t.clone()) for n, t in init.items()} + shards = {n: nn.Parameter(shard_of(t, world_size, rank).clone()) for n, t in init.items()} + test = optimizer_cls(groups(working), lr=LR, weight_decay=0.1) + for group, sharded in zip(test.param_groups, groups(shards)): + group["params"] = sharded["params"] + test.setup_distributed(None, None, {id(shards[n]): working[n] for n in shapes}, None, True) + + for step_grads in grads: + for n in shapes: + whole[n].grad = step_grads[n].clone() + shards[n].grad = shard_of(step_grads[n], world_size, rank).clone() + ref.step() + test.step() + + def gathered(t, shape): + parts = [torch.empty_like(t) for _ in range(world_size)] + dist.all_gather(parts, t.detach().contiguous()) + return torch.cat(parts)[: torch.Size(shape).numel()].view(shape) + + # updates, not weights: next to unit-variance weights a wrong update would hide inside any relative bound + result = {f"update/{n}": gathered(shards[n], shapes[n]) - init[n] for n in shapes} + result.update({f"momentum/{n}": gathered(test.state[shards[n]]["momentum_buffer"], shapes[n]) for n in matrices}) + expected = {f"update/{n}": whole[n].detach() - init[n] for n in shapes} + expected.update({f"momentum/{n}": ref.state[whole[n]]["momentum_buffer"] for n in matrices}) + if optimizer_cls is Muon: + # the gathered input is the bf16 cast the whole-matrix step makes, so the results are the same bits + judge_equal(result, expected).assert_pass() + else: + report = judge_relative_bound(result, expected, torch.bfloat16) + assert {e.name for e in report.failures} >= {f"update/{n}" for n in matrices}, report.format() + dist.destroy_process_group() + + +@pytest.mark.parametrize("world_size", [2, 3]) +def test_shards_match_whole_matrices(world_size): + torch.multiprocessing.spawn(run_shards, args=(world_size, free_port(), Muon), nprocs=world_size) + + +def test_shard_local_orthogonalization_detected(): + torch.multiprocessing.spawn(run_shards, args=(2, free_port(), ShardLocalMuon), nprocs=2) + + +def run_setup_checks(rank, world_size, port): + os.environ.update(MASTER_ADDR="127.0.0.1", MASTER_PORT=str(port)) + dist.init_process_group("gloo", rank=rank, world_size=world_size) + weight = nn.Parameter(torch.randn(4, 6)) + optimizer = Muon([weight]) + with pytest.raises(NotImplementedError, match="tensor parallelism"): + optimizer.setup_distributed(tp_group=dist.group.WORLD) + # not one of world_size equal slices, as with expert parameters ZeRO shards over their own group + shard = nn.Parameter(torch.randn(5)) + optimizer.param_groups[0]["params"] = [shard] + with pytest.raises(ValueError, match="do not span the data-parallel group"): + optimizer.setup_distributed(None, None, {id(shard): weight}, None, True) + dist.destroy_process_group() + + +def test_setup_rejects_tensor_parallel_and_foreign_shards(): + torch.multiprocessing.spawn(run_setup_checks, args=(2, free_port()), nprocs=2) + + +# --------------------------------------------------------------------------- +# LowLevelZeRO on GPUs: N ranks against one rank of the same plugin +# --------------------------------------------------------------------------- + + +class MLP(nn.Module): + def __init__(self): + super().__init__() + self.fc1 = nn.Linear(DIM_IN, HIDDEN) + self.norm = nn.LayerNorm(HIDDEN) + self.fc2 = nn.Linear(HIDDEN, DIM_OUT) + + def forward(self, x): + return {"y": self.fc2(self.norm(F.gelu(self.fc1(x))))} + + +def _data(): + return {"x": torch.randn(8, DIM_IN)} + + +def _identity(output): + return output + + +# module-level parts only: the entry is sent to spawned processes +ENTRY = (MLP, _data, _identity, None, SimpleNamespace(numerics=NumericsSpec())) + + +def run_zero(rank, world_size, port, plugin_fn, optimizer_fn, path): + disable_existing_loggers() + colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) + # every rank trains on the same batch: averaging N equal gradients is exact for N = 2, so only sharding differs + trace = run_steps(prepare(ENTRY, STEPS), plugin_fn(), 0, STEPS, optimizer_fn=optimizer_fn) + if rank == 0: + trace.save(path) + dist.destroy_process_group() + + +def zero_trace(plugin_fn, world_size, optimizer_fn=muon_optimizer): + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "trace.safetensors") + spawn(run_zero, world_size, plugin_fn=plugin_fn, optimizer_fn=optimizer_fn, path=path) + return Trace.load(path) + + +def comparable(test, world_size, ref): + """Losses, weights and the optimizer's state (float32 master weights, momentum, AdamW moments) of both runs. + + The optimizer's tensors are recorded per rank in ZeRO's flat layout; the slices of ``test`` are joined and the + padding cut to the length of ``ref``'s, which one rank holds unpadded.""" + keep = lambda k: k.startswith(("loss/", "param/")) # noqa: E731 + t, r = {k: v for k, v in test.items() if keep(k)}, {k: v for k, v in ref.items() if keep(k)} + for key, value in ref.items(): + if key.startswith("optim/"): + parts = [test[f"rank{i}/{key}"].flatten() for i in range(world_size)] + r[key] = value.flatten() + t[key] = torch.cat(parts)[: r[key].numel()] + return t, r + + +def updates(state, precision): + """The master weights' change over the run instead of the weights, plus the optimizer's other state. + + For a relative bound: three steps at lr = 0.02 change these weights by a few percent, so a wrong update would stay + within the bound if the weights themselves were compared.""" + params = [p for p in prepare(ENTRY, STEPS).model.parameters() if p.requires_grad] + groups = [[p for p in params if p.ndim == 2], [p for p in params if p.ndim != 2]] # as muon_optimizer builds them + # the plugin casts the model to its precision before ZeRO copies the float32 master weights + dtype = torch.bfloat16 if precision == "bf16" else torch.float32 + start = { + f"optim/{g}.{i}/param": p.detach().to(dtype).float().flatten() + for g, ps in enumerate(groups) + for i, p in enumerate(ps) + } + out = {} + for key, value in state.items(): + if key in start: + out[f"update/{key}"] = value - start[key] + elif key.startswith("optim/"): + out[key] = value + return out + + +ZERO_CONFIGS = [(1, "fp32", False), (2, "fp32", False), (1, "bf16", False), (2, "bf16", False), (2, "bf16", True)] + + +@pytest.mark.dist +@requires_gpus(2) +@pytest.mark.parametrize("stage, precision, cpu_offload", ZERO_CONFIGS) +@rerun_if_address_is_in_use() +def test_zero_two_ranks_equal_one(stage, precision, cpu_offload): + plugin_fn = partial(LowLevelZeroPlugin, stage=stage, precision=precision, cpu_offload=cpu_offload) + test, ref = comparable(zero_trace(plugin_fn, 2), 2, zero_trace(plugin_fn, 1)) + judge_equal(test, ref).assert_pass() + + +@pytest.mark.dist +@requires_gpus(4) +@rerun_if_address_is_in_use() +def test_zero_four_ranks_within_bound(): + # averaging 4 equal gradients rounds, so the runs differ in the last bits: Megatron-LM's relative bound + plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="bf16") + test, ref = comparable(zero_trace(plugin_fn, 4), 4, zero_trace(plugin_fn, 1)) + judge_relative_bound(updates(test, "bf16"), updates(ref, "bf16"), torch.bfloat16).assert_pass() + + +@pytest.mark.dist +@requires_gpus(2) +@rerun_if_address_is_in_use() +def test_zero_shard_local_orthogonalization_detected(): + plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="bf16") + shard_local = partial(muon_optimizer, cls=ShardLocalMuon) + test, ref = comparable(zero_trace(plugin_fn, 2, shard_local), 2, zero_trace(plugin_fn, 1)) + report = judge_relative_bound(updates(test, "bf16"), updates(ref, "bf16"), torch.bfloat16) + assert {e.name for e in report.failures} >= {"update/optim/0.0/param", "update/optim/0.1/param"}, report.format() + + +@pytest.mark.dist +@requires_gpus(2) +@rerun_if_address_is_in_use() +def test_zero_resume_is_exact(): + # fp32: in bf16 no optimizer resumes exactly, as LowLevelZero rebuilds the float32 master weights from the bf16 + # ones on load (tests/kit/numerics/README.md, "Resuming from a checkpoint") + plugin_fn = partial(LowLevelZeroPlugin, stage=2, precision="fp32") + resume.check(ENTRY, plugin_fn, world_size=2, optimizer_fn=muon_optimizer).assert_pass() + + +def run_plugins(rank, world_size, port): + disable_existing_loggers() + colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) + for plugin in (GeminiPlugin(), TorchFSDPPlugin()): + with pytest.raises(NotImplementedError, match="does not support Muon"): + model = MLP().cuda() + Booster(plugin=plugin).boost(model, muon_optimizer(model.parameters())) + model = MLP().cuda() + muon = muon_optimizer(model.parameters()) + _, optimizer, *_ = Booster(plugin=LowLevelZeroPlugin(stage=1)).boost(model, muon) + # set up in place, not replaced: an LR scheduler built on `muon` before boost keeps driving it + assert optimizer.optim is muon and muon.shard_to_working_param + dist.destroy_process_group() + + +@pytest.mark.dist +@requires_gpus(1) +@rerun_if_address_is_in_use() +def test_plugins_with_muon(): + spawn(run_plugins, 1)