Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/scripts/ci/run_cpu_unit_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@ TESTS=(
tests/test_numerics/test_numerics_data_parallel.py
tests/test_numerics/test_numerics_independent_impl.py
tests/test_numerics/test_numerics_resume.py
tests/test_optimizer/test_dist_muon.py
tests/test_optimizer/test_lr_scheduler.py
tests/test_optimizer/test_muon.py
tests/test_pipeline/test_pipeline_utils/test_t5_pipeline_utils.py
tests/test_pipeline/test_pipeline_utils/test_whisper_pipeline_utils.py
tests/test_pipeline/test_schedule/test_pipeline_schedule_utils.py
Expand Down
6 changes: 6 additions & 0 deletions colossalai/booster/plugin/gemini_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from colossalai.cluster import DistCoordinator, ProcessGroupMesh
from colossalai.interface import ModelWrapper, OptimizerWrapper
from colossalai.logging import get_dist_logger
from colossalai.nn.optimizer import Muon
from colossalai.shardformer import ShardConfig, ShardFormer
from colossalai.zero import GeminiDDP, GeminiOptimizer
from colossalai.zero.gemini.memory_tracer import MemStats
Expand Down Expand Up @@ -665,6 +666,11 @@ def configure(
dataloader: Optional[DataLoader] = None,
lr_scheduler: Optional[LRScheduler] = None,
) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]:
if isinstance(optimizer, Muon):
raise NotImplementedError(
"GeminiPlugin does not support Muon: its chunks split matrices at arbitrary boundaries. "
"Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin."
)
params_info = get_param_info(optimizer)
if not isinstance(model, ModelWrapper):
# convert model to sync bn
Expand Down
6 changes: 6 additions & 0 deletions colossalai/booster/plugin/torch_fsdp_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
from colossalai.cluster import DistCoordinator
from colossalai.interface import ModelWrapper, OptimizerWrapper
from colossalai.logging import get_dist_logger
from colossalai.nn.optimizer import Muon
from colossalai.utils.safetensors import load_flat

from .dp_plugin_base import DPPluginBase
Expand Down Expand Up @@ -536,6 +537,11 @@ def configure(
dataloader: Optional[DataLoader] = None,
lr_scheduler: Optional[LRScheduler] = None,
) -> Tuple[nn.Module, OptimizerWrapper, Callable, DataLoader, LRScheduler]:
if isinstance(optimizer, Muon):
raise NotImplementedError(
"TorchFSDPPlugin does not support Muon: FSDP hands the optimizer flattened parameters in one group. "
"Use LowLevelZeroPlugin (ZeRO stage 1 or 2) or TorchDDPPlugin."
)
# wrap the model with PyTorch FSDP
fsdp_model = TorchFSDPModel(model, device_id=torch.cuda.current_device(), **self.fsdp_kwargs)

Expand Down
3 changes: 3 additions & 0 deletions colossalai/nn/optimizer/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from .hybrid_adam import HybridAdam
from .lamb import Lamb
from .lars import Lars
from .muon import Muon, muon_param_groups

from .adafactor import Adafactor # noqa

Expand All @@ -36,6 +37,8 @@
"DistributedCAME",
"Adafactor",
"DistributedAdaFactor",
"Muon",
"muon_param_groups",
]

optim2DistOptim = {
Expand Down
Loading
Loading