Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions tests/kit/benchmark/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,14 @@ tokenized and packed to the fixed shape: a MoE router needs real tokens) and, fo
`LatentVideo`, `LatentTokens` and `flow_matching_loss`; a model whose sequence is set by its frames and resolution
declares `searchable=("micro_batch",)`.

`batch_fn(model, micro_batch, seq_len, generator)` builds the batch at the size it is given, never a fixed one: the
maximum search changes `micro_batch` or `seq_len` from call to call. It takes every random number from `generator`
(e.g. `torch.randn(shape, generator=generator)`), never from the global random state: each rank's generator is seeded
differently, so the ranks of a multi-device measurement train different data of the same shape, as in training. With
the global state every rank gets the same batch, which does not change a dense model's speed but makes a MoE router
send the same tokens to the same experts on every rank. These functions are part of the entry, so they go into the
same pull request; one that several models can use goes into `config/data.py`.

A model that has no model_zoo entry yet can be measured from `entries.py` (benchmark-only
entries, looked up after model_zoo): `qwen3_moe` (Qwen3-30B-A3B) and `wan2_2_ti2v` (the Wan2.2-TI2V-5B transformer at
480×832, 81 frames). Their specs move into model_zoo when the models get their entries; they are kept out of it until
Expand Down
46 changes: 26 additions & 20 deletions tests/kit/benchmark/runner/measure.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@
import torch.distributed as dist
from torch.multiprocessing.spawn import ProcessExitedException, ProcessRaisedException

import colossalai
from colossalai.accelerator import get_accelerator
from colossalai.booster import Booster
from colossalai.testing import spawn
from tests.kit.numerics.capture.plugins import make_optimizer
from tests.kit.numerics.scenarios._distributed import worker_process_group

from ..config.model import PRETRAINED, BenchmarkSpec, spec_of
from ..config.optimizers import build_optimizer, for_cpu
Expand Down Expand Up @@ -90,25 +90,31 @@ def measure(setup: Setup, spec: Optional[BenchmarkSpec] = None) -> Dict[str, Any


def _worker(rank, world_size, port, setup, spec, path):
colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port)
spec = spec or spec_of(setup.entry)
used_at_start = device_used()
record = _train(spec, setup)
record["device_used_at_start"] = used_at_start
ranks = [None] * world_size
# objects, not tensors: works with every backend (gloo, NCCL, HCCL) and keeps Python ints and floats exact
dist.all_gather_object(ranks, record)
if rank == 0:
shared = {key: ranks[0].pop(key) for key in ("parameters", "optimizer")}
for other in ranks[1:]:
for key in shared:
other.pop(key)
with open(path, "w") as f:
json.dump(
{"status": COMPLETED, "setup": asdict(setup), **shared, "environment": environment(), "ranks": ranks},
f,
)
dist.destroy_process_group()
# releases the process group on the way out, also when training raises (see the numerics kit)
with worker_process_group(rank, world_size, port):
spec = spec or spec_of(setup.entry)
used_at_start = device_used()
record = _train(spec, setup)
record["device_used_at_start"] = used_at_start
ranks = [None] * world_size
# objects, not tensors: works with every backend (gloo, NCCL, HCCL) and keeps Python ints and floats exact
dist.all_gather_object(ranks, record)
if rank == 0:
shared = {key: ranks[0].pop(key) for key in ("parameters", "optimizer")}
for other in ranks[1:]:
for key in shared:
other.pop(key)
with open(path, "w") as f:
json.dump(
{
"status": COMPLETED,
"setup": asdict(setup),
**shared,
"environment": environment(),
"ranks": ranks,
},
f,
)


def _train(spec: BenchmarkSpec, setup: Setup) -> Dict[str, Any]:
Expand Down
8 changes: 5 additions & 3 deletions tests/kit/numerics/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -186,18 +186,20 @@ model_zoo.register(
```

Pass the entry's model_zoo name and the plugin exactly as training code would build it. Plugins need
`torch.distributed`, so run inside `colossalai.testing.spawn`:
`torch.distributed`, so run inside `colossalai.testing.spawn`. `@worker` launches ColossalAI for the worker and
releases its process group when it returns or raises; a worker that exits with its group alive can abort or hang on
the way out (see `scenarios/_distributed.py`):

```python
import colossalai
from colossalai.booster.plugin import LowLevelZeroPlugin
from colossalai.testing import spawn
from tests.kit.numerics import judge
from tests.kit.numerics.scenarios._distributed import worker
from tests.kit.numerics.scenarios.independent_impl import blank_control, training_traces


@worker
def run(rank, world_size, port):
colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port)
blank_control("transformers_mixtral")
plugin = LowLevelZeroPlugin(stage=2, precision="bf16")
test, ref, truth = training_traces("transformers_mixtral", plugin, steps=3)
Expand Down
5 changes: 3 additions & 2 deletions tests/kit/numerics/acceptance.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
from .capture.recipes import UnsupportedRecipe
from .judge import AlignmentReport
from .scenarios import data_parallel, independent_impl, resume
from .scenarios._distributed import worker_process_group
from .spec import NumericsSpec
from .trace import environment_meta, repository_state

Expand Down Expand Up @@ -259,8 +260,8 @@ def _in_process_group(fn: Callable, *args) -> AlignmentReport:


def _worker(rank, world_size, port, fn, args, path):
colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port)
torch.save(fn(*args), path)
with worker_process_group(rank, world_size, port):
torch.save(fn(*args), path)


def _blank_control(entry, steps, cache_dir) -> AlignmentReport:
Expand Down
38 changes: 37 additions & 1 deletion tests/kit/numerics/scenarios/_distributed.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
"""Release the default process group owned by a spawned numerical worker."""
"""The default process group owned by a spawned numerical worker: create it, and release it on the way out."""

import functools
from contextlib import contextmanager
from typing import Callable, Iterator

import torch
import torch.distributed as dist

import colossalai


def destroy_worker_process_group():
"""Keep Gloo's Python owner alive until the generic process group is gone.
Expand All @@ -23,3 +29,33 @@ def destroy_worker_process_group():
# Deliberately release through Gloo's no-GIL Python holder, not as a
# side effect of the generic ProcessGroup's C++ destructor.
del backend


@contextmanager
def worker_process_group(rank: int, world_size: int, port: int) -> Iterator[None]:
"""Launch ColossalAI on ``localhost:port`` for a spawned worker and release its group when the block exits.

A worker that finished passes a barrier first, so no rank tears down the shared transport while a peer still uses
it. A worker that failed skips it: a failing peer may never reach a collective. Either way the group is released by
:func:`destroy_worker_process_group`, and an error propagates unchanged.
"""
try:
colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port)
yield
dist.barrier()
finally:
destroy_worker_process_group()


def worker(fn: Callable) -> Callable:
"""Run a ``spawn`` target ``fn(rank, world_size, port, **kwargs)`` inside :func:`worker_process_group`.

The wrapper keeps ``fn``'s name, so a module-level worker stays picklable for ``spawn``.
"""

@functools.wraps(fn)
def run(rank: int, world_size: int, port: int, **kwargs):
with worker_process_group(rank, world_size, port):
return fn(rank, world_size, port, **kwargs)

return run
13 changes: 2 additions & 11 deletions tests/kit/numerics/scenarios/data_parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,14 @@
from typing import Callable, Optional, Tuple, Union

import torch
import torch.distributed as dist

import colossalai
from colossalai.booster.mixed_precision import MixedPrecision
from colossalai.testing import spawn

from ..capture.training import default_optimizer, prepare, run_data_parallel
from ..judge import AlignmentReport, judge_relative_bound, require_gradients
from ..trace import Trace
from ._distributed import destroy_worker_process_group
from ._distributed import worker_process_group


def traces(
Expand Down Expand Up @@ -104,20 +102,13 @@ def _run(
path,
role,
):
try:
colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port)
with worker_process_group(rank, world_size, port):
s = prepare(entry, micro_batches, seed)
trace = run_data_parallel(
s, plugin_fn(), micro_batches, mixed_precision, optimizer_fn, grad_seed, token_weighted, role=role
)
if rank == 0:
trace.save(path)
# Finish all work on every rank before tearing down the shared transport.
# Keep this out of finally: a failing peer may never reach a collective.
dist.barrier()
finally:
# Each spawned worker owns its group; release it before interpreter shutdown, even on errors.
destroy_worker_process_group()


class _keep:
Expand Down
14 changes: 2 additions & 12 deletions tests/kit/numerics/scenarios/resume.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,13 @@
import tempfile
from typing import Callable, Iterable, Optional, Tuple, Union

import torch.distributed as dist

import colossalai
from colossalai.booster.mixed_precision import MixedPrecision
from colossalai.testing import spawn

from ..capture.training import CHECKPOINT_PARTS, default_optimizer, prepare, run_steps
from ..judge import AlignmentReport, Entry, judge_equal
from ..trace import Trace
from ._distributed import destroy_worker_process_group
from ._distributed import worker_process_group


def traces(
Expand Down Expand Up @@ -162,8 +159,7 @@ def _run(
load=None,
save=None,
):
try:
colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port)
with worker_process_group(rank, world_size, port):
s = prepare(entry, total, seed)
trace = run_steps(
s,
Expand All @@ -181,12 +177,6 @@ def _run(
)
if rank == 0:
trace.save(path)
# Finish all work on every rank before tearing down the shared transport.
# Keep this out of finally: a failing peer may never reach a collective.
dist.barrier()
finally:
# Each spawned worker owns its group; release it before interpreter shutdown, even on errors.
destroy_worker_process_group()


def _from_pretrained(entry, checkpoint: str):
Expand Down
5 changes: 2 additions & 3 deletions tests/test_numerics/test_gemma4_checkpoint_regressions.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from tests.kit.numerics import NumericsSpec
from tests.kit.numerics.acceptance import CONFIGS, _entry_config, run_acceptance
from tests.kit.numerics.scenarios import resume
from tests.kit.numerics.scenarios._distributed import worker

ENTRY = "transformers_gemma4_e2b_for_causal_lm"
GPU = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires one CUDA GPU")
Expand Down Expand Up @@ -91,14 +92,12 @@ def test_gemma4_fp16_finite_updates(name):
assert report["recipe_overrides"] == {"fp16_initial_loss_scale": 1024.0}


@worker
def _amp_checkpoint_worker(rank, world_size, port, directory, shard):
import colossalai
from colossalai.booster import Booster
from colossalai.booster.mixed_precision.fp16_torch import FP16TorchMixedPrecision
from colossalai.booster.plugin import TorchDDPPlugin

colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port)

def build(scale, growth_interval):
model = torch.nn.Linear(3, 2).cuda()
optimizer = torch.optim.AdamW(model.parameters())
Expand Down
4 changes: 2 additions & 2 deletions tests/test_numerics/test_numerics_data_parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from tests.kit.numerics import NumericsSpec, TokenLoss, judge_relative_bound, relative_bound
from tests.kit.numerics.capture.training import default_optimizer
from tests.kit.numerics.scenarios import data_parallel, resume
from tests.kit.numerics.scenarios._distributed import worker

GRADS = {f"grad/param/fc{i}.{p}" for i in (1, 2) for p in ("weight", "bias")}

Expand Down Expand Up @@ -185,11 +186,10 @@ def test_step_without_tokens_rejected():
data_parallel.check(entry, TorchDDPPlugin, world_size=2, micro_batches=4, token_weighted=True)


@worker
def _without_token_loss(rank, world_size, port):
import colossalai
from tests.kit.numerics.capture.training import prepare, run_data_parallel

colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port)
with pytest.raises(ValueError, match="token_loss"):
run_data_parallel(prepare(ENTRY, 2), TorchDDPPlugin(), 2, token_weighted=True)

Expand Down
Loading
Loading