From 2deeb6958dad42191e221f50e51015a00236b987 Mon Sep 17 00:00:00 2001 From: JazyJiang Date: Wed, 18 Nov 2026 17:44:52 +0800 Subject: [PATCH 1/2] [test] release the process group of every spawned numerical and benchmark worker The data-parallel and resume scenarios already synchronize on success and release their group through destroy_worker_process_group, also on errors. The other workers still return with the group alive, or, in the benchmark, destroy it directly right after all_gather_object and not at all when training raises. On the torch 2.5.1 CPU check that can end in SIGABRT at interpreter exit (pytorch/pytorch#179238) or in a hang inside destroy_process_group, which holds the GIL before torch 2.8 (pytorch/pytorch#154976). Move that pattern into worker_process_group (and a @worker decorator) next to destroy_worker_process_group, keep the two scenarios' behaviour unchanged, and use it for the acceptance worker, the 14 independent-impl test workers, the token-loss and Gemma 4 checkpoint test workers, and the benchmark worker. The README example uses it too. --- tests/kit/benchmark/runner/measure.py | 46 +++++++++++-------- tests/kit/numerics/README.md | 8 ++-- tests/kit/numerics/acceptance.py | 5 +- tests/kit/numerics/scenarios/_distributed.py | 38 ++++++++++++++- tests/kit/numerics/scenarios/data_parallel.py | 13 +----- tests/kit/numerics/scenarios/resume.py | 14 +----- .../test_gemma4_checkpoint_regressions.py | 5 +- .../test_numerics_data_parallel.py | 4 +- .../test_numerics_independent_impl.py | 34 ++++++-------- 9 files changed, 94 insertions(+), 73 deletions(-) diff --git a/tests/kit/benchmark/runner/measure.py b/tests/kit/benchmark/runner/measure.py index cc1f2d8bc4c6..09364b3e1f3e 100644 --- a/tests/kit/benchmark/runner/measure.py +++ b/tests/kit/benchmark/runner/measure.py @@ -35,11 +35,11 @@ import torch.distributed as dist from torch.multiprocessing.spawn import ProcessExitedException, ProcessRaisedException -import colossalai from colossalai.accelerator import get_accelerator from colossalai.booster import Booster from colossalai.testing import spawn from tests.kit.numerics.capture.plugins import make_optimizer +from tests.kit.numerics.scenarios._distributed import worker_process_group from ..config.model import PRETRAINED, BenchmarkSpec, spec_of from ..config.optimizers import build_optimizer, for_cpu @@ -90,25 +90,31 @@ def measure(setup: Setup, spec: Optional[BenchmarkSpec] = None) -> Dict[str, Any def _worker(rank, world_size, port, setup, spec, path): - colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) - spec = spec or spec_of(setup.entry) - used_at_start = device_used() - record = _train(spec, setup) - record["device_used_at_start"] = used_at_start - ranks = [None] * world_size - # objects, not tensors: works with every backend (gloo, NCCL, HCCL) and keeps Python ints and floats exact - dist.all_gather_object(ranks, record) - if rank == 0: - shared = {key: ranks[0].pop(key) for key in ("parameters", "optimizer")} - for other in ranks[1:]: - for key in shared: - other.pop(key) - with open(path, "w") as f: - json.dump( - {"status": COMPLETED, "setup": asdict(setup), **shared, "environment": environment(), "ranks": ranks}, - f, - ) - dist.destroy_process_group() + # releases the process group on the way out, also when training raises (see the numerics kit) + with worker_process_group(rank, world_size, port): + spec = spec or spec_of(setup.entry) + used_at_start = device_used() + record = _train(spec, setup) + record["device_used_at_start"] = used_at_start + ranks = [None] * world_size + # objects, not tensors: works with every backend (gloo, NCCL, HCCL) and keeps Python ints and floats exact + dist.all_gather_object(ranks, record) + if rank == 0: + shared = {key: ranks[0].pop(key) for key in ("parameters", "optimizer")} + for other in ranks[1:]: + for key in shared: + other.pop(key) + with open(path, "w") as f: + json.dump( + { + "status": COMPLETED, + "setup": asdict(setup), + **shared, + "environment": environment(), + "ranks": ranks, + }, + f, + ) def _train(spec: BenchmarkSpec, setup: Setup) -> Dict[str, Any]: diff --git a/tests/kit/numerics/README.md b/tests/kit/numerics/README.md index 5f2d75315025..995cf8a2300a 100644 --- a/tests/kit/numerics/README.md +++ b/tests/kit/numerics/README.md @@ -186,18 +186,20 @@ model_zoo.register( ``` Pass the entry's model_zoo name and the plugin exactly as training code would build it. Plugins need -`torch.distributed`, so run inside `colossalai.testing.spawn`: +`torch.distributed`, so run inside `colossalai.testing.spawn`. `@worker` launches ColossalAI for the worker and +releases its process group when it returns or raises; a worker that exits with its group alive can abort or hang on +the way out (see `scenarios/_distributed.py`): ```python -import colossalai from colossalai.booster.plugin import LowLevelZeroPlugin from colossalai.testing import spawn from tests.kit.numerics import judge +from tests.kit.numerics.scenarios._distributed import worker from tests.kit.numerics.scenarios.independent_impl import blank_control, training_traces +@worker def run(rank, world_size, port): - colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) blank_control("transformers_mixtral") plugin = LowLevelZeroPlugin(stage=2, precision="bf16") test, ref, truth = training_traces("transformers_mixtral", plugin, steps=3) diff --git a/tests/kit/numerics/acceptance.py b/tests/kit/numerics/acceptance.py index ecaf03338d8b..fa6cdc547c22 100644 --- a/tests/kit/numerics/acceptance.py +++ b/tests/kit/numerics/acceptance.py @@ -39,6 +39,7 @@ from .capture.recipes import UnsupportedRecipe from .judge import AlignmentReport from .scenarios import data_parallel, independent_impl, resume +from .scenarios._distributed import worker_process_group from .spec import NumericsSpec from .trace import environment_meta, repository_state @@ -259,8 +260,8 @@ def _in_process_group(fn: Callable, *args) -> AlignmentReport: def _worker(rank, world_size, port, fn, args, path): - colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) - torch.save(fn(*args), path) + with worker_process_group(rank, world_size, port): + torch.save(fn(*args), path) def _blank_control(entry, steps, cache_dir) -> AlignmentReport: diff --git a/tests/kit/numerics/scenarios/_distributed.py b/tests/kit/numerics/scenarios/_distributed.py index 84ae93b858a2..f4d26f4ac05c 100644 --- a/tests/kit/numerics/scenarios/_distributed.py +++ b/tests/kit/numerics/scenarios/_distributed.py @@ -1,8 +1,14 @@ -"""Release the default process group owned by a spawned numerical worker.""" +"""The default process group owned by a spawned numerical worker: create it, and release it on the way out.""" + +import functools +from contextlib import contextmanager +from typing import Callable, Iterator import torch import torch.distributed as dist +import colossalai + def destroy_worker_process_group(): """Keep Gloo's Python owner alive until the generic process group is gone. @@ -23,3 +29,33 @@ def destroy_worker_process_group(): # Deliberately release through Gloo's no-GIL Python holder, not as a # side effect of the generic ProcessGroup's C++ destructor. del backend + + +@contextmanager +def worker_process_group(rank: int, world_size: int, port: int) -> Iterator[None]: + """Launch ColossalAI on ``localhost:port`` for a spawned worker and release its group when the block exits. + + A worker that finished passes a barrier first, so no rank tears down the shared transport while a peer still uses + it. A worker that failed skips it: a failing peer may never reach a collective. Either way the group is released by + :func:`destroy_worker_process_group`, and an error propagates unchanged. + """ + try: + colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) + yield + dist.barrier() + finally: + destroy_worker_process_group() + + +def worker(fn: Callable) -> Callable: + """Run a ``spawn`` target ``fn(rank, world_size, port, **kwargs)`` inside :func:`worker_process_group`. + + The wrapper keeps ``fn``'s name, so a module-level worker stays picklable for ``spawn``. + """ + + @functools.wraps(fn) + def run(rank: int, world_size: int, port: int, **kwargs): + with worker_process_group(rank, world_size, port): + return fn(rank, world_size, port, **kwargs) + + return run diff --git a/tests/kit/numerics/scenarios/data_parallel.py b/tests/kit/numerics/scenarios/data_parallel.py index cab234af2433..72f2ae2aef55 100644 --- a/tests/kit/numerics/scenarios/data_parallel.py +++ b/tests/kit/numerics/scenarios/data_parallel.py @@ -20,16 +20,14 @@ from typing import Callable, Optional, Tuple, Union import torch -import torch.distributed as dist -import colossalai from colossalai.booster.mixed_precision import MixedPrecision from colossalai.testing import spawn from ..capture.training import default_optimizer, prepare, run_data_parallel from ..judge import AlignmentReport, judge_relative_bound, require_gradients from ..trace import Trace -from ._distributed import destroy_worker_process_group +from ._distributed import worker_process_group def traces( @@ -104,20 +102,13 @@ def _run( path, role, ): - try: - colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) + with worker_process_group(rank, world_size, port): s = prepare(entry, micro_batches, seed) trace = run_data_parallel( s, plugin_fn(), micro_batches, mixed_precision, optimizer_fn, grad_seed, token_weighted, role=role ) if rank == 0: trace.save(path) - # Finish all work on every rank before tearing down the shared transport. - # Keep this out of finally: a failing peer may never reach a collective. - dist.barrier() - finally: - # Each spawned worker owns its group; release it before interpreter shutdown, even on errors. - destroy_worker_process_group() class _keep: diff --git a/tests/kit/numerics/scenarios/resume.py b/tests/kit/numerics/scenarios/resume.py index b833fa0336c9..85c83200c619 100644 --- a/tests/kit/numerics/scenarios/resume.py +++ b/tests/kit/numerics/scenarios/resume.py @@ -27,16 +27,13 @@ import tempfile from typing import Callable, Iterable, Optional, Tuple, Union -import torch.distributed as dist - -import colossalai from colossalai.booster.mixed_precision import MixedPrecision from colossalai.testing import spawn from ..capture.training import CHECKPOINT_PARTS, default_optimizer, prepare, run_steps from ..judge import AlignmentReport, Entry, judge_equal from ..trace import Trace -from ._distributed import destroy_worker_process_group +from ._distributed import worker_process_group def traces( @@ -162,8 +159,7 @@ def _run( load=None, save=None, ): - try: - colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) + with worker_process_group(rank, world_size, port): s = prepare(entry, total, seed) trace = run_steps( s, @@ -181,12 +177,6 @@ def _run( ) if rank == 0: trace.save(path) - # Finish all work on every rank before tearing down the shared transport. - # Keep this out of finally: a failing peer may never reach a collective. - dist.barrier() - finally: - # Each spawned worker owns its group; release it before interpreter shutdown, even on errors. - destroy_worker_process_group() def _from_pretrained(entry, checkpoint: str): diff --git a/tests/test_numerics/test_gemma4_checkpoint_regressions.py b/tests/test_numerics/test_gemma4_checkpoint_regressions.py index bd59eab42754..b54886370b87 100644 --- a/tests/test_numerics/test_gemma4_checkpoint_regressions.py +++ b/tests/test_numerics/test_gemma4_checkpoint_regressions.py @@ -13,6 +13,7 @@ from tests.kit.numerics import NumericsSpec from tests.kit.numerics.acceptance import CONFIGS, _entry_config, run_acceptance from tests.kit.numerics.scenarios import resume +from tests.kit.numerics.scenarios._distributed import worker ENTRY = "transformers_gemma4_e2b_for_causal_lm" GPU = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires one CUDA GPU") @@ -91,14 +92,12 @@ def test_gemma4_fp16_finite_updates(name): assert report["recipe_overrides"] == {"fp16_initial_loss_scale": 1024.0} +@worker def _amp_checkpoint_worker(rank, world_size, port, directory, shard): - import colossalai from colossalai.booster import Booster from colossalai.booster.mixed_precision.fp16_torch import FP16TorchMixedPrecision from colossalai.booster.plugin import TorchDDPPlugin - colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) - def build(scale, growth_interval): model = torch.nn.Linear(3, 2).cuda() optimizer = torch.optim.AdamW(model.parameters()) diff --git a/tests/test_numerics/test_numerics_data_parallel.py b/tests/test_numerics/test_numerics_data_parallel.py index 611be7979f54..6e188e006853 100644 --- a/tests/test_numerics/test_numerics_data_parallel.py +++ b/tests/test_numerics/test_numerics_data_parallel.py @@ -15,6 +15,7 @@ from tests.kit.numerics import NumericsSpec, TokenLoss, judge_relative_bound, relative_bound from tests.kit.numerics.capture.training import default_optimizer from tests.kit.numerics.scenarios import data_parallel, resume +from tests.kit.numerics.scenarios._distributed import worker GRADS = {f"grad/param/fc{i}.{p}" for i in (1, 2) for p in ("weight", "bias")} @@ -185,11 +186,10 @@ def test_step_without_tokens_rejected(): data_parallel.check(entry, TorchDDPPlugin, world_size=2, micro_batches=4, token_weighted=True) +@worker def _without_token_loss(rank, world_size, port): - import colossalai from tests.kit.numerics.capture.training import prepare, run_data_parallel - colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) with pytest.raises(ValueError, match="token_loss"): run_data_parallel(prepare(ENTRY, 2), TorchDDPPlugin(), 2, token_weighted=True) diff --git a/tests/test_numerics/test_numerics_independent_impl.py b/tests/test_numerics/test_numerics_independent_impl.py index a39e2f6010e3..9eb67f884289 100644 --- a/tests/test_numerics/test_numerics_independent_impl.py +++ b/tests/test_numerics/test_numerics_independent_impl.py @@ -6,7 +6,6 @@ import torch from torch import nn -import colossalai from colossalai.accelerator import get_accelerator from colossalai.booster.mixed_precision import BF16MixedPrecision from colossalai.booster.plugin import HybridParallelPlugin, LowLevelZeroPlugin, TorchDDPPlugin @@ -14,6 +13,7 @@ from tests.kit.numerics import NumericsSpec, Routers, Trace from tests.kit.numerics.capture.recipes import Recipe, UnsupportedRecipe, recipe_for from tests.kit.numerics.capture.training import prepare, run_reference +from tests.kit.numerics.scenarios._distributed import worker from tests.kit.numerics.scenarios.independent_impl import ( _worse_of, blank_control, @@ -87,12 +87,8 @@ def _entry(spec=NumericsSpec()): BUGGY = NumericsSpec(reference_fn=lambda: RenamedMLP(drop_column=True), param_map=_rename) -def _launch(rank, world_size, port): - colossalai.launch(rank=rank, world_size=world_size, host="localhost", port=port) - - +@worker def _check_aligned(rank, world_size, port, plugin_fn, mixed_precision): - _launch(rank, world_size, port) for spec in (NumericsSpec(), NumericsSpec(reference_fn=RenamedMLP, param_map=_rename)): report = check_training(_entry(spec), plugin_fn(), mixed_precision, steps=STEPS) assert {e.name for e in report.entries} == TRACE_NAMES @@ -101,8 +97,8 @@ def _check_aligned(rank, world_size, port, plugin_fn, mixed_precision): assert all(e.ratio == pytest.approx(1.0) for e in report.entries), report.format() +@worker def _check_bug_detected(rank, world_size, port, plugin_fn, mixed_precision): - _launch(rank, world_size, port) report = check_training(_entry(BUGGY), plugin_fn(), mixed_precision, steps=STEPS) failed = {e.name for e in report.failures} # the tensors a dropped column moves far beyond bf16 noise (ratio > 15); weights after one AdamW step are not @@ -111,8 +107,8 @@ def _check_bug_detected(rank, world_size, port, plugin_fn, mixed_precision): assert expected <= failed, report.format() +@worker def _check_blank_control(rank, world_size, port): - _launch(rank, world_size, port) blank_control(_entry(), steps=STEPS) with pytest.raises(AssertionError, match="blank control"): blank_control(_entry(BUGGY)) @@ -210,8 +206,8 @@ def test_nondeterministic_reference_uses_worse_run(): ] +@worker def _check_rejected(rank, world_size, port): - _launch(rank, world_size, port) for plugin_fn, mixed_precision, match in REJECTED: with pytest.raises(UnsupportedRecipe, match=match): recipe_for(plugin_fn(), mixed_precision) @@ -237,8 +233,8 @@ def __init__(self): self.head = nn.Linear(32, 8) # never called +@worker def _check_missing_gradients(rank, world_size, port): - _launch(rank, world_size, port) # DDP must be told about the unused branch, or it waits for its gradient plugin_fn = partial(TorchDDPPlugin, find_unused_parameters=True) entry = (_WithUnusedBranch, _data, lambda out: out, None, _attribute(NumericsSpec())) @@ -305,8 +301,8 @@ def _hidden(output): return {"hidden": output.last_hidden_state} +@worker def _check_shardformer(rank, world_size, port, precision): - _launch(rank, world_size, port) entry = (_tiny_llama, _tokens, _hidden, None, _attribute(NumericsSpec())) plugin = HybridParallelPlugin(tp_size=1, pp_size=1, precision=precision) check_training(entry, plugin, steps=STEPS).assert_pass() @@ -343,8 +339,8 @@ def forward(self, x): return {"y": out} +@worker def _check_routing_in_training(rank, world_size, port): - _launch(rank, world_size, port) spec = NumericsSpec(routers=Routers(select=lambda name, module: name == "gate", top_k=2)) entry = (_TinyMoE, _data, lambda out: out, None, _attribute(spec)) report = check_training(entry, TorchDDPPlugin(), steps=STEPS) @@ -375,8 +371,8 @@ def _plugin(name): }[name]() +@worker def _check_plugin(rank, world_size, port, name): - _launch(rank, world_size, port) entry = (_tiny_llama, _tokens, _hidden, None, _attribute(NumericsSpec())) report = check_training(entry, _plugin(name), steps=STEPS) report.assert_pass() @@ -397,8 +393,8 @@ def test_plugin_aligned(name): spawn(_check_plugin, 1, name=name) +@worker def _unverified_plugins(rank, world_size, port): - _launch(rank, world_size, port) from torch.distributed.fsdp import CPUOffload, MixedPrecision from colossalai.booster.plugin import GeminiPlugin, TorchFSDPPlugin @@ -436,8 +432,8 @@ def _freeze_tower(model): model.tower.requires_grad_(False) +@worker def _check_prepare_fn(rank, world_size, port): - _launch(rank, world_size, port) entry = (_Tower, _data, lambda out: out, None, _attribute(NumericsSpec(prepare_fn=_freeze_tower))) s = prepare(entry) assert not s.model.tower.weight.requires_grad and s.model.head.weight.requires_grad @@ -492,10 +488,10 @@ def test_optimizer_update_rule(): assert {e.name for e in buggy.failures} >= {f"param/0/{p}" for p in PARAMS} +@worker def _check_cache(rank, world_size, port, directory): import os - _launch(rank, world_size, port) plugin = TorchDDPPlugin() first = training_traces(_entry(), plugin, steps=STEPS, cache_dir=directory) files = sorted(os.listdir(os.path.join(directory, "custom"))) @@ -519,8 +515,8 @@ def _matrices_only(params): return torch.optim.AdamW([p for p in params if p.dim() > 1], lr=1e-3) +@worker def _check_optimizer_coverage(rank, world_size, port): - _launch(rank, world_size, port) report = check_training(_entry(), TorchDDPPlugin(), steps=STEPS, optimizer_fn=_matrices_only) # both sides leave the biases untouched, so every comparison passes; only the contract catches it failed = {e.name: e.reason for e in report.failures} @@ -562,8 +558,8 @@ def _tied_entry(spec): return (_TiedLM, _ids, lambda out: out, None, _attribute(spec)) +@worker def _check_tied(rank, world_size, port, name): - _launch(rank, world_size, port) plugin_fn = TorchDDPPlugin if name is None else partial(_plugin, name) report = check_training(_tied_entry(NumericsSpec(tied_params=TIED)), plugin_fn(), steps=STEPS) report.assert_pass() @@ -609,8 +605,8 @@ def _checkpointed(model): _count_layer_calls(model) +@worker def _check_gradient_checkpointing(rank, world_size, port, name): - _launch(rank, world_size, port) plugin_fn = { "ddp_fp32": TorchDDPPlugin, "shardformer_bf16": partial(HybridParallelPlugin, tp_size=1, pp_size=1, precision="bf16"), From 03f25d82c871b5f167566cb6c9b16b4c7d01c47b Mon Sep 17 00:00:00 2001 From: JazyJiang Date: Wed, 18 Nov 2026 17:47:37 +0800 Subject: [PATCH 2/2] [test] Benchmark README: batch functions take their size and random numbers from their arguments --- tests/kit/benchmark/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/kit/benchmark/README.md b/tests/kit/benchmark/README.md index 8783178dd67b..98be3d6a586f 100644 --- a/tests/kit/benchmark/README.md +++ b/tests/kit/benchmark/README.md @@ -134,6 +134,14 @@ tokenized and packed to the fixed shape: a MoE router needs real tokens) and, fo `LatentVideo`, `LatentTokens` and `flow_matching_loss`; a model whose sequence is set by its frames and resolution declares `searchable=("micro_batch",)`. +`batch_fn(model, micro_batch, seq_len, generator)` builds the batch at the size it is given, never a fixed one: the +maximum search changes `micro_batch` or `seq_len` from call to call. It takes every random number from `generator` +(e.g. `torch.randn(shape, generator=generator)`), never from the global random state: each rank's generator is seeded +differently, so the ranks of a multi-device measurement train different data of the same shape, as in training. With +the global state every rank gets the same batch, which does not change a dense model's speed but makes a MoE router +send the same tokens to the same experts on every rank. These functions are part of the entry, so they go into the +same pull request; one that several models can use goes into `config/data.py`. + A model that has no model_zoo entry yet can be measured from `entries.py` (benchmark-only entries, looked up after model_zoo): `qwen3_moe` (Qwen3-30B-A3B) and `wan2_2_ti2v` (the Wan2.2-TI2V-5B transformer at 480×832, 81 frames). Their specs move into model_zoo when the models get their entries; they are kept out of it until