From a7c766b721028dfa93e50461d64bf82c2a2b0839 Mon Sep 17 00:00:00 2001 From: zhouaihui <34807769+zhouaihui@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:49:48 +0800 Subject: [PATCH 1/5] feat: add Megatron Core backends for Qwen3 Target and DSpark Draft --- .dockerignore | 1 + aspec/_environment.py | 11 +- aspec/_target_worker.py | 3 + aspec/cli.py | 5 +- aspec/commands/_checkpoint_audit.py | 31 +- aspec/commands/_recipe_smoke.py | 5 + aspec/commands/doctor.py | 72 +- aspec/config/inference_config.py | 10 + aspec/config/megatron.py | 96 +++ aspec/config/train_config.py | 34 +- aspec/controller/eval.py | 22 +- aspec/controller/inference_manager.py | 2 +- aspec/controller/loop.py | 2 +- aspec/evidence.py | 40 +- aspec/feature_contract.py | 10 +- aspec/inference/context.py | 7 + aspec/inference/engine/megatron_engine.py | 294 ++++++++ aspec/inference/engine/target_actor.py | 39 +- aspec/inference/factory.py | 13 +- aspec/inference/managed.py | 8 +- aspec/inference/service.py | 5 +- aspec/inference/service_protocol.py | 3 +- aspec/inference/target_runtime.py | 13 +- aspec/models/megatron.py | 198 ++++++ aspec/models/megatron_draft.py | 121 ++++ aspec/models/megatron_target.py | 156 +++++ aspec/planning.py | 90 ++- aspec/ray/placement_group.py | 2 +- aspec/ray/train_group.py | 5 + aspec/runtime_identity.py | 12 + aspec/train_entry.py | 15 +- aspec/training/checkpoint.py | 65 +- aspec/training/data_fetcher.py | 132 +++- aspec/training/dflash_trainer.py | 39 +- aspec/training/megatron_optimizer.py | 47 ++ aspec/training/megatron_trainer.py | 206 ++++++ aspec/training/optimizer.py | 32 +- aspec/training/tp_source_prefetch.py | 63 ++ aspec/training/trainer.py | 19 +- aspec/training/trainer_actor.py | 29 +- aspec/utils/env.py | 7 + aspec/utils/train_dump.py | 5 +- docs/README.md | 1 + docs/framework/REFERENCES.md | 2 + docs/releasing.md | 11 +- docs/support.md | 4 + docs/user-guide/installation.md | 2 + docs/user-guide/megatron.md | 260 +++++++ requirements/megatron-build.txt | 10 + requirements/megatron-v0.20.0.txt | 21 + requirements/sglang-v0.5.21.txt | 2 + setup.py | 4 +- setup.sh | 20 +- tests/cpu/test_alignment_rel_diff.py | 59 ++ tests/cpu/test_megatron.py | 267 +++++++ tests/cpu/test_release_audit.py | 6 +- tests/cpu/test_target_service.py | 2 +- tests/cpu/test_torch_tp_validation.py | 61 ++ tests/test_optimizer_state_keys.py | 24 + tools/audit_release.py | 24 +- tools/build_release.py | 11 +- tools/install.sh | 26 +- .../megatron_pipeline_acceptance.py | 125 ++++ .../megatron_target_aspec_alignment.py | 96 +++ tools/validation/megatron_torch_alignment.py | 662 ++++++++++++++++++ tools/validation/torch_tp_dspark.py | 187 +++++ 66 files changed, 3691 insertions(+), 165 deletions(-) create mode 100644 aspec/config/megatron.py create mode 100644 aspec/inference/engine/megatron_engine.py create mode 100644 aspec/models/megatron.py create mode 100644 aspec/models/megatron_draft.py create mode 100644 aspec/models/megatron_target.py create mode 100644 aspec/training/megatron_optimizer.py create mode 100644 aspec/training/megatron_trainer.py create mode 100644 aspec/training/tp_source_prefetch.py create mode 100644 docs/user-guide/megatron.md create mode 100644 requirements/megatron-build.txt create mode 100644 requirements/megatron-v0.20.0.txt create mode 100644 tests/cpu/test_alignment_rel_diff.py create mode 100644 tests/cpu/test_megatron.py create mode 100644 tests/cpu/test_torch_tp_validation.py create mode 100644 tools/validation/megatron_pipeline_acceptance.py create mode 100644 tools/validation/megatron_target_aspec_alignment.py create mode 100644 tools/validation/megatron_torch_alignment.py create mode 100644 tools/validation/torch_tp_dspark.py diff --git a/.dockerignore b/.dockerignore index c042738..6a33b46 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,6 +16,7 @@ dist .env.* **/ray-auth-token remote_exec.sh +*.override.md **/*.safetensors **/*.pt container-cache diff --git a/aspec/_environment.py b/aspec/_environment.py index 696710f..8c1a5ad 100644 --- a/aspec/_environment.py +++ b/aspec/_environment.py @@ -92,7 +92,15 @@ def command_environment(): raise ValueError("ASPEC_CUDA_COMPAT_DIR must contain libcuda.so.1.") paths.append(str(path)) purelib = Path(sysconfig.get_paths()["purelib"]) + cudnn = purelib / "nvidia/cudnn" + if not env.get("CUDNN_HOME") and (cudnn / "lib/libcudnn.so.9").is_file(): + # TE searches CUDA_HOME and the host toolkit before LD_LIBRARY_PATH. + # Bind its explicit search to this interpreter's cuDNN wheel. + env["CUDNN_HOME"] = str(cudnn) + env["ASPEC_AUTO_CUDNN_HOME"] = str(cudnn) toolkit = purelib / "nvidia/cu13" + if (toolkit / "lib/libcudart.so.13").is_file(): + env.setdefault("CUDNN_FRONTEND_CUDART_LIB_NAME", "libcudart.so.13") if not env.get("CUDA_HOME") and not env.get("CUDA_PATH") and (toolkit / "bin/nvcc").is_file(): view = _toolkit_view(toolkit, env) env["CUDA_HOME"] = str(view) @@ -101,7 +109,8 @@ def command_environment(): previous = env.get("LIBRARY_PATH", "") env["LIBRARY_PATH"] = str(view / "lib64") + (os.pathsep + previous if previous else "") paths.append(str(view / "lib64")) - paths.extend(str(p) for p in (purelib / "nvidia/cuda_runtime/lib", purelib / "torch/lib") + paths.extend(str(p) for p in (purelib / "nvidia/cuda_runtime/lib", purelib / "torch/lib", + purelib / "nvidia/cudnn/lib", purelib / "nvidia/nccl/lib") if p.is_dir()) if (toolkit / "lib").is_dir(): paths.append(str(toolkit / "lib")) diff --git a/aspec/_target_worker.py b/aspec/_target_worker.py index 5655ad4..b4068dc 100644 --- a/aspec/_target_worker.py +++ b/aspec/_target_worker.py @@ -13,6 +13,9 @@ def main(): automatic_toolkit = os.environ.pop("ASPEC_AUTO_CUDA_HOME", None) if automatic_toolkit and os.environ.get("CUDA_HOME") == automatic_toolkit: os.environ.pop("CUDA_HOME", None) + automatic_cudnn = os.environ.pop("ASPEC_AUTO_CUDNN_HOME", None) + if automatic_cudnn and os.environ.get("CUDNN_HOME") == automatic_cudnn: + os.environ.pop("CUDNN_HOME", None) os.environ["PATH"] = str(Path(sys.executable).parent) + os.pathsep + os.environ.get("PATH", "") if cache := os.environ.get("ASPEC_CACHE_HOME"): if Path(cache).name != "target": diff --git a/aspec/cli.py b/aspec/cli.py index fec66ca..49b923a 100644 --- a/aspec/cli.py +++ b/aspec/cli.py @@ -48,7 +48,10 @@ def main(): else: module = "aspec.train_entry" help_requested = arguments in (["--help"], ["-h"]) - if module == "aspec.inference.service" and not help_requested and not is_target_environment(): + megatron_target = ("--backend=megatron" in arguments or any( + a == "--backend" and b == "megatron" for a, b in zip(arguments, arguments[1:]) + )) + if module == "aspec.inference.service" and not help_requested and not is_target_environment() and not megatron_target: raise SystemExit("Start aspec target in the Target environment (pip install ling-aspec-target)") try: execute([sys.executable, "-m", module, *arguments]) diff --git a/aspec/commands/_checkpoint_audit.py b/aspec/commands/_checkpoint_audit.py index 8b616ae..10a3c75 100644 --- a/aspec/commands/_checkpoint_audit.py +++ b/aspec/commands/_checkpoint_audit.py @@ -21,19 +21,19 @@ import math -def cpu_values(value): +def cpu_values(value, *, full_tensors=False): import torch if isinstance(value, torch.Tensor): if hasattr(value, "to_local"): - value = value.to_local() + value = value.full_tensor() if full_tensors else value.to_local() return value.detach().cpu().clone() if isinstance(value, dict): - return {key: cpu_values(item) for key, item in value.items()} + return {key: cpu_values(item, full_tensors=full_tensors) for key, item in value.items()} if isinstance(value, list): - return [cpu_values(item) for item in value] + return [cpu_values(item, full_tensors=full_tensors) for item in value] if isinstance(value, tuple): - return tuple(cpu_values(item) for item in value) + return tuple(cpu_values(item, full_tensors=full_tensors) for item in value) return value @@ -80,8 +80,12 @@ def compare_checkpoint(trainer, directory): candidate_model = ModelState(trainer.model).state_dict() candidate_optimizer = OptimizerState(trainer.model, trainer.optimizer).state_dict() - original_model = cpu_values(candidate_model) - original_optimizer = cpu_values(candidate_optimizer) + # Core compute parameters are ordinary local tensors, while its DCP state + # describes logical TP tensors. Read full independent CPU references rather + # than treating a local shard as the checkpoint's complete destination. + logical_tp = getattr(trainer, "execution_backend", "fsdp") == "megatron" + original_model = cpu_values(candidate_model, full_tensors=logical_tp) + original_optimizer = cpu_values(candidate_optimizer, full_tensors=logical_tp) dcp.load({"model_state": original_model}, checkpoint_id=str(directory / "model")) dcp.load({"optim_state": original_optimizer}, checkpoint_id=str(directory / "optimizer")) statistics = { @@ -91,6 +95,8 @@ def compare_checkpoint(trainer, directory): worst_parameters = {key: {"relative_l2": 0, "name": None} for key in statistics} def compare(key, name, reference, actual): + if logical_tp and hasattr(actual, "full_tensor"): + actual = actual.full_tensor() reference = reference.reshape(-1) actual = actual.reshape(-1) if reference.shape != actual.shape: @@ -127,16 +133,25 @@ def compare(key, name, reference, actual): } +def backend_rng_identity(state): + return {"backend": state["backend"], + "cuda_tracker": tensor_hash(sorted(state["cuda_tracker"].items()))} + + def audit_state(trainer): import torch - return { + result = { "global_step": trainer.global_step, "state": state_hashes(trainer), "scheduler": cpu_values(trainer.lr_scheduler.state_dict()), "torch_rng_sha256": hashlib.sha256(torch.random.get_rng_state().numpy().tobytes()).hexdigest(), "cuda_rng_sha256": hashlib.sha256(torch.cuda.get_rng_state().cpu().numpy().tobytes()).hexdigest(), } + snapshot = getattr(trainer, "_checkpoint_backend_rng", None) + if snapshot is not None: + result["backend_rng"] = backend_rng_identity(snapshot()) + return result def add_statistics(total, reference, actual): diff --git a/aspec/commands/_recipe_smoke.py b/aspec/commands/_recipe_smoke.py index 57890da..ca07910 100644 --- a/aspec/commands/_recipe_smoke.py +++ b/aspec/commands/_recipe_smoke.py @@ -111,6 +111,11 @@ def init(self, args, role, mooncake_config=None, with_ref=False): } if any(initial[key] != value for key, value in hashes.items()): raise AssertionError("Restored RNG differs from saved checkpoint") + if "backend_rng" in initial: + from aspec.commands._checkpoint_audit import backend_rng_identity + + if backend_rng_identity(rng["backend_rng"]) != initial["backend_rng"]: + raise AssertionError("Restored model-parallel RNG differs from saved checkpoint") (self._audit_root / f"recipe-resume-diff{self._audit_suffix}.json").write_text( json.dumps({"state": compared, "rng_exact": True}, sort_keys=True) ) diff --git a/aspec/commands/doctor.py b/aspec/commands/doctor.py index 410eeaa..a2b2032 100644 --- a/aspec/commands/doctor.py +++ b/aspec/commands/doctor.py @@ -53,8 +53,10 @@ def check_dependency_metadata(distributions=None): return result -def expected_versions(target=False): - source = asset_path("requirements/sglang-v0.5.21.txt" if target else "requirements/draft.txt") +def expected_versions(target=False, *, backend=None): + manifest = ("megatron-v0.20.0.txt" if backend == "megatron" else + "sglang-v0.5.21.txt" if target else "draft.txt") + source = asset_path("requirements/" + manifest) requirements = [Requirement(line) for line in source.read_text().splitlines() if line.strip() and not line.startswith("#")] return {requirement.name: next((spec.version for spec in requirement.specifier @@ -62,7 +64,68 @@ def expected_versions(target=False): for requirement in requirements} -def inspect_environment(*, gpu=False, target=False): +def _inspect_megatron_environment(*, gpu, target): + result = {"python": sys.executable, "prefix": sys.prefix, "platform": platform.platform(), + "gpu_requested": gpu, "target_requested": target, "backend": "megatron", + "role": "target" if target else "draft", "patch": "not_required", + "checks": [], "errors": [], "dependencies": check_dependency_metadata()} + result["errors"].extend(result["dependencies"]["conflicts"]) + if error := venv_error(): + result["errors"].append(error) + if sys.version_info[:2] < (3, 12) or sys.platform != "linux": + result["errors"].append("Megatron requires Linux with Python >=3.12") + for name, expected in expected_versions(backend="megatron").items(): + try: + actual = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + actual = None + ok = actual is not None and (expected is None or actual.split("+", 1)[0] == expected) + result["checks"].append({"package": name, "expected": expected, "installed": actual, "ok": ok}) + if not ok: + result["errors"].append(f"{name}: expected {expected}, found {actual}") + for name in ("ling-aspec", "ling-aspec-target", "sglang"): + try: + importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + pass + else: + result["errors"].append(f"{name} belongs in a separate backend environment") + if gpu: + try: + import torch + import transformer_engine.pytorch as te + from mooncake.engine import TransferEngine + + from aspec.models.megatron import require_megatron + + require_megatron() + layer = te.Linear(16, 16, params_dtype=torch.bfloat16, device="cuda") + output = layer(torch.ones(2, 16, dtype=torch.bfloat16, device="cuda")) + if not torch.isfinite(output).all().item(): + raise RuntimeError("Transformer Engine BF16 linear produced non-finite values") + result["cuda"] = {"torch": torch.__version__, "runtime": torch.version.cuda, + "device_capability": list(torch.cuda.get_device_capability()), + "transformer_engine": te.__file__, + "mooncake_native_import": TransferEngine is not None} + except Exception as error: + result["errors"].append(f"Megatron GPU/native-library check failed: {error}") + result["ok"] = not result["errors"] + return result + + +def inspect_environment(*, gpu=False, target=False, backend=None): + if backend is None: + marker = Path(sys.prefix) / ".aspec-backend" + if marker.is_file() and marker.read_text().strip() == "megatron": + backend = "megatron" + else: + try: + importlib.metadata.version("ling-aspec-megatron") + backend = "megatron" + except importlib.metadata.PackageNotFoundError: + pass + if backend == "megatron": + return _inspect_megatron_environment(gpu=gpu, target=target) result = {"python": sys.executable, "prefix": sys.prefix, "platform": platform.platform(), "gpu_requested": gpu, "target_requested": target, "checks": [], "errors": []} result["dependencies"] = check_dependency_metadata() @@ -137,9 +200,10 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--gpu", action="store_true", help="Also allocate a small CUDA tensor and import Mooncake") parser.add_argument("--target", action="store_true", help="Also require SGLang and its training patch") + parser.add_argument("--backend", choices=("sglang", "megatron"), default=None) parser.add_argument("--json", action="store_true") args = parser.parse_args() - result = inspect_environment(gpu=args.gpu, target=args.target) + result = inspect_environment(gpu=args.gpu, target=args.target, backend=args.backend) if args.json: print(json.dumps(result, indent=2)) else: diff --git a/aspec/config/inference_config.py b/aspec/config/inference_config.py index 1f8a7b0..324bc68 100644 --- a/aspec/config/inference_config.py +++ b/aspec/config/inference_config.py @@ -34,6 +34,15 @@ from aspec.config.mooncake_config import MooncakeConfig +@dataclass +class MegatronConfig: + """Node-local, frozen Qwen3 prefill through upstream Megatron Core.""" + + tp_size: int = 1 + init_timeout: int = 600 + implementation: str = "transformer_engine" + + @dataclass class SGLangConfig: """Essential SGLang engine configuration. @@ -195,6 +204,7 @@ class InferenceConfig: # performance default; the Plan must explicitly admit or omit it. target_prefill_memory_optimized: bool = False offline: OfflineTrainingConfig = field(default_factory=OfflineTrainingConfig) + megatron: MegatronConfig = field(default_factory=MegatronConfig) sglang: SGLangConfig = field(default_factory=SGLangConfig) vllm: VllmConfig = field(default_factory=VllmConfig) trtllm: TrtllmConfig = field(default_factory=TrtllmConfig) diff --git a/aspec/config/megatron.py b/aspec/config/megatron.py new file mode 100644 index 0000000..a197b6f --- /dev/null +++ b/aspec/config/megatron.py @@ -0,0 +1,96 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Dependency-light capability boundary for the upstream Megatron integration.""" + +MEGATRON_VERSION = "0.20.0" +MEGATRON_REVISION = "685fb8bd970185e9ac3283fe32a5d6667bf4b268" + + +def resolve_megatron_gradient_precision(args): + """Resolve Core main-gradient/DP precision without importing the backend.""" + dtype = getattr(args, "megatron_grad_dtype", "fp32") + if not isinstance(dtype, str) or dtype not in {"fp32", "bf16"}: + raise ValueError("training.megatron_grad_dtype must be fp32 or bf16") + if getattr(args, "train_backend", "fsdp") != "megatron" and dtype != "fp32": + raise ValueError("training.megatron_grad_dtype applies only to train_backend=megatron") + resolved = "float32" if dtype == "fp32" else "bfloat16" + return {"gradient_accumulation_dtype": resolved, "gradient_reduce_dtype": resolved, + "tp_partial_gradient_reduce_dtype": "float32", "optimizer_state_dtype": "float32", + "core_grad_reduce_in_fp32": dtype == "fp32"} + + +def validate_megatron_execution(args, draft): + """Reject combinations outside the runnable dense DSpark CP1/PP1 lane.""" + training = getattr(args, "train_backend", "fsdp") + target = getattr(args, "target_model_backend", "sglang") + resolve_megatron_gradient_precision(args) + tp = getattr(args, "draft_tensor_parallel_size", 1) + if type(tp) is not int or tp < 1: + raise ValueError("training.draft_tensor_parallel_size must be a positive integer") + if training != "megatron" and tp != 1: + raise ValueError("Draft TP requires training.train_backend=megatron") + if training != "megatron" and target != "megatron": + return + if getattr(args, "cluster_mode", "single_node") != "single_node": + raise ValueError("Megatron currently requires single-node execution") + if getattr(args, "aspec_shared_source", False) or getattr(args, "aspec_shared_reader", None): + raise ValueError("Megatron shared Target execution is not supported yet") + if draft["family"] != "dspark": + raise ValueError("Megatron currently supports the DSpark objective") + if draft.get("mtp_conditioning"): + raise ValueError("Megatron currently requires an unconditioned DSpark Draft") + if getattr(args, "draft_context_parallel_size", 1) != 1: + raise ValueError("Megatron currently requires Draft CP1") + if target == "megatron": + if getattr(args, "sglang_extra_args", None) or getattr(args, "sglang_allow_context_length_override", False): + raise ValueError("Megatron Target does not apply SGLang backend overrides") + if getattr(args, "megatron_implementation", "transformer_engine") not in {"local", "transformer_engine"}: + raise ValueError("Unknown Megatron model implementation") + if not getattr(args, "target_prefill_memory_optimized", False): + raise ValueError("Megatron Target requires target_prefill_memory_optimized=true") + if getattr(args, "sglang_nnodes", 1) != 1 or getattr(args, "sglang_pp_size", 1) != 1: + raise ValueError("Megatron Target currently requires node-local TP and PP1") + if getattr(args, "train_with_decode", False): + raise ValueError("Megatron Target supports prefill-only execution") + target_tp = getattr(args, "megatron_tp_size", 1) + if type(target_tp) is not int or target_tp < 1: + raise ValueError("inference.megatron.tp_size must be a positive integer") + if target_tp not in {1, 2}: + raise ValueError("Megatron Target currently validates TP1/TP2") + if target_tp != getattr(args, "inference_num_gpus_per_engine", 1): + raise ValueError("inference.megatron.tp_size must equal inference_num_gpus_per_engine") + slots = (getattr(args, "mooncake_source_gpu_ring_slots", None) + or getattr(args, "mooncake_gpu_ring_slots", 2)) + if getattr(args, "inference_batch_size", 1) > slots: + raise ValueError("Megatron Target source ring must retain one complete inference batch") + if training == "megatron": + if getattr(args, "distributed_backend", "nccl") != "nccl": + raise ValueError("Megatron Draft requires the NCCL distributed backend") + if getattr(args, "micro_batch_size", 1) != 1: + raise ValueError("Megatron Draft currently requires micro_batch_size=1") + prefetch = max(int(getattr(args, "prefetch_depth", 0)), 0) + destination_slots = (getattr(args, "mooncake_destination_gpu_ring_slots", None) + or getattr(args, "mooncake_gpu_ring_slots", 2)) + if prefetch and destination_slots < prefetch + 2: + raise ValueError("Megatron source prefetch requires destination_gpu_ring_slots >= prefetch_depth + 2") + if tp not in {1, 2}: + raise ValueError("Megatron Draft currently validates TP1/TP2") + if getattr(args, "inference_engine_type", "sgl") == "offline": + raise ValueError("Offline replay with Megatron Draft is not validated") + if getattr(args, "fsdp_strategy", "REPLICATE") != "REPLICATE": + raise ValueError("Megatron Draft currently uses replicated DDP rather than FSDP sharding") + if getattr(args, "fsdp_cpu_offload", False): + raise ValueError("Megatron Draft CPU offload is not supported") + if getattr(args, "world_size", 1) % tp: + raise ValueError("Draft world size must be divisible by draft_tensor_parallel_size") + if getattr(args, "dflash_packing", False): + raise ValueError("Megatron Draft packed execution is not supported yet") + if getattr(args, "compile_model", False): + raise ValueError("Megatron Draft currently compiles individual optimized kernels only") + if draft.get("structure", {}).get("ffn_type", "dense") != "dense": + raise ValueError("Megatron Draft currently requires a dense FFN") + attention = draft.get("attention", {}) + if (attention.get("head_layout", {}).get("type", "gqa") not in {"gqa", "mha"} + or any(window is not None for window in attention.get("context_windows", [])) + or any(attention.get(kind) is not None for kind in ("csa", "qsa", "dsa"))): + raise ValueError("Megatron Draft currently requires dense full GQA/MHA attention") diff --git a/aspec/config/train_config.py b/aspec/config/train_config.py index 4fb5a92..c98d5e7 100644 --- a/aspec/config/train_config.py +++ b/aspec/config/train_config.py @@ -34,6 +34,7 @@ resolve_draft_execution_capability, ) from aspec.config.inference_config import InferenceConfig, _validate_inference_batch_config +from aspec.config.megatron import resolve_megatron_gradient_precision from aspec.config.mooncake_config import resolve_rdma_routes from aspec.inference.concurrency import direct_output_sample_capacity from aspec.utils.logging import logger @@ -140,6 +141,7 @@ class TrainingConfig: # Attention backend and context parallelism are independent execution # dimensions. CP>1 currently requires the FA4 backend. draft_context_parallel_size: int = 1 + draft_tensor_parallel_size: int = 1 colocate: bool = False continual_training: bool = False # A weights-only phase transition normally starts a fresh data stream. Set @@ -214,6 +216,9 @@ class TrainingConfig: # Anchor budget per physical row (legacy) or original source sample. anchor_sampling_scope: str = "row" train_backend: str = "fsdp" + # Core main-gradient accumulation and DP reduction. Optimizer state and + # the dedicated TP-partial norm-gradient reduction remain FP32. + megatron_grad_dtype: str = "fp32" train_env_vars: str = "{}" train_with_decode: bool = False training_num_gpus_per_node: int = 1 @@ -380,12 +385,14 @@ def _validate_current_scope(config: DictConfig) -> None: num_proc = config.get("num_proc", 64) if isinstance(num_proc, bool) or not isinstance(num_proc, int) or num_proc < 1: raise ValueError("num_proc must be an integer >= 1") + resolve_megatron_gradient_precision(config.training) engine_type = config.inference.inference_engine_type target_backend = config.model.target_model_backend - if engine_type not in {"sgl", "offline"} or target_backend != "sglang": + supported = {"sgl": "sglang", "offline": "sglang", "megatron": "megatron"} + if supported.get(engine_type) != target_backend: raise NotImplementedError( - "ASpec currently supports inference.inference_engine_type=sgl or offline with " - "model.target_model_backend=sglang; " + "ASpec currently supports sgl/offline with model.target_model_backend=sglang, " + "or megatron with model.target_model_backend=megatron; " f"got engine_type={engine_type!r}, target_backend={target_backend!r}" ) @@ -867,19 +874,20 @@ def _validate_execution_capabilities(config: DictConfig, *, source_context_paral if not config.inference.target_prefill_memory_optimized: return - if config.inference.inference_engine_type != "sgl": + if config.inference.inference_engine_type not in {"sgl", "megatron"}: raise NotImplementedError( - "target_prefill_memory_optimized currently supports SGLang only" + "target_prefill_memory_optimized requires a live Target backend" ) if config.training.train_with_decode: raise NotImplementedError( "target_prefill_memory_optimized requires prefill-only Target execution" ) - if config.inference.inference_num_gpus_per_engine != 1: + if config.inference.inference_engine_type == "sgl" and config.inference.inference_num_gpus_per_engine != 1: raise NotImplementedError( "target_prefill_memory_optimized currently requires one GPU per Target engine" ) - if config.inference.sglang.tp_size != 1 or config.inference.sglang.pp_size != 1: + if (config.inference.inference_engine_type == "sgl" + and (config.inference.sglang.tp_size != 1 or config.inference.sglang.pp_size != 1)): raise NotImplementedError( "target_prefill_memory_optimized currently requires SGLang TP1/PP1" ) @@ -948,6 +956,10 @@ def _validate_execution_capabilities(config: DictConfig, *, source_context_paral logical_dp_size = world_size // cp_size if is_direct_cp_backend( config.training.attention_backend ) else world_size + tp_size = config.training.draft_tensor_parallel_size if config.training.train_backend == "megatron" else 1 + if type(tp_size) is not int or tp_size < 1: + raise ValueError("training.draft_tensor_parallel_size must be a positive integer") + logical_dp_size //= tp_size dispatch_batch_size = config.training.micro_batch_size * logical_dp_size inference_gpus = config.inference.inference_num_gpus if not isinstance(inference_gpus, int) or inference_gpus < 1: @@ -955,15 +967,16 @@ def _validate_execution_capabilities(config: DictConfig, *, source_context_paral "target_prefill_memory_optimized requires a positive inference_num_gpus" ) if node_local_features: - engine_count = config.inference.inference_num_gpus_per_node + engine_count = (config.inference.inference_num_gpus_per_node + // config.inference.inference_num_gpus_per_engine) dispatch_batch_size = ( config.training.micro_batch_size * config.training.training_num_gpus_per_node - // cp_size + // cp_size // tp_size ) capacity_scope = "per node" else: - engine_count = inference_gpus + engine_count = inference_gpus // config.inference.inference_num_gpus_per_engine capacity_scope = "across the run" resident_sample_capacity = direct_output_sample_capacity( engine_count=engine_count, @@ -1063,6 +1076,7 @@ def load_config( "mooncake": "mooncake_", "offline": "offline_", "sglang": "sglang_", + "megatron": "megatron_", "vllm": "vllm_", "trtllm": "trtllm_", "tokenspeed": "tokenspeed_", diff --git a/aspec/controller/eval.py b/aspec/controller/eval.py index 667e141..e9452d8 100644 --- a/aspec/controller/eval.py +++ b/aspec/controller/eval.py @@ -72,15 +72,23 @@ def eval_cache_identity(args, dataset_identity, count): identity = OmegaConf.to_container(OmegaConf.create(identity), resolve=True) from aspec.runtime_identity import SGLANG_RUNTIME_RELATIVE_PATHS, module_runtime_identity - versions = {} - for name in ("sglang", "torch", "transformers", "flash-attn-4", "flashinfer-python", "sglang-kernel"): - try: - versions[name] = importlib.metadata.version(name) - except importlib.metadata.PackageNotFoundError: - versions[name] = None + if getattr(args, "target_model_backend", None) == "megatron": + identity["megatron_tp_size"] = args.megatron_tp_size + identity["megatron_implementation"] = args.megatron_implementation + producer_code = args.megatron_target_cache_identity + versions = producer_code["dependencies"] + else: + versions = {} + for name in ("sglang", "torch", "transformers", "flash-attn-4", "flashinfer-python", "sglang-kernel"): + try: + versions[name] = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + versions[name] = None + producer_code = module_runtime_identity( + "sglang", relative_paths=SGLANG_RUNTIME_RELATIVE_PATHS)["effective_tree_sha256"] return {"schema": "aspec.eval-cache/v2", "dataset": dataset_identity, "samples": count, "target_snapshot": snapshot, "inputs": identity, - "producer_code": module_runtime_identity("sglang", relative_paths=SGLANG_RUNTIME_RELATIVE_PATHS)["effective_tree_sha256"], + "producer_code": producer_code, "consumer_cache_code": {name: module_runtime_identity(name)["effective_tree_sha256"] for name in ("aspec.training.data_fetcher", "aspec.utils.processing", "aspec.data.utils")}, diff --git a/aspec/controller/inference_manager.py b/aspec/controller/inference_manager.py index 9018e41..9efe859 100644 --- a/aspec/controller/inference_manager.py +++ b/aspec/controller/inference_manager.py @@ -1216,7 +1216,7 @@ async def leased_engine(): multimodal_inputs=kwargs["multimodal_inputs"], ) if ( - getattr(self.args, "inference_engine_type", "sgl") == "sgl" + getattr(self.args, "inference_engine_type", "sgl") in {"sgl", "megatron"} and not self._train_with_decode ): generate_kwargs["request_ids"] = request_ids diff --git a/aspec/controller/loop.py b/aspec/controller/loop.py index a038e21..8408a59 100644 --- a/aspec/controller/loop.py +++ b/aspec/controller/loop.py @@ -436,7 +436,7 @@ def _safe_training_cleanup( feature_contract_error = None read_owner_fence = closed_reader_fence if train_group is None else None source_teardown_safe = True - external_target = (getattr(args, "inference_engine_type", None) == "sgl" + external_target = (getattr(args, "inference_engine_type", None) in {"sgl", "megatron"} and bool(getattr(args, "target_service", None))) def record(component: str, state: str, error: Exception | None = None) -> None: diff --git a/aspec/evidence.py b/aspec/evidence.py index ccfc32e..0d829d7 100644 --- a/aspec/evidence.py +++ b/aspec/evidence.py @@ -34,6 +34,7 @@ from aspec.planning import ExecutionPlan from aspec.runtime_control import is_termination_exception from aspec.runtime_identity import ( + MEGATRON_RUNTIME_RELATIVE_PATHS, SGLANG_RUNTIME_RELATIVE_PATHS, module_runtime_identity, ) @@ -43,6 +44,9 @@ "ling-aspec", "ling-aspec-runtime", "ling-aspec-target", + "ling-aspec-megatron", + "megatron-core", + "transformer-engine", "torch", "ray", "sglang", @@ -208,7 +212,8 @@ def _module_source_identity(module: str) -> dict[str, Any]: runtime_identity = module_runtime_identity( module, relative_paths=( - SGLANG_RUNTIME_RELATIVE_PATHS if module == "sglang" else None + SGLANG_RUNTIME_RELATIVE_PATHS if module == "sglang" else + MEGATRON_RUNTIME_RELATIVE_PATHS if module == "megatron.core" else None ), ) identity: dict[str, Any] = { @@ -306,7 +311,9 @@ def __init__( "aspec": _source_identity(), "packages": _package_versions(), "package_scope": "draft_driver", - "backends": {"sglang": _module_source_identity("sglang")}, + "backends": ({"megatron": _module_source_identity("megatron.core")} + if plan.policy.backends["target_features"] == "megatron" else + {"sglang": _module_source_identity("sglang")}), "host": { "hostname": platform.node(), "system": platform.system(), @@ -433,13 +440,16 @@ def record_actor_initialization( target_runtimes = [dict(status["runtime"]) for status in (inference_engine_statuses or ()) if isinstance(status.get("runtime"), Mapping)] if target_runtimes: + target_backend = self._plan.policy.backends["target_features"] backend_identities = [item.get("backend_identity") for item in target_runtimes] if (len(target_runtimes) != inference_engine_count + or any(item.get("backend", "sglang") != target_backend for item in target_runtimes) or any(not isinstance(item, Mapping) or not item.get("effective_tree_sha256") or item.get("missing_files") for item in backend_identities) or len({item["effective_tree_sha256"] for item in backend_identities}) != 1): - raise RuntimeError("Target workers reported incomplete or different SGLang identities") - self._data["source"]["backends"]["sglang"] = dict(backend_identities[0]) + name = "SGLang" if target_backend == "sglang" else "Megatron" + raise RuntimeError(f"Target workers reported incomplete or different {name} identities") + self._data["source"]["backends"][target_backend] = dict(backend_identities[0]) self._data["source"]["target_runtimes"] = target_runtimes prior_runtime = self._data["observations"]["runtime"] @@ -500,12 +510,12 @@ def record_actor_initialization( capacity = status.get("token_capacity") if not isinstance(capacity, Mapping): raise RuntimeError( - f"SGLang engine {status.get('rank')} did not report token capacity" + f"Target engine {status.get('rank')} did not report token capacity" ) observed_context = capacity.get("context") if observed_context != planned_context: raise RuntimeError( - "Observed SGLang context contract does not match the Plan: " + "Observed Target context contract does not match the Plan: " f"rank={status.get('rank')}, planned={planned_context}, " f"observed={observed_context}" ) @@ -513,7 +523,7 @@ def record_actor_initialization( "backend_context_length" ]: raise RuntimeError( - "Observed SGLang context length does not match the Plan: " + "Observed Target context length does not match the Plan: " f"rank={status.get('rank')}, " f"planned={backend_tokens['backend_context_length']}, " f"observed={capacity.get('context_length')}" @@ -570,7 +580,7 @@ def record_actor_initialization( { "predicate": ("all_planned_offline_replay_initializers_completed" if self._plan.policy.backends["target_features"] == "offline_replay" - else "all_planned_sglang_engine_initializers_completed"), + else f"all_planned_{self._plan.policy.backends['target_features']}_engine_initializers_completed"), "observed_engine_count": inference_engine_count, }, ) @@ -599,6 +609,18 @@ def _validate_actor_placement( training = [dict(item) for item in (training_actor_statuses or ())] inference = [dict(item) for item in (inference_engine_statuses or ())] + if self._plan.policy.backends["training"] == "megatron": + expected_tp = self._plan.policy.parallelism["training"]["tensor_parallel"] + expected_dp = self._plan.policy.parallelism["training"]["data_parallel"] + if (not training or any(item.get("backend") != "megatron" + or item.get("tensor_parallel_size") != expected_tp + or item.get("data_parallel_size") != expected_dp + or not item.get("initialized") for item in training)): + raise RuntimeError("Megatron trainer/backend TP was not observed as planned") + identities = [item.get("runtime", {}).get("backend_identity", {}) for item in training] + if (any(not item.get("effective_tree_sha256") or item.get("missing_files") for item in identities) + or len({item.get("effective_tree_sha256") for item in identities}) != 1): + raise RuntimeError("Megatron trainers reported incomplete or different Core identities") if training and len(training) != self._plan.policy.resources["training"]["world_size"]: raise RuntimeError("trainer placement status count does not match the Plan") if inference and len(inference) != self._plan.policy.resources["inference"]["engines"]: @@ -833,7 +855,7 @@ def record_training(self, summary: Mapping[str, Any]) -> None: transfer_observation = { "predicate": ("optimizer_step_consumed_validated_offline_features_via_mooncake" if self._plan.policy.backends["target_features"] == "offline_replay" - else "optimizer_step_consumed_validated_sglang_features_via_mooncake"), + else f"optimizer_step_consumed_validated_{self._plan.policy.backends['target_features']}_features_via_mooncake"), "optimizer_steps_observed": training["optimizer_steps_observed"], "feature_contract_id": planned_contract_id, "producer_count": feature_contract["producer_count"], diff --git a/aspec/feature_contract.py b/aspec/feature_contract.py index ae40798..5a370d1 100644 --- a/aspec/feature_contract.py +++ b/aspec/feature_contract.py @@ -67,6 +67,7 @@ def build_target_feature_contract( target_num_hidden_layers: int, aux_hidden_state_layers: list[int], store_last_hidden_states: bool, + producer_backend: str = "sglang", mtp: Mapping[str, Any] | None = None, ) -> dict[str, Any]: """Build the payload contract without starting a backend. @@ -76,6 +77,8 @@ def build_target_feature_contract( differences still produce distinct identities. """ + if producer_backend not in {"sglang", "megatron"}: + raise FeatureContractError(f"Unsupported Target producer backend: {producer_backend!r}") if not isinstance(target_hidden_size, int) or target_hidden_size <= 0: raise FeatureContractError( f"draft target_hidden_size must be a positive integer, got {target_hidden_size!r}" @@ -168,7 +171,7 @@ def build_target_feature_contract( contract: dict[str, Any] = { "schema_version": FEATURE_CONTRACT_SCHEMA_VERSION, - "producer": {"backend": "sglang", "mode": "prefill_only"}, + "producer": {"backend": producer_backend, "mode": "prefill_only"}, "consumer": { "target_distribution": target_distribution, }, @@ -197,7 +200,10 @@ def validate_feature_contract(contract: Mapping[str, Any]) -> None: raise FeatureContractError( f"Target feature contract identity mismatch: {observed_id!r} != {expected_id!r}" ) - if contract.get("producer") != {"backend": "sglang", "mode": "prefill_only"}: + if contract.get("producer") not in ( + {"backend": "sglang", "mode": "prefill_only"}, + {"backend": "megatron", "mode": "prefill_only"}, + ): raise FeatureContractError( f"Unsupported target feature producer contract: {contract.get('producer')!r}" ) diff --git a/aspec/inference/context.py b/aspec/inference/context.py index b6c034d..59bf8d4 100644 --- a/aspec/inference/context.py +++ b/aspec/inference/context.py @@ -269,6 +269,13 @@ def configure_sglang_context_override(contract: Mapping[str, Any]) -> bool: return enabled +def compile_megatron_context_contract(**kwargs): + """Frozen prefill consumes exactly the requested sequence, without decode reserve.""" + contract = compile_offline_context_contract(**kwargs) + contract["backend_context_length"] = contract["effective_training_max_sequence_length"] + return contract + + def compile_offline_context_contract( *, max_sequence_length, context_parallel_size, model_config, native_context_length, allow_context_length_override, context_shard_alignment=1, diff --git a/aspec/inference/engine/megatron_engine.py b/aspec/inference/engine/megatron_engine.py new file mode 100644 index 0000000..8366afb --- /dev/null +++ b/aspec/inference/engine/megatron_engine.py @@ -0,0 +1,294 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Frozen upstream Megatron Target workers behind ASpec's Target service. + +Each engine owns a role-local TP world. Only the publishing rank owns source +storage; TP activations are replicated because sequence parallelism is disabled. +Control pipes carry tokens and descriptors, never feature tensors. +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import os +import sys +import threading +import traceback +import uuid +from datetime import timedelta + +import torch + +from aspec.feature_contract import FeatureContractRuntime +from aspec.inference.target_prefill import TargetPrefillDirectSink, TargetPrefillRequest +from aspec.ray.ray_actor import RayActor +from aspec.utils.misc import get_current_node_ip + + +def _target_worker(connection, local_rank, gpu_ids, args, mooncake_config, port, engine_rank): + import torch.distributed as dist + + from aspec.models.megatron import initialize_parallelism, load_qwen3_target + + store = None + try: + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(gpu) for gpu in gpu_ids) + torch.cuda.set_device(local_rank) + dist.init_process_group( + "nccl", init_method=f"tcp://127.0.0.1:{port}", + rank=local_rank, world_size=len(gpu_ids), + timeout=timedelta(seconds=getattr(args, "megatron_init_timeout", 600)), + ) + groups = initialize_parallelism(len(gpu_ids)) + model, config = load_qwen3_target( + args.target_model_path, groups, + implementation=getattr(args, "megatron_implementation", "transformer_engine"), + ) + captures = list(args.aux_hidden_states_layers) + runtime = FeatureContractRuntime( + args.aspec_feature_contract, role="producer", participant=f"megatron_engine_{engine_rank}", + ) + runtime.validate_producer_binding( + hidden_size=config.hidden_size, num_hidden_layers=config.num_hidden_layers, + aux_hidden_state_layers=captures, store_last_hidden_states=args.store_last_hidden_states, + ) + if local_rank == 0: + from aspec.transfer.mooncake.eagle_store import EagleMooncakeStore + + source_config = dataclasses.replace(mooncake_config, global_segment_size=0, async_put_pool_size=0) + store = EagleMooncakeStore(source_config) + store.setup(device=local_rank, p2p_role="source") + status = { + "pid": os.getpid(), "hidden_size": config.hidden_size, + "python": sys.executable, "prefix": sys.prefix, + "aspec": __import__("aspec").__file__, + "megatron": __import__("megatron.core", fromlist=["__file__"]).__file__, + "torch": torch.__version__, "cuda": torch.version.cuda, + "tensor_parallel_size": len(gpu_ids), "local_rank": local_rank, + "device_capability": list(torch.cuda.get_device_capability()), + "model_implementation": getattr(args, "megatron_implementation", "transformer_engine"), + "model_class": type(model).__name__, + "hybrid_layer_pattern": model.hybrid_layer_pattern, + "target_frozen": not any(p.requires_grad for p in model.parameters()), + } + connection.send({"ok": True, "value": status}) + while True: + method, payload = connection.recv() + if method == "shutdown": + if store is not None: + store.close() + dist.destroy_process_group() + connection.send({"ok": True, "value": None}) + return + if method == "status": + features = runtime.status() + if store is not None: + features["transport"] = store.get_p2p_status() + connection.send({"ok": True, "value": features}) + continue + if method != "generate": + raise ValueError(f"Unknown Megatron Target operation: {method}") + outputs = [] + for sample in payload: + ids = torch.tensor(sample["tokens"], dtype=torch.long, device="cuda") + length = ids.numel() + if not 0 < length <= args.max_seq_length: + raise ValueError("Target input length is outside the planned context") + request = TargetPrefillRequest(sample["request_id"], sample["data_id"], length) + sink = None + try: + if local_rank == 0: + sink = TargetPrefillDirectSink( + store=store, requests=[request], capture_layers=captures, + hidden_size=config.hidden_size, store_last_hidden_states=args.store_last_hidden_states, + source_engine=f"megatron_{engine_rank}", context_parallel_size=1, + ) + sink.begin(input_ids=ids, request_ids=[request.request_id], + sequence_lengths=[length], prefix_lengths=[0]) + final = model.prefill(ids.unsqueeze(0), captures, sink.capture if sink is not None else None) + if local_rank == 0: + sink.finish(final) + key = sink.commit_request(request.request_id) + shapes = {"hidden_states": (length, len(captures) * config.hidden_size), + "input_ids": (length,)} + dtypes = {"hidden_states": torch.bfloat16, "input_ids": torch.int64} + if args.store_last_hidden_states: + shapes["last_hidden_states"] = (length, config.hidden_size) + dtypes["last_hidden_states"] = torch.bfloat16 + metadata = { + "request_id": request.request_id, "engine_rank": engine_rank, + "aspec_feature_contract_id": runtime.contract_id, + "source_node_ip": get_current_node_ip(), + "feature_affinity": getattr(args, "mooncake_feature_affinity", "any"), + "target_feature_output": "direct_to_transport", + "transport_data_plane": "transfer_engine", + "transport_protocol": mooncake_config.protocol, + "feature_layout": "full", + "draft_context_parallel_size": 1, "draft_feature_layout": "full", + "draft_attention_backend": args.attention_backend, + } + from aspec.config.draft_execution import is_direct_cp_backend + + if is_direct_cp_backend(args.attention_backend): + metadata.update(direct_cp_sharded=True, direct_cp_size=1) + runtime.validate_metadata(tensor_shapes=shapes, tensor_dtypes=dtypes, metadata=metadata) + sink.validate_publishable() + sink.publish() + outputs.append({"mooncake_key": key, "tensor_shapes": shapes, + "tensor_dtypes": dtypes, "metadata": metadata}) + except BaseException: + if sink is not None: + sink.abort() + raise + connection.send({"ok": True, "value": outputs if local_rank == 0 else None}) + except BaseException as error: + try: + connection.send({"ok": False, "error": repr(error), "traceback": traceback.format_exc()}) + finally: + # Published storage remains owned until the service fences readers + # and observes this process exiting, including native-call failures. + connection.close() + if store is not None: + # Keep all published CUDA registrations alive on failure. The + # Target service terminates this world only after its read fence. + threading.Event().wait() + + +class MegatronEngine(RayActor): + """Ray control facade; model and source storage live in owned child workers.""" + + def __init__(self, args, rank, base_gpu_id=None, num_gpus_per_engine=1, + node_rank=0, engine_group=0, gpu_ids=None): + self.args, self.rank = args, rank + self.base_gpu_id = base_gpu_id + self.gpu_ids = list(gpu_ids or range(base_gpu_id, base_gpu_id + num_gpus_per_engine)) + self.num_gpus_per_engine = num_gpus_per_engine + self._connections, self._processes = [], [] + self._status = [] + self._lock = threading.Lock() + self._mooncake_config = None + self._features = None + self._failed = False + + def _receive(self): + try: + return self._receive_workers() + except BaseException: + self._failed = True + raise + + def _receive_workers(self): + results = [] + for connection, process in zip(self._connections, self._processes, strict=True): + if not connection.poll(getattr(self.args, "megatron_init_timeout", 600)): + raise TimeoutError(f"Megatron Target worker {process.pid} did not complete") + result = connection.recv() + if not result["ok"]: + raise RuntimeError(f"Megatron Target worker failed: {result['error']}\n{result['traceback']}") + results.append(result["value"]) + return results + + def _call(self, method, payload=None): + with self._lock: + if not self.health_check(): + raise RuntimeError("Megatron Target worker world is unavailable") + try: + for connection in self._connections: + connection.send((method, payload)) + return self._receive()[0] + except BaseException: + self._failed = True + raise + + def init(self, mooncake_config=None, pre_allocated_port=None): + if self._processes: + raise RuntimeError("Megatron Target engine cannot be initialized twice") + if mooncake_config is None or mooncake_config.data_plane != "transfer_engine": + raise ValueError("Megatron Target requires the Mooncake transfer_engine data plane") + if pre_allocated_port is None: + raise ValueError("Megatron Target requires a reserved initialization port") + self._mooncake_config = mooncake_config + context = torch.multiprocessing.get_context("spawn") + for rank in range(self.num_gpus_per_engine): + parent, child = context.Pipe() + process = context.Process(target=_target_worker, args=( + child, rank, self.gpu_ids, self.args, mooncake_config, pre_allocated_port, self.rank, + )) + process.start() + child.close() + self._connections.append(parent) + self._processes.append(process) + self._status = self._receive() + + async def generate(self, data_id, input_ids_ref=None, packed_loss_mask_list=None, + formatted_prompts=None, request_ids=None, return_last_hidden_states=False, + return_logits=True, multimodal_inputs=None): + if multimodal_inputs: + raise ValueError("Megatron Target currently supports text-only Qwen3") + if (input_ids_ref is None) == (formatted_prompts is None): + raise ValueError("Provide exactly one of input_ids_ref or formatted_prompts") + if formatted_prompts is not None: + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(self.args.target_model_path, trust_remote_code=False) + tokens = tokenizer(formatted_prompts, add_special_tokens=False)["input_ids"] + else: + tokens = [value.reshape(-1).tolist() for value in input_ids_ref] + ids = ([f"{data_id}_{index}" for index in range(len(tokens))] + if isinstance(data_id, str) else list(data_id)) + if len(ids) != len(tokens): + raise ValueError("Target data identities and token sequences differ in length") + if request_ids is None: + request_ids = [uuid.uuid4().hex for _ in tokens] + elif isinstance(request_ids, str): + request_ids = [f"{request_ids}_{index}" for index in range(len(tokens))] + if len(request_ids) != len(tokens): + raise ValueError("Target request identities and token sequences differ in length") + payload = [dict(tokens=value, data_id=data, request_id=request) + for value, data, request in zip(tokens, ids, request_ids, strict=True)] + return await asyncio.to_thread(self._call, "generate", payload) + + def get_status(self): + return { + "rank": self.rank, "pid": os.getpid(), "node_ip": self.get_node_ip(), + "initialized": bool(self._status) and self.health_check(), + "base_gpu_id": self.base_gpu_id, "engine_gpu_ids": self.gpu_ids, + "num_gpus_per_engine": self.num_gpus_per_engine, + "hidden_size": self._status[0]["hidden_size"] if self._status else None, + "target_prefill_memory_optimized": True, "workers": self._status, + "token_capacity": {"context": self.args.aspec_context_contract, + "context_length": self.args.aspec_context_contract["backend_context_length"], + "profiled_max_total_tokens": None, + "required_max_total_tokens": None, + "max_prefill_tokens": self.args.max_seq_length, + "requested_max_running_requests": 1, + "allocation_policy": "sequential_full_prefill_without_kv_pool"}, + } + + def health_check(self, timeout=5.0): + return (not self._failed and bool(self._processes) + and all(process.is_alive() for process in self._processes)) + + def get_mooncake_config(self): + return self._mooncake_config + + def get_feature_contract_status(self): + if self._processes and self.health_check(): + self._features = self._call("status") + return self._features or {"role": "producer", "state": "pending_observation"} + + async def prepare_feature_contract_status(self): + return await asyncio.to_thread(self.get_feature_contract_status) + + async def shutdown(self): + if self._processes: + await asyncio.to_thread(self._call, "shutdown") + for process in self._processes: + process.join(timeout=30) + if process.is_alive(): + raise RuntimeError("Megatron Target worker did not exit after source drain") + for connection in self._connections: + connection.close() + self._processes.clear() + self._connections.clear() diff --git a/aspec/inference/engine/target_actor.py b/aspec/inference/engine/target_actor.py index 308a294..2da486f 100644 --- a/aspec/inference/engine/target_actor.py +++ b/aspec/inference/engine/target_actor.py @@ -18,9 +18,16 @@ def __init__(self, *args, **kwargs): self.runtime_identity = code_identity() self.gpu_ids = kwargs.pop("gpu_ids") - from aspec.inference.engine.sgl_engine import SglEngine + arguments = kwargs.get("args", args[0] if args else None) + self.backend = getattr(arguments, "target_model_backend", "sglang") + if self.backend == "megatron": + from aspec.inference.engine.megatron_engine import MegatronEngine - self.engine = SglEngine(*args, **kwargs) + self.engine = MegatronEngine(*args, gpu_ids=self.gpu_ids, **kwargs) + else: + from aspec.inference.engine.sgl_engine import SglEngine + + self.engine = SglEngine(*args, **kwargs) def init(self, **kwargs): return self.engine.init(**kwargs) @@ -37,13 +44,19 @@ def find_free_port(self, *args, **kwargs): def get_status(self): import psutil import ray - import sglang import torch import aspec - from aspec.runtime_identity import SGLANG_RUNTIME_RELATIVE_PATHS, module_runtime_identity - - backend = module_runtime_identity("sglang", relative_paths=SGLANG_RUNTIME_RELATIVE_PATHS) + from aspec.runtime_identity import ( + MEGATRON_RUNTIME_RELATIVE_PATHS, + SGLANG_RUNTIME_RELATIVE_PATHS, + module_runtime_identity, + ) + + if self.backend == "megatron": + backend = module_runtime_identity("megatron.core", relative_paths=MEGATRON_RUNTIME_RELATIVE_PATHS) + else: + backend = module_runtime_identity("sglang", relative_paths=SGLANG_RUNTIME_RELATIVE_PATHS) backend_identity = { "path": backend["path"], "git_revision": None, "git_dirty": None, "effective_tree_sha256": backend["effective_tree_sha256"], @@ -52,18 +65,24 @@ def get_status(self): result = self.engine.get_status() result["engine_gpu_ids"] = self.gpu_ids result["created"] = psutil.Process().create_time() + from aspec.inference.target_runtime import dependency_versions + result["runtime"] = { **self.runtime_identity, "python": sys.executable, "prefix": sys.prefix, "aspec": str(Path(aspec.__file__).resolve()), "ray": str(Path(ray.__file__).resolve()), - "sglang": str(Path(sglang.__file__).resolve()), "backend_identity": backend_identity, - "sglang_version": importlib.metadata.version("sglang"), + "backend": self.backend, + "dependencies": dependency_versions("sglang", "megatron-core", "transformer-engine", + "flashinfer-python", "sglang-kernel"), "torch": torch.__version__, "cuda": torch.version.cuda, - "cutlass": importlib.metadata.version("nvidia-cutlass-dsl"), - "quack": importlib.metadata.version("quack-kernels"), + "cutlass": dependency_versions("nvidia-cutlass-dsl")["nvidia-cutlass-dsl"], + "quack": dependency_versions("quack-kernels")["quack-kernels"], } + if self.backend == "sglang": + result["runtime"]["sglang"] = backend["path"] + result["runtime"]["sglang_version"] = importlib.metadata.version("sglang") return result def get_mooncake_config(self): diff --git a/aspec/inference/factory.py b/aspec/inference/factory.py index 45fdccc..9305f81 100644 --- a/aspec/inference/factory.py +++ b/aspec/inference/factory.py @@ -37,10 +37,9 @@ def _require_supported_engine(engine_type: str) -> None: - if engine_type not in {"sgl", "offline"}: + if engine_type not in {"sgl", "offline", "megatron"}: raise NotImplementedError( - "ASpec currently supports the SGLang or offline replay path " - f"(inference_engine_type='sgl' or 'offline'); got {engine_type!r}" + f"Supported inference engines are sgl, megatron and offline; got {engine_type!r}" ) @@ -59,7 +58,7 @@ def create_inference_engines(args, inference_pg, mooncake_config, engine_group: _wait_for_init(init_refs, "OfflineReplay", timeout=300) return engines - if engine_type not in ("hf", "sgl", "vllm", "trtllm", "tokenspeed"): + if engine_type not in ("hf", "sgl", "vllm", "trtllm", "tokenspeed", "megatron"): raise ValueError(f"Unknown inference_engine_type: {engine_type}") logger.info(f"Using {engine_type} engine for inference") @@ -94,14 +93,14 @@ def prepare_inference_engines(args, inference_pg, mooncake_config, engine_group: if engine_type == "offline": return _prepare_offline_replay_engines(args, mooncake_config, engine_group) - if engine_type not in ("hf", "sgl", "vllm", "trtllm", "tokenspeed"): + if engine_type not in ("hf", "sgl", "vllm", "trtllm", "tokenspeed", "megatron"): raise ValueError(f"Unknown inference_engine_type: {engine_type}") logger.info(f"Preparing {engine_type} inference engines...") if engine_type == "hf": engines, init_refs = _prepare_hf_engines(args, inference_pg, mooncake_config, engine_group) - elif engine_type == "sgl": + elif engine_type in {"sgl", "megatron"}: engines, init_refs = _prepare_sgl_engines(args, inference_pg, mooncake_config, engine_group) elif engine_type == "trtllm": engines, init_refs = _prepare_trtllm_engines( @@ -159,7 +158,7 @@ def init_engines(args, pg, engine_type: str, mooncake_config=None, engine_group: _require_supported_engine(engine_type) if engine_type == "hf": return _init_hf_engines(args, pg, mooncake_config, engine_group) - elif engine_type == "sgl": + elif engine_type in {"sgl", "megatron"}: return _init_sgl_engines(args, pg, mooncake_config, engine_group) elif engine_type == "vllm": return _init_vllm_engines(args, pg, mooncake_config, engine_group) diff --git a/aspec/inference/managed.py b/aspec/inference/managed.py index 47a89ca..f2c12e3 100644 --- a/aspec/inference/managed.py +++ b/aspec/inference/managed.py @@ -53,7 +53,8 @@ def managed_target(args, *, draft_gpu_count=None): config = root / "target-service.json" config.write_text(json.dumps({"model": {"target_model_path": args.target_model_path}})) command = [settings["python_executable"], "-m", "aspec._target_worker", "-m", - "aspec.inference.service", "--config", str(config), "--address", address] + "aspec.inference.service", "--config", str(config), "--address", address, + "--backend", getattr(args, "target_model_backend", "sglang")] log_path = root / "target-service.log" with log_path.open("w") as log: process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, @@ -61,7 +62,10 @@ def managed_target(args, *, draft_gpu_count=None): ready = False prior_devices = os.environ.get("CUDA_VISIBLE_DEVICES") try: - deadline = time.monotonic() + float(getattr(args, "sglang_init_timeout", 300)) + timeout_field = ("megatron_init_timeout" + if getattr(args, "target_model_backend", "sglang") == "megatron" + else "sglang_init_timeout") + deadline = time.monotonic() + float(getattr(args, timeout_field, 300)) while time.monotonic() < deadline: if process.poll() is not None: raise RuntimeError(f"Target exited before readiness; see {log_path}") diff --git a/aspec/inference/service.py b/aspec/inference/service.py index af37079..5b6b696 100644 --- a/aspec/inference/service.py +++ b/aspec/inference/service.py @@ -93,6 +93,8 @@ def create(self, value): if set(arguments) - TARGET_ARGUMENTS: raise ValueError("Unknown Target configuration fields") args = Namespace(**arguments) + if getattr(args, "target_model_backend", "sglang") != self.runtime.get("backend", "sglang"): + raise ValueError("Requested Target backend differs from the service environment") validate_target_tp(getattr(args, "sglang_nnodes", 1), value["node_rank"]) if self.model and args.target_model_path != self.model: raise ValueError("Target model differs from the service configuration") @@ -437,6 +439,7 @@ def main(): logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser(description="Run Target in this environment; Draft connects by address.") parser.add_argument("--config", type=Path) + parser.add_argument("--backend", choices=("sglang", "megatron"), default="sglang") parser.add_argument("--address", help="Listening http://host:port; defaults to the config or localhost:30000") args = parser.parse_args() config = OmegaConf.load(args.config) if args.config else OmegaConf.create({}) @@ -446,7 +449,7 @@ def main(): devices = visible_gpu_ids() if not devices: parser.error("Select allocated Target GPUs with CUDA_VISIBLE_DEVICES") - runtime = inventory() # Check this environment and patch before starting processes. + runtime = inventory(args.backend) service = TargetService(model=model, devices=devices, runtime=runtime) parsed = urlsplit(address) server = http_server(service, parsed.hostname, parsed.port) diff --git a/aspec/inference/service_protocol.py b/aspec/inference/service_protocol.py index 9de6d54..c66f6e0 100644 --- a/aspec/inference/service_protocol.py +++ b/aspec/inference/service_protocol.py @@ -23,7 +23,8 @@ # to this interface. Optimizers, datasets, logging credentials and driver # objects remain in Draft. Unknown Draft settings do not change this protocol. TARGET_ARGUMENTS = frozenset(""" -target_model_path trust_remote_code aux_hidden_states_layers store_last_hidden_states native_context_length +target_model_path target_model_backend trust_remote_code aux_hidden_states_layers store_last_hidden_states native_context_length +megatron_tp_size megatron_init_timeout megatron_implementation draft_algorithm mooncake_hidden_dim sglang_mem_fraction_static sglang_pp_size sglang_nnodes sglang_tp_size sglang_extra_args sglang_allow_context_length_override sglang_enable_multimodal diff --git a/aspec/inference/target_runtime.py b/aspec/inference/target_runtime.py index 3527c93..35851de 100644 --- a/aspec/inference/target_runtime.py +++ b/aspec/inference/target_runtime.py @@ -35,8 +35,19 @@ def dependency_versions(*names): return result -def inventory(): +def inventory(backend="sglang"): """Read Target identity in its own interpreter, without starting GPU services.""" + if backend == "megatron": + from aspec.models.megatron import require_megatron + from aspec.runtime_identity import MEGATRON_RUNTIME_RELATIVE_PATHS, module_runtime_identity + + require_megatron() + return {**code_identity(), "backend": backend, "python": sys.executable, "prefix": sys.prefix, + "megatron": module_runtime_identity("megatron.core", relative_paths=MEGATRON_RUNTIME_RELATIVE_PATHS), + "dependencies": dependency_versions("torch", "ray", "megatron-core", "transformer-engine", + "flashinfer-python", "sglang-kernel")} + if backend != "sglang": + raise ValueError(f"Unsupported Target backend: {backend}") from aspec.commands.doctor import inspect_environment from aspec.runtime_identity import SGLANG_RUNTIME_RELATIVE_PATHS, module_runtime_identity diff --git a/aspec/models/megatron.py b/aspec/models/megatron.py new file mode 100644 index 0000000..8b3327f --- /dev/null +++ b/aspec/models/megatron.py @@ -0,0 +1,198 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Public Megatron Core construction and strict Qwen3 weight mapping. + +Uses Megatron Core 0.20's public model/spec/process-group interfaces. Qwen3 +mapping follows the grouped QKV and gate/up layouts documented by NVIDIA's +Megatron Bridge Qwen3 adapter. +""" + +from __future__ import annotations + +import importlib.metadata +import json +from pathlib import Path + +import torch +import torch.distributed as dist +from safetensors import safe_open + +from aspec.config.megatron import MEGATRON_VERSION + + +def require_megatron(): + from packaging.version import Version + + try: + version = importlib.metadata.version("megatron-core") + except importlib.metadata.PackageNotFoundError as exc: + raise RuntimeError("Install the pinned ASpec Megatron environment") from exc + if Version(version).base_version != MEGATRON_VERSION: + raise RuntimeError(f"Megatron Core {MEGATRON_VERSION} is required; found {version}") + return version + + +def initialize_parallelism(tp_size: int, *, seed: int = 42): + """The caller's default process group contains only one execution role.""" + require_megatron() + from megatron.core import parallel_state + from megatron.core.process_groups_config import ProcessGroupCollection + + if not dist.is_initialized(): + raise RuntimeError("Initialize the role-local process group before Megatron") + if parallel_state.model_parallel_is_initialized(): + raise RuntimeError("A Megatron model-parallel world is already initialized") + parallel_state.initialize_model_parallel( + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=1, + context_parallel_size=1, + create_gloo_process_groups=False, + ) + from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed + + model_parallel_cuda_manual_seed(seed) + return ProcessGroupCollection.use_mpu_process_groups() + + +def transformer_config(config, tp_size: int): + from megatron.core.transformer.transformer_config import TransformerConfig + + hidden = int(config.hidden_size) + heads = int(config.num_attention_heads) + groups = int(config.num_key_value_heads) + intermediate = int(config.intermediate_size) + if any(value % tp_size for value in (heads, groups, intermediate, config.vocab_size)): + raise ValueError("Qwen3 heads, KV groups, FFN and vocabulary must divide Target TP") + if config.model_type != "qwen3" or getattr(config, "num_experts", None): + raise ValueError("Megatron Target currently requires dense Qwen3") + if getattr(config, "attention_bias", False): + raise ValueError("Megatron Qwen3 prefill requires bias-free attention projections") + if getattr(config, "quantization_config", None) or getattr(config, "hidden_act", "silu") != "silu": + raise ValueError("Megatron Qwen3 requires unquantized dense SwiGLU weights") + rope = getattr(config, "rope_parameters", None) or {} + scaling = getattr(config, "rope_scaling", None) or rope + if scaling.get("rope_type", scaling.get("type", "default")) != "default": + raise ValueError("Megatron Qwen3 currently requires the original default RoPE policy") + return TransformerConfig( + num_layers=2 * int(config.num_hidden_layers), + hidden_size=hidden, + ffn_hidden_size=intermediate, + num_attention_heads=heads, + num_query_groups=groups, + kv_channels=int(getattr(config, "head_dim", None) or hidden // heads), + normalization="RMSNorm", + layernorm_epsilon=float(config.rms_norm_eps), + qk_layernorm=True, + gated_linear_unit=True, + activation_func=torch.nn.functional.silu, + add_bias_linear=False, + add_qkv_bias=False, + attention_dropout=0.0, + hidden_dropout=0.0, + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=1, + context_parallel_size=1, + params_dtype=torch.bfloat16, + bf16=True, + perform_initialization=False, + use_cpu_initialization=False, + gradient_accumulation_fusion=False, + ) + + +class SafetensorReader: + """Read one source parameter at a time, with an explicit model identity.""" + + def __init__(self, path): + from huggingface_hub import snapshot_download + + self.root = Path(path) + if not self.root.is_dir(): + self.root = Path(snapshot_download(path, allow_patterns=["*.json", "*.safetensors"])) + index = self.root / "model.safetensors.index.json" + if index.is_file(): + self.files = json.loads(index.read_text())["weight_map"] + else: + with safe_open(self.root / "model.safetensors", framework="pt", device="cpu") as source: + self.files = {key: "model.safetensors" for key in source.keys()} + self.read = set() + + def tensor(self, name): + if name not in self.files: + raise ValueError(f"Target checkpoint is missing required parameter {name}") + self.read.add(name) + with safe_open(self.root / self.files[name], framework="pt", device="cpu") as source: + return source.get_tensor(name) + + +def _shard(value, axis, rank, size): + if value.shape[axis] % size: + raise ValueError(f"Tensor dimension {axis} cannot be partitioned over TP{size}") + return value.chunk(size, dim=axis)[rank].contiguous() + + +def qwen3_parameter(reader, name, config, rank, tp_size): + """Map public MCore parameter names, preserving native Qwen3 values.""" + if name == "embedding.word_embeddings.weight": + return _shard(reader.tensor("model.embed_tokens.weight"), 0, rank, tp_size) + if name == "final_norm.weight": + return reader.tensor("model.norm.weight") + pieces = name.split(".") + if pieces[:2] != ["decoder", "layers"] or len(pieces) < 5: + raise ValueError(f"Unsupported Megatron Qwen3 parameter: {name}") + prefix = f"model.layers.{int(pieces[2]) // 2}." + suffix = ".".join(pieces[3:]) + aliases = { + "input_layernorm.weight": "input_layernorm.weight", + "pre_mlp_layernorm.weight": "post_attention_layernorm.weight", + "self_attention.q_layernorm.weight": "self_attn.q_norm.weight", + "self_attention.k_layernorm.weight": "self_attn.k_norm.weight", + } + if suffix in aliases: + return reader.tensor(prefix + aliases[suffix]) + if suffix == "self_attention.linear_qkv.weight": + groups = int(config.num_key_value_heads) + heads_per_group = int(config.num_attention_heads) // groups + width = int(getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads) + query = reader.tensor(prefix + "self_attn.q_proj.weight").view(groups, heads_per_group, width, -1) + key = reader.tensor(prefix + "self_attn.k_proj.weight").view(groups, 1, width, -1) + value = reader.tensor(prefix + "self_attn.v_proj.weight").view(groups, 1, width, -1) + return _shard(torch.cat((query, key, value), dim=1), 0, rank, tp_size).flatten(0, 2) + if suffix == "self_attention.linear_proj.weight": + return _shard(reader.tensor(prefix + "self_attn.o_proj.weight"), 1, rank, tp_size) + if suffix == "mlp.linear_fc1.weight": + return torch.cat([ + _shard(reader.tensor(prefix + "mlp." + part + ".weight"), 0, rank, tp_size) + for part in ("gate_proj", "up_proj") + ], dim=0) + if suffix == "mlp.linear_fc2.weight": + return _shard(reader.tensor(prefix + "mlp.down_proj.weight"), 1, rank, tp_size) + raise ValueError(f"Unsupported Megatron Qwen3 parameter: {name}") + + +def load_qwen3_target(path, groups, *, implementation="transformer_engine"): + """Load the frozen feature-only HybridModel, including its final norm.""" + from transformers import AutoConfig + + from aspec.models.megatron_target import build_target + + reader = SafetensorReader(path) + config = AutoConfig.from_pretrained(reader.root, trust_remote_code=False) + tp_size = dist.get_world_size(groups.tp) + rank = dist.get_rank(groups.tp) + core_config = transformer_config(config, tp_size) + model = build_target(config, core_config, groups, implementation) + model.to(dtype=torch.bfloat16) + with torch.no_grad(): + for name, parameter in model.named_parameters(): + value = qwen3_parameter(reader, name, config, rank, tp_size) + if tuple(value.shape) != tuple(parameter.shape): + raise ValueError(f"Qwen3 mapping shape mismatch for {name}: {value.shape} != {parameter.shape}") + parameter.copy_(value) + unused = set(reader.files) - reader.read + # The feature-only Target never constructs the vocabulary projection. + unused.discard("lm_head.weight") + if unused: + raise ValueError(f"Unmapped Target checkpoint parameters: {sorted(unused)}") + model.eval().requires_grad_(False) + return model, config diff --git a/aspec/models/megatron_draft.py b/aspec/models/megatron_draft.py new file mode 100644 index 0000000..970cccc --- /dev/null +++ b/aspec/models/megatron_draft.py @@ -0,0 +1,121 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Megatron TP projections around the retained optimized DSpark computation. + +The vocabulary objective and its bounded/tiled caches remain replicated in this +CP1 implementation. Only attention heads and dense FFN channels are partitioned. +This avoids gathering vocabulary logits or changing global probability semantics. +""" + +import torch +from torch.distributed.tensor import DTensor, Shard + +from aspec.models.megatron import require_megatron + + +def draft_transformer_config(config, tp_size): + require_megatron() + from megatron.core.transformer.transformer_config import TransformerConfig + + for name in ("num_attention_heads", "num_key_value_heads", "intermediate_size"): + if getattr(config, name) % tp_size: + raise ValueError(f"Draft {name} must be divisible by TP{tp_size}") + return TransformerConfig( + num_layers=config.num_hidden_layers, hidden_size=config.hidden_size, + ffn_hidden_size=config.intermediate_size, + num_attention_heads=config.num_attention_heads, + num_query_groups=config.num_key_value_heads, + kv_channels=int(getattr(config, "head_dim", None) + or config.hidden_size // config.num_attention_heads), normalization="RMSNorm", + layernorm_epsilon=config.rms_norm_eps, gated_linear_unit=True, + activation_func=torch.nn.functional.silu, add_bias_linear=False, + params_dtype=torch.bfloat16, bf16=True, + tensor_model_parallel_size=tp_size, context_parallel_size=1, + pipeline_model_parallel_size=1, sequence_parallel=False, + perform_initialization=False, use_cpu_initialization=False, + gradient_accumulation_fusion=False, attention_dropout=0.0, hidden_dropout=0.0, + ) + + +def checkpoint_tensor(tensor, parameter, tp_mesh): + """Expose the logical tensor to DCP without changing compute parameters.""" + axis = getattr(parameter, "_aspec_tp_axis", None) + if isinstance(tensor, DTensor) or axis is None or tensor.ndim == 0: + return tensor + return DTensor.from_local(tensor, device_mesh=tp_mesh, placements=(Shard(axis),), + run_check=False) + + +def partition_dspark(draft, core_config, groups, tp_mesh): + from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear + + class Column(ColumnParallelLinear): + def forward(self, value): + output, bias = super().forward(value) + if bias is not None: + raise RuntimeError("DSpark TP projections must be bias-free") + return output + + class Row(RowParallelLinear): + def forward(self, value): + output, bias = super().forward(value) + if bias is not None: + raise RuntimeError("DSpark TP projections must be bias-free") + return output + + rank, size = groups.tp.rank(), groups.tp.size() + + def replace(owner, name, axis): + original = getattr(owner, name) + if original.bias is not None: + raise ValueError("Megatron DSpark requires bias-free projections") + options = {"gather_output": False} if axis == 0 else {"input_is_parallel": True} + layer_type = Column if axis == 0 else Row + replacement = layer_type( + original.in_features, original.out_features, config=core_config, + init_method=core_config.init_method, bias=False, skip_bias_add=False, + tp_group=groups.tp, **options, + ) + with torch.no_grad(): + replacement.weight.copy_(original.weight.chunk(size, dim=axis)[rank]) + replacement.weight.requires_grad_(original.weight.requires_grad) + replacement.weight._aspec_tp_axis = axis + setattr(owner, name, replacement) + + for layer in draft.layers: + attention = layer.self_attn + for name in ("q_proj", "k_proj", "v_proj"): + replace(attention, name, 0) + replace(attention, "o_proj", 1) + attention.num_heads //= size + attention.num_kv_heads //= size + for name in ("gate_proj", "up_proj"): + replace(layer.mlp, name, 0) + replace(layer.mlp, "down_proj", 1) + draft.cuda() + for layer in draft.layers: + for norm in (layer.self_attn.q_norm, layer.self_attn.k_norm): + norm.weight._aspec_tp_partial_gradient = True + draft._aspec_tp_mesh = tp_mesh + draft._aspec_tp_group = groups.tp + return draft + + +def model_checkpoint_state(model, tp_mesh): + parameters = dict(model.named_parameters()) + return {name: checkpoint_tensor(value, parameters[name], tp_mesh) + if name in parameters else value + for name, value in model.state_dict().items() if isinstance(value, torch.Tensor)} + + +def load_model_checkpoint_state(model, state): + current = {name: value for name, value in model.state_dict().items() + if isinstance(value, torch.Tensor)} + if set(current) != set(state): + raise ValueError("Megatron checkpoint does not cover the complete DSpark state") + with torch.no_grad(): + for name, value in state.items(): + local = value.to_local() if isinstance(value, DTensor) else value + if current[name].shape != local.shape: + raise ValueError(f"Megatron checkpoint shape mismatch: {name}") + current[name].copy_(local) diff --git a/aspec/models/megatron_target.py b/aspec/models/megatron_target.py new file mode 100644 index 0000000..84b3803 --- /dev/null +++ b/aspec/models/megatron_target.py @@ -0,0 +1,156 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Frozen single-sequence Qwen3 prefill on the upstream Core HybridStack.""" + +from contextvars import ContextVar +from dataclasses import dataclass +from functools import partial + +import torch +from flashinfer.norm import fused_add_rmsnorm, rmsnorm +from flashinfer.rope import apply_rope_with_cos_sin_cache_inplace +from megatron.core.models.backends import get_backend +from megatron.core.models.hybrid.hybrid_block import HybridStack, HybridStackSubmodules +from megatron.core.models.hybrid.hybrid_model import HybridModel +from megatron.core.transformer.attention import SelfAttention, SelfAttentionSubmodules +from megatron.core.transformer.enums import AttnMaskType +from megatron.core.transformer.mlp import MLP, MLPSubmodules +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_layer import TransformerLayer, TransformerLayerSubmodules +from sgl_kernel import silu_and_mul +from sgl_kernel.flash_attn import flash_attn_varlen_func + + +class NormWeight(torch.nn.Module): + """Checkpoint gamma for a normalization executed by the prefill kernels.""" + + def __init__(self, *, config, hidden_size, eps): + super().__init__() + self.weight = torch.nn.Parameter(torch.empty(hidden_size, dtype=config.params_dtype)) + self.eps = eps + + +@dataclass +class _Prefill: + positions: torch.Tensor + bounds: torch.Tensor + layers: frozenset + capture: object + residual: torch.Tensor | None = None + rope_cache: torch.Tensor | None = None + + def normalize(self, branch, norm): + if self.residual is None: + self.residual = branch + return rmsnorm(branch, norm.weight, norm.eps) + fused_add_rmsnorm(branch, self.residual, norm.weight, norm.eps) + return branch + + +# Core passes a single activation between HybridStack positions. The fused norm +# also needs the pending residual. Scope that second stream to one model call. +_prefill = ContextVar("qwen3_target_prefill") + + +def _normalize_heads(value, norm): + # Warp reduction order for Qwen3 Q/K gamma, with FP32 lane accumulators. + x = value.float() + lanes = x.unflatten(-1, (32, -1)) + sums = lanes[..., 0].square() + for component in lanes.unbind(-1)[1:]: + sums = torch.addcmul(sums, component, component) + for stride in (16, 8, 4, 2, 1): + sums = sums[..., :stride] + sums[..., stride:] + inv_rms = torch.rsqrt(sums / x.shape[-1] + norm.eps) + return (x * inv_rms * norm.weight.float()).to(value.dtype) + + +class Qwen3AttentionLayer(TransformerLayer): + def forward(self, hidden_states, rotary_pos_emb, **kwargs): + state = _prefill.get() + attention = self.self_attention + width = self.config.kv_channels + length = hidden_states.size(0) + if state.rope_cache is None: + angles = rotary_pos_emb[:length, 0, 0, :width // 2].contiguous() + state.rope_cache = torch.cat((angles.cos(), angles.sin()), dim=-1) + normalized = state.normalize(hidden_states.squeeze(1), self.input_layernorm) + packed, _ = attention.linear_qkv(normalized.unsqueeze(1)) + queries = self.config.num_attention_heads // self.config.num_query_groups + packed = packed.reshape(length, -1, queries + 2, width) + q = _normalize_heads(packed[:, :, :queries].reshape(length, -1, width), attention.q_layernorm) + k = _normalize_heads(packed[:, :, queries], attention.k_layernorm) + v = packed[:, :, queries + 1].contiguous() + apply_rope_with_cos_sin_cache_inplace( + state.positions, q.flatten(1), k.flatten(1), width, state.rope_cache, True, + ) + context = flash_attn_varlen_func(q, k, v, state.bounds, state.bounds, length, length, causal=True) + branch, _ = attention.linear_proj(context.reshape(length, 1, -1)) + return branch, None + + +class Qwen3MLPLayer(TransformerLayer): + def forward(self, hidden_states, **kwargs): + state = _prefill.get() + normalized = state.normalize(hidden_states.squeeze(1), self.pre_mlp_layernorm) + gate_up, _ = self.mlp.linear_fc1(normalized.unsqueeze(1)) + branch, _ = self.mlp.linear_fc2(silu_and_mul(gate_up.squeeze(1)).unsqueeze(1)) + logical_layer = (self.layer_number - 1) // 2 + if state.capture is not None and logical_layer in state.layers: + state.capture(logical_layer, branch.squeeze(1), state.residual) + return branch, None + + +class Qwen3Target(HybridModel): + """ASpec feature endpoint. Core owns embeddings, layer order and TP projections.""" + + @torch.inference_mode() + def prefill(self, ids, layers, capture=None): + if ids.ndim != 2 or ids.size(0) != 1 or not 0 < ids.size(1) <= self.max_sequence_length: + raise ValueError("Qwen3 Target expects one nonempty sequence within its context") + if any(layer < 0 or layer >= self.config.num_layers // 2 for layer in layers): + raise ValueError("Qwen3 Target capture is outside the logical decoder") + positions = torch.arange(ids.size(1), device=ids.device) + bounds = torch.tensor([0, ids.size(1)], device=ids.device, dtype=torch.int32) + state = _Prefill(positions, bounds, frozenset(layers), capture) + token = _prefill.set(state) + try: + branch = self(input_ids=ids, position_ids=positions.unsqueeze(0), attention_mask=None) + return state.normalize(branch.squeeze(1), self.final_norm) + finally: + _prefill.reset(token) + + +def build_target(config, core_config, groups, implementation): + if core_config.kv_channels % 32: + raise ValueError("Qwen3 prefill requires heads divisible by a warp") + backend = get_backend(implementation) + attention = ModuleSpec( + module=SelfAttention, params={"attn_mask_type": AttnMaskType.causal}, + submodules=SelfAttentionSubmodules( + linear_qkv=backend.column_parallel_linear(), linear_proj=backend.row_parallel_linear(), + core_attention=backend.core_attention(), q_layernorm=NormWeight, k_layernorm=NormWeight, + ), + ) + mlp = partial(MLP.as_mlp_submodule, submodules=MLPSubmodules( + linear_fc1=backend.column_parallel_linear(), linear_fc2=backend.row_parallel_linear(), + )) + stack = ModuleSpec(module=HybridStack, submodules=HybridStackSubmodules( + attention_layer=ModuleSpec(module=Qwen3AttentionLayer, submodules=TransformerLayerSubmodules( + input_layernorm=NormWeight, self_attention=attention)), + mlp_layer=ModuleSpec(module=Qwen3MLPLayer, submodules=TransformerLayerSubmodules( + pre_mlp_layernorm=NormWeight, mlp=mlp)), + )) + rope = getattr(config, "rope_parameters", None) or {} + with torch.device("cuda"): + model = Qwen3Target( + config=core_config, hybrid_stack_spec=stack, + hybrid_layer_pattern="*-" * config.num_hidden_layers, + vocab_size=config.vocab_size, max_sequence_length=config.max_position_embeddings, + post_process=False, position_embedding_type="rope", + rotary_base=rope.get("rope_theta", getattr(config, "rope_theta", 1000000.0)), + pg_collection=groups, + ) + model.final_norm = NormWeight(config=core_config, hidden_size=config.hidden_size, + eps=core_config.layernorm_epsilon) + return model diff --git a/aspec/planning.py b/aspec/planning.py index b83445d..cd70bf7 100644 --- a/aspec/planning.py +++ b/aspec/planning.py @@ -47,12 +47,14 @@ resolve_draft_execution_capability, ) from aspec.config.draft_structure import resolve_draft_structure +from aspec.config.megatron import resolve_megatron_gradient_precision, validate_megatron_execution from aspec.config.mooncake_config import MooncakeConfig, resolve_rdma_routes from aspec.config.train_config import config_to_flat_args from aspec.feature_contract import FeatureContractError, build_target_feature_contract from aspec.inference.concurrency import direct_output_sample_capacity from aspec.inference.context import ( ContextContractError, + compile_megatron_context_contract, compile_offline_context_contract, compile_sglang_context_contract, ) @@ -841,7 +843,16 @@ def _resolve_batch_size(args: argparse.Namespace) -> None: ) args.attention_backend = execution.attention_backend args.draft_context_parallel_size = execution.context_parallel_size - if execution.direct_cp_backend: + if getattr(args, "train_backend", "fsdp") == "megatron": + tp_size = getattr(args, "draft_tensor_parallel_size", 1) + if type(tp_size) is not int or tp_size < 1 or world_size % tp_size: + raise PlanCompileError("Draft world size must be divisible by positive draft_tensor_parallel_size") + dp_size = world_size // tp_size + args.dp_size = dp_size + args.sp_size = 1 + args.per_dp_rank_batch_size = args.micro_batch_size + sp_size = 1 + elif execution.direct_cp_backend: cp_size = execution.context_parallel_size if world_size % cp_size != 0: raise PlanCompileError( @@ -936,10 +947,14 @@ def _validate_usp_args(args: argparse.Namespace) -> None: def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: - if getattr(args, "train_backend", None) != "fsdp": + if getattr(args, "train_backend", None) not in {"fsdp", "megatron"}: raise PlanCompileError( - f"The current training backend is fsdp; got {getattr(args, 'train_backend', None)!r}" + f"Training backend must be fsdp or megatron; got {getattr(args, 'train_backend', None)!r}" ) + try: + validate_megatron_execution(args, draft) + except ValueError as exc: + raise PlanCompileError(str(exc)) from exc cluster_mode = getattr(args, "cluster_mode", "single_node") if cluster_mode not in {"single_node", "static_multi_node"}: raise PlanCompileError(f"Unsupported cluster.mode {cluster_mode!r}") @@ -1042,7 +1057,10 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: raise PlanCompileError("static_multi_node requires cluster.shared_root") expected_tp = getattr(args, "sglang_nnodes", 1) * gpus_per_engine - if not offline and getattr(args, "sglang_tp_size", None) != expected_tp: + target_tp = (getattr(args, "megatron_tp_size", 1) + if getattr(args, "target_model_backend", "sglang") == "megatron" + else getattr(args, "sglang_tp_size", None)) + if not offline and target_tp != expected_tp: raise PlanCompileError( "inference.sglang.tp_size must equal sglang.nnodes * " f"inference_num_gpus_per_engine ({expected_tp})" @@ -1195,15 +1213,16 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: getattr(args, "target_prefill_memory_optimized", False) ) if direct_target_output: - if getattr(args, "inference_engine_type", "sgl") != "sgl": + if getattr(args, "inference_engine_type", "sgl") not in {"sgl", "megatron"}: raise PlanCompileError( - "target_prefill_memory_optimized currently requires SGLang" + "target_prefill_memory_optimized requires a live Target backend" ) if getattr(args, "train_with_decode", False): raise PlanCompileError( "target_prefill_memory_optimized requires prefill-only Target execution" ) - if gpus_per_engine != 1 or getattr(args, "sglang_tp_size", 1) != 1: + if (getattr(args, "inference_engine_type", "sgl") == "sgl" + and (gpus_per_engine != 1 or getattr(args, "sglang_tp_size", 1) != 1)): raise PlanCompileError( "target_prefill_memory_optimized currently requires Target TP1" ) @@ -1265,6 +1284,7 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: dispatch_batch_size = ( int(getattr(args, "training_num_gpus_per_node", 0)) // int(getattr(args, "draft_context_parallel_size", 1)) + // int(getattr(args, "draft_tensor_parallel_size", 1)) * args.per_dp_rank_batch_size ) capacity_scope = "per node" @@ -1565,6 +1585,7 @@ def _workload( target_num_hidden_layers=draft["target_num_hidden_layers"], aux_hidden_state_layers=list(draft["target_layer_ids"]), store_last_hidden_states=store_last, + producer_backend=_select(config, "model.target_model_backend", "sglang"), **({"mtp": {"native_config": draft["mtp_conditioning"]["native_config"], "embedding_key": _select(config, "model.embedding_key")}} if draft.get("mtp_conditioning") and draft["mtp_conditioning"]["mode"] == "hidden" @@ -1691,6 +1712,22 @@ def _policy( _select(config, "inference.inference_batch_size"), ), ) + megatron_input_memory = None + if runtime.get("train_backend") == "megatron": + # Receive/collate copies on the consumer thread. Leader source prefetch + # borrows registered destination slots, counted separately as rings. + resident_batches = 2 + feature_width = int(_select(config, "mooncake.hidden_dim", 4096)) + feature_bytes_per_token = 2 * feature_width * ( + len(runtime["aux_hidden_states_layers"]) + int(runtime.get("store_last_hidden_states", False))) + # Token IDs plus masks/positions alongside the captured feature ABI. + bytes_per_batch = (feature_bytes_per_token + 40) * int(runtime["max_seq_length"]) * int(runtime["micro_batch_size"]) + megatron_input_memory = { + "storage": "consumer_cuda_payloads_replicated_over_tp", + "resident_batch_bound_per_rank": resident_batches, + "payload_bytes_bound_per_rank": resident_batches * bytes_per_batch, + "excludes": "registered_rings_model_activations_and_projection_caches", + } cluster_nodes: list[dict[str, Any]] = [] locality_domains: list[dict[str, Any]] = [] ready_watermark_by_domain: dict[str, int] = {} @@ -1807,7 +1844,8 @@ def _policy( return ExecutionPolicy( backends={ - "target_features": "offline_replay" if runtime["inference_engine_type"] == "offline" else "sglang", + "target_features": ("offline_replay" if runtime["inference_engine_type"] == "offline" + else _select(config, "model.target_model_backend")), "training": _select(config, "training.train_backend"), "transport": "mooncake", }, @@ -1836,7 +1874,7 @@ def _policy( "world_size": runtime["world_size"], }, "inference": { - "gpu_owner": "target_service" if runtime["inference_engine_type"] == "sgl" else None, + "gpu_owner": "target_service" if runtime["inference_engine_type"] != "offline" else None, "nodes": ( len(_select(config, "inference.inference_node_ips") or ()) if cluster_mode == "static_multi_node" @@ -1857,15 +1895,22 @@ def _policy( parallelism={ "training": { "data_parallel": runtime["dp_size"], + "tensor_parallel": _select(config, "training.draft_tensor_parallel_size", 1), "sequence_parallel": runtime["sp_size"], "context_parallel": _select( config, "training.draft_context_parallel_size", 1 ), "attention_backend": _select(config, "training.attention_backend"), - "fsdp_strategy": _select(config, "training.fsdp_strategy"), + "fsdp_strategy": (None if runtime["train_backend"] == "megatron" + else _select(config, "training.fsdp_strategy")), + **({**resolve_megatron_gradient_precision(argparse.Namespace(**runtime)), + "vocabulary": "replicated"} + if runtime["train_backend"] == "megatron" else {}), }, "inference": { - "tensor_parallel": _select(config, "inference.sglang.tp_size"), + "tensor_parallel": (_select(config, "inference.megatron.tp_size") + if runtime["inference_engine_type"] == "megatron" + else _select(config, "inference.sglang.tp_size")), "pipeline_parallel": _select(config, "inference.sglang.pp_size"), "source_context_parallel_size": runtime["source_context_parallel_size"], }, @@ -1909,15 +1954,18 @@ def _policy( }, capacity={ "device_memory": { + **({"megatron_input_staging": megatron_input_memory} if megatron_input_memory else {}), **({"mtp_conditioning": conditioning_memory} if conditioning_memory else {}), **({"target_mtp_features": target_mtp_memory} if target_mtp_memory else {}), - "sglang_static_fraction": (None if runtime["inference_engine_type"] == "offline" + "sglang_static_fraction": (None if runtime["inference_engine_type"] != "sgl" else _select(config, "inference.sglang.mem_fraction_static")) }, "backend_tokens": { **copy.deepcopy(dict(context_contract)), "profiled_max_total_tokens": None, - "state": "not_applicable_offline" if runtime["inference_engine_type"] == "offline" else "pending_backend_probe", + "state": ("not_applicable_offline" if runtime["inference_engine_type"] == "offline" + else "no_persistent_kv_pool" if runtime["inference_engine_type"] == "megatron" + else "pending_backend_probe"), }, "transfer_storage": { "data_plane": data_plane, @@ -2049,9 +2097,18 @@ def compile_execution_plan( source["resolved_config_sha256"] = _resolved_config_sha256(config) draft = _load_draft_descriptor(config) target_family = _resolve_target_family(target_path) + if (_select(config, "training.train_backend") == "megatron" + or _select(config, "model.target_model_backend") == "megatron"): + target_raw = _read_local_target_config(target_path) + if target_family != "qwen3" or target_raw and target_raw.get("model_type") != "qwen3": + raise PlanCompileError("Megatron integration currently requires dense Qwen3 Target weights") runtime = _runtime_values(config, draft, source_context_parallel_size=source_context_parallel_size) try: - context_compiler = compile_offline_context_contract if offline else compile_sglang_context_contract + context_compiler = ( + compile_offline_context_contract if offline else + compile_megatron_context_contract if runtime["inference_engine_type"] == "megatron" + else compile_sglang_context_contract + ) context_contract = context_compiler( max_sequence_length=_select(config, "training.max_seq_length"), context_parallel_size=int( @@ -2139,7 +2196,8 @@ def compile_execution_plan( name="target_feature_engine", requirement="required", requested=_select(config, "inference.inference_engine_type"), - resolved="offline_replay" if offline else "sglang", + resolved=("offline_replay" if offline else + _select(config, "model.target_model_backend", "sglang")), ), CapabilityDecision( name="draft_trainer", @@ -2151,7 +2209,7 @@ def compile_execution_plan( name="training_backend", requirement="required", requested=_select(config, "training.train_backend"), - resolved="fsdp", + resolved=_select(config, "training.train_backend", "fsdp"), ), CapabilityDecision( name="static_multi_node_execution", diff --git a/aspec/ray/placement_group.py b/aspec/ray/placement_group.py index c945e3f..07624e0 100644 --- a/aspec/ray/placement_group.py +++ b/aspec/ray/placement_group.py @@ -559,7 +559,7 @@ def create_placement_groups(args, roles: set[str] | None = None): _ensure_ray_initialized() - if getattr(args, "inference_engine_type", None) == "sgl": + if getattr(args, "inference_engine_type", None) in {"sgl", "megatron"}: managed_local = (getattr(args, "target_service", None) or {}).get("mode") == "managed-local" if managed_local: live_nodes = [node for node in ray.nodes() if node.get("Alive")] diff --git a/aspec/ray/train_group.py b/aspec/ray/train_group.py index 1699f43..2faca83 100644 --- a/aspec/ray/train_group.py +++ b/aspec/ray/train_group.py @@ -298,6 +298,11 @@ def set_eval_queues( def _expand_queues_for_usp(self, queues): actor_count = len(self._actor_handlers) + if getattr(self.args, "train_backend", "fsdp") == "megatron": + tp = self.args.draft_tensor_parallel_size + if actor_count % tp or len(queues) != actor_count // tp: + raise ValueError("Megatron needs one input queue per data-parallel replica") + return [queues[rank // tp] for rank in range(actor_count)] if len(queues) == actor_count: return queues diff --git a/aspec/runtime_identity.py b/aspec/runtime_identity.py index 6c8461a..d9014c4 100644 --- a/aspec/runtime_identity.py +++ b/aspec/runtime_identity.py @@ -24,6 +24,18 @@ # Patched files and their native v0.5.21 request/capture/runner dependencies. # A bounded set catches divergent checkouts without hashing the entire repo. +MEGATRON_RUNTIME_RELATIVE_PATHS = ( + "package_info.py", + "models/hybrid/hybrid_model.py", + "models/hybrid/hybrid_block.py", + "models/hybrid/hybrid_layer_allocation.py", + "tensor_parallel/layers.py", + "tensor_parallel/mappings.py", + "distributed/distributed_data_parallel.py", + "transformer/spec_utils.py", + "transformer/transformer_block.py", +) + SGLANG_RUNTIME_RELATIVE_PATHS = ( "kernels/ops/attention/triton_gdn_fused_proj.py", "kernels/ops/attention/fla/chunk_delta_h.py", diff --git a/aspec/train_entry.py b/aspec/train_entry.py index 0a3602d..1c62922 100644 --- a/aspec/train_entry.py +++ b/aspec/train_entry.py @@ -302,9 +302,9 @@ def _validate_and_configure_dflash(args, draft_model_config) -> None: algo = "DSpark" if is_dspark else "DFlash" engine_type = getattr(args, "inference_engine_type", "sgl") - if engine_type not in {"sgl", "offline"}: + if engine_type not in {"sgl", "megatron", "offline"}: raise NotImplementedError( - f"{algo} currently supports inference_engine_type='sgl' or 'offline', got '{engine_type}'." + f"{algo} requires inference_engine_type=sgl, megatron or offline; got {engine_type!r}" ) if getattr(args, "defer_tokenization", False): raise NotImplementedError(f"{algo} does not support defer_tokenization=True.") @@ -458,6 +458,15 @@ def train_async_no_generation(args, evidence: RunEvidenceRecorder | None = None) inference_engine_statuses = ray.get( [engine.get_status.remote() for engine in inference_engines] ) + if args.inference_engine_type == "megatron": + # The Draft interpreter may not have any Target dependencies installed. + runtime = inference_engine_statuses[0]["runtime"] + args.megatron_target_cache_identity = { + "adapter": runtime["code_sha256"], + "core": runtime["backend_identity"]["effective_tree_sha256"], + "dependencies": runtime["dependencies"], + "torch": runtime["torch"], "cuda": runtime["cuda"], + } if evidence is not None: evidence.record_actor_initialization( training_actor_count=n_train, @@ -489,7 +498,7 @@ def train_async_no_generation(args, evidence: RunEvidenceRecorder | None = None) timer.log_summary() except BaseException: - if args.inference_engine_type == "sgl": + if args.inference_engine_type in {"sgl", "megatron"}: rollback_unused_targets(inference_engines) raise diff --git a/aspec/training/checkpoint.py b/aspec/training/checkpoint.py index 3b9fab2..67d9ed7 100644 --- a/aspec/training/checkpoint.py +++ b/aspec/training/checkpoint.py @@ -60,6 +60,8 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: "training_num_gpus_per_node", "attention_backend", "draft_context_parallel_size", + "draft_tensor_parallel_size", + "train_backend", "sp_ulysses_size", "sp_ring_size", "fsdp_strategy", @@ -68,6 +70,8 @@ def _logical_training_topology(args: Any, world_size: int) -> dict[str, Any]: value = getattr(args, field, None) if value is not None: topology[field] = value + if getattr(args, "train_backend", "fsdp") == "megatron": + topology["megatron_grad_dtype"] = getattr(args, "megatron_grad_dtype", "fp32") if getattr(args, "draft_algorithm", None) == "mtp": topology["training_objective"] = { "steps": args.mtp_steps, "step_weight_beta": args.mtp_step_weight_beta, @@ -182,6 +186,15 @@ def _validate_exact_resume_topology(args: Any, metadata: dict[str, Any], *, chec if saved_topology is not None and not isinstance(saved_topology, dict): raise RuntimeError("Exact resume checkpoint has invalid training_topology metadata") current_topology = _logical_training_topology(args, current_world_size) + if ((saved_topology or {}).get("train_backend", "fsdp") + != current_topology.get("train_backend", "fsdp") + or (saved_topology or {}).get("draft_tensor_parallel_size", 1) + != current_topology.get("draft_tensor_parallel_size", 1)): + raise RuntimeError("Exact resume cannot change the training backend or Draft TP; use weights-only initialization") + if (current_topology.get("train_backend") == "megatron" + and (saved_topology or {}).get("megatron_grad_dtype", "fp32") + != current_topology["megatron_grad_dtype"]): + raise RuntimeError("Exact resume cannot change Megatron gradient precision; use weights-only initialization") if getattr(args, "draft_algorithm", None) == "dspark": saved_objective = (saved_topology or {}).get("training_objective") or {} if saved_objective.get("lm_head_precision", "bf16") != getattr(args, "dspark_lm_head_precision", "bf16"): @@ -357,7 +370,8 @@ def _rng_state_path(checkpoint_dir: Path, rank: int) -> Path: return checkpoint_dir / "rng" / f"rank-{int(rank):05d}.pt" -def _save_rank_rng_state(checkpoint_dir: Path, *, rank: int, world_size: int) -> None: +def _save_rank_rng_state(checkpoint_dir: Path, *, rank: int, world_size: int, + backend_rng_state=None) -> None: path = _rng_state_path(checkpoint_dir, rank) temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") state = { @@ -366,6 +380,7 @@ def _save_rank_rng_state(checkpoint_dir: Path, *, rank: int, world_size: int) -> "world_size": int(world_size), "torch": torch.get_rng_state(), "cuda": torch.cuda.get_rng_state() if torch.cuda.is_available() else None, + **({"backend_rng": backend_rng_state} if backend_rng_state is not None else {}), } try: torch.save(state, temporary) @@ -472,10 +487,15 @@ def __init__(self, model): self.model = model def state_dict(self): + if callable(getattr(self.model, "_aspec_checkpoint_state", None)): + return {"model": self.model._aspec_checkpoint_state()} model_state_dict, _ = get_state_dict(self.model, optimizers=[]) return {"model": model_state_dict} def load_state_dict(self, state_dict): + if callable(getattr(self.model, "_aspec_load_checkpoint_state", None)): + self.model._aspec_load_checkpoint_state(state_dict["model"]) + return set_state_dict( self.model, optimizers=[], model_state_dict=state_dict["model"], optim_state_dict=None ) @@ -520,7 +540,8 @@ def _optimizer_load_template(self): master = self.optimizer.fp32_params[index] if len(parts) != 2 or field not in {"step", "exp_avg", "exp_avg_sq"}: raise ValueError(f"Unsupported BF16 Adam checkpoint state: {key}") - expected_shape = () if field == "step" else tuple(master.shape) + logical_master = self._checkpoint_tensor(index, master) + expected_shape = () if field == "step" else tuple(logical_master.shape) dtype = value.properties.dtype valid_dtype = ( dtype in {torch.float32, torch.float64} @@ -537,11 +558,24 @@ def _optimizer_load_template(self): self._load_template = {**template, "state": states} return self._load_template + def _checkpoint_tensor(self, index, tensor): + wrap = getattr(self.optimizer, "checkpoint_tensor", None) + return wrap(index, tensor) if wrap is not None else tensor + + def _checkpoint_optimizer_state(self, state): + if not hasattr(self.optimizer, "checkpoint_tensor"): + return state + return {**state, "state": { + index: {key: self._checkpoint_tensor(int(index), value) + if isinstance(value, torch.Tensor) else value for key, value in values.items()} + for index, values in state["state"].items() + }} + def state_dict(self): if self._is_bf16_optimizer: names = _fp32_param_names(self.optimizer) return { - "optim": ( + "optim": self._checkpoint_optimizer_state( self._optimizer_load_template() if self._load_path is not None else self.optimizer.state_dict() @@ -549,8 +583,8 @@ def state_dict(self): # Keyed by name: which parameters are trainable can differ between the run # that saved and the run that loads. "fp32_params": { - name: param.data - for name, param in zip(names, self.optimizer.fp32_params, strict=True) + name: self._checkpoint_tensor(index, param.data) + for index, (name, param) in enumerate(zip(names, self.optimizer.fp32_params, strict=True)) }, } _, optimizer_state_dict = get_state_dict(self.model, optimizers=[self.optimizer]) @@ -558,7 +592,16 @@ def state_dict(self): def load_state_dict(self, state_dict): if self._is_bf16_optimizer: - self.optimizer.load_state_dict(state_dict["optim"]) + from torch.distributed.tensor import DTensor + + optim = state_dict["optim"] + if hasattr(self.optimizer, "checkpoint_tensor"): + optim = {**optim, "state": { + index: {key: value.to_local() if isinstance(value, DTensor) else value + for key, value in values.items()} + for index, values in optim["state"].items() + }} + self.optimizer.load_state_dict(optim) saved = state_dict.get("fp32_params") if saved: names = _fp32_param_names(self.optimizer) @@ -573,7 +616,10 @@ def load_state_dict(self, state_dict): ) with torch.no_grad(): for name, master in zip(names, self.optimizer.fp32_params, strict=True): - master.data.copy_(saved[name]) + value = saved[name] + if isinstance(value, DTensor) and not isinstance(master, DTensor): + value = value.to_local() + master.copy_(value) logger.info("Restored fp32 master params from checkpoint") return set_state_dict( @@ -1111,6 +1157,9 @@ def finalize_load(actor: Any, checkpoint_payload: dict[str, Any] | None) -> None rng_state, legacy=isinstance(rng_state, dict) and "schema_version" not in rng_state, ) + restore_backend_rng = getattr(actor, "_restore_backend_rng", None) + if restore_backend_rng is not None: + restore_backend_rng(rng_state.get("backend_rng")) torch.set_rng_state(rng_state["torch"]) if torch.cuda.is_available() and rng_state.get("cuda") is not None: cuda_state = rng_state["cuda"] @@ -1180,6 +1229,8 @@ def save(actor: Any, step: int) -> None: checkpoint_dir, rank=dist.get_rank(), world_size=dist.get_world_size(), + backend_rng_state=(actor._checkpoint_backend_rng() + if hasattr(actor, "_checkpoint_backend_rng") else None), ) if dist.get_rank() == 0: diff --git a/aspec/training/data_fetcher.py b/aspec/training/data_fetcher.py index d807116..2f535bf 100644 --- a/aspec/training/data_fetcher.py +++ b/aspec/training/data_fetcher.py @@ -139,8 +139,18 @@ def __init__( packing_enabled: bool = False, feature_affinity: str = "any", consumer_node_ip: str | None = None, + tensor_parallel_group=None, + tensor_parallel_control_group=None, ): self.ray_queue = ray_queue + if (tensor_parallel_group is None) != (tensor_parallel_control_group is None): + raise ValueError("TP input requires paired GPU-data and CPU-control groups") + self.tensor_parallel_group = tensor_parallel_group + self.tensor_parallel_control_group = tensor_parallel_control_group + self._tensor_parallel_device = (torch.cuda.current_device() + if tensor_parallel_group is not None else None) + self._tp_source_prefetch = None + self._tensor_parallel_samples = self._tensor_parallel_payload_bytes = 0 self.mooncake_store = mooncake_store self.device = device self.prefetch_factor = prefetch_factor @@ -186,11 +196,21 @@ def __init__( ): raise ValueError("packing requires non-USP training and positive max_seq_length") self._init_sp_context() + if tensor_parallel_group is not None and prefetch_factor > 0 and dist.get_rank(tensor_parallel_group) == 0: + from aspec.training.tp_source_prefetch import TPSourcePrefetch + + self._tp_source_prefetch = TPSourcePrefetch( + self._read_tp_source, lambda item: _release_p2p_receive_leases(item[1]), depth=prefetch_factor) def request_stop(self) -> None: """Cancel future queue reads without interrupting a native transfer.""" self._stop_requested.set() + def close(self): + self.request_stop() + if self._tp_source_prefetch is not None: + self._tp_source_prefetch.close() + def _next_queue_item(self): # Poll only the queue wait, not Mooncake itself: close must be able to # join an idle prefetch worker before its registered buffers are freed. @@ -540,8 +560,14 @@ def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: logger.debug(f"__iter__: waiting for item from ray_queue (yield_count={yield_count})") try: - item = self._next_queue_item() + if self.tensor_parallel_group is not None: + item, data = self._tensor_parallel_item() + else: + item = self._next_queue_item() + data = None except Exception as e: + if self.tensor_parallel_group is not None: + raise logger.warning(f"__iter__: Exception waiting for data: {e}, timeout={self.timeout}") break @@ -550,7 +576,8 @@ def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: break logger.debug(f"__iter__: got item, mooncake_key={item.mooncake_key}") - data = self._load_from_mooncake(item) + if self.tensor_parallel_group is None: + data = self._load_from_mooncake(item) neutralized_count = self._resolve_and_neutralize_loss_mask( data, item.mooncake_key, neutralized_count @@ -584,6 +611,86 @@ def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: logger.debug(f"__iter__: yielding batch {yield_count}, keys={list(data.keys())}") yield data + def _tensor_parallel_item(self): + with torch.cuda.device(self._tensor_parallel_device): + return self._tensor_parallel_item_on_device() + + def _read_tp_source(self): + # No collation, GPU clones or collectives run on this producer thread. + # P2P GET writes preallocated destination slots; leases bound prefetch. + with torch.cuda.device(self._tensor_parallel_device): + item = self._next_queue_item() + return item, self._load_from_mooncake(item) if item is not None else {} + + def _tensor_parallel_item_on_device(self): + """Read a source lease once, then broadcast its full CP1 payload over TP. + + Source release and work-ledger acknowledgement belong to the TP leader. + Source GET may be prefetched by the leader. Every TP broadcast executes + on the consumer thread between microbatches, in model collective order. + Its destination lease remains attached through the consumer microbatch. + """ + group = self.tensor_parallel_group + leader = dist.get_global_rank(group, 0) + is_leader = dist.get_rank(group) == 0 + control, data, leases = [None], None, [] + if is_leader: + try: + if self._tp_source_prefetch is not None: + item, data = self._tp_source_prefetch.next() + else: + item, data = self._read_tp_source() + leases = _take_p2p_receive_leases(data) + control[0] = { + "item": item, + "values": {k: v for k, v in data.items() if not isinstance(v, torch.Tensor)}, + "tensors": [(k, tuple(v.shape), v.dtype, v.device.type) + for k, v in data.items() if isinstance(v, torch.Tensor)], + } + except BaseException as error: + control[0] = {"error": f"{type(error).__name__}: {error}"} + try: + # Waiting for the next queue item must never launch a waiting GPU + # collective on followers: CUDA allocation/synchronization in the + # current model step could otherwise wait for undispatched input. + dist.broadcast_object_list(control, src=leader, group=self.tensor_parallel_control_group) + message = control[0] + if "error" in message: + raise RuntimeError("TP input leader failed: " + message["error"]) + item = message["item"] + if item is None: + return None, None + if not is_leader: + data = dict(message["values"]) + self._feature_contract_runtime.validate_metadata( + tensor_shapes=item.tensor_shapes, tensor_dtypes=item.tensor_dtypes, + metadata=item.metadata, + ) + for key, shape, dtype, original_device in message["tensors"]: + buffer = (data[key].to(device="cuda").contiguous() if is_leader + else torch.empty(shape, device="cuda", dtype=dtype)) + dist.broadcast(buffer, src=leader, group=group) + if not is_leader: + data[key] = buffer.cpu() if original_device == "cpu" else buffer + if not is_leader: + names = self._feature_contract_runtime.contract["tensors"] + self._validate_received_feature_tensors( + {name: data[name] for name in names}, expected_shapes=item.tensor_shapes, + expected_dtypes={name: data[name].dtype for name in names}, + ) + if leases: + data["_mooncake_p2p_receive_leases"] = leases + leases = [] + self._tensor_parallel_samples += 1 + self._tensor_parallel_payload_bytes += sum( + data[key].numel() * data[key].element_size() + for key, _shape, _dtype, _device in message["tensors"] + ) + return item, data + finally: + for lease in leases: + lease.release() + def _direct_cp_get_sharded_item( self, neutralized_count: int ) -> tuple[Dict[str, torch.Tensor] | None, int]: @@ -957,6 +1064,14 @@ def get_feature_contract_status(self) -> dict[str, Any]: status["data_admission"] = { "neutralized_rank_samples": self._neutralized_rank_samples, } + if getattr(self, "tensor_parallel_group", None) is not None: + status["tensor_parallel_input"] = { + "rank": dist.get_rank(self.tensor_parallel_group), + "size": dist.get_world_size(self.tensor_parallel_group), + "broadcast_samples": self._tensor_parallel_samples, + "broadcast_payload_bytes": self._tensor_parallel_payload_bytes, + "source_read_owner": dist.get_rank(self.tensor_parallel_group) == 0, + } get_transport_status = getattr(self.mooncake_store, "get_p2p_status", None) transport = ( get_transport_status() if callable(get_transport_status) else None @@ -1011,6 +1126,8 @@ def create_mooncake_dataloader( packing_enabled: bool = False, feature_affinity: str = "any", consumer_node_ip: str | None = None, + tensor_parallel_group=None, + tensor_parallel_control_group=None, ) -> DataLoader: """Create a DataLoader that fetches from mooncake via queue. @@ -1028,7 +1145,7 @@ def create_mooncake_dataloader( collator: Collator for padding and batching samples. device: Target device for tensors. batch_size: Number of samples per batch (= per_dp_rank_batch_size). - prefetch_factor: Unused, kept for API compatibility. + prefetch_factor: Source GET queue depth for TP input; unused by other paths. timeout: Timeout in seconds for waiting on queue. None means wait forever. assistant_header_ids: Token IDs for assistant header (for loss mask skip check). end_token_ids: Token IDs for end of turn (for loss mask skip check). @@ -1061,6 +1178,8 @@ def create_mooncake_dataloader( packing_enabled=packing_enabled, feature_affinity=feature_affinity, consumer_node_ip=consumer_node_ip, + tensor_parallel_group=tensor_parallel_group, + tensor_parallel_control_group=tensor_parallel_control_group, ) def dataloader_collator(samples): @@ -1128,6 +1247,8 @@ def __init__( packing_enabled: bool = False, feature_affinity: str = "any", consumer_node_ip: str | None = None, + tensor_parallel_group=None, + tensor_parallel_control_group=None, ): self.batch_size = batch_size self._dataloader = create_mooncake_dataloader( @@ -1154,6 +1275,8 @@ def __init__( packing_enabled=packing_enabled, feature_affinity=feature_affinity, consumer_node_ip=consumer_node_ip, + tensor_parallel_group=tensor_parallel_group, + tensor_parallel_control_group=tensor_parallel_control_group, ) def __iter__(self) -> Iterator[Dict[str, torch.Tensor]]: @@ -1165,6 +1288,9 @@ def get_feature_contract_status(self) -> dict[str, Any]: def request_stop(self) -> None: self._dataloader.dataset.request_stop() + def close(self): + self._dataloader.dataset.close() + class PrefetchedDataFetcher: """Wraps MooncakeDataFetcher with async pre-fetching. diff --git a/aspec/training/dflash_trainer.py b/aspec/training/dflash_trainer.py index 447c680..a944f8e 100644 --- a/aspec/training/dflash_trainer.py +++ b/aspec/training/dflash_trainer.py @@ -94,6 +94,22 @@ def __init__(self, args: Namespace): self.verifier_norm: Optional[torch.nn.Module] = None self._mtp_binding_validated = False + _optimizer_class = BF16Optimizer + + def _loads_initial_weights_here(self): + return dist.get_rank() == 0 + + def _wrap_training_model(self, model, draft_model): + full_state = model.state_dict() if dist.get_rank() == 0 else {} + model = apply_fsdp2( + model, mesh=self.dp_mesh, cpu_offload=self.fsdp_cpu_offload, + args=self.args, modules_to_shard=list(draft_model.layers), + ) + return fsdp2_load_full_state_dict( + model, full_state, self.dp_mesh, + cpu_offload=True if self.fsdp_cpu_offload else None, + ) + def init_model( self, draft_model_config, @@ -145,7 +161,7 @@ def init_model( draft_model = self._build_draft_model(config) conditioning = getattr(config, "mtp_conditioning", None) - if dist.get_rank() == 0: + if self._loads_initial_weights_here(): if conditioning: draft_model.initialize_mtp_conditioning(target_model_path) initial_path = getattr(self.args, "initial_draft_model_path", None) @@ -175,22 +191,7 @@ def init_model( dflash_model = self._build_training_wrapper(draft_model) - full_state = dflash_model.state_dict() if dist.get_rank() == 0 else {} - - dflash_model = apply_fsdp2( - dflash_model, - mesh=self.dp_mesh, - cpu_offload=self.fsdp_cpu_offload, - args=self.args, - modules_to_shard=list(draft_model.layers), - ) - - dflash_model = fsdp2_load_full_state_dict( - dflash_model, - full_state, - self.dp_mesh, - cpu_offload=True if self.fsdp_cpu_offload else None, - ) + dflash_model = self._wrap_training_model(dflash_model, draft_model) if getattr(self.args, "compile_model", False): logger.info("Compiling DFlash model with torch.compile (inductor backend)") @@ -215,7 +216,7 @@ def init_model( decay_style = getattr(self.args, "lr_decay_style", "cosine") warmup_ratio = getattr(self.args, "warmup_ratio", 0.1) - self.optimizer = BF16Optimizer( + self.optimizer = self._optimizer_class( self.draft_model, lr=self.args.learning_rate, weight_decay=getattr(self.args, "weight_decay", 0.0), @@ -274,7 +275,7 @@ def init_model( self.prof.on_init_end() - logger.info(f"[Rank {self.dp_rank}] DFlash model initialized with FSDP2") + logger.info("[DP rank %s] Draft initialized with %s", self.dp_rank, self.execution_backend) return 0 diff --git a/aspec/training/megatron_optimizer.py b/aspec/training/megatron_optimizer.py new file mode 100644 index 0000000..1117b0f --- /dev/null +++ b/aspec/training/megatron_optimizer.py @@ -0,0 +1,47 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Retain ASpec's BF16 master-AdamW semantics over Megatron TP/DDP buffers.""" + +import torch +import torch.distributed as dist + +from aspec.models.megatron_draft import checkpoint_tensor +from aspec.training.optimizer import BF16Optimizer + + +class MegatronBF16Optimizer(BF16Optimizer): + def _model_gradient(self, parameter): + return parameter.main_grad + + def checkpoint_tensor(self, index, value): + return checkpoint_tensor(value, self.model_params[index], self.model._aspec_tp_mesh) + + def _clip_gradients(self): + group = self.model._aspec_tp_group + # Replicated vocabulary/confidence/norm parameters contribute once; + # every distinct attention/FFN shard contributes to the global norm. + gradients = [master.grad for parameter, master in zip( + self.model_params, self.fp32_params, strict=True) + if master.grad is not None + and (getattr(parameter, "_aspec_tp_axis", None) is not None or group.rank() == 0)] + norm = torch.nn.utils.get_total_norm(gradients).square() + dist.all_reduce(norm, op=dist.ReduceOp.SUM, group=group) + norm.sqrt_() + coefficient = (self.max_grad_norm / (norm + 1e-6)).clamp(max=1.0) + if self._fuse_clip_scaling: + self.optimizer.grad_scale = coefficient.reciprocal() + else: + from aspec.models.ops.gradient_scaling import scale_gradients_ + + self.optimizer.grad_scale = None + scale_gradients_([p.grad for p in self.fp32_params if p.grad is not None], coefficient) + return norm + + def step(self, closure=None): + norm = super().step(closure) + self.model._aspec_ddp.zero_grad_buffer() + return norm + + def zero_grad(self, set_to_none=True): + super().zero_grad(set_to_none=set_to_none) + self.model._aspec_ddp.zero_grad_buffer() diff --git a/aspec/training/megatron_trainer.py b/aspec/training/megatron_trainer.py new file mode 100644 index 0000000..209253b --- /dev/null +++ b/aspec/training/megatron_trainer.py @@ -0,0 +1,206 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""DSpark's retained training loop with upstream Megatron TP and DDP. + +Inspired by the public Megatron Core 0.20 custom-model DDP interface. DSpark +algorithm, batching, optimizer-step normalization and recovery remain ASpec-owned. +""" + +import weakref +from pathlib import Path + +import torch +import torch.distributed as dist +from torch.distributed.device_mesh import DeviceMesh +from torch.distributed.tensor import DTensor + +from aspec.config.megatron import resolve_megatron_gradient_precision +from aspec.models.megatron import initialize_parallelism +from aspec.models.megatron_draft import ( + draft_transformer_config, + load_model_checkpoint_state, + model_checkpoint_state, + partition_dspark, +) +from aspec.training.dspark_trainer import DSparkTrainer +from aspec.training.megatron_optimizer import MegatronBF16Optimizer + + +class MegatronDSparkTrainer(DSparkTrainer): + execution_backend = "megatron" + _optimizer_class = MegatronBF16Optimizer + + def _setup_device_mesh(self): + self.groups = initialize_parallelism(self.args.draft_tensor_parallel_size, + seed=getattr(self.args, "seed", 42)) + self.tp_size, self.tp_rank = self.groups.tp.size(), self.groups.tp.rank() + self.dp_size, self.dp_rank = self.groups.dp.size(), self.groups.dp.rank() + self.dp_group = self.groups.dp + self.cp_size, self.cp_rank, self.cp_group = 1, 0, None + self.cache_rank = dist.get_rank() + self.tp_mesh = DeviceMesh.from_group(self.groups.tp, "cuda") + self.dp_mesh = DeviceMesh.from_group(self.groups.dp, "cuda") + self.mesh = self.grad_sync_mesh = self.dp_mesh + # Only source GET is prefetched. Input collectives run on the model's + # consumer thread, in a consistent order across overlapping TP/DP groups. + for replica in range(self.dp_size): + ranks = list(range(replica * self.tp_size, (replica + 1) * self.tp_size)) + group = dist.new_group(ranks, backend="nccl") + control_group = dist.new_group(ranks, backend="gloo") + eval_group = dist.new_group(ranks, backend="nccl") + eval_control_group = dist.new_group(ranks, backend="gloo") + if replica == self.dp_rank: + self.input_tp_group = group + self.input_tp_control_group = control_group + self.eval_input_tp_group = eval_group + self.eval_input_tp_control_group = eval_control_group + + def _get_init_weight_context_manager(self): + return lambda: torch.device("cpu") + + def _loads_initial_weights_here(self): + return True + + def _build_training_wrapper(self, draft_model): + wrapper = super()._build_training_wrapper(draft_model) + sample = wrapper._sample_anchor_positions + wrapper._aspec_anchor_tp_broadcasts = 0 + + def shared_anchors(*args, **kwargs): + anchors, keep = sample(*args, **kwargs) + if self.tp_size > 1: + leader = dist.get_global_rank(self.groups.tp, 0) + dist.broadcast(anchors, src=leader, group=self.groups.tp) + dist.broadcast(keep, src=leader, group=self.groups.tp) + wrapper._aspec_anchor_tp_broadcasts += 1 + return anchors, keep + + wrapper._sample_anchor_positions = shared_anchors + return wrapper + + def _wrap_training_model(self, model, draft_model): + from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig + from megatron.core.tensor_parallel import ( + set_defaults_if_not_set_tensor_model_parallel_attributes, + ) + + config = draft_transformer_config(draft_model.config, self.tp_size) + partition_dspark(draft_model, config, self.groups, self.tp_mesh) + for parameter in model.parameters(): + set_defaults_if_not_set_tensor_model_parallel_attributes(parameter) + precision = resolve_megatron_gradient_precision(self.args) + ddp = DistributedDataParallel( + config, DistributedDataParallelConfig( + grad_reduce_in_fp32=precision["core_grad_reduce_in_fp32"], overlap_grad_reduce=True, + average_in_collective=True, use_distributed_optimizer=False), + model, pg_collection=self.groups, + ) + ddp.broadcast_params() + self._tp_partial_gradients = [p.main_grad for p in model.parameters() + if getattr(p, "_aspec_tp_partial_gradient", False)] + self._tp_partial_buffer = torch.empty( + sum(g.numel() for g in self._tp_partial_gradients), device="cuda", dtype=torch.float32, + ) + self._tp_partial_views = list(self._tp_partial_buffer.split( + [g.numel() for g in self._tp_partial_gradients])) + self._gradient_precision = { + "configured": getattr(self.args, "megatron_grad_dtype", "fp32"), + **precision, + "observed_main_grad_dtypes": sorted({str(p.main_grad.dtype) for p in model.parameters() + if p.requires_grad}), + "observed_core_grad_reduce_in_fp32": ddp.ddp_config.grad_reduce_in_fp32, + "observed_tp_partial_buffer_dtype": str(self._tp_partial_buffer.dtype), + } + # A weak reference avoids making a second module parent / state-dict cycle. + object.__setattr__(draft_model, "_aspec_ddp", weakref.proxy(ddp)) + ddp._aspec_checkpoint_state = lambda: model_checkpoint_state(model, self.tp_mesh) + ddp._aspec_load_checkpoint_state = lambda state: load_model_checkpoint_state(model, state) + return ddp + + def _normalize_accumulated_gradients(self, local_denominator, accumulation_steps): + self.model.finish_grad_sync() + if self.tp_size > 1: + flattened = [g.view(-1) for g in self._tp_partial_gradients] + torch._foreach_copy_(self._tp_partial_views, flattened) + dist.all_reduce(self._tp_partial_buffer, group=self.groups.tp) + torch._foreach_copy_(flattened, self._tp_partial_views) + denominator = local_denominator.detach().clone() + dist.all_reduce(denominator, group=self.dp_group) + scale = torch.where( + denominator > 0, + self.dp_size * accumulation_steps / denominator.clamp_min(1e-6), + torch.zeros_like(denominator), + ) + from aspec.models.ops.gradient_scaling import scale_gradients_ + + scale_gradients_([p.main_grad for p in self.optimizer.model_params], scale) + return denominator + + def _checkpoint_backend_rng(self): + from megatron.core.tensor_parallel.random import get_cuda_rng_tracker + + return {"backend": "megatron", "cuda_tracker": get_cuda_rng_tracker().get_states()} + + def _restore_backend_rng(self, state): + from megatron.core.tensor_parallel.random import get_cuda_rng_tracker + + if not isinstance(state, dict) or state.get("backend") != "megatron": + raise RuntimeError("Exact Megatron resume requires the model-parallel RNG tracker") + saved = state.get("cuda_tracker") + if not isinstance(saved, dict) or set(saved) != set(get_cuda_rng_tracker().get_states()): + raise RuntimeError("Megatron RNG tracker streams differ from the checkpoint") + if any(not isinstance(value, torch.Tensor) or value.dtype != torch.uint8 + or value.device.type != "cpu" or value.ndim != 1 for value in saved.values()): + raise RuntimeError("Invalid Megatron RNG tracker state") + get_cuda_rng_tracker().set_states(saved) + + def pop_committed_work(self, optimizer_step): + local = super().pop_committed_work(optimizer_step) + replicas = [None] * self.tp_size + dist.all_gather_object(replicas, local, group=self.groups.tp) + if any(items != local for items in replicas): + raise RuntimeError("TP ranks completed different ASpec work identities") + return local if self.tp_rank == 0 else [] + + def _kernel_observation(self): + wrapper = getattr(self, "dflash", None) + if getattr(wrapper, "l1_loss_backend", None) is None: + return None + return { + "l1_backend": wrapper.l1_loss_backend, + "l1_kernel_invocations": dict(wrapper.l1_kernel_invocations), + "anchor_tp_broadcasts": wrapper._aspec_anchor_tp_broadcasts, + "qk_preparation_backends": [layer.self_attn.qk_prepare_backend + for layer in self.draft_model.layers], + "gradient_checkpointing_enabled": self.draft_model.gradient_checkpointing, + "optimizer_fused_clip_scaling": self.optimizer._fuse_clip_scaling, + "gradient_precision": dict(self._gradient_precision), + "last_batch": {name.removesuffix("_observed"): value + for name, value in vars(wrapper).items() + if name.endswith("_observed") and type(value) in (int, float)}, + "attention_layers": list(self.draft_model.last_attention_observations), + } + + def get_feature_contract_status(self): + status = super().get_feature_contract_status() + if observation := self._kernel_observation(): + status["dspark_execution"] = observation + return status + + def save_draft_model_for_serving(self, output_dir): + error = None + try: + if self.dp_rank == 0: + state = model_checkpoint_state(self.draft_model, self.tp_mesh) + state = {name: (value.full_tensor() if isinstance(value, DTensor) else value).cpu() + for name, value in state.items()} + if self.tp_rank == 0: + root = Path(output_dir) + root.mkdir(parents=True, exist_ok=True) + self.draft_model.config.save_pretrained(root) + torch.save(state, root / "pytorch_model.bin") + except Exception as caught: + error = caught + message = self._synchronize_serving_error(error) + if message is not None: + raise RuntimeError("Megatron Draft export failed: " + message) diff --git a/aspec/training/optimizer.py b/aspec/training/optimizer.py index bb5a06e..929a414 100644 --- a/aspec/training/optimizer.py +++ b/aspec/training/optimizer.py @@ -85,6 +85,20 @@ def __init__( wsd_decay_style=wsd_decay_style, ) + def _model_gradient(self, parameter): + return parameter.grad + + def _clip_gradients(self): + if self._fuse_clip_scaling: + grad_norm = torch.nn.utils.get_total_norm( + [p.grad for p in self.fp32_params if p.grad is not None] + ) + coefficient = (self.max_grad_norm / (grad_norm + 1e-6)).clamp(max=1.0) + self.optimizer.grad_scale = coefficient.reciprocal() + return grad_norm + self.optimizer.grad_scale = None + return torch.nn.utils.clip_grad_norm_(self.fp32_params, self.max_grad_norm) + def step(self, closure=None): """Perform optimizer step with gradient clipping. @@ -98,27 +112,17 @@ def step(self, closure=None): grad_destinations = [] grad_sources = [] for p, mp, g in zip(self.model_params, self.fp32_params, self.fp32_grads): - if p.grad is not None: + gradient = self._model_gradient(p) + if gradient is not None: grad_destinations.append(g) - grad_sources.append(p.grad) + grad_sources.append(gradient) mp.grad = g else: mp.grad = None if grad_destinations: _copy_tensors_(grad_destinations, grad_sources) - if self._fuse_clip_scaling: - grad_norm = torch.nn.utils.get_total_norm( - [p.grad for p in self.fp32_params if p.grad is not None] - ) - # Fused AdamW already supports device-side gradient unscaling. - # Apply the same clipping coefficient there, avoiding a complete - # read/write pass over the FP32 gradient buffers on Hopper. - coefficient = (self.max_grad_norm / (grad_norm + 1e-6)).clamp(max=1.0) - self.optimizer.grad_scale = coefficient.reciprocal() - else: - self.optimizer.grad_scale = None - grad_norm = torch.nn.utils.clip_grad_norm_(self.fp32_params, self.max_grad_norm) + grad_norm = self._clip_gradients() # Fused AdamW consumes this device scalar through the same path used by # GradScaler. A nonzero value skips parameter, moment, weight-decay, and diff --git a/aspec/training/tp_source_prefetch.py b/aspec/training/tp_source_prefetch.py new file mode 100644 index 0000000..460aa55 --- /dev/null +++ b/aspec/training/tp_source_prefetch.py @@ -0,0 +1,63 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Bounded source GET prefetch; GPU collectives stay on the consumer thread.""" + +import queue +import threading + + +class TPSourcePrefetch: + def __init__(self, read, release, *, depth): + self.read, self.release = read, release + self.queue = queue.Queue(maxsize=depth) + self.stop = threading.Event() + self.thread = None + self.error = None + self.exhausted = False + + def _run(self): + try: + while not self.stop.is_set(): + item = self.read() + while not self.stop.is_set(): + try: + self.queue.put(item, timeout=0.1) + break + except queue.Full: + continue + else: + self.release(item) + return + if item[0] is None: + return + except BaseException as error: + self.error = error + + def next(self): + if self.exhausted: + return None, {} + if self.thread is None: + self.thread = threading.Thread(target=self._run, daemon=True, name="tp-source-get") + self.thread.start() + while not self.stop.is_set(): + if self.error is not None: + raise self.error + try: + item = self.queue.get(timeout=0.1) + self.exhausted = item[0] is None + return item + except queue.Empty: + continue + raise StopIteration + + def close(self): + self.stop.set() + if self.thread is not None: + self.thread.join(timeout=10) + if self.thread.is_alive(): + raise RuntimeError("TP source GET did not stop; retain backing storage until the read fence") + while True: + try: + self.release(self.queue.get_nowait()) + except queue.Empty: + return diff --git a/aspec/training/trainer.py b/aspec/training/trainer.py index adf4900..f4eb9d9 100644 --- a/aspec/training/trainer.py +++ b/aspec/training/trainer.py @@ -74,12 +74,15 @@ def _consumer_participant( *, dp_rank: int, sequence_rank: int | None = None, + tensor_rank: int = 0, ) -> str: """Build a stable, unique identity for one feature/storage consumer.""" participant = f"draft_{role}_dp_rank_{dp_rank}" if sequence_rank is not None: participant = f"{participant}_sequence_rank_{sequence_rank}" + if tensor_rank: + participant = f"{participant}_tensor_rank_{tensor_rank}" return participant @@ -140,6 +143,8 @@ class Trainer(abc.ABC): logic via ``init_model``, ``_train_step``, and ``_aggregate_metrics``. """ + execution_backend = "fsdp" + def __init__(self, args: Namespace): self.args = args @@ -328,6 +333,7 @@ def set_train_queue( ) prefetch_depth = getattr(self.args, "prefetch_depth", 0) + tp_input = getattr(self, "input_tp_group", None) is not None gpu_device = torch.cuda.current_device() # When prefetching, stage data on CPU to avoid GPU contention between @@ -340,6 +346,7 @@ def set_train_queue( collator=collator, device=fetch_device, batch_size=per_dp_rank_batch_size, + prefetch_factor=prefetch_depth if tp_input else 2, assistant_header_ids=self.assistant_header_ids, end_token_ids=self.end_token_ids, dynamic_loss_mask=self.dynamic_loss_mask, @@ -355,15 +362,18 @@ def set_train_queue( "trainer", dp_rank=self.dp_rank, sequence_rank=sequence_rank, + tensor_rank=getattr(self, "tp_rank", 0), ), lifecycle_observer=lifecycle_observer, packing_enabled=packing_enabled, feature_affinity=getattr( self.args, "mooncake_feature_affinity", "any" ), + tensor_parallel_group=getattr(self, "input_tp_group", None), + tensor_parallel_control_group=getattr(self, "input_tp_control_group", None), ) - if prefetch_depth > 0: + if prefetch_depth > 0 and not tp_input: self.data_fetcher = PrefetchedDataFetcher( inner_fetcher, prefetch_depth=prefetch_depth, @@ -372,12 +382,13 @@ def set_train_queue( logger.info( f"[Rank {self.dp_rank}] Prefetched data fetcher initialized " f"(batch_size={per_dp_rank_batch_size}, prefetch_depth={prefetch_depth}, " - f"staging=CPU, target={gpu_device})" + f"staging={'GPU TP' if getattr(self, 'input_tp_group', None) is not None else 'CPU'}, target={gpu_device})" ) else: self.data_fetcher = inner_fetcher logger.info( f"[Rank {self.dp_rank}] Data fetcher initialized with batch_size={per_dp_rank_batch_size}" + + (f", source_prefetch_depth={prefetch_depth}, TP broadcasts on consumer thread" if tp_input else "") ) # ------------------------------------------------------------------ @@ -412,6 +423,7 @@ def set_eval_queue( collator=collator, device=torch.cuda.current_device(), batch_size=per_dp_rank_batch_size, + prefetch_factor=0, assistant_header_ids=self.assistant_header_ids, end_token_ids=self.end_token_ids, dynamic_loss_mask=self.dynamic_loss_mask, @@ -427,8 +439,11 @@ def set_eval_queue( "eval", dp_rank=self.dp_rank, sequence_rank=sequence_rank, + tensor_rank=getattr(self, "tp_rank", 0), ), lifecycle_observer=lifecycle_observer, + tensor_parallel_group=getattr(self, "eval_input_tp_group", None), + tensor_parallel_control_group=getattr(self, "eval_input_tp_control_group", None), ) self._eval_collator = collator self._eval_cache: list[dict] = [] diff --git a/aspec/training/trainer_actor.py b/aspec/training/trainer_actor.py index 3077962..2a6339d 100644 --- a/aspec/training/trainer_actor.py +++ b/aspec/training/trainer_actor.py @@ -127,7 +127,14 @@ def init(self, args: Namespace, role: str, mooncake_config=None, with_ref: bool if draft_model_config is None and getattr(args, "draft_model_config", None): draft_model_config = AutoDraftModelConfig.from_file(args.draft_model_config) - trainer_class = _trainer_class_for_config(draft_model_config) + if getattr(args, "train_backend", "fsdp") == "megatron": + from aspec.training.megatron_trainer import MegatronDSparkTrainer + + if not isinstance(draft_model_config, DSparkConfig): + raise ValueError("Megatron Draft currently requires DSpark") + trainer_class = MegatronDSparkTrainer + else: + trainer_class = _trainer_class_for_config(draft_model_config) self._trainer = trainer_class(args) target_model_path = getattr(args, "target_model_path", None) @@ -194,7 +201,8 @@ def get_status(self) -> dict: import aspec from aspec.inference.target_runtime import code_identity, dependency_versions - return { + trainer = getattr(self, "_trainer", None) + result = { "rank": self._rank, "world_size": self._world_size, "node_ip": self.get_node_ip(), @@ -204,11 +212,26 @@ def get_status(self) -> dict: "assigned_gpu_ids": [int(float(value)) for value in ray.get_gpu_ids()], "local_gpu_id": int(os.environ["LOCAL_RANK"]), "initialized": hasattr(self, "_trainer"), + "backend": getattr(trainer, "execution_backend", None), + "tensor_parallel_size": getattr(trainer, "tp_size", 1), + "tensor_parallel_rank": getattr(trainer, "tp_rank", 0), + "data_parallel_rank": getattr(trainer, "dp_rank", None), + "data_parallel_size": getattr(trainer, "dp_size", None), "runtime": {**code_identity(), "python": sys.executable, "prefix": sys.prefix, "aspec": aspec.__file__, "ray": ray.__file__, "torch": torch.__version__, "cuda": torch.version.cuda, - "dependencies": dependency_versions("nvidia-cutlass-dsl", "quack-kernels")}, + "dependencies": dependency_versions("nvidia-cutlass-dsl", "quack-kernels", + "megatron-core", "transformer-engine")}, } + if getattr(trainer, "execution_backend", None) == "megatron": + from aspec.runtime_identity import ( + MEGATRON_RUNTIME_RELATIVE_PATHS, + module_runtime_identity, + ) + + result["runtime"]["backend_identity"] = module_runtime_identity( + "megatron.core", relative_paths=MEGATRON_RUNTIME_RELATIVE_PATHS) + return result def save_model(self, step: int, force_sync: bool = False) -> None: self._trainer.save_model(step, force_sync) diff --git a/aspec/utils/env.py b/aspec/utils/env.py index 9e00c0c..1d37846 100644 --- a/aspec/utils/env.py +++ b/aspec/utils/env.py @@ -21,6 +21,10 @@ # compatibility directory. Ray runtime_env workers do not reliably retain # the driver's process loader path, so forward the resolved value exactly. "LD_LIBRARY_PATH", + "CUDA_HOME", + "CUDNN_HOME", + "CUDNN_FRONTEND_CUDART_LIB_NAME", + "CUDA_DEVICE_MAX_CONNECTIONS", # Source snapshots and explicitly selected backend adapters must resolve to # the same code in the driver and every Ray actor. "PYTHONPATH", @@ -81,6 +85,9 @@ def get_aspec_env_vars(args=None) -> dict[str, str]: **{k: "1" for k in _RAY_NOSET_VISIBLE_DEVICES_KEYS}, } env.update({k: os.environ[k] for k in _ASPEC_ENV_KEYS if k in os.environ}) + if (getattr(args, "train_backend", None) == "megatron" + or getattr(args, "target_model_backend", None) == "megatron"): + env.setdefault("CUDA_DEVICE_MAX_CONNECTIONS", "1") if getattr(args, "cluster_mode", "single_node") == "static_multi_node": # Ray is started from each node's own venv/environment. A driver's # library/cache paths must not overwrite another node's local layout. diff --git a/aspec/utils/train_dump.py b/aspec/utils/train_dump.py index 5a19912..c8c42f1 100644 --- a/aspec/utils/train_dump.py +++ b/aspec/utils/train_dump.py @@ -43,8 +43,9 @@ def extract_gradients(model: nn.Module) -> Dict[str, torch.Tensor]: """ gradients = {} for name, param in model.named_parameters(): - if param.grad is not None: - gradients[name] = param.grad.detach().clone().cpu() + gradient = param.grad if param.grad is not None else getattr(param, "main_grad", None) + if gradient is not None: + gradients[name] = gradient.detach().clone().cpu() return gradients diff --git a/docs/README.md b/docs/README.md index 18ba362..1abb6fe 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,6 +18,7 @@ The [project roadmap](../ROADMAP.md) lists implemented features and planned exte | Task | Guide | | --- | --- | | Configure resources, batches and Plan | [Configuration](user-guide/configuration.md) | +| Select Megatron versions and isolated environments | [Megatron integration](user-guide/megatron.md) | | Choose DSpark, DFlash, EAGLE or native MTP | [Algorithms](user-guide/algorithms.md) | | Choose Dense/MoE/MLA, SWA/CSA/QSA or packing/CP | [Structures](user-guide/structures.md) | | Use Ling Targets | [Ling recipes](user-guide/ling.md) | diff --git a/docs/framework/REFERENCES.md b/docs/framework/REFERENCES.md index fa68237..838df17 100644 --- a/docs/framework/REFERENCES.md +++ b/docs/framework/REFERENCES.md @@ -12,6 +12,8 @@ the retained source identities; applicable license and notice texts are distribu | [AngelSpec](https://github.com/Tencent/AngelSpec) `d3412bed231025ea85d35dbf5e0af44f1ac62a5b` | Thread-default reference retained in `aspec/utils/env.py` | | Retained [AngelSpec](https://github.com/Tencent/AngelSpec) snapshot `0966eb139d1c4054767e4a37bfc8a33db5989a72` | Concurrency, Direct-CP, Target-prefill, model-configuration and tensor-transfer regression cases; original source/license ancestry remains applicable | | [Megatron-LM](https://github.com/NVIDIA/Megatron-LM) and [FlashAttention](https://github.com/Dao-AILab/flash-attention) | Declared Direct CP, sparse metadata, output compaction and attention bridge derivations | +| Upstream [Megatron Core 0.20.0](https://github.com/NVIDIA/Megatron-LM/tree/685fb8bd970185e9ac3283fe32a5d6667bf4b268) | External, unmodified runtime through public model/spec, TP and DDP interfaces; independently authored ASpec Qwen3/DSpark adapters reuse the retained Target norm and training graph | +| [Megatron Bridge Qwen3 mapping](https://github.com/NVIDIA-NeMo/Megatron-Bridge/blob/f46e89eafb935fb0c75209afdb4502322250d0c7/src/megatron/bridge/models/qwen/qwen3_bridge.py) | Public grouped-QKV/gated-MLP mapping reference; inspired ASpec's independent strict HF-to-Core loader, with explicit BF16 norm order and per-feature comparisons | | Official [SGLang](https://github.com/sgl-project/sglang) v0.5.21 `e00930c5489053f26d86b179cee0d087f846acbb` | Current Target runtime plus `aspec/compat/patches/sglang-v0.5.21.patch` | | [cuDNN Frontend](https://github.com/NVIDIA/cudnn-frontend) `cb8cec8a9884f77fa514767237502bef7b7ad9f0` | Source of the adapted ASpec-owned DSpark Q/K CuTeDSL specialization and selected Frost primitives | | cuDNN Frontend **1.31.0 wheel**, with the FlashAttention/CuTe source chain retained | Adapted SM90/SM100 joint GQA backward, pre/postprocess, layouts and host dispatch; [source paths, hashes and licenses](../../aspec/kernels/cutedsl/cudnn_gqa/README.md) | diff --git a/docs/releasing.md b/docs/releasing.md index 08f8116..551de73 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1,7 +1,10 @@ # Releasing -ASpec publishes the Draft package `ling-aspec`, Target package `ling-aspec-target`, and -shared code package `ling-aspec-runtime` at the same version. Python imports and the CLI use `aspec`. +ASpec builds the Draft package `ling-aspec`, SGLang Target package `ling-aspec-target`, +Megatron environment package `ling-aspec-megatron`, and shared code package +`ling-aspec-runtime` at the same version. Python imports and the CLI use `aspec`. +The [Megatron guide](user-guide/megatron.md) describes its native build prerequisites +and independent environment profile. ## Versioning @@ -39,8 +42,8 @@ maintenance tools. `setup.py` collects resources from their canonical source loc In the release repository, configure a GitHub environment named `pypi`: - Add the environment secret `PYPI_API_TOKEN` with an account-wide PyPI token whose account can - publish all three distributions. This workflow uploads them together; a token scoped to one - project cannot authorize the other two. + publish all four distributions. This workflow uploads them together; a token scoped to one + project cannot authorize the others. - Allow deployments from `v*` tags. - Add required reviewers if releases need approval before upload. diff --git a/docs/support.md b/docs/support.md index c5f82d0..b6c058c 100644 --- a/docs/support.md +++ b/docs/support.md @@ -29,6 +29,10 @@ passing Plan validation establishes legality, not performance, convergence or do ## Execution and assets +- Experimental upstream Megatron Core Target/Draft adapters cover Qwen3-8B dense DSpark, + single-node CP1/PP1, Target TP1/TP2 and Draft TP1/TP2 with DP. The roles select backends + independently and use separate environments. See [scope and validation](user-guide/megatron.md) + for current evidence and numerical/replay boundaries; no Core fork is required. - Single-node online training supports required NVLink and explicitly selected Store/transport paths. With packing enabled, DSpark/DFlash full and sliding attention preserve document semantics with FA4 or native cuDNN flattened-token execution at CP1 and above. Selected-entry CSA/QSA/DSA diff --git a/docs/user-guide/installation.md b/docs/user-guide/installation.md index 85b1d1a..c83ad4d 100644 --- a/docs/user-guide/installation.md +++ b/docs/user-guide/installation.md @@ -5,6 +5,8 @@ GPUs and a CUDA 13-compatible driver. The Target package supplies its matching C the host still needs the C++20 compiler and native libraries listed below. See [native prerequisites](installation-advanced.md#requirements) for the compiler, glibc and RDMA library requirements, or use the [container](containers.md). +The experimental [Megatron profile](megatron.md) uses Python >=3.12, a separate environment +package and staged Transformer Engine build; follow that guide when choosing Megatron for either role. ## Install from PyPI diff --git a/docs/user-guide/megatron.md b/docs/user-guide/megatron.md new file mode 100644 index 0000000..7a57ba8 --- /dev/null +++ b/docs/user-guide/megatron.md @@ -0,0 +1,260 @@ +# Megatron Target and Draft + +ASpec supports the public Megatron Core 0.20.0 API +(`core_v0.20.0`, revision `685fb8bd970185e9ac3283fe32a5d6667bf4b268`). +Select the Target and Draft independently. The validated configuration uses +Qwen3-8B prefill and dense full-attention DSpark, BF16, microbatch size one, +single-node CP1/PP1, and TP1 or TP2. The Megatron Draft supports DP replicas. +Shared Target, decode, MTP conditioning and Megatron Draft packing are outside +this configuration. + +## Dependency profiles + +| Distribution | Environment | +| --- | --- | +| `ling-aspec-runtime` | Shared ASpec code, CLI and service protocol | +| `ling-aspec` | Existing ASpec Draft dependencies | +| `ling-aspec-target` | SGLang Target dependencies | +| `ling-aspec-megatron` | Megatron dependencies for either role | + +Install each role in its own isolated venv. All ASpec distributions in a run use +the same release version. Install Megatron with the common runtime directly; +combining its profile with either default environment package creates unsupported +dependency mixtures. Backend imports remain lazy. + +Megatron requires Python >=3.12. Its profile pins Torch 2.13.0, CUDA toolkit 13.0.3, +Transformer Engine 2.20.2, FA4 b34, CUTLASS DSL 4.7.1 and Quack 0.6.5. The feature-only Target uses FlashInfer 0.6.18 +and the standalone sglang-kernel 0.4.7 package on SM90. The ordinary +Draft and SGLang profiles retain their independent versions. + +Transformer Engine's PyTorch extension must be built against the execution +environment's Torch ABI. An unconstrained build environment can select a different +Torch. Source setup installs the build prerequisites first, then builds with +`--no-build-isolation`, selecting the venv's CUDA 13 toolkit and NCCL/cuDNN headers +explicitly. Including all NVIDIA wheel headers can put the transport's CUDA 12 +runtime headers before CUDA 13; the setup helper avoids this mixture. + +From a source checkout: + +```bash +bash setup.sh --megatron --python python3.12 --venv /absolute/path/to/draft-venv +bash setup.sh --megatron --target --python python3.12 --venv /absolute/path/to/target-venv +``` + +For a built release directory, create a Python 3.12 venv, activate it, and install: + +```bash +pip install -r /path/to/release/runtime/megatron-build.txt +aspec_nvidia=$(python -c 'import pathlib,sysconfig; print(pathlib.Path(sysconfig.get_paths()["purelib"]) / "nvidia")') +CUDA_HOME="$aspec_nvidia/cu13" \ + NVTE_CUDA_INCLUDE_PATH="$aspec_nvidia/cu13/include" \ + CPLUS_INCLUDE_PATH="$aspec_nvidia/cudnn/include:$aspec_nvidia/nccl/include" \ + NVTE_BUILD_USE_NVIDIA_WHEELS=0 MAX_JOBS=4 pip install --no-build-isolation \ + --find-links /path/to/release ling-aspec-megatron +aspec doctor --backend megatron --gpu +``` + +The host needs a compatible CUDA driver and the native prerequisites in the +[installation guide](installation-advanced.md). + +Each process prepares native-library paths from its own interpreter. The optional +local supervisor removes automatically selected Draft library paths before starting +Target. A separate Target service requires a service address; managed-local mode +also requires an explicit Target interpreter. Neither role imports the other's +Python dependency graph or transfers Python model objects across the boundary. + +## Target and Draft adapters + +The frozen Target uses Core's `HybridModel` and `HybridStack`. Each logical +Qwen3 block contributes an attention position (`*`) and an MLP position (`-`). +For Qwen3-8B the stack executes 72 positions. The adapter maps the grouped QKV +and gate/up weights into the Core TP projections and loads the final norm. +The feature-only model omits the vocabulary head. + +ASpec supplies two Target-specific layer specs for the single-sequence prefill. +They preserve the BF16 residual stream across fused add/RMSNorm boundaries, +the Q/K normalization reduction order, RoPE, causal FA3 and fused SwiGLU. +At an auxiliary MLP boundary the TP leader adds the pending branch and residual +directly into the transport slot. The final feature uses the checkpoint's RMSNorm. +Each request goes through the same Core model forward. Each engine processes one +sequence at a time. + +Megatron Draft partitions attention heads and dense FFN channels using public +Core TP projections, and uses Core DDP gradient buffers with ASpec's FP32-master +AdamW update and schedule. The retained DSpark graph still owns attention/QK +kernels, block compaction, bounded projection caches, tiled/indexed L1, statistics, +selective recomputation and deferred metrics. Vocabulary, Markov and confidence +computations remain replicated across TP. This preserves the existing global +softmax/L1 semantics; vocabulary sharding is not a claimed capability. + +Megatron Draft gradient precision is selected through the normal training +configuration: + +```yaml +training: + train_backend: megatron + megatron_grad_dtype: bf16 # fp32 (default) or bf16 +``` + +`fp32` preserves the existing default. `bf16` passes +`grad_reduce_in_fp32=False` to Core's ordinary DDP configuration, selecting BF16 +main-gradient accumulation and DP reduction for the BF16 model parameters. The dedicated TP-partial Q/K norm-gradient +reduction, optimizer masters and Adam moments remain FP32 under both settings. +This option applies only to Megatron Draft; the existing Torch/REPLICATE BF16 +gradient path is unchanged. + +The Plan records accumulation, DP-reduction and TP-partial-reduction precision +separately. Runtime execution observations include actual main-gradient dtypes +and the Core DDP flag. Exact resume preserves this setting; checkpoints without +the field are interpreted as FP32. Changing gradient precision requires +a weights-only new stage rather than an exact continuation. + +One TP leader prefetches source GETs into bounded, preallocated destination rings. +TP broadcasts and collation run on the consumer thread between microbatches, in +the same order as model collectives. Availability and tensor metadata use a CPU +Gloo group; no background reader launches NCCL work. Training and evaluation own +separate input groups. With source prefetch enabled, destination slots must cover +`prefetch_depth + 2` samples, including the active consumer and an in-progress GET. +The destination lease remains attached until the consumer finishes. Source release +occurs once per DP replica. Anchors/keep masks are shared from the TP leader; +commit collection checks matching work identity and reports it once per replica. + +Gradients are normalized by the complete optimizer window's supervision weight, +pooled across DP. TP-partial Q/K norm gradients are reduced once, and clipping +counts each replicated parameter once. Checkpoints describe logical TP shards and +restore model, FP32 masters, moments, schedule and both ASpec and Core RNG streams. +Exact resume requires the same Draft backend, TP topology and world size. Cross-backend weights-only +initialization is a separate stage. Native export gathers the logical Draft weights +for the existing strict loader. + +## Run Qwen3-8B with DSpark + +Start with SGLang Target and Megatron Draft in separate environments. Activate +the Megatron Draft venv and select two allocated GPUs: + +```bash +source /absolute/path/to/megatron-draft-venv/bin/activate +CUDA_VISIBLE_DEVICES=0,1 aspec quickstart qwen3-8b \ + --target /absolute/path/to/Qwen3-8B --output outputs/megatron-draft-001 \ + --managed-target /absolute/path/to/sglang-target-venv/bin/python \ + --set training.train_backend=megatron \ + --set training.megatron_grad_dtype=bf16 \ + --set mooncake.destination_gpu_ring_slots=3 +``` + +This exercises training, fresh-process recovery, native export and strict loading. +The two-GPU Quickstart budget covers TP1 for both roles. Megatron Draft TP2 uses +`training.draft_tensor_parallel_size=2` and two Draft GPUs per DP replica. +Megatron Target is selected independently with `model.target_model_backend=megatron` +and `inference.inference_engine_type=megatron`; its TP2 settings are +`inference.megatron.tp_size=2` and `inference.inference_num_gpus_per_engine=2`. + +## Verification + +Run the fixed-feature comparison in each Draft's own environment using the same +initial weights, SGLang feature cache, anchors, schedule and gradient precision. +Prepare the initial weights with `megatron_torch_alignment.py --backend fsdp +--phase prepare`. For Torch TP1 use `--backend fsdp`. For Torch TP2 the +verification tool uses a native PyTorch DTensor adapter around the existing +DSpark trainer and optimizer. This adapter is available only in the validation +tool. The public Torch training entry point supports TP1. + +A Torch TP2/DP1 reference run is: + +```bash +CUDA_VISIBLE_DEVICES=0,1 python -m aspec._environment \ + /absolute/path/to/torch-venv/bin/torchrun --standalone --nproc_per_node=2 \ + /absolute/path/to/tools/validation/megatron_torch_alignment.py \ + --attention-backward-deterministic \ + --backend torch-tp --phase checkpoint-control --tp 2 \ + --steps 100 --split-step 50 \ + --tensor-steps 1,2,5,10,20,30,40,50,60,70,80,90,100 \ + --state-steps 1,10,25,50,75,100 \ + --source /absolute/path/to/ASpec \ + --model /absolute/path/to/Qwen3-8B \ + --teacher /absolute/path/to/aspec-eval-cache.pt \ + --initial /absolute/path/to/shared-initial/model.safetensors \ + --output /tmp/torch-tp2-dp1-reference +``` + +For the Megatron run, use its venv and checkout, replace `--backend torch-tp` +with `--backend megatron --megatron-grad-dtype bf16`, and add +`--reference /tmp/torch-tp2-dp1-reference` after the reference completes. Use four +workers and TP2 for DP2 in both roles; add update 11 to both tensor milestone lists. +The deterministic option verifies the cuDNN-FE attention backward policy and +writes its call counts on every rank. + +`checkpoint-control` saves after update 50 and continues to 100. In fresh +processes, use `--phase resume --resume .../checkpoints/iter_0000051` to restore +the saved state. Compare against the uninterrupted run with `--reference` and +`--no-save-reference-tensors`. The tool records loss components, gradient norm +and learning rate each step, plus logical gradients and optimizer state at the +requested milestones. + +For Target comparison, run `tools/validation/megatron_target_aspec_alignment.py` +with `torchrun` in the Megatron environment: + +```bash +python -m aspec._environment /absolute/path/to/megatron-venv/bin/torchrun \ + --standalone --nproc_per_node=2 \ + /absolute/path/to/tools/validation/megatron_target_aspec_alignment.py \ + --model /absolute/path/to/Qwen3-8B \ + --reference-cache /absolute/path/to/aspec-eval-cache.pt \ + --length 4096 --output /tmp/target-tp2.json +``` + +This compares requested residual captures and final post-norm features with the +existing ASpec/SGLang producer, using the same checkpoint and token IDs. The +cache's producer and value-stage identity are checked. Choose an SGLang cache produced +at the same TP size as the Target for the matched comparison. + +To exercise online training and recovery with pretokenized Parquet inputs: + +```bash +CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 python -m aspec._environment \ + /absolute/path/to/megatron-draft-venv/bin/python \ + -m tools.validation.megatron_pipeline_acceptance \ + --model /absolute/path/to/Qwen3-8B \ + --train-data /absolute/path/to/train.parquet \ + --eval-data /absolute/path/to/eval.parquet \ + --target-python /absolute/path/to/megatron-target-venv/bin/python \ + --target-tp 2 --draft-tp 2 --dp 2 --accumulation 2 \ + --sequence-length 4096 --anchors 512 --steps 4 \ + --output /tmp/megatron-online-check +``` + +This uses the ordinary launcher and recipe audits, restarts the driver/workers +from a native checkpoint, checks state/RNG/work restoration, continues training, +and exports an asset for strict loading. Use compiler caches on a filesystem +supporting the toolchain's concurrent cache operations. Functional pipeline +completion and Target feature equivalence remain separate gates. + +## Validated configurations + +The deterministic online comparison on SM90 used 200 Open PerfectBlend prompts +with Qwen3-8B responses, two samples per optimizer update, a 4096-token context +limit and 512 anchors. DP1 accumulated two microbatches and DP2 accumulated one +per replica. Each pair used the same Target TP size, Draft topology, initial +weights and BF16 gradients. + +| Draft topology | SGLang Target + Torch Draft mean objective | Megatron Target + Megatron Draft mean objective | Max absolute per-step rel diff | +| --- | ---: | ---: | ---: | +| TP1/DP1 | 2.83849453 | 2.83849453 | 0% | +| TP2/DP1 | 2.83903998 | 2.83903998 | 0% | +| TP2/DP2 | 2.85058107 | 2.85058107 | 0% | + +The objective is 0.1 × CE + 0.9 × L1 + confidence loss, averaged over 100 +optimizer updates. Final Draft weights, FP32 masters and Adam moments matched +at every topology. For TP1, set `inference.sglang.extra_args.prefill_max_requests=1` +in the reference profile to match Megatron's single-request prefill. Batching +requests changes the token-row count of the first MLP down projection and can +change BF16 rounding. + +The independent fixed-feature checks at TP2/DP1 and TP2/DP2 verified that +restoring after update 50 reproduced the remaining 50 updates and final state +on each backend. Online mixed-backend checks covered SGLang Target with +Megatron Draft, Megatron Target with Torch Draft, feature transport, export +and strict native loading. The 4K fixture exercised CuTeDSL Q/K preparation, +tiled FP32 L1 and fused optimizer clipping. Multi-chunk projection caching, +optional backbone checkpointing, throughput, peak memory and held-out quality +require separate measurements. diff --git a/requirements/megatron-build.txt b/requirements/megatron-build.txt new file mode 100644 index 0000000..1099fb4 --- /dev/null +++ b/requirements/megatron-build.txt @@ -0,0 +1,10 @@ +# Install before building native Megatron/TE extensions, in the same venv. +torch==2.13.0 +cuda-toolkit[cccl,nvcc]==13.0.3.0 +numpy<2.4 +setuptools>=80 +wheel +packaging>=24.2 +pybind11 +cmake +ninja diff --git a/requirements/megatron-v0.20.0.txt b/requirements/megatron-v0.20.0.txt new file mode 100644 index 0000000..9c980b2 --- /dev/null +++ b/requirements/megatron-v0.20.0.txt @@ -0,0 +1,21 @@ +# Independent Megatron environment for either Target or Draft (Python >=3.12). +# Upstream core_v0.20.0: 685fb8bd970185e9ac3283fe32a5d6667bf4b268. +megatron-core==0.20.0 +transformer-engine[pytorch,core_cu13]==2.20.2 +torch==2.13.0 +torchvision==0.28.0 +transformers==5.12.1 +cuda-toolkit[cccl,nvcc]==13.0.3.0 +apache-tvm-ffi==0.1.11 +# Native prefill kernels for the feature-only HybridModel. +flashinfer-python==0.6.18 +sglang-kernel==0.4.7 +cuda-tile==1.6.0rc5 +flash-attn-4==4.0.0b34 +nvidia-cutlass-dsl[cu13]==4.7.1 +quack-kernels[cu13]==0.6.5 +nvidia-cudnn-frontend==1.31.0 +einops==0.8.2 +cmake +ninja +numba diff --git a/requirements/sglang-v0.5.21.txt b/requirements/sglang-v0.5.21.txt index eb37893..7a43663 100644 --- a/requirements/sglang-v0.5.21.txt +++ b/requirements/sglang-v0.5.21.txt @@ -1,5 +1,7 @@ # Target-only environment: retain SGLang's declared dependency versions. sglang==0.5.21 +# Already required by SGLang; explicit pin admits this prerelease in uv too. +cuda-tile==1.6.0rc5 flash-attn-4==4.0.0b34 cuda-toolkit[nvcc]==13.0.3.0 # Mooncake's native wheel needs libcudart.so.12 alongside Torch's CUDA 13. diff --git a/setup.py b/setup.py index 62324ac..9c61c64 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,9 @@ def run(self): ROOT / "examples/data/sample_conversations.jsonl", ROOT / "tools/recipe_catalog.json", ROOT / "requirements/sglang-v0.5.21.txt", - ROOT / "requirements/draft.txt"] + ROOT / "requirements/draft.txt", + ROOT / "requirements/megatron-v0.20.0.txt", + ROOT / "requirements/megatron-build.txt"] for source in assets: destination = package / "resources" / source.relative_to(ROOT) destination.parent.mkdir(parents=True, exist_ok=True) diff --git a/setup.sh b/setup.sh index d65f0a1..45c66a8 100755 --- a/setup.sh +++ b/setup.sh @@ -5,6 +5,7 @@ venv="$repo_root/.venv" base_python="" options=() role=draft +backend=default explicit_venv=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -13,10 +14,11 @@ while [[ $# -gt 0 ]]; do if [[ $1 == --venv ]]; then venv=$2; explicit_venv=1; else base_python=$2; fi shift 2 ;; --target) role=target; options+=("$1"); shift ;; + --megatron) backend=megatron; options+=("$1"); shift ;; --dev|--gpu) options+=("$1"); shift ;; --help) - echo 'Usage: bash setup.sh [--venv PATH] [--python EXECUTABLE] [--target] [--dev] [--gpu]' - echo 'Creates an isolated Linux Python >=3.11 venv and installs pinned dependencies.' + echo 'Usage: bash setup.sh [--venv PATH] [--python EXECUTABLE] [--target] [--megatron] [--dev] [--gpu]' + echo 'Creates an isolated Linux Python >=3.11 venv (>=3.12 for Megatron) and installs pinned dependencies.' echo 'Defaults to .venv. Existing venvs require this checkout owner.' echo 'Does not install host packages/drivers, download models, start training or clean clusters.' exit 0 ;; @@ -24,24 +26,28 @@ while [[ $# -gt 0 ]]; do esac done if [[ $role == target && $explicit_venv == 0 ]]; then venv="$repo_root/.venv-target"; fi +if [[ $backend == megatron && $explicit_venv == 0 ]]; then venv="$repo_root/.venv-megatron-$role"; fi [[ $(uname -s) == Linux && $(uname -m) == x86_64 ]] || { echo 'Use Linux x86-64; see docs/user-guide/installation.md.' >&2; exit 2; } [[ -z ${PYTHONPATH:-} && -z ${PYTHONHOME:-} ]] || { echo 'Unset PYTHONPATH and PYTHONHOME before setup.' >&2; exit 2; } +minimum_python=11 +if [[ $backend == megatron ]]; then minimum_python=12; fi if [[ -z $base_python ]]; then for candidate in python3.12 python3.11 python3; do - if command -v "$candidate" >/dev/null && "$candidate" -c 'import sys; sys.exit(sys.version_info[:2] < (3,11))'; then + if command -v "$candidate" >/dev/null && "$candidate" -c 'import sys; sys.exit(sys.version_info[:2] < (3, int(sys.argv[1])))' "$minimum_python"; then base_python=$candidate; break fi done fi -[[ -n $base_python ]] || { echo 'Install Python >=3.11 or pass --python EXECUTABLE.' >&2; exit 2; } -"$base_python" - <<'CHECK' +[[ -n $base_python ]] || { echo "Install Python >=3.$minimum_python or pass --python EXECUTABLE." >&2; exit 2; } +"$base_python" - "$minimum_python" <<'CHECK' import sys -if sys.version_info[:2] < (3, 11): - raise SystemExit("Python >=3.11 is required") +minimum = (3, int(sys.argv[1])) +if sys.version_info[:2] < minimum: + raise SystemExit(f"Python >=3.{minimum[1]} is required") CHECK venv=$("$base_python" -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$venv") if [[ -e $venv || -L $venv ]]; then diff --git a/tests/cpu/test_alignment_rel_diff.py b/tests/cpu/test_alignment_rel_diff.py new file mode 100644 index 0000000..5647f7c --- /dev/null +++ b/tests/cpu/test_alignment_rel_diff.py @@ -0,0 +1,59 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Guard numerical reports against hidden normalization/near-zero errors.""" + +import math + +import pytest +import torch + +from tools.validation.megatron_torch_alignment import compare_vectors, scalar_diff, selected_steps + + +def test_scalar_rel_diff_preserves_sign_and_exposes_zero_floor(): + assert scalar_diff(9, 10)["signed_rel_diff"] == -.1 + assert scalar_diff(9, 10)["abs_rel_diff"] == .1 + row = scalar_diff(1e-14, 0) + assert row["denominator_floored"] + assert row["absolute_difference"] == 1e-14 + assert row["signed_rel_diff"] == .01 + with pytest.raises(ValueError, match="Nonfinite"): + scalar_diff(float("nan"), 1) + + +def test_tensor_rel_diff_uses_logical_elements_and_reports_each_parameter(): + expected = {"large": torch.tensor([1., 2., 3.]), "small": torch.tensor([4.])} + actual = {"large": torch.tensor([2., 2., 3.]), "small": torch.tensor([2.])} + row = compare_vectors(actual, expected, device="cpu") + assert row["relative_l2"] == pytest.approx(math.sqrt(5 / 30)) + assert row["relative_l1"] == pytest.approx(3 / 10) + assert row["relative_max"] == .5 + assert row["mean_absolute"] == .75 + assert row["pointwise_rel_diff_mean"] == .375 + assert row["pointwise_rel_diff_quantiles"]["p50"] == .25 + assert row["pointwise_exceed_counts"]["0.1"] == 2 + assert row["different_elements"] == 2 + assert row["numel"] == 4 + assert {r["parameter"] for r in row["per_parameter"]} == set(expected) + + +def test_tensor_rel_diff_zero_reference_is_explicit_and_does_not_drop_elements(): + row = compare_vectors({"w": torch.tensor([0., 1e-12])}, + {"w": torch.zeros(2)}, device="cpu") + assert row["relative_l2"] is None + assert row["zero_reference"] + assert row["denominator_floored_count"] == 2 + assert row["pointwise_rel_diff_mean"] == pytest.approx(.5) + equal = compare_vectors({"w": torch.zeros(2)}, {"w": torch.zeros(2)}, device="cpu") + assert equal["relative_l2"] == 0 + assert equal["bitwise_equal"] + with pytest.raises(ValueError, match="Nonfinite"): + compare_vectors({"w": torch.tensor([float("inf")])}, + {"w": torch.ones(1)}, device="cpu") + + +def test_declared_observation_steps_do_not_extend_the_trajectory(): + assert selected_steps("1,2,10,100", 100) == {1, 2, 10, 100} + assert selected_steps("", 100) == set() + with pytest.raises(ValueError, match="within"): + selected_steps("101", 100) diff --git a/tests/cpu/test_megatron.py b/tests/cpu/test_megatron.py new file mode 100644 index 0000000..1879783 --- /dev/null +++ b/tests/cpu/test_megatron.py @@ -0,0 +1,267 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +import subprocess +import sys +import threading +from argparse import Namespace +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch +import torch.nn.functional as F + +from aspec.config.megatron import validate_megatron_execution +from aspec.config.train_config import load_config +from aspec.feature_contract import build_target_feature_contract, validate_feature_contract +from aspec.models.megatron import qwen3_parameter +from aspec.planning import compile_execution_plan + + +def test_baseline_import_does_not_require_megatron(): + result = subprocess.run([sys.executable, "-c", """ +import sys +class RejectMegatron: + def find_spec(self, name, *args): + if name == 'megatron' or name.startswith('megatron.'): + raise AssertionError('Baseline imported an optional backend') +sys.meta_path.insert(0, RejectMegatron()) +import aspec.inference.factory +import aspec.training.trainer_actor +"""], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + +def settings(**overrides): + return Namespace(**{"train_backend": "megatron", "target_model_backend": "megatron", + "draft_tensor_parallel_size": 2, "world_size": 4, + "megatron_tp_size": 2, "inference_num_gpus_per_engine": 2, + "target_prefill_memory_optimized": True, **overrides}) + + +@pytest.mark.parametrize("target,training", [("megatron", "fsdp"), ("sglang", "megatron"), + ("megatron", "megatron")]) +def test_independent_target_draft_pairings(target, training): + validate_megatron_execution(settings(target_model_backend=target, train_backend=training, + draft_tensor_parallel_size=1 if training == "fsdp" else 2), + {"family": "dspark"}) + + +@pytest.mark.parametrize("dtype,resolved", [("fp32", "float32"), ("bf16", "bfloat16")]) +def test_gradient_precision_configuration_reaches_plan_and_runtime(dtype, resolved): + recipe = Path(__file__).resolve().parents[2] / "configs/recipes/quickstart_qwen3_8b.yaml" + config = load_config(str(recipe), cli_args=[ + "training.train_backend=megatron", "training.draft_tensor_parallel_size=2", + "training.training_num_gpus_per_node=2", "training.prefetch_depth=0", + f"training.megatron_grad_dtype={dtype}", + ]) + plan = compile_execution_plan(config) + assert plan.to_runtime_args().megatron_grad_dtype == dtype + policy = plan.policy.parallelism["training"] + assert policy["gradient_accumulation_dtype"] == policy["gradient_reduce_dtype"] == resolved + assert policy["core_grad_reduce_in_fp32"] == (dtype == "fp32") + assert policy["tp_partial_gradient_reduce_dtype"] == policy["optimizer_state_dtype"] == "float32" + + +@pytest.mark.parametrize("backend,dtype", [("megatron", "fp16"), ("megatron", "float32"), + ("megatron", None), ("fsdp", "bf16")]) +def test_gradient_precision_rejects_invalid_or_wrong_backend_configuration(backend, dtype): + with pytest.raises(ValueError, match="megatron_grad_dtype"): + validate_megatron_execution(settings(train_backend=backend, draft_tensor_parallel_size=1, + megatron_grad_dtype=dtype), {"family": "dspark"}) + + +@pytest.mark.parametrize("saved,current,allowed", [ + (None, "fp32", True), (None, "bf16", False), + ("fp32", "fp32", True), ("bf16", "bf16", True), + ("fp32", "bf16", False), ("bf16", "fp32", False), +]) +def test_exact_resume_preserves_megatron_gradient_precision(saved, current, allowed): + from aspec.training.checkpoint import ( + _logical_training_topology, + _validate_exact_resume_topology, + ) + + args = Namespace(train_backend="megatron", draft_tensor_parallel_size=1, + world_size=1, megatron_grad_dtype=current) + topology = _logical_training_topology(args, 1) + if saved is None: + topology.pop("megatron_grad_dtype") + else: + topology["megatron_grad_dtype"] = saved + metadata = {"world_size": 1, "training_topology": topology} + if allowed: + _validate_exact_resume_topology(args, metadata) + else: + with pytest.raises(RuntimeError, match="gradient precision"): + _validate_exact_resume_topology(args, metadata) + + +@pytest.mark.parametrize("overrides,message", [ + ({"world_size": 3}, "divisible"), + ({"draft_context_parallel_size": 2}, "CP1"), + ({"dflash_packing": True}, "packed"), + ({"fsdp_cpu_offload": True}, "offload"), + ({"distributed_backend": "gloo"}, "NCCL"), + ({"micro_batch_size": 2}, "micro_batch_size"), + ({"prefetch_depth": 1, "mooncake_destination_gpu_ring_slots": 1}, "destination_gpu_ring_slots"), + ({"sglang_pp_size": 2}, "PP1"), + ({"target_prefill_memory_optimized": False}, "memory_optimized"), + ({"inference_num_gpus_per_engine": 1}, "must equal"), + ({"train_backend": "fsdp", "draft_tensor_parallel_size": 2}, "Draft TP"), +]) +def test_megatron_fails_closed_before_process_launch(overrides, message): + with pytest.raises(ValueError, match=message): + validate_megatron_execution(settings(**overrides), {"family": "dspark"}) + + +def test_producer_backend_is_part_of_the_feature_identity(): + kwargs = dict(drafter_family="dspark", target_hidden_size=16, target_num_hidden_layers=4, + aux_hidden_state_layers=[0, 2], store_last_hidden_states=True) + sglang = build_target_feature_contract(**kwargs) + megatron = build_target_feature_contract(**kwargs, producer_backend="megatron") + validate_feature_contract(megatron) + assert sglang["contract_id"] != megatron["contract_id"] + + +def test_tp_source_capacity_counts_dp_replicas_and_target_engines(): + recipe = Path(__file__).resolve().parents[2] / "configs/recipes/quickstart_qwen3_8b.yaml" + overrides = [ + "model.target_model_backend=megatron", "inference.inference_engine_type=megatron", + "training.train_backend=megatron", "training.draft_tensor_parallel_size=2", + "training.training_num_gpus_per_node=4", "training.draft_accumulation_steps=1", + "inference.megatron.tp_size=2", "inference.inference_num_gpus=2", + "inference.inference_num_gpus_per_engine=2", "inference.inference_batch_size=1", + "mooncake.source_gpu_ring_slots=2", + "mooncake.destination_gpu_ring_slots=3", + ] + config = load_config(str(recipe), cli_args=overrides) + plan = compile_execution_plan(config) + assert plan.to_runtime_args().dp_size == 2 + assert plan.policy.pipeline["global_batch_size"] == 2 + # Two Target GPUs still represent one engine with one source ring. + with pytest.raises(ValueError, match="cannot retain one complete Draft dispatch"): + load_config(base_config=config, cli_args=["mooncake.source_gpu_ring_slots=1"]) + + +def test_tp_receive_slot_remains_owned_until_consumer_releases_it(monkeypatch): + from aspec.training.data_fetcher import MooncakeDataset, _release_p2p_receive_leases + + dataset = object.__new__(MooncakeDataset) + dataset.tensor_parallel_group = object() + dataset.tensor_parallel_control_group = object() + dataset._tp_source_prefetch = None + dataset._tensor_parallel_samples = dataset._tensor_parallel_payload_bytes = 0 + dataset._next_queue_item = lambda: "sample" + receive_buffer = torch.zeros(3) + ring = {"busy": False, "reads": 0, "releases": 0} + + def release(): + assert ring["busy"] + ring["busy"] = False + ring["releases"] += 1 + + def load(_item): + assert not ring["busy"], "next prefetch cannot wait on the active consumer" + ring["busy"] = True + ring["reads"] += 1 + receive_buffer.fill_(ring["reads"]) + return {"features": receive_buffer, + "_mooncake_p2p_receive_lease": SimpleNamespace(release=release)} + + dataset._load_from_mooncake = load + dataset._read_tp_source = lambda: ("sample", load("sample")) + monkeypatch.setattr(torch.distributed, "get_global_rank", lambda *_: 0) + monkeypatch.setattr(torch.distributed, "get_rank", lambda *_: 0) + def broadcast_control(*_, group, **__): + assert group is dataset.tensor_parallel_control_group + + monkeypatch.setattr(torch.distributed, "broadcast_object_list", broadcast_control) + monkeypatch.setattr(torch.distributed, "broadcast", lambda *_, **__: None) + original_to = torch.Tensor.to + + def to(tensor, *args, **kwargs): + return tensor if kwargs.get("device") == "cuda" else original_to(tensor, *args, **kwargs) + + monkeypatch.setattr(torch.Tensor, "to", to) + _, first = dataset._tensor_parallel_item_on_device() + assert first["features"].tolist() == [1, 1, 1] + assert ring["busy"] and ring["releases"] == 0 + _release_p2p_receive_leases(first) + _, second = dataset._tensor_parallel_item_on_device() + assert second["features"].tolist() == [2, 2, 2] + _release_p2p_receive_leases(second) + assert ring == {"busy": False, "reads": 2, "releases": 2} + + +def test_grouped_qkv_tp_mapping_preserves_independent_hf_projections(): + generator = torch.Generator().manual_seed(1729) + tensors = {f"model.layers.0.self_attn.{name}_proj.weight": torch.randn(rows, 16, generator=generator) + for name, rows in (("q", 16), ("k", 8), ("v", 8))} + reader = SimpleNamespace(tensor=lambda key: tensors[key]) + config = SimpleNamespace(num_attention_heads=4, num_key_value_heads=2, hidden_size=16, head_dim=4) + inputs = torch.randn(3, 16, generator=generator) + queries, keys, values = [], [], [] + for rank in range(2): + mapped = qwen3_parameter(reader, "decoder.layers.0.self_attention.linear_qkv.weight", + config, rank, 2) + projected = F.linear(inputs, mapped).view(3, 1, 4, 4) + queries.append(projected[:, :, :2].flatten(1)) + keys.append(projected[:, :, 2].flatten(1)) + values.append(projected[:, :, 3].flatten(1)) + for name, shards in (("q", queries), ("k", keys), ("v", values)): + torch.testing.assert_close(torch.cat(shards, dim=-1), + F.linear(inputs, tensors[f"model.layers.0.self_attn.{name}_proj.weight"])) + + +def test_source_prefetch_only_reads_in_background_and_drains_unused_leases(): + from aspec.training.tp_source_prefetch import TPSourcePrefetch + + main_thread = threading.get_ident() + calls, released = [], [] + second_read = threading.Event() + values = iter((("one", {"id": 1}), ("two", {"id": 2}), (None, {}))) + + def read(): + calls.append(threading.get_ident()) + item = next(values) + if item[0] == "two": + second_read.set() + return item + + prefetch = TPSourcePrefetch(read, lambda item: released.append(item[1]), depth=1) + assert prefetch.next() == ("one", {"id": 1}) + assert second_read.wait(timeout=2) + prefetch.close() + assert calls and all(thread != main_thread for thread in calls) + assert all(item.get("id") in (None, 2) for item in released) + assert {"id": 2} in released + + +def test_hybrid_physical_layers_map_to_qwen3_logical_block(): + tensors = {name: torch.rand(8) for name in ( + "model.layers.9.input_layernorm.weight", "model.layers.9.post_attention_layernorm.weight", + "model.norm.weight", + )} + reader = SimpleNamespace(tensor=tensors.__getitem__) + for core_name, source_name in ( + ("decoder.layers.18.input_layernorm.weight", "model.layers.9.input_layernorm.weight"), + ("decoder.layers.19.pre_mlp_layernorm.weight", "model.layers.9.post_attention_layernorm.weight"), + ("final_norm.weight", "model.norm.weight"), + ): + assert qwen3_parameter(reader, core_name, None, 0, 1) is tensors[source_name] + + +def test_megatron_eval_cache_binds_target_worker_and_topology(tmp_path): + from aspec.controller.eval import eval_cache_identity + + args = SimpleNamespace( + target_model_path=str(tmp_path), target_model_backend="megatron", + megatron_tp_size=1, megatron_implementation="transformer_engine", + megatron_target_cache_identity={"adapter": "candidate", "core": "core", "dependencies": {}}, + ) + first = eval_cache_identity(args, "sha256:dataset", 8) + assert first["producer_code"] == args.megatron_target_cache_identity + args.megatron_tp_size = 2 + assert first != eval_cache_identity(args, "sha256:dataset", 8) diff --git a/tests/cpu/test_release_audit.py b/tests/cpu/test_release_audit.py index 0d79272..30ad450 100644 --- a/tests/cpu/test_release_audit.py +++ b/tests/cpu/test_release_audit.py @@ -61,7 +61,7 @@ def _artifacts( link.type = tarfile.SYMTYPE link.linkname = "target" output.addfile(link) - for role in ("ling-aspec", "ling-aspec-target"): + for role in ("ling-aspec", "ling-aspec-target", "ling-aspec-megatron"): normalized = role.replace("-", "_") metadata = (f"Name: {role}\nVersion: {wheel_version}\n" f"Requires-Dist: ling-aspec-runtime=={wheel_version}\n") @@ -81,7 +81,7 @@ def test_release_audit_accepts_complete_artifact_pair(tmp_path): result = audit(tmp_path) assert result[wheel.name] == len(WHEEL_REQUIRED) + 1 assert result[source.name] == len(SDIST_REQUIRED) - assert len(result) == 6 + assert len(result) == 8 @pytest.mark.parametrize("member", ["aspec/compat/sglang_patch.py", @@ -119,7 +119,7 @@ def test_release_audit_rejects_non_source_members(tmp_path, member): def test_release_audit_requires_one_wheel_and_one_sdist(tmp_path): - with pytest.raises(ValueError, match="exactly three wheels and three sdists"): + with pytest.raises(ValueError, match="exactly four wheels and four sdists"): audit(tmp_path) diff --git a/tests/cpu/test_target_service.py b/tests/cpu/test_target_service.py index ddcb8d5..d790327 100644 --- a/tests/cpu/test_target_service.py +++ b/tests/cpu/test_target_service.py @@ -57,7 +57,7 @@ def register(sig, handler): monkeypatch.setattr(module, "TargetService", lambda **_: instance) monkeypatch.setattr(module, "http_server", lambda *_: server) monkeypatch.setattr(module, "visible_gpu_ids", lambda: [0]) - monkeypatch.setattr(module, "inventory", lambda: {"python": sys.executable}) + monkeypatch.setattr(module, "inventory", lambda backend="sglang": {"python": sys.executable}) monkeypatch.setattr(module, "ray", MagicMock()) monkeypatch.setattr(module.tempfile, "mkdtemp", lambda **_: str(tmp_path)) monkeypatch.setattr(module.time, "sleep", lambda seconds: events.append(("wait", seconds))) diff --git a/tests/cpu/test_torch_tp_validation.py b/tests/cpu/test_torch_tp_validation.py new file mode 100644 index 0000000..dc26184 --- /dev/null +++ b/tests/cpu/test_torch_tp_validation.py @@ -0,0 +1,61 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Check native DTensor TP gradients against independent split-linear autograd.""" + +from datetime import timedelta + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +from torch.distributed.device_mesh import init_device_mesh + +from tools.validation.torch_tp_dspark import TorchTPLinear + + +def _projection_worker(rank, rendezvous): + torch.set_num_threads(1) + dist.init_process_group("gloo", init_method=rendezvous, rank=rank, world_size=2, + timeout=timedelta(seconds=60)) + try: + mesh = init_device_mesh("cpu", (2,)) + for dtype in (torch.float32, torch.bfloat16): + for axis in (0, 1): + torch.manual_seed(114) + full = torch.nn.Linear(8, 12, bias=False, dtype=dtype) + layer = TorchTPLinear(full, mesh, axis) + reference_weight = full.weight.detach().clone().requires_grad_(True) + reference_input = torch.randn(2, 3, 8, dtype=dtype, requires_grad=True) + gradient = torch.randn(2, 3, 12, dtype=dtype) + pieces = reference_weight.chunk(2, dim=axis) + if axis == 0: + expected = torch.cat([torch.nn.functional.linear(reference_input, w) + for w in pieces], dim=-1) + local = reference_input.detach().clone().requires_grad_(True) + local_gradient = gradient.chunk(2, dim=-1)[rank] + else: + partials = [torch.nn.functional.linear(x.contiguous(), w.contiguous()) + for x, w in zip(reference_input.chunk(2, dim=-1), pieces, strict=True)] + expected = partials[0] + partials[1] + local = reference_input.detach().chunk(2, dim=-1)[rank].clone().requires_grad_(True) + local_gradient = gradient + actual = layer(local) + expected_output = expected.chunk(2, dim=-1)[rank] if axis == 0 else expected + torch.testing.assert_close(actual, expected_output, rtol=0, atol=0) + # Exercise native BF16 leaf accumulation across two microbatches. + for _ in range(2): + (layer(local).float() * local_gradient.float()).sum().backward() + (expected.float() * gradient.float()).sum().backward(retain_graph=True) + torch.testing.assert_close(layer.weight.grad, + reference_weight.grad.chunk(2, dim=axis)[rank], rtol=0, atol=0) + expected_input_gradient = (reference_input.grad if axis == 0 + else reference_input.grad.chunk(2, dim=-1)[rank]) + torch.testing.assert_close(local.grad, expected_input_gradient, rtol=0, atol=0) + assert layer.weight.grad.dtype == dtype + finally: + dist.destroy_process_group() + + +@pytest.mark.skipif(not dist.is_gloo_available(), reason="Requires native Gloo collectives") +def test_dtensor_column_and_row_shards_preserve_forward_and_accumulated_backward(tmp_path): + mp.spawn(_projection_worker, args=((tmp_path / "rendezvous").as_uri(),), nprocs=2, join=True) diff --git a/tests/test_optimizer_state_keys.py b/tests/test_optimizer_state_keys.py index 80c192c..d08e05f 100644 --- a/tests/test_optimizer_state_keys.py +++ b/tests/test_optimizer_state_keys.py @@ -146,6 +146,30 @@ def load_state_dict(self, value): self.optimizer.load_state_dict(value) +def test_restore_preserves_dtensor_master_weights(tmp_path): + import torch.distributed as dist + from torch.distributed.device_mesh import init_device_mesh + from torch.distributed.tensor import Shard, distribute_tensor + + dist.init_process_group("gloo", init_method=(tmp_path / "rendezvous").as_uri(), + rank=0, world_size=1) + try: + model = nn.Linear(4, 2, bias=False) + mesh = init_device_mesh("cpu", (1,)) + model.weight = nn.Parameter(distribute_tensor(model.weight.detach(), mesh, [Shard(0)])) + optimizer = _StubBF16Optimizer(model) + state = OptimizerState(model, optimizer) + saved = state.state_dict() + expected = saved["fp32_params"]["weight"].clone() + saved["fp32_params"]["weight"] = expected + optimizer.fp32_params[0].zero_() + + state.load_state_dict(saved) + torch.testing.assert_close(optimizer.fp32_params[0].to_local(), expected.to_local(), rtol=0, atol=0) + finally: + dist.destroy_process_group() + + def test_dcp_exact_resume_restores_lazy_adam_moments_without_an_extra_update(tmp_path): original = _RealAdamBF16Surface(nn.Linear(3, 2)) original.fp32_params[0].grad = torch.ones_like(original.fp32_params[0]) diff --git a/tools/audit_release.py b/tools/audit_release.py index 663a75d..a59fcd4 100644 --- a/tools/audit_release.py +++ b/tools/audit_release.py @@ -75,6 +75,8 @@ "aspec/compat/patches/sglang-v0.5.21.patch", "requirements/sglang-v0.5.21.txt", "requirements/draft.txt", + "requirements/megatron-v0.20.0.txt", + "requirements/megatron-build.txt", "tools/audit_release.py", "tools/recipe_catalog.json", "tools/build_release.py", @@ -107,9 +109,19 @@ "aspec/commands/convert_pretokenized_jsonl.py", "aspec/commands/_recipe_smoke.py", "aspec/commands/_checkpoint_audit.py", + "aspec/config/megatron.py", + "aspec/models/megatron.py", + "aspec/models/megatron_target.py", + "aspec/models/megatron_draft.py", + "aspec/training/megatron_trainer.py", + "aspec/training/megatron_optimizer.py", + "aspec/training/tp_source_prefetch.py", + "aspec/inference/engine/megatron_engine.py", "aspec/commands/_recipe_capacity.py", "aspec/resources/requirements/sglang-v0.5.21.txt", "aspec/resources/requirements/draft.txt", + "aspec/resources/requirements/megatron-v0.20.0.txt", + "aspec/resources/requirements/megatron-build.txt", "aspec/resources/tools/recipe_catalog.json", "aspec/resources/examples/data/sample_conversations.jsonl", } @@ -146,7 +158,7 @@ def _identity(raw: bytes, artifact: Path) -> tuple[str, str]: version = metadata.get("Version") if not name or not version: raise ValueError(f"{artifact.name}: package metadata lacks Name or Version") - if name not in {"ling-aspec", "ling-aspec-target", "ling-aspec-runtime"}: + if name not in {"ling-aspec", "ling-aspec-target", "ling-aspec-runtime", "ling-aspec-megatron"}: raise ValueError(f"{artifact.name}: unexpected distribution name {name!r}") if name != "ling-aspec-runtime": requirements = {value.replace(" ", "") for value in metadata.get_all("Requires-Dist", [])} @@ -198,7 +210,7 @@ def _sdist_members(path: Path) -> tuple[set[str], tuple[str, str]]: missing = SDIST_REQUIRED - names if root.startswith("ling_aspec_runtime-") else {"PKG-INFO", "LICENSE", "README.md", "pyproject.toml"} - names if missing: raise ValueError(f"{path.name}: missing source members {sorted(missing)}") - if not root.startswith(("ling_aspec-", "ling_aspec_target-", "ling_aspec_runtime-")): + if not root.startswith(("ling_aspec-", "ling_aspec_target-", "ling_aspec_runtime-", "ling_aspec_megatron-")): raise ValueError(f"{path.name}: unexpected source root {root!r}") with tarfile.open(path, mode="r:gz") as source: metadata_member = source.getmember(f"{root}/PKG-INFO") @@ -212,19 +224,19 @@ def _sdist_members(path: Path) -> tuple[set[str], tuple[str, str]]: def audit(directory: Path) -> dict[str, int]: wheels = sorted(directory.glob("*.whl")) sdists = sorted(directory.glob("*.tar.gz")) - if len(wheels) != 3 or len(sdists) != 3: - raise ValueError(f"{directory}: expected exactly three wheels and three sdists") + if len(wheels) != 4 or len(sdists) != 4: + raise ValueError(f"{directory}: expected exactly four wheels and four sdists") result, identities = {}, {} for artifact in [*wheels, *sdists]: members, identity = (_wheel_members(artifact) if artifact.suffix == ".whl" else _sdist_members(artifact)) result[artifact.name] = len(members) identities.setdefault(identity, []).append(artifact.suffix == ".whl") - expected = {"ling-aspec", "ling-aspec-target", "ling-aspec-runtime"} + expected = {"ling-aspec", "ling-aspec-target", "ling-aspec-runtime", "ling-aspec-megatron"} if ({name for name, version in identities} != expected or len({version for name, version in identities}) != 1 or any(sorted(kinds) != [False, True] for kinds in identities.values())): - raise ValueError("wheel identity does not match the three version-aligned source identities") + raise ValueError("wheel identity does not match the four version-aligned source identities") return result diff --git a/tools/build_release.py b/tools/build_release.py index ee00aef..befd036 100644 --- a/tools/build_release.py +++ b/tools/build_release.py @@ -33,10 +33,13 @@ def build_release(directory): subprocess.run([sys.executable, "-m", "build", "--outdir", str(directory)], cwd=root, check=True) project = tomllib.loads((root / "pyproject.toml").read_text())["project"] - for role, manifest in (("draft", "draft.txt"), ("target", "sglang-v0.5.21.txt")): + for role, manifest, name, python_min in ( + ("draft", "draft.txt", "ling-aspec", "3.11"), + ("target", "sglang-v0.5.21.txt", "ling-aspec-target", "3.11"), + ("megatron", "megatron-v0.20.0.txt", "ling-aspec-megatron", "3.12"), + ): with tempfile.TemporaryDirectory(prefix="aspec-role-") as staging: stage = Path(staging) - name = "ling-aspec" if role == "draft" else "ling-aspec-target" dependencies = [f"ling-aspec-runtime=={project['version']}"] dependencies += [line.strip() for line in (root / "requirements" / manifest).read_text().splitlines() if line.strip() and not line.lstrip().startswith("#")] @@ -45,7 +48,7 @@ def build_release(directory): f"name = {json.dumps(name)}\nversion = {json.dumps(project['version'])}\n" f"description = {json.dumps('ASpec ' + role + ' environment')}\n" "readme = \"README.md\"\nlicense = \"Apache-2.0\"\n" - "license-files = [\"LICENSE\"]\nrequires-python = \">=3.11\"\n" + f"license-files = [\"LICENSE\"]\nrequires-python = \">={python_min}\"\n" f"dependencies = {json.dumps(dependencies)}\n\n" "[project.urls]\nHomepage = \"https://github.com/inclusionAI/ASpec\"\n" "\n[tool.setuptools]\npackages = []\n") @@ -59,7 +62,7 @@ def build_release(directory): print(f"PASS {artifact}: {count} reviewed members", flush=True) runtime = directory / "runtime" runtime.mkdir() - for name in ("sglang-v0.5.21.txt", "draft.txt"): + for name in ("sglang-v0.5.21.txt", "draft.txt", "megatron-v0.20.0.txt", "megatron-build.txt"): shutil.copyfile(root / "requirements" / name, runtime / name) files = sorted(path for path in directory.rglob("*") if path.is_file()) (directory / "SHA256SUMS").write_text("".join( diff --git a/tools/install.sh b/tools/install.sh index 0a088ab..d030c4e 100755 --- a/tools/install.sh +++ b/tools/install.sh @@ -2,14 +2,16 @@ set -euo pipefail dev=0 role=draft +backend=default gpu=() for argument in "$@"; do case "$argument" in --dev) dev=1 ;; --target) role=target ;; + --megatron) backend=megatron ;; --gpu) gpu=(--gpu) ;; --help) - echo 'Usage: bash tools/install.sh [--target] [--dev] [--gpu]' + echo 'Usage: bash tools/install.sh [--target] [--megatron] [--dev] [--gpu]' echo 'For an already activated isolated venv. New users: bash setup.sh' exit 0 ;; *) printf 'Unknown argument: %s\n' "$argument" >&2; exit 2 ;; @@ -48,9 +50,27 @@ if [[ $role == target ]]; then fi extras="." (( dev == 0 )) || extras=".[dev]" -python -m pip install -r "$manifest" -e "$extras" +if [[ $backend == megatron ]]; then + python -c 'import sys; sys.exit("Megatron requires Python >=3.12" if sys.version_info[:2] < (3,12) else 0)' + manifest=requirements/megatron-v0.20.0.txt + check+=(--backend megatron) + # Build TE against the actual execution Torch, including its native headers. + # An unconstrained isolated build can select a different Torch ABI. + python -m pip install -r requirements/megatron-build.txt + aspec_nvidia=$(python -c 'import pathlib,sysconfig; print(pathlib.Path(sysconfig.get_paths()["purelib"]) / "nvidia")') + # The transport also needs CUDA 12's runtime. Including every NVIDIA wheel + # would put its headers ahead of CUDA 13 and break TE's C++ compilation. + CUDA_HOME="$aspec_nvidia/cu13" \ + NVTE_CUDA_INCLUDE_PATH="$aspec_nvidia/cu13/include" \ + CPLUS_INCLUDE_PATH="$aspec_nvidia/cudnn/include:$aspec_nvidia/nccl/include:${CPLUS_INCLUDE_PATH:-}" \ + NVTE_BUILD_USE_NVIDIA_WHEELS=0 MAX_JOBS="${MAX_JOBS:-4}" \ + python -m pip install --no-build-isolation -r "$manifest" -e "$extras" + printf 'megatron\n' > "$VIRTUAL_ENV/.aspec-backend" +else + python -m pip install -r "$manifest" -e "$extras" +fi printf '%s\n' "$role" > "$VIRTUAL_ENV/.aspec-role" -if [[ $role == target ]]; then +if [[ $role == target && $backend != megatron ]]; then aspec patch-sglang aspec patch-sglang --check fi diff --git a/tools/validation/megatron_pipeline_acceptance.py b/tools/validation/megatron_pipeline_acceptance.py new file mode 100644 index 0000000..dd07ffa --- /dev/null +++ b/tools/validation/megatron_pipeline_acceptance.py @@ -0,0 +1,125 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Bounded mixed-backend train/resume/export through the normal ASpec launcher. + +Reuses ASpec's existing recipe audits. Unlike the bundled two-GPU Quickstart, +this validation entry keeps explicit TP, DP and gradient-accumulation settings. +Input capacity, future bitwise replay, quality and throughput are separate gates. +""" + +import argparse +import json +import os +import sys +from pathlib import Path + +from omegaconf import OmegaConf + +from aspec.commands.quickstart import build_profile +from aspec.commands.recipes import audit_run, digest, execute, export_asset + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--train-data", required=True, type=Path) + parser.add_argument("--eval-data", type=Path) + parser.add_argument("--target-python", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--target-backend", choices=("sglang", "megatron"), default="megatron") + parser.add_argument("--draft-backend", choices=("fsdp", "megatron"), default="megatron") + parser.add_argument("--target-tp", type=int, choices=(1, 2), default=1) + parser.add_argument("--draft-tp", type=int, choices=(1, 2), default=1) + parser.add_argument("--dp", type=int, default=1) + parser.add_argument("--accumulation", type=int, default=2) + parser.add_argument("--sequence-length", type=int, default=4096) + parser.add_argument("--anchors", type=int, default=512) + parser.add_argument("--steps", type=int, default=4) + parser.add_argument("--gradient-checkpointing", action="store_true") + parser.add_argument("--attention-backend", choices=("fa4", "cudnn_flex_attention"), + default="cudnn_flex_attention") + parser.add_argument("--target-address", default="http://127.0.0.1:30000") + args = parser.parse_args() + if min(args.dp, args.accumulation, args.sequence_length, args.anchors) < 1: + parser.error("DP, accumulation, sequence length and anchors must be positive") + if args.steps < 2 or args.steps % 2: + parser.error("Use an even step count >=2") + if args.target_backend == "sglang" and args.target_tp != 1: + parser.error("The retained SGLang Target path requires TP1") + if args.draft_backend == "fsdp" and args.draft_tp != 1: + parser.error("Draft TP requires the Megatron backend") + if not args.train_data.is_file() or not args.target_python.is_absolute() or not args.target_python.is_file(): + parser.error("Provide existing data and an absolute Target interpreter path") + if args.eval_data is not None and not args.eval_data.is_file(): + parser.error("Evaluation data must exist") + world = args.draft_tp * args.dp + budget = world + args.target_tp + devices = os.environ.get("CUDA_VISIBLE_DEVICES", "").split(",") + if len(devices) != budget or any(not item.strip() for item in devices) or len(set(devices)) != budget: + parser.error(f"Select exactly {budget} distinct Draft+Target GPUs") + root = args.output.resolve() + root.mkdir(parents=True, exist_ok=False) + config = build_profile("qwen3-8b", args.model) + config = OmegaConf.merge(config, OmegaConf.create({ + "dataset": {"train_data_path": str(args.train_data.resolve()), "chat_template": None, + "eval_data_path": str(args.eval_data.resolve()) if args.eval_data else None, + "eval_interval": args.steps // 2}, + "model": {"target_model_backend": args.target_backend}, + "training": { + "train_backend": args.draft_backend, "draft_tensor_parallel_size": args.draft_tp, + "training_num_gpus_per_node": world, "draft_accumulation_steps": args.accumulation, + "attention_backend": args.attention_backend, "dflash_packing": False, + "gradient_checkpointing": args.gradient_checkpointing, + "max_seq_length": args.sequence_length, "dspark_num_anchors": args.anchors, + "num_epochs": 100, "lr_total_steps": args.steps, + "save_interval": args.steps // 2, "save_final_checkpoint": True, + }, + "inference": { + "inference_engine_type": "megatron" if args.target_backend == "megatron" else "sgl", + "inference_num_gpus": args.target_tp, "inference_num_gpus_per_engine": args.target_tp, + "inference_num_gpus_per_node": budget, + "megatron": {"tp_size": args.target_tp}, + "target_service": {"mode": "managed-local", "address": args.target_address, + "python_executable": str(args.target_python)}, + }, + "mooncake": {"max_seq_len": args.sequence_length, + "destination_gpu_ring_slots": 3 if args.draft_backend == "megatron" else 1, + "source_gpu_ring_slots": max(12, 2 * args.dp)}, + })) + profile = root / "profile.yaml" + OmegaConf.save(config, profile) + result = {"status": "running", "scope": "functional train/resume/native export", + "source_input": {"path": str(args.train_data.resolve()), "sha256": digest(args.train_data)}, + "world_size": world, "dp": args.dp, "draft_tp": args.draft_tp, + "target_tp": args.target_tp, "accumulation": args.accumulation} + result["gradient_checkpointing"] = args.gradient_checkpointing + if args.eval_data: + result["eval_input"] = {"path": str(args.eval_data.resolve()), "sha256": digest(args.eval_data)} + try: + doctor = [sys.executable, "-m", "aspec.commands.doctor", "--gpu"] + if args.draft_backend == "megatron": + doctor += ["--backend", "megatron"] + execute(doctor, root / "doctor.log") + execute([sys.executable, "-m", "aspec.train_entry", "--config", profile, "--dry-run"], + root / "plan.log") + for label, limit in (("first", args.steps // 2), ("resumed", args.steps)): + run = root / label + overrides = [f"training.num_train_steps={limit}", f"cache_dir={root / 'cache'}"] + if label == "resumed": + overrides.append(f"training.load_path={root / 'first/checkpoints'}") + execute([sys.executable, "-m", "aspec.commands._recipe_smoke", "--config", profile, + "--output", run, "--gpus", budget, *overrides], root / f"{label}.log") + audit_run(run, limit, label == "resumed", require_eval=bool(args.eval_data), expected_ranks=world) + export_asset(root / "resumed/checkpoints" / f"iter_{args.steps + 1:07d}", root / "asset", + log_prefix=root / "asset") + result["status"] = "passed" + except BaseException as error: + result.update(status="failed", error=str(error)) + raise + finally: + (root / "acceptance.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result), flush=True) + + +if __name__ == "__main__": + main() diff --git a/tools/validation/megatron_target_aspec_alignment.py b/tools/validation/megatron_target_aspec_alignment.py new file mode 100644 index 0000000..153db02 --- /dev/null +++ b/tools/validation/megatron_target_aspec_alignment.py @@ -0,0 +1,96 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Compare frozen Qwen3 Target features with a same-TP ASpec/SGLang cache. + +Run with torchrun in the Megatron environment. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path + +import torch +import torch.distributed as dist + +from aspec.models.megatron import initialize_parallelism, load_qwen3_target + + +def sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(8 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--reference-cache", type=Path, required=True) + parser.add_argument("--sample", type=int, default=0) + parser.add_argument("--length", type=int, default=4096) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + dist.init_process_group("nccl") + cached = torch.load(args.reference_cache, map_location="cpu", weights_only=False, mmap=True) + identity = cached["identity"] + contract = identity["inputs"]["aspec_feature_contract"] + if contract["producer"]["backend"] != "sglang": + raise ValueError("Reference cache must come from the existing ASpec SGLang producer") + if contract["tensors"]["last_hidden_states"]["value_stage"] != "post_norm": + raise ValueError("Expected the existing post-norm Target feature ABI") + sample = cached["samples"][args.sample] + if sample["input_ids"].shape[0] != 1 or sample["input_ids"].shape[1] < args.length: + raise ValueError("Select one complete reference sequence") + layers = contract["aux_hidden_state_layers"] + groups = initialize_parallelism(dist.get_world_size()) + model, config = load_qwen3_target(args.model, groups) + captured = {} + + def capture(layer, branch, residual): + captured[str(layer)] = branch + residual + + versions = [p._version for p in model.parameters()] + ids = sample["input_ids"][:, :args.length].cuda() + captured["final"] = model.prefill(ids, layers, capture) + frozen = (not any(p.requires_grad for p in model.parameters()) + and versions == [p._version for p in model.parameters()]) + verdict = [None] + if dist.get_rank() == 0: + expected = {str(layer): sample["hidden_states"][0, :args.length, + index * config.hidden_size:(index + 1) * config.hidden_size].cuda() + for index, layer in enumerate(layers)} + expected["final"] = sample["last_hidden_states"][0, :args.length].cuda() + measurements = {} + for name, target in expected.items(): + value, target = captured[name].double(), target.double() + measurements[name] = { + "mean": value.mean().item(), "reference_mean": target.mean().item(), + "equal": torch.equal(value, target), + "relative_l2": ((value - target).norm() / target.norm().clamp_min(1e-12)).item(), + "max_absolute": (value - target).abs().max().item(), + "cosine": torch.nn.functional.cosine_similarity(value.flatten(), target.flatten(), dim=0).item()} + passed = frozen and all(row["relative_l2"] <= 0.02 and row["cosine"] >= 0.999 + for row in measurements.values()) + result = {"reference_tp": identity["inputs"].get("sglang_tp_size"), "model_class": type(model).__name__, + "passed": passed, "reference": "existing_aspec_sglang_feature_cache", + "reference_sha256": sha(args.reference_cache), "reference_identity": identity, + "sample": args.sample, "length": args.length, "tp": dist.get_world_size(), + "target_frozen": frozen, "features": measurements, + "tolerances": {"relative_l2_max": 0.02, "cosine_min": 0.999}} + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("x") as stream: + json.dump(result, stream, indent=2) + print(json.dumps({key: value for key, value in result.items() if key != "reference_identity"}), flush=True) + verdict[0] = passed + dist.broadcast_object_list(verdict, src=0) + dist.destroy_process_group() + if not verdict[0]: + raise SystemExit("Megatron features did not align with the existing ASpec Target") + + +if __name__ == "__main__": + main() diff --git a/tools/validation/megatron_torch_alignment.py b/tools/validation/megatron_torch_alignment.py new file mode 100644 index 0000000..7f8aa0a --- /dev/null +++ b/tools/validation/megatron_torch_alignment.py @@ -0,0 +1,662 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Compare native ASpec Torch and Megatron DSpark updates on fixed Target features. + +Run each backend in its own environment. Measure the objective at every update +and compare logical gradients, parameters and optimizer state at selected steps. +A fresh process can restore a checkpoint and compare with a continuous run. +""" + +import argparse +import copy +import hashlib +import importlib.util +import inspect +import json +import math +import os +import time +from pathlib import Path +from unittest.mock import patch + +import torch +import torch.distributed as dist +from torch.distributed.checkpoint.state_dict import StateDictOptions, get_model_state_dict +from torch.distributed.tensor import DTensor + +import aspec + + +def file_sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(8 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def tensor_record(value): + value = value.detach().cpu().contiguous() + return {"shape": list(value.shape), "dtype": str(value.dtype), + "sha256": hashlib.sha256(value.reshape(-1).view(torch.uint8).numpy().tobytes()).hexdigest()} + + +def write_json(path, value): + path = Path(path) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n") + temporary.replace(path) + + +def local_state(trainer): + def record(value): + if isinstance(value, DTensor): + value = value.to_local() + if isinstance(value, torch.Tensor): + return tensor_record(value) + if isinstance(value, dict): + return {str(k): record(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [record(v) for v in value] + return value + + return {"model": record(trainer.model.state_dict()), + "masters": record(trainer.optimizer.fp32_params), + "optimizer": record(trainer.optimizer.state_dict()), + "scheduler": record(trainer.lr_scheduler.state_dict()), + "torch_rng": record(torch.get_rng_state()), + "cuda_rng": record(torch.cuda.get_rng_state()), + "backend_rng": (record(trainer._checkpoint_backend_rng()) + if hasattr(trainer, "_checkpoint_backend_rng") else None), + "frozen_target_head": tensor_record(trainer.target_lm_head_weight), + "global_step": trainer.global_step} + + +def logical_cpu(trainer, value, parameter): + if value is None: + return None + if isinstance(value, DTensor): + value = value.full_tensor() + elif getattr(parameter, "_aspec_tp_axis", None) is not None: + if hasattr(trainer, "logical_tensor"): + value = trainer.logical_tensor(value, parameter).full_tensor() + else: + from aspec.models.megatron_draft import checkpoint_tensor + + value = checkpoint_tensor(value, parameter, trainer.tp_mesh).full_tensor() + return value.detach().cpu().clone() + + +def named_trainable(trainer): + result = [(name, parameter) for name, parameter in trainer.draft_model.named_parameters() + if parameter.requires_grad] + assert all(p is expected for (_, p), expected in zip( + result, trainer.optimizer.model_params, strict=True)) + return result + + +def logical_optimizer_state(trainer): + result = {key: {} for key in ("model", "masters", "exp_avg", "exp_avg_sq")} + for index, (name, parameter) in enumerate(named_trainable(trainer)): + master = trainer.optimizer.fp32_params[index] + result["model"][name] = logical_cpu(trainer, parameter, parameter) + result["masters"][name] = logical_cpu(trainer, master, parameter) + state = trainer.optimizer.optimizer.state.get(master, {}) + for key in ("exp_avg", "exp_avg_sq"): + if key in state: + result[key][name] = logical_cpu(trainer, state[key], parameter) + return result + + +def scalar_diff(actual, reference, epsilon=1e-12): + """Reference-relative signed difference, with an explicit denominator floor.""" + actual, reference = float(actual), float(reference) + if not math.isfinite(actual) or not math.isfinite(reference): + raise ValueError("Nonfinite scalar in numerical comparison") + delta = actual - reference + return {"actual": actual, "reference": reference, "signed_difference": delta, + "absolute_difference": abs(delta), + "signed_rel_diff": delta / max(abs(reference), epsilon), + "abs_rel_diff": abs(delta) / max(abs(reference), epsilon), + "denominator_floored": abs(reference) < epsilon, "epsilon": epsilon} + + +def compare_vectors(actual, expected, *, device=None): + """Full-vector reductions and explicit, near-zero-aware pointwise rel diffs. + + Norms/means/maxima/threshold counts use every element and FP64 reductions. + Only quantiles are sampled: systematic coordinates in the logical vector, + independent of values, with separate per-parameter samples. No training state + or RNG is changed. The denominator floor is max(1e-12, 1e-6 * reference RMS) + per parameter; floored coordinates are counted rather than silently removed. + """ + if set(actual) != set(expected): + raise ValueError("Compared trainable parameter sets differ") + device = device or ("cuda" if torch.cuda.is_available() else "cpu") + count = sum(v.numel() for v in expected.values() if v is not None) + stride = max(1, math.ceil(count / 262144)) + totals = torch.zeros(14, dtype=torch.float64, device=device) + global_samples, per_parameter = [], [] + exceed_counts = torch.zeros(4, dtype=torch.int64, device=device) + thresholds = (.01, .03, .1, 1.0) + exact, offset = True, 0 + + def ratio(numerator, denominator): + return numerator / denominator if denominator else (0.0 if numerator == 0 else None) + + def summarize(values): + n, e2, b2, a2, e1, b1, es, dot, floors, point_sum, maximum, reference_max, point_max, mismatches = values + return {"numel": int(n), "relative_l2": ratio(math.sqrt(e2), math.sqrt(b2)), + "relative_l1": ratio(e1, b1), "relative_max": ratio(maximum, reference_max), + "rmse": math.sqrt(e2 / max(n, 1)), "mean_absolute": e1 / max(n, 1), + "max_absolute": maximum, "signed_mean_difference": es / max(n, 1), + "reference_rms": math.sqrt(b2 / max(n, 1)), + "cosine": ratio(dot, math.sqrt(a2 * b2)), + "pointwise_rel_diff_mean": point_sum / max(n, 1), + "pointwise_rel_diff_max": point_max, "denominator_floored_count": int(floors), + "different_elements": int(mismatches), "zero_reference": b2 == 0} + + def quantiles(values): + if not values or not sum(v.numel() for v in values): + return {"sample_size": 0, "p50": None, "p95": None, "p99": None} + values = torch.cat(values).float() + return {"sample_size": values.numel(), + **{label: val for label, val in zip(("p50", "p95", "p99"), + torch.quantile(values, torch.tensor([.5, .95, .99], device=device)).tolist(), strict=True)}} + + for name, value in actual.items(): + reference = expected[name] + if value is None or reference is None: + if value is not None or reference is not None: + raise ValueError(f"Missing gradient differs: {name}") + continue + if value.shape != reference.shape: + raise ValueError(f"Compared tensor shapes differ: {name}") + a, b = value.to(device).float().flatten(), reference.to(device).float().flatten() + if not (torch.isfinite(a).all() & torch.isfinite(b).all()).item(): + raise ValueError(f"Nonfinite tensor in numerical comparison: {name}") + ad, bd = a.double(), b.double() + dd = delta = ad - bd + e2, b2 = dd.square().sum(), bd.square().sum() + floor = max(1e-12, 1e-6 * (b2 / max(b.numel(), 1)).sqrt().item()) + pointwise = delta.abs() / bd.abs().clamp_min(floor) + exceeds = torch.stack([(pointwise > threshold).sum() for threshold in thresholds]) + exceed_counts += exceeds + row = torch.stack((b.new_tensor(b.numel()), e2, b2, ad.square().sum(), dd.abs().sum(), + bd.abs().sum(), dd.sum(), (ad * bd).sum(), (b.abs() < floor).sum(), + pointwise.double().sum(), delta.abs().max(), b.abs().max(), + pointwise.max(), (a != b).sum())).double() + totals[:10] += row[:10] + totals[10:13] = torch.maximum(totals[10:13], row[10:13]) + totals[13] += row[13] + values = row.tolist() + exact = exact and value.dtype == reference.dtype and values[13] == 0 + row_summary = summarize(values) + per_parameter.append({"parameter": name, "squared_error": values[1], + "denominator_floor": floor, **row_summary, + "pointwise_exceed_counts": dict(zip(map(str, thresholds), exceeds.tolist(), strict=True)), + "pointwise_rel_diff_quantiles": quantiles([ + pointwise[::max(1, math.ceil(pointwise.numel() / 4096))]])}) + global_samples.append(pointwise[(-offset) % stride::stride].clone()) + offset += b.numel() + del a, b, delta, ad, bd, dd, pointwise + return {**summarize(totals.tolist()), "bitwise_equal": exact, + "pointwise_exceed_counts": dict(zip(map(str, thresholds), exceed_counts.tolist(), strict=True)), + "pointwise_rel_diff_quantiles": quantiles(global_samples), + "quantile_method": "systematic logical coordinates, global <=262144, per-parameter <=4096", + "denominator_floor_policy": "max(1e-12, 1e-6 * per-parameter reference RMS)", + "per_parameter": per_parameter, + "largest_errors": sorted(per_parameter, key=lambda row: row["squared_error"], reverse=True)[:5]} + + +def selected_steps(value, maximum): + if value == "all": + return set(range(1, maximum + 1)) + result = {int(step) for step in value.split(",") if step} + if any(step < 1 or step > maximum for step in result): + raise ValueError("Observation steps must be within the training trajectory") + return result + + +def wait_reference(path, timeout): + deadline = time.monotonic() + timeout + while not path.is_file(): + if time.monotonic() >= deadline: + raise TimeoutError(f"Reference observation unavailable: {path}") + time.sleep(.5) + + +def source_identity(trainer, expected): + from aspec.training import checkpoint + + root = Path(aspec.__file__).resolve().parent + if root != expected.resolve() / "aspec": + raise RuntimeError(f"Wrong ASpec checkout: {root}, expected {expected}") + files = {} + for method in (type(trainer), trainer._train_core_from_queue, trainer._train_step, + trainer._normalize_accumulated_gradients, type(trainer.optimizer), + type(trainer.optimizer).step, checkpoint.save, checkpoint.load): + path = Path(inspect.getfile(method)).resolve() + if path == Path(__file__).with_name("torch_tp_dspark.py").resolve(): + files["validation/torch_tp_dspark.py"] = file_sha(path) + else: + files[str(path.relative_to(root))] = file_sha(path) + return {"aspec": str(root), "python": os.sys.executable, "prefix": os.sys.prefix, + "torch": torch.__version__, "trainer": type(trainer).__name__, "files": files, + "harness_sha256": file_sha(__file__)} + + +def save_native_checkpoint(trainer, runtime, committed, fixture_sha, output, steps): + from aspec.training import checkpoint + + rank = dist.get_rank() + if rank == 0: + checkpoint.prepare_checkpoint_save(runtime.checkpoint_dir, trainer.global_step) + dist.barrier() + trainer.save_model(trainer.global_step, force_sync=True) + if rank == 0: + checkpoint.save_checkpoint_work_state(runtime.checkpoint_dir, trainer.global_step, { + "committed_work_ids": committed, "dataset_fingerprint": "sha256:" + fixture_sha, + "dataset_size": steps * 2 * trainer.dp_size}) + dist.barrier() + write_json(output / f"checkpoint-state-rank{rank}.json", local_state(trainer)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--backend", choices=("fsdp", "megatron", "torch-tp"), required=True) + parser.add_argument("--phase", choices=("prepare", "uninterrupted", "checkpoint-control", "split", "resume"), required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--teacher", type=Path, required=True) + parser.add_argument("--initial", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--reference", type=Path, action="append", default=[]) + parser.add_argument("--resume", type=Path) + parser.add_argument("--tp", type=int, default=1) + parser.add_argument("--megatron-grad-dtype", choices=("fp32", "bf16")) + parser.add_argument("--steps", type=int, default=4) + parser.add_argument("--stop-after", type=int, help="Stop at this update without shortening the LR horizon") + parser.add_argument("--split-step", type=int, default=2) + parser.add_argument("--length", type=int, default=4096) + parser.add_argument("--anchors", type=int, default=512) + parser.add_argument("--tensor-steps", default="all", help="Full gradient/update observations: all or comma-separated steps") + parser.add_argument("--state-steps", default="", help="Model/master/moment observations: comma-separated steps") + parser.add_argument("--save-reference-tensors", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--reference-wait-seconds", type=float, default=0) + parser.add_argument("--attention-backend", choices=("fa4", "cudnn_flex_attention"), default="cudnn_flex_attention") + parser.add_argument("--attention-backward-deterministic", action="store_true", + help="Require native deterministic cuDNN-FE attention backward on every rank") + args = parser.parse_args() + if args.backend == "fsdp" and args.tp != 1: + parser.error("The Torch baseline uses its existing TP1 execution") + if args.backend != "megatron" and args.megatron_grad_dtype is not None: + parser.error("Megatron gradient precision applies only to the Megatron backend") + if not args.attention_backward_deterministic: + run(args) + return + if args.attention_backend != "cudnn_flex_attention" or args.phase == "prepare": + parser.error("The deterministic backward check requires a cuDNN training run") + + from cudnn.flex_attention import autograd, flex_attn_func + + backward = autograd._flex_attention_backward + calls = {"forward": 0, "backward": 0} + + def deterministic_forward(*inputs, **options): + options["deterministic"] = True + calls["forward"] += 1 + return flex_attn_func(*inputs, **options) + + def checked_backward(*inputs, **options): + if not options.get("deterministic"): + raise RuntimeError("cuDNN attention backward did not receive deterministic=True") + calls["backward"] += 1 + return backward(*inputs, **options) + + with (patch("cudnn.flex_attention.flex_attn_func", deterministic_forward), + patch.object(autograd, "_flex_attention_backward", checked_backward)): + run(args, attention_calls=calls) + + +def run(args, *, attention_calls=None): + from aspec.config.train_config import load_config + from aspec.models.draft.auto import AutoDraftModelConfig + from aspec.planning import compile_execution_plan + from aspec.training import checkpoint + from aspec.training.dspark_trainer import DSparkTrainer + from aspec.utils.distributed import init_direct_cp_groups, init_gloo_group + + if args.backend == "torch-tp": + class RejectMegatronImports: + def find_spec(self, fullname, path=None, target=None): + if fullname == "megatron" or fullname.startswith("megatron."): + raise RuntimeError("The independent Torch TP validation cannot import Megatron Core") + + os.sys.meta_path.insert(0, RejectMegatronImports()) + tensor_steps = selected_steps(args.tensor_steps, args.steps) + state_steps = selected_steps(args.state_steps, args.steps) + torch.set_num_threads(4) + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + dist.init_process_group("nccl", device_id=torch.device("cuda", torch.cuda.current_device())) + init_gloo_group() + init_direct_cp_groups(1) + rank, world = dist.get_rank(), dist.get_world_size() + output = args.output.resolve() + if rank == 0: + output.mkdir(parents=True, exist_ok=False) + dist.barrier() + os.environ.setdefault("ASPEC_TRAIN_DATA", str(args.source / "examples/data/sample_conversations.jsonl")) + os.environ.setdefault("ASPEC_EVAL_DATA", str(args.source / "examples/data/sample_conversations.jsonl")) + base_backend = "fsdp" if args.backend == "torch-tp" else args.backend + overrides = [f"model.target_model_path={args.model}", + f"model.draft_model_config={args.source / 'aspec/config/dspark_draft_config.json'}", + f"training.train_backend={base_backend}", "training.dflash_packing=false", + f"training.training_num_gpus_per_node={world}", "training.draft_accumulation_steps=2", + "training.deterministic_anchor_sampling=true", f"training.max_seq_length={args.length}", + f"training.dspark_num_anchors={args.anchors}", f"training.num_train_steps={args.steps}", + f"training.lr_total_steps={args.steps}", f"training.attention_backend={args.attention_backend}", + "inference.target_prefill_memory_optimized=true", + "mooncake.source_gpu_ring_slots=12", "mooncake.destination_gpu_ring_slots=3", + f"output_dir={output}"] + if args.initial: + overrides.append(f"model.initial_draft_model_path={args.initial}") + if args.resume: + overrides.append(f"training.load_path={args.resume}") + if args.backend == "megatron": + overrides.append(f"training.draft_tensor_parallel_size={args.tp}") + if args.megatron_grad_dtype is not None: + overrides.append(f"training.megatron_grad_dtype={args.megatron_grad_dtype}") + config = load_config(str(args.source / "configs/recipes/qwen3_8b_dspark.yaml"), cli_args=overrides) + plan = compile_execution_plan(config) + runtime = plan.to_runtime_args() + runtime.rank, runtime.world_size = rank, world + if args.backend == "megatron": + from aspec.training.megatron_trainer import MegatronDSparkTrainer + + trainer = MegatronDSparkTrainer(runtime) + elif args.backend == "torch-tp": + # The maintained public Plan still rejects Torch Draft TP. This + # explicitly recorded validation extension tests its native DTensor + # arithmetic without claiming production Plan/backend support. + runtime.draft_tensor_parallel_size = args.tp + path = Path(__file__).with_name("torch_tp_dspark.py") + spec = importlib.util.spec_from_file_location("aspec_torch_tp_validation", path) + module = importlib.util.module_from_spec(spec) + os.sys.modules[spec.name] = module + spec.loader.exec_module(module) + trainer = module.TorchTPDSparkTrainer(runtime) + write_json(output / f"torch-tp-validation-rank{rank}.json", { + "scope": "validation-only extension of existing Torch DSpark trainer; not a production backend", + "base_plan_backend": "fsdp", "base_plan_draft_tp": 1, + "observed_draft_tp": trainer.tp_size, "observed_dp": trainer.dp_size, + "tp_operators": "native PyTorch DTensor linear/redistribute/autograd", + "adapter_sha256": file_sha(path), + }) + else: + trainer = DSparkTrainer(runtime) + trainer.init_model(AutoDraftModelConfig.from_file(runtime.draft_model_config), args.model) + identity = source_identity(trainer, args.source) + if args.backend == "megatron": + write_json(output / f"gradient-precision-rank{rank}.json", trainer._gradient_precision) + if args.backend != "megatron" and any(name == "megatron" or name.startswith("megatron.") for name in os.sys.modules): + raise RuntimeError("The baseline imported Megatron") + write_json(output / f"origin-rank{rank}.json", identity) + if args.phase == "prepare": + if args.backend != "fsdp" or world != 1: + raise ValueError("Prepare the common initialization with the actual TP1/DP1 Torch trainer") + from safetensors.torch import save_file + + state = get_model_state_dict(trainer.draft_model, options=StateDictOptions(full_state_dict=True, cpu_offload=True)) + save_file({name: value.contiguous() for name, value in state.items()}, str(output / "model.safetensors")) + trainer.draft_model.config.save_pretrained(output) + write_json(output / "prepared.json", {"source": identity, "initial_sha256": file_sha(output / "model.safetensors")}) + dist.destroy_process_group() + return + + initial_state = local_state(trainer) + write_json(output / f"initial-state-rank{rank}.json", initial_state) + if args.phase == "resume": + if args.resume is None: + raise ValueError("Resume requires a native checkpoint") + expected = json.loads((args.resume.parent.parent / f"checkpoint-state-rank{rank}.json").read_text()) + restored = initial_state == expected + write_json(output / f"restore-rank{rank}.json", {"exact": restored}) + if not restored: + differences = [key for key in expected if expected[key] != initial_state[key]] + raise RuntimeError(f"Native checkpoint state did not restore exactly on rank {rank}: {differences}") + del initial_state + saved_work = checkpoint.load_checkpoint_work_state(runtime) if args.resume else None + committed = list(saved_work["committed_work_ids"]) if saved_work else [] + samples = torch.load(args.teacher, map_location="cpu", weights_only=False, mmap=True)["samples"] + fixture_sha = file_sha(args.teacher) + manifest = {"teacher_sha256": fixture_sha, "initial_sha256": file_sha(args.initial) if args.initial else None, + "steps": args.steps, "length": args.length, "anchors": args.anchors, + "accumulation": 2, "attention_backend": args.attention_backend, + "dp_size": trainer.dp_size, "seed": runtime.seed, + "learning_rate": runtime.learning_rate, "warmup_ratio": runtime.warmup_ratio, + "objective_weights": [trainer.ce_loss_alpha, trainer.l1_loss_alpha, trainer.confidence_head_alpha]} + if rank == 0: + write_json(output / "run-manifest.json", manifest) + write_json(output / "runtime-config.json", json.loads(json.dumps(vars(runtime), default=str))) + for reference in args.reference: + wait_reference(reference / "run-manifest.json", args.reference_wait_seconds) + if manifest != json.loads((reference / "run-manifest.json").read_text()): + raise RuntimeError("Compared trajectories differ in fixed inputs or training contract") + trainable = named_trainable(trainer) + publish = trainer.dp_rank == 0 + writer = publish and getattr(trainer, "tp_rank", 0) == 0 + if args.phase != "resume" and publish: + logical = logical_optimizer_state(trainer) + if writer: + initial = {key: {name: tensor_record(value) for name, value in tensors.items()} + for key, tensors in logical.items()} + write_json(output / "initial-logical.json", initial) + for reference in args.reference: + wait_reference(reference / "initial-logical.json", args.reference_wait_seconds) + expected = json.loads((reference / "initial-logical.json").read_text()) + if initial != expected: + raise RuntimeError("Compared trainers did not start from identical logical weights/masters") + del logical + observation = {"terms": [], "anchors": [], "gradients": {}, "before": {}} + original_forward = trainer._forward + + def observed_forward(*values, **kwargs): + result = original_forward(*values, **kwargs) + observation["terms"].append(torch.stack([part.detach() for part in result[-1]])) + return result + + trainer._forward = observed_forward + original_sample = trainer.dflash._sample_anchor_positions + + def observed_sample(*values, **kwargs): + result = original_sample(*values, **kwargs) + observation["anchors"].append(tuple(value.detach().clone() for value in result)) + return result + + trainer.dflash._sample_anchor_positions = observed_sample + original_step = trainer.optimizer.step + + def observed_step(*values, **kwargs): + if publish and trainer.global_step + 1 in tensor_steps: + for index, (name, parameter) in enumerate(trainable): + gradient = (parameter.main_grad if hasattr(parameter, "main_grad") else parameter.grad) + observation["gradients"][name] = logical_cpu(trainer, gradient, parameter) + observation["before"][name] = logical_cpu(trainer, trainer.optimizer.fp32_params[index], parameter) + norm = original_step(*values, **kwargs) + observation["norm"] = norm.detach().clone() + return norm + + trainer.optimizer.step = observed_step + trainer._work_lifecycle_enabled = True + stop = args.split_step if args.phase == "split" else args.steps + if args.stop_after is not None: + if not trainer.global_step < args.stop_after <= stop: + raise ValueError("stop-after must follow the initial step and stay within the declared horizon") + stop = args.stop_after + while trainer.global_step < stop: + step = trainer.global_step + 1 + observation.update(terms=[], anchors=[], gradients={}, before={}) + batches = [] + for micro in range(2): + index = ((step - 1) * 2 * trainer.dp_size + micro * trainer.dp_size + trainer.dp_rank) % len(samples) + batch = {key: value[:, :args.length] if isinstance(value, torch.Tensor) else copy.deepcopy(value) + for key, value in samples[index].items()} + batch["_aspec_work"] = [{"execution_id": f"window-{step}/dp-{trainer.dp_rank}/micro-{micro}"}] + batches.append(batch) + trainer.data_fetcher = batches + metrics = trainer._train_core_from_queue(step=step - 1, num_batches=2) + if hasattr(metrics, "finalize"): + metrics = metrics.finalize() + local_work = trainer.pop_committed_work(step) + participants = [None] * world + dist.all_gather_object(participants, local_work) + new_work = [item for replica in participants for item in replica] + if len(new_work) != 2 * trainer.dp_size or set(new_work).intersection(committed): + raise RuntimeError("Committed work was duplicated or omitted") + committed.extend(new_work) + totals = torch.stack(observation["terms"]).sum(0) + dist.all_reduce(totals, group=trainer.dp_group) + anchors = [{"positions": tensor_record(a), "keep": tensor_record(k)} for a, k in observation["anchors"]] + write_json(output / f"anchors-step{step}-rank{rank}.json", anchors) + participant_anchors = [None] * world + dist.all_gather_object(participant_anchors, {"replica": trainer.dp_rank, "anchors": anchors}) + by_replica = {} + for participant in participant_anchors: + replica = str(participant["replica"]) + if replica in by_replica and by_replica[replica] != participant["anchors"]: + raise RuntimeError("TP participants used different anchors") + by_replica[replica] = participant["anchors"] + updates = {} + if publish and step in tensor_steps: + for index, (name, parameter) in enumerate(trainable): + current = logical_cpu(trainer, trainer.optimizer.fp32_params[index], parameter) + if writer: + updates[name] = current - observation["before"][name] + if writer: + vectors = {"gradients": observation["gradients"], "updates": updates} + record = {"step": step, "loss": (totals[0] / totals[1]).item(), + "numerator": totals[0].item(), "denominator": totals[1].item(), + "gradient_norm": observation["norm"].item(), + "learning_rate_after_update": trainer.optimizer.get_learning_rate(), + "gradient_dtypes": sorted({str(v.dtype) for v in vectors["gradients"].values() if v is not None}), + "anchors": anchors, "anchors_by_replica": by_replica, + "metrics": metrics, "comparisons": {}, + "full_tensor_observation": step in tensor_steps, + "fixture_sample_indices_by_replica": { + str(replica): [((step - 1) * 2 * trainer.dp_size + micro * trainer.dp_size + replica) + % len(samples) for micro in range(2)] + for replica in range(trainer.dp_size)}} + if args.save_reference_tensors and step in tensor_steps and args.phase in {"uninterrupted", "checkpoint-control"}: + torch.save(vectors, output / f"vectors-step{step}.pt") + for reference in args.reference: + wait_reference(reference / f"step{step}.json", args.reference_wait_seconds) + expected_record = json.loads((reference / f"step{step}.json").read_text()) + measurements = {} + if step in tensor_steps: + expected = torch.load(reference / f"vectors-step{step}.pt", weights_only=True, mmap=True) + measurements = {key: compare_vectors(value, expected[key]) for key, value in vectors.items()} + del expected + scalar_values = {"objective": record["loss"], "gradient_norm": record["gradient_norm"], + "learning_rate": record["learning_rate_after_update"], + **{key: value for key, value in metrics.items() if key.startswith("train/") + and isinstance(value, (int, float))}} + expected_scalars = {"objective": expected_record["loss"], + "gradient_norm": expected_record["gradient_norm"], + "learning_rate": expected_record["learning_rate_after_update"], + **expected_record["metrics"]} + measurements["scalars"] = {key: scalar_diff(value, expected_scalars[key]) + for key, value in scalar_values.items()} + measurements.update(loss_relative_error=abs(record["loss"] / expected_record["loss"] - 1), + denominator_exact=record["denominator"] == expected_record["denominator"], + anchors_exact=by_replica == expected_record["anchors_by_replica"], + learning_rate_exact=record["learning_rate_after_update"] == expected_record["learning_rate_after_update"]) + measurements["tolerance_passed"] = (measurements["loss_relative_error"] <= 0.01 + and measurements["denominator_exact"] and measurements["anchors_exact"] + and measurements["learning_rate_exact"] + and (step not in tensor_steps or all( + measurements[key]["relative_l2"] is not None + and measurements[key]["relative_l2"] <= 0.03 for key in vectors))) + measurements["tolerance_scope"] = "loss/gradient/update" if step in tensor_steps else "loss only; full tensors not observed" + record["comparisons"][str(reference)] = measurements + if publish and step in state_steps: + logical = logical_optimizer_state(trainer) + if writer: + if args.save_reference_tensors: + torch.save(logical, output / f"logical-state-step{step}.pt") + state_comparisons = {} + for reference in args.reference: + wait_reference(reference / f"step{step}.json", args.reference_wait_seconds) + expected = torch.load(reference / f"logical-state-step{step}.pt", weights_only=True, mmap=True) + state_comparisons[str(reference)] = {key: compare_vectors(value, expected[key]) + for key, value in logical.items()} + del expected + write_json(output / f"logical-state-step{step}.json", state_comparisons) + del logical + if writer: + write_json(output / f"step{step}.json", record) + print(json.dumps({"step": step, "backend": args.backend, "loss": record["loss"], + "full_tensors": step in tensor_steps, + "scalar_rel_diffs": {ref: {key: values["abs_rel_diff"] for key, values in c["scalars"].items() + if key in {"objective", "train/ce_loss", "train/l1_loss", "train/confidence_loss"}} + for ref, c in record["comparisons"].items()}}), flush=True) + observation.update(gradients={}, before={}, anchors=[], terms=[]) + if writer: + del vectors + del updates, batches + dist.barrier() + if args.phase == "checkpoint-control" and step == args.split_step: + save_native_checkpoint(trainer, runtime, committed, fixture_sha, output, args.steps) + + if args.phase == "split": + save_native_checkpoint(trainer, runtime, committed, fixture_sha, output, args.steps) + write_json(output / f"final-state-rank{rank}.json", local_state(trainer)) + if trainer.global_step == args.steps and publish and args.steps not in state_steps: + logical = logical_optimizer_state(trainer) + if writer: + if args.save_reference_tensors and args.phase in {"uninterrupted", "checkpoint-control"}: + torch.save(logical, output / "final-logical.pt") + comparisons = {} + for reference in args.reference: + wait_reference(reference / "completed.json", args.reference_wait_seconds) + expected = torch.load(reference / "final-logical.pt", weights_only=True, mmap=True) + comparisons[str(reference)] = {key: compare_vectors(values, expected[key]) + for key, values in logical.items()} + write_json(output / "final-logical-comparisons.json", comparisons) + del logical + elif trainer.global_step == args.steps and writer: + write_json(output / "final-logical-comparisons.json", + json.loads((output / f"logical-state-step{args.steps}.json").read_text())) + if attention_calls is not None: + if not attention_calls["backward"]: + raise RuntimeError("No native deterministic cuDNN attention backward calls were observed") + write_json(output / f"attention-backward-rank{rank}.json", attention_calls) + if rank == 0: + if args.backend == "torch-tp": + write_json(output / "torch-tp-execution.json", { + "optimizer_fused_clip_scaling": trainer.optimizer._fuse_clip_scaling, + "qk_preparation_backends": [layer.self_attn.qk_prepare_backend + for layer in trainer.draft_model.layers], + "l1_backend": trainer.dflash.l1_loss_backend, + "l1_kernel_invocations": dict(trainer.dflash.l1_kernel_invocations), + "megatron_core_imported": any(name == "megatron" or name.startswith("megatron.") + for name in os.sys.modules), + }) + write_json(output / "completed.json", {"phase": args.phase, "steps": trainer.global_step, + "backend": args.backend, "world_size": world, "tp": args.tp, + "committed_work_ids": committed, "teacher_sha256": fixture_sha, + "runtime": identity, "attention_backend": args.attention_backend}) + write_json(output / "observation-policy.json", { + "tensor_steps": sorted(tensor_steps), "state_steps": sorted(state_steps), + "save_reference_tensors": args.save_reference_tensors, + "scalar_rel_diff": "(actual-reference) / max(abs(reference), 1e-12)", + "tensor_pointwise_rel_diff": "abs(actual-reference) / max(abs(reference), max(1e-12, 1e-6 * per-parameter reference RMS))", + "trajectory": "independent native updates from identical initial weights; subsequent differences include accumulated drift", + "fixture_samples": len(samples), "fixture_repeats": True, + "initial_sha256": file_sha(args.initial) if args.initial else None}) + dist.barrier() + dist.destroy_process_group() + + +if __name__ == "__main__": + main() diff --git a/tools/validation/torch_tp_dspark.py b/tools/validation/torch_tp_dspark.py new file mode 100644 index 0000000..8d46ef2 --- /dev/null +++ b/tools/validation/torch_tp_dspark.py @@ -0,0 +1,187 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Validation-only native PyTorch TP for the retained ASpec DSpark trainer. + +Independently implements the public DTensor linear placements; no Megatron +operators or optimizer are imported. Plain local Parameters preserve ASpec's +fused master-AdamW/clipping path. DTensor wraps operands inside each projection +so its native autograd owns TP collectives. This is not a production backend: +the public Torch configuration continues to reject Draft TP greater than one. +""" + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.distributed.device_mesh import init_device_mesh +from torch.distributed.tensor import DTensor, Replicate, Shard + +from aspec.training.dspark_trainer import DSparkTrainer +from aspec.training.fsdp import apply_fsdp2 +from aspec.training.optimizer import BF16Optimizer + + +class TorchTPLinear(torch.nn.Module): + def __init__(self, original, mesh, axis): + super().__init__() + if original.bias is not None: + raise ValueError("The Torch TP validation path requires bias-free projections") + self.mesh, self.axis = mesh, axis + self.weight = torch.nn.Parameter( + original.weight.chunk(mesh.size(), dim=axis)[mesh.get_local_rank()].clone(), + requires_grad=original.weight.requires_grad, + ) + self.weight._aspec_tp_axis = axis + + def forward(self, value): + placement = Replicate() if self.axis == 0 else Shard(value.ndim - 1) + inputs = DTensor.from_local(value, self.mesh, (placement,), run_check=False) + weight = DTensor.from_local(self.weight, self.mesh, (Shard(self.axis),), run_check=False) + result = F.linear(inputs, weight) + if self.axis == 1: + result = result.redistribute(placements=(Replicate(),)) + return result.to_local() + + +class TorchTPOptimizer(BF16Optimizer): + def checkpoint_tensor(self, index, value): + return self.model._validation_logical_tensor(value, self.model_params[index]) + + def _clip_gradients(self): + group = self.model._validation_tp_group + gradients = [master.grad for parameter, master in zip( + self.model_params, self.fp32_params, strict=True) + if master.grad is not None + and (hasattr(parameter, "_aspec_tp_axis") or group.rank() == 0)] + # The global vector contains each replicated parameter once, and every + # distinct shard. Retain the native Torch norm and fused clip-scaling. + norm_squared = torch.nn.utils.get_total_norm(gradients).square() + dist.all_reduce(norm_squared, group=group) + norm = norm_squared.sqrt_() + coefficient = (self.max_grad_norm / (norm + 1e-6)).clamp(max=1.0) + if self._fuse_clip_scaling: + self.optimizer.grad_scale = coefficient.reciprocal() + else: + from aspec.models.ops.gradient_scaling import scale_gradients_ + + self.optimizer.grad_scale = None + scale_gradients_([p.grad for p in self.fp32_params if p.grad is not None], coefficient) + return norm + + +class TorchTPDSparkTrainer(DSparkTrainer): + execution_backend = "torch-tp-validation" + _optimizer_class = TorchTPOptimizer + + def _setup_device_mesh(self): + size = self.args.draft_tensor_parallel_size + if size != 2 or dist.get_world_size() % size: + raise ValueError("The Torch TP validation adapter requires TP2 and complete DP replicas") + mesh = init_device_mesh("cuda", (dist.get_world_size() // size, size), + mesh_dim_names=("dp", "tp")) + self.tp_mesh, self.dp_mesh = mesh["tp"], mesh["dp"] + self.tp_group, self.dp_group = self.tp_mesh.get_group(), self.dp_mesh.get_group() + self.tp_size, self.tp_rank = size, self.tp_mesh.get_local_rank() + self.dp_size, self.dp_rank = self.dp_mesh.size(), self.dp_mesh.get_local_rank() + self.mesh = self.grad_sync_mesh = self.dp_mesh + self.cp_size, self.cp_rank, self.cp_group = 1, 0, None + self.cache_rank = dist.get_rank() + + def _get_init_weight_context_manager(self): + return lambda: torch.device("cpu") + + def _loads_initial_weights_here(self): + return True + + def _build_training_wrapper(self, draft_model): + wrapper = super()._build_training_wrapper(draft_model) + original = wrapper._sample_anchor_positions + + def sample(*args, **kwargs): + anchors, keep = original(*args, **kwargs) + leader = dist.get_global_rank(self.tp_group, 0) + dist.broadcast(anchors, src=leader, group=self.tp_group) + dist.broadcast(keep, src=leader, group=self.tp_group) + return anchors, keep + + wrapper._sample_anchor_positions = sample + return wrapper + + def logical_tensor(self, value, parameter): + axis = getattr(parameter, "_aspec_tp_axis", None) + if axis is None or value.ndim == 0: + return value + return DTensor.from_local(value, self.tp_mesh, (Shard(axis),), run_check=False) + + def _wrap_training_model(self, model, draft_model): + if (self.args.fsdp_strategy != "REPLICATE" or self.args.draft_context_parallel_size != 1 + or self.args.dflash_packing or self.args.compile_model): + raise ValueError("Torch TP validation requires native REPLICATE, padded CP1 execution") + config = draft_model.config + if any(getattr(config, key) % self.tp_size for key in ( + "num_attention_heads", "num_key_value_heads", "intermediate_size", + )): + raise ValueError("Attention heads and FFN width must divide the Torch TP size") + draft_model.cuda() + for layer in draft_model.layers: + for owner, names, axis in ( + (layer.self_attn, ("q_proj", "k_proj", "v_proj"), 0), + (layer.self_attn, ("o_proj",), 1), + (layer.mlp, ("gate_proj", "up_proj"), 0), + (layer.mlp, ("down_proj",), 1), + ): + for name in names: + setattr(owner, name, TorchTPLinear(getattr(owner, name), self.tp_mesh, axis)) + layer.self_attn.num_heads //= self.tp_size + layer.self_attn.num_kv_heads //= self.tp_size + self._partial_norm_parameters = [norm.weight for layer in draft_model.layers + for norm in (layer.self_attn.q_norm, layer.self_attn.k_norm)] + # Head-shared norm weights receive contributions from local heads only. + # Sum their partial derivatives in FP32, retaining BF16 leaf gradients. + self._partial_buffer = torch.empty(sum(p.numel() for p in self._partial_norm_parameters), + device="cuda", dtype=torch.float32) + self._partial_views = list(self._partial_buffer.split( + [p.numel() for p in self._partial_norm_parameters])) + model = apply_fsdp2(model, mesh=self.dp_mesh, args=self.args) + object.__setattr__(draft_model, "_validation_tp_group", self.tp_group) + object.__setattr__(draft_model, "_validation_logical_tensor", self.logical_tensor) + + def state(): + parameters = dict(model.named_parameters()) + return {name: self.logical_tensor(value, parameters[name]) if name in parameters else value + for name, value in model.state_dict().items() if isinstance(value, torch.Tensor)} + + def load(values): + current = model.state_dict() + if set(current) != set(values): + raise ValueError("The Torch TP checkpoint must cover the full model") + with torch.no_grad(): + for name, value in values.items(): + local = value.to_local() if isinstance(value, DTensor) else value + if current[name].shape != local.shape: + raise ValueError(f"Torch TP checkpoint shape mismatch: {name}") + current[name].copy_(local) + + model._aspec_checkpoint_state, model._aspec_load_checkpoint_state = state, load + return model + + def _normalize_accumulated_gradients(self, local_denominator, accumulation_steps): + flat = [p.grad.view(-1) for p in self._partial_norm_parameters] + torch._foreach_copy_(self._partial_views, flat) + dist.all_reduce(self._partial_buffer, group=self.tp_group) + torch._foreach_copy_(flat, self._partial_views) + return super()._normalize_accumulated_gradients(local_denominator, accumulation_steps) + + def pop_committed_work(self, optimizer_step): + local = super().pop_committed_work(optimizer_step) + participants = [None] * self.tp_size + dist.all_gather_object(participants, local, group=self.tp_group) + if any(work != local for work in participants): + raise RuntimeError("Torch TP participants committed different work") + return local if self.tp_rank == 0 else [] + + def _checkpoint_backend_rng(self): + return {"backend": "torch-tp-validation", "tp_size": self.tp_size} + + def _restore_backend_rng(self, state): + if state != self._checkpoint_backend_rng(): + raise RuntimeError("Torch TP validation resume cannot change backend or TP size") From e77064b1f6f497eaa1852385063f203fb34bfff6 Mon Sep 17 00:00:00 2001 From: zhouaihui <34807769+zhouaihui@users.noreply.github.com> Date: Fri, 20 Nov 2026 10:46:08 +0800 Subject: [PATCH 2/5] fix: coordinate Target publication and TP input preparation Drain published Target leases before shutdown and synchronize TP input readiness before broadcasting. Tighten Target configuration and alignment checks, and publish the online TP2 validation driver and per-update objectives. --- aspec/cli.py | 10 +- aspec/inference/engine/megatron_engine.py | 59 +++-- aspec/inference/factory.py | 6 +- aspec/inference/service.py | 12 +- aspec/inference/service_client.py | 9 +- aspec/inference/service_protocol.py | 16 ++ aspec/inference/target_runtime.py | 2 + aspec/models/megatron.py | 10 + aspec/runtime_identity.py | 1 + aspec/training/data_fetcher.py | 55 +++-- aspec/utils/env.py | 2 +- docs/benchmarks/megatron-online-tp2.csv | 201 ++++++++++++++++++ docs/user-guide/megatron.md | 124 ++++++++--- tests/cpu/test_megatron.py | 113 ++++++++++ tests/cpu/test_target_environment.py | 20 ++ tests/cpu/test_target_service.py | 9 + tests/cpu/test_torch_tp_validation.py | 2 + tools/validation/megatron_online_alignment.py | 153 +++++++++++++ .../megatron_target_aspec_alignment.py | 5 +- tools/validation/megatron_torch_alignment.py | 51 ++++- tools/validation/torch_tp_dspark.py | 2 + 21 files changed, 772 insertions(+), 90 deletions(-) create mode 100644 docs/benchmarks/megatron-online-tp2.csv create mode 100644 tools/validation/megatron_online_alignment.py diff --git a/aspec/cli.py b/aspec/cli.py index 49b923a..6199379 100644 --- a/aspec/cli.py +++ b/aspec/cli.py @@ -48,11 +48,11 @@ def main(): else: module = "aspec.train_entry" help_requested = arguments in (["--help"], ["-h"]) - megatron_target = ("--backend=megatron" in arguments or any( - a == "--backend" and b == "megatron" for a, b in zip(arguments, arguments[1:]) - )) - if module == "aspec.inference.service" and not help_requested and not is_target_environment() and not megatron_target: - raise SystemExit("Start aspec target in the Target environment (pip install ling-aspec-target)") + if module == "aspec.inference.service" and not help_requested and not is_target_environment(): + from aspec.inference.service_protocol import target_service_parser + + if target_service_parser().parse_args(arguments).backend == "sglang": + raise SystemExit("Start aspec target in the Target environment (pip install ling-aspec-target)") try: execute([sys.executable, "-m", module, *arguments]) except (ValueError, OSError) as error: diff --git a/aspec/inference/engine/megatron_engine.py b/aspec/inference/engine/megatron_engine.py index 8366afb..8896e61 100644 --- a/aspec/inference/engine/megatron_engine.py +++ b/aspec/inference/engine/megatron_engine.py @@ -16,6 +16,7 @@ import threading import traceback import uuid +from contextlib import ExitStack, contextmanager from datetime import timedelta import torch @@ -26,6 +27,32 @@ from aspec.utils.misc import get_current_node_ip +@contextmanager +def _prefill_batch(synchronize): + """Hold committed source slots until every TP worker finishes the batch.""" + sinks = [] + with ExitStack() as cleanup: + def stage(sink): + cleanup.callback(sink.abort) + sinks.append(sink) + + yield stage + for sink in sinks: + sink.validate_publishable() + if synchronize is not None: + synchronize() + for sink in sinks: + sink.publish() + cleanup.pop_all() + + +def _close_source(store): + timeout = max(float(store.config.get_retry_max_wait_seconds), 5.0) + if not store.wait_for_p2p_source_releases(timeout=timeout): + raise TimeoutError("Megatron Target source leases still have active readers") + store.close() + + def _target_worker(connection, local_rank, gpu_ids, args, mooncake_config, port, engine_rank): import torch.distributed as dist @@ -45,6 +72,9 @@ def _target_worker(connection, local_rank, gpu_ids, args, mooncake_config, port, args.target_model_path, groups, implementation=getattr(args, "megatron_implementation", "transformer_engine"), ) + planned_context = args.aspec_context_contract["backend_context_length"] + if max(planned_context, args.max_seq_length) > model.max_sequence_length: + raise ValueError("Planned Target context exceeds the loaded model's maximum sequence length") captures = list(args.aux_hidden_states_layers) runtime = FeatureContractRuntime( args.aspec_feature_contract, role="producer", participant=f"megatron_engine_{engine_rank}", @@ -68,7 +98,7 @@ def _target_worker(connection, local_rank, gpu_ids, args, mooncake_config, port, "tensor_parallel_size": len(gpu_ids), "local_rank": local_rank, "device_capability": list(torch.cuda.get_device_capability()), "model_implementation": getattr(args, "megatron_implementation", "transformer_engine"), - "model_class": type(model).__name__, + "model_class": type(model).__name__, "max_sequence_length": model.max_sequence_length, "hybrid_layer_pattern": model.hybrid_layer_pattern, "target_frozen": not any(p.requires_grad for p in model.parameters()), } @@ -77,7 +107,7 @@ def _target_worker(connection, local_rank, gpu_ids, args, mooncake_config, port, method, payload = connection.recv() if method == "shutdown": if store is not None: - store.close() + _close_source(store) dist.destroy_process_group() connection.send({"ok": True, "value": None}) return @@ -90,20 +120,21 @@ def _target_worker(connection, local_rank, gpu_ids, args, mooncake_config, port, if method != "generate": raise ValueError(f"Unknown Megatron Target operation: {method}") outputs = [] - for sample in payload: - ids = torch.tensor(sample["tokens"], dtype=torch.long, device="cuda") - length = ids.numel() - if not 0 < length <= args.max_seq_length: - raise ValueError("Target input length is outside the planned context") - request = TargetPrefillRequest(sample["request_id"], sample["data_id"], length) - sink = None - try: + with _prefill_batch(dist.barrier if len(gpu_ids) > 1 else None) as stage: + for sample in payload: + ids = torch.tensor(sample["tokens"], dtype=torch.long, device="cuda") + length = ids.numel() + if not 0 < length <= args.max_seq_length: + raise ValueError("Target input length is outside the planned context") + request = TargetPrefillRequest(sample["request_id"], sample["data_id"], length) + sink = None if local_rank == 0: sink = TargetPrefillDirectSink( store=store, requests=[request], capture_layers=captures, hidden_size=config.hidden_size, store_last_hidden_states=args.store_last_hidden_states, source_engine=f"megatron_{engine_rank}", context_parallel_size=1, ) + stage(sink) sink.begin(input_ids=ids, request_ids=[request.request_id], sequence_lengths=[length], prefix_lengths=[0]) final = model.prefill(ids.unsqueeze(0), captures, sink.capture if sink is not None else None) @@ -133,14 +164,8 @@ def _target_worker(connection, local_rank, gpu_ids, args, mooncake_config, port, if is_direct_cp_backend(args.attention_backend): metadata.update(direct_cp_sharded=True, direct_cp_size=1) runtime.validate_metadata(tensor_shapes=shapes, tensor_dtypes=dtypes, metadata=metadata) - sink.validate_publishable() - sink.publish() outputs.append({"mooncake_key": key, "tensor_shapes": shapes, "tensor_dtypes": dtypes, "metadata": metadata}) - except BaseException: - if sink is not None: - sink.abort() - raise connection.send({"ok": True, "value": outputs if local_rank == 0 else None}) except BaseException as error: try: @@ -235,6 +260,8 @@ async def generate(self, data_id, input_ids_ref=None, packed_loss_mask_list=None tokens = tokenizer(formatted_prompts, add_special_tokens=False)["input_ids"] else: tokens = [value.reshape(-1).tolist() for value in input_ids_ref] + if len(tokens) > self._mooncake_config.ring_slots_for_role("source"): + raise ValueError("Megatron Target batch exceeds the number of source ring slots") ids = ([f"{data_id}_{index}" for index in range(len(tokens))] if isinstance(data_id, str) else list(data_id)) if len(ids) != len(tokens): diff --git a/aspec/inference/factory.py b/aspec/inference/factory.py index 9305f81..339e61e 100644 --- a/aspec/inference/factory.py +++ b/aspec/inference/factory.py @@ -311,9 +311,11 @@ def rollback_unused_targets(engines): def _init_sgl_engines(args, pg, mooncake_config=None, engine_group: int = 0) -> list: """Initialize SGLang engines with Ray placement groups (blocking).""" engines, init_handles = _prepare_sgl_engines(args, pg, mooncake_config, engine_group) - init_timeout = getattr(args, "sglang_init_timeout", 300) + from aspec.inference.service_protocol import target_init_timeout + + init_timeout = target_init_timeout(args, sglang_default=300) try: - _wait_for_init(init_handles, "Sgl", timeout=init_timeout) + _wait_for_init(init_handles, getattr(args, "target_model_backend", "sglang"), timeout=init_timeout) except BaseException: rollback_unused_targets(engines) raise diff --git a/aspec/inference/service.py b/aspec/inference/service.py index 5b6b696..2ced888 100644 --- a/aspec/inference/service.py +++ b/aspec/inference/service.py @@ -6,7 +6,6 @@ cross-environment Ray calls; ASpec's GPU leases, feature contracts and tensor transport remain in use. No SpecForge server code is copied. """ -import argparse import json import logging import os @@ -17,7 +16,6 @@ import uuid from argparse import Namespace from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path from urllib.parse import urlsplit import ray @@ -29,6 +27,7 @@ TARGET_ARGUMENTS, decode, encode, + target_service_parser, validate_address, validate_target_tp, ) @@ -105,6 +104,10 @@ def create(self, value): count = value["num_gpus_per_engine"] if type(count) is not int or count <= 0: raise ValueError("Invalid Target GPU allocation") + if getattr(args, "target_model_backend", "sglang") == "megatron": + tp = getattr(args, "megatron_tp_size", 1) + if type(tp) is not int or tp != count: + raise ValueError("Megatron Target GPU allocation must equal its tensor parallel size") key = value["key"] with self.lock: if self.draining: @@ -437,10 +440,7 @@ def main(): from omegaconf import OmegaConf logging.basicConfig(level=logging.INFO) - parser = argparse.ArgumentParser(description="Run Target in this environment; Draft connects by address.") - parser.add_argument("--config", type=Path) - parser.add_argument("--backend", choices=("sglang", "megatron"), default="sglang") - parser.add_argument("--address", help="Listening http://host:port; defaults to the config or localhost:30000") + parser = target_service_parser() args = parser.parse_args() config = OmegaConf.load(args.config) if args.config else OmegaConf.create({}) address = validate_address(args.address or OmegaConf.select( diff --git a/aspec/inference/service_client.py b/aspec/inference/service_client.py index 13367f7..7475be2 100644 --- a/aspec/inference/service_client.py +++ b/aspec/inference/service_client.py @@ -7,7 +7,12 @@ import ray -from aspec.inference.service_protocol import PROTOCOL, engine_arguments, request +from aspec.inference.service_protocol import ( + PROTOCOL, + engine_arguments, + request, + target_init_timeout, +) from aspec.inference.target_runtime import release_version @@ -16,7 +21,7 @@ def __init__(self, *, address, args, rank, num_gpus_per_engine, node_ip=None, node_rank=0, engine_group=0): self.address = address self.key = uuid.uuid4().hex - self.timeout = float(getattr(args, "sglang_init_timeout", 1200)) + 60 + self.timeout = float(target_init_timeout(args, sglang_default=1200)) + 60 self._closed = threading.Event() info = request(address, "/v1/info", timeout=30) if info["version"] != release_version(): diff --git a/aspec/inference/service_protocol.py b/aspec/inference/service_protocol.py index c66f6e0..febb53d 100644 --- a/aspec/inference/service_protocol.py +++ b/aspec/inference/service_protocol.py @@ -5,8 +5,10 @@ Inspired by SpecForge f8cc3dc's external capture adapter. ASpec retains its SglEngine, feature contracts and transport rather than copying that adapter. """ +import argparse import json from dataclasses import asdict, is_dataclass +from pathlib import Path from urllib.error import HTTPError from urllib.parse import urlsplit from urllib.request import ProxyHandler, Request, build_opener @@ -40,6 +42,20 @@ """.split()) +def target_service_parser(): + parser = argparse.ArgumentParser(description="Run Target in this environment; Draft connects by address.") + parser.add_argument("--config", type=Path) + parser.add_argument("--backend", choices=("sglang", "megatron"), default="sglang") + parser.add_argument("--address", help="Listening http://host:port; defaults to the config or localhost:30000") + return parser + + +def target_init_timeout(args, *, sglang_default): + if getattr(args, "target_model_backend", "sglang") == "megatron": + return getattr(args, "megatron_init_timeout", 600) + return getattr(args, "sglang_init_timeout", sglang_default) + + def validate_target_tp(nnodes, node_rank=0): if nnodes != 1 or node_rank != 0: raise ValueError("Target TP groups must be node-local (inference.sglang.nnodes=1, node_rank=0)") diff --git a/aspec/inference/target_runtime.py b/aspec/inference/target_runtime.py index 35851de..7fafd95 100644 --- a/aspec/inference/target_runtime.py +++ b/aspec/inference/target_runtime.py @@ -42,6 +42,8 @@ def inventory(backend="sglang"): from aspec.runtime_identity import MEGATRON_RUNTIME_RELATIVE_PATHS, module_runtime_identity require_megatron() + # Load the same native kernel symbols the Target worker needs before advertising readiness. + from aspec.models.megatron_target import Qwen3Target # noqa: F401 return {**code_identity(), "backend": backend, "python": sys.executable, "prefix": sys.prefix, "megatron": module_runtime_identity("megatron.core", relative_paths=MEGATRON_RUNTIME_RELATIVE_PATHS), "dependencies": dependency_versions("torch", "ray", "megatron-core", "transformer-engine", diff --git a/aspec/models/megatron.py b/aspec/models/megatron.py index 8b3327f..6211db7 100644 --- a/aspec/models/megatron.py +++ b/aspec/models/megatron.py @@ -65,6 +65,16 @@ def transformer_config(config, tp_size: int): raise ValueError("Qwen3 heads, KV groups, FFN and vocabulary must divide Target TP") if config.model_type != "qwen3" or getattr(config, "num_experts", None): raise ValueError("Megatron Target currently requires dense Qwen3") + layer_types = getattr(config, "layer_types", None) + if layer_types is not None: + full_attention = all(kind == "full_attention" for kind in layer_types) + else: + full_attention = not ( + getattr(config, "use_sliding_window", False) and getattr(config, "sliding_window", None) + and getattr(config, "max_window_layers", 0) < config.num_hidden_layers + ) + if not full_attention: + raise ValueError("Megatron Qwen3 Target requires full attention in every layer") if getattr(config, "attention_bias", False): raise ValueError("Megatron Qwen3 prefill requires bias-free attention projections") if getattr(config, "quantization_config", None) or getattr(config, "hidden_act", "silu") != "silu": diff --git a/aspec/runtime_identity.py b/aspec/runtime_identity.py index d9014c4..af8b6fc 100644 --- a/aspec/runtime_identity.py +++ b/aspec/runtime_identity.py @@ -26,6 +26,7 @@ # A bounded set catches divergent checkouts without hashing the entire repo. MEGATRON_RUNTIME_RELATIVE_PATHS = ( "package_info.py", + "models/common/embeddings/rotary_pos_embedding.py", "models/hybrid/hybrid_model.py", "models/hybrid/hybrid_block.py", "models/hybrid/hybrid_layer_allocation.py", diff --git a/aspec/training/data_fetcher.py b/aspec/training/data_fetcher.py index 2f535bf..5516d47 100644 --- a/aspec/training/data_fetcher.py +++ b/aspec/training/data_fetcher.py @@ -145,6 +145,8 @@ def __init__( self.ray_queue = ray_queue if (tensor_parallel_group is None) != (tensor_parallel_control_group is None): raise ValueError("TP input requires paired GPU-data and CPU-control groups") + if tensor_parallel_group is not None and (usp_enabled or direct_cp_enabled): + raise ValueError("TP input cannot be combined with USP or direct CP") self.tensor_parallel_group = tensor_parallel_group self.tensor_parallel_control_group = tensor_parallel_control_group self._tensor_parallel_device = (torch.cuda.current_device() @@ -660,24 +662,45 @@ def _tensor_parallel_item_on_device(self): item = message["item"] if item is None: return None, None - if not is_leader: - data = dict(message["values"]) - self._feature_contract_runtime.validate_metadata( - tensor_shapes=item.tensor_shapes, tensor_dtypes=item.tensor_dtypes, - metadata=item.metadata, - ) - for key, shape, dtype, original_device in message["tensors"]: - buffer = (data[key].to(device="cuda").contiguous() if is_leader - else torch.empty(shape, device="cuda", dtype=dtype)) - dist.broadcast(buffer, src=leader, group=group) + buffers = {} + error = None + try: if not is_leader: - data[key] = buffer.cpu() if original_device == "cpu" else buffer + data = dict(message["values"]) + if self._feature_contract_runtime is not None: + self._feature_contract_runtime.validate_metadata( + tensor_shapes=item.tensor_shapes, tensor_dtypes=item.tensor_dtypes, + metadata=item.metadata, + ) + for key, shape, dtype, original_device in message["tensors"]: + if is_leader: + buffers[key] = data[key].to(device="cuda").contiguous() + else: + buffers[key] = torch.empty(shape, device="cuda", dtype=dtype) + data[key] = (torch.empty(shape, dtype=dtype, device="cpu") + if original_device == "cpu" else buffers[key]) + if not is_leader and self._feature_contract_runtime is not None: + names = self._feature_contract_runtime.contract["tensors"] + self._validate_received_feature_tensors( + {name: data[name] for name in names}, expected_shapes=item.tensor_shapes, + expected_dtypes=item.tensor_dtypes, + ) + except Exception as exc: + error = f"TP input rank {dist.get_rank(group)}: {type(exc).__name__}: {exc}" + # One small CPU collective on the normal path. Exchange diagnostics + # only after all participants know they must skip the NCCL payload. + failed = torch.tensor(int(error is not None), dtype=torch.int32, device="cpu") + dist.all_reduce(failed, group=self.tensor_parallel_control_group) + if failed.item(): + errors = [None] * dist.get_world_size(group) + dist.all_gather_object(errors, error, group=self.tensor_parallel_control_group) + raise RuntimeError("TP input preparation failed: " + "; ".join(filter(None, errors))) + for buffer in buffers.values(): + dist.broadcast(buffer, src=leader, group=group) if not is_leader: - names = self._feature_contract_runtime.contract["tensors"] - self._validate_received_feature_tensors( - {name: data[name] for name in names}, expected_shapes=item.tensor_shapes, - expected_dtypes={name: data[name].dtype for name in names}, - ) + for key, buffer in buffers.items(): + if data[key].device.type == "cpu": + data[key].copy_(buffer) if leases: data["_mooncake_p2p_receive_leases"] = leases leases = [] diff --git a/aspec/utils/env.py b/aspec/utils/env.py index 1d37846..6c85143 100644 --- a/aspec/utils/env.py +++ b/aspec/utils/env.py @@ -91,7 +91,7 @@ def get_aspec_env_vars(args=None) -> dict[str, str]: if getattr(args, "cluster_mode", "single_node") == "static_multi_node": # Ray is started from each node's own venv/environment. A driver's # library/cache paths must not overwrite another node's local layout. - for key in ("LD_LIBRARY_PATH", "PYTHONPATH", "HF_HOME", "ASPEC_CUDA_COMPAT_DIR", "CUTE_DSL_CACHE_DIR", + for key in ("CUDA_HOME", "CUDNN_HOME", "LD_LIBRARY_PATH", "PYTHONPATH", "HF_HOME", "ASPEC_CUDA_COMPAT_DIR", "CUTE_DSL_CACHE_DIR", "FLASH_ATTENTION_CUTE_DSL_CACHE_DIR", "ASPEC_LOG_DIR"): env.pop(key, None) return env diff --git a/docs/benchmarks/megatron-online-tp2.csv b/docs/benchmarks/megatron-online-tp2.csv new file mode 100644 index 0000000..3c07e7b --- /dev/null +++ b/docs/benchmarks/megatron-online-tp2.csv @@ -0,0 +1,201 @@ +draft_dp,step,sglang_torch_objective,megatron_objective,abs_rel_diff +1,1,3.755715511504049,3.755715511504049,0.0 +1,2,3.9489112779566504,3.9489112779566504,0.0 +1,3,6.394355260938833,6.394355260938833,0.0 +1,4,3.535158431425255,3.535158431425255,0.0 +1,5,4.663684525596108,4.663684525596108,0.0 +1,6,3.749276704044223,3.749276704044223,0.0 +1,7,3.5474174187751575,3.5474174187751575,0.0 +1,8,3.0037692179428155,3.0037692179428155,0.0 +1,9,2.9286285638446965,2.9286285638446965,0.0 +1,10,2.8512743744382094,2.8512743744382094,0.0 +1,11,2.8926897694750062,2.8926897694750062,0.0 +1,12,3.02247194831582,3.02247194831582,0.0 +1,13,2.8976240265700253,2.8976240265700253,0.0 +1,14,2.877538289851966,2.877538289851966,0.0 +1,15,2.8095168082610757,2.8095168082610757,0.0 +1,16,3.129611097838802,3.129611097838802,0.0 +1,17,3.1787790933339597,3.1787790933339597,0.0 +1,18,3.4997262969871734,3.4997262969871734,0.0 +1,19,3.7776471957951863,3.7776471957951863,0.0 +1,20,3.840914629934873,3.840914629934873,0.0 +1,21,3.4023869609110227,3.4023869609110227,0.0 +1,22,3.2538292889779212,3.2538292889779212,0.0 +1,23,2.8009793685047,2.8009793685047,0.0 +1,24,2.770695361125896,2.770695361125896,0.0 +1,25,2.751982983455716,2.751982983455716,0.0 +1,26,2.640023846530543,2.640023846530543,0.0 +1,27,2.629589393391599,2.629589393391599,0.0 +1,28,2.689536324175185,2.689536324175185,0.0 +1,29,2.637508801597702,2.637508801597702,0.0 +1,30,2.6985999692561586,2.6985999692561586,0.0 +1,31,2.8182637159534716,2.8182637159534716,0.0 +1,32,3.046031029642253,3.046031029642253,0.0 +1,33,2.908732976026226,2.908732976026226,0.0 +1,34,2.704845400480617,2.704845400480617,0.0 +1,35,2.7998925998870843,2.7998925998870843,0.0 +1,36,2.7895445065984465,2.7895445065984465,0.0 +1,37,2.7051980497705594,2.7051980497705594,0.0 +1,38,2.7344043637331596,2.7344043637331596,0.0 +1,39,2.509439197941317,2.509439197941317,0.0 +1,40,2.610893462817923,2.610893462817923,0.0 +1,41,2.7377426596989554,2.7377426596989554,0.0 +1,42,2.637927188461201,2.637927188461201,0.0 +1,43,2.63986896828625,2.63986896828625,0.0 +1,44,2.5871017643620826,2.5871017643620826,0.0 +1,45,2.614345435366796,2.614345435366796,0.0 +1,46,2.728988705976587,2.728988705976587,0.0 +1,47,2.9411069995793713,2.9411069995793713,0.0 +1,48,2.744398289571361,2.744398289571361,0.0 +1,49,2.7640358536119223,2.7640358536119223,0.0 +1,50,2.550890729314269,2.550890729314269,0.0 +1,51,2.758689432357313,2.758689432357313,0.0 +1,52,2.6676574237710717,2.6676574237710717,0.0 +1,53,2.7532351885667175,2.7532351885667175,0.0 +1,54,2.6753928620036223,2.6753928620036223,0.0 +1,55,2.4969498746958547,2.4969498746958547,0.0 +1,56,2.6121784501779954,2.6121784501779954,0.0 +1,57,2.516519891283509,2.516519891283509,0.0 +1,58,2.499191305850258,2.499191305850258,0.0 +1,59,2.6497694971882195,2.6497694971882195,0.0 +1,60,2.7132600963527285,2.7132600963527285,0.0 +1,61,2.494319326659656,2.494319326659656,0.0 +1,62,2.4896776952369706,2.4896776952369706,0.0 +1,63,2.479453488774173,2.479453488774173,0.0 +1,64,2.6111824505102126,2.6111824505102126,0.0 +1,65,2.768552953618063,2.768552953618063,0.0 +1,66,2.9187781203037084,2.9187781203037084,0.0 +1,67,2.7825301413240213,2.7825301413240213,0.0 +1,68,2.5703179640860765,2.5703179640860765,0.0 +1,69,2.6785484744511066,2.6785484744511066,0.0 +1,70,2.661630147819428,2.661630147819428,0.0 +1,71,2.614331327050197,2.614331327050197,0.0 +1,72,2.5786759450558794,2.5786759450558794,0.0 +1,73,2.4932433535481113,2.4932433535481113,0.0 +1,74,2.473171408172292,2.473171408172292,0.0 +1,75,2.4908180082554847,2.4908180082554847,0.0 +1,76,2.5538707407834096,2.5538707407834096,0.0 +1,77,2.5955664232245903,2.5955664232245903,0.0 +1,78,2.560071540041583,2.560071540041583,0.0 +1,79,2.470817293538922,2.470817293538922,0.0 +1,80,2.4769347611925236,2.4769347611925236,0.0 +1,81,2.5118219406956674,2.5118219406956674,0.0 +1,82,2.820946677014538,2.820946677014538,0.0 +1,83,2.7461472202139428,2.7461472202139428,0.0 +1,84,2.880561826497822,2.880561826497822,0.0 +1,85,2.846046559966212,2.846046559966212,0.0 +1,86,2.778524320199631,2.778524320199631,0.0 +1,87,2.4985565028420567,2.4985565028420567,0.0 +1,88,2.8256218199105594,2.8256218199105594,0.0 +1,89,2.4703508998171473,2.4703508998171473,0.0 +1,90,2.4885303900059057,2.4885303900059057,0.0 +1,91,2.5112198135012354,2.5112198135012354,0.0 +1,92,2.469125517465879,2.469125517465879,0.0 +1,93,2.495883009904303,2.495883009904303,0.0 +1,94,2.66443476881672,2.66443476881672,0.0 +1,95,2.759431852677762,2.759431852677762,0.0 +1,96,2.600510991457831,2.600510991457831,0.0 +1,97,2.7924760935162354,2.7924760935162354,0.0 +1,98,2.5236881607009267,2.5236881607009267,0.0 +1,99,2.4488236482157943,2.4488236482157943,0.0 +1,100,2.5124660479242937,2.5124660479242937,0.0 +2,1,3.7533354075036023,3.7533354075036023,0.0 +2,2,3.9454312436274273,3.9454312436274273,0.0 +2,3,6.3899630176929305,6.3899630176929305,0.0 +2,4,3.518234228698406,3.518234228698406,0.0 +2,5,4.737220968457079,4.737220968457079,0.0 +2,6,3.7578405075717747,3.7578405075717747,0.0 +2,7,3.4900298794508915,3.4900298794508915,0.0 +2,8,2.995217554173092,2.995217554173092,0.0 +2,9,2.944386699142634,2.944386699142634,0.0 +2,10,2.884274754370698,2.884274754370698,0.0 +2,11,2.8726153555099887,2.8726153555099887,0.0 +2,12,3.0069363112598615,3.0069363112598615,0.0 +2,13,2.8884170757660343,2.8884170757660343,0.0 +2,14,2.8516478334094453,2.8516478334094453,0.0 +2,15,2.7772170183813802,2.7772170183813802,0.0 +2,16,3.2027737129662994,3.2027737129662994,0.0 +2,17,3.3568134943755563,3.3568134943755563,0.0 +2,18,3.768032100869773,3.768032100869773,0.0 +2,19,4.059996545319195,4.059996545319195,0.0 +2,20,4.124869476005608,4.124869476005608,0.0 +2,21,3.6403117303698513,3.6403117303698513,0.0 +2,22,3.434617180902799,3.434617180902799,0.0 +2,23,2.855240683755184,2.855240683755184,0.0 +2,24,2.8041677300526975,2.8041677300526975,0.0 +2,25,2.7672158933785447,2.7672158933785447,0.0 +2,26,2.633489174725221,2.633489174725221,0.0 +2,27,2.621632506061803,2.621632506061803,0.0 +2,28,2.671955199488743,2.671955199488743,0.0 +2,29,2.627090470254716,2.627090470254716,0.0 +2,30,2.695290064797739,2.695290064797739,0.0 +2,31,2.812074229285662,2.812074229285662,0.0 +2,32,3.0645571448778237,3.0645571448778237,0.0 +2,33,2.9025393768495302,2.9025393768495302,0.0 +2,34,2.6930148422330515,2.6930148422330515,0.0 +2,35,2.776083532059291,2.776083532059291,0.0 +2,36,2.766260312284372,2.766260312284372,0.0 +2,37,2.6710309753019748,2.6710309753019748,0.0 +2,38,2.725291217641412,2.725291217641412,0.0 +2,39,2.5073755701430778,2.5073755701430778,0.0 +2,40,2.624145321730768,2.624145321730768,0.0 +2,41,2.744018338926749,2.744018338926749,0.0 +2,42,2.616422026076212,2.616422026076212,0.0 +2,43,2.599268784243094,2.599268784243094,0.0 +2,44,2.5167422698167052,2.5167422698167052,0.0 +2,45,2.577373907224785,2.577373907224785,0.0 +2,46,2.7309660324237526,2.7309660324237526,0.0 +2,47,2.958283445945888,2.958283445945888,0.0 +2,48,2.7666011042852934,2.7666011042852934,0.0 +2,49,2.7997332672224813,2.7997332672224813,0.0 +2,50,2.617134169921016,2.617134169921016,0.0 +2,51,2.7524866888931214,2.7524866888931214,0.0 +2,52,2.6531086153224672,2.6531086153224672,0.0 +2,53,2.7223538550282327,2.7223538550282327,0.0 +2,54,2.6714445544589847,2.6714445544589847,0.0 +2,55,2.5029756245839985,2.5029756245839985,0.0 +2,56,2.602785934398324,2.602785934398324,0.0 +2,57,2.516950585214147,2.516950585214147,0.0 +2,58,2.500728115283564,2.500728115283564,0.0 +2,59,2.647509887712718,2.647509887712718,0.0 +2,60,2.7093939085000547,2.7093939085000547,0.0 +2,61,2.4776345465020504,2.4776345465020504,0.0 +2,62,2.4770722615816334,2.4770722615816334,0.0 +2,63,2.455684748280687,2.455684748280687,0.0 +2,64,2.612189807386492,2.612189807386492,0.0 +2,65,2.7507026587273318,2.7507026587273318,0.0 +2,66,2.920574731046472,2.920574731046472,0.0 +2,67,2.784464239158551,2.784464239158551,0.0 +2,68,2.5539501715567425,2.5539501715567425,0.0 +2,69,2.6489925047182252,2.6489925047182252,0.0 +2,70,2.6410449985023736,2.6410449985023736,0.0 +2,71,2.6067708034095323,2.6067708034095323,0.0 +2,72,2.571881981422283,2.571881981422283,0.0 +2,73,2.484595016317446,2.484595016317446,0.0 +2,74,2.4725457930860912,2.4725457930860912,0.0 +2,75,2.4900859412273406,2.4900859412273406,0.0 +2,76,2.5541016549921136,2.5541016549921136,0.0 +2,77,2.5962512324664107,2.5962512324664107,0.0 +2,78,2.5669269495602283,2.5669269495602283,0.0 +2,79,2.4750354707258606,2.4750354707258606,0.0 +2,80,2.4764291164483234,2.4764291164483234,0.0 +2,81,2.5094883285579987,2.5094883285579987,0.0 +2,82,2.823899941123214,2.823899941123214,0.0 +2,83,2.750830955420957,2.750830955420957,0.0 +2,84,2.872251495655924,2.872251495655924,0.0 +2,85,2.851715903421645,2.851715903421645,0.0 +2,86,2.7827560208279825,2.7827560208279825,0.0 +2,87,2.5000441771327937,2.5000441771327937,0.0 +2,88,2.8183667602897438,2.8183667602897438,0.0 +2,89,2.4709944357601596,2.4709944357601596,0.0 +2,90,2.488414342435299,2.488414342435299,0.0 +2,91,2.5110512336021906,2.5110512336021906,0.0 +2,92,2.4686226641898417,2.4686226641898417,0.0 +2,93,2.497109374945498,2.497109374945498,0.0 +2,94,2.666725258160556,2.666725258160556,0.0 +2,95,2.754750948399796,2.754750948399796,0.0 +2,96,2.6006765922746284,2.6006765922746284,0.0 +2,97,2.7878805480392193,2.7878805480392193,0.0 +2,98,2.5005059924586943,2.5005059924586943,0.0 +2,99,2.448249555069334,2.448249555069334,0.0 +2,100,2.511920188263162,2.511920188263162,0.0 diff --git a/docs/user-guide/megatron.md b/docs/user-guide/megatron.md index 7a57ba8..9954d1f 100644 --- a/docs/user-guide/megatron.md +++ b/docs/user-guide/megatron.md @@ -159,43 +159,69 @@ verification tool uses a native PyTorch DTensor adapter around the existing DSpark trainer and optimizer. This adapter is available only in the validation tool. The public Torch training entry point supports TP1. -A Torch TP2/DP1 reference run is: +Run the following Bash commands from the checkout. Supply the role venvs, model, +SGLang feature cache and an empty output root. The same backend that wrote a +checkpoint also supplies its uninterrupted replay reference. ```bash -CUDA_VISIBLE_DEVICES=0,1 python -m aspec._environment \ - /absolute/path/to/torch-venv/bin/torchrun --standalone --nproc_per_node=2 \ - /absolute/path/to/tools/validation/megatron_torch_alignment.py \ - --attention-backward-deterministic \ - --backend torch-tp --phase checkpoint-control --tp 2 \ - --steps 100 --split-step 50 \ - --tensor-steps 1,2,5,10,20,30,40,50,60,70,80,90,100 \ - --state-steps 1,10,25,50,75,100 \ - --source /absolute/path/to/ASpec \ - --model /absolute/path/to/Qwen3-8B \ - --teacher /absolute/path/to/aspec-eval-cache.pt \ - --initial /absolute/path/to/shared-initial/model.safetensors \ - --output /tmp/torch-tp2-dp1-reference +src=$PWD +torch_env=/absolute/path/to/torch-draft-venv +meg_env=/absolute/path/to/megatron-draft-venv +model=/absolute/path/to/Qwen3-8B +teacher=/absolute/path/to/aspec-eval-cache.pt +out=/absolute/path/to/fresh-fixed-feature-runs +export CUDA_DEVICE_MAX_CONNECTIONS=1 +unset PYTHONPATH PYTHONHOME + +fixed_run() { + local env_dir=$1 devices=$2 workers=$3 + shift 3 + VIRTUAL_ENV="$env_dir" PATH="$env_dir/bin:$PATH" CUDA_VISIBLE_DEVICES="$devices" \ + "$env_dir/bin/python" -m aspec._environment \ + "$env_dir/bin/torchrun" --standalone --nproc_per_node="$workers" \ + "$src/tools/validation/megatron_torch_alignment.py" \ + --source "$src" --model "$model" --teacher "$teacher" "$@" +} +fixed_run "$torch_env" 0 1 --backend fsdp --tp 1 --phase prepare --output "$out/initial" + +for dp in 1 2; do + devices=0,1 + milestones=1,2,5,10,20,30,40,50,60,70,80,90,100 + if [[ $dp == 2 ]]; then + devices=0,1,2,3 + milestones=1,2,5,10,11,20,30,40,50,60,70,80,90,100 + fi + workers=$((2 * dp)) + common=(--tp 2 --steps 100 --split-step 50 --attention-backward-deterministic + --tensor-steps "$milestones" --state-steps 1,10,25,50,75,100 + --initial "$out/initial/model.safetensors") + torch_ref="$out/torch-tp2-dp$dp" + meg_ref="$out/megatron-tp2-dp$dp" + fixed_run "$torch_env" "$devices" "$workers" --backend torch-tp "${common[@]}" \ + --phase checkpoint-control --output "$torch_ref" + fixed_run "$meg_env" "$devices" "$workers" --backend megatron --megatron-grad-dtype bf16 \ + "${common[@]}" --phase checkpoint-control --reference "$torch_ref" --output "$meg_ref" + fixed_run "$torch_env" "$devices" "$workers" --backend torch-tp "${common[@]}" \ + --phase resume --resume "$torch_ref/checkpoints/iter_0000051" --reference "$torch_ref" \ + --no-save-reference-tensors --output "$torch_ref-resumed" + fixed_run "$meg_env" "$devices" "$workers" --backend megatron --megatron-grad-dtype bf16 \ + "${common[@]}" --phase resume --resume "$meg_ref/checkpoints/iter_0000051" --reference "$meg_ref" \ + --no-save-reference-tensors --output "$meg_ref-resumed" +done ``` -For the Megatron run, use its venv and checkout, replace `--backend torch-tp` -with `--backend megatron --megatron-grad-dtype bf16`, and add -`--reference /tmp/torch-tp2-dp1-reference` after the reference completes. Use four -workers and TP2 for DP2 in both roles; add update 11 to both tensor milestone lists. -The deterministic option verifies the cuDNN-FE attention backward policy and -writes its call counts on every rank. - -`checkpoint-control` saves after update 50 and continues to 100. In fresh -processes, use `--phase resume --resume .../checkpoints/iter_0000051` to restore -the saved state. Compare against the uninterrupted run with `--reference` and -`--no-save-reference-tensors`. The tool records loss components, gradient norm -and learning rate each step, plus logical gradients and optimizer state at the -requested milestones. +The control saves after update 50 and continues to 100. The fresh processes +compare the remaining updates with their respective controls. Each run records +the backward policy on every rank, the loss components and gradient norm at every +update, and logical gradients and optimizer state at the requested milestones. For Target comparison, run `tools/validation/megatron_target_aspec_alignment.py` with `torchrun` in the Megatron environment: ```bash -python -m aspec._environment /absolute/path/to/megatron-venv/bin/torchrun \ +target_env=/absolute/path/to/megatron-target-venv +VIRTUAL_ENV="$target_env" PATH="$target_env/bin:$PATH" CUDA_VISIBLE_DEVICES=0,1 \ + "$target_env/bin/python" -m aspec._environment "$target_env/bin/torchrun" \ --standalone --nproc_per_node=2 \ /absolute/path/to/tools/validation/megatron_target_aspec_alignment.py \ --model /absolute/path/to/Qwen3-8B \ @@ -211,8 +237,8 @@ at the same TP size as the Target for the matched comparison. To exercise online training and recovery with pretokenized Parquet inputs: ```bash -CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 python -m aspec._environment \ - /absolute/path/to/megatron-draft-venv/bin/python \ +VIRTUAL_ENV="$meg_env" PATH="$meg_env/bin:$PATH" CUDA_VISIBLE_DEVICES=0,1,2,3,4,5 \ + "$meg_env/bin/python" -m aspec._environment "$meg_env/bin/python" \ -m tools.validation.megatron_pipeline_acceptance \ --model /absolute/path/to/Qwen3-8B \ --train-data /absolute/path/to/train.parquet \ @@ -250,6 +276,44 @@ in the reference profile to match Megatron's single-request prefill. Batching requests changes the token-row count of the first MLP down projection and can change BF16 rounding. +The two TP2 reference runs used live SGLang prefill and the validation-only +native Torch DTensor Draft. Each run had one two-GPU Target and either two or +four Draft GPUs. The original observer revision (SHA-256) was +`1c5fea43a3fe6859f802bee6bc2eac2b06d70c606d5425fb18bccba024bcce64`. The run evidence identifies +the installed ASpec source trees by digest: Torch +`e79301553a64bfd6cf12d9e50047e9d2dc82c29ddb8b0976943c2e3d39fcd30f` and Megatron +`26d72e38a3fc8e492a283acb7f6ba608dee3bf4ca74c9e70085a609d19458e8b`. The [per-update objectives](../benchmarks/megatron-online-tp2.csv) +contain both measurements and their absolute relative difference. Each objective +pools the two microbatch numerators and denominators, counting each TP replica once. + +The source checkout includes a portable version of that online driver. Run both +roles sequentially with the same pretokenized training and evaluation Parquet +files. Each invocation starts its own managed Target and writes +`objectives.json` in a fresh directory. The Torch validation plan uses Megatron's +TP2 scheduling topology and selects native DTensor operations in the Draft workers. + +```bash +sgl_target=/absolute/path/to/sglang-target-venv/bin/python +meg_target=/absolute/path/to/megatron-target-venv/bin/python +train=/absolute/path/to/train.parquet +eval_data=/absolute/path/to/eval.parquet +export CUDA_DEVICE_MAX_CONNECTIONS=1 +for dp in 1 2; do + devices=0,1,2,3 + if [[ $dp == 2 ]]; then devices=0,1,2,3,4,5; fi + for draft in torch megatron; do + env_dir=$torch_env + target_python=$sgl_target + if [[ $draft == megatron ]]; then env_dir=$meg_env; target_python=$meg_target; fi + VIRTUAL_ENV="$env_dir" PATH="$env_dir/bin:$PATH" CUDA_VISIBLE_DEVICES="$devices" \ + "$env_dir/bin/python" -m aspec._environment \ + "$env_dir/bin/python" -m tools.validation.megatron_online_alignment \ + --draft "$draft" --dp "$dp" --model "$model" --train-data "$train" --eval-data "$eval_data" \ + --target-python "$target_python" --steps 100 --output "$out/online-$draft-dp$dp" + done +done +``` + The independent fixed-feature checks at TP2/DP1 and TP2/DP2 verified that restoring after update 50 reproduced the remaining 50 updates and final state on each backend. Online mixed-backend checks covered SGLang Target with diff --git a/tests/cpu/test_megatron.py b/tests/cpu/test_megatron.py index 1879783..4f5c2ef 100644 --- a/tests/cpu/test_megatron.py +++ b/tests/cpu/test_megatron.py @@ -5,6 +5,7 @@ import sys import threading from argparse import Namespace +from datetime import timedelta from pathlib import Path from types import SimpleNamespace @@ -174,6 +175,8 @@ def load(_item): dataset._read_tp_source = lambda: ("sample", load("sample")) monkeypatch.setattr(torch.distributed, "get_global_rank", lambda *_: 0) monkeypatch.setattr(torch.distributed, "get_rank", lambda *_: 0) + monkeypatch.setattr(torch.distributed, "all_reduce", lambda *_, **__: None) + def broadcast_control(*_, group, **__): assert group is dataset.tensor_parallel_control_group @@ -265,3 +268,113 @@ def test_megatron_eval_cache_binds_target_worker_and_topology(tmp_path): assert first["producer_code"] == args.megatron_target_cache_identity args.megatron_tp_size = 2 assert first != eval_cache_identity(args, "sha256:dataset", 8) + + +@pytest.mark.parametrize("failure", ["prefill", "validate", "peer", None]) +def test_target_batch_defers_publication_until_all_workers_finish(failure): + from unittest.mock import Mock + + from aspec.inference.engine.megatron_engine import _prefill_batch + + first, second = Mock(), Mock() + calls = [] + + def synchronize(): + calls.append("ready") + if failure == "peer": + raise ValueError("peer") + + def generate(): + with _prefill_batch(synchronize) as stage: + stage(first) + stage(second) + if failure == "prefill": + raise ValueError("prefill") + if failure == "validate": + second.validate_publishable.side_effect = ValueError("validate") + first.publish.side_effect = lambda: calls.append("published") + + if failure: + with pytest.raises(ValueError, match=failure): + generate() + for sink in (first, second): + sink.publish.assert_not_called() + sink.abort.assert_called_once() + else: + generate() + assert calls == ["ready", "published"] + second.publish.assert_called_once() + + +@pytest.mark.parametrize("released", [False, True]) +def test_target_closes_source_only_after_release(released): + from unittest.mock import Mock + + from aspec.inference.engine.megatron_engine import _close_source + + store = Mock() + store.config.get_retry_max_wait_seconds = 7 + store.wait_for_p2p_source_releases.return_value = released + if released: + _close_source(store) + store.close.assert_called_once() + else: + with pytest.raises(TimeoutError, match="active readers"): + _close_source(store) + store.close.assert_not_called() + store.wait_for_p2p_source_releases.assert_called_once_with(timeout=7) + + +def test_bitwise_equality_and_final_reference_policy(tmp_path): + from tools.validation.megatron_torch_alignment import compare_vectors, reference_state_path + + reference = {"weight": torch.tensor([0.0])} + result = compare_vectors({"weight": torch.tensor([-0.0])}, reference, device="cpu") + assert result["max_absolute"] == 0 and not result["bitwise_equal"] + (tmp_path / "completed.json").touch() + milestone = tmp_path / "logical-state-step4.pt" + milestone.touch() + assert reference_state_path(tmp_path, 4, 4, 0) == milestone + final = tmp_path / "final-logical.pt" + final.touch() + assert reference_state_path(tmp_path, 4, 4, 0) == final + + +def _tp_input_worker(rank, init_method, use_cuda): + import torch.distributed as dist + + from aspec.training.data_fetcher import MooncakeDataset + + dist.init_process_group("gloo", init_method=init_method, rank=rank, world_size=2, + timeout=timedelta(seconds=60)) + try: + dataset = object.__new__(MooncakeDataset) + dataset.tensor_parallel_control_group = dist.group.WORLD + if use_cuda: + torch.cuda.set_device(rank) + group = dist.new_group(backend="nccl") if use_cuda else dist.group.WORLD + dataset.tensor_parallel_group = group + dataset._tp_source_prefetch = None + dataset._tensor_parallel_samples = dataset._tensor_parallel_payload_bytes = 0 + dataset._feature_contract_runtime = None + sample = SimpleNamespace(tensor_shapes={}, tensor_dtypes={}, metadata={}) + dataset._read_tp_source = lambda: (sample, {"ids": torch.tensor([5])} if use_cuda else {}) + _, data = dataset._tensor_parallel_item_on_device() + if use_cuda: + assert data["ids"].tolist() == [5] + assert data["ids"].device.type == "cpu" + + def reject(**kwargs): + raise ValueError("invalid follower metadata") + + dataset._feature_contract_runtime = SimpleNamespace(validate_metadata=reject) + with pytest.raises(RuntimeError, match="rank 1.*invalid follower metadata"): + dataset._tensor_parallel_item_on_device() + finally: + dist.destroy_process_group() + + +@pytest.mark.parametrize("use_cuda", [False, pytest.param(True, marks=pytest.mark.skipif( + torch.cuda.device_count() < 2, reason="requires two GPUs for the NCCL payload"))]) +def test_tp_preparation_error_reaches_both_ranks(tmp_path, use_cuda): + torch.multiprocessing.spawn(_tp_input_worker, args=((tmp_path / "rdzv").as_uri(), use_cuda), nprocs=2) diff --git a/tests/cpu/test_target_environment.py b/tests/cpu/test_target_environment.py index 1d2edeb..bb2340f 100644 --- a/tests/cpu/test_target_environment.py +++ b/tests/cpu/test_target_environment.py @@ -236,3 +236,23 @@ def test_cross_node_target_tp_is_rejected_before_creating_facades(): with pytest.raises(ValueError, match="node-local"): _prepare_sgl_engines(Namespace(sglang_nnodes=2), None) ray.remote.assert_not_called() + + +@pytest.mark.parametrize("flags,allowed", [ + (["--back", "megatron"], True), + (["--backend=megatron", "--backend=sglang"], False), +]) +def test_target_cli_guards_the_parsed_backend(monkeypatch, flags, allowed): + from aspec import cli + + calls = [] + monkeypatch.setattr(cli, "is_target_environment", lambda: False) + monkeypatch.setattr(cli, "execute", calls.append) + monkeypatch.setattr(sys, "argv", ["aspec", "target", *flags]) + if allowed: + cli.main() + assert calls == [[sys.executable, "-m", "aspec.inference.service", *flags]] + else: + with pytest.raises(SystemExit, match="Target environment"): + cli.main() + assert not calls diff --git a/tests/cpu/test_target_service.py b/tests/cpu/test_target_service.py index d790327..0fa5ebc 100644 --- a/tests/cpu/test_target_service.py +++ b/tests/cpu/test_target_service.py @@ -811,3 +811,12 @@ def test_cross_node_tp_is_rejected_before_target_allocation(service, nnodes, nod with pytest.raises(ValueError, match="node-local"): service.create(value) assert not service.engines and service.idle() + + +def test_megatron_create_rejects_mismatched_tp_before_allocating(service): + service.runtime["backend"] = "megatron" + payload = creation(service, count=1) + payload["arguments"].update(target_model_backend="megatron", megatron_tp_size=2) + with pytest.raises(ValueError, match="tensor parallel size"): + service.create(payload) + assert not service.engines diff --git a/tests/cpu/test_torch_tp_validation.py b/tests/cpu/test_torch_tp_validation.py index dc26184..aca61f9 100644 --- a/tests/cpu/test_torch_tp_validation.py +++ b/tests/cpu/test_torch_tp_validation.py @@ -19,6 +19,8 @@ def _projection_worker(rank, rendezvous): timeout=timedelta(seconds=60)) try: mesh = init_device_mesh("cpu", (2,)) + with pytest.raises(ValueError, match="divide evenly"): + TorchTPLinear(torch.nn.Linear(4, 5, bias=False), mesh, axis=0) for dtype in (torch.float32, torch.bfloat16): for axis in (0, 1): torch.manual_seed(114) diff --git a/tools/validation/megatron_online_alignment.py b/tools/validation/megatron_online_alignment.py new file mode 100644 index 0000000..3bcf9e2 --- /dev/null +++ b/tools/validation/megatron_online_alignment.py @@ -0,0 +1,153 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Measure the deterministic TP2 DSpark objective with a live, same-TP Target.""" + +import argparse +import json +import sys +from contextlib import ExitStack +from dataclasses import replace +from pathlib import Path +from statistics import mean +from unittest.mock import patch + +import torch.distributed as dist + +from aspec import train_entry +from aspec.commands import _recipe_smoke +from aspec.ray.train_group import RayTrainGroup +from aspec.training import trainer_actor +from tools.validation.megatron_torch_alignment import deterministic_attention_backward +from tools.validation.torch_tp_dspark import TorchTPDSparkTrainer + + +class OnlineTorchTPTrainer(TorchTPDSparkTrainer): + def _setup_device_mesh(self): + super()._setup_device_mesh() + for replica in range(self.dp_size): + ranks = list(range(replica * self.tp_size, (replica + 1) * self.tp_size)) + groups = [dist.new_group(ranks, backend=backend) for backend in ("nccl", "gloo", "nccl", "gloo")] + if replica == self.dp_rank: + (self.input_tp_group, self.input_tp_control_group, + self.eval_input_tp_group, self.eval_input_tp_control_group) = groups + + +class ObjectiveActor(trainer_actor.TrainerActor): + def init(self, args, *rest, **kwargs): + with ExitStack() as stack: + if args.train_backend == "torch-tp-validation": + stack.enter_context(patch.object(trainer_actor, "_trainer_class_for_config", + return_value=OnlineTorchTPTrainer)) + result = super().init(args, *rest, **kwargs) + trainer = self._trainer + if trainer.tp_size != 2 or trainer.execution_backend != args.train_backend: + raise ValueError("Online comparison requires the selected TP2 trainer") + if args.train_backend == "torch-tp-validation" and "megatron.core" in sys.modules: + raise RuntimeError("Torch TP validation worker imported Megatron Core") + self.objectives = [] + forward = trainer._forward + + def observe(*inputs, **options): + outputs = forward(*inputs, **options) + if trainer.model.training and trainer.tp_rank == 0: + numerator, denominator = outputs[-1] + self.objectives.append([trainer.global_step + 1, numerator.item(), denominator.item()]) + return outputs + + trainer._forward = observe + return result + + def train_from_queue(self, step, num_batches): + with deterministic_attention_backward() as calls: + result = super().train_from_queue(step, num_batches) + if not calls["backward"]: + raise RuntimeError("Missing deterministic cuDNN attention backward on this rank") + return result + + def save_model(self, step, force_sync=False): + super().save_model(step, force_sync) + name = f"recipe-objective-dp{self._trainer.dp_rank}.json" + files = {name: json.dumps(self.objectives)} if self._trainer.tp_rank == 0 else {} + return {"output_dir": self.args.output_dir, "files": files} + + +def torch_validation_plan(plan): + policy = plan.policy + if (policy.backends["training"], policy.backends["target_features"]) != ("megatron", "sglang"): + raise ValueError("Torch validation needs a TP2 scheduling plan and a live SGLang Target") + if any(policy.parallelism[role]["tensor_parallel"] != 2 for role in ("training", "inference")): + raise ValueError("Both Target and Draft must use TP2") + training = {**policy.parallelism["training"], "fsdp_strategy": "REPLICATE"} + training.pop("core_grad_reduce_in_fp32", None) + policy = replace(policy, backends={**policy.backends, "training": "torch-tp-validation"}, + parallelism={**policy.parallelism, "training": training}) + return replace(plan, policy=policy, _runtime_values={**plan._runtime_values, + "train_backend": "torch-tp-validation"}) + + +def replica_queues(group, queues): + if len(group._actor_handlers) != 2 * len(queues): + raise ValueError("Expected one input queue per TP2 replica") + return [queue for queue in queues for _ in range(2)] + + +def objectives(root, dp, steps): + windows = [[] for _ in range(steps)] + for rank in range(dp): + rows = json.loads((root / f"recipe-objective-dp{rank}.json").read_text()) + expected = [step for step in range(1, steps + 1) for _ in range(2 // dp)] + if [row[0] for row in rows] != expected: + raise ValueError(f"Replica {rank} did not observe the expected optimizer windows") + for step, numerator, denominator in rows: + windows[step - 1].append((numerator, denominator)) + return [sum(n for n, _ in window) / max(sum(d for _, d in window), 1e-6) for window in windows] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--draft", choices=("torch", "megatron"), required=True) + parser.add_argument("--dp", type=int, choices=(1, 2), required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--train-data", type=Path, required=True) + parser.add_argument("--eval-data", type=Path, required=True) + parser.add_argument("--target-python", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--steps", type=int, default=100) + args = parser.parse_args() + if args.steps < 1: + parser.error("steps must be positive") + root = args.output.resolve() + target = "sglang" if args.draft == "torch" else "megatron" + config = Path(__file__).resolve().parents[2] / "configs/recipes/quickstart_qwen3_8b.yaml" + budget = args.dp * 2 + 2 + argv = [sys.argv[0], "--config", str(config), "--gpus", str(budget), "--output", str(root), + f"model.target_model_path={args.model.resolve()}", f"model.target_model_backend={target}", + f"dataset.train_data_path={args.train_data.resolve()}", "dataset.chat_template=null", + f"dataset.eval_data_path={args.eval_data.resolve()}", f"dataset.eval_interval={args.steps}", + f"cache_dir={root / 'cache'}", "training.train_backend=megatron", "training.megatron_grad_dtype=bf16", + "training.draft_tensor_parallel_size=2", f"training.training_num_gpus_per_node={args.dp * 2}", + f"training.draft_accumulation_steps={2 // args.dp}", "training.num_epochs=100", + "training.max_seq_length=4096", "training.dspark_num_anchors=512", "training.dflash_packing=false", + "training.gradient_checkpointing=false", f"training.num_train_steps={args.steps}", + f"training.save_interval={args.steps}", f"training.lr_total_steps={args.steps}", + "inference.target_service.mode=managed-local", "inference.target_service.address=http://127.0.0.1:30000", + f"inference.target_service.python_executable={args.target_python.absolute()}", + f"inference.inference_engine_type={'sgl' if target == 'sglang' else target}", + f"inference.target_prefill_memory_optimized={'false' if target == 'sglang' else 'true'}", + "inference.inference_num_gpus=2", "inference.inference_num_gpus_per_engine=2", + f"inference.inference_num_gpus_per_node={budget}", "inference.sglang.tp_size=2", + "inference.megatron.tp_size=2", "mooncake.max_seq_len=4096", "mooncake.destination_gpu_ring_slots=3"] + with ExitStack() as stack: + stack.enter_context(patch.object(sys, "argv", argv)) + if args.draft == "torch": + compile_plan = train_entry.compile_execution_plan + stack.enter_context(patch.object(train_entry, "compile_execution_plan", + lambda *a, **kw: torch_validation_plan(compile_plan(*a, **kw)))) + stack.enter_context(patch.object(RayTrainGroup, "_expand_queues_for_usp", replica_queues)) + _recipe_smoke.main(trainer_class=ObjectiveActor) + values = objectives(root, args.dp, args.steps) + (root / "objectives.json").write_text(json.dumps({"steps": values, "mean": mean(values)}, allow_nan=False)) + + +if __name__ == "__main__": + main() diff --git a/tools/validation/megatron_target_aspec_alignment.py b/tools/validation/megatron_target_aspec_alignment.py index 153db02..1ff97ec 100644 --- a/tools/validation/megatron_target_aspec_alignment.py +++ b/tools/validation/megatron_target_aspec_alignment.py @@ -35,8 +35,11 @@ def main(): args = parser.parse_args() torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) dist.init_process_group("nccl") - cached = torch.load(args.reference_cache, map_location="cpu", weights_only=False, mmap=True) + cached = torch.load(args.reference_cache, map_location="cpu", weights_only=True, mmap=True) identity = cached["identity"] + reference_tp = identity["inputs"].get("sglang_tp_size") + if type(reference_tp) is not int or reference_tp != dist.get_world_size(): + raise ValueError("SGLang reference cache must declare the same TP size as the Target") contract = identity["inputs"]["aspec_feature_contract"] if contract["producer"]["backend"] != "sglang": raise ValueError("Reference cache must come from the existing ASpec SGLang producer") diff --git a/tools/validation/megatron_torch_alignment.py b/tools/validation/megatron_torch_alignment.py index 7f8aa0a..368723a 100644 --- a/tools/validation/megatron_torch_alignment.py +++ b/tools/validation/megatron_torch_alignment.py @@ -16,6 +16,7 @@ import math import os import time +from contextlib import contextmanager from pathlib import Path from unittest.mock import patch @@ -172,7 +173,8 @@ def quantiles(values): continue if value.shape != reference.shape: raise ValueError(f"Compared tensor shapes differ: {name}") - a, b = value.to(device).float().flatten(), reference.to(device).float().flatten() + value, reference = value.to(device), reference.to(device) + a, b = value.float().flatten(), reference.float().flatten() if not (torch.isfinite(a).all() & torch.isfinite(b).all()).item(): raise ValueError(f"Nonfinite tensor in numerical comparison: {name}") ad, bd = a.double(), b.double() @@ -190,7 +192,10 @@ def quantiles(values): totals[10:13] = torch.maximum(totals[10:13], row[10:13]) totals[13] += row[13] values = row.tolist() - exact = exact and value.dtype == reference.dtype and values[13] == 0 + exact = exact and value.dtype == reference.dtype and torch.equal( + value.contiguous().reshape(-1).view(torch.uint8), + reference.contiguous().reshape(-1).view(torch.uint8), + ) row_summary = summarize(values) per_parameter.append({"parameter": name, "squared_error": values[1], "denominator_floor": floor, **row_summary, @@ -226,6 +231,17 @@ def wait_reference(path, timeout): time.sleep(.5) +def reference_state_path(reference, step, final_step, timeout): + if step == final_step: + wait_reference(reference / "completed.json", timeout) + final = reference / "final-logical.pt" + if final.is_file(): + return final + else: + wait_reference(reference / f"step{step}.json", timeout) + return reference / f"logical-state-step{step}.pt" + + def source_identity(trainer, expected): from aspec.training import checkpoint @@ -288,6 +304,9 @@ def main(): parser.add_argument("--attention-backward-deterministic", action="store_true", help="Require native deterministic cuDNN-FE attention backward on every rank") args = parser.parse_args() + if args.steps < 1 or (args.phase in {"split", "checkpoint-control"} + and not 1 <= args.split_step <= args.steps): + parser.error("Require positive steps and a split within the training horizon") if args.backend == "fsdp" and args.tp != 1: parser.error("The Torch baseline uses its existing TP1 execution") if args.backend != "megatron" and args.megatron_grad_dtype is not None: @@ -298,6 +317,12 @@ def main(): if args.attention_backend != "cudnn_flex_attention" or args.phase == "prepare": parser.error("The deterministic backward check requires a cuDNN training run") + with deterministic_attention_backward() as calls: + run(args, attention_calls=calls) + + +@contextmanager +def deterministic_attention_backward(): from cudnn.flex_attention import autograd, flex_attn_func backward = autograd._flex_attention_backward @@ -316,7 +341,7 @@ def checked_backward(*inputs, **options): with (patch("cudnn.flex_attention.flex_attn_func", deterministic_forward), patch.object(autograd, "_flex_attention_backward", checked_backward)): - run(args, attention_calls=calls) + yield calls def run(args, *, attention_calls=None): @@ -428,8 +453,10 @@ def find_spec(self, fullname, path=None, target=None): del initial_state saved_work = checkpoint.load_checkpoint_work_state(runtime) if args.resume else None committed = list(saved_work["committed_work_ids"]) if saved_work else [] - samples = torch.load(args.teacher, map_location="cpu", weights_only=False, mmap=True)["samples"] fixture_sha = file_sha(args.teacher) + if args.resume and (saved_work or {}).get("dataset_fingerprint") != "sha256:" + fixture_sha: + raise ValueError("Resume teacher fixture differs from checkpoint dataset fingerprint") + samples = torch.load(args.teacher, map_location="cpu", weights_only=True, mmap=True)["samples"] manifest = {"teacher_sha256": fixture_sha, "initial_sha256": file_sha(args.initial) if args.initial else None, "steps": args.steps, "length": args.length, "anchors": args.anchors, "accumulation": 2, "attention_backend": args.attention_backend, @@ -512,7 +539,9 @@ def observed_step(*values, **kwargs): participants = [None] * world dist.all_gather_object(participants, local_work) new_work = [item for replica in participants for item in replica] - if len(new_work) != 2 * trainer.dp_size or set(new_work).intersection(committed): + issued = [f"window-{step}/dp-{replica}/micro-{micro}" + for replica in range(trainer.dp_size) for micro in range(2)] + if sorted(new_work) != sorted(issued) or set(new_work).intersection(committed): raise RuntimeError("Committed work was duplicated or omitted") committed.extend(new_work) totals = torch.stack(observation["terms"]).sum(0) @@ -535,7 +564,7 @@ def observed_step(*values, **kwargs): updates[name] = current - observation["before"][name] if writer: vectors = {"gradients": observation["gradients"], "updates": updates} - record = {"step": step, "loss": (totals[0] / totals[1]).item(), + record = {"step": step, "loss": (totals[0] / totals[1].clamp_min(1e-6)).item(), "numerator": totals[0].item(), "denominator": totals[1].item(), "gradient_norm": observation["norm"].item(), "learning_rate_after_update": trainer.optimizer.get_learning_rate(), @@ -567,7 +596,7 @@ def observed_step(*values, **kwargs): **expected_record["metrics"]} measurements["scalars"] = {key: scalar_diff(value, expected_scalars[key]) for key, value in scalar_values.items()} - measurements.update(loss_relative_error=abs(record["loss"] / expected_record["loss"] - 1), + measurements.update(loss_relative_error=measurements["scalars"]["objective"]["abs_rel_diff"], denominator_exact=record["denominator"] == expected_record["denominator"], anchors_exact=by_replica == expected_record["anchors_by_replica"], learning_rate_exact=record["learning_rate_after_update"] == expected_record["learning_rate_after_update"]) @@ -586,8 +615,8 @@ def observed_step(*values, **kwargs): torch.save(logical, output / f"logical-state-step{step}.pt") state_comparisons = {} for reference in args.reference: - wait_reference(reference / f"step{step}.json", args.reference_wait_seconds) - expected = torch.load(reference / f"logical-state-step{step}.pt", weights_only=True, mmap=True) + path = reference_state_path(reference, step, args.steps, args.reference_wait_seconds) + expected = torch.load(path, weights_only=True, mmap=True) state_comparisons[str(reference)] = {key: compare_vectors(value, expected[key]) for key, value in logical.items()} del expected @@ -618,8 +647,8 @@ def observed_step(*values, **kwargs): torch.save(logical, output / "final-logical.pt") comparisons = {} for reference in args.reference: - wait_reference(reference / "completed.json", args.reference_wait_seconds) - expected = torch.load(reference / "final-logical.pt", weights_only=True, mmap=True) + path = reference_state_path(reference, args.steps, args.steps, args.reference_wait_seconds) + expected = torch.load(path, weights_only=True, mmap=True) comparisons[str(reference)] = {key: compare_vectors(values, expected[key]) for key, values in logical.items()} write_json(output / "final-logical-comparisons.json", comparisons) diff --git a/tools/validation/torch_tp_dspark.py b/tools/validation/torch_tp_dspark.py index 8d46ef2..f2a7e18 100644 --- a/tools/validation/torch_tp_dspark.py +++ b/tools/validation/torch_tp_dspark.py @@ -25,6 +25,8 @@ def __init__(self, original, mesh, axis): super().__init__() if original.bias is not None: raise ValueError("The Torch TP validation path requires bias-free projections") + if axis not in (0, 1) or original.weight.size(axis) % mesh.size(): + raise ValueError("The TP projection dimension must divide evenly across ranks") self.mesh, self.axis = mesh, axis self.weight = torch.nn.Parameter( original.weight.chunk(mesh.size(), dim=axis)[mesh.get_local_rank()].clone(), From 5b98a42dd30d54b1118867eec26b57187770fa17 Mon Sep 17 00:00:00 2001 From: zhouaihui <34807769+zhouaihui@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:19:22 +0800 Subject: [PATCH 3/5] fix: roll back staged Target outputs when a TP peer exits Use a Gloo readiness barrier after CUDA work finishes. Cover Target context limits, GPU lease teardown, TP failure rollback, and RoPE cache invalidation. --- aspec/inference/engine/megatron_engine.py | 16 +- tests/cpu/test_megatron.py | 40 ++++ tests/test_megatron_target_gpu.py | 276 ++++++++++++++++++++++ 3 files changed, 330 insertions(+), 2 deletions(-) create mode 100644 tests/test_megatron_target_gpu.py diff --git a/aspec/inference/engine/megatron_engine.py b/aspec/inference/engine/megatron_engine.py index 8896e61..15d3f06 100644 --- a/aspec/inference/engine/megatron_engine.py +++ b/aspec/inference/engine/megatron_engine.py @@ -62,11 +62,23 @@ def _target_worker(connection, local_rank, gpu_ids, args, mooncake_config, port, try: os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(str(gpu) for gpu in gpu_ids) torch.cuda.set_device(local_rank) + timeout = timedelta(seconds=getattr(args, "megatron_init_timeout", 600)) dist.init_process_group( "nccl", init_method=f"tcp://127.0.0.1:{port}", rank=local_rank, world_size=len(gpu_ids), - timeout=timedelta(seconds=getattr(args, "megatron_init_timeout", 600)), + timeout=timeout, ) + ready = None + if len(gpu_ids) > 1: + control = dist.new_group(backend="gloo", timeout=timeout) + + def wait_for_peers(): + # Finish local GPU work first. A failed peer must not strand the + # publishing rank inside a CUDA collective with unreturned slots. + torch.cuda.current_stream().synchronize() + dist.monitored_barrier(group=control, timeout=timeout) + + ready = wait_for_peers groups = initialize_parallelism(len(gpu_ids)) model, config = load_qwen3_target( args.target_model_path, groups, @@ -120,7 +132,7 @@ def _target_worker(connection, local_rank, gpu_ids, args, mooncake_config, port, if method != "generate": raise ValueError(f"Unknown Megatron Target operation: {method}") outputs = [] - with _prefill_batch(dist.barrier if len(gpu_ids) > 1 else None) as stage: + with _prefill_batch(ready) as stage: for sample in payload: ids = torch.tensor(sample["tokens"], dtype=torch.long, device="cuda") length = ids.numel() diff --git a/tests/cpu/test_megatron.py b/tests/cpu/test_megatron.py index 4f5c2ef..447ceb9 100644 --- a/tests/cpu/test_megatron.py +++ b/tests/cpu/test_megatron.py @@ -378,3 +378,43 @@ def reject(**kwargs): torch.cuda.device_count() < 2, reason="requires two GPUs for the NCCL payload"))]) def test_tp_preparation_error_reaches_both_ranks(tmp_path, use_cuda): torch.multiprocessing.spawn(_tp_input_worker, args=((tmp_path / "rdzv").as_uri(), use_cuda), nprocs=2) + + +def test_rotary_source_change_invalidates_megatron_eval_cache(tmp_path, monkeypatch): + from unittest.mock import Mock + + from aspec.controller import eval as evaluation + from aspec.runtime_identity import MEGATRON_RUNTIME_RELATIVE_PATHS, module_runtime_identity + + core = tmp_path / "fixture_core" + core.mkdir() + (core / "__init__.py").touch() + for relative in MEGATRON_RUNTIME_RELATIVE_PATHS: + path = core / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# original source\n") + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.setattr(evaluation.ray, "get", lambda value: value) + controller, group = Mock(), Mock() + controller.get_dataset_admission_evidence.remote.return_value = {"eval": {"dataset_fingerprint": "sha256:data"}} + cached = set() + group.load_eval_cache.side_effect = lambda path, identity: [int(path in cached)] + args = SimpleNamespace(target_model_path=str(tmp_path), target_model_backend="megatron", + eval_interval=1, checkpoint_dir=None, cache_dir=str(tmp_path / "cache"), + dp_size=1, inference_batch_size=1, megatron_tp_size=1, + megatron_implementation="transformer_engine") + + def lookup(): + digest = module_runtime_identity(core.name, relative_paths=MEGATRON_RUNTIME_RELATIVE_PATHS) + assert not digest["missing_files"] + args.megatron_target_cache_identity = {"core": digest["effective_tree_sha256"], + "dependencies": {"megatron-core": "0.20.0"}} + return evaluation.setup_eval(controller, group, args, eval_dataset_size=1) + + initial = lookup() + cached.add(initial.eval_cache_path) + assert lookup().eval_cache_loaded + (core / "models/common/embeddings/rotary_pos_embedding.py").write_text("# changed source\n") + changed = lookup() + assert not changed.eval_cache_loaded + assert changed.eval_cache_path != initial.eval_cache_path diff --git a/tests/test_megatron_target_gpu.py b/tests/test_megatron_target_gpu.py new file mode 100644 index 0000000..bcde0d3 --- /dev/null +++ b/tests/test_megatron_target_gpu.py @@ -0,0 +1,276 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 +"""Opt-in Target worker checks. Set ASPEC_RUN_GPU_INTEGRATION=1 and ASPEC_TARGET_MODEL.""" + +import asyncio +import json +import os +import socket +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from aspec.config.mooncake_config import MooncakeConfig +from aspec.feature_contract import build_target_feature_contract +from aspec.inference.engine.megatron_engine import MegatronEngine + +pytestmark = pytest.mark.skipif( + os.getenv("ASPEC_RUN_GPU_INTEGRATION") != "1" or not os.getenv("ASPEC_TARGET_MODEL"), + reason="requires a local Qwen3 checkpoint, the Megatron Target environment and NVLink GPUs", +) + + +def _config(): + return MooncakeConfig(local_hostname="127.0.0.1", data_plane="transfer_engine", + protocol="nvlink_intra", require_nvlink=True, enable_gpu_direct=True, + allow_host_fallback=False, gpu_buffer_size=2 << 20, gpu_ring_slots=2, + get_retry_max_wait_seconds=5) + + +def _gpu_ids(count): + import torch + + if torch.cuda.device_count() < count: + pytest.skip(f"requires {count} visible GPUs") + visible = os.environ.get("CUDA_VISIBLE_DEVICES") + return visible.split(",")[:count] if visible else list(range(count)) + + +@contextmanager +def _engine(tp, *, context_offset=0, request_offset=0): + model = Path(os.environ["ASPEC_TARGET_MODEL"]) + config = json.loads((model / "config.json").read_text()) + limit = config["max_position_embeddings"] + contract = build_target_feature_contract( + drafter_family="dspark", target_hidden_size=config["hidden_size"], + target_num_hidden_layers=config["num_hidden_layers"], aux_hidden_state_layers=[0], + store_last_hidden_states=True, producer_backend="megatron", + ) + args = SimpleNamespace(target_model_path=str(model), aspec_feature_contract=contract, + aspec_context_contract={"backend_context_length": limit + context_offset}, + max_seq_length=limit + request_offset, aux_hidden_states_layers=[0], + store_last_hidden_states=True, attention_backend="cudnn_flex_attention", + megatron_implementation=os.getenv("ASPEC_MEGATRON_IMPLEMENTATION", "transformer_engine")) + engine = MegatronEngine(args, rank=0, num_gpus_per_engine=tp, gpu_ids=_gpu_ids(tp)) + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + try: + engine.init(_config(), pre_allocated_port=port) + yield engine, limit + finally: + # Failed workers may intentionally retain registrations. These tests own + # all clients and finish fencing them before leaving this context. + for process in engine._processes: + if process.is_alive(): + process.terminate() + process.join(timeout=10) + if process.is_alive(): + process.kill() + process.join(timeout=10) + for connection in engine._connections: + connection.close() + + +@pytest.mark.parametrize("tp", [1, 2]) +@pytest.mark.parametrize("context_offset,request_offset", [(0, 0), (1, 0), (0, 1)]) +def test_worker_readiness_at_model_limit(tp, context_offset, request_offset): + if context_offset or request_offset: + with pytest.raises(RuntimeError, match="Planned Target context exceeds the loaded model"): + with _engine(tp, context_offset=context_offset, request_offset=request_offset): + pytest.fail("worker advertised readiness above the model limit") + return + with _engine(tp) as (engine, limit): + assert len(engine._status) == tp + assert all(s["max_sequence_length"] == limit and s["tensor_parallel_size"] == tp for s in engine._status) + assert engine._call("status")["transport"]["ring"]["in_use"] == 0 + asyncio.run(engine.shutdown()) + + +def _transport_worker(audit, draining, connection, rank, gpu_ids, *args): + """Exercise the worker RPC and real source ring with small known CUDA payloads.""" + from contextlib import ExitStack + from unittest.mock import patch + + import torch + import torch.distributed as dist + + from aspec.inference.engine import megatron_engine + from aspec.inference.target_prefill import TargetPrefillDirectSink + from aspec.models import megatron + from aspec.transfer.mooncake.eagle_store import EagleMooncakeStore + + def record(event, value): + with audit.open("a") as stream: + stream.write(json.dumps([event, value]) + "\n") + + load = megatron.load_qwen3_target + + def load_with_known_features(*values, **options): + model, config = load(*values, **options) + + def prefill(ids, layers, capture): + hidden = ids.view(-1, 1).expand(-1, config.hidden_size).to(torch.bfloat16).contiguous() + dist.all_reduce(hidden) + hidden.div_(dist.get_world_size()) + if capture: + capture(layers[0], hidden, None) + if ids[0, 0].item() == 98 + rank: + raise RuntimeError("injected second prefill failure") + return hidden + + model.prefill = prefill + return model, config + + class ObservedSink(TargetPrefillDirectSink): + def commit_request(self, request_id): + key = super().commit_request(request_id) + record("commit", request_id) + return key + + def publish(self): + super().publish() + record("publish", self.requests[0].request_id) + + def abort(self): + before = self.store.get_p2p_status()["ring"]["in_use"] + super().abort() + after = self.store.get_p2p_status()["ring"]["in_use"] + record("abort", [before, after]) + if after == 0: + with ExitStack() as cleanup: + for i in range(2): + slot = self.store.reserve_p2p_output( + key=f"reuse-{i}", sequence_length=1, hidden_size=32, num_aux_layers=1, + store_last_hidden_states=True, data_id=str(i), request_id=str(i), source_engine="0") + cleanup.callback(slot.discard) + record("reacquired", self.store.get_p2p_status()["ring"]["in_use"]) + + wait = EagleMooncakeStore.wait_for_p2p_source_releases + + def observe_drain(store, timeout): + draining.set() + return wait(store, timeout=timeout) + + with (patch.object(megatron, "load_qwen3_target", load_with_known_features), + patch.object(megatron_engine, "TargetPrefillDirectSink", ObservedSink), + patch.object(EagleMooncakeStore, "wait_for_p2p_source_releases", observe_drain)): + megatron_engine._target_worker(connection, rank, gpu_ids, *args) + + +@pytest.fixture +def observed_transport(tmp_path, monkeypatch): + from functools import partial + + import torch.multiprocessing as mp + + from aspec.inference.engine import megatron_engine + + audit = tmp_path / "ring.jsonl" + draining = mp.get_context("spawn").Event() + # Each spawned child imports the original worker from the module afresh. + monkeypatch.setattr(megatron_engine, "_target_worker", partial(_transport_worker, audit, draining)) + monkeypatch.setenv("ASPEC_RUN_ID", "target-test") + monkeypatch.setenv("ASPEC_ATTEMPT_ID", "target-test-attempt") + return audit, draining + + +@pytest.mark.parametrize("failing_rank", [0, 1], ids=["publisher", "follower"]) +def test_tp2_failed_rpc_returns_no_keys_and_releases_slots(observed_transport, failing_rank): + import torch + + audit, _ = observed_transport + with _engine(2) as (engine, _): + engine.args.megatron_init_timeout = 30 # Bound the RPC after checkpoint loading. + with pytest.raises(RuntimeError): + asyncio.run(engine.generate(["A", "B"], input_ids_ref=[torch.tensor([7]), torch.tensor([98 + failing_rank])], + request_ids=["A", "B"])) + if failing_rank == 1: + # The facade receives the publishing rank's rollback error first. + assert engine._connections[1].poll(5) + assert "injected second prefill failure" in engine._connections[1].recv()["error"] + assert engine._processes[0].is_alive() + events = [json.loads(line) for line in audit.read_text().splitlines()] + committed = [["commit", "A"], ["commit", "B"]] if failing_rank == 1 else [["commit", "A"]] + assert events == committed + [["abort", [2, 1]], ["abort", [1, 0]], ["reacquired", 2]] + + +def _read_on_second_gpu(connection): + import torch + + from aspec.transfer.mooncake.p2p import MooncakeP2PDescriptor, MooncakeP2PTransfer + + torch.cuda.set_device(1) + transfer = MooncakeP2PTransfer(_config()) + try: + transfer.setup(torch.device("cuda", 1), role="destination") + connection.send("ready") + result = transfer.read(MooncakeP2PDescriptor.from_key(connection.recv())) + try: + ids = result.tensors["input_ids"] + assert ids.tolist() == [7, 11] + for name in ("hidden_states", "last_hidden_states"): + features = result.tensors[name] + torch.testing.assert_close(features, ids[:, None].expand_as(features).to(features.dtype)) + finally: + result.lease.release() + connection.send("read") + assert connection.recv() == "exit" + finally: + transfer.close() + connection.close() + + +@pytest.mark.parametrize("timeout", [False, True], ids=["active-release", "timeout-fence"]) +def test_shutdown_keeps_published_source_readable(observed_transport, timeout): + from concurrent.futures import ThreadPoolExecutor + + import psutil + import torch + + from aspec.inference.service import readers_exited, stop_owned_processes + + _gpu_ids(2) + _, draining = observed_transport + ctx = torch.multiprocessing.get_context("spawn") + # The active-release case also exercises a successful TP2 publication. + with _engine(1 if timeout else 2) as (engine, _): + client, child = ctx.Pipe() + reader = ctx.Process(target=_read_on_second_gpu, args=(child,)) + reader.start() + child.close() + try: + assert client.poll(120) and client.recv() == "ready" + owners = [{"node_ip": "127.0.0.1", "pid": reader.pid, "created": psutil.Process(reader.pid).create_time()}] + source = engine._processes[0] + identity = {source.pid: psutil.Process(source.pid).create_time()} + outputs = asyncio.run(engine.generate("sample", input_ids_ref=[torch.tensor([7, 11])])) + assert engine._call("status")["transport"]["ring"]["in_use"] == 1 + with ThreadPoolExecutor(max_workers=1) as executor: + shutdown = executor.submit(asyncio.run, engine.shutdown()) + assert draining.wait(timeout=30) + if timeout: + with pytest.raises(RuntimeError, match="source leases still have active readers"): + shutdown.result(timeout=30) + else: + assert not shutdown.done() + assert source.is_alive() and not readers_exited(owners, "127.0.0.1") + client.send(outputs[0]["mooncake_key"]) + assert client.poll(30) and client.recv() == "read" + if not timeout: + shutdown.result(timeout=30) + client.send("exit") + reader.join(timeout=15) + assert reader.exitcode == 0 and readers_exited(owners, "127.0.0.1") + if timeout: + assert source.is_alive() + stop_owned_processes(identity) + source.join(timeout=10) + assert not source.is_alive() + finally: + if reader.is_alive(): + reader.kill() + reader.join(timeout=10) + client.close() From 12d911ca49476bc3bf5f72219b38ae0f05188d37 Mon Sep 17 00:00:00 2001 From: zhouaihui <34807769+zhouaihui@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:22:29 +0800 Subject: [PATCH 4/5] fix: reject packed TP input before dataset setup Keep the NCCL input test opt-in and make the online alignment example runnable from a fresh shell. --- aspec/training/data_fetcher.py | 2 ++ docs/user-guide/megatron.md | 11 ++++++-- tests/cpu/test_megatron.py | 51 ++++++++++------------------------ tests/test_tp_input_gpu.py | 17 ++++++++++++ tests/tp_input.py | 42 ++++++++++++++++++++++++++++ 5 files changed, 83 insertions(+), 40 deletions(-) create mode 100644 tests/test_tp_input_gpu.py create mode 100644 tests/tp_input.py diff --git a/aspec/training/data_fetcher.py b/aspec/training/data_fetcher.py index 5516d47..3006b4a 100644 --- a/aspec/training/data_fetcher.py +++ b/aspec/training/data_fetcher.py @@ -147,6 +147,8 @@ def __init__( raise ValueError("TP input requires paired GPU-data and CPU-control groups") if tensor_parallel_group is not None and (usp_enabled or direct_cp_enabled): raise ValueError("TP input cannot be combined with USP or direct CP") + if tensor_parallel_group is not None and packing_enabled: + raise ValueError("TP input cannot be combined with packing") self.tensor_parallel_group = tensor_parallel_group self.tensor_parallel_control_group = tensor_parallel_control_group self._tensor_parallel_device = (torch.cuda.current_device() diff --git a/docs/user-guide/megatron.md b/docs/user-guide/megatron.md index 9954d1f..0ebfcdf 100644 --- a/docs/user-guide/megatron.md +++ b/docs/user-guide/megatron.md @@ -286,17 +286,22 @@ the installed ASpec source trees by digest: Torch contain both measurements and their absolute relative difference. Each objective pools the two microbatch numerators and denominators, counting each TP replica once. -The source checkout includes a portable version of that online driver. Run both -roles sequentially with the same pretokenized training and evaluation Parquet -files. Each invocation starts its own managed Target and writes +Run the following Bash commands from the root of the ASpec checkout. Set the +paths to the Draft environments, Target interpreters, Qwen3-8B checkpoint, +pretokenized Parquet files and a fresh output directory. Both backends use the +same training and evaluation files. Each invocation starts its own managed Target and writes `objectives.json` in a fresh directory. The Torch validation plan uses Megatron's TP2 scheduling topology and selects native DTensor operations in the Draft workers. ```bash +torch_env=/absolute/path/to/torch-draft-venv +meg_env=/absolute/path/to/megatron-draft-venv sgl_target=/absolute/path/to/sglang-target-venv/bin/python meg_target=/absolute/path/to/megatron-target-venv/bin/python +model=/absolute/path/to/Qwen3-8B train=/absolute/path/to/train.parquet eval_data=/absolute/path/to/eval.parquet +out=/absolute/path/to/fresh-online-runs export CUDA_DEVICE_MAX_CONNECTIONS=1 for dp in 1 2; do devices=0,1,2,3 diff --git a/tests/cpu/test_megatron.py b/tests/cpu/test_megatron.py index 447ceb9..f980ec8 100644 --- a/tests/cpu/test_megatron.py +++ b/tests/cpu/test_megatron.py @@ -5,7 +5,6 @@ import sys import threading from argparse import Namespace -from datetime import timedelta from pathlib import Path from types import SimpleNamespace @@ -340,44 +339,22 @@ def test_bitwise_equality_and_final_reference_policy(tmp_path): assert reference_state_path(tmp_path, 4, 4, 0) == final -def _tp_input_worker(rank, init_method, use_cuda): - import torch.distributed as dist - +@pytest.mark.parametrize("prefetch_factor", [0, 2]) +def test_tp_input_rejects_packing_before_setup(prefetch_factor): from aspec.training.data_fetcher import MooncakeDataset - dist.init_process_group("gloo", init_method=init_method, rank=rank, world_size=2, - timeout=timedelta(seconds=60)) - try: - dataset = object.__new__(MooncakeDataset) - dataset.tensor_parallel_control_group = dist.group.WORLD - if use_cuda: - torch.cuda.set_device(rank) - group = dist.new_group(backend="nccl") if use_cuda else dist.group.WORLD - dataset.tensor_parallel_group = group - dataset._tp_source_prefetch = None - dataset._tensor_parallel_samples = dataset._tensor_parallel_payload_bytes = 0 - dataset._feature_contract_runtime = None - sample = SimpleNamespace(tensor_shapes={}, tensor_dtypes={}, metadata={}) - dataset._read_tp_source = lambda: (sample, {"ids": torch.tensor([5])} if use_cuda else {}) - _, data = dataset._tensor_parallel_item_on_device() - if use_cuda: - assert data["ids"].tolist() == [5] - assert data["ids"].device.type == "cpu" - - def reject(**kwargs): - raise ValueError("invalid follower metadata") - - dataset._feature_contract_runtime = SimpleNamespace(validate_metadata=reject) - with pytest.raises(RuntimeError, match="rank 1.*invalid follower metadata"): - dataset._tensor_parallel_item_on_device() - finally: - dist.destroy_process_group() - - -@pytest.mark.parametrize("use_cuda", [False, pytest.param(True, marks=pytest.mark.skipif( - torch.cuda.device_count() < 2, reason="requires two GPUs for the NCCL payload"))]) -def test_tp_preparation_error_reaches_both_ranks(tmp_path, use_cuda): - torch.multiprocessing.spawn(_tp_input_worker, args=((tmp_path / "rdzv").as_uri(), use_cuda), nprocs=2) + with pytest.raises(ValueError, match="TP input cannot be combined with packing"): + MooncakeDataset( + ray_queue=None, mooncake_store=None, device=torch.device("cpu"), + tensor_parallel_group=object(), tensor_parallel_control_group=object(), + packing_enabled=True, max_seq_length=16, prefetch_factor=prefetch_factor, + ) + + +def test_tp_preparation_error_reaches_both_ranks(tmp_path): + from tests.tp_input import tp_input_worker + + torch.multiprocessing.spawn(tp_input_worker, args=((tmp_path / "rdzv").as_uri(), False), nprocs=2) def test_rotary_source_change_invalidates_megatron_eval_cache(tmp_path, monkeypatch): diff --git a/tests/test_tp_input_gpu.py b/tests/test_tp_input_gpu.py new file mode 100644 index 0000000..018081f --- /dev/null +++ b/tests/test_tp_input_gpu.py @@ -0,0 +1,17 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +import os + +import pytest +import torch + +from tests.tp_input import tp_input_worker + + +@pytest.mark.skipif( + os.getenv("ASPEC_RUN_GPU_INTEGRATION") != "1" or torch.cuda.device_count() < 2, + reason="requires ASPEC_RUN_GPU_INTEGRATION=1 and two visible GPUs", +) +def test_tp_preparation_error_reaches_both_ranks_nccl(tmp_path): + torch.multiprocessing.spawn(tp_input_worker, args=((tmp_path / "rdzv").as_uri(), True), nprocs=2) diff --git a/tests/tp_input.py b/tests/tp_input.py new file mode 100644 index 0000000..d77efc8 --- /dev/null +++ b/tests/tp_input.py @@ -0,0 +1,42 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +from datetime import timedelta +from types import SimpleNamespace + +import pytest +import torch + + +def tp_input_worker(rank, init_method, use_cuda): + import torch.distributed as dist + + from aspec.training.data_fetcher import MooncakeDataset + + dist.init_process_group("gloo", init_method=init_method, rank=rank, world_size=2, + timeout=timedelta(seconds=60)) + try: + dataset = object.__new__(MooncakeDataset) + dataset.tensor_parallel_control_group = dist.group.WORLD + if use_cuda: + torch.cuda.set_device(rank) + group = dist.new_group(backend="nccl") if use_cuda else dist.group.WORLD + dataset.tensor_parallel_group = group + dataset._tp_source_prefetch = None + dataset._tensor_parallel_samples = dataset._tensor_parallel_payload_bytes = 0 + dataset._feature_contract_runtime = None + sample = SimpleNamespace(tensor_shapes={}, tensor_dtypes={}, metadata={}) + dataset._read_tp_source = lambda: (sample, {"ids": torch.tensor([5])} if use_cuda else {}) + _, data = dataset._tensor_parallel_item_on_device() + if use_cuda: + assert data["ids"].tolist() == [5] + assert data["ids"].device.type == "cpu" + + def reject(**kwargs): + raise ValueError("invalid follower metadata") + + dataset._feature_contract_runtime = SimpleNamespace(validate_metadata=reject) + with pytest.raises(RuntimeError, match="rank 1.*invalid follower metadata"): + dataset._tensor_parallel_item_on_device() + finally: + dist.destroy_process_group() From 864afcb23d8af3ac2f685e1c885da02256c7498d Mon Sep 17 00:00:00 2001 From: Longxmas <1185267696@qq.com> Date: Fri, 20 Nov 2026 14:56:59 +0800 Subject: [PATCH 5/5] Fix Megatron source ring admission and disabled confidence gradients --- aspec/config/train_config.py | 4 ++++ aspec/inference/concurrency.py | 16 +++++++------- aspec/planning.py | 4 ++++ aspec/training/megatron_trainer.py | 3 +++ docs/user-guide/megatron.md | 3 +-- tests/cpu/test_megatron.py | 34 +++++++++++++++++++++++++++++- 6 files changed, 53 insertions(+), 11 deletions(-) diff --git a/aspec/config/train_config.py b/aspec/config/train_config.py index c98d5e7..1eec42f 100644 --- a/aspec/config/train_config.py +++ b/aspec/config/train_config.py @@ -982,6 +982,10 @@ def _validate_execution_capabilities(config: DictConfig, *, source_context_paral engine_count=engine_count, source_slots=source_slots, slots_per_sample=slots_per_sample, + publication_batch_size=( + config.inference.inference_batch_size + if config.inference.inference_engine_type == "megatron" else 1 + ), ) if resident_sample_capacity < dispatch_batch_size: raise ValueError( diff --git a/aspec/inference/concurrency.py b/aspec/inference/concurrency.py index 95cb56c..579d310 100644 --- a/aspec/inference/concurrency.py +++ b/aspec/inference/concurrency.py @@ -46,19 +46,19 @@ def effective_engine_concurrency( def direct_output_sample_capacity( - *, engine_count: int, source_slots: int, slots_per_sample: int = 1 + *, engine_count: int, source_slots: int, slots_per_sample: int = 1, + publication_batch_size: int = 1, ) -> int: - """Return how many direct-output samples can remain resident at once. + """Return the sample capacity available for a complete Draft dispatch. - A Target engine may own more than one live sample as long as its - generation-checked source ring has enough slots. Source diversity helps - balance releases, but one distinct engine per Draft rank is not required. + With atomic publication, a blocked Target can hold a partial batch of + up to publication_batch_size - 1 unpublished samples per engine. """ - if engine_count < 1 or source_slots < 1 or slots_per_sample < 1: + if min(engine_count, source_slots, slots_per_sample, publication_batch_size) < 1: raise ValueError( - "engine count, source slots, and slots per sample must be positive" + "engine count, source slots, slots per sample, and publication batch size must be positive" ) - return engine_count * (source_slots // slots_per_sample) + return engine_count * max(0, source_slots // slots_per_sample - publication_batch_size + 1) def required_target_prefill_tokens( diff --git a/aspec/planning.py b/aspec/planning.py index cd70bf7..b8ac5d8 100644 --- a/aspec/planning.py +++ b/aspec/planning.py @@ -1296,6 +1296,10 @@ def _validate_scope(args: argparse.Namespace, draft: Mapping[str, Any]) -> None: engine_count=engine_count, source_slots=int(source_slots), slots_per_sample=slots_per_sample, + publication_batch_size=( + args.inference_batch_size + if getattr(args, "inference_engine_type", "sgl") == "megatron" else 1 + ), ) packing_supply_threshold = 0 if getattr(args, "dflash_packing", False): diff --git a/aspec/training/megatron_trainer.py b/aspec/training/megatron_trainer.py index 209253b..a129baa 100644 --- a/aspec/training/megatron_trainer.py +++ b/aspec/training/megatron_trainer.py @@ -62,6 +62,9 @@ def _loads_initial_weights_here(self): return True def _build_training_wrapper(self, draft_model): + # Core DDP expects gradient-ready hooks for every trainable bucket member. + if draft_model.confidence_head is not None and self.confidence_head_alpha <= 0: + draft_model.confidence_head.requires_grad_(False) wrapper = super()._build_training_wrapper(draft_model) sample = wrapper._sample_anchor_positions wrapper._aspec_anchor_tp_broadcasts = 0 diff --git a/docs/user-guide/megatron.md b/docs/user-guide/megatron.md index 0ebfcdf..fc2e868 100644 --- a/docs/user-guide/megatron.md +++ b/docs/user-guide/megatron.md @@ -76,8 +76,7 @@ They preserve the BF16 residual stream across fused add/RMSNorm boundaries, the Q/K normalization reduction order, RoPE, causal FA3 and fused SwiGLU. At an auxiliary MLP boundary the TP leader adds the pending branch and residual directly into the transport slot. The final feature uses the checkpoint's RMSNorm. -Each request goes through the same Core model forward. Each engine processes one -sequence at a time. +Each request goes through the same Core model forward. Each engine processes one sequence at a time and publishes the batch together. For one Draft dispatch of `D` samples across `E` Target engines, configure at least `inference_batch_size - 1 + ceil(D / E)` source ring slots per engine. This leaves room for a complete Draft dispatch while an unfinished Target batch holds slots. Megatron Draft partitions attention heads and dense FFN channels using public Core TP projections, and uses Core DDP gradient buffers with ASpec's FP32-master diff --git a/tests/cpu/test_megatron.py b/tests/cpu/test_megatron.py index f980ec8..77e9914 100644 --- a/tests/cpu/test_megatron.py +++ b/tests/cpu/test_megatron.py @@ -16,7 +16,7 @@ from aspec.config.train_config import load_config from aspec.feature_contract import build_target_feature_contract, validate_feature_contract from aspec.models.megatron import qwen3_parameter -from aspec.planning import compile_execution_plan +from aspec.planning import PlanCompileError, compile_execution_plan def test_baseline_import_does_not_require_megatron(): @@ -116,6 +116,26 @@ def test_megatron_fails_closed_before_process_launch(overrides, message): validate_megatron_execution(settings(**overrides), {"family": "dspark"}) +@pytest.mark.parametrize("alpha", [0.0, 1.0]) +def test_megatron_freezes_disabled_confidence_head(monkeypatch, alpha): + from aspec.training.dspark_trainer import DSparkTrainer + from aspec.training.megatron_trainer import MegatronDSparkTrainer + + trainer = object.__new__(MegatronDSparkTrainer) + trainer.confidence_head_alpha = alpha + draft = torch.nn.Module() + draft.confidence_head = torch.nn.Linear(2, 1) + draft.markov_head = torch.nn.Linear(2, 2) + + def build_wrapper(_trainer, model): + assert all(p.requires_grad == (alpha > 0) for p in model.confidence_head.parameters()) + assert all(p.requires_grad for p in model.markov_head.parameters()) + return SimpleNamespace(_sample_anchor_positions=lambda: None) + + monkeypatch.setattr(DSparkTrainer, "_build_training_wrapper", build_wrapper) + trainer._build_training_wrapper(draft) + + def test_producer_backend_is_part_of_the_feature_identity(): kwargs = dict(drafter_family="dspark", target_hidden_size=16, target_num_hidden_layers=4, aux_hidden_state_layers=[0, 2], store_last_hidden_states=True) @@ -144,6 +164,18 @@ def test_tp_source_capacity_counts_dp_replicas_and_target_engines(): with pytest.raises(ValueError, match="cannot retain one complete Draft dispatch"): load_config(base_config=config, cli_args=["mooncake.source_gpu_ring_slots=1"]) + # A second batch can stage two samples while the first leaves one ready. + # With only three slots, the next complete Draft dispatch cannot be fed. + config = load_config(base_config=config, cli_args=[ + "inference.inference_batch_size=3", "mooncake.source_gpu_ring_slots=4", + ]) + assert compile_execution_plan(config).to_runtime_args().dp_size == 2 + with pytest.raises(ValueError, match="capacity=1"): + load_config(base_config=config, cli_args=["mooncake.source_gpu_ring_slots=3"]) + config.mooncake.source_gpu_ring_slots = 3 + with pytest.raises(PlanCompileError, match="capacity=1"): + compile_execution_plan(config) + def test_tp_receive_slot_remains_owned_until_consumer_releases_it(monkeypatch): from aspec.training.data_fetcher import MooncakeDataset, _release_p2p_receive_leases