From 388e010bf2b03158c20e398f4a48aafc01d9c1a4 Mon Sep 17 00:00:00 2001 From: Zhipeng Wang <7927074+PKUWZP@users.noreply.github.com> Date: Fri, 20 Nov 2026 01:03:07 +0000 Subject: [PATCH 1/2] Add a subchannel FP8/INT8 quantized training example Companion to deepspeedai/DeepSpeed#9131: continues training a causal LM (dense or AutoEP MoE) on WikiText-103 under BF16 and several quantization recipes, and reports validation loss, step time and peak memory. Co-Authored-By: Claude Fable 5.2 Signed-off-by: Zhipeng Wang <7927074+PKUWZP@users.noreply.github.com> --- training/subchannel_quantization/README.md | 107 +++++++++++ .../subchannel_quantization/requirements.txt | 6 + training/subchannel_quantization/train.py | 173 ++++++++++++++++++ 3 files changed, 286 insertions(+) create mode 100644 training/subchannel_quantization/README.md create mode 100644 training/subchannel_quantization/requirements.txt create mode 100644 training/subchannel_quantization/train.py diff --git a/training/subchannel_quantization/README.md b/training/subchannel_quantization/README.md new file mode 100644 index 000000000..903051598 --- /dev/null +++ b/training/subchannel_quantization/README.md @@ -0,0 +1,107 @@ +# Subchannel FP8/INT8 Quantized Training Example + +DeepSpeed's `quantization.subchannel` config section runs the matrix multiplications of `nn.Linear` layers and AutoEP grouped experts in FP8 or INT8 instead of BF16, with no model code changes. +This example continues training a pretrained causal LM on WikiText-103 under several quantization recipes, so that validation loss, step time and peak memory can be compared with a BF16 baseline. + +The feature is proposed in [DeepSpeed issue #9132](https://github.com/deepspeedai/DeepSpeed/issues/9132) and implemented in [PR #9131](https://github.com/deepspeedai/DeepSpeed/pull/9131), which has not been merged. See the tutorial in that PR (`docs/_tutorials/subchannel-quantized-training.md`) for what each setting does. + +## Prerequisites + +- An NVIDIA GPU of compute capability 8.9 or newer (for example L40S, H100, H200) for the FP8 kernels. On other devices the GEMMs fall back to a PyTorch reference implementation that reproduces the numerics but is not meant for speed. +- `pip install -r requirements.txt`, which installs DeepSpeed from the tip of PR #9131. +- For MoE models, a GPU count that divides the model's expert count (the script sets `autoep_size` to the GPU count). + +## Recipes + +`--recipe` selects what goes into the `quantization` section of the DeepSpeed config: + +| Recipe | Format | Scaling | Purpose | +| ------ | ------ | ------- | ------- | +| `bf16` | none | none | Baseline: no `quantization` section. | +| `e4m3_row` | FP8 e4m3 | one scale per operand row (`block_size: 0`) | The fast mode. Faster than BF16 on wide layers only. | +| `e4m3` | FP8 e4m3 | 128-element blocks | The feature's defaults. Closer to BF16 in loss, slower than BF16. | +| `int8` | INT8 | 128-element blocks | INT8 operands. | +| `int8_h128` | INT8 | 128-element blocks, Hadamard rotation of 128 | INT8 with outliers spread across each block. | + +All quantized recipes round to nearest in the forward GEMM and use stochastic rounding in the two gradient GEMMs. For example, `e4m3_row` produces: + +```json +"quantization": { + "subchannel": { + "enabled": true, + "forward": {"block_size": 0, "stochastic_rounding": false}, + "dx": {"block_size": 0, "stochastic_rounding": true}, + "dw": {"block_size": 0, "stochastic_rounding": true} + } +} +``` + +## Run + +Launch one process per GPU with `torchrun`. Run the `bf16` recipe first, then any quantized recipe with the same arguments. + +A dense model with wide layers, where row-scaled FP8 is faster than BF16: + +```bash +for recipe in bf16 e4m3_row e4m3; do + torchrun --nproc_per_node 4 train.py --recipe $recipe \ + --model Qwen/Qwen2.5-14B --dense --grad_checkpointing --bf16_states \ + --steps 100 --eval_every 50 --seq_len 2048 --batch_size 4 --eval_blocks 32 \ + --out results/qwen2.5-14b_$recipe.json +done +``` + +An MoE model with AutoEP, where the quantized recipes match BF16 in loss but are slower: + +```bash +for recipe in bf16 e4m3_row e4m3 int8 int8_h128; do + torchrun --nproc_per_node 4 train.py --recipe $recipe \ + --model Qwen/Qwen1.5-MoE-A2.7B \ + --out results/qwen1.5-moe_$recipe.json +done +``` + +Each run prints validation loss at every `--eval_every` steps and, at the end, the median step time and peak GPU memory. The same numbers are written to the `--out` JSON file together with the per-step training loss and step times. + +Options: + +- `--dense`: the model is not an MoE model; train with plain data parallelism instead of AutoEP. +- `--grad_checkpointing`: enable gradient checkpointing. Needed to fit wide dense models at thousands of tokens per step. +- `--bf16_states`: keep master weights and optimizer state in BF16. Needed to fit models of roughly 14B parameters and up on 141 GiB GPUs. +- `--zero_stage`: ZeRO stage (default 2). Use 3 when parameters plus gradients do not fit on one GPU. +- `--steps`, `--eval_every`, `--seq_len`, `--batch_size`, `--eval_blocks`, `--lr`: training and evaluation sizes. + +The script reads at most the first 60,000 rows of WikiText-103, about enough for 100 steps at 4 GPUs x batch 4 x sequence 2048. Lower `--steps` or the batch shape if you raise any of these. + +## Reference results + +Measured on 4x NVIDIA H200 (driver 595.112.07) with torch 2.8.0+cu129, Triton 3.4.0 and transformers 5.23.0, one run per configuration. + +Dense models, with the first command above (Qwen2.5-32B additionally with `--zero_stage 3`): + +| Model | Recipe | Val loss @100 | Step p50 | vs BF16 | Peak memory | +| ----- | ------ | ------------- | -------- | ------- | ----------- | +| Qwen2.5-14B | `bf16` | 1.6783 | 2145 ms | 1.00x | 66.2 GiB | +| | `e4m3_row` | 1.6839 | 1965 ms | 1.09x | 66.2 GiB | +| | `e4m3` | 1.6816 | 2896 ms | 0.74x | 66.2 GiB | +| Qwen2.5-32B | `bf16` | 1.6561 | 4594 ms | 1.00x | 82.2 GiB | +| | `e4m3_row` | 1.6604 | 3801 ms | 1.21x | 82.2 GiB | +| | `e4m3` | 1.6584 | 6175 ms | 0.74x | 82.2 GiB | + +MoE models, 300 steps at the default batch 4 x sequence 512 (Qwen3-30B-A3B additionally with `--bf16_states`): + +| Model | Recipe | Val loss @300 | Step p50 | Peak memory | +| ----- | ------ | ------------- | -------- | ----------- | +| Qwen1.5-MoE-A2.7B | `bf16` | 1.8900 | 341 ms | 64.5 GiB | +| | `e4m3_row` | 1.8941 | 518 ms | 64.5 GiB | +| | `e4m3` | 1.8923 | 500 ms | 64.5 GiB | +| Qwen3-30B-A3B | `bf16` | 1.9084 | 622 ms | 91.8 GiB | +| | `e4m3_row` | 1.9116 | 947 ms | 91.8 GiB | +| | `e4m3` | 1.9118 | 789 ms | 91.8 GiB | + +What to expect: + +- Row-scaled FP8 beats BF16 only when layers are wide (about 5000 and up) and each GPU processes several thousand tokens per step. On the MoE models above, whose layers are about 2048 wide, it is about 1.5x slower. +- Block-scaled recipes are slower than BF16 on every model measured. +- Peak memory does not change: only the GEMM operands are quantized, on the fly. +- These are short continued-pretraining runs. They show that loss tracks the baseline over a few hundred steps, not final model quality. diff --git a/training/subchannel_quantization/requirements.txt b/training/subchannel_quantization/requirements.txt new file mode 100644 index 000000000..8c5df25cd --- /dev/null +++ b/training/subchannel_quantization/requirements.txt @@ -0,0 +1,6 @@ +# Subchannel quantization is not merged yet: install the tip of DeepSpeed PR #9131. +deepspeed @ git+https://github.com/deepspeedai/DeepSpeed.git@refs/pull/9131/head +torch>=2.8 +triton>=3.4 +transformers>=5.2 +datasets>=2.14.0 diff --git a/training/subchannel_quantization/train.py b/training/subchannel_quantization/train.py new file mode 100644 index 000000000..5cd63177b --- /dev/null +++ b/training/subchannel_quantization/train.py @@ -0,0 +1,173 @@ +# SPDX-License-Identifier: Apache-2.0 +# DeepSpeed Team +"""Continue training a causal LM on WikiText-103 with or without DeepSpeed subchannel FP8/INT8 quantization. + +Every recipe sees the same data in the same order and starts from the same pretrained weights, so validation loss, +step time and peak memory can be compared against the ``bf16`` baseline. See README.md for launch commands. +""" +import argparse +import json +import math +import os +import time + +import torch + +import deepspeed +import deepspeed.comm as dist +from deepspeed.accelerator import get_accelerator +from datasets import load_dataset +from transformers import AutoModelForCausalLM, AutoTokenizer + +RECIPES = { + "bf16": None, + # The feature's defaults: e4m3, 128-wide blocks, nearest rounding forward, stochastic rounding for dx and dw. + "e4m3": {}, + # One scale per operand row (per expert in dw): the fast FP8 path. + "e4m3_row": {name: { + "block_size": 0, + "stochastic_rounding": name != "forward" + } + for name in ("forward", "dx", "dw")}, + "int8": {name: { + "format": "int8", + "stochastic_rounding": name != "forward" + } + for name in ("forward", "dx", "dw")}, + "int8_h128": { + name: { + "format": "int8", + "hadamard_size": 128, + "stochastic_rounding": name != "forward" + } + for name in ("forward", "dx", "dw") + }, +} + + +def token_blocks(tokenizer, split, seq_len, max_blocks): + text = "\n\n".join(row for row in load_dataset("Salesforce/wikitext", "wikitext-103-raw-v1", split=split)["text"][:60000] + if row.strip()) + ids = tokenizer(text[:max_blocks * seq_len * 8], return_tensors="pt").input_ids[0] + count = min(max_blocks, ids.numel() // seq_len) + return ids[:count * seq_len].view(count, seq_len) + + +def global_mean(value, device): + tensor = torch.tensor([value], dtype=torch.float64, device=device) + dist.all_reduce(tensor) + return tensor.item() / dist.get_world_size() + + +def evaluate(engine, blocks, batch_size): + engine.eval() + rank, world = dist.get_rank(), dist.get_world_size() + local = blocks[rank::world] + total = 0.0 + with torch.no_grad(): + for start in range(0, local.shape[0], batch_size): + ids = local[start:start + batch_size].to(engine.device) + total += engine(input_ids=ids, labels=ids).loss.item() * ids.shape[0] + engine.train() + return global_mean(total / local.shape[0], engine.device) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--recipe", choices=RECIPES, required=True) + parser.add_argument("--model", required=True, help="Hugging Face model id or local path") + parser.add_argument("--out", required=True) + parser.add_argument("--steps", type=int, default=300) + parser.add_argument("--eval_every", type=int, default=100) + parser.add_argument("--seq_len", type=int, default=512) + parser.add_argument("--batch_size", type=int, default=4) + parser.add_argument("--eval_blocks", type=int, default=128) + parser.add_argument("--lr", type=float, default=1e-5) + parser.add_argument("--bf16_states", action="store_true", help="bf16 master weights and optimizer states") + parser.add_argument("--zero_stage", type=int, default=2) + parser.add_argument("--dense", action="store_true", help="not an MoE model: plain data parallelism, no AutoEP") + parser.add_argument("--grad_checkpointing", action="store_true", help="trade a second forward for memory") + args = parser.parse_args() + + torch.manual_seed(0) + world = int(os.environ.get("WORLD_SIZE", 1)) + config = { + "train_micro_batch_size_per_gpu": args.batch_size, + "bf16": { + "enabled": True + }, + "zero_optimization": { + "stage": args.zero_stage + }, + "gradient_clipping": 1.0, + "optimizer": { + "type": "AdamW", + "params": { + "lr": args.lr, + "weight_decay": 0.0, + "torch_adam": True + } + }, + "expert_parallel": { + "enabled": True, + "autoep_size": world + }, + } + if args.dense: + del config["expert_parallel"] + if args.bf16_states: + config["bf16"].update(bf16_master_weights_and_grads=True, bf16_optimizer_states=True) + if RECIPES[args.recipe] is not None: + config["quantization"] = {"subchannel": {"enabled": True, **RECIPES[args.recipe]}} + + tokenizer = AutoTokenizer.from_pretrained(args.model) + model = AutoModelForCausalLM.from_pretrained(args.model, dtype=torch.bfloat16) + if args.grad_checkpointing: + model.gradient_checkpointing_enable() + engine, _, _, _ = deepspeed.initialize(model=model, config=config) + rank = dist.get_rank() + + train = token_blocks(tokenizer, "train", args.seq_len, args.steps * args.batch_size * world) + val = token_blocks(tokenizer, "validation", args.seq_len, args.eval_blocks) + + record = {"recipe": args.recipe, "args": vars(args), "world_size": world, "train_loss": [], "eval": {}, "step_ms": []} + record["eval"][0] = evaluate(engine, val, args.batch_size) + if rank == 0: + print(f"[{args.recipe}] step 0 eval loss {record['eval'][0]:.4f} ppl {math.exp(record['eval'][0]):.3f}", flush=True) + + get_accelerator().reset_peak_memory_stats() + for step in range(args.steps): + first = (step * world + rank) * args.batch_size + ids = train[first:first + args.batch_size].to(engine.device) + get_accelerator().synchronize() + start = time.perf_counter() + loss = engine(input_ids=ids, labels=ids).loss + engine.backward(loss) + engine.step() + get_accelerator().synchronize() + record["step_ms"].append((time.perf_counter() - start) * 1e3) + record["train_loss"].append(global_mean(loss.item(), engine.device)) + if rank == 0 and (step + 1) % 20 == 0: + recent = record["train_loss"][-20:] + print(f"[{args.recipe}] step {step + 1} train loss {sum(recent) / len(recent):.4f} " + f"step {sorted(record['step_ms'][-20:])[10]:.0f} ms", flush=True) + if (step + 1) % args.eval_every == 0: + record["eval"][step + 1] = evaluate(engine, val, args.batch_size) + if rank == 0: + loss_value = record["eval"][step + 1] + print(f"[{args.recipe}] step {step + 1} eval loss {loss_value:.4f} ppl {math.exp(loss_value):.3f}", + flush=True) + + record["peak_mem_gib"] = get_accelerator().max_memory_allocated() / 2**30 + steady = sorted(record["step_ms"][min(10, len(record["step_ms"]) // 2):]) + record["step_ms_p50"] = steady[len(steady) // 2] + if rank == 0: + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(record, handle) + print(f"[{args.recipe}] done: p50 step {record['step_ms_p50']:.0f} ms, peak {record['peak_mem_gib']:.1f} GiB", + flush=True) + + +if __name__ == "__main__": + main() From facf46e17a0d53c6b10300987a53177189231930 Mon Sep 17 00:00:00 2001 From: Zhipeng Wang <7927074+PKUWZP@users.noreply.github.com> Date: Fri, 20 Nov 2026 20:44:13 +0000 Subject: [PATCH 2/2] Add a from-scratch convergence script to the subchannel quantization example from_scratch.py trains a Llama-architecture model from random initialization on all of WikiText-103 with a warmup + cosine schedule and reports final validation and test loss, so FP8 recipes can be compared with BF16 at convergence. The README gains the launch commands and two-seed reference results. Co-Authored-By: Claude Fable 5.2 Signed-off-by: Zhipeng Wang <7927074+PKUWZP@users.noreply.github.com> --- training/subchannel_quantization/README.md | 53 +++- .../subchannel_quantization/from_scratch.py | 255 ++++++++++++++++++ 2 files changed, 306 insertions(+), 2 deletions(-) create mode 100644 training/subchannel_quantization/from_scratch.py diff --git a/training/subchannel_quantization/README.md b/training/subchannel_quantization/README.md index 903051598..597491a1b 100644 --- a/training/subchannel_quantization/README.md +++ b/training/subchannel_quantization/README.md @@ -2,6 +2,7 @@ DeepSpeed's `quantization.subchannel` config section runs the matrix multiplications of `nn.Linear` layers and AutoEP grouped experts in FP8 or INT8 instead of BF16, with no model code changes. This example continues training a pretrained causal LM on WikiText-103 under several quantization recipes, so that validation loss, step time and peak memory can be compared with a BF16 baseline. +A second script, `from_scratch.py`, trains a model from random initialization to convergence to compare final quality. The feature is proposed in [DeepSpeed issue #9132](https://github.com/deepspeedai/DeepSpeed/issues/9132) and implemented in [PR #9131](https://github.com/deepspeedai/DeepSpeed/pull/9131), which has not been merged. See the tutorial in that PR (`docs/_tutorials/subchannel-quantized-training.md`) for what each setting does. @@ -73,9 +74,44 @@ Options: The script reads at most the first 60,000 rows of WikiText-103, about enough for 100 steps at 4 GPUs x batch 4 x sequence 2048. Lower `--steps` or the batch shape if you raise any of these. +## Full training from scratch + +`from_scratch.py` trains a Llama-architecture model built from a config (default: 8 layers, hidden 4096, MLP 11008, 1.82B parameters, GPT-2 tokenizer) on all of WikiText-103 with a warmup + cosine schedule that decays to zero, and reports final validation and test loss. +It supports the `bf16`, `e4m3_row` and `e4m3` recipes and plain data parallelism under ZeRO-2. + +Tokenize the dataset once, in a single process: + +```bash +python from_scratch.py --prep +``` + +Then train each recipe with the same `--seed`, which fixes both the initial weights and the data order: + +```bash +for seed in 0 1; do + for recipe in bf16 e4m3_row e4m3; do + torchrun --nproc_per_node 2 from_scratch.py --recipe $recipe --seed $seed \ + --out results/fs_${recipe}_s${seed}.json + done +done +``` + +The defaults are 4 epochs (28,788 steps on 2 GPUs), batch 4 x sequence 2048 per GPU, AdamW with peak lr 2e-4, 1000 warmup steps, weight decay 0.1 and gradient clipping 1.0. +One run takes about 2.5 to 3 hours on 2 H200s and needs about 30 GiB per GPU. +Validation loss is evaluated and the `--out` file rewritten every `--eval_every` steps (default 500). + +Options: + +- `--seed`, `--epochs`, `--lr`, `--warmup`, `--seq_len`, `--batch_size`, `--eval_every`: training schedule and sizes. +- `--layers`, `--hidden`, `--intermediate`: model shape. The head count is `hidden / 128`. +- `--max_steps`: stop early without changing the schedule, for timing runs. +- `--data_dir`: where `--prep` writes the tokenized splits (default `data/` next to the script). + +The number of steps per epoch, and so the schedule, depends on the GPU count. + ## Reference results -Measured on 4x NVIDIA H200 (driver 595.112.07) with torch 2.8.0+cu129, Triton 3.4.0 and transformers 5.23.0, one run per configuration. +Measured on 4x NVIDIA H200 (driver 595.112.07) with torch 2.8.0+cu129, Triton 3.4.0 and transformers 5.23.0. The full-training table has two seeds; the others are one run per configuration. Dense models, with the first command above (Qwen2.5-32B additionally with `--zero_stage 3`): @@ -99,9 +135,22 @@ MoE models, 300 steps at the default batch 4 x sequence 512 (Qwen3-30B-A3B addit | | `e4m3_row` | 1.9116 | 947 ms | 91.8 GiB | | | `e4m3` | 1.9118 | 789 ms | 91.8 GiB | +Full training from scratch with the `from_scratch.py` defaults, 2 GPUs per run, two runs sharing the machine at a time: + +| Recipe | Seed | Final val loss | Test loss | Test ppl | Step p50 | Peak memory | +| ------ | ---- | -------------- | --------- | -------- | -------- | ----------- | +| `bf16` | 0 | 2.9597 | 2.9761 | 19.61 | 323 ms | 29.9 GiB | +| `e4m3_row` | 0 | 2.9557 | 2.9747 | 19.58 | 307 ms | 29.9 GiB | +| `e4m3` | 0 | 2.9518 | 2.9738 | 19.57 | 386 ms | 29.9 GiB | +| `bf16` | 1 | 2.9650 | 2.9860 | 19.81 | 326 ms | 29.9 GiB | +| `e4m3_row` | 1 | 2.9631 | 2.9828 | 19.74 | 305 ms | 29.9 GiB | +| `e4m3` | 1 | 2.9545 | 2.9792 | 19.67 | 386 ms | 29.9 GiB | + What to expect: +- In full training, both FP8 recipes end within seed-to-seed variation of BF16: the two BF16 seeds differ by 0.010 in test loss, and each FP8 run is within 0.007 of BF16 at the same seed. This covers one dense architecture and one dataset; MoE models and the INT8 recipes have not been trained to convergence. + - Row-scaled FP8 beats BF16 only when layers are wide (about 5000 and up) and each GPU processes several thousand tokens per step. On the MoE models above, whose layers are about 2048 wide, it is about 1.5x slower. - Block-scaled recipes are slower than BF16 on every model measured. - Peak memory does not change: only the GEMM operands are quantized, on the fly. -- These are short continued-pretraining runs. They show that loss tracks the baseline over a few hundred steps, not final model quality. +- The `train.py` runs are short continued pretraining. They show that loss tracks the baseline over a few hundred steps, not final model quality. diff --git a/training/subchannel_quantization/from_scratch.py b/training/subchannel_quantization/from_scratch.py new file mode 100644 index 000000000..f9399eea7 --- /dev/null +++ b/training/subchannel_quantization/from_scratch.py @@ -0,0 +1,255 @@ +# SPDX-License-Identifier: Apache-2.0 +# DeepSpeed Team +"""Train a wide Llama-architecture LM from scratch on WikiText-103 to convergence, with or without subchannel FP8. + +For a given seed, every recipe starts from the same initial weights and sees the same data order, so final validation +and test loss can be compared against the ``bf16`` baseline. Run ``python from_scratch.py --prep`` once (single +process) to tokenize the dataset, then launch with torchrun. See README.md. +""" +import argparse +import json +import math +import os +import time + +import torch + +import deepspeed +import deepspeed.comm as dist +from deepspeed.accelerator import get_accelerator + +RECIPES = { + "bf16": None, + "e4m3": {}, + "e4m3_row": { + name: { + "block_size": 0, + "stochastic_rounding": name != "forward" + } + for name in ("forward", "dx", "dw") + }, +} + + +def prep(data_dir): + from datasets import load_dataset + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained("gpt2") + os.makedirs(data_dir, exist_ok=True) + for split in ("train", "validation", "test"): + rows = load_dataset("Salesforce/wikitext", + "wikitext-103-raw-v1", + split=split)["text"] + ids = [] + for start in range(0, len(rows), 20000): + chunk = [row for row in rows[start:start + 20000] if row.strip()] + for encoded in tokenizer(chunk)["input_ids"]: + ids.extend(encoded) + tensor = torch.tensor(ids, dtype=torch.int32) + torch.save(tensor, os.path.join(data_dir, f"{split}.pt")) + print(split, tensor.numel(), "tokens", flush=True) + + +def blocks_of(data_dir, split, seq_len): + ids = torch.load(os.path.join(data_dir, f"{split}.pt")).long() + count = ids.numel() // seq_len + return ids[:count * seq_len].view(count, seq_len) + + +def global_mean(value, device): + tensor = torch.tensor([value], dtype=torch.float64, device=device) + dist.all_reduce(tensor) + return tensor.item() / dist.get_world_size() + + +def evaluate(engine, blocks, batch_size): + engine.eval() + rank, world = dist.get_rank(), dist.get_world_size() + usable = blocks[:blocks.shape[0] // world * world] + local = usable[rank::world] + total = 0.0 + with torch.no_grad(): + for start in range(0, local.shape[0], batch_size): + ids = local[start:start + batch_size].to(engine.device) + total += engine(input_ids=ids, + labels=ids).loss.item() * ids.shape[0] + engine.train() + return global_mean(total / local.shape[0], engine.device) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--prep", + action="store_true", + help="tokenize WikiText-103 into --data_dir and exit") + parser.add_argument("--data_dir", + default=os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data")) + parser.add_argument("--recipe", choices=RECIPES) + parser.add_argument("--out") + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--epochs", type=float, default=4.0) + parser.add_argument( + "--max_steps", + type=int, + default=0, + help="stop early (timing runs); the schedule is unchanged") + parser.add_argument("--eval_every", type=int, default=500) + parser.add_argument("--seq_len", type=int, default=2048) + parser.add_argument("--batch_size", type=int, default=4) + parser.add_argument("--lr", type=float, default=2e-4) + parser.add_argument("--warmup", type=int, default=1000) + parser.add_argument("--layers", type=int, default=8) + parser.add_argument("--hidden", type=int, default=4096) + parser.add_argument("--intermediate", type=int, default=11008) + args = parser.parse_args() + if args.prep: + return prep(args.data_dir) + if args.recipe is None or args.out is None: + parser.error("--recipe and --out are required for training") + + from transformers import LlamaConfig, LlamaForCausalLM + world = int(os.environ.get("WORLD_SIZE", 1)) + train = blocks_of(args.data_dir, "train", args.seq_len) + val = blocks_of(args.data_dir, "validation", args.seq_len) + test = blocks_of(args.data_dir, "test", args.seq_len) + steps_per_epoch = train.shape[0] // (world * args.batch_size) + total_steps = int(args.epochs * steps_per_epoch) + + config = { + "train_micro_batch_size_per_gpu": args.batch_size, + "bf16": { + "enabled": True + }, + "zero_optimization": { + "stage": 2 + }, + "gradient_clipping": 1.0, + "optimizer": { + "type": "AdamW", + "params": { + "lr": args.lr, + "betas": [0.9, 0.95], + "weight_decay": 0.1, + "torch_adam": True + } + }, + "scheduler": { + "type": "WarmupCosineLR", + "params": { + "total_num_steps": total_steps, + "warmup_num_steps": args.warmup, + "warmup_min_ratio": 0.0, + "cos_min_ratio": 0.0 + } + }, + } + if RECIPES[args.recipe] is not None: + config["quantization"] = { + "subchannel": { + "enabled": True, + **RECIPES[args.recipe] + } + } + + torch.manual_seed(args.seed) + model = LlamaForCausalLM( + LlamaConfig(vocab_size=50257, + hidden_size=args.hidden, + intermediate_size=args.intermediate, + num_hidden_layers=args.layers, + num_attention_heads=args.hidden // 128, + num_key_value_heads=args.hidden // 128, + max_position_embeddings=args.seq_len, + tie_word_embeddings=True, + bos_token_id=50256, + eos_token_id=50256)) + engine, _, _, scheduler = deepspeed.initialize(model=model, config=config) + rank = dist.get_rank() + params = sum(p.numel() for p in model.parameters()) + + record = { + "recipe": args.recipe, + "args": vars(args), + "world_size": world, + "params": params, + "steps_per_epoch": steps_per_epoch, + "total_steps": total_steps, + "train_loss": [], + "eval": {}, + "step_ms": [] + } + + def save(): + if rank == 0: + os.makedirs(os.path.dirname(os.path.abspath(args.out)), + exist_ok=True) + with open(args.out, "w") as handle: + json.dump(record, handle) + + if rank == 0: + print( + f"[{args.recipe}] {params / 1e9:.2f}B params, {steps_per_epoch} steps/epoch, {total_steps} steps", + flush=True) + last_step = min(total_steps, + args.max_steps) if args.max_steps else total_steps + window_loss, window_start, perm, epoch = [], None, None, -1 + get_accelerator().reset_peak_memory_stats() + for step in range(last_step): + if step // steps_per_epoch != epoch: + epoch = step // steps_per_epoch + perm = torch.randperm( + train.shape[0], + generator=torch.Generator().manual_seed(1000 * args.seed + + epoch)) + first = ((step % steps_per_epoch) * world + rank) * args.batch_size + ids = train[perm[first:first + args.batch_size]].to(engine.device) + if step % 50 == 0: + get_accelerator().synchronize() + now = time.perf_counter() + if window_start is not None: + record["step_ms"].append((now - window_start) / 50 * 1e3) + record["train_loss"].append( + global_mean( + sum(window_loss) / len(window_loss), engine.device)) + if rank == 0 and step % 250 == 0: + print( + f"[{args.recipe}] step {step} train {record['train_loss'][-1]:.4f} " + f"{record['step_ms'][-1]:.0f} ms/step lr {scheduler.get_last_lr()[0]:.2e}", + flush=True) + window_loss = [] + get_accelerator().synchronize() + window_start = time.perf_counter() + loss = engine(input_ids=ids, labels=ids).loss + engine.backward(loss) + engine.step() + window_loss.append(loss.detach()) + if step % 50 == 49: + window_loss = [float(torch.stack(window_loss).float().mean())] + if (step + 1) % args.eval_every == 0 or step + 1 == last_step: + record["eval"][step + 1] = evaluate(engine, val, args.batch_size) + if rank == 0: + value = record["eval"][step + 1] + print( + f"[{args.recipe}] step {step + 1} (epoch {(step + 1) / steps_per_epoch:.2f}) val loss {value:.4f} " + f"ppl {math.exp(value):.2f}", + flush=True) + save() + window_start = None + window_loss = [] + + record["test_loss"] = evaluate(engine, test, args.batch_size) + record["peak_mem_gib"] = get_accelerator().max_memory_allocated() / 2**30 + steady = sorted(record["step_ms"][1:] or record["step_ms"]) + record["step_ms_p50"] = steady[len(steady) // 2] if steady else None + save() + if rank == 0: + print( + f"[{args.recipe}] done: test loss {record['test_loss']:.4f} ppl {math.exp(record['test_loss']):.2f}, " + f"p50 step {record['step_ms_p50']} ms, peak {record['peak_mem_gib']:.1f} GiB", + flush=True) + + +if __name__ == "__main__": + main()