diff --git a/modelopt_recipes/models/zai-org/GLM-5.3-Flash/ptq/nvfp4_experts_dense_mlp-kv_fp8_cast.yaml b/modelopt_recipes/models/zai-org/GLM-5.3-Flash/ptq/nvfp4_experts_dense_mlp-kv_fp8_cast.yaml new file mode 100644 index 00000000000..36dc00ebe64 --- /dev/null +++ b/modelopt_recipes/models/zai-org/GLM-5.3-Flash/ptq/nvfp4_experts_dense_mlp-kv_fp8_cast.yaml @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Recipe for zai-org/GLM-5.3-Flash: NVFP4 on the routed experts and the dense MLP, FP8 KV cache. +# +# The wider sibling of `nvfp4_experts_only-kv_fp8_cast`. GLM-5.3-Flash is a `glm5_next` VLM +# MoE -- 45 decoder layers, 288 routed experts, and HYBRID attention: KDA (linear-attention) +# layers interleaved with NoPE sparse-MLA layers. It needs transformers >= 5.16.1 (earlier +# releases cannot parse the config). +# +# Quantized: the routed experts AND the dense MLP, W4A4 -- both the weight and the input +# quantizers are NVFP4 -- plus an FP8 KV cache in cast mode. Everything else stays BF16: +# shared experts, the router gate, both attention families, the vision tower, embeddings +# and lm_head. +# +# How little "dense MLP" adds: `mlp_layer_types` marks layers 0-2 `dense` and 3-44 `sparse`, +# so only 3 of the 45 layers have a plain MLP -- 9 modules in total (`mlp.gate_proj`, +# `mlp.up_proj`, `mlp.down_proj` each). The other 42 layers carry `mlp.experts..*`. +# +# The one rule that is not obvious is the trailing `*visual*` disable, and it is load-bearing. +# The vision tower reuses the same leaf names -- `model.visual.blocks..mlp.gate_proj` and +# friends, across 24 blocks -- so the dense-MLP patterns match 144 modules inside it. Entries +# apply in order, so the disable has to come LAST or the vision tower is quantized by accident. +# (The experts-only sibling needs no such rule: `*.experts.*` matches nothing under +# `model.visual.*`.) +# +# Two model-specific notes: +# +# * `layerwise.enable=false` is required, not incidental. This is a VLM, so the decoder +# layers nest under `model.language_model.layers` and layerwise_calibrate cannot find +# them. +# +# * The MTP head is inlined as decoder layer 45 and is NOT quantized. The config declares +# `num_hidden_layers: 45`, so transformers builds layers 0-44 and never instantiates +# layer 45; no quantizer attaches to it and it stays BF16. +# +# Like the sibling, the shared `default_disabled_quantizers` unit is deliberately not imported. +# Of its patterns only `*visual*` changes anything here; every other one either matches no +# module in this model or matches a module `base_disable_all` already left off (`lm_head`, the +# `mlp.gate.` routers), and none of them are re-enabled below. Two patterns that look riskier +# than they are: `*.experts.*` needs a literal `.experts.`, so `mlp.shared_experts.*` does not +# match; and `*mlp.gate_proj*` needs `mlp.gate_proj`, which the router `mlp.gate` lacks. + +imports: + base_disable_all: configs/ptq/units/base_disable_all + nvfp4: configs/numerics/nvfp4 + kv_fp8_cast: configs/ptq/units/kv_fp8_cast + +metadata: + recipe_type: ptq + description: >- + GLM-5.3-Flash: NVFP4 (W4A4) on the routed experts and on the dense MLP of layers 0-2, + plus an FP8 KV cache in cast mode using constant amax; max calibration. Shared experts, + router gate, KDA and MLA attention, vision tower, embeddings, lm_head and the inlined + MTP layer all stay BF16. +quantize: + algorithm: + method: max + # Max calibration is fast and does not typically need checkpointing. + # layerwise.enable=false required for VLMs where the decoder layers are nested under + # `model.language_model.layers` (layerwise_calibrate can't find them otherwise). + layerwise: {enable: false} + quant_cfg: + - $import: base_disable_all + # Routed experts: layers 3-44, 288 per layer. + - quantizer_name: '*.experts.*weight_quantizer' + cfg: + $import: nvfp4 + - quantizer_name: '*.experts.*input_quantizer' + cfg: + $import: nvfp4 + # Dense MLP: layers 0-2 -- and, until the last entry undoes it, the vision tower. + - quantizer_name: '*mlp.gate_proj*weight_quantizer' + cfg: + $import: nvfp4 + - quantizer_name: '*mlp.gate_proj*input_quantizer' + cfg: + $import: nvfp4 + - quantizer_name: '*mlp.up_proj*weight_quantizer' + cfg: + $import: nvfp4 + - quantizer_name: '*mlp.up_proj*input_quantizer' + cfg: + $import: nvfp4 + - quantizer_name: '*mlp.down_proj*weight_quantizer' + cfg: + $import: nvfp4 + - quantizer_name: '*mlp.down_proj*input_quantizer' + cfg: + $import: nvfp4 + - $import: kv_fp8_cast + # MUST stay last: keeps the vision tower BF16 after the dense-MLP patterns matched it. + - {quantizer_name: '*visual*', enable: false} diff --git a/modelopt_recipes/models/zai-org/GLM-5.3-Flash/ptq/nvfp4_experts_only-kv_fp8_cast.yaml b/modelopt_recipes/models/zai-org/GLM-5.3-Flash/ptq/nvfp4_experts_only-kv_fp8_cast.yaml new file mode 100644 index 00000000000..5b61c83ff95 --- /dev/null +++ b/modelopt_recipes/models/zai-org/GLM-5.3-Flash/ptq/nvfp4_experts_only-kv_fp8_cast.yaml @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Recipe for zai-org/GLM-5.3-Flash: NVFP4 on the routed experts only, FP8 KV cache. +# +# GLM-5.3-Flash is a `glm5_next` VLM MoE -- 45 decoder layers, 288 routed experts, and +# HYBRID attention: KDA (linear-attention) layers interleaved with NoPE sparse-MLA +# layers. It needs transformers >= 5.16.1 (earlier releases cannot parse the config). +# +# Quantized: the routed experts only, W4A4 -- both the weight and the input quantizers +# are NVFP4 -- plus an FP8 KV cache in cast mode. Everything else stays BF16: the dense +# MLP, shared experts, the router gate, both attention families, the vision tower, +# embeddings and lm_head. +# +# The quant_cfg is deliberately minimal. `base_disable_all` sets every quantizer to +# `enable: false`, and the only entries after it re-enable the routed experts and the KV +# cache -- so nothing else needs listing. In particular the shared `default_disabled_quantizers` +# unit is not imported: every pattern it turns off (router gates, `lm_head`, the linear-attn +# `conv1d` and `in_proj_a`/`in_proj_b`, ...) is already off and is not matched by +# `*.experts.*`. Note `*.experts.*` requires a literal `.experts.`, so `mlp.shared_experts.*` +# does not match it and the shared experts stay BF16. The same literal is why the vision tower +# needs no explicit disable here: it is a dense ViT whose 224 modules carry no `experts` in any +# name, so `*.experts.*` cannot reach `model.visual.*`. The wider +# `nvfp4_experts_dense_mlp-kv_fp8_cast` sibling does need one -- its `*mlp.*_proj*` patterns +# match the vision tower's identically named `gate_proj`/`up_proj`/`down_proj` leaves. +# +# Two model-specific notes: +# +# * `layerwise.enable=false` is required, not incidental. This is a VLM, so the decoder +# layers nest under `model.language_model.layers` and layerwise_calibrate cannot find +# them. +# +# * The MTP head is inlined as decoder layer 45 and is NOT quantized. The config declares +# `num_hidden_layers: 45`, so transformers builds layers 0-44 and never instantiates +# layer 45; no quantizer attaches to it and it stays BF16. + +imports: + base_disable_all: configs/ptq/units/base_disable_all + nvfp4: configs/numerics/nvfp4 + kv_fp8_cast: configs/ptq/units/kv_fp8_cast + +metadata: + recipe_type: ptq + description: >- + GLM-5.3-Flash: NVFP4 (W4A4) on the routed experts only, plus an FP8 KV cache in cast + mode using constant amax; max calibration. The dense MLP, shared experts, router gate, + KDA and MLA attention, vision tower, embeddings, lm_head and the inlined MTP layer all + stay BF16. +quantize: + algorithm: + method: max + # Max calibration is fast and does not typically need checkpointing. + # layerwise.enable=false required for VLMs where the decoder layers are nested under + # `model.language_model.layers` (layerwise_calibrate can't find them otherwise). + layerwise: {enable: false} + quant_cfg: + - $import: base_disable_all + - quantizer_name: '*.experts.*weight_quantizer' + cfg: + $import: nvfp4 + - quantizer_name: '*.experts.*input_quantizer' + cfg: + $import: nvfp4 + - $import: kv_fp8_cast diff --git a/modelopt_recipes/ptq.md b/modelopt_recipes/ptq.md index 5e6b332de1c..ec4a6955795 100644 --- a/modelopt_recipes/ptq.md +++ b/modelopt_recipes/ptq.md @@ -406,6 +406,20 @@ checkpoint's** quant config verbatim: Q4_K/Q5_0 linears → NVFP4 W4A4 (attention q/k/v/o kept uniform so export can fuse them), the Q6_K MLP `down_proj` layers → FP8 W8A8, embeddings → NVFP4 W4A16, `lm_head` → FP8 W8A16, and the F32 tensors (conv1d, norms) → BF16. +- **`models/zai-org/GLM-5.3-Flash/ptq/nvfp4_experts_only-kv_fp8_cast`** is the + NVFP4 config for `zai-org/GLM-5.3-Flash`, a `glm5_next` VLM MoE with **hybrid + attention** — KDA (linear-attention) layers interleaved with NoPE sparse-MLA + layers. Routed experts → NVFP4 W4A4; KV cache → FP8 cast; everything else, + including the MTP block inlined as decoder layer 45, stays BF16. It carries + the experts-only scope rather than a mixed-precision map, and pins + `layerwise.enable=false`, which this VLM requires because its decoder layers + nest under `model.language_model.layers`. +- **`models/zai-org/GLM-5.3-Flash/ptq/nvfp4_experts_dense_mlp-kv_fp8_cast`** + widens that scope to the dense MLP as well. `mlp_layer_types` marks only layers + 0-2 `dense`, so this adds just 9 modules (`mlp.gate_proj` / `up_proj` / + `down_proj`) on top of the routed experts. The vision tower reuses those same + leaf names, so a single `*visual*` disable is appended **last** to keep + `model.visual.*` in BF16 — the experts-only variant needs no such rule. *Why special:* unlike any general recipe, these **mix FP8 and NVFP4 across different component types — or individual layers** — and hardcode the precise