diff --git a/modelopt_recipes/models/Qwen/Qwen3.8-2.4T-A95B/ptq/nvfp4_experts_mse-fp8_self_attn-fp8_linear_attn-kv_fp8_cast.yaml b/modelopt_recipes/models/Qwen/Qwen3.8-2.4T-A95B/ptq/nvfp4_experts_mse-fp8_self_attn-fp8_linear_attn-kv_fp8_cast.yaml new file mode 100644 index 00000000000..5295b3ad640 --- /dev/null +++ b/modelopt_recipes/models/Qwen/Qwen3.8-2.4T-A95B/ptq/nvfp4_experts_mse-fp8_self_attn-fp8_linear_attn-kv_fp8_cast.yaml @@ -0,0 +1,91 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Recipe for nvidia/Qwen3.8-2.4T-A95B-NVFP4 +# (https://huggingface.co/nvidia/Qwen3.8-2.4T-A95B-NVFP4), produced from +# Qwen/Qwen3.8-2.4T-A95B with ModelOpt using this file. +# +# Qwen/Qwen3.8-2.4T-A95B is a `qwen3_5_moe_text` MoE: 92 layers, 512 routed experts (top-10) +# plus a shared expert, and HYBRID attention -- gated-delta (linear-attention) layers +# interleaved with full-attention layers. It is transformers-native from >= 5.9, and its +# config ships `base_model_ep_plan`, so no ModelOpt plugin is required. +# +# What this quantizes: +# routed experts NVFP4 (MSE-searched static weight scales, dynamic input scales) +# self-attention FP8 (W8A8, all projections) +# linear-attention FP8 (W8A8, the gated-delta projections: +# in_proj_qkv / in_proj_z / in_proj_a / in_proj_b, out_proj) +# KV cache FP8 (cast mode) +# everything else BF16 -- including MTP, which is left unquantized +# +# The gated-delta conv1d and the norms carry no Linear quantizer, so the recurrent state +# path itself is never quantized; only the projections around it are. Quantizing those +# projections was validated not to degrade the model: the exported checkpoint was +# evaluated against the BF16 baseline on GPQA, AA-LCR, SciCode, IFBench and +# Terminal-Bench 2.1, with no meaningful accuracy regression on any of them. +# +# The source checkpoint ships as native block-FP8 (`quant_method=fp8`, +# `weight_block_size [128, 128]`, dynamic activations). The loader dequantizes it to +# BF16 before quantizers are inserted, so the scales below are calibrated against BF16 +# weights rather than against the shipped FP8. + +imports: + base_disable_all: configs/ptq/units/base_disable_all + nvfp4: configs/numerics/nvfp4 + nvfp4_static: configs/numerics/nvfp4_static + fp8: configs/numerics/fp8 + kv_fp8_cast: configs/ptq/units/kv_fp8_cast + +metadata: + recipe_type: ptq + description: >- + qwen3_5_moe_text: NVFP4 (MSE static weights, dynamic inputs) on the routed experts + FP8 (W8A8) + on all self-attention AND all linear-attention projections; everything else BF16; FP8 KV cache + (cast); MTP left unquantized. Validated against the BF16 baseline with no meaningful + accuracy regression. + +quantize: + algorithm: + method: mse + fp8_scale_sweep: true + layerwise: + enable: false + quant_cfg: + - $import: base_disable_all + # NVFP4 on the routed experts: + - quantizer_name: '*mlp.experts.*weight_quantizer' + cfg: + $import: nvfp4_static + - quantizer_name: '*mlp.experts.*input_quantizer' + cfg: + $import: nvfp4 + # FP8 on all self-attention projections: + - quantizer_name: '*self_attn*weight_quantizer' + cfg: + $import: fp8 + - quantizer_name: '*self_attn*input_quantizer' + cfg: + $import: fp8 + # FP8 on all linear-attention (gated-delta) projections: + - quantizer_name: '*linear_attn*weight_quantizer' + cfg: + $import: fp8 + - quantizer_name: '*linear_attn*input_quantizer' + cfg: + $import: fp8 + # FP8 KV cache -- CAST mode: + - $import: kv_fp8_cast + # Keep the MTP block BF16 (last so it wins over the broad enables above): + - {quantizer_name: '*mtp*', enable: false} diff --git a/modelopt_recipes/ptq.md b/modelopt_recipes/ptq.md index 5e6b332de1c..fca1dfdc557 100644 --- a/modelopt_recipes/ptq.md +++ b/modelopt_recipes/ptq.md @@ -406,6 +406,18 @@ checkpoint's** quant config verbatim: Q4_K/Q5_0 linears → NVFP4 W4A4 (attention q/k/v/o kept uniform so export can fuse them), the Q6_K MLP `down_proj` layers → FP8 W8A8, embeddings → NVFP4 W4A16, `lm_head` → FP8 W8A16, and the F32 tensors (conv1d, norms) → BF16. +- **`models/Qwen/Qwen3.8-2.4T-A95B/ptq/nvfp4_experts_mse-fp8_self_attn-fp8_linear_attn-kv_fp8_cast`** + mirrors `nvidia/Qwen3.8-2.4T-A95B-NVFP4`: a `qwen3_5_moe_text` MoE with **hybrid + attention** — gated-delta (linear-attention) layers interleaved with + full-attention layers. Routed experts → NVFP4 (MSE-searched static weight + scales, dynamic input scales); **both** self-attention and the gated-delta + linear-attention projections (`in_proj_qkv/z/a/b`, `out_proj`) → FP8 W8A8; KV + cache → FP8 cast; everything else, including the MTP block, stays BF16. The + gated-delta `conv1d` and norms carry no Linear quantizer, so the recurrent + state path itself is never quantized — only the projections around it are. The + source ships as native block-FP8 (`weight_block_size [128, 128]`); the loader + dequantizes it to BF16 before quantizers are inserted, so the scales are + calibrated against BF16 weights, not the shipped FP8. *Why special:* unlike any general recipe, these **mix FP8 and NVFP4 across different component types — or individual layers** — and hardcode the precise