[Quant] Add nvfp4_per_token online MoE quantization (#48538)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-07-16 14:25:27 -07:00
committed by GitHub
parent ab3c1aedf3
commit c95c663049
7 changed files with 285 additions and 5 deletions
+6
View File
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt8StaticChannelSym,
kMxfp4Dynamic,
kMxfp8Dynamic,
kNvfp4Static,
)
# User-facing names addressable from quantization_config.
@@ -134,6 +135,11 @@ _ONLINE_SHORTHANDS: dict[str, QuantizationConfigArgs] = {
"int8_per_channel_weight_only": QuantizationConfigArgs(
moe=QuantSpec(weight=kInt8StaticChannelSym),
),
# Online NVFP4 on MoE with per-token dynamic activation scales (Blackwell +
# FlashInfer TRTLLM only); linear stays unquantized (no `linear` field).
"nvfp4_per_token": QuantizationConfigArgs(
moe=QuantSpec(weight=kNvfp4Static),
),
}