[Quant] Add nvfp4_per_token online MoE quantization (#48538)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp8Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
|
||||
# User-facing names addressable from quantization_config.
|
||||
@@ -134,6 +135,11 @@ _ONLINE_SHORTHANDS: dict[str, QuantizationConfigArgs] = {
|
||||
"int8_per_channel_weight_only": QuantizationConfigArgs(
|
||||
moe=QuantSpec(weight=kInt8StaticChannelSym),
|
||||
),
|
||||
# Online NVFP4 on MoE with per-token dynamic activation scales (Blackwell +
|
||||
# FlashInfer TRTLLM only); linear stays unquantized (no `linear` field).
|
||||
"nvfp4_per_token": QuantizationConfigArgs(
|
||||
moe=QuantSpec(weight=kNvfp4Static),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user