[quantization] Add per-token NVFP4 quantization MoE support for ReLU2 (#56740)

Signed-off-by: Xuanteng Huang <xuantengh@nvidia.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Xuanteng Huang
2026-10-07 22:01:09 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 OpenAI Codex
parent b803edbc9f
commit f38c6778ce
18 changed files with 450 additions and 61 deletions
@@ -12,7 +12,7 @@ from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm.config import get_current_vllm_config_or_none
from vllm.config.quantization import QuantSpec
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
init_fp8_linear_kernel,
@@ -63,6 +63,9 @@ from vllm.model_executor.layers.quantization.base_config import (
)
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
trtllm_nvfp4_hidden_alignment,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
FP8_SCALE_SENTINEL,
process_fp8_input_tensor_strategy_moe,
@@ -90,6 +93,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp8Dynamic,
kMxfp8Static,
kNvfp4Dynamic,
kNvfp4DynamicToken,
kNvfp4Static,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
@@ -724,12 +728,18 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
kv_cache_quant_algo: str | None = None,
exclude_modules: list[str] | None = None,
group_size: int = 16,
*,
quantization_args: QuantizationConfigArgs | None = None,
) -> None:
if exclude_modules is None:
exclude_modules = []
super().__init__(exclude_modules)
self.quant_method = quant_method
self.is_checkpoint_nvfp4_serialized = is_checkpoint_nvfp4_serialized
moe_spec = quantization_args.moe if quantization_args is not None else None
self.moe_activation_override: QuantKey | None = (
moe_spec.activation if moe_spec is not None else None
)
if is_checkpoint_nvfp4_serialized:
self.group_size = group_size
self.kv_cache_quant_algo = kv_cache_quant_algo
@@ -815,6 +825,7 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
kv_cache_quant_method,
exclude_modules,
group_size,
quantization_args=original_config.get("_online_quantization_args"),
)
@@ -835,18 +846,36 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
) -> None:
super().__init__(moe_config)
self.quant_config = quant_config
# W4A16 mode fires for W4A16_NVFP4 on-disk checkpoints. With
# activation_key=None every W4A4 backend's _supports_quant_scheme
# rejects itself (they all require (kNvfp4Static, kNvfp4Dynamic)
# exactly); only Marlin survives. Marlin's MoE path drops
# activation scales in convert_to_nvfp4_moe_kernel_format, so no
# other change is needed.
self.use_a16 = quant_config.quant_method == "W4A16_NVFP4"
activation_key = None if self.use_a16 else kNvfp4Dynamic
if quant_config.moe_activation_override is not None:
if self.use_a16:
raise ValueError(
"NVFP4 activation overrides require a W4A4 checkpoint."
)
activation_key = quant_config.moe_activation_override
if activation_key not in (kNvfp4Dynamic, kNvfp4DynamicToken):
raise ValueError(
"Unsupported ModelOpt NVFP4 MoE activation override: "
f"{activation_key}. Use quantization_config.moe.activation="
"'nvfp4_per_token' or omit the override."
)
self.per_token_activation = activation_key == kNvfp4DynamicToken
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
config=self.moe,
weight_key=kNvfp4Static,
activation_key=None if self.use_a16 else kNvfp4Dynamic,
activation_key=activation_key,
)
if (
self.per_token_activation
and self.nvfp4_backend != NvFp4MoeBackend.FLASHINFER_TRTLLM
):
raise ValueError(
"Per-token NVFP4 activation for pre-quantized weights "
"requires the FlashInfer TRTLLM MoE backend."
)
self.use_global_sf = is_global_sf_supported_for_nvfp4_backend(
self.nvfp4_backend
@@ -984,6 +1013,10 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
self._build_moe_kernel(layer)
return
if self.per_token_activation:
layer.w13_input_scale.data.fill_(1.0)
layer.w2_input_scale.data.fill_(1.0)
# Use a single gscale for w13.
if self.moe.is_act_and_mul and not torch.allclose(
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
@@ -1016,6 +1049,10 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
a2_scale=layer.w2_input_scale,
is_act_and_mul=self.moe.is_act_and_mul,
use_a16=self.use_a16,
trtllm_hidden_alignment=trtllm_nvfp4_hidden_alignment(
per_token_activation=self.per_token_activation,
is_act_and_mul=self.moe.is_act_and_mul,
),
)
replace_parameter(layer, "w13_weight", w13)
@@ -1039,6 +1076,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
per_token_activation=self.per_token_activation,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -1588,6 +1626,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
group_size: int | None,
**kwargs: Any,
) -> "ModelOptMixedPrecisionConfig":
quantization_args = original_config.get("_online_quantization_args")
if "quantization" in original_config:
quantized_layers = original_config["quantization"].get(
"quantized_layers", {}
@@ -1626,6 +1665,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
kv_cache_quant_algo=kv_cache_quant_method,
exclude_modules=[],
group_size=group_size,
quantization_args=quantization_args,
)
# Sibling config for layers that declare quant_algo: "W4A16_NVFP4".
# get_quant_method resolves this sub-config to the (kNvfp4Static, None)
@@ -1638,6 +1678,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
kv_cache_quant_algo=kv_cache_quant_method,
exclude_modules=[],
group_size=group_size,
quantization_args=quantization_args,
)
mxfp8_config = ModelOptMxFp8Config(
@@ -19,6 +19,9 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
trtllm_nvfp4_hidden_alignment,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
FLOAT4_E2M1_MAX,
)
@@ -87,6 +90,8 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
(SM100) only.
"""
activation_quant_key = kNvfp4DynamicToken
def __init__(
self,
*,
@@ -100,9 +105,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
config=self.moe,
weight_key=kNvfp4Static,
activation_key=kNvfp4DynamicToken,
activation_key=self.activation_quant_key,
)
@property
def per_token_activation(self) -> bool:
return self.activation_quant_key == kNvfp4DynamicToken
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
@@ -156,6 +165,10 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
w2_scale_2=layer.w2_weight_scale_2,
a2_scale=layer.w2_input_scale,
is_act_and_mul=self.moe.is_act_and_mul,
trtllm_hidden_alignment=trtllm_nvfp4_hidden_alignment(
per_token_activation=self.per_token_activation,
is_act_and_mul=self.moe.is_act_and_mul,
),
)
replace_parameter(layer, "w13_weight", w13)
@@ -176,7 +189,7 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
per_token_activation=True,
per_token_activation=self.per_token_activation,
)
else:
# Reload creates new scale tensors; derived kernel scales must use
@@ -344,6 +344,7 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
w2_scale_2: torch.Tensor,
a2_scale: torch.Tensor,
is_act_and_mul: bool,
trtllm_hidden_alignment: int = 256,
) -> tuple[
torch.Tensor,
torch.Tensor,
@@ -398,7 +399,11 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
# Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels.
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
w13, w13_scale, w2, w2_scale
w13,
w13_scale,
w2,
w2_scale,
min_alignment=trtllm_hidden_alignment,
)
if layer.moe_config.hidden_dim_unpadded is None:
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
@@ -280,6 +280,20 @@ def align_fp4_moe_weights_for_fi(
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
def trtllm_nvfp4_hidden_alignment(
per_token_activation: bool, is_act_and_mul: bool
) -> int:
"""Hidden-dim alignment for the TRTLLM-Gen NVFP4 MoE cubins.
The per-token (dynamic activation scale) non-gated variant has far fewer
tile configs for 256-aligned K and its heuristic default tactic fails with
"No valid config found" (e.g. Nemotron's 2688 -> 2816), so pad to 512.
"""
if per_token_activation and not is_act_and_mul:
return 512
return 256
def align_fp4_moe_hidden_dim_for_fi(
w13: torch.Tensor,
w13_scale: torch.Tensor,