[quantization] Add per-token NVFP4 quantization MoE support for ReLU2 (#56740)
Signed-off-by: Xuanteng Huang <xuantengh@nvidia.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
OpenAI Codex
parent
b803edbc9f
commit
f38c6778ce
@@ -12,7 +12,7 @@ from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config_or_none
|
||||
from vllm.config.quantization import QuantSpec
|
||||
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
init_fp8_linear_kernel,
|
||||
@@ -63,6 +63,9 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
trtllm_nvfp4_hidden_alignment,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
FP8_SCALE_SENTINEL,
|
||||
process_fp8_input_tensor_strategy_moe,
|
||||
@@ -90,6 +93,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4DynamicToken,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
@@ -724,12 +728,18 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
|
||||
kv_cache_quant_algo: str | None = None,
|
||||
exclude_modules: list[str] | None = None,
|
||||
group_size: int = 16,
|
||||
*,
|
||||
quantization_args: QuantizationConfigArgs | None = None,
|
||||
) -> None:
|
||||
if exclude_modules is None:
|
||||
exclude_modules = []
|
||||
super().__init__(exclude_modules)
|
||||
self.quant_method = quant_method
|
||||
self.is_checkpoint_nvfp4_serialized = is_checkpoint_nvfp4_serialized
|
||||
moe_spec = quantization_args.moe if quantization_args is not None else None
|
||||
self.moe_activation_override: QuantKey | None = (
|
||||
moe_spec.activation if moe_spec is not None else None
|
||||
)
|
||||
if is_checkpoint_nvfp4_serialized:
|
||||
self.group_size = group_size
|
||||
self.kv_cache_quant_algo = kv_cache_quant_algo
|
||||
@@ -815,6 +825,7 @@ class ModelOptNvFp4Config(ModelOptQuantConfigBase):
|
||||
kv_cache_quant_method,
|
||||
exclude_modules,
|
||||
group_size,
|
||||
quantization_args=original_config.get("_online_quantization_args"),
|
||||
)
|
||||
|
||||
|
||||
@@ -835,18 +846,36 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
) -> None:
|
||||
super().__init__(moe_config)
|
||||
self.quant_config = quant_config
|
||||
# W4A16 mode fires for W4A16_NVFP4 on-disk checkpoints. With
|
||||
# activation_key=None every W4A4 backend's _supports_quant_scheme
|
||||
# rejects itself (they all require (kNvfp4Static, kNvfp4Dynamic)
|
||||
# exactly); only Marlin survives. Marlin's MoE path drops
|
||||
# activation scales in convert_to_nvfp4_moe_kernel_format, so no
|
||||
# other change is needed.
|
||||
self.use_a16 = quant_config.quant_method == "W4A16_NVFP4"
|
||||
activation_key = None if self.use_a16 else kNvfp4Dynamic
|
||||
|
||||
if quant_config.moe_activation_override is not None:
|
||||
if self.use_a16:
|
||||
raise ValueError(
|
||||
"NVFP4 activation overrides require a W4A4 checkpoint."
|
||||
)
|
||||
activation_key = quant_config.moe_activation_override
|
||||
if activation_key not in (kNvfp4Dynamic, kNvfp4DynamicToken):
|
||||
raise ValueError(
|
||||
"Unsupported ModelOpt NVFP4 MoE activation override: "
|
||||
f"{activation_key}. Use quantization_config.moe.activation="
|
||||
"'nvfp4_per_token' or omit the override."
|
||||
)
|
||||
self.per_token_activation = activation_key == kNvfp4DynamicToken
|
||||
|
||||
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=kNvfp4Static,
|
||||
activation_key=None if self.use_a16 else kNvfp4Dynamic,
|
||||
activation_key=activation_key,
|
||||
)
|
||||
if (
|
||||
self.per_token_activation
|
||||
and self.nvfp4_backend != NvFp4MoeBackend.FLASHINFER_TRTLLM
|
||||
):
|
||||
raise ValueError(
|
||||
"Per-token NVFP4 activation for pre-quantized weights "
|
||||
"requires the FlashInfer TRTLLM MoE backend."
|
||||
)
|
||||
|
||||
self.use_global_sf = is_global_sf_supported_for_nvfp4_backend(
|
||||
self.nvfp4_backend
|
||||
@@ -984,6 +1013,10 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
self._build_moe_kernel(layer)
|
||||
return
|
||||
|
||||
if self.per_token_activation:
|
||||
layer.w13_input_scale.data.fill_(1.0)
|
||||
layer.w2_input_scale.data.fill_(1.0)
|
||||
|
||||
# Use a single gscale for w13.
|
||||
if self.moe.is_act_and_mul and not torch.allclose(
|
||||
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
|
||||
@@ -1016,6 +1049,10 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
a2_scale=layer.w2_input_scale,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
use_a16=self.use_a16,
|
||||
trtllm_hidden_alignment=trtllm_nvfp4_hidden_alignment(
|
||||
per_token_activation=self.per_token_activation,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
),
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
@@ -1039,6 +1076,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
per_token_activation=self.per_token_activation,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
@@ -1588,6 +1626,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
group_size: int | None,
|
||||
**kwargs: Any,
|
||||
) -> "ModelOptMixedPrecisionConfig":
|
||||
quantization_args = original_config.get("_online_quantization_args")
|
||||
if "quantization" in original_config:
|
||||
quantized_layers = original_config["quantization"].get(
|
||||
"quantized_layers", {}
|
||||
@@ -1626,6 +1665,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
kv_cache_quant_algo=kv_cache_quant_method,
|
||||
exclude_modules=[],
|
||||
group_size=group_size,
|
||||
quantization_args=quantization_args,
|
||||
)
|
||||
# Sibling config for layers that declare quant_algo: "W4A16_NVFP4".
|
||||
# get_quant_method resolves this sub-config to the (kNvfp4Static, None)
|
||||
@@ -1638,6 +1678,7 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
kv_cache_quant_algo=kv_cache_quant_method,
|
||||
exclude_modules=[],
|
||||
group_size=group_size,
|
||||
quantization_args=quantization_args,
|
||||
)
|
||||
|
||||
mxfp8_config = ModelOptMxFp8Config(
|
||||
|
||||
@@ -19,6 +19,9 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
trtllm_nvfp4_hidden_alignment,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
)
|
||||
@@ -87,6 +90,8 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
(SM100) only.
|
||||
"""
|
||||
|
||||
activation_quant_key = kNvfp4DynamicToken
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -100,9 +105,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=kNvfp4Static,
|
||||
activation_key=kNvfp4DynamicToken,
|
||||
activation_key=self.activation_quant_key,
|
||||
)
|
||||
|
||||
@property
|
||||
def per_token_activation(self) -> bool:
|
||||
return self.activation_quant_key == kNvfp4DynamicToken
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
@@ -156,6 +165,10 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
w2_scale_2=layer.w2_weight_scale_2,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
trtllm_hidden_alignment=trtllm_nvfp4_hidden_alignment(
|
||||
per_token_activation=self.per_token_activation,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
),
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
@@ -176,7 +189,7 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
per_token_activation=True,
|
||||
per_token_activation=self.per_token_activation,
|
||||
)
|
||||
else:
|
||||
# Reload creates new scale tensors; derived kernel scales must use
|
||||
|
||||
@@ -344,6 +344,7 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
w2_scale_2: torch.Tensor,
|
||||
a2_scale: torch.Tensor,
|
||||
is_act_and_mul: bool,
|
||||
trtllm_hidden_alignment: int = 256,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
@@ -398,7 +399,11 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
# Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels.
|
||||
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:
|
||||
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
|
||||
w13, w13_scale, w2, w2_scale
|
||||
w13,
|
||||
w13_scale,
|
||||
w2,
|
||||
w2_scale,
|
||||
min_alignment=trtllm_hidden_alignment,
|
||||
)
|
||||
if layer.moe_config.hidden_dim_unpadded is None:
|
||||
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
|
||||
|
||||
@@ -280,6 +280,20 @@ def align_fp4_moe_weights_for_fi(
|
||||
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
|
||||
|
||||
|
||||
def trtllm_nvfp4_hidden_alignment(
|
||||
per_token_activation: bool, is_act_and_mul: bool
|
||||
) -> int:
|
||||
"""Hidden-dim alignment for the TRTLLM-Gen NVFP4 MoE cubins.
|
||||
|
||||
The per-token (dynamic activation scale) non-gated variant has far fewer
|
||||
tile configs for 256-aligned K and its heuristic default tactic fails with
|
||||
"No valid config found" (e.g. Nemotron's 2688 -> 2816), so pad to 512.
|
||||
"""
|
||||
if per_token_activation and not is_act_and_mul:
|
||||
return 512
|
||||
return 256
|
||||
|
||||
|
||||
def align_fp4_moe_hidden_dim_for_fi(
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user