[quantization] Add per-token NVFP4 quantization MoE support for ReLU2 (#56740)

Signed-off-by: Xuanteng Huang <xuantengh@nvidia.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Xuanteng Huang
2026-10-07 22:01:09 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 OpenAI Codex
parent b803edbc9f
commit f38c6778ce
18 changed files with 450 additions and 61 deletions
@@ -19,6 +19,9 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
trtllm_nvfp4_hidden_alignment,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
FLOAT4_E2M1_MAX,
)
@@ -87,6 +90,8 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
(SM100) only.
"""
activation_quant_key = kNvfp4DynamicToken
def __init__(
self,
*,
@@ -100,9 +105,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
config=self.moe,
weight_key=kNvfp4Static,
activation_key=kNvfp4DynamicToken,
activation_key=self.activation_quant_key,
)
@property
def per_token_activation(self) -> bool:
return self.activation_quant_key == kNvfp4DynamicToken
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
@@ -156,6 +165,10 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
w2_scale_2=layer.w2_weight_scale_2,
a2_scale=layer.w2_input_scale,
is_act_and_mul=self.moe.is_act_and_mul,
trtllm_hidden_alignment=trtllm_nvfp4_hidden_alignment(
per_token_activation=self.per_token_activation,
is_act_and_mul=self.moe.is_act_and_mul,
),
)
replace_parameter(layer, "w13_weight", w13)
@@ -176,7 +189,7 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
per_token_activation=True,
per_token_activation=self.per_token_activation,
)
else:
# Reload creates new scale tensors; derived kernel scales must use