[quantization] Add per-token NVFP4 quantization MoE support for ReLU2 (#56740)
Signed-off-by: Xuanteng Huang <xuantengh@nvidia.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
OpenAI Codex
parent
b803edbc9f
commit
f38c6778ce
@@ -19,6 +19,9 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
trtllm_nvfp4_hidden_alignment,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
)
|
||||
@@ -87,6 +90,8 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
(SM100) only.
|
||||
"""
|
||||
|
||||
activation_quant_key = kNvfp4DynamicToken
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -100,9 +105,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
|
||||
config=self.moe,
|
||||
weight_key=kNvfp4Static,
|
||||
activation_key=kNvfp4DynamicToken,
|
||||
activation_key=self.activation_quant_key,
|
||||
)
|
||||
|
||||
@property
|
||||
def per_token_activation(self) -> bool:
|
||||
return self.activation_quant_key == kNvfp4DynamicToken
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
@@ -156,6 +165,10 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
w2_scale_2=layer.w2_weight_scale_2,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
trtllm_hidden_alignment=trtllm_nvfp4_hidden_alignment(
|
||||
per_token_activation=self.per_token_activation,
|
||||
is_act_and_mul=self.moe.is_act_and_mul,
|
||||
),
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
@@ -176,7 +189,7 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
experts_cls=self.experts_cls,
|
||||
backend=self.nvfp4_backend,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
per_token_activation=True,
|
||||
per_token_activation=self.per_token_activation,
|
||||
)
|
||||
else:
|
||||
# Reload creates new scale tensors; derived kernel scales must use
|
||||
|
||||
Reference in New Issue
Block a user