[Quantization] Add per-token NVFP4 CuTe-DSL MoE backend (#50030)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
This commit is contained in:
Matej Sirovatka
2026-09-30 10:42:03 +00:00
committed by GitHub
co-authored by aoshen02 OpenAI Codex aoshen02
parent d5ee309314
commit 0103a9b96f
9 changed files with 186 additions and 66 deletions
@@ -14,11 +14,14 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
activation_to_flashinfer_int,
quantize_nvfp4_per_token_input,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kNvfp4Dynamic,
kNvfp4DynamicToken,
kNvfp4Static,
)
from vllm.platforms import current_platform
@@ -40,6 +43,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
per_token_activation: bool = False,
):
super().__init__(
moe_config=moe_config,
@@ -63,6 +67,14 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
self.situ_beta = moe_config.activation_situ_beta
self.situ_linear_beta = moe_config.activation_situ_linear_beta
self.per_token_activation = per_token_activation
self.per_token_global_scale = None
if per_token_activation:
assert quant_config.a2_gscale is not None
self.per_token_global_scale = quant_config.a2_gscale.new_full(
(1,),
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
@@ -92,6 +104,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
) -> bool:
SUPPORTED_W_A = [
(kNvfp4Static, kNvfp4Dynamic),
(kNvfp4Static, kNvfp4DynamicToken),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@@ -114,6 +127,10 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
@property
def expects_unquantized_inputs(self) -> bool:
return self.per_token_activation
def workspace_shapes(
self,
M: int,
@@ -127,8 +144,8 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
workspace1 = (0,)
workspace2 = (0,)
# K is packed (K//2 for uint8), so output uses hidden_dim.
assert self.hidden_dim == K * 2
expected_hidden_dim = K if self.expects_unquantized_inputs else K * 2
assert self.hidden_dim == expected_hidden_dim
output = (M, self.hidden_dim)
return (workspace1, workspace2, output)
@@ -151,13 +168,26 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
apply_router_weight_on_input: bool | None,
):
assert self.quant_dtype == "nvfp4"
assert a1q_scale is not None
assert self.w1_scale is not None
assert self.w2_scale is not None
if self.expects_unquantized_inputs:
hidden_states, block_scale, per_token_scale = (
quantize_nvfp4_per_token_input(hidden_states)
)
fc2_input_scale = self.per_token_global_scale
else:
assert a1q_scale is not None
block_scale = a1q_scale
per_token_scale = None
fc2_input_scale = self.a2_gscale
assert block_scale is not None
assert fc2_input_scale is not None
# a1q_scale is (M, K//16) float8_e4m3fn from fp4_quantize.
# The functional API expects x_sf with trailing dim: (M, K//16, 1).
x_sf = a1q_scale.unsqueeze(-1)
x_sf = block_scale.unsqueeze(-1)
# The kernel defaults swiglu_{alpha,beta,limit} to the plain-SwiGLU
# values, so only forward the ones the model actually sets.
@@ -188,6 +218,9 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
}
swiglu_kwargs = {k: v for k, v in swiglu_params.items() if v is not None}
per_token_kwargs = (
{"per_token_scale": per_token_scale} if self.per_token_activation else {}
)
flashinfer_cute_dsl_fused_moe_nvfp4(
x=hidden_states,
x_sf=x_sf,
@@ -196,7 +229,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
w1_weight=w1,
w1_weight_sf=self.w1_scale,
w1_alpha=self.g1_alphas,
fc2_input_scale=self.a2_gscale,
fc2_input_scale=fc2_input_scale,
w2_weight=w2,
w2_weight_sf=self.w2_scale,
w2_alpha=self.g2_alphas,
@@ -209,4 +242,5 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
MoEActivation.SILU if activation == MoEActivation.SITU else activation
),
**swiglu_kwargs,
**per_token_kwargs,
)
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
activation_to_flashinfer_int,
has_flashinfer_situ_activation,
quantize_nvfp4_per_token_input,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -37,10 +38,6 @@ from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
logger = init_logger(__name__)
# Base scale for per-token NVFP4 activation quant; the kernel folds the
# per-token global scale (from the activation amax) on top of it.
_PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0)
class TrtLlmNvFp4ExpertsBase:
"""NvFp4 TRTLLM-Gen MoE kernels. Supports modular and monolithic interface."""
@@ -259,23 +256,6 @@ class TrtLlmNvFp4ExpertsBase:
def expects_unquantized_inputs(self) -> bool:
return self.per_token_activation
def _quantize_per_token_input(
self, hidden_states: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""NVFP4-quantize activations with a per-token global scale.
Returns ``(packed_fp4, block_scale, per_token_scale)``.
"""
from flashinfer import SfLayout, nvfp4_quantize
hs_fp4, hs_block_scale, per_token_scale = nvfp4_quantize(
hidden_states,
_PER_TOKEN_BASE_GLOBAL_SCALE,
sfLayout=SfLayout.layout_linear,
per_token_activation=True,
)
return hs_fp4, hs_block_scale, per_token_scale
def _get_chunk_size(self) -> int:
MAX_GRID_Y = 65535
MAX_TILE_TOKENS_DIM = 128
@@ -361,9 +341,9 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
# Per-token: input is unquantized, quantize it here. Otherwise it was
# already quantized in prepare() with the static global scale.
if self.per_token_activation:
if self.expects_unquantized_inputs:
hidden_states, block_scale, per_token_scale = (
self._quantize_per_token_input(hidden_states)
quantize_nvfp4_per_token_input(hidden_states)
)
else:
block_scale, per_token_scale = a1q_scale, None
@@ -437,7 +417,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
) -> UnfinalizedMoEOutput | None:
assert self._supports_activation(activation)
# Per-token defers input quant to _invoke_kernel, so a1q_scale is None.
assert a1q_scale is not None or self.per_token_activation
assert a1q_scale is not None or self.expects_unquantized_inputs
# DeepEP produces int64 indexes.
topk_ids = topk_ids.to(dtype=torch.int32)
@@ -539,7 +519,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
import flashinfer
assert self._supports_activation(activation)
assert a1q_scale is not None or self.per_token_activation
assert a1q_scale is not None or self.expects_unquantized_inputs
assert self.quant_config.w1_scale is not None
assert self.quant_config.w2_scale is not None
assert (
@@ -551,9 +531,9 @@ class TrtLlmNvFp4ExpertsMonolithic(
)
# Per-token: input is unquantized, quantize it here (see modular apply).
if self.per_token_activation:
if self.expects_unquantized_inputs:
hidden_states, block_scale, per_token_scale = (
self._quantize_per_token_input(hidden_states)
quantize_nvfp4_per_token_input(hidden_states)
)
else:
block_scale, per_token_scale = a1q_scale, None