[Quantization] Add per-token NVFP4 CuTe-DSL MoE backend (#50030)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: S1ro1 <matej.sirovatka@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
This commit is contained in:
Matej Sirovatka
2026-09-30 10:42:03 +00:00
committed by GitHub
co-authored by aoshen02 OpenAI Codex aoshen02
parent d5ee309314
commit 0103a9b96f
9 changed files with 186 additions and 66 deletions
@@ -9,8 +9,8 @@ import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
align_fp4_moe_hidden_dim_for_fi,
align_fp4_moe_weights_for_fi,
align_trtllm_fp4_moe_hidden_dim_for_fi,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
swizzle_blockscale,
@@ -128,9 +128,9 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
]:
"""Prepare weights for the CuteDSL wrapper-based NvFP4 MoE backend.
Converts weight scale factors to MMA layout expected by CuteDslMoEWrapper,
and interleaves w13 gate/linear rows for gated activations. Non-gated
activations use a single w13 projection and keep its row order unchanged.
Pads the runtime expert tensors to the kernel's GEMM alignment, converts
weight scale factors to the MMA layout expected by CuteDslMoEWrapper, and
interleaves w13 gate/linear rows for gated activations.
"""
# Global scaling factors (same as other FlashInfer backends).
num_experts = w13.shape[0]
@@ -140,12 +140,38 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
)
a2_scale = amax_for_moe_activation_quant(a2_scale, enable_eplb).repeat(num_experts)
if layer.activation.is_gated:
gated = layer.activation.is_gated
if gated:
w13, w13_scale = reorder_w13_to_w31_for_flashinfer_cutedsl(
layer.activation, w13, w13_scale
)
# Interleave up/gate rows for w13 weights and scales.
# GEMM1's output dimension must be a multiple of 128: 2I for gated
# activations (also required by interleaving), but only I for non-gated.
# Keep the checkpoint tensors unchanged and pad only the kernel's runtime
# representation. Zero rows also make the padded GEMM2 contraction a no-op.
w13, w13_scale, w2, w2_scale, padded_intermediate = align_fp4_moe_weights_for_fi(
w13,
w13_scale,
w2,
w2_scale,
is_act_and_mul=gated,
min_alignment=64 if gated else 128,
)
layer.moe_config.intermediate_size_per_partition = padded_intermediate
# GEMM1 gathers full 256-element K tiles of activations and their scales
# without a K-tail predicate. Pad runtime weights to match; the MoE runner
# pads activations and trims the output using the original hidden size.
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
w13, w13_scale, w2, w2_scale
)
if layer.moe_config.hidden_dim_unpadded is None:
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
layer.moe_config.hidden_dim = padded_hidden
if gated:
# Interleave up/gate rows for the fused gated activation.
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
@@ -371,8 +397,8 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
# Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels.
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:
w13, w13_scale, w2, w2_scale, padded_hidden = (
align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
w13, w13_scale, w2, w2_scale
)
if layer.moe_config.hidden_dim_unpadded is None:
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
@@ -13,6 +13,8 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0)
def activation_to_flashinfer_int(activation: MoEActivation) -> int:
return activation_to_flashinfer_type(activation).value
@@ -50,6 +52,20 @@ def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType"
return ACTIVATION_TO_FI_ACTIVATION[activation]
def quantize_nvfp4_per_token_input(
hidden_states: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Quantize NVFP4 activations with one FP32 decode scale per token."""
from flashinfer import SfLayout, nvfp4_quantize
return nvfp4_quantize(
hidden_states,
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
sfLayout=SfLayout.layout_linear,
per_token_activation=True,
)
def swap_w13_to_w31(x: torch.Tensor) -> torch.Tensor:
return (
x.reshape(-1, 2, x.shape[-2] // 2, x.shape[-1]).flip(dims=[1]).reshape(x.shape)
@@ -264,7 +280,7 @@ def align_fp4_moe_weights_for_fi(
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
def align_trtllm_fp4_moe_hidden_dim_for_fi(
def align_fp4_moe_hidden_dim_for_fi(
w13: torch.Tensor,
w13_scale: torch.Tensor,
w2: torch.Tensor,
@@ -279,7 +295,7 @@ def align_trtllm_fp4_moe_hidden_dim_for_fi(
return w13, w13_scale, w2, w2_scale, hidden_size
logger.warning_once(
"Padding hidden size from %d to %d for TRTLLM NVFP4 MoE weights. "
"Padding hidden size from %d to %d for FlashInfer NVFP4 MoE weights. "
"This requires activation slicing at runtime and may cause "
"performance degradation.",
hidden_size,