[Quantization] Add per-token NVFP4 CuTe-DSL MoE backend (#50030)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: S1ro1 <matej.sirovatka@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
aoshen02
parent
d5ee309314
commit
0103a9b96f
@@ -9,8 +9,8 @@ import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
align_fp4_moe_hidden_dim_for_fi,
|
||||
align_fp4_moe_weights_for_fi,
|
||||
align_trtllm_fp4_moe_hidden_dim_for_fi,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
swizzle_blockscale,
|
||||
@@ -128,9 +128,9 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
]:
|
||||
"""Prepare weights for the CuteDSL wrapper-based NvFP4 MoE backend.
|
||||
|
||||
Converts weight scale factors to MMA layout expected by CuteDslMoEWrapper,
|
||||
and interleaves w13 gate/linear rows for gated activations. Non-gated
|
||||
activations use a single w13 projection and keep its row order unchanged.
|
||||
Pads the runtime expert tensors to the kernel's GEMM alignment, converts
|
||||
weight scale factors to the MMA layout expected by CuteDslMoEWrapper, and
|
||||
interleaves w13 gate/linear rows for gated activations.
|
||||
"""
|
||||
# Global scaling factors (same as other FlashInfer backends).
|
||||
num_experts = w13.shape[0]
|
||||
@@ -140,12 +140,38 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
)
|
||||
a2_scale = amax_for_moe_activation_quant(a2_scale, enable_eplb).repeat(num_experts)
|
||||
|
||||
if layer.activation.is_gated:
|
||||
gated = layer.activation.is_gated
|
||||
if gated:
|
||||
w13, w13_scale = reorder_w13_to_w31_for_flashinfer_cutedsl(
|
||||
layer.activation, w13, w13_scale
|
||||
)
|
||||
|
||||
# Interleave up/gate rows for w13 weights and scales.
|
||||
# GEMM1's output dimension must be a multiple of 128: 2I for gated
|
||||
# activations (also required by interleaving), but only I for non-gated.
|
||||
# Keep the checkpoint tensors unchanged and pad only the kernel's runtime
|
||||
# representation. Zero rows also make the padded GEMM2 contraction a no-op.
|
||||
w13, w13_scale, w2, w2_scale, padded_intermediate = align_fp4_moe_weights_for_fi(
|
||||
w13,
|
||||
w13_scale,
|
||||
w2,
|
||||
w2_scale,
|
||||
is_act_and_mul=gated,
|
||||
min_alignment=64 if gated else 128,
|
||||
)
|
||||
layer.moe_config.intermediate_size_per_partition = padded_intermediate
|
||||
|
||||
# GEMM1 gathers full 256-element K tiles of activations and their scales
|
||||
# without a K-tail predicate. Pad runtime weights to match; the MoE runner
|
||||
# pads activations and trims the output using the original hidden size.
|
||||
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
|
||||
w13, w13_scale, w2, w2_scale
|
||||
)
|
||||
if layer.moe_config.hidden_dim_unpadded is None:
|
||||
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
|
||||
layer.moe_config.hidden_dim = padded_hidden
|
||||
|
||||
if gated:
|
||||
# Interleave up/gate rows for the fused gated activation.
|
||||
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
|
||||
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
|
||||
|
||||
@@ -371,8 +397,8 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
|
||||
# Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels.
|
||||
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:
|
||||
w13, w13_scale, w2, w2_scale, padded_hidden = (
|
||||
align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
|
||||
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
|
||||
w13, w13_scale, w2, w2_scale
|
||||
)
|
||||
if layer.moe_config.hidden_dim_unpadded is None:
|
||||
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
|
||||
|
||||
@@ -13,6 +13,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0)
|
||||
|
||||
|
||||
def activation_to_flashinfer_int(activation: MoEActivation) -> int:
|
||||
return activation_to_flashinfer_type(activation).value
|
||||
@@ -50,6 +52,20 @@ def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType"
|
||||
return ACTIVATION_TO_FI_ACTIVATION[activation]
|
||||
|
||||
|
||||
def quantize_nvfp4_per_token_input(
|
||||
hidden_states: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Quantize NVFP4 activations with one FP32 decode scale per token."""
|
||||
from flashinfer import SfLayout, nvfp4_quantize
|
||||
|
||||
return nvfp4_quantize(
|
||||
hidden_states,
|
||||
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
|
||||
sfLayout=SfLayout.layout_linear,
|
||||
per_token_activation=True,
|
||||
)
|
||||
|
||||
|
||||
def swap_w13_to_w31(x: torch.Tensor) -> torch.Tensor:
|
||||
return (
|
||||
x.reshape(-1, 2, x.shape[-2] // 2, x.shape[-1]).flip(dims=[1]).reshape(x.shape)
|
||||
@@ -264,7 +280,7 @@ def align_fp4_moe_weights_for_fi(
|
||||
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
|
||||
|
||||
|
||||
def align_trtllm_fp4_moe_hidden_dim_for_fi(
|
||||
def align_fp4_moe_hidden_dim_for_fi(
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
@@ -279,7 +295,7 @@ def align_trtllm_fp4_moe_hidden_dim_for_fi(
|
||||
return w13, w13_scale, w2, w2_scale, hidden_size
|
||||
|
||||
logger.warning_once(
|
||||
"Padding hidden size from %d to %d for TRTLLM NVFP4 MoE weights. "
|
||||
"Padding hidden size from %d to %d for FlashInfer NVFP4 MoE weights. "
|
||||
"This requires activation slicing at runtime and may cause "
|
||||
"performance degradation.",
|
||||
hidden_size,
|
||||
|
||||
Reference in New Issue
Block a user