[Kernel] Support Nvfp4 Cutedsl Moe Swiglu-oai and Relu2(non-gated) Activation (#47106)
Signed-off-by: Tiekai Bi <tiekaib@nvidia.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5
Yongye Zhu
parent
b1e12d142d
commit
e08111211b
@@ -814,6 +814,8 @@ def nvfp4_moe_quant_config(
|
||||
w1_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
is_scale_swizzled: bool = True,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
@@ -833,6 +835,8 @@ def nvfp4_moe_quant_config(
|
||||
per_out_ch_quant=False,
|
||||
block_shape=None,
|
||||
is_scale_swizzled=is_scale_swizzled,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
)
|
||||
|
||||
|
||||
@@ -59,6 +59,9 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
self.global_num_experts = moe_config.num_experts
|
||||
self.ep_rank = moe_config.moe_parallel_config.ep_rank
|
||||
self.local_expert_offset = self.ep_rank * self.local_num_experts
|
||||
self.gemm1_alpha = quant_config.gemm1_alpha
|
||||
self.gemm1_beta = quant_config.gemm1_beta
|
||||
self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
|
||||
@@ -93,7 +96,12 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in (MoEActivation.SILU, MoEActivation.RELU2_NO_MUL)
|
||||
return activation in (
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
@@ -149,6 +157,22 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
# The functional API expects x_sf with trailing dim: (M, K//16, 1).
|
||||
x_sf = a1q_scale.unsqueeze(-1)
|
||||
|
||||
# The kernel defaults swiglu_{alpha,beta,limit} to the plain-SwiGLU
|
||||
# values, so only forward the ones the model actually sets.
|
||||
swiglu_params: dict[str, float | None] = {}
|
||||
if activation == MoEActivation.SILU:
|
||||
swiglu_params = {"swiglu_limit": self.gemm1_clamp_limit}
|
||||
elif activation in (
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
):
|
||||
swiglu_params = {
|
||||
"swiglu_alpha": self.gemm1_alpha,
|
||||
"swiglu_beta": self.gemm1_beta,
|
||||
"swiglu_limit": self.gemm1_clamp_limit,
|
||||
}
|
||||
swiglu_kwargs = {k: v for k, v in swiglu_params.items() if v is not None}
|
||||
|
||||
flashinfer_cute_dsl_fused_moe_nvfp4(
|
||||
x=hidden_states,
|
||||
x_sf=x_sf,
|
||||
@@ -167,4 +191,5 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
|
||||
local_expert_offset=self.local_expert_offset,
|
||||
moe_output=output,
|
||||
activation_type=activation_to_flashinfer_int(activation),
|
||||
**swiglu_kwargs,
|
||||
)
|
||||
|
||||
@@ -190,6 +190,7 @@ def select_nvfp4_moe_backend(
|
||||
NVFP4_BACKENDS_WITH_CLAMP = {
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
NvFp4MoeBackend.VLLM_CUTLASS,
|
||||
NvFp4MoeBackend.MARLIN,
|
||||
NvFp4MoeBackend.HUMMING,
|
||||
@@ -261,8 +262,8 @@ def select_nvfp4_moe_backend(
|
||||
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
|
||||
f"explicitly requested moe_backend={runner_backend!r} does "
|
||||
f"not apply the SwiGLU clamp. Use 'flashinfer_trtllm', "
|
||||
f"'flashinfer_cutlass', 'cutlass', 'marlin', or 'humming' "
|
||||
f"instead."
|
||||
f"'flashinfer_cutlass', 'flashinfer_cutedsl', 'cutlass', "
|
||||
f"'marlin', or 'humming' instead."
|
||||
)
|
||||
return _return_or_raise(
|
||||
requested_backend, config, weight_key, activation_key, activation_format
|
||||
@@ -471,6 +472,8 @@ def make_nvfp4_moe_quant_config(
|
||||
a13_scale: torch.Tensor,
|
||||
a2_scale: torch.Tensor,
|
||||
swiglu_limit: float | None = None,
|
||||
swiglu_alpha: float | None = None,
|
||||
swiglu_beta: float | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
if backend == NvFp4MoeBackend.HUMMING:
|
||||
@@ -497,6 +500,8 @@ def make_nvfp4_moe_quant_config(
|
||||
a2_gscale=a2_scale,
|
||||
w1_scale=w13_scale,
|
||||
w2_scale=w2_scale,
|
||||
gemm1_alpha=swiglu_alpha,
|
||||
gemm1_beta=swiglu_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
@@ -521,6 +526,8 @@ def make_nvfp4_moe_quant_config(
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
)
|
||||
),
|
||||
gemm1_alpha=swiglu_alpha,
|
||||
gemm1_beta=swiglu_beta,
|
||||
gemm1_clamp_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
|
||||
+2
@@ -251,6 +251,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
a13_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
swiglu_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
swiglu_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
|
||||
@@ -1591,6 +1591,8 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
a13_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
swiglu_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
swiglu_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
|
||||
@@ -1659,6 +1659,9 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
w2_scale_2=layer.w2_weight_scale_2,
|
||||
a13_scale=layer.w13_input_scale_2,
|
||||
a2_scale=layer.w2_input_scale_2,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
swiglu_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
swiglu_beta=getattr(layer, "swiglu_beta", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
align_fp4_moe_weights_for_fi,
|
||||
align_trtllm_fp4_moe_hidden_dim_for_fi,
|
||||
@@ -80,6 +81,28 @@ def interleave_linear_and_gate(
|
||||
return x
|
||||
|
||||
|
||||
def reorder_w13_to_w31_for_flashinfer_cutedsl(
|
||||
activation: MoEActivation,
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Normalize gated w13 rows to the [up; gate] order used by FlashInfer."""
|
||||
if activation == MoEActivation.SWIGLUOAI:
|
||||
# gpt-oss checkpoints store w13 interleaved as [gate0, up0, gate1, ...].
|
||||
gate, up = w13[:, 0::2], w13[:, 1::2]
|
||||
gate_scale, up_scale = w13_scale[:, 0::2], w13_scale[:, 1::2]
|
||||
return (
|
||||
torch.cat([up, gate], dim=1).contiguous(),
|
||||
torch.cat([up_scale, gate_scale], dim=1).contiguous(),
|
||||
)
|
||||
|
||||
half = w13.shape[1] // 2
|
||||
return (
|
||||
torch.cat([w13[:, half:], w13[:, :half]], dim=1).contiguous(),
|
||||
torch.cat([w13_scale[:, half:], w13_scale[:, :half]], dim=1).contiguous(),
|
||||
)
|
||||
|
||||
|
||||
def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
layer: "RoutedExperts",
|
||||
w13: torch.Tensor,
|
||||
@@ -114,9 +137,9 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
|
||||
a2_scale = a2_scale.max().to(torch.float32).repeat(num_experts)
|
||||
|
||||
if layer.activation.is_gated:
|
||||
half = w13.shape[1] // 2
|
||||
w13 = torch.cat([w13[:, half:], w13[:, :half]], dim=1)
|
||||
w13_scale = torch.cat([w13_scale[:, half:], w13_scale[:, :half]], dim=1)
|
||||
w13, w13_scale = reorder_w13_to_w31_for_flashinfer_cutedsl(
|
||||
layer.activation, w13, w13_scale
|
||||
)
|
||||
|
||||
# Interleave up/gate rows for w13 weights and scales.
|
||||
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
|
||||
|
||||
Reference in New Issue
Block a user