Integrate CuTeDSL MoE for ReLU2 NVFP4 (#49580)

Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
danielafrimi
2026-07-28 19:46:38 -07:00
committed by GitHub
co-authored by OpenAI Codex Michael Goin
parent 30c2718eaa
commit 32a423ac0a
3 changed files with 245 additions and 9 deletions
@@ -13,6 +13,9 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
activation_to_flashinfer_int,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kNvfp4Dynamic,
@@ -76,7 +79,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
return True
@staticmethod
def _supports_quant_scheme(
@@ -90,7 +93,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation == MoEActivation.SILU
return activation in (MoEActivation.SILU, MoEActivation.RELU2_NO_MUL)
@staticmethod
def _supports_parallel_config(
@@ -163,4 +166,5 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular):
num_local_experts=self.local_num_experts,
local_expert_offset=self.local_expert_offset,
moe_output=output,
activation_type=activation_to_flashinfer_int(activation),
)
@@ -103,7 +103,8 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
"""Prepare weights for the CuteDSL wrapper-based NvFP4 MoE backend.
Converts weight scale factors to MMA layout expected by CuteDslMoEWrapper,
and interleaves w13 gate/linear rows.
and interleaves w13 gate/linear rows for gated activations. Non-gated
activations use a single w13 projection and keep its row order unchanged.
"""
from flashinfer.cute_dsl.utils import convert_sf_to_mma_layout
@@ -112,13 +113,14 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl(
a13_scale = a13_scale.max().to(torch.float32).repeat(num_experts)
a2_scale = a2_scale.max().to(torch.float32).repeat(num_experts)
half = w13.shape[1] // 2
w13 = torch.cat([w13[:, half:], w13[:, :half]], dim=1)
w13_scale = torch.cat([w13_scale[:, half:], w13_scale[:, :half]], dim=1)
if layer.activation.is_gated:
half = w13.shape[1] // 2
w13 = torch.cat([w13[:, half:], w13[:, :half]], dim=1)
w13_scale = torch.cat([w13_scale[:, half:], w13_scale[:, :half]], dim=1)
# Interleave up/gate rows for w13 weights and scales.
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
# Interleave up/gate rows for w13 weights and scales.
w13 = interleave_linear_and_gate(w13, group_size=64, dim=1)
w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1)
# Convert w13 scale factors: linear → swizzled → MMA layout.
w13_scale = swizzle_blockscale(w13_scale)