[MoE Refactor] MXFP4 Cutlass Experts to MK (#34542)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Yongye Zhu
2026-02-25 17:32:39 -08:00
committed by GitHub
parent cbf8f7028c
commit 1976356ee6
19 changed files with 454 additions and 169 deletions
@@ -242,6 +242,10 @@ class FusedMoEQuantConfig:
def quant_dtype(self) -> torch.dtype | str | None:
return self._a1.dtype
@property
def weight_quant_dtype(self) -> torch.dtype | str | None:
return self._w1.dtype
@property
def is_quantized(self) -> bool:
return self.quant_dtype is not None
@@ -4,6 +4,7 @@
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
@@ -18,6 +19,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
kMxfp4Static,
kMxfp8Dynamic,
kNvfp4Dynamic,
kNvfp4Static,
)
@@ -64,10 +67,18 @@ class FlashInferExperts(mk.FusedMoEPermuteExpertsUnpermute):
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
assert quant_config.quant_dtype in ("nvfp4", torch.float8_e4m3fn, None), (
"Only nvfp4, fp8, bfloat16 and"
assert quant_config.weight_quant_dtype in (
"mxfp4",
"nvfp4",
torch.float8_e4m3fn,
None,
), (
"Only mxfp4, nvfp4, fp8, bfloat16 and"
" float16 quantization are currently supported."
)
self.device = moe_config.device
self.num_experts = moe_config.num_local_experts
self.ep_rank = moe_config.moe_parallel_config.ep_rank
self.ep_size = moe_config.moe_parallel_config.ep_size
self.tp_rank = moe_config.moe_parallel_config.tp_rank
@@ -78,6 +89,28 @@ class FlashInferExperts(mk.FusedMoEPermuteExpertsUnpermute):
# - pass per-block weight scales to the kernel
# - skip input activation quantization (kernel applies scaling)
self.use_deepseek_fp8_block_scale = quant_config.is_block_quantized
self.max_capture_size = (
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
)
if quant_config.weight_quant_dtype == "mxfp4":
# This value is used specifically for gpt-oss,
# Need to revisit this for other models
self.gemm1_alpha = torch.tensor(
[1.702] * self.num_experts, dtype=torch.float32, device=self.device
)
self.gemm1_beta = torch.tensor(
[1.0] * self.num_experts, dtype=torch.float32, device=self.device
)
self.gemm1_clamp_limit = torch.tensor(
[7.0] * self.num_experts, dtype=torch.float32, device=self.device
)
if quant_config.quant_dtype == "mxfp8":
self.fake_input_scale = torch.ones(
self.num_experts,
device=self.device,
dtype=torch.float32,
)
@property
def expects_unquantized_inputs(self) -> bool:
@@ -119,20 +152,33 @@ class FlashInferExperts(mk.FusedMoEPermuteExpertsUnpermute):
]
and p.has_device_capability(90)
)
# fp8 block-scale on 9.0
# fp8 block-scale, wmxfp4a16 on 9.0
or (
scheme == (kFp8Static128BlockSym, kFp8Dynamic128Sym)
scheme
in [
(kMxfp4Static, None),
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
]
and p.is_device_capability(90)
)
# nvfp4 on 10.0+
# nvfp4, wmxfp4amxfp8 on 10.0+
or (
scheme == (kNvfp4Static, kNvfp4Dynamic) and p.has_device_capability(100)
scheme
in [
(kMxfp4Static, kMxfp8Dynamic),
(kNvfp4Static, kNvfp4Dynamic),
]
and p.has_device_capability(100)
)
)
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL]
return activation in [
MoEActivation.SILU,
MoEActivation.RELU2_NO_MUL,
MoEActivation.SWIGLUOAI,
]
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
@@ -216,12 +262,23 @@ class FlashInferExperts(mk.FusedMoEPermuteExpertsUnpermute):
activation_str_to_value_map = {
MoEActivation.SILU: ActivationType.Swiglu, # This is the default
MoEActivation.SWIGLUOAI: ActivationType.Swiglu, # gpt-oss alias
MoEActivation.RELU2_NO_MUL: ActivationType.Relu2,
}
assert activation in activation_str_to_value_map, (
f"{activation=} missing from {activation_str_to_value_map.keys()=}"
)
quant_scales = None
fc1_expert_weights = None
fc2_expert_weights = None
fc1_expert_biases = None
fc2_expert_biases = None
swiglu_alpha = None
swiglu_beta = None
swiglu_limit = None
use_mxfp8_act_scaling = False
use_w4_group_scaling = False
# Select quantization metadata based on FP8 format/path
if (
self.quant_dtype == torch.float8_e4m3fn
@@ -256,6 +313,43 @@ class FlashInferExperts(mk.FusedMoEPermuteExpertsUnpermute):
# FlashInfer API requires weight to be long for nvfp4
fc1_expert_weights = w1.view(torch.long)
fc2_expert_weights = w2.view(torch.long)
elif self.weight_quant_dtype == "mxfp4":
assert self.w1_scale is not None and self.w2_scale is not None
assert w1.is_contiguous() and w2.is_contiguous()
assert self.gemm1_alpha is not None
assert self.gemm1_beta is not None
assert self.gemm1_clamp_limit is not None
assert topk_ids.is_contiguous()
fc1_expert_biases = self.w1_bias
fc2_expert_biases = self.w2_bias
swiglu_alpha = self.gemm1_alpha
swiglu_beta = self.gemm1_beta
swiglu_limit = self.gemm1_clamp_limit
if self.quant_dtype == "mxfp8":
assert self.fake_input_scale is not None
fc1_expert_weights = w1.view(torch.long)
fc2_expert_weights = w2.view(torch.long)
quant_scales = [
self.w1_scale.view(torch.int32),
self.fake_input_scale,
self.w2_scale.view(torch.int32),
self.fake_input_scale,
]
use_mxfp8_act_scaling = True
else:
assert hidden_states.dtype == torch.bfloat16
fc1_expert_weights = w1
fc2_expert_weights = w2
quant_scales = [
self.w1_scale,
self.w2_scale,
]
a1q_scale = None
use_w4_group_scaling = True
elif self.use_deepseek_fp8_block_scale:
# FP8 block-scale path: provide block-scale weights, omit a1q_scale
quant_scales = [
@@ -277,6 +371,12 @@ class FlashInferExperts(mk.FusedMoEPermuteExpertsUnpermute):
token_final_scales=topk_weights,
fc1_expert_weights=fc1_expert_weights,
fc2_expert_weights=fc2_expert_weights,
fc1_expert_biases=fc1_expert_biases,
fc2_expert_biases=fc2_expert_biases,
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
swiglu_limit=swiglu_limit,
output=output,
output_dtype=self.out_dtype,
quant_scales=quant_scales,
input_sf=a1q_scale,
@@ -284,10 +384,12 @@ class FlashInferExperts(mk.FusedMoEPermuteExpertsUnpermute):
tp_rank=self.tp_rank,
ep_size=self.ep_size,
ep_rank=self.ep_rank,
output=output,
activation_type=activation_str_to_value_map[activation],
# Informs FlashInfer to use the block-scale decoding path when True
use_deepseek_fp8_block_scale=self.use_deepseek_fp8_block_scale,
use_mxfp8_act_scaling=use_mxfp8_act_scaling,
use_w4_group_scaling=use_w4_group_scaling,
tune_max_num_tokens=max(self.max_capture_size, 1),
)
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
@@ -564,9 +564,13 @@ class FusedMoEPermuteExpertsUnpermute(ABC):
#
@property
def quant_dtype(self) -> torch.dtype | None:
def quant_dtype(self) -> torch.dtype | str | None:
return self.quant_config.quant_dtype
@property
def weight_quant_dtype(self) -> torch.dtype | str | None:
return self.quant_config.weight_quant_dtype
@property
def block_shape(self) -> list[int] | None:
return self.quant_config.block_shape
@@ -25,15 +25,20 @@ class TrtLlmGenExperts(mk.FusedMoEPermuteExpertsUnpermute):
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
gemm1_alpha,
gemm1_beta,
gemm1_clamp_limit,
max_capture_size,
):
super().__init__(moe_config, quant_config)
self.gemm1_alpha = gemm1_alpha
self.gemm1_beta = gemm1_beta
self.gemm1_clamp_limit = gemm1_clamp_limit
self.device = torch.cuda.current_device()
self.num_experts = moe_config.num_local_experts
self.gemm1_alpha = torch.tensor(
[1.702] * self.num_experts, dtype=torch.float32, device=self.device
)
self.gemm1_beta = torch.tensor(
[1.0] * self.num_experts, dtype=torch.float32, device=self.device
)
self.gemm1_clamp_limit = torch.tensor(
[7.0] * self.num_experts, dtype=torch.float32, device=self.device
)
self.max_capture_size = max_capture_size
@staticmethod
@@ -195,11 +195,12 @@ def _mxfp8_e4m3_quantize(
A_scale: torch.Tensor | None,
per_act_token_quant: bool,
block_shape: list[int] | None = None,
is_sf_swizzled_layout: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
assert A_scale is None
assert not per_act_token_quant
assert block_shape is None
return mxfp8_e4m3_quantize(A)
return mxfp8_e4m3_quantize(A, is_sf_swizzled_layout)
def _mxfp6_e3m2_quantize(
@@ -275,7 +276,13 @@ def moe_kernel_quantize_input(
elif quant_dtype == "mxfp8":
# TODO: `quant_dtype == "mxfp8"` is ambiguous,
# should be fp8_e4m3. OCP MX also defines `fp8_e5m2`.
return _mxfp8_e4m3_quantize(A, A_scale, per_act_token_quant, block_shape)
return _mxfp8_e4m3_quantize(
A,
A_scale,
per_act_token_quant,
block_shape,
is_sf_swizzled_layout=is_fp4_scale_swizzled,
)
elif quant_dtype == "mxfp6_e3m2":
return _mxfp6_e3m2_quantize(A, A_scale, per_act_token_quant, block_shape)
elif quant_dtype == "mxfp6_e2m3":
@@ -256,6 +256,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
"Please check your environment and try again."
)
self._cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
# Initialized in process_weights_after_loading for CUTLASS/SM90 backends
self.moe_mk: mk.FusedMoEModularKernel | None = None
def create_weights(
@@ -648,19 +649,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS
or self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16
):
layer.gemm1_alpha = Parameter(
torch.tensor([1.702] * self.num_experts, dtype=torch.float32).cuda(),
requires_grad=False,
)
layer.gemm1_beta = Parameter(
torch.tensor([1.0] * self.num_experts, dtype=torch.float32).cuda(),
requires_grad=False,
)
layer.gemm1_clamp_limit = Parameter(
torch.tensor([7.0] * self.num_experts, dtype=torch.float32).cuda(),
requires_grad=False,
)
sf_block_size = 32 # mxfp4 block size
# Common shape assertions
@@ -772,6 +760,30 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
layer.w2_weight_scale = torch.nn.Parameter(
w2_scales_interleaved, requires_grad=False
)
# theses two kernels go through the `flashinfer_cutlass_fused_moe` path
from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
FlashInferExperts,
)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
prepare_finalize = maybe_make_prepare_finalize(
moe=self.moe,
quant_config=self.moe_quant_config,
routing_tables=layer._maybe_init_expert_routing_tables(),
allow_new_interface=True,
)
assert prepare_finalize is not None
self.moe_mk = mk.FusedMoEModularKernel(
prepare_finalize,
FlashInferExperts(
moe_config=self.moe,
quant_config=self.moe_quant_config,
),
shared_experts=None,
)
elif self.mxfp4_backend == Mxfp4Backend.TRITON:
from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig
@@ -847,7 +859,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
)
elif self.mxfp4_backend in [Mxfp4Backend.SM100_FI_MXFP4_BF16]:
elif self.mxfp4_backend in [
Mxfp4Backend.SM100_FI_MXFP4_BF16,
Mxfp4Backend.SM90_FI_MXFP4_BF16,
]:
return mxfp4_w4a16_moe_quant_config(
w1_bias=layer.w13_bias,
w2_bias=layer.w2_bias,
@@ -897,9 +912,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
):
# B200 code-path
kwargs = {
"gemm1_alpha": layer.gemm1_alpha,
"gemm1_beta": layer.gemm1_beta,
"gemm1_clamp_limit": layer.gemm1_clamp_limit,
# TODO(bnell): part of quant_config
"max_capture_size": self.max_capture_size,
}
@@ -935,20 +947,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
if layer.enable_eplb:
raise NotImplementedError("EPLB is not supported for mxfp4")
if self.mxfp4_backend == Mxfp4Backend.MARLIN:
assert self.moe_mk is not None
return self.moe_mk(
hidden_states=x,
w1=layer.w13_weight,
w2=layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
)
assert _can_support_mxfp4(
layer.use_grouped_topk,
layer.topk_group,
@@ -967,68 +965,22 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
assert (
self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS
or self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16
)
from vllm.utils.flashinfer import flashinfer_cutlass_fused_moe
# Backend-specific preparation
if self.mxfp4_backend == Mxfp4Backend.SM100_FI_MXFP4_MXFP8_CUTLASS:
from flashinfer import mxfp8_quantize
x_quant, x_scale = mxfp8_quantize(x, True, 32)
fake_input_scale = torch.ones(self.num_experts, device=x.device)
quant_scales = [
layer.w13_weight_scale.contiguous().view(torch.int32),
fake_input_scale,
layer.w2_weight_scale.contiguous().view(torch.int32),
fake_input_scale,
]
fi_input = x_quant
extra_kwargs = dict(
use_mxfp8_act_scaling=True,
input_sf=x_scale,
fc1_expert_weights=layer.w13_weight.contiguous().view(torch.long),
fc2_expert_weights=layer.w2_weight.contiguous().view(torch.long),
)
elif self.mxfp4_backend == Mxfp4Backend.SM90_FI_MXFP4_BF16:
assert x.dtype == torch.bfloat16
quant_scales = [
layer.w13_weight_scale,
layer.w2_weight_scale,
]
fi_input = x
extra_kwargs = dict(
use_w4_group_scaling=True,
fc1_expert_weights=layer.w13_weight,
fc2_expert_weights=layer.w2_weight,
)
output = torch.empty_like(x, dtype=torch.bfloat16)
flashinfer_cutlass_fused_moe(
input=fi_input,
token_selected_experts=topk_ids.to(torch.int).contiguous(),
token_final_scales=topk_weights,
output_dtype=torch.bfloat16,
output=output,
quant_scales=quant_scales,
fc1_expert_biases=layer.w13_bias,
fc2_expert_biases=layer.w2_bias,
swiglu_alpha=layer.gemm1_alpha,
swiglu_beta=layer.gemm1_beta,
swiglu_limit=layer.gemm1_clamp_limit,
tp_size=self.moe.tp_size,
tp_rank=self.moe.tp_rank,
ep_size=self.moe.ep_size,
ep_rank=self.moe.ep_rank,
tune_max_num_tokens=max(self.max_capture_size, 1),
**extra_kwargs,
or self.mxfp4_backend == Mxfp4Backend.MARLIN
)
return output
assert self.moe_mk is not None
return self.moe_mk(
hidden_states=x,
w1=layer.w13_weight,
w2=layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
expert_map=layer.expert_map,
shared_experts_input=shared_experts_input,
)
def apply_monolithic(
self,
@@ -19,6 +19,7 @@ if TYPE_CHECKING:
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
MXFP_SCALE_DTYPE = torch.uint8
def get_fp8_min_max() -> tuple[float, float]:
@@ -151,6 +152,18 @@ kFp8Static128BlockSym = QuantKey(FP8_DTYPE, kStatic128BlockScale, symmetric=True
kDynamic64Scale = ScaleDesc(torch.float32, False, GroupShape(1, 64))
kFp8Dynamic64Sym = QuantKey(FP8_DTYPE, kDynamic64Scale, symmetric=True)
# TODO (zyongye): Convert all the torch.dtype to scale_dtype
# Changing that requires changing torch compile fused AR+Quant Quant key
# to avoid assertion error
kMxfp4DynamicGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, False, GroupShape(1, 32))
kMxfp4Dynamic = QuantKey(FP4_DTYPE, scale=kMxfp4DynamicGroupScale, symmetric=True)
kMxfp8DynamicGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, False, GroupShape(1, 32))
kMxfp8Dynamic = QuantKey(FP8_DTYPE, scale=kMxfp8DynamicGroupScale, symmetric=True)
kMxfp4StaticGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32))
kMxfp4Static = QuantKey(FP4_DTYPE, scale=kMxfp4StaticGroupScale, symmetric=True)
# Normalize the group_shape to the full extent for any dims that are -1
def _normalize_quant_group_shape(x: torch.Tensor, group_shape: GroupShape):