[CPU] Add MXFP4 W4A16 MoE support (#41922)

Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
Signed-off-by: Yuwen Zhou <yuwen.zhou@intel.com>
This commit is contained in:
Yuwen Zhou
2026-05-18 03:04:45 -07:00
committed by GitHub
parent cac81b6eda
commit 88a860d754
16 changed files with 1172 additions and 349 deletions
@@ -213,6 +213,10 @@ class SGLFusedMOE:
None, # w1_zero
None, # w2_zero
None, # block_size
None, # w1_bias
None, # w2_bias
None, # alpha
None, # limit
True, # is_vnni
)
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU FP8 W8A16 block-quantized fused MoE experts."""
"""CPU FP8 W8A16 and MXFP4 W4A16 fused MoE experts."""
import torch
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
kMxfp4Static,
)
from vllm.platforms import current_platform
@@ -26,14 +27,8 @@ def prepare_fp8_moe_layer_for_cpu(
w2: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""VNNI-prepack FP8 MoE weights for CPU kernel."""
num_experts = w13.size(0)
packed_w13_list = []
packed_w2_list = []
for i in range(num_experts):
packed_w13_list.append(torch.ops._C.convert_weight_packed(w13[i]))
packed_w2_list.append(torch.ops._C.convert_weight_packed(w2[i]))
packed_w13 = torch.stack(packed_w13_list)
packed_w2 = torch.stack(packed_w2_list)
packed_w13 = torch.ops._C.convert_weight_packed(w13)
packed_w2 = torch.ops._C.convert_weight_packed(w2)
return packed_w13, packed_w2
@@ -171,5 +166,161 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
None, # w1_zero
None, # w2_zero
block_shape, # block_size
None, # w1_bias
None, # w2_bias
None, # alpha
None, # limit
True, # is_vnni
)
def prepare_mxfp4_moe_layer_for_cpu(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""VNNI-prepack MXFP4 MoE weights and repack scales for CPU AMX kernel."""
packed_w13 = torch.ops._C.convert_weight_packed(w13)
packed_w2 = torch.ops._C.convert_weight_packed(w2)
packed_w13_scale = torch.ops._C.convert_scale_packed(w13_scale)
packed_w2_scale = torch.ops._C.convert_scale_packed(w2_scale)
return packed_w13, packed_w2, packed_w13_scale, packed_w2_scale
class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
"""CPU MXFP4 W4A16 monolithic MoE experts."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(
moe_config,
quant_config,
)
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in (MoEActivation.SILU, MoEActivation.SWIGLUOAI)
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kMxfp4Static, None),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def supports_expert_map(self) -> bool:
return False
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
select_experts,
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
# Get bias and swiglu params from quant config
w1_bias = self.quant_config.w1_bias
w2_bias = self.quant_config.w2_bias
alpha = getattr(self.quant_config, "gemm1_alpha", None)
limit = getattr(self.quant_config, "gemm1_clamp_limit", None)
return fused_experts_cpu(
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
False, # inplace
CPUQuantMethod.MXFP4, # moe_comp_method
self.w1_scale, # w1_scale
self.w2_scale, # w2_scale
None, # w1_zero
None, # w2_zero
None, # block_size
w1_bias,
w2_bias,
alpha,
limit,
True, # is_vnni
)
@@ -79,6 +79,8 @@ class Mxfp4MoeBackend(Enum):
TRITON_UNFUSED = "TRITON_UNFUSED"
# XPU
XPU = "XPU"
# CPU
CPU = "CPU"
# Emulation
EMULATION = "EMULATION"
# Humming
@@ -198,6 +200,11 @@ def backend_to_kernel_cls(
return [XPUExpertsMXFp4]
elif backend == Mxfp4MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import CPUExpertsMxfp4
return [CPUExpertsMxfp4]
elif backend == Mxfp4MoeBackend.EMULATION:
from vllm.model_executor.layers.fused_moe.experts.ocp_mx_emulation_moe import (
OCP_MXQuantizationEmulationTritonExperts,
@@ -237,6 +244,7 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
],
"aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8],
"xpu": [Mxfp4MoeBackend.XPU],
"cpu": [Mxfp4MoeBackend.CPU],
"emulation": [Mxfp4MoeBackend.EMULATION],
}
if backends := mapping.get(runner_backend):
@@ -535,6 +543,17 @@ def select_mxfp4_moe_backend(
activation_format,
)
if current_platform.is_cpu():
backend = Mxfp4MoeBackend.CPU
logger.info_once(_make_log_backend(backend))
return _return_or_raise(
Mxfp4MoeBackend.CPU,
config,
kMxfp4Static,
None,
activation_format,
)
if current_platform.is_cuda() or current_platform.is_rocm():
raise NotImplementedError(
"No MXFP4 MoE backend supports the deployment configuration."
@@ -639,6 +658,10 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
elif current_platform.is_rocm():
intermediate_size = round_up(intermediate_size, 256)
hidden_size = round_up(hidden_size, 256)
elif backend == Mxfp4MoeBackend.CPU:
# CPU AMX kernel uses BLOCK_N=32, align to 32
intermediate_size = round_up(intermediate_size, 32)
hidden_size = round_up(hidden_size, 32)
else:
intermediate_size = round_up(intermediate_size, 64)
return hidden_size, intermediate_size
@@ -1098,6 +1121,31 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
w13_bias,
w2_bias,
)
elif mxfp4_backend == Mxfp4MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_mxfp4_moe_layer_for_cpu,
)
packed_w13, packed_w2, packed_w13_scale, packed_w2_scale = (
prepare_mxfp4_moe_layer_for_cpu(
w13_weight.data,
w2_weight.data,
w13_weight_scale.data,
w2_weight_scale.data,
)
)
if w13_bias is not None:
w13_bias = w13_bias.data.to(torch.float32)
if w2_bias is not None:
w2_bias = w2_bias.data.to(torch.float32)
return (
packed_w13,
packed_w2,
packed_w13_scale,
packed_w2_scale,
w13_bias,
w2_bias,
)
elif mxfp4_backend == Mxfp4MoeBackend.EMULATION:
# No additional transformation needed for emulation backend,
# weights are dequantized on the fly in the experts class.
@@ -1501,6 +1549,7 @@ def make_mxfp4_moe_quant_config(
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
Mxfp4MoeBackend.AITER_MXFP4_BF16,
Mxfp4MoeBackend.CPU,
):
return mxfp4_w4a16_moe_quant_config(
w1_bias=w1_bias,