[CPU] Add FP8 W8A16 MoE support (#41314)

Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
This commit is contained in:
Yuwen Zhou
2026-05-06 23:17:07 -07:00
committed by GitHub
parent 51f22dcfd0
commit 713b28bd0b
6 changed files with 498 additions and 1 deletions
@@ -0,0 +1,175 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU FP8 W8A16 block-quantized fused MoE experts."""
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
)
from vllm.platforms import current_platform
def prepare_fp8_moe_layer_for_cpu(
w13: torch.Tensor,
w2: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""VNNI-prepack FP8 MoE weights for CPU kernel."""
num_experts = w13.size(0)
packed_w13_list = []
packed_w2_list = []
for i in range(num_experts):
packed_w13_list.append(torch.ops._C.convert_weight_packed(w13[i]))
packed_w2_list.append(torch.ops._C.convert_weight_packed(w2[i]))
packed_w13 = torch.stack(packed_w13_list)
packed_w2 = torch.stack(packed_w2_list)
return packed_w13, packed_w2
class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
"""CPU FP8 W8A16 block-quantized monolithic MoE experts."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(
moe_config,
quant_config,
)
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation == MoEActivation.SILU
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def supports_expert_map(self) -> bool:
return False
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
select_experts,
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
block_shape = (
list(self.quant_config.block_shape)
if self.quant_config.block_shape
else (
[self.quant_config._w1.shape.row, self.quant_config._w1.shape.col]
if self.quant_config._w1.shape is not None
else None
)
)
return torch.ops._C.fused_experts_cpu(
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
False, # inplace
False, # use_int8_w8a8
True, # use_fp8_w8a16
self.w1_scale, # w1_scale
self.w2_scale, # w2_scale
block_shape, # block_size
None, # a1_scale (W8A16: no activation quantization)
None, # a2_scale
True, # is_vnni
)
@@ -55,6 +55,7 @@ class Fp8MoeBackend(Enum):
VLLM_CUTLASS = "VLLM_CUTLASS"
BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS"
XPU = "XPU"
CPU = "CPU"
def _get_priority_backends(
@@ -80,6 +81,7 @@ def _get_priority_backends(
Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
Fp8MoeBackend.BATCHED_TRITON,
Fp8MoeBackend.XPU,
Fp8MoeBackend.CPU,
]
def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None:
@@ -102,6 +104,10 @@ def _get_priority_backends(
# move XPU backend to the front.
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.XPU)
if current_platform.is_cpu():
# CPU platform uses FP8 W8A16 fused MoE kernel.
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.CPU)
return _AVAILABLE_BACKENDS
@@ -186,6 +192,13 @@ def backend_to_kernel_cls(
return [XPUExpertsFp8]
elif backend == Fp8MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsFp8,
)
return [CPUExpertsFp8]
else:
raise ValueError(f"Unknown FP8 MoE backend: {backend.value}")
@@ -472,6 +485,12 @@ def convert_to_fp8_moe_kernel_format(
)
w13, w2 = prepare_fp8_moe_layer_for_xpu(w13, w2)
elif fp8_backend == Fp8MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_fp8_moe_layer_for_cpu,
)
w13, w2 = prepare_fp8_moe_layer_for_cpu(w13, w2)
else:
if fp8_backend not in [
Fp8MoeBackend.TRITON,
@@ -516,6 +535,14 @@ def make_fp8_moe_quant_config(
block_shape=block_shape,
)
# CPU is mixed precision W8A16 config.
if fp8_backend == Fp8MoeBackend.CPU:
return fp8_w8a16_moe_quant_config(
w1_scale=w1_scale,
w2_scale=w2_scale,
block_shape=block_shape,
)
# Flashinfer CUTLASS per-tensor uses single dq scale
# (alpha = w_scale * a_scale) and inverse a2 scale.
if fp8_backend == Fp8MoeBackend.FLASHINFER_CUTLASS and block_shape is None: