[CPU] Add CPU FP8 W8A8 linear/MoE support (#49942)

Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Yuwen Zhou
2026-09-18 17:59:42 +08:00
committed by GitHub
co-authored by Li, Jiang <jiang1.li@intel.com>
parent 6df2b1a8c6
commit b346479065
20 changed files with 2924 additions and 46 deletions
@@ -1055,7 +1055,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
mqa_q_nope,
self.impl._w_uk_packed, # type: ignore[attr-defined]
True,
None,
self.impl._w_scale, # type: ignore[attr-defined]
)
mqa_ql_nope = mqa_ql_nope.transpose(0, 1)
else:
@@ -1401,7 +1401,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
x,
self.impl._w_uv_packed, # type: ignore[attr-defined]
True,
None,
self.impl._w_scale, # type: ignore[attr-defined]
)
else:
# Multiply + Transpose (N, B, L) x (N, L, V)->(N, B, V)->(B, N, V)
@@ -15,6 +15,7 @@ from vllm._custom_ops import (
convert_weight_packed_scale_zp,
cpu_fused_moe,
cpu_fused_moe_int8,
cpu_has_amx_fp8,
cpu_prepack_moe_weight,
cpu_prepack_moe_weight_int8,
fused_experts_cpu,
@@ -467,6 +468,7 @@ class CPUExpertsFp8(mk.FusedMoEExpertsModular):
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepSeekV3,
RoutingMethodType.DeepseekV4,
RoutingMethodType.DeepSeekV3,
]
@@ -1612,3 +1614,218 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
self._w1_scale_bf16,
self._w2_scale_bf16,
)
# ===========================================================================
# FP8 W8A8 MoE
# ===========================================================================
def prepare_fp8_w8a8_moe_layer_for_cpu(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Prepack FP8 W8A8 MoE weights for CPU kernel."""
E = w13.size(0)
packed_w13_list = []
packed_w13_scale_list = []
for i in range(E):
w1_exp = w13[i] # [2N, K] FP8
ws_exp = w13_scale[i] # [2N, K/128] or [2N_blocks, K/128]
# Align scale rows to weight rows if necessary (for block-128 checkpoints
# where scale may have fewer rows: [2N/128, K/128] → [2N, K/128])
if ws_exp.size(0) < w1_exp.size(0):
repeat_factor = w1_exp.size(0) // ws_exp.size(0)
ws_exp = torch.repeat_interleave(ws_exp, repeat_factor, dim=0)
ws_exp = ws_exp[: w1_exp.size(0), :].contiguous()
pw, ps = torch.ops._C.float8_linear_prepack_cpu(w1_exp, ws_exp)
packed_w13_list.append(pw)
packed_w13_scale_list.append(ps)
packed_w13 = torch.stack(packed_w13_list)
packed_w13_scale = torch.stack(packed_w13_scale_list)
# w2 uses the W8A16 path (BF16 activation × FP8 weight), VNNI packed
packed_w2 = torch.ops._C.convert_weight_packed(w2)
return packed_w13, packed_w13_scale, packed_w2, w2_scale
class CPUExpertsFp8W8A8(mk.FusedMoEExpertsModular):
"""CPU FP8 W8A8 block-quantized modular MoE experts."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
@property
def expects_unquantized_inputs(self) -> bool:
# Receives BF16 hidden states; quantization to FP8 is done inside apply()
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
# Requires native AMX-FP8 hardware.
return current_platform.is_cpu() and cpu_has_amx_fp8()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation == MoEActivation.SILU
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
# Block-quantized FP8.
return (weight_key, activation_key) == (
kFp8Static128BlockSym,
kFp8Dynamic128Sym,
)
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepSeekV3,
RoutingMethodType.DeepseekV4,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def moe_problem_size(
self,
a1: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_ids: torch.Tensor,
) -> tuple[int, int, int, int, int]:
# float8_linear_prepack_cpu blocks w1 into [E, Nc, Kc, block_k, block_n],
# not the (E, N, K) layout the base implementation assumes -- N isn't
# recoverable from that shape, so read it from moe_config instead.
E = w1.shape[0]
K = a1.size(-1)
N = (
self.moe_config.intermediate_size_per_partition
* self.moe_config.w13_num_shards
)
M = a1.size(0) if a1.dim() == 2 else a1.size(1)
topk = topk_ids.size(1)
return E, M, N, K, topk
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# fused_experts_cpu manages its own scratch space.
return (0,), (0,), (M, K)
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
if apply_router_weight_on_input:
# fused_experts_cpu always applies topk_weights internally on
# combine; MoEPrepareAndFinalizeNoDPEPModular.prepare() would
# also pre-apply it to hidden_states, double-weighting the
# output. Not needed by any CPU FP8 model today.
raise NotImplementedError(
"CPUExpertsFp8W8A8 does not support apply_router_weight_on_input=True."
)
block_shape = (
list(self.quant_config.block_shape)
if self.quant_config.block_shape
else (
[
self.quant_config._w1.shape.row,
self.quant_config._w1.shape.col,
]
if self.quant_config._w1.shape is not None
else None
)
)
# Quantize hidden_states (BF16) → FP8 with per-token scales
hidden_states_2d = hidden_states.reshape(-1, hidden_states.shape[-1])
x_fp8, x_scales = torch.ops._C.quantize_fp8e4m3_vec(
hidden_states_2d, True, None
)
fused_experts_cpu(
output,
x_fp8,
w1,
w2,
topk_weights,
topk_ids,
CPUQuantMethod.FP8_W8A8,
self.w1_scale, # w1_scale
self.w2_scale, # w2_scale
None, # w1_zero
None, # w2_zero
block_shape, # block_size
None, # w1_bias
None, # w2_bias
None, # alpha
None, # limit
True, # is_vnni
x_scales, # a1_scale: per-token FP8 activation scales
)
@@ -57,7 +57,9 @@ class Fp8MoeBackend(Enum):
VLLM_CUTLASS = "VLLM_CUTLASS"
BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS"
XPU = "XPU"
# CPU FP8 W8A16 (BF16 activations); CPU_W8A8 needs native AMX-FP8.
CPU = "CPU"
CPU_W8A8 = "CPU_W8A8"
HPC = "HPC"
# Dequantize-to-BF16 emulation for MXFP8 on devices without a native
# MXFP8 MoE kernel (e.g. ROCm). Weights pass through unchanged here.
@@ -92,6 +94,7 @@ def _get_priority_backends(
Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
Fp8MoeBackend.BATCHED_TRITON,
Fp8MoeBackend.XPU,
Fp8MoeBackend.CPU_W8A8,
Fp8MoeBackend.CPU,
Fp8MoeBackend.HPC,
]
@@ -129,8 +132,10 @@ def _get_priority_backends(
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.XPU)
if current_platform.is_cpu():
# CPU platform uses FP8 W8A16 fused MoE kernel.
# W8A8 first: it falls through to the W8A16 backend whenever the
# hardware (AMX-FP8) or the config isn't supported.
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.CPU)
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.CPU_W8A8)
return _AVAILABLE_BACKENDS
@@ -238,6 +243,13 @@ def backend_to_kernel_cls(
return [CPUExpertsFp8]
elif backend == Fp8MoeBackend.CPU_W8A8:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsFp8W8A8,
)
return [CPUExpertsFp8W8A8]
elif backend == Fp8MoeBackend.HPC:
from vllm.model_executor.layers.fused_moe.hpc_moe import (
HPCExperts,
@@ -652,6 +664,14 @@ def convert_to_fp8_moe_kernel_format(
)
w13, w2 = prepare_fp8_moe_layer_for_cpu(w13, w2)
elif fp8_backend == Fp8MoeBackend.CPU_W8A8:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_fp8_w8a8_moe_layer_for_cpu,
)
w13, w13_scale, w2, w2_scale = prepare_fp8_w8a8_moe_layer_for_cpu(
w13, w2, w13_scale, w2_scale
)
else:
if fp8_backend not in [
Fp8MoeBackend.TRITON,
@@ -697,7 +717,15 @@ def make_fp8_moe_quant_config(
In a future PR, we will have this function should be
a method of the modular kernel itself.
"""
# MARLIN and CPU are mixed precision W8A16 config.
if fp8_backend == Fp8MoeBackend.CPU_W8A8:
return fp8_w8a8_moe_quant_config(
w1_scale=w1_scale,
w2_scale=w2_scale,
a1_scale=a1_scale,
block_shape=block_shape,
)
# MARLIN and CPU (W8A16) are mixed precision W8A16 configs.
if fp8_backend == Fp8MoeBackend.MARLIN or fp8_backend == Fp8MoeBackend.CPU:
return fp8_w8a16_moe_quant_config(
w1_scale=w1_scale,
@@ -737,11 +737,13 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
)
# Reject backends whose make_fp8_moe_quant_config branch silently
# drops per_act_token_quant / per_out_ch_quant or collapses scales:
# MARLIN / CPU route through fp8_w8a16_moe_quant_config; FLASHINFER_*
# fold scales into a per-tensor alpha (oracle/fp8.py).
# MARLIN / CPU route through fp8_w8a16_moe_quant_config, CPU_W8A8
# ignores both flags, and FLASHINFER_* fold scales into a per-tensor
# alpha (oracle/fp8.py).
if self.fp8_backend in (
Fp8MoeBackend.MARLIN,
Fp8MoeBackend.CPU,
Fp8MoeBackend.CPU_W8A8,
Fp8MoeBackend.FLASHINFER_CUTLASS,
Fp8MoeBackend.FLASHINFER_TRTLLM,
):