[CPU] Add FP8 W8A16 MoE support (#41314)

Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
This commit is contained in:
Yuwen Zhou
2026-05-06 23:17:07 -07:00
committed by GitHub
parent 51f22dcfd0
commit 713b28bd0b
6 changed files with 498 additions and 1 deletions
+2
View File
@@ -12,6 +12,7 @@ steps:
- vllm/_custom_ops.py
- tests/kernels/attention/test_cpu_attn.py
- tests/kernels/moe/test_cpu_fused_moe.py
- tests/kernels/moe/test_cpu_fp8_fused_moe.py
- tests/kernels/test_onednn.py
- tests/kernels/test_awq_int4_to_int8.py
- tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
@@ -20,6 +21,7 @@ steps:
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
pytest -x -v -s tests/kernels/moe/test_cpu_fp8_fused_moe.py
pytest -x -v -s tests/kernels/test_onednn.py
pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py
pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py"
+258
View File
@@ -0,0 +1,258 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for CPU FP8 W8A16 block-scaled fused MoE kernel."""
import math
import sys
import pytest
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
import vllm._custom_ops as ops # noqa: E402
if not hasattr(torch.ops._C, "fused_experts_cpu"):
pytest.skip("fused_experts_cpu op not available", allow_module_level=True)
BLOCK_SIZE = [128, 128] # [block_n, block_k]
# FP8 weight generation parameters
_FP8_INFO = torch.finfo(torch.float8_e4m3fn)
FP8_SCALE = _FP8_INFO.max # 448.0
FACTOR_FOR_SCALE = 1e-3
# Tolerance for FP8 W8A16
FP8_W8A16_ATOL = 1e-2
FP8_W8A16_RTOL = 1e-2
def _silu_and_mul(x: torch.Tensor) -> torch.Tensor:
d = x.shape[-1] // 2
return F.silu(x[..., :d]) * x[..., d:]
def _block_dequant_weight(
weight: torch.Tensor,
scales: torch.Tensor,
block_size: list[int],
) -> torch.Tensor:
"""Block-dequantize FP8 weight [E, N, K] → float [E, N, K].
Each (block_n × block_k) tile is multiplied by its per-block scale.
"""
E, N, K = weight.shape
block_n, block_k = block_size
pad_N = (block_n - N % block_n) % block_n
pad_K = (block_k - K % block_k) % block_k
if pad_N > 0 or pad_K > 0:
weight = F.pad(weight, (0, pad_K, 0, pad_N))
n_tiles = math.ceil(N / block_n)
k_tiles = math.ceil(K / block_k)
weight_block = (
weight.view(E, n_tiles, block_n, k_tiles, block_k)
.permute(0, 1, 3, 2, 4)
.float()
.contiguous()
)
weight_scaled = (
(weight_block * scales.view(E, n_tiles, k_tiles, 1, 1))
.permute(0, 1, 3, 2, 4)
.contiguous()
)
if pad_N > 0 or pad_K > 0:
weight_scaled = weight_scaled.view(E, N + pad_N, K + pad_K)
weight_scaled = weight_scaled[..., :N, :K].contiguous()
else:
weight_scaled = weight_scaled.view(E, N, K)
return weight_scaled
def ref_w8a16_block_fp8_moe(
a: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
w1_s: torch.Tensor,
w2_s: torch.Tensor,
topk_weight: torch.Tensor,
topk_ids: torch.Tensor,
block_size: list[int],
) -> torch.Tensor:
"""Reference FP8 W8A16 block-scaled fused MoE in pure torch.
Steps:
1. Block-dequant FP8 weights → float
2. For each expert: matmul → SiLU+Mul → matmul
3. Weighted sum across top-k experts
"""
B, D = a.shape
topk = topk_ids.size(1)
w1_dq = _block_dequant_weight(w1, w1_s, block_size)
w2_dq = _block_dequant_weight(w2, w2_s, block_size)
a_exp = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D).float()
out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32)
topk_weight_flat = topk_weight.view(-1)
topk_ids_flat = topk_ids.view(-1)
for i in range(w1_dq.shape[0]):
mask = topk_ids_flat == i
if mask.sum():
ic0 = torch.matmul(a_exp[mask], w1_dq[i].transpose(0, 1))
ic1 = _silu_and_mul(ic0)
out[mask] = torch.matmul(ic1, w2_dq[i].transpose(0, 1))
return (
(out.view(B, -1, w2_dq.shape[1]) * topk_weight_flat.view(B, -1, 1))
.sum(dim=1)
.to(a.dtype)
)
def _make_fp8_moe_weights(
E: int,
N: int,
K: int,
block_size: list[int],
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Generate random FP8 MoE weights with random block scales.
Weight generation follows SGLang: ``randn * FP8_SCALE`` → clamp → cast.
Scales are small random values (``FACTOR_FOR_SCALE``), independent of
the actual weight magnitudes — this is sufficient to test the kernel's
block-dequant + matmul correctness.
Returns: (w1, w2, w1_s, w2_s)
"""
block_n, block_k = block_size
w1 = (
(torch.randn(E, 2 * N, K) * FP8_SCALE)
.clamp(min=-FP8_SCALE, max=FP8_SCALE)
.to(torch.float8_e4m3fn)
)
w2 = (
(torch.randn(E, K, N) * FP8_SCALE)
.clamp(min=-FP8_SCALE, max=FP8_SCALE)
.to(torch.float8_e4m3fn)
)
w1_s = (
torch.randn(E, math.ceil(2 * N / block_n), math.ceil(K / block_k))
* FACTOR_FOR_SCALE
)
w2_s = (
torch.randn(E, math.ceil(K / block_n), math.ceil(N / block_k))
* FACTOR_FOR_SCALE
)
return w1, w2, w1_s, w2_s
def _prepack_experts(w: torch.Tensor) -> torch.Tensor:
"""VNNI-prepack each expert's weight via ``convert_weight_packed``."""
return torch.stack(
[torch.ops._C.convert_weight_packed(w[e]) for e in range(w.shape[0])]
)
NUM_TOKENS = [1, 2, 64, 121]
# (M, intermediate_size N, hidden_size K, num_experts E, topk)
MoE_CONFIGS = [
(256, 512, 8, 2),
(256, 512, 8, 4),
(512, 256, 8, 2),
(512, 256, 8, 4),
(512, 512, 8, 2),
(512, 512, 8, 4),
(768, 2048, 8, 2),
(768, 2048, 8, 4),
(768, 2048, 128, 8),
]
SEEDS = [0]
@pytest.mark.parametrize("M", NUM_TOKENS)
@pytest.mark.parametrize("N,K,E,topk", MoE_CONFIGS)
@pytest.mark.parametrize("seed", SEEDS)
def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
"""Test fused_experts_cpu FP8 W8A16 against dequantised torch reference."""
set_random_seed(seed)
a = torch.randn(M, K, dtype=torch.bfloat16) / math.sqrt(K)
w1, w2, w1_s, w2_s = _make_fp8_moe_weights(E, N, K, BLOCK_SIZE)
score = torch.randn(M, E, dtype=torch.bfloat16)
score = torch.softmax(score, dim=-1, dtype=torch.float32)
topk_weight, topk_ids = torch.topk(score, topk)
topk_ids = topk_ids.to(torch.int32)
ref_out = ref_w8a16_block_fp8_moe(
a,
w1,
w2,
w1_s,
w2_s,
topk_weight,
topk_ids,
BLOCK_SIZE,
)
pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2)
# Test inplace=False against reference
out = ops.fused_experts_cpu(
a.clone(),
pw1,
pw2,
topk_weight,
topk_ids,
False,
False,
True,
w1_s,
w2_s,
BLOCK_SIZE,
None,
None,
True,
)
torch.testing.assert_close(
ref_out.bfloat16(),
out,
atol=FP8_W8A16_ATOL,
rtol=FP8_W8A16_RTOL,
)
# Test inplace=True produces identical output
out_inplace = ops.fused_experts_cpu(
a.clone(),
pw1,
pw2,
topk_weight,
topk_ids,
True,
False,
True,
w1_s,
w2_s,
BLOCK_SIZE,
None,
None,
True,
)
torch.testing.assert_close(out_inplace, out, atol=0, rtol=0)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
+2 -1
View File
@@ -13,7 +13,8 @@ MODELS = [
"Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4", # without g_idx
"RedHatAI/Qwen3-1.7B-quantized.w4a16", # with zp
"OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
"Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized
"Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized linear
"Qwen/Qwen3-30B-A3B-FP8", # FP8 W8A16 block-quantized MoE
]
DTYPE = ["bfloat16"]
+34
View File
@@ -3172,6 +3172,40 @@ if hasattr(torch.ops._C, "fused_experts_cpu"):
return torch.empty_like(hidden_states)
def fused_experts_cpu(
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
inplace: bool,
use_int8_w8a8: bool,
use_fp8_w8a16: bool,
w1_scale: torch.Tensor | None,
w2_scale: torch.Tensor | None,
block_size: list[int] | None,
a1_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
is_vnni: bool,
) -> torch.Tensor:
return torch.ops._C.fused_experts_cpu(
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
inplace,
use_int8_w8a8,
use_fp8_w8a16,
w1_scale,
w2_scale,
block_size,
a1_scale,
a2_scale,
is_vnni,
)
if hasattr(torch.ops._C, "int8_scaled_mm_with_quant"):
@register_fake("_C::int8_scaled_mm_with_quant")
@@ -0,0 +1,175 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU FP8 W8A16 block-quantized fused MoE experts."""
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
)
from vllm.platforms import current_platform
def prepare_fp8_moe_layer_for_cpu(
w13: torch.Tensor,
w2: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""VNNI-prepack FP8 MoE weights for CPU kernel."""
num_experts = w13.size(0)
packed_w13_list = []
packed_w2_list = []
for i in range(num_experts):
packed_w13_list.append(torch.ops._C.convert_weight_packed(w13[i]))
packed_w2_list.append(torch.ops._C.convert_weight_packed(w2[i]))
packed_w13 = torch.stack(packed_w13_list)
packed_w2 = torch.stack(packed_w2_list)
return packed_w13, packed_w2
class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
"""CPU FP8 W8A16 block-quantized monolithic MoE experts."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(
moe_config,
quant_config,
)
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation == MoEActivation.SILU
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def supports_expert_map(self) -> bool:
return False
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
select_experts,
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
block_shape = (
list(self.quant_config.block_shape)
if self.quant_config.block_shape
else (
[self.quant_config._w1.shape.row, self.quant_config._w1.shape.col]
if self.quant_config._w1.shape is not None
else None
)
)
return torch.ops._C.fused_experts_cpu(
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
False, # inplace
False, # use_int8_w8a8
True, # use_fp8_w8a16
self.w1_scale, # w1_scale
self.w2_scale, # w2_scale
block_shape, # block_size
None, # a1_scale (W8A16: no activation quantization)
None, # a2_scale
True, # is_vnni
)
@@ -55,6 +55,7 @@ class Fp8MoeBackend(Enum):
VLLM_CUTLASS = "VLLM_CUTLASS"
BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS"
XPU = "XPU"
CPU = "CPU"
def _get_priority_backends(
@@ -80,6 +81,7 @@ def _get_priority_backends(
Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
Fp8MoeBackend.BATCHED_TRITON,
Fp8MoeBackend.XPU,
Fp8MoeBackend.CPU,
]
def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None:
@@ -102,6 +104,10 @@ def _get_priority_backends(
# move XPU backend to the front.
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.XPU)
if current_platform.is_cpu():
# CPU platform uses FP8 W8A16 fused MoE kernel.
_move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.CPU)
return _AVAILABLE_BACKENDS
@@ -186,6 +192,13 @@ def backend_to_kernel_cls(
return [XPUExpertsFp8]
elif backend == Fp8MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsFp8,
)
return [CPUExpertsFp8]
else:
raise ValueError(f"Unknown FP8 MoE backend: {backend.value}")
@@ -472,6 +485,12 @@ def convert_to_fp8_moe_kernel_format(
)
w13, w2 = prepare_fp8_moe_layer_for_xpu(w13, w2)
elif fp8_backend == Fp8MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_fp8_moe_layer_for_cpu,
)
w13, w2 = prepare_fp8_moe_layer_for_cpu(w13, w2)
else:
if fp8_backend not in [
Fp8MoeBackend.TRITON,
@@ -516,6 +535,14 @@ def make_fp8_moe_quant_config(
block_shape=block_shape,
)
# CPU is mixed precision W8A16 config.
if fp8_backend == Fp8MoeBackend.CPU:
return fp8_w8a16_moe_quant_config(
w1_scale=w1_scale,
w2_scale=w2_scale,
block_shape=block_shape,
)
# Flashinfer CUTLASS per-tensor uses single dq scale
# (alpha = w_scale * a_scale) and inverse a2 scale.
if fp8_backend == Fp8MoeBackend.FLASHINFER_CUTLASS and block_shape is None: