From 713b28bd0bf4d7f8226ee20940e3ae171fabd09b Mon Sep 17 00:00:00 2001 From: Yuwen Zhou Date: Thu, 7 May 2026 14:17:07 +0800 Subject: [PATCH] [CPU] Add FP8 W8A16 MoE support (#41314) Signed-off-by: yuwenzho --- .buildkite/hardware_tests/cpu.yaml | 2 + tests/kernels/moe/test_cpu_fp8_fused_moe.py | 258 ++++++++++++++++++ tests/quantization/test_cpu_wna16.py | 3 +- vllm/_custom_ops.py | 34 +++ .../layers/fused_moe/experts/cpu_moe.py | 175 ++++++++++++ .../layers/fused_moe/oracle/fp8.py | 27 ++ 6 files changed, 498 insertions(+), 1 deletion(-) create mode 100644 tests/kernels/moe/test_cpu_fp8_fused_moe.py create mode 100644 vllm/model_executor/layers/fused_moe/experts/cpu_moe.py diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml index 77abb5532d..1f6c6e61c9 100644 --- a/.buildkite/hardware_tests/cpu.yaml +++ b/.buildkite/hardware_tests/cpu.yaml @@ -12,6 +12,7 @@ steps: - vllm/_custom_ops.py - tests/kernels/attention/test_cpu_attn.py - tests/kernels/moe/test_cpu_fused_moe.py + - tests/kernels/moe/test_cpu_fp8_fused_moe.py - tests/kernels/test_onednn.py - tests/kernels/test_awq_int4_to_int8.py - tests/kernels/quantization/test_cpu_fp8_scaled_mm.py @@ -20,6 +21,7 @@ steps: bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m " pytest -x -v -s tests/kernels/attention/test_cpu_attn.py pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py + pytest -x -v -s tests/kernels/moe/test_cpu_fp8_fused_moe.py pytest -x -v -s tests/kernels/test_onednn.py pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py" diff --git a/tests/kernels/moe/test_cpu_fp8_fused_moe.py b/tests/kernels/moe/test_cpu_fp8_fused_moe.py new file mode 100644 index 0000000000..695a0d8d46 --- /dev/null +++ b/tests/kernels/moe/test_cpu_fp8_fused_moe.py @@ -0,0 +1,258 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for CPU FP8 W8A16 block-scaled fused MoE kernel.""" + +import math +import sys + +import pytest +import torch +import torch.nn.functional as F + +from vllm.platforms import current_platform +from vllm.utils.torch_utils import set_random_seed + +if not current_platform.is_cpu(): + pytest.skip("skipping CPU-only tests", allow_module_level=True) + +import vllm._custom_ops as ops # noqa: E402 + +if not hasattr(torch.ops._C, "fused_experts_cpu"): + pytest.skip("fused_experts_cpu op not available", allow_module_level=True) + + +BLOCK_SIZE = [128, 128] # [block_n, block_k] + +# FP8 weight generation parameters +_FP8_INFO = torch.finfo(torch.float8_e4m3fn) +FP8_SCALE = _FP8_INFO.max # 448.0 +FACTOR_FOR_SCALE = 1e-3 + +# Tolerance for FP8 W8A16 +FP8_W8A16_ATOL = 1e-2 +FP8_W8A16_RTOL = 1e-2 + + +def _silu_and_mul(x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + return F.silu(x[..., :d]) * x[..., d:] + + +def _block_dequant_weight( + weight: torch.Tensor, + scales: torch.Tensor, + block_size: list[int], +) -> torch.Tensor: + """Block-dequantize FP8 weight [E, N, K] → float [E, N, K]. + + Each (block_n × block_k) tile is multiplied by its per-block scale. + """ + E, N, K = weight.shape + block_n, block_k = block_size + pad_N = (block_n - N % block_n) % block_n + pad_K = (block_k - K % block_k) % block_k + + if pad_N > 0 or pad_K > 0: + weight = F.pad(weight, (0, pad_K, 0, pad_N)) + + n_tiles = math.ceil(N / block_n) + k_tiles = math.ceil(K / block_k) + + weight_block = ( + weight.view(E, n_tiles, block_n, k_tiles, block_k) + .permute(0, 1, 3, 2, 4) + .float() + .contiguous() + ) + weight_scaled = ( + (weight_block * scales.view(E, n_tiles, k_tiles, 1, 1)) + .permute(0, 1, 3, 2, 4) + .contiguous() + ) + if pad_N > 0 or pad_K > 0: + weight_scaled = weight_scaled.view(E, N + pad_N, K + pad_K) + weight_scaled = weight_scaled[..., :N, :K].contiguous() + else: + weight_scaled = weight_scaled.view(E, N, K) + return weight_scaled + + +def ref_w8a16_block_fp8_moe( + a: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + w1_s: torch.Tensor, + w2_s: torch.Tensor, + topk_weight: torch.Tensor, + topk_ids: torch.Tensor, + block_size: list[int], +) -> torch.Tensor: + """Reference FP8 W8A16 block-scaled fused MoE in pure torch. + + Steps: + 1. Block-dequant FP8 weights → float + 2. For each expert: matmul → SiLU+Mul → matmul + 3. Weighted sum across top-k experts + """ + B, D = a.shape + topk = topk_ids.size(1) + + w1_dq = _block_dequant_weight(w1, w1_s, block_size) + w2_dq = _block_dequant_weight(w2, w2_s, block_size) + + a_exp = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D).float() + out = torch.zeros(B * topk, w2_dq.shape[1], dtype=torch.float32) + + topk_weight_flat = topk_weight.view(-1) + topk_ids_flat = topk_ids.view(-1) + + for i in range(w1_dq.shape[0]): + mask = topk_ids_flat == i + if mask.sum(): + ic0 = torch.matmul(a_exp[mask], w1_dq[i].transpose(0, 1)) + ic1 = _silu_and_mul(ic0) + out[mask] = torch.matmul(ic1, w2_dq[i].transpose(0, 1)) + + return ( + (out.view(B, -1, w2_dq.shape[1]) * topk_weight_flat.view(B, -1, 1)) + .sum(dim=1) + .to(a.dtype) + ) + + +def _make_fp8_moe_weights( + E: int, + N: int, + K: int, + block_size: list[int], +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Generate random FP8 MoE weights with random block scales. + + Weight generation follows SGLang: ``randn * FP8_SCALE`` → clamp → cast. + Scales are small random values (``FACTOR_FOR_SCALE``), independent of + the actual weight magnitudes — this is sufficient to test the kernel's + block-dequant + matmul correctness. + + Returns: (w1, w2, w1_s, w2_s) + """ + block_n, block_k = block_size + + w1 = ( + (torch.randn(E, 2 * N, K) * FP8_SCALE) + .clamp(min=-FP8_SCALE, max=FP8_SCALE) + .to(torch.float8_e4m3fn) + ) + w2 = ( + (torch.randn(E, K, N) * FP8_SCALE) + .clamp(min=-FP8_SCALE, max=FP8_SCALE) + .to(torch.float8_e4m3fn) + ) + + w1_s = ( + torch.randn(E, math.ceil(2 * N / block_n), math.ceil(K / block_k)) + * FACTOR_FOR_SCALE + ) + w2_s = ( + torch.randn(E, math.ceil(K / block_n), math.ceil(N / block_k)) + * FACTOR_FOR_SCALE + ) + return w1, w2, w1_s, w2_s + + +def _prepack_experts(w: torch.Tensor) -> torch.Tensor: + """VNNI-prepack each expert's weight via ``convert_weight_packed``.""" + return torch.stack( + [torch.ops._C.convert_weight_packed(w[e]) for e in range(w.shape[0])] + ) + + +NUM_TOKENS = [1, 2, 64, 121] +# (M, intermediate_size N, hidden_size K, num_experts E, topk) +MoE_CONFIGS = [ + (256, 512, 8, 2), + (256, 512, 8, 4), + (512, 256, 8, 2), + (512, 256, 8, 4), + (512, 512, 8, 2), + (512, 512, 8, 4), + (768, 2048, 8, 2), + (768, 2048, 8, 4), + (768, 2048, 128, 8), +] +SEEDS = [0] + + +@pytest.mark.parametrize("M", NUM_TOKENS) +@pytest.mark.parametrize("N,K,E,topk", MoE_CONFIGS) +@pytest.mark.parametrize("seed", SEEDS) +def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed): + """Test fused_experts_cpu FP8 W8A16 against dequantised torch reference.""" + set_random_seed(seed) + + a = torch.randn(M, K, dtype=torch.bfloat16) / math.sqrt(K) + w1, w2, w1_s, w2_s = _make_fp8_moe_weights(E, N, K, BLOCK_SIZE) + + score = torch.randn(M, E, dtype=torch.bfloat16) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_ids = topk_ids.to(torch.int32) + + ref_out = ref_w8a16_block_fp8_moe( + a, + w1, + w2, + w1_s, + w2_s, + topk_weight, + topk_ids, + BLOCK_SIZE, + ) + + pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2) + + # Test inplace=False against reference + out = ops.fused_experts_cpu( + a.clone(), + pw1, + pw2, + topk_weight, + topk_ids, + False, + False, + True, + w1_s, + w2_s, + BLOCK_SIZE, + None, + None, + True, + ) + torch.testing.assert_close( + ref_out.bfloat16(), + out, + atol=FP8_W8A16_ATOL, + rtol=FP8_W8A16_RTOL, + ) + + # Test inplace=True produces identical output + out_inplace = ops.fused_experts_cpu( + a.clone(), + pw1, + pw2, + topk_weight, + topk_ids, + True, + False, + True, + w1_s, + w2_s, + BLOCK_SIZE, + None, + None, + True, + ) + torch.testing.assert_close(out_inplace, out, atol=0, rtol=0) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/tests/quantization/test_cpu_wna16.py b/tests/quantization/test_cpu_wna16.py index f6e54fc147..1fbe222019 100644 --- a/tests/quantization/test_cpu_wna16.py +++ b/tests/quantization/test_cpu_wna16.py @@ -13,7 +13,8 @@ MODELS = [ "Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4", # without g_idx "RedHatAI/Qwen3-1.7B-quantized.w4a16", # with zp "OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", - "Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized + "Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized linear + "Qwen/Qwen3-30B-A3B-FP8", # FP8 W8A16 block-quantized MoE ] DTYPE = ["bfloat16"] diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 430536ecca..9f3f033b48 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -3172,6 +3172,40 @@ if hasattr(torch.ops._C, "fused_experts_cpu"): return torch.empty_like(hidden_states) +def fused_experts_cpu( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + inplace: bool, + use_int8_w8a8: bool, + use_fp8_w8a16: bool, + w1_scale: torch.Tensor | None, + w2_scale: torch.Tensor | None, + block_size: list[int] | None, + a1_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + is_vnni: bool, +) -> torch.Tensor: + return torch.ops._C.fused_experts_cpu( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + inplace, + use_int8_w8a8, + use_fp8_w8a16, + w1_scale, + w2_scale, + block_size, + a1_scale, + a2_scale, + is_vnni, + ) + + if hasattr(torch.ops._C, "int8_scaled_mm_with_quant"): @register_fake("_C::int8_scaled_mm_with_quant") diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py new file mode 100644 index 0000000000..273b5356f6 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CPU FP8 W8A16 block-quantized fused MoE experts.""" + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + FusedMoEQuantConfig, + RoutingMethodType, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kFp8Dynamic128Sym, + kFp8Static128BlockSym, +) +from vllm.platforms import current_platform + + +def prepare_fp8_moe_layer_for_cpu( + w13: torch.Tensor, + w2: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """VNNI-prepack FP8 MoE weights for CPU kernel.""" + num_experts = w13.size(0) + packed_w13_list = [] + packed_w2_list = [] + for i in range(num_experts): + packed_w13_list.append(torch.ops._C.convert_weight_packed(w13[i])) + packed_w2_list.append(torch.ops._C.convert_weight_packed(w2[i])) + packed_w13 = torch.stack(packed_w13_list) + packed_w2 = torch.stack(packed_w2_list) + return packed_w13, packed_w2 + + +class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic): + """CPU FP8 W8A16 block-quantized monolithic MoE experts.""" + + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + ): + super().__init__( + moe_config, + quant_config, + ) + + @property + def expects_unquantized_inputs(self) -> bool: + return True + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def _supports_current_device() -> bool: + return current_platform.is_cpu() + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return activation == MoEActivation.SILU + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + return True + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + SUPPORTED_W_A = [ + (kFp8Static128BlockSym, kFp8Dynamic128Sym), + ] + return (weight_key, activation_key) in SUPPORTED_W_A + + @staticmethod + def _supports_routing_method( + routing_method: RoutingMethodType, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return routing_method in [ + RoutingMethodType.Default, + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ] + + @staticmethod + def _supports_router_logits_dtype( + router_logits_dtype: torch.dtype | None, + routing_method: RoutingMethodType, + ) -> bool: + return True + + def supports_expert_map(self) -> bool: + return False + + def apply( + self, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + router_logits: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + apply_router_weight_on_input: bool, + # grouped topk + fused topk bias parameters + num_expert_group: int | None = None, + e_score_correction_bias: torch.Tensor | None = None, + routed_scaling_factor: float | None = None, + topk_group: int | None = None, + ) -> torch.Tensor: + from vllm.model_executor.layers.fused_moe.cpu_fused_moe import ( + select_experts, + ) + + topk_weights, topk_ids = select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + use_grouped_topk=num_expert_group is not None, + top_k=self.moe_config.experts_per_token, + renormalize=self.moe_config.routing_method + in ( + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ), + topk_group=topk_group, + num_expert_group=num_expert_group, + scoring_func="softmax", + routed_scaling_factor=( + routed_scaling_factor if routed_scaling_factor is not None else 1.0 + ), + e_score_correction_bias=e_score_correction_bias, + ) + + block_shape = ( + list(self.quant_config.block_shape) + if self.quant_config.block_shape + else ( + [self.quant_config._w1.shape.row, self.quant_config._w1.shape.col] + if self.quant_config._w1.shape is not None + else None + ) + ) + + return torch.ops._C.fused_experts_cpu( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + False, # inplace + False, # use_int8_w8a8 + True, # use_fp8_w8a16 + self.w1_scale, # w1_scale + self.w2_scale, # w2_scale + block_shape, # block_size + None, # a1_scale (W8A16: no activation quantization) + None, # a2_scale + True, # is_vnni + ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 74be17eaa5..7ce0c58b71 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -55,6 +55,7 @@ class Fp8MoeBackend(Enum): VLLM_CUTLASS = "VLLM_CUTLASS" BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS" XPU = "XPU" + CPU = "CPU" def _get_priority_backends( @@ -80,6 +81,7 @@ def _get_priority_backends( Fp8MoeBackend.BATCHED_VLLM_CUTLASS, Fp8MoeBackend.BATCHED_TRITON, Fp8MoeBackend.XPU, + Fp8MoeBackend.CPU, ] def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None: @@ -102,6 +104,10 @@ def _get_priority_backends( # move XPU backend to the front. _move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.XPU) + if current_platform.is_cpu(): + # CPU platform uses FP8 W8A16 fused MoE kernel. + _move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.CPU) + return _AVAILABLE_BACKENDS @@ -186,6 +192,13 @@ def backend_to_kernel_cls( return [XPUExpertsFp8] + elif backend == Fp8MoeBackend.CPU: + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + CPUExpertsFp8, + ) + + return [CPUExpertsFp8] + else: raise ValueError(f"Unknown FP8 MoE backend: {backend.value}") @@ -472,6 +485,12 @@ def convert_to_fp8_moe_kernel_format( ) w13, w2 = prepare_fp8_moe_layer_for_xpu(w13, w2) + elif fp8_backend == Fp8MoeBackend.CPU: + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + prepare_fp8_moe_layer_for_cpu, + ) + + w13, w2 = prepare_fp8_moe_layer_for_cpu(w13, w2) else: if fp8_backend not in [ Fp8MoeBackend.TRITON, @@ -516,6 +535,14 @@ def make_fp8_moe_quant_config( block_shape=block_shape, ) + # CPU is mixed precision W8A16 config. + if fp8_backend == Fp8MoeBackend.CPU: + return fp8_w8a16_moe_quant_config( + w1_scale=w1_scale, + w2_scale=w2_scale, + block_shape=block_shape, + ) + # Flashinfer CUTLASS per-tensor uses single dq scale # (alpha = w_scale * a_scale) and inverse a2 scale. if fp8_backend == Fp8MoeBackend.FLASHINFER_CUTLASS and block_shape is None: