[CPU] Migrate unquantized MoE to the modular-kernel experts structure (#50133)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
Li, Jiang
2026-08-03 14:55:04 +08:00
committed by GitHub
co-authored by Claude Opus 5 Fadi Arafeh
parent b3f97dae24
commit 0a6446005d
20 changed files with 972 additions and 839 deletions
+161 -48
View File
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
from vllm._custom_ops import (
cpu_fused_moe,
@@ -12,9 +15,18 @@ from vllm._custom_ops import (
cpu_prepack_moe_weight_int8,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
_CPU_MOE_ACT_FN,
CPUFusedMOE,
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
biased_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
ArmCPUUnquantizedExperts,
CPUUnquantizedExperts,
X86CPUUnquantizedExperts,
select_experts,
)
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -36,7 +48,10 @@ ACT = [
]
USE_BIAS = [False, True]
ISA = ["vec"]
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
if (
current_platform.get_cpu_architecture() == CpuArchEnum.ARM
and sys.platform != "darwin"
):
ISA.append("neon")
if torch.cpu._is_amx_tile_supported():
ISA.append("amx")
@@ -44,6 +59,27 @@ if torch.cpu._is_amx_tile_supported():
DTYPE = [torch.bfloat16]
def _ref_moe_activation(
input: torch.Tensor,
activation: MoEActivation,
) -> torch.Tensor:
if activation == MoEActivation.SWIGLUOAI:
gate = input[..., ::2].clamp(max=7.0)
up = input[..., 1::2].clamp(min=-7.0, max=7.0)
glu = gate * torch.sigmoid(gate * 1.702)
return (up + 1) * glu
d = input.shape[-1] // 2
gate, up = input[..., :d], input[..., d:]
if activation == MoEActivation.SILU:
return torch.nn.functional.silu(gate) * up
if activation == MoEActivation.GELU:
return torch.nn.functional.gelu(gate, approximate="none") * up
if activation == MoEActivation.GELU_TANH:
return torch.nn.functional.gelu(gate, approximate="tanh") * up
raise ValueError(f"Unsupported activation: {activation}")
def ref_fused_moe(
input: torch.Tensor,
w13: torch.Tensor,
@@ -87,7 +123,8 @@ def ref_fused_moe(
tokens_for_this_expert, curr_w13, curr_w13_bias
)
# Note: to simulate the kernel implementation
gate_up = _CPU_MOE_ACT_FN[activation](gate_up).to(dtype=input.dtype).float()
gate_up = _ref_moe_activation(gate_up, activation)
gate_up = gate_up.to(dtype=input.dtype).float()
expert_out = torch.nn.functional.linear(gate_up, curr_w2, curr_w2_bias)
outputs.append(expert_out)
@@ -166,7 +203,7 @@ def ref_fused_moe_int8(
if w13_bias is not None:
gate_up += w13_bias[expert_idx].float()
intermediate = _CPU_MOE_ACT_FN[activation](gate_up).to(input.dtype)
intermediate = _ref_moe_activation(gate_up, activation).to(input.dtype)
intermediate_int8, intermediate_scale = quantize_per_token(intermediate)
output = torch.matmul(
intermediate_int8.float(),
@@ -356,8 +393,8 @@ UNALIGNED_INTERMEDIATE_DIM = 176
class _StubMoELayer(torch.nn.Module):
"""Minimal stand-in for the real MoE layer module, exposing just what
CPUFusedMOE reads/replaces (w13_weight, w2_weight, activation, and
optionally w13_bias/w2_bias)."""
the unquantized CPU experts read/replace (w13_weight, w2_weight, the
router configuration, and optionally w13_bias/w2_bias)."""
def __init__(
self,
@@ -375,12 +412,77 @@ class _StubMoELayer(torch.nn.Module):
self.w13_bias = torch.nn.Parameter(w13_bias, requires_grad=False)
if w2_bias is not None:
self.w2_bias = torch.nn.Parameter(w2_bias, requires_grad=False)
self.use_grouped_topk = False
self.renormalize = False
self.scoring_func = "softmax"
self.custom_routing_function = None
def _make_moe_config(
expert_num: int,
hidden_size: int,
intermediate_size: int,
topk_num: int,
dtype: torch.dtype,
act: MoEActivation,
has_bias: bool = False,
) -> FusedMoEConfig:
return FusedMoEConfig(
num_experts=expert_num,
experts_per_token=topk_num,
hidden_dim=hidden_size,
intermediate_size=intermediate_size,
num_local_experts=expert_num,
num_logical_experts=expert_num,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=act,
in_dtype=dtype,
device="cpu",
routing_method=RoutingMethodType.Default,
has_bias=has_bias,
)
@pytest.mark.parametrize(
("hidden_size", "intermediate_size", "act", "expected"),
[
(128, 128, MoEActivation.SILU, True),
# Hidden size is not 32-aligned.
(112, 128, MoEActivation.SILU, False),
(128, 176, MoEActivation.SILU, True),
# SwiGLUOAI's interleaved gate/up layout cannot be padded.
(128, 176, MoEActivation.SWIGLUOAI, False),
],
)
def test_cpu_vec_fused_moe_shape_support(
hidden_size: int,
intermediate_size: int,
act: MoEActivation,
expected: bool,
):
moe_config = _make_moe_config(
expert_num=8,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
topk_num=4,
dtype=torch.bfloat16,
act=act,
)
supported, _ = CPUUnquantizedExperts.is_supported_config(
CPUUnquantizedExperts,
moe_config,
None,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported is expected
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
@pytest.mark.parametrize("use_bias", USE_BIAS)
@pytest.mark.parametrize("dtype", DTYPE)
@pytest.mark.parametrize("isa", ISA)
@pytest.mark.parametrize(
"act",
[MoEActivation.SILU, MoEActivation.GELU, MoEActivation.GELU_TANH],
@@ -391,14 +493,11 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
hidden_size: int,
use_bias: bool,
dtype: torch.dtype,
isa: str,
act: MoEActivation,
):
"""An unaligned per-partition moe_intermediate_size must still hit the
grouped-gemm fast path via automatic zero-padding, with numerically
correct output, instead of silently falling back to the much slower
per-expert torch loop."""
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
pytest.skip("padding is only applied on the x86 AMX/vector kernels")
"""CPU kernels handle unaligned intermediate sizes by zero-padding the
weights before prepacking."""
set_random_seed(0)
batch_size = 64
@@ -423,9 +522,15 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
score = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
topk_weight, topk_ids = torch.topk(score, topk_num)
topk_ids = topk_ids.to(torch.int32)
# Route with the same helper apply() uses internally, so the reference
# only differs from the kernel in how the experts are evaluated.
topk_weight, topk_ids = select_experts(
hidden_states=input,
router_logits=router_logits,
top_k=topk_num,
use_grouped_topk=False,
renormalize=False,
)
ref_output = ref_fused_moe(
input, w13, w2, w13_bias, w2_bias, topk_weight, topk_ids, act
@@ -439,40 +544,48 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
w2_bias.clone() if w2_bias is not None else None,
)
cpu_moe = CPUFusedMOE(layer)
assert cpu_moe.forward_method == cpu_moe.forward_grouped_gemm, (
"expected the padded intermediate size to hit the grouped-gemm fast path"
moe_config = _make_moe_config(
expert_num, hidden_size, intermediate_size, topk_num, dtype, act, use_bias
)
experts_cls = {
"vec": CPUUnquantizedExperts,
"amx": X86CPUUnquantizedExperts,
"neon": ArmCPUUnquantizedExperts,
}[isa]
supported, reason = experts_cls.is_supported_config(
experts_cls,
moe_config,
None,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, (
f"expected the padded intermediate size to be supported, got {reason}"
)
output = cpu_moe.forward_method(
layer, input, topk_weight, topk_ids, act, expert_num, False
# Mirror UnquantizedFusedMoEMethod._setup_kernel: the quant config is
# built from the layer first, then the experts shuffle the layer into the
# kernel's runtime format.
quant_config = (
biased_moe_quant_config(layer.w13_bias, layer.w2_bias)
if use_bias
else FusedMoEQuantConfig.make()
)
experts = experts_cls(moe_config, quant_config)
assert experts.isa == isa
experts.process_weights_after_loading(layer)
output = experts.apply(
hidden_states=input,
w1=layer.w13_weight,
w2=layer.w2_weight,
router_logits=router_logits,
activation=act,
global_num_experts=expert_num,
expert_map=None,
a1q_scale=None,
apply_router_weight_on_input=False,
)
atol, rtol = get_default_atol(output), get_default_rtol(output)
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
def test_cpu_fused_moe_unaligned_intermediate_size_swigluoai(default_vllm_config):
"""swigluoai's interleaved gate/up layout isn't padded automatically. On
AMX-capable CPUs this must raise rather than silently falling back to
the (correct but much slower) per-expert torch loop; elsewhere it should
fall back exactly as before."""
set_random_seed(0)
expert_num = 8
hidden_size = 128
intermediate_size = UNALIGNED_INTERMEDIATE_DIM
dtype = torch.bfloat16
up_dim = 2 * intermediate_size
layer = _StubMoELayer(
torch.randn((expert_num, up_dim, hidden_size), dtype=dtype),
torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype),
MoEActivation.SWIGLUOAI,
)
if torch.cpu._is_amx_tile_supported():
with pytest.raises(RuntimeError):
CPUFusedMOE(layer)
else:
cpu_moe = CPUFusedMOE(layer)
assert cpu_moe.forward_method == cpu_moe.forward_torch
-79
View File
@@ -1482,85 +1482,6 @@ def test_moe_sum_pad_aware(topk: int, dtype: torch.dtype, topk_ids_dtype: torch.
opcheck(torch.ops._moe_C.moe_sum, (input, actual, topk_ids, expert_map))
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize("m", [1, 33])
@pytest.mark.parametrize("n,k", [(128, 128)])
@pytest.mark.parametrize("e", [8])
@pytest.mark.parametrize("topk", [2])
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize("with_bias", [False, True])
@pytest.mark.parametrize("activation", [MoEActivation.SILU])
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only test")
def test_cpu_fused_moe_basic(
m: int,
n: int,
k: int,
e: int,
topk: int,
dtype: torch.dtype,
with_bias: bool,
activation: MoEActivation,
):
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import CPUFusedMOE
device = "cpu"
set_random_seed(7)
a = torch.randn((m, k), device=device, dtype=dtype) / 10
w13 = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / 10
w2 = torch.randn((e, k, n), device=device, dtype=dtype) / 10
router_logits = torch.randn((m, e), device=device, dtype=dtype)
b1 = b2 = None
if with_bias:
b1 = torch.randn((e, 2 * n), device=device, dtype=dtype) / 10
b2 = torch.randn((e, k), device=device, dtype=dtype) / 10
ref = (
torch_moe(a, w13, w2, router_logits, topk, b1, b2)
if with_bias
else torch_moe(a, w13, w2, router_logits, topk)
)
class _Dummy(torch.nn.Module):
def __init__(self, w13, w2, b1=None, b2=None):
super().__init__()
self.w13_weight = torch.nn.Parameter(w13, requires_grad=False)
self.w2_weight = torch.nn.Parameter(w2, requires_grad=False)
if b1 is not None:
self.w13_bias = torch.nn.Parameter(b1, requires_grad=False)
if b2 is not None:
self.w2_bias = torch.nn.Parameter(b2, requires_grad=False)
layer = _Dummy(w13, w2, b1, b2).to(dtype)
fused = CPUFusedMOE(layer)
out = fused(
layer=layer,
x=a,
use_grouped_topk=False,
top_k=topk,
router_logits=router_logits,
renormalize=False,
global_num_experts=e,
expert_map=None,
custom_routing_function=None,
scoring_func="softmax",
routed_scaling_factor=1.0,
e_score_correction_bias=None,
apply_router_weight_on_input=False,
activation=activation,
)
# Tolerances: fp32 tight; bf16 looser (esp. with bias)
if dtype == torch.float32:
atol = 1e-3
elif with_bias:
atol = 8e-2
else:
atol = 5e-2
torch.testing.assert_close(out, ref, atol=atol, rtol=0)
def _batched_fused_marlin_moe_cases() -> list[Any]:
cases = [
pytest.param(
@@ -3,14 +3,16 @@
from unittest.mock import patch
import pytest
import torch
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
backend_to_kernel_cls,
select_unquantized_moe_backend,
)
from vllm.platforms import current_platform
from vllm.platforms import CpuArchEnum, current_platform
skipif_not_cuda_rocm = pytest.mark.skipif(
not (current_platform.is_cuda() or current_platform.is_rocm()),
@@ -18,6 +20,109 @@ skipif_not_cuda_rocm = pytest.mark.skipif(
)
@pytest.mark.parametrize(
("amx_supported", "in_dtype", "expect_amx_kernel"),
[
(True, torch.bfloat16, True),
(False, torch.bfloat16, False),
(True, torch.float16, False),
],
)
def test_x86_cpu_unquantized_kernel_selection(
amx_supported: bool,
in_dtype: torch.dtype,
expect_amx_kernel: bool,
):
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUUnquantizedExperts,
X86CPUUnquantizedExperts,
)
with (
patch.object(current_platform, "is_cpu", return_value=True),
patch.object(
current_platform,
"get_cpu_architecture",
return_value=CpuArchEnum.X86,
),
patch("torch.cpu._is_amx_tile_supported", return_value=amx_supported),
):
moe_config = make_dummy_moe_config(
hidden_dim=128,
intermediate_size=128,
in_dtype=in_dtype,
)
kernel_cls = next(
cls
for cls in backend_to_kernel_cls(UnquantizedMoeBackend.CPU)
if cls.is_supported_config(
cls,
moe_config,
None,
None,
CPUUnquantizedExperts.activation_format(),
)[0]
)
expected_kernel_cls = (
X86CPUUnquantizedExperts if expect_amx_kernel else CPUUnquantizedExperts
)
assert kernel_cls is expected_kernel_cls
@pytest.mark.parametrize(
("platform", "in_dtype", "expect_arm_kernel"),
[
("linux", torch.bfloat16, True),
("linux", torch.float16, False),
("darwin", torch.bfloat16, False),
],
)
def test_arm_cpu_unquantized_kernel_selection(
platform: str,
in_dtype: torch.dtype,
expect_arm_kernel: bool,
):
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
ArmCPUUnquantizedExperts,
CPUUnquantizedExperts,
)
with (
patch.object(current_platform, "is_cpu", return_value=True),
patch.object(
current_platform,
"get_cpu_architecture",
return_value=CpuArchEnum.ARM,
),
patch(
"vllm.model_executor.layers.fused_moe.experts.cpu_moe.sys.platform",
platform,
),
):
moe_config = make_dummy_moe_config(
hidden_dim=128,
intermediate_size=128,
in_dtype=in_dtype,
)
kernel_cls = next(
cls
for cls in backend_to_kernel_cls(UnquantizedMoeBackend.CPU)
if cls.is_supported_config(
cls,
moe_config,
None,
None,
CPUUnquantizedExperts.activation_format(),
)[0]
)
expected_kernel_cls = (
ArmCPUUnquantizedExperts if expect_arm_kernel else CPUUnquantizedExperts
)
assert kernel_cls is expected_kernel_cls
@pytest.mark.parametrize(
"platform_method,expected_backend",
[
@@ -69,14 +174,20 @@ def test_select_default_backend_by_platform(
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(current_platform, platform_method, return_value=True),
):
moe_config = make_dummy_moe_config()
# CPU's grouped-gemm kernels require hidden/intermediate sizes
# aligned to 32; the size-1 defaults only work for backends that
# don't check shapes at selection time.
moe_config = (
make_dummy_moe_config(hidden_dim=128, intermediate_size=128)
if expected_backend == UnquantizedMoeBackend.CPU
else make_dummy_moe_config()
)
selected_backend, expert_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == expected_backend
if expected_backend in [
UnquantizedMoeBackend.CPU,
UnquantizedMoeBackend.OOT,
UnquantizedMoeBackend.TPU,
]: