[CPU] Migrate unquantized MoE to the modular-kernel experts structure (#50133)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
Li, Jiang
2026-08-03 14:55:04 +08:00
committed by GitHub
co-authored by Claude Opus 5 Fadi Arafeh
parent b3f97dae24
commit 0a6446005d
20 changed files with 972 additions and 839 deletions
+161 -48
View File
@@ -1,9 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
from vllm._custom_ops import (
cpu_fused_moe,
@@ -12,9 +15,18 @@ from vllm._custom_ops import (
cpu_prepack_moe_weight_int8,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
_CPU_MOE_ACT_FN,
CPUFusedMOE,
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
biased_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
ArmCPUUnquantizedExperts,
CPUUnquantizedExperts,
X86CPUUnquantizedExperts,
select_experts,
)
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -36,7 +48,10 @@ ACT = [
]
USE_BIAS = [False, True]
ISA = ["vec"]
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
if (
current_platform.get_cpu_architecture() == CpuArchEnum.ARM
and sys.platform != "darwin"
):
ISA.append("neon")
if torch.cpu._is_amx_tile_supported():
ISA.append("amx")
@@ -44,6 +59,27 @@ if torch.cpu._is_amx_tile_supported():
DTYPE = [torch.bfloat16]
def _ref_moe_activation(
input: torch.Tensor,
activation: MoEActivation,
) -> torch.Tensor:
if activation == MoEActivation.SWIGLUOAI:
gate = input[..., ::2].clamp(max=7.0)
up = input[..., 1::2].clamp(min=-7.0, max=7.0)
glu = gate * torch.sigmoid(gate * 1.702)
return (up + 1) * glu
d = input.shape[-1] // 2
gate, up = input[..., :d], input[..., d:]
if activation == MoEActivation.SILU:
return torch.nn.functional.silu(gate) * up
if activation == MoEActivation.GELU:
return torch.nn.functional.gelu(gate, approximate="none") * up
if activation == MoEActivation.GELU_TANH:
return torch.nn.functional.gelu(gate, approximate="tanh") * up
raise ValueError(f"Unsupported activation: {activation}")
def ref_fused_moe(
input: torch.Tensor,
w13: torch.Tensor,
@@ -87,7 +123,8 @@ def ref_fused_moe(
tokens_for_this_expert, curr_w13, curr_w13_bias
)
# Note: to simulate the kernel implementation
gate_up = _CPU_MOE_ACT_FN[activation](gate_up).to(dtype=input.dtype).float()
gate_up = _ref_moe_activation(gate_up, activation)
gate_up = gate_up.to(dtype=input.dtype).float()
expert_out = torch.nn.functional.linear(gate_up, curr_w2, curr_w2_bias)
outputs.append(expert_out)
@@ -166,7 +203,7 @@ def ref_fused_moe_int8(
if w13_bias is not None:
gate_up += w13_bias[expert_idx].float()
intermediate = _CPU_MOE_ACT_FN[activation](gate_up).to(input.dtype)
intermediate = _ref_moe_activation(gate_up, activation).to(input.dtype)
intermediate_int8, intermediate_scale = quantize_per_token(intermediate)
output = torch.matmul(
intermediate_int8.float(),
@@ -356,8 +393,8 @@ UNALIGNED_INTERMEDIATE_DIM = 176
class _StubMoELayer(torch.nn.Module):
"""Minimal stand-in for the real MoE layer module, exposing just what
CPUFusedMOE reads/replaces (w13_weight, w2_weight, activation, and
optionally w13_bias/w2_bias)."""
the unquantized CPU experts read/replace (w13_weight, w2_weight, the
router configuration, and optionally w13_bias/w2_bias)."""
def __init__(
self,
@@ -375,12 +412,77 @@ class _StubMoELayer(torch.nn.Module):
self.w13_bias = torch.nn.Parameter(w13_bias, requires_grad=False)
if w2_bias is not None:
self.w2_bias = torch.nn.Parameter(w2_bias, requires_grad=False)
self.use_grouped_topk = False
self.renormalize = False
self.scoring_func = "softmax"
self.custom_routing_function = None
def _make_moe_config(
expert_num: int,
hidden_size: int,
intermediate_size: int,
topk_num: int,
dtype: torch.dtype,
act: MoEActivation,
has_bias: bool = False,
) -> FusedMoEConfig:
return FusedMoEConfig(
num_experts=expert_num,
experts_per_token=topk_num,
hidden_dim=hidden_size,
intermediate_size=intermediate_size,
num_local_experts=expert_num,
num_logical_experts=expert_num,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=act,
in_dtype=dtype,
device="cpu",
routing_method=RoutingMethodType.Default,
has_bias=has_bias,
)
@pytest.mark.parametrize(
("hidden_size", "intermediate_size", "act", "expected"),
[
(128, 128, MoEActivation.SILU, True),
# Hidden size is not 32-aligned.
(112, 128, MoEActivation.SILU, False),
(128, 176, MoEActivation.SILU, True),
# SwiGLUOAI's interleaved gate/up layout cannot be padded.
(128, 176, MoEActivation.SWIGLUOAI, False),
],
)
def test_cpu_vec_fused_moe_shape_support(
hidden_size: int,
intermediate_size: int,
act: MoEActivation,
expected: bool,
):
moe_config = _make_moe_config(
expert_num=8,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
topk_num=4,
dtype=torch.bfloat16,
act=act,
)
supported, _ = CPUUnquantizedExperts.is_supported_config(
CPUUnquantizedExperts,
moe_config,
None,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported is expected
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
@pytest.mark.parametrize("use_bias", USE_BIAS)
@pytest.mark.parametrize("dtype", DTYPE)
@pytest.mark.parametrize("isa", ISA)
@pytest.mark.parametrize(
"act",
[MoEActivation.SILU, MoEActivation.GELU, MoEActivation.GELU_TANH],
@@ -391,14 +493,11 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
hidden_size: int,
use_bias: bool,
dtype: torch.dtype,
isa: str,
act: MoEActivation,
):
"""An unaligned per-partition moe_intermediate_size must still hit the
grouped-gemm fast path via automatic zero-padding, with numerically
correct output, instead of silently falling back to the much slower
per-expert torch loop."""
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
pytest.skip("padding is only applied on the x86 AMX/vector kernels")
"""CPU kernels handle unaligned intermediate sizes by zero-padding the
weights before prepacking."""
set_random_seed(0)
batch_size = 64
@@ -423,9 +522,15 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
score = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
topk_weight, topk_ids = torch.topk(score, topk_num)
topk_ids = topk_ids.to(torch.int32)
# Route with the same helper apply() uses internally, so the reference
# only differs from the kernel in how the experts are evaluated.
topk_weight, topk_ids = select_experts(
hidden_states=input,
router_logits=router_logits,
top_k=topk_num,
use_grouped_topk=False,
renormalize=False,
)
ref_output = ref_fused_moe(
input, w13, w2, w13_bias, w2_bias, topk_weight, topk_ids, act
@@ -439,40 +544,48 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
w2_bias.clone() if w2_bias is not None else None,
)
cpu_moe = CPUFusedMOE(layer)
assert cpu_moe.forward_method == cpu_moe.forward_grouped_gemm, (
"expected the padded intermediate size to hit the grouped-gemm fast path"
moe_config = _make_moe_config(
expert_num, hidden_size, intermediate_size, topk_num, dtype, act, use_bias
)
experts_cls = {
"vec": CPUUnquantizedExperts,
"amx": X86CPUUnquantizedExperts,
"neon": ArmCPUUnquantizedExperts,
}[isa]
supported, reason = experts_cls.is_supported_config(
experts_cls,
moe_config,
None,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, (
f"expected the padded intermediate size to be supported, got {reason}"
)
output = cpu_moe.forward_method(
layer, input, topk_weight, topk_ids, act, expert_num, False
# Mirror UnquantizedFusedMoEMethod._setup_kernel: the quant config is
# built from the layer first, then the experts shuffle the layer into the
# kernel's runtime format.
quant_config = (
biased_moe_quant_config(layer.w13_bias, layer.w2_bias)
if use_bias
else FusedMoEQuantConfig.make()
)
experts = experts_cls(moe_config, quant_config)
assert experts.isa == isa
experts.process_weights_after_loading(layer)
output = experts.apply(
hidden_states=input,
w1=layer.w13_weight,
w2=layer.w2_weight,
router_logits=router_logits,
activation=act,
global_num_experts=expert_num,
expert_map=None,
a1q_scale=None,
apply_router_weight_on_input=False,
)
atol, rtol = get_default_atol(output), get_default_rtol(output)
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
def test_cpu_fused_moe_unaligned_intermediate_size_swigluoai(default_vllm_config):
"""swigluoai's interleaved gate/up layout isn't padded automatically. On
AMX-capable CPUs this must raise rather than silently falling back to
the (correct but much slower) per-expert torch loop; elsewhere it should
fall back exactly as before."""
set_random_seed(0)
expert_num = 8
hidden_size = 128
intermediate_size = UNALIGNED_INTERMEDIATE_DIM
dtype = torch.bfloat16
up_dim = 2 * intermediate_size
layer = _StubMoELayer(
torch.randn((expert_num, up_dim, hidden_size), dtype=dtype),
torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype),
MoEActivation.SWIGLUOAI,
)
if torch.cpu._is_amx_tile_supported():
with pytest.raises(RuntimeError):
CPUFusedMOE(layer)
else:
cpu_moe = CPUFusedMOE(layer)
assert cpu_moe.forward_method == cpu_moe.forward_torch