[CPU] Migrate unquantized MoE to the modular-kernel experts structure (#50133)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
Li, Jiang
2026-08-03 14:55:04 +08:00
committed by GitHub
co-authored by Claude Opus 5 Fadi Arafeh
parent b3f97dae24
commit 0a6446005d
20 changed files with 972 additions and 839 deletions
@@ -1,577 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
from collections.abc import Callable
import torch
from torch.nn import functional as F
from vllm import _custom_ops as ops
from vllm._custom_ops import (
CPUQuantMethod,
cpu_fused_moe,
cpu_prepack_moe_weight,
fused_experts_cpu,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.quantization.utils.layer_utils import replace_parameter
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import direct_register_custom_op
_CPU_MOE_LAYER_CACHE = {}
# The CPU grouped-gemm MoE kernels (AMX and vector) tile the expert
# intermediate ("N") dimension in blocks of this size and have no tail/
# remainder handling, so a shard is only eligible for the fast path when
# its per-partition intermediate size is a multiple of it.
_MOE_GROUPED_GEMM_N_TILE = 32
def _swigluoai_forward_native(
x: torch.Tensor,
alpha: float = 1.702,
limit: float = 7.0,
) -> torch.Tensor:
"""PyTorch-native implementation of SwigluOAIAndMul.forward_native.
Standalone function to avoid instantiating SwigluOAIAndMul (a CustomOp)
which would trigger get_current_vllm_config() before config is set.
"""
gate, up = x[..., ::2], x[..., 1::2]
gate = gate.clamp(min=None, max=limit)
up = up.clamp(min=-limit, max=limit)
glu = gate * torch.sigmoid(gate * alpha)
gated_output = (up + 1) * glu
return gated_output
def _gelu_and_mul(
x: torch.Tensor,
) -> torch.Tensor:
d = x.shape[-1] // 2
return F.gelu(x[..., :d], approximate="none") * x[..., d:]
# Map activation names to their native forward functions.
# Uses static methods or standalone functions to avoid instantiating CustomOp
# classes, which would call get_current_vllm_config() before config is set.
_CPU_MOE_ACT_FN: dict[MoEActivation, Callable[[torch.Tensor], torch.Tensor]] = {
MoEActivation.SILU: SiluAndMul.forward_native,
MoEActivation.SWIGLUOAI: _swigluoai_forward_native,
MoEActivation.GELU: _gelu_and_mul,
MoEActivation.GELU_TANH: (
lambda x: F.gelu(x[..., : x.shape[-1] // 2], approximate="tanh")
* x[..., x.shape[-1] // 2 :]
),
}
def grouped_topk(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
num_expert_group: int = 0,
topk_group: int = 0,
scoring_func: str = "softmax",
routed_scaling_factor: float = 1.0,
e_score_correction_bias: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"
gating_output = gating_output.float()
if scoring_func == "softmax":
scores = torch.softmax(gating_output, dim=-1)
elif scoring_func == "sigmoid":
scores = gating_output.sigmoid()
else:
raise ValueError(f"Unsupported scoring function: {scoring_func}")
num_token = scores.shape[0]
if e_score_correction_bias is not None:
original_scores = scores
scores = scores + e_score_correction_bias.unsqueeze(0)
group_scores = (
scores.view(num_token, num_expert_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
)
else:
group_scores = (
scores.view(num_token, num_expert_group, -1).max(dim=-1).values
) # [n, n_group]
group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[
1
] # [n, top_k_group]
group_mask = torch.zeros_like(group_scores) # [n, n_group]
group_mask.scatter_(1, group_idx, 1) # [n, n_group]
score_mask = (
group_mask.unsqueeze(-1)
.expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group)
.reshape(num_token, -1)
) # [n, e]
tmp_scores = scores.masked_fill(~score_mask.bool(), float("-inf")) # [n, e]
if e_score_correction_bias is not None:
topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)[1]
topk_weights = original_scores.gather(1, topk_ids)
else:
topk_weights, topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)
if renormalize:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
if routed_scaling_factor != 1.0:
topk_weights = topk_weights * routed_scaling_factor
return topk_weights, topk_ids.to(torch.int32)
def select_experts(
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
top_k: int,
use_grouped_topk: bool,
renormalize: bool,
topk_group: int | None = None,
num_expert_group: int | None = None,
custom_routing_function: Callable | None = None,
scoring_func: str = "softmax",
routed_scaling_factor: float = 1.0,
e_score_correction_bias: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if use_grouped_topk:
assert topk_group is not None
assert num_expert_group is not None
return grouped_topk(
hidden_states=hidden_states,
gating_output=router_logits,
topk=top_k,
renormalize=renormalize,
num_expert_group=num_expert_group,
topk_group=topk_group,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
)
elif custom_routing_function is None:
assert scoring_func == "softmax"
topk_logit_vals, topk_idx = torch.topk(
router_logits, k=top_k, dim=-1, sorted=False
)
if renormalize:
topk_vals = torch.softmax(topk_logit_vals, dim=-1)
else:
logZ = torch.logsumexp(router_logits, dim=-1, keepdim=True)
topk_vals = (topk_logit_vals - logZ).exp()
return topk_vals.to(torch.float32), topk_idx.to(torch.int32)
else:
return custom_routing_function(
hidden_states=hidden_states,
gating_output=router_logits,
topk=top_k,
renormalize=renormalize,
)
class SGLFusedMOE:
def __init__(self, layer: torch.nn.Module) -> None:
pass
def __call__(
self,
layer: torch.nn.Module,
x: torch.Tensor,
use_grouped_topk: bool,
top_k: int,
router_logits: torch.Tensor,
renormalize: bool,
topk_group: int | None = None,
num_expert_group: int | None = None,
global_num_experts: int = -1,
expert_map: torch.Tensor | None = None,
custom_routing_function: Callable | None = None,
scoring_func: str = "softmax",
routed_scaling_factor: float = 1.0,
e_score_correction_bias: torch.Tensor | None = None,
apply_router_weight_on_input: bool = False,
activation: MoEActivation = MoEActivation.SILU,
) -> torch.Tensor:
assert activation == MoEActivation.SILU, f"{activation} is not supported."
assert not apply_router_weight_on_input
topk_weights, topk_ids = select_experts(
hidden_states=x,
router_logits=router_logits,
use_grouped_topk=use_grouped_topk,
top_k=top_k,
renormalize=renormalize,
topk_group=topk_group,
num_expert_group=num_expert_group,
custom_routing_function=custom_routing_function,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
)
return fused_experts_cpu(
x,
layer.w13_weight,
layer.w2_weight,
topk_weights,
topk_ids,
False, # inplace
CPUQuantMethod.UNQUANT, # moe_comp_method
None, # w1_scale
None, # w2_scale
None, # w1_zero
None, # w2_zero
None, # block_size
None, # w1_bias
None, # w2_bias
None, # alpha
None, # limit
True, # is_vnni
)
class CPUFusedMOE:
"""CPU-based fused MoE implementation."""
def __init__(self, layer: torch.nn.Module) -> None:
self._pad_moe_intermediate_for_grouped_gemm(layer)
use_grouped_gemm, isa = self.check_grouped_gemm(layer)
self.isa = isa
if use_grouped_gemm:
self.forward_method = self.forward_grouped_gemm
self.init_moe_grouped_gemm(layer=layer)
else:
self.forward_method = self.forward_torch
self.init_moe_torch(layer=layer)
def __call__(
self,
layer: torch.nn.Module,
x: torch.Tensor,
use_grouped_topk: bool,
top_k: int,
router_logits: torch.Tensor,
renormalize: bool,
topk_group: int | None = None,
num_expert_group: int | None = None,
global_num_experts: int = -1,
expert_map: torch.Tensor | None = None,
custom_routing_function: Callable | None = None,
scoring_func: str = "softmax",
routed_scaling_factor: float = 1.0,
e_score_correction_bias: torch.Tensor | None = None,
apply_router_weight_on_input: bool = False,
activation: MoEActivation = MoEActivation.SILU,
) -> torch.Tensor:
assert activation in _CPU_MOE_ACT_FN, f"{activation} is not supported."
topk_weights, topk_ids = select_experts(
hidden_states=x,
router_logits=router_logits,
use_grouped_topk=use_grouped_topk,
top_k=top_k,
renormalize=renormalize,
topk_group=topk_group,
num_expert_group=num_expert_group,
custom_routing_function=custom_routing_function,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
)
return self.forward_method(
layer,
x,
topk_weights,
topk_ids,
activation,
global_num_experts,
apply_router_weight_on_input,
)
def _pad_moe_intermediate_for_grouped_gemm(self, layer: torch.nn.Module) -> None:
"""Zero-pad the per-partition MoE intermediate dim up to a multiple
of _MOE_GROUPED_GEMM_N_TILE, so the AMX/vector grouped-gemm kernels
can be used even when TP sharding (moe_intermediate_size // tp_size)
lands on an unaligned value (e.g. moe_intermediate_size=704 at tp=4
-> 176). Only applies to the x86 (AMX/vec) kernels and half-split
gate/up activations; interleaved layouts (swigluoai) are left
untouched.
"""
if not hasattr(torch.ops._C, "prepack_moe_weight"):
return
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
return
intermediate_size = layer.w2_weight.size(2)
remainder = intermediate_size % _MOE_GROUPED_GEMM_N_TILE
if remainder == 0:
return
if layer.activation == MoEActivation.SWIGLUOAI:
return
pad = _MOE_GROUPED_GEMM_N_TILE - remainder
padded_size = intermediate_size + pad
num_experts, _, hidden_size = layer.w13_weight.shape
new_w13 = layer.w13_weight.new_zeros(num_experts, 2 * padded_size, hidden_size)
new_w13[:, :intermediate_size] = layer.w13_weight[:, :intermediate_size]
new_w13[:, padded_size : padded_size + intermediate_size] = layer.w13_weight[
:, intermediate_size:
]
replace_parameter(layer, "w13_weight", new_w13)
new_w2 = layer.w2_weight.new_zeros(num_experts, hidden_size, padded_size)
new_w2[:, :, :intermediate_size] = layer.w2_weight
replace_parameter(layer, "w2_weight", new_w2)
if hasattr(layer, "w13_bias"):
new_bias = layer.w13_bias.new_zeros(num_experts, 2 * padded_size)
new_bias[:, :intermediate_size] = layer.w13_bias[:, :intermediate_size]
new_bias[:, padded_size : padded_size + intermediate_size] = layer.w13_bias[
:, intermediate_size:
]
replace_parameter(layer, "w13_bias", new_bias)
def _grouped_gemm_alignment_error(self, layer: torch.nn.Module) -> str:
# w2's input size is the per-partition MoE intermediate size (the
# dimension TP-sharding splits), and it's what most commonly breaks
# alignment, e.g. moe_intermediate_size=704 at tp=4 gives
# 704 // 4 == 176, which isn't a multiple of 32.
intermediate_size_per_partition = layer.w2_weight.size(2)
return (
"CPU fused-MoE AMX grouped-gemm kernel cannot be used for a "
f"layer with w13 shape {tuple(layer.w13_weight.shape)} / w2 "
f"shape {tuple(layer.w2_weight.shape)}: the per-partition MoE "
f"intermediate size ({intermediate_size_per_partition}) is not "
f"a multiple of {_MOE_GROUPED_GEMM_N_TILE}, and automatic "
"zero-padding could not resolve this (typically because the "
"activation uses an interleaved gate/up layout, e.g. "
"swigluoai). vLLM refuses to silently fall back to the much "
"slower per-expert torch loop on AMX-capable CPUs; consider a "
"different --tensor-parallel-size."
)
def check_grouped_gemm(
self,
layer: torch.nn.Module,
) -> tuple[bool, str]:
if not hasattr(torch.ops._C, "prepack_moe_weight"):
return False, "none"
dtype = layer.w13_weight.dtype
w13_input_size = layer.w13_weight.size(2)
w13_output_size = layer.w13_weight.size(1)
w2_input_size = layer.w2_weight.size(2)
w2_output_size = layer.w2_weight.size(1)
supports_amx = torch.cpu._is_amx_tile_supported()
if supports_amx:
if (
dtype == torch.bfloat16
and w13_output_size % 32 == 0
and w2_output_size % 32 == 0
and w13_input_size % 32 == 0
and w2_input_size % 32 == 0
):
return True, "amx"
raise RuntimeError(self._grouped_gemm_alignment_error(layer))
if not (w13_output_size % 32 == 0 and w2_output_size % 32 == 0):
return False, "none"
if (
layer.activation == MoEActivation.SWIGLUOAI
and w2_input_size % _MOE_GROUPED_GEMM_N_TILE != 0
):
return False, "none"
supports_neon = current_platform.get_cpu_architecture() == CpuArchEnum.ARM
if supports_neon:
if (
dtype == torch.bfloat16
and w13_input_size % 4 == 0
and w2_input_size % 4 == 0
):
return True, "neon"
return False, "none"
return True, "vec"
def init_moe_grouped_gemm(
self,
layer: torch.nn.Module,
) -> None:
new_w13 = cpu_prepack_moe_weight(layer.w13_weight, self.isa)
replace_parameter(layer, "w13_weight", new_w13)
new_w2 = cpu_prepack_moe_weight(layer.w2_weight, self.isa)
replace_parameter(layer, "w2_weight", new_w2)
def init_moe_torch(
self,
layer: torch.nn.Module,
) -> None:
use_onednn_mm = ops._supports_onednn and ops.is_onednn_acl_supported()
num_experts = layer.w13_weight.size(0)
has_w13_bias = hasattr(layer, "w13_bias")
has_w2_bias = hasattr(layer, "w2_bias")
layer.gate_up_linear = []
layer.down_linear = []
for i in range(num_experts):
layer_w13_weight = layer.w13_weight[i]
layer_w13_bias = layer.w13_bias[i] if has_w13_bias else None
layer_w2_weight = layer.w2_weight[i]
layer_w2_bias = layer.w2_bias[i] if has_w2_bias else None
if use_onednn_mm:
gate_up_handle = ops.create_onednn_mm(layer_w13_weight.t(), 32)
layer.gate_up_linear.append(
lambda x, handle=gate_up_handle, bias=layer_w13_bias: ops.onednn_mm(
handle, x, bias
)
)
down_handle = ops.create_onednn_mm(layer_w2_weight.t(), 32)
layer.down_linear.append(
lambda x, handle=down_handle, bias=layer_w2_bias: ops.onednn_mm(
handle, x, bias
)
)
else:
layer.gate_up_linear.append(
lambda x, w=layer_w13_weight, b=layer_w13_bias: F.linear(x, w, b)
)
layer.down_linear.append(
lambda x, w=layer_w2_weight, b=layer_w2_bias: F.linear(x, w, b)
)
if use_onednn_mm: # remove weight
layer.w13_weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
layer.w2_weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
_CPU_MOE_LAYER_CACHE[id(layer)] = weakref.ref(layer)
def forward_grouped_gemm(
self,
layer: torch.nn.Module,
input: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int = -1,
skip_weighted: bool = False,
) -> torch.Tensor:
if skip_weighted:
assert topk_ids.size(1) == 1, (
"apply_router_weight_on_input is only implemented for topk=1"
)
input.mul_(topk_weights.to(input.dtype))
output = cpu_fused_moe(
input,
layer.w13_weight,
layer.w2_weight,
getattr(layer, "w13_bias", None),
getattr(layer, "w2_bias", None),
topk_weights,
topk_ids,
activation.value,
self.isa,
skip_weighted,
)
return output
def forward_torch(
self,
layer: torch.nn.Module,
input: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int = -1,
skip_weighted: bool = False,
) -> torch.Tensor:
if skip_weighted:
assert topk_ids.size(1) == 1, (
"apply_router_weight_on_input is only implemented for topk=1"
)
input.mul_(topk_weights.to(input.dtype))
output = torch.empty_like(input)
layer_id = id(layer)
torch.ops.vllm.cpu_fused_moe_torch(
layer_id,
output,
input,
topk_weights,
topk_ids,
activation.value,
global_num_experts,
skip_weighted,
)
return output
def cpu_fused_moe_torch(
layer_id: int,
output: torch.Tensor,
input: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: str,
global_num_experts: int = -1,
skip_weighted: bool = False,
) -> None:
act = MoEActivation.from_str(activation)
layer = _CPU_MOE_LAYER_CACHE[layer_id]()
# Ref code from https://github.com/sgl-project/sglang/blob/716e682721397df103f347d22da8bd46c6016dab/python/sglang/srt/layers/moe/fused_moe_native.py#L53
len_experts = global_num_experts
cnts = topk_ids.new_zeros((topk_ids.shape[0], len_experts))
cnts.scatter_(1, topk_ids.to(torch.int64), 1)
tokens_per_expert = cnts.sum(dim=0)
idxs = topk_ids.view(-1).argsort()
sorted_tokens = input[idxs // topk_ids.shape[1]]
tokens_per_expert = tokens_per_expert.cpu().numpy()
outputs = []
start_idx = 0
for i, num_tokens in enumerate(tokens_per_expert):
end_idx = start_idx + num_tokens
if num_tokens == 0:
continue
tokens_for_this_expert = sorted_tokens[start_idx:end_idx]
gate_up = layer.gate_up_linear[i](tokens_for_this_expert) # type: ignore
gate_up = _CPU_MOE_ACT_FN[act](gate_up)
expert_out = layer.down_linear[i](gate_up) # type: ignore
outputs.append(expert_out)
start_idx = end_idx
outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)
new_x = torch.empty_like(outs)
new_x[idxs] = outs
if skip_weighted:
final_out = new_x
else:
final_out = (
new_x.view(*topk_ids.shape, -1)
.type(topk_weights.dtype)
.mul_(topk_weights.unsqueeze(dim=-1))
.sum(dim=1)
.type(new_x.dtype)
)
output.copy_(final_out)
direct_register_custom_op(
op_name="cpu_fused_moe_torch",
op_func=cpu_fused_moe_torch,
mutates_args=["output"],
)
@@ -218,7 +218,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
Returns:
Output tensor after MoE computation
"""
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
select_experts,
)
@@ -1,6 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU quantized fused MoE experts."""
"""CPU fused MoE experts."""
import math
import sys
from collections.abc import Callable
from typing import cast
import torch
@@ -9,7 +14,9 @@ from vllm._custom_ops import (
CPUQuantAlgo,
CPUQuantMethod,
convert_weight_packed_scale_zp,
cpu_fused_moe,
cpu_fused_moe_int8,
cpu_prepack_moe_weight,
cpu_prepack_moe_weight_int8,
fused_experts_cpu,
)
@@ -29,7 +36,439 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt8StaticChannelSym,
kMxfp4Static,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.math_utils import round_up
# ===========================================================================
# Routing
# ===========================================================================
def grouped_topk(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
num_expert_group: int = 0,
topk_group: int = 0,
scoring_func: str = "softmax",
routed_scaling_factor: float = 1.0,
e_score_correction_bias: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"
gating_output = gating_output.float()
if scoring_func == "softmax":
scores = torch.softmax(gating_output, dim=-1)
elif scoring_func == "sigmoid":
scores = gating_output.sigmoid()
else:
raise ValueError(f"Unsupported scoring function: {scoring_func}")
num_token = scores.shape[0]
if e_score_correction_bias is not None:
original_scores = scores
scores = scores + e_score_correction_bias.unsqueeze(0)
group_scores = (
scores.view(num_token, num_expert_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
)
else:
group_scores = (
scores.view(num_token, num_expert_group, -1).max(dim=-1).values
) # [n, n_group]
group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[
1
] # [n, top_k_group]
group_mask = torch.zeros_like(group_scores) # [n, n_group]
group_mask.scatter_(1, group_idx, 1) # [n, n_group]
score_mask = (
group_mask.unsqueeze(-1)
.expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group)
.reshape(num_token, -1)
) # [n, e]
tmp_scores = scores.masked_fill(~score_mask.bool(), float("-inf")) # [n, e]
if e_score_correction_bias is not None:
topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)[1]
topk_weights = original_scores.gather(1, topk_ids)
else:
topk_weights, topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)
if renormalize:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
if routed_scaling_factor != 1.0:
topk_weights = topk_weights * routed_scaling_factor
return topk_weights, topk_ids.to(torch.int32)
def select_experts(
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
top_k: int,
use_grouped_topk: bool,
renormalize: bool,
topk_group: int | None = None,
num_expert_group: int | None = None,
custom_routing_function: Callable | None = None,
scoring_func: str = "softmax",
routed_scaling_factor: float = 1.0,
e_score_correction_bias: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if use_grouped_topk:
assert topk_group is not None
assert num_expert_group is not None
return grouped_topk(
hidden_states=hidden_states,
gating_output=router_logits,
topk=top_k,
renormalize=renormalize,
num_expert_group=num_expert_group,
topk_group=topk_group,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
)
elif custom_routing_function is None:
assert scoring_func == "softmax"
topk_logit_vals, topk_idx = torch.topk(
router_logits, k=top_k, dim=-1, sorted=False
)
if renormalize:
topk_vals = torch.softmax(topk_logit_vals, dim=-1)
else:
logZ = torch.logsumexp(router_logits, dim=-1, keepdim=True)
topk_vals = (topk_logit_vals - logZ).exp()
return topk_vals.to(torch.float32), topk_idx.to(torch.int32)
else:
topk_weights, topk_ids = custom_routing_function(
hidden_states=hidden_states,
gating_output=router_logits,
topk=top_k,
renormalize=renormalize,
)
# cpu_fused_moe reads routing tensors as contiguous float32/int32
# buffers and does not account for tensor strides.
topk_weights = topk_weights.to(torch.float32).contiguous()
topk_ids = topk_ids.to(torch.int32).contiguous()
return topk_weights, topk_ids
# ===========================================================================
# Unquantized (BF16/FP16/FP32) MoE
# ===========================================================================
class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
"""Portable vector grouped-gemm unquantized MoE experts."""
isa = "vec"
output_alignment = 32
reduction_alignment = 1
@classmethod
def _intermediate_alignment(cls) -> int:
return math.lcm(cls.output_alignment, cls.reduction_alignment)
@classmethod
def _padded_intermediate_size(cls, moe_config: FusedMoEConfig) -> int:
intermediate_size = moe_config.intermediate_size_per_partition
if moe_config.activation == MoEActivation.SWIGLUOAI:
return intermediate_size
return round_up(intermediate_size, cls._intermediate_alignment())
@classmethod
def _supports_grouped_gemm(
cls,
moe_config: FusedMoEConfig,
) -> tuple[bool, str | None]:
intermediate_size = cls._padded_intermediate_size(moe_config)
if (
moe_config.hidden_dim % cls.output_alignment != 0
or intermediate_size % cls.output_alignment != 0
):
return False, (
"kernel requires hidden and intermediate dimensions divisible by "
f"{cls.output_alignment}"
)
if (
moe_config.hidden_dim % cls.reduction_alignment != 0
or intermediate_size % cls.reduction_alignment != 0
):
return False, (
"kernel requires reduction dimensions divisible by "
f"{cls.reduction_alignment}"
)
return True, None
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
# Router configuration that the monolithic apply() signature cannot
# carry. Captured off the layer in process_weights_after_loading.
self.use_grouped_topk = False
self.renormalize = False
self.scoring_func = "softmax"
self.custom_routing_function: Callable | None = None
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cpu()
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
supported, reason = mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
if not supported:
return supported, reason
cpu_cls = cast(type[CPUUnquantizedExperts], cls)
return cpu_cls._supports_grouped_gemm(moe_config)
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in (
MoEActivation.SILU,
MoEActivation.SWIGLUOAI,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) == (None, None)
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return True
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
# Routing runs in select_experts(), which covers every routing method
# a layer can be configured with, including custom routing functions.
return True
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self._pad_moe_intermediate(layer)
self.use_grouped_topk = layer.use_grouped_topk
self.renormalize = layer.renormalize
self.scoring_func = layer.scoring_func
self.custom_routing_function = layer.custom_routing_function
replace_parameter(
layer, "w13_weight", cpu_prepack_moe_weight(layer.w13_weight, self.isa)
)
replace_parameter(
layer, "w2_weight", cpu_prepack_moe_weight(layer.w2_weight, self.isa)
)
def _pad_moe_intermediate(self, layer: torch.nn.Module) -> None:
"""Zero-pad the per-partition MoE intermediate dim of both weights and
the expert bias, see `_padded_intermediate_size`."""
intermediate_size = self.moe_config.intermediate_size_per_partition
padded_size = self._padded_intermediate_size(self.moe_config)
if padded_size == intermediate_size:
return
num_experts, _, hidden_size = layer.w13_weight.shape
new_w13 = layer.w13_weight.new_zeros(num_experts, 2 * padded_size, hidden_size)
new_w13[:, :intermediate_size] = layer.w13_weight[:, :intermediate_size]
new_w13[:, padded_size : padded_size + intermediate_size] = layer.w13_weight[
:, intermediate_size:
]
replace_parameter(layer, "w13_weight", new_w13)
new_w2 = layer.w2_weight.new_zeros(num_experts, hidden_size, padded_size)
new_w2[:, :, :intermediate_size] = layer.w2_weight
replace_parameter(layer, "w2_weight", new_w2)
if hasattr(layer, "w13_bias"):
new_bias = layer.w13_bias.new_zeros(num_experts, 2 * padded_size)
new_bias[:, :intermediate_size] = layer.w13_bias[:, :intermediate_size]
new_bias[:, padded_size : padded_size + intermediate_size] = layer.w13_bias[
:, intermediate_size:
]
# Assign through .data rather than replacing the Parameter: the
# quant config is built before this runs and holds a reference to
# this very object, which is what feeds self.w1_bias in apply().
layer.w13_bias.data = new_bias
def _select_experts(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
num_expert_group: int | None,
topk_group: int | None,
e_score_correction_bias: torch.Tensor | None,
routed_scaling_factor: float | None,
) -> tuple[torch.Tensor, torch.Tensor]:
return select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
top_k=self.moe_config.experts_per_token,
use_grouped_topk=self.use_grouped_topk,
renormalize=self.renormalize,
topk_group=topk_group,
num_expert_group=num_expert_group,
custom_routing_function=self.custom_routing_function,
scoring_func=self.scoring_func,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
topk_weights, topk_ids = self._select_experts(
hidden_states,
router_logits,
num_expert_group,
topk_group,
e_score_correction_bias,
routed_scaling_factor,
)
if apply_router_weight_on_input:
assert topk_ids.size(1) == 1, (
"apply_router_weight_on_input is only implemented for topk=1"
)
hidden_states.mul_(topk_weights.to(hidden_states.dtype))
return cpu_fused_moe(
hidden_states,
w1,
w2,
self.w1_bias,
self.w2_bias,
topk_weights,
topk_ids,
activation.value,
self.isa,
apply_router_weight_on_input,
)
class X86CPUUnquantizedExperts(CPUUnquantizedExperts):
"""x86 AMX grouped-gemm unquantized MoE experts."""
isa = "amx"
output_alignment = 32
reduction_alignment = 32
@staticmethod
def _supports_current_device() -> bool:
return (
current_platform.is_cpu()
and current_platform.get_cpu_architecture() == CpuArchEnum.X86
and torch.cpu._is_amx_tile_supported()
)
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
supported, reason = mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
if not supported:
return supported, reason
if moe_config.in_dtype != torch.bfloat16:
return False, "kernel requires bfloat16 activations"
cpu_cls = cast(type[CPUUnquantizedExperts], cls)
return cpu_cls._supports_grouped_gemm(moe_config)
class ArmCPUUnquantizedExperts(CPUUnquantizedExperts):
"""Arm NEON grouped-gemm unquantized MoE experts."""
isa = "neon"
output_alignment = 32
reduction_alignment = 4
@staticmethod
def _supports_current_device() -> bool:
return (
current_platform.is_cpu()
and current_platform.get_cpu_architecture() == CpuArchEnum.ARM
and sys.platform != "darwin"
)
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
supported, reason = mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
if not supported:
return supported, reason
if moe_config.in_dtype != torch.bfloat16:
return False, "kernel requires bfloat16 activations"
cpu_cls = cast(type[CPUUnquantizedExperts], cls)
return cpu_cls._supports_grouped_gemm(moe_config)
# ===========================================================================
# FP8 W8A16 MoE
@@ -131,10 +570,6 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
select_experts,
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
@@ -289,10 +724,6 @@ class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
select_experts,
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
@@ -505,10 +936,6 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
"apply_router_weight_on_input=True. "
)
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
select_experts,
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
@@ -631,7 +1058,6 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""VNNI-prepack INT8 MoE weights for CPU kernel."""
from vllm.model_executor.utils import replace_parameter
w13 = torch.ops._C.convert_weight_packed(layer.w13_weight)
w2 = torch.ops._C.convert_weight_packed(layer.w2_weight)
@@ -655,10 +1081,6 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
select_experts,
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
@@ -796,8 +1218,6 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
return True
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
from vllm.model_executor.utils import replace_parameter
w13 = cpu_prepack_moe_weight_int8(layer.w13_weight, "neon")
w2 = cpu_prepack_moe_weight_int8(layer.w2_weight, "neon")
replace_parameter(layer, "w13_weight", w13)
@@ -819,10 +1239,6 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
select_experts,
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
@@ -143,6 +143,20 @@ def backend_to_kernel_cls(
return [XPUExperts]
elif backend == UnquantizedMoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
ArmCPUUnquantizedExperts,
CPUUnquantizedExperts,
X86CPUUnquantizedExperts,
)
# Prefer architecture-specific kernels before the portable vector path.
return [
X86CPUUnquantizedExperts,
ArmCPUUnquantizedExperts,
CPUUnquantizedExperts,
]
else:
raise ValueError(f"Unknown unquantized MoE backend: {backend.value}")
@@ -199,10 +213,6 @@ def select_unquantized_moe_backend(
Note: Shape-specific fallbacks may still occur at runtime.
"""
if current_platform.is_cpu():
# TODO: migrate to MK structure.
return UnquantizedMoeBackend.CPU, None
if current_platform.is_tpu():
return UnquantizedMoeBackend.TPU, None
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from typing import TYPE_CHECKING
import torch
@@ -33,7 +32,6 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.platforms import current_platform
from vllm.platforms.interface import CpuArchEnum
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
@@ -54,13 +52,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
moe_config=self.moe,
)
@property
def is_monolithic(self) -> bool:
# Escape hatch for CPU, which stays on the old monolithic path.
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
return True
return super().is_monolithic
@property
def supports_eplb(self) -> bool:
return True
@@ -71,8 +62,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
):
raise ValueError(
f"{self.__class__.__name__} uses the new modular kernel initialization "
"logic for all but the CPU backend. CPU backend is monolithic. "
"So this function should not be called."
"logic. So this function should not be called."
)
def select_gemm_impl(
@@ -201,6 +191,14 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
routing_tables=layer._expert_routing_tables(),
)
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
# The CPU experts need the layer itself for the setup that
# convert_to_unquantized_kernel_format cannot express, since
# it only sees the two weight tensors: padding and prepacking
# into the grouped-gemm layout (bias included), and capturing
# the router config that monolithic apply() cannot carry.
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def process_weights_after_loading(self, layer: "RoutedExperts") -> None:
super().process_weights_after_loading(layer)
@@ -221,38 +219,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
# OOT handles internally.
return
elif self.unquantized_backend == UnquantizedMoeBackend.CPU:
# CPU stays on the old path — no oracle, no moe_kernel.
from vllm.model_executor.layers.fused_moe import cpu_fused_moe
if current_platform.get_cpu_architecture() == CpuArchEnum.X86:
from vllm.model_executor.layers.utils import check_cpu_sgl_kernel
dtype_w13 = layer.w13_weight.dtype
_, n_w13, k_w13 = layer.w13_weight.size()
dtype_w2 = layer.w2_weight.dtype
_, n_w2, k_w2 = layer.w2_weight.size()
if (
envs.VLLM_CPU_SGL_KERNEL
and check_cpu_sgl_kernel(n_w13, k_w13, dtype_w13)
and check_cpu_sgl_kernel(n_w2, k_w2, dtype_w2)
):
packed_w13_weight = torch.ops._C.convert_weight_packed(
layer.w13_weight
)
assert packed_w13_weight.size() == layer.w13_weight.size()
layer.w13_weight.copy_(packed_w13_weight)
del packed_w13_weight
packed_w2_weight = torch.ops._C.convert_weight_packed(
layer.w2_weight
)
assert packed_w2_weight.size() == layer.w2_weight.size()
layer.w2_weight.copy_(packed_w2_weight)
self.cpu_fused_moe: Callable = cpu_fused_moe.SGLFusedMOE(layer)
else:
self.cpu_fused_moe = cpu_fused_moe.CPUFusedMOE(layer)
else:
self.cpu_fused_moe = cpu_fused_moe.CPUFusedMOE(layer)
elif self.unquantized_backend == UnquantizedMoeBackend.XPU:
w13 = layer.w13_weight
w2 = layer.w2_weight
@@ -363,39 +329,18 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.is_monolithic
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
assert self.moe_kernel is None
return self.cpu_fused_moe(
layer,
x,
layer.use_grouped_topk,
layer.top_k,
router_logits,
layer.renormalize,
layer.topk_group,
layer.num_expert_group,
layer.global_num_experts,
layer.expert_map,
layer.custom_routing_function,
layer.scoring_func,
layer.routed_scaling_factor,
layer.e_score_correction_bias,
layer.apply_router_weight_on_input,
layer.activation,
)
else:
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
x,
layer.w13_weight,
layer.w2_weight,
router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
x,
layer.w13_weight,
layer.w2_weight,
router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)