[CPU] Migrate unquantized MoE to the modular-kernel experts structure (#50133)
Signed-off-by: jiang1.li <jiang1.li@intel.com> Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
co-authored by
Claude Opus 5
Fadi Arafeh
parent
b3f97dae24
commit
0a6446005d
@@ -1,577 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import weakref
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
from torch.nn import functional as F
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._custom_ops import (
|
||||
CPUQuantMethod,
|
||||
cpu_fused_moe,
|
||||
cpu_prepack_moe_weight,
|
||||
fused_experts_cpu,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.quantization.utils.layer_utils import replace_parameter
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
_CPU_MOE_LAYER_CACHE = {}
|
||||
# The CPU grouped-gemm MoE kernels (AMX and vector) tile the expert
|
||||
# intermediate ("N") dimension in blocks of this size and have no tail/
|
||||
# remainder handling, so a shard is only eligible for the fast path when
|
||||
# its per-partition intermediate size is a multiple of it.
|
||||
_MOE_GROUPED_GEMM_N_TILE = 32
|
||||
|
||||
|
||||
def _swigluoai_forward_native(
|
||||
x: torch.Tensor,
|
||||
alpha: float = 1.702,
|
||||
limit: float = 7.0,
|
||||
) -> torch.Tensor:
|
||||
"""PyTorch-native implementation of SwigluOAIAndMul.forward_native.
|
||||
|
||||
Standalone function to avoid instantiating SwigluOAIAndMul (a CustomOp)
|
||||
which would trigger get_current_vllm_config() before config is set.
|
||||
"""
|
||||
gate, up = x[..., ::2], x[..., 1::2]
|
||||
gate = gate.clamp(min=None, max=limit)
|
||||
up = up.clamp(min=-limit, max=limit)
|
||||
glu = gate * torch.sigmoid(gate * alpha)
|
||||
gated_output = (up + 1) * glu
|
||||
return gated_output
|
||||
|
||||
|
||||
def _gelu_and_mul(
|
||||
x: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
return F.gelu(x[..., :d], approximate="none") * x[..., d:]
|
||||
|
||||
|
||||
# Map activation names to their native forward functions.
|
||||
# Uses static methods or standalone functions to avoid instantiating CustomOp
|
||||
# classes, which would call get_current_vllm_config() before config is set.
|
||||
_CPU_MOE_ACT_FN: dict[MoEActivation, Callable[[torch.Tensor], torch.Tensor]] = {
|
||||
MoEActivation.SILU: SiluAndMul.forward_native,
|
||||
MoEActivation.SWIGLUOAI: _swigluoai_forward_native,
|
||||
MoEActivation.GELU: _gelu_and_mul,
|
||||
MoEActivation.GELU_TANH: (
|
||||
lambda x: F.gelu(x[..., : x.shape[-1] // 2], approximate="tanh")
|
||||
* x[..., x.shape[-1] // 2 :]
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def grouped_topk(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
num_expert_group: int = 0,
|
||||
topk_group: int = 0,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"
|
||||
|
||||
gating_output = gating_output.float()
|
||||
if scoring_func == "softmax":
|
||||
scores = torch.softmax(gating_output, dim=-1)
|
||||
elif scoring_func == "sigmoid":
|
||||
scores = gating_output.sigmoid()
|
||||
else:
|
||||
raise ValueError(f"Unsupported scoring function: {scoring_func}")
|
||||
|
||||
num_token = scores.shape[0]
|
||||
if e_score_correction_bias is not None:
|
||||
original_scores = scores
|
||||
scores = scores + e_score_correction_bias.unsqueeze(0)
|
||||
group_scores = (
|
||||
scores.view(num_token, num_expert_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
|
||||
)
|
||||
else:
|
||||
group_scores = (
|
||||
scores.view(num_token, num_expert_group, -1).max(dim=-1).values
|
||||
) # [n, n_group]
|
||||
group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[
|
||||
1
|
||||
] # [n, top_k_group]
|
||||
group_mask = torch.zeros_like(group_scores) # [n, n_group]
|
||||
group_mask.scatter_(1, group_idx, 1) # [n, n_group]
|
||||
score_mask = (
|
||||
group_mask.unsqueeze(-1)
|
||||
.expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group)
|
||||
.reshape(num_token, -1)
|
||||
) # [n, e]
|
||||
tmp_scores = scores.masked_fill(~score_mask.bool(), float("-inf")) # [n, e]
|
||||
|
||||
if e_score_correction_bias is not None:
|
||||
topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)[1]
|
||||
topk_weights = original_scores.gather(1, topk_ids)
|
||||
else:
|
||||
topk_weights, topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)
|
||||
|
||||
if renormalize:
|
||||
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights = topk_weights * routed_scaling_factor
|
||||
return topk_weights, topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
def select_experts(
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
top_k: int,
|
||||
use_grouped_topk: bool,
|
||||
renormalize: bool,
|
||||
topk_group: int | None = None,
|
||||
num_expert_group: int | None = None,
|
||||
custom_routing_function: Callable | None = None,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if use_grouped_topk:
|
||||
assert topk_group is not None
|
||||
assert num_expert_group is not None
|
||||
return grouped_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=top_k,
|
||||
renormalize=renormalize,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
elif custom_routing_function is None:
|
||||
assert scoring_func == "softmax"
|
||||
topk_logit_vals, topk_idx = torch.topk(
|
||||
router_logits, k=top_k, dim=-1, sorted=False
|
||||
)
|
||||
if renormalize:
|
||||
topk_vals = torch.softmax(topk_logit_vals, dim=-1)
|
||||
else:
|
||||
logZ = torch.logsumexp(router_logits, dim=-1, keepdim=True)
|
||||
topk_vals = (topk_logit_vals - logZ).exp()
|
||||
return topk_vals.to(torch.float32), topk_idx.to(torch.int32)
|
||||
else:
|
||||
return custom_routing_function(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=top_k,
|
||||
renormalize=renormalize,
|
||||
)
|
||||
|
||||
|
||||
class SGLFusedMOE:
|
||||
def __init__(self, layer: torch.nn.Module) -> None:
|
||||
pass
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
use_grouped_topk: bool,
|
||||
top_k: int,
|
||||
router_logits: torch.Tensor,
|
||||
renormalize: bool,
|
||||
topk_group: int | None = None,
|
||||
num_expert_group: int | None = None,
|
||||
global_num_experts: int = -1,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
custom_routing_function: Callable | None = None,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
activation: MoEActivation = MoEActivation.SILU,
|
||||
) -> torch.Tensor:
|
||||
assert activation == MoEActivation.SILU, f"{activation} is not supported."
|
||||
assert not apply_router_weight_on_input
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=x,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
top_k=top_k,
|
||||
renormalize=renormalize,
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
custom_routing_function=custom_routing_function,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
return fused_experts_cpu(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
CPUQuantMethod.UNQUANT, # moe_comp_method
|
||||
None, # w1_scale
|
||||
None, # w2_scale
|
||||
None, # w1_zero
|
||||
None, # w2_zero
|
||||
None, # block_size
|
||||
None, # w1_bias
|
||||
None, # w2_bias
|
||||
None, # alpha
|
||||
None, # limit
|
||||
True, # is_vnni
|
||||
)
|
||||
|
||||
|
||||
class CPUFusedMOE:
|
||||
"""CPU-based fused MoE implementation."""
|
||||
|
||||
def __init__(self, layer: torch.nn.Module) -> None:
|
||||
self._pad_moe_intermediate_for_grouped_gemm(layer)
|
||||
use_grouped_gemm, isa = self.check_grouped_gemm(layer)
|
||||
self.isa = isa
|
||||
if use_grouped_gemm:
|
||||
self.forward_method = self.forward_grouped_gemm
|
||||
self.init_moe_grouped_gemm(layer=layer)
|
||||
else:
|
||||
self.forward_method = self.forward_torch
|
||||
self.init_moe_torch(layer=layer)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
use_grouped_topk: bool,
|
||||
top_k: int,
|
||||
router_logits: torch.Tensor,
|
||||
renormalize: bool,
|
||||
topk_group: int | None = None,
|
||||
num_expert_group: int | None = None,
|
||||
global_num_experts: int = -1,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
custom_routing_function: Callable | None = None,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
activation: MoEActivation = MoEActivation.SILU,
|
||||
) -> torch.Tensor:
|
||||
assert activation in _CPU_MOE_ACT_FN, f"{activation} is not supported."
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=x,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
top_k=top_k,
|
||||
renormalize=renormalize,
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
custom_routing_function=custom_routing_function,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
return self.forward_method(
|
||||
layer,
|
||||
x,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation,
|
||||
global_num_experts,
|
||||
apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
def _pad_moe_intermediate_for_grouped_gemm(self, layer: torch.nn.Module) -> None:
|
||||
"""Zero-pad the per-partition MoE intermediate dim up to a multiple
|
||||
of _MOE_GROUPED_GEMM_N_TILE, so the AMX/vector grouped-gemm kernels
|
||||
can be used even when TP sharding (moe_intermediate_size // tp_size)
|
||||
lands on an unaligned value (e.g. moe_intermediate_size=704 at tp=4
|
||||
-> 176). Only applies to the x86 (AMX/vec) kernels and half-split
|
||||
gate/up activations; interleaved layouts (swigluoai) are left
|
||||
untouched.
|
||||
"""
|
||||
if not hasattr(torch.ops._C, "prepack_moe_weight"):
|
||||
return
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
return
|
||||
|
||||
intermediate_size = layer.w2_weight.size(2)
|
||||
remainder = intermediate_size % _MOE_GROUPED_GEMM_N_TILE
|
||||
if remainder == 0:
|
||||
return
|
||||
if layer.activation == MoEActivation.SWIGLUOAI:
|
||||
return
|
||||
|
||||
pad = _MOE_GROUPED_GEMM_N_TILE - remainder
|
||||
padded_size = intermediate_size + pad
|
||||
num_experts, _, hidden_size = layer.w13_weight.shape
|
||||
|
||||
new_w13 = layer.w13_weight.new_zeros(num_experts, 2 * padded_size, hidden_size)
|
||||
new_w13[:, :intermediate_size] = layer.w13_weight[:, :intermediate_size]
|
||||
new_w13[:, padded_size : padded_size + intermediate_size] = layer.w13_weight[
|
||||
:, intermediate_size:
|
||||
]
|
||||
replace_parameter(layer, "w13_weight", new_w13)
|
||||
|
||||
new_w2 = layer.w2_weight.new_zeros(num_experts, hidden_size, padded_size)
|
||||
new_w2[:, :, :intermediate_size] = layer.w2_weight
|
||||
replace_parameter(layer, "w2_weight", new_w2)
|
||||
|
||||
if hasattr(layer, "w13_bias"):
|
||||
new_bias = layer.w13_bias.new_zeros(num_experts, 2 * padded_size)
|
||||
new_bias[:, :intermediate_size] = layer.w13_bias[:, :intermediate_size]
|
||||
new_bias[:, padded_size : padded_size + intermediate_size] = layer.w13_bias[
|
||||
:, intermediate_size:
|
||||
]
|
||||
replace_parameter(layer, "w13_bias", new_bias)
|
||||
|
||||
def _grouped_gemm_alignment_error(self, layer: torch.nn.Module) -> str:
|
||||
# w2's input size is the per-partition MoE intermediate size (the
|
||||
# dimension TP-sharding splits), and it's what most commonly breaks
|
||||
# alignment, e.g. moe_intermediate_size=704 at tp=4 gives
|
||||
# 704 // 4 == 176, which isn't a multiple of 32.
|
||||
intermediate_size_per_partition = layer.w2_weight.size(2)
|
||||
return (
|
||||
"CPU fused-MoE AMX grouped-gemm kernel cannot be used for a "
|
||||
f"layer with w13 shape {tuple(layer.w13_weight.shape)} / w2 "
|
||||
f"shape {tuple(layer.w2_weight.shape)}: the per-partition MoE "
|
||||
f"intermediate size ({intermediate_size_per_partition}) is not "
|
||||
f"a multiple of {_MOE_GROUPED_GEMM_N_TILE}, and automatic "
|
||||
"zero-padding could not resolve this (typically because the "
|
||||
"activation uses an interleaved gate/up layout, e.g. "
|
||||
"swigluoai). vLLM refuses to silently fall back to the much "
|
||||
"slower per-expert torch loop on AMX-capable CPUs; consider a "
|
||||
"different --tensor-parallel-size."
|
||||
)
|
||||
|
||||
def check_grouped_gemm(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
) -> tuple[bool, str]:
|
||||
if not hasattr(torch.ops._C, "prepack_moe_weight"):
|
||||
return False, "none"
|
||||
|
||||
dtype = layer.w13_weight.dtype
|
||||
w13_input_size = layer.w13_weight.size(2)
|
||||
w13_output_size = layer.w13_weight.size(1)
|
||||
w2_input_size = layer.w2_weight.size(2)
|
||||
w2_output_size = layer.w2_weight.size(1)
|
||||
|
||||
supports_amx = torch.cpu._is_amx_tile_supported()
|
||||
if supports_amx:
|
||||
if (
|
||||
dtype == torch.bfloat16
|
||||
and w13_output_size % 32 == 0
|
||||
and w2_output_size % 32 == 0
|
||||
and w13_input_size % 32 == 0
|
||||
and w2_input_size % 32 == 0
|
||||
):
|
||||
return True, "amx"
|
||||
raise RuntimeError(self._grouped_gemm_alignment_error(layer))
|
||||
|
||||
if not (w13_output_size % 32 == 0 and w2_output_size % 32 == 0):
|
||||
return False, "none"
|
||||
|
||||
if (
|
||||
layer.activation == MoEActivation.SWIGLUOAI
|
||||
and w2_input_size % _MOE_GROUPED_GEMM_N_TILE != 0
|
||||
):
|
||||
return False, "none"
|
||||
|
||||
supports_neon = current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
if supports_neon:
|
||||
if (
|
||||
dtype == torch.bfloat16
|
||||
and w13_input_size % 4 == 0
|
||||
and w2_input_size % 4 == 0
|
||||
):
|
||||
return True, "neon"
|
||||
return False, "none"
|
||||
|
||||
return True, "vec"
|
||||
|
||||
def init_moe_grouped_gemm(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
) -> None:
|
||||
new_w13 = cpu_prepack_moe_weight(layer.w13_weight, self.isa)
|
||||
replace_parameter(layer, "w13_weight", new_w13)
|
||||
new_w2 = cpu_prepack_moe_weight(layer.w2_weight, self.isa)
|
||||
replace_parameter(layer, "w2_weight", new_w2)
|
||||
|
||||
def init_moe_torch(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
) -> None:
|
||||
use_onednn_mm = ops._supports_onednn and ops.is_onednn_acl_supported()
|
||||
num_experts = layer.w13_weight.size(0)
|
||||
has_w13_bias = hasattr(layer, "w13_bias")
|
||||
has_w2_bias = hasattr(layer, "w2_bias")
|
||||
|
||||
layer.gate_up_linear = []
|
||||
layer.down_linear = []
|
||||
|
||||
for i in range(num_experts):
|
||||
layer_w13_weight = layer.w13_weight[i]
|
||||
layer_w13_bias = layer.w13_bias[i] if has_w13_bias else None
|
||||
layer_w2_weight = layer.w2_weight[i]
|
||||
layer_w2_bias = layer.w2_bias[i] if has_w2_bias else None
|
||||
if use_onednn_mm:
|
||||
gate_up_handle = ops.create_onednn_mm(layer_w13_weight.t(), 32)
|
||||
layer.gate_up_linear.append(
|
||||
lambda x, handle=gate_up_handle, bias=layer_w13_bias: ops.onednn_mm(
|
||||
handle, x, bias
|
||||
)
|
||||
)
|
||||
down_handle = ops.create_onednn_mm(layer_w2_weight.t(), 32)
|
||||
layer.down_linear.append(
|
||||
lambda x, handle=down_handle, bias=layer_w2_bias: ops.onednn_mm(
|
||||
handle, x, bias
|
||||
)
|
||||
)
|
||||
else:
|
||||
layer.gate_up_linear.append(
|
||||
lambda x, w=layer_w13_weight, b=layer_w13_bias: F.linear(x, w, b)
|
||||
)
|
||||
layer.down_linear.append(
|
||||
lambda x, w=layer_w2_weight, b=layer_w2_bias: F.linear(x, w, b)
|
||||
)
|
||||
|
||||
if use_onednn_mm: # remove weight
|
||||
layer.w13_weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
|
||||
layer.w2_weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
|
||||
|
||||
_CPU_MOE_LAYER_CACHE[id(layer)] = weakref.ref(layer)
|
||||
|
||||
def forward_grouped_gemm(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int = -1,
|
||||
skip_weighted: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if skip_weighted:
|
||||
assert topk_ids.size(1) == 1, (
|
||||
"apply_router_weight_on_input is only implemented for topk=1"
|
||||
)
|
||||
input.mul_(topk_weights.to(input.dtype))
|
||||
|
||||
output = cpu_fused_moe(
|
||||
input,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
getattr(layer, "w13_bias", None),
|
||||
getattr(layer, "w2_bias", None),
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation.value,
|
||||
self.isa,
|
||||
skip_weighted,
|
||||
)
|
||||
return output
|
||||
|
||||
def forward_torch(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
input: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int = -1,
|
||||
skip_weighted: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if skip_weighted:
|
||||
assert topk_ids.size(1) == 1, (
|
||||
"apply_router_weight_on_input is only implemented for topk=1"
|
||||
)
|
||||
input.mul_(topk_weights.to(input.dtype))
|
||||
|
||||
output = torch.empty_like(input)
|
||||
layer_id = id(layer)
|
||||
torch.ops.vllm.cpu_fused_moe_torch(
|
||||
layer_id,
|
||||
output,
|
||||
input,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation.value,
|
||||
global_num_experts,
|
||||
skip_weighted,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def cpu_fused_moe_torch(
|
||||
layer_id: int,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: str,
|
||||
global_num_experts: int = -1,
|
||||
skip_weighted: bool = False,
|
||||
) -> None:
|
||||
act = MoEActivation.from_str(activation)
|
||||
layer = _CPU_MOE_LAYER_CACHE[layer_id]()
|
||||
|
||||
# Ref code from https://github.com/sgl-project/sglang/blob/716e682721397df103f347d22da8bd46c6016dab/python/sglang/srt/layers/moe/fused_moe_native.py#L53
|
||||
len_experts = global_num_experts
|
||||
|
||||
cnts = topk_ids.new_zeros((topk_ids.shape[0], len_experts))
|
||||
cnts.scatter_(1, topk_ids.to(torch.int64), 1)
|
||||
tokens_per_expert = cnts.sum(dim=0)
|
||||
idxs = topk_ids.view(-1).argsort()
|
||||
|
||||
sorted_tokens = input[idxs // topk_ids.shape[1]]
|
||||
tokens_per_expert = tokens_per_expert.cpu().numpy()
|
||||
|
||||
outputs = []
|
||||
start_idx = 0
|
||||
|
||||
for i, num_tokens in enumerate(tokens_per_expert):
|
||||
end_idx = start_idx + num_tokens
|
||||
if num_tokens == 0:
|
||||
continue
|
||||
tokens_for_this_expert = sorted_tokens[start_idx:end_idx]
|
||||
|
||||
gate_up = layer.gate_up_linear[i](tokens_for_this_expert) # type: ignore
|
||||
gate_up = _CPU_MOE_ACT_FN[act](gate_up)
|
||||
expert_out = layer.down_linear[i](gate_up) # type: ignore
|
||||
outputs.append(expert_out)
|
||||
start_idx = end_idx
|
||||
|
||||
outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0)
|
||||
new_x = torch.empty_like(outs)
|
||||
|
||||
new_x[idxs] = outs
|
||||
if skip_weighted:
|
||||
final_out = new_x
|
||||
else:
|
||||
final_out = (
|
||||
new_x.view(*topk_ids.shape, -1)
|
||||
.type(topk_weights.dtype)
|
||||
.mul_(topk_weights.unsqueeze(dim=-1))
|
||||
.sum(dim=1)
|
||||
.type(new_x.dtype)
|
||||
)
|
||||
output.copy_(final_out)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="cpu_fused_moe_torch",
|
||||
op_func=cpu_fused_moe_torch,
|
||||
mutates_args=["output"],
|
||||
)
|
||||
@@ -218,7 +218,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
Returns:
|
||||
Output tensor after MoE computation
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
select_experts,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU quantized fused MoE experts."""
|
||||
"""CPU fused MoE experts."""
|
||||
|
||||
import math
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
@@ -9,7 +14,9 @@ from vllm._custom_ops import (
|
||||
CPUQuantAlgo,
|
||||
CPUQuantMethod,
|
||||
convert_weight_packed_scale_zp,
|
||||
cpu_fused_moe,
|
||||
cpu_fused_moe_int8,
|
||||
cpu_prepack_moe_weight,
|
||||
cpu_prepack_moe_weight_int8,
|
||||
fused_experts_cpu,
|
||||
)
|
||||
@@ -29,7 +36,439 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
# ===========================================================================
|
||||
# Routing
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def grouped_topk(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
num_expert_group: int = 0,
|
||||
topk_group: int = 0,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"
|
||||
|
||||
gating_output = gating_output.float()
|
||||
if scoring_func == "softmax":
|
||||
scores = torch.softmax(gating_output, dim=-1)
|
||||
elif scoring_func == "sigmoid":
|
||||
scores = gating_output.sigmoid()
|
||||
else:
|
||||
raise ValueError(f"Unsupported scoring function: {scoring_func}")
|
||||
|
||||
num_token = scores.shape[0]
|
||||
if e_score_correction_bias is not None:
|
||||
original_scores = scores
|
||||
scores = scores + e_score_correction_bias.unsqueeze(0)
|
||||
group_scores = (
|
||||
scores.view(num_token, num_expert_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
|
||||
)
|
||||
else:
|
||||
group_scores = (
|
||||
scores.view(num_token, num_expert_group, -1).max(dim=-1).values
|
||||
) # [n, n_group]
|
||||
group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[
|
||||
1
|
||||
] # [n, top_k_group]
|
||||
group_mask = torch.zeros_like(group_scores) # [n, n_group]
|
||||
group_mask.scatter_(1, group_idx, 1) # [n, n_group]
|
||||
score_mask = (
|
||||
group_mask.unsqueeze(-1)
|
||||
.expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group)
|
||||
.reshape(num_token, -1)
|
||||
) # [n, e]
|
||||
tmp_scores = scores.masked_fill(~score_mask.bool(), float("-inf")) # [n, e]
|
||||
|
||||
if e_score_correction_bias is not None:
|
||||
topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)[1]
|
||||
topk_weights = original_scores.gather(1, topk_ids)
|
||||
else:
|
||||
topk_weights, topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)
|
||||
|
||||
if renormalize:
|
||||
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights = topk_weights * routed_scaling_factor
|
||||
return topk_weights, topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
def select_experts(
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
top_k: int,
|
||||
use_grouped_topk: bool,
|
||||
renormalize: bool,
|
||||
topk_group: int | None = None,
|
||||
num_expert_group: int | None = None,
|
||||
custom_routing_function: Callable | None = None,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if use_grouped_topk:
|
||||
assert topk_group is not None
|
||||
assert num_expert_group is not None
|
||||
return grouped_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=top_k,
|
||||
renormalize=renormalize,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
elif custom_routing_function is None:
|
||||
assert scoring_func == "softmax"
|
||||
topk_logit_vals, topk_idx = torch.topk(
|
||||
router_logits, k=top_k, dim=-1, sorted=False
|
||||
)
|
||||
if renormalize:
|
||||
topk_vals = torch.softmax(topk_logit_vals, dim=-1)
|
||||
else:
|
||||
logZ = torch.logsumexp(router_logits, dim=-1, keepdim=True)
|
||||
topk_vals = (topk_logit_vals - logZ).exp()
|
||||
return topk_vals.to(torch.float32), topk_idx.to(torch.int32)
|
||||
else:
|
||||
topk_weights, topk_ids = custom_routing_function(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=top_k,
|
||||
renormalize=renormalize,
|
||||
)
|
||||
# cpu_fused_moe reads routing tensors as contiguous float32/int32
|
||||
# buffers and does not account for tensor strides.
|
||||
topk_weights = topk_weights.to(torch.float32).contiguous()
|
||||
topk_ids = topk_ids.to(torch.int32).contiguous()
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Unquantized (BF16/FP16/FP32) MoE
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
|
||||
"""Portable vector grouped-gemm unquantized MoE experts."""
|
||||
|
||||
isa = "vec"
|
||||
output_alignment = 32
|
||||
reduction_alignment = 1
|
||||
|
||||
@classmethod
|
||||
def _intermediate_alignment(cls) -> int:
|
||||
return math.lcm(cls.output_alignment, cls.reduction_alignment)
|
||||
|
||||
@classmethod
|
||||
def _padded_intermediate_size(cls, moe_config: FusedMoEConfig) -> int:
|
||||
intermediate_size = moe_config.intermediate_size_per_partition
|
||||
if moe_config.activation == MoEActivation.SWIGLUOAI:
|
||||
return intermediate_size
|
||||
return round_up(intermediate_size, cls._intermediate_alignment())
|
||||
|
||||
@classmethod
|
||||
def _supports_grouped_gemm(
|
||||
cls,
|
||||
moe_config: FusedMoEConfig,
|
||||
) -> tuple[bool, str | None]:
|
||||
intermediate_size = cls._padded_intermediate_size(moe_config)
|
||||
if (
|
||||
moe_config.hidden_dim % cls.output_alignment != 0
|
||||
or intermediate_size % cls.output_alignment != 0
|
||||
):
|
||||
return False, (
|
||||
"kernel requires hidden and intermediate dimensions divisible by "
|
||||
f"{cls.output_alignment}"
|
||||
)
|
||||
if (
|
||||
moe_config.hidden_dim % cls.reduction_alignment != 0
|
||||
or intermediate_size % cls.reduction_alignment != 0
|
||||
):
|
||||
return False, (
|
||||
"kernel requires reduction dimensions divisible by "
|
||||
f"{cls.reduction_alignment}"
|
||||
)
|
||||
return True, None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
# Router configuration that the monolithic apply() signature cannot
|
||||
# carry. Captured off the layer in process_weights_after_loading.
|
||||
self.use_grouped_topk = False
|
||||
self.renormalize = False
|
||||
self.scoring_func = "softmax"
|
||||
self.custom_routing_function: Callable | None = None
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cpu()
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
supported, reason = mk.FusedMoEExperts.is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if not supported:
|
||||
return supported, reason
|
||||
cpu_cls = cast(type[CPUUnquantizedExperts], cls)
|
||||
return cpu_cls._supports_grouped_gemm(moe_config)
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in (
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return (weight_key, activation_key) == (None, None)
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
routing_method: RoutingMethodType,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
# Routing runs in select_experts(), which covers every routing method
|
||||
# a layer can be configured with, including custom routing functions.
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self._pad_moe_intermediate(layer)
|
||||
self.use_grouped_topk = layer.use_grouped_topk
|
||||
self.renormalize = layer.renormalize
|
||||
self.scoring_func = layer.scoring_func
|
||||
self.custom_routing_function = layer.custom_routing_function
|
||||
replace_parameter(
|
||||
layer, "w13_weight", cpu_prepack_moe_weight(layer.w13_weight, self.isa)
|
||||
)
|
||||
replace_parameter(
|
||||
layer, "w2_weight", cpu_prepack_moe_weight(layer.w2_weight, self.isa)
|
||||
)
|
||||
|
||||
def _pad_moe_intermediate(self, layer: torch.nn.Module) -> None:
|
||||
"""Zero-pad the per-partition MoE intermediate dim of both weights and
|
||||
the expert bias, see `_padded_intermediate_size`."""
|
||||
intermediate_size = self.moe_config.intermediate_size_per_partition
|
||||
padded_size = self._padded_intermediate_size(self.moe_config)
|
||||
if padded_size == intermediate_size:
|
||||
return
|
||||
|
||||
num_experts, _, hidden_size = layer.w13_weight.shape
|
||||
|
||||
new_w13 = layer.w13_weight.new_zeros(num_experts, 2 * padded_size, hidden_size)
|
||||
new_w13[:, :intermediate_size] = layer.w13_weight[:, :intermediate_size]
|
||||
new_w13[:, padded_size : padded_size + intermediate_size] = layer.w13_weight[
|
||||
:, intermediate_size:
|
||||
]
|
||||
replace_parameter(layer, "w13_weight", new_w13)
|
||||
|
||||
new_w2 = layer.w2_weight.new_zeros(num_experts, hidden_size, padded_size)
|
||||
new_w2[:, :, :intermediate_size] = layer.w2_weight
|
||||
replace_parameter(layer, "w2_weight", new_w2)
|
||||
|
||||
if hasattr(layer, "w13_bias"):
|
||||
new_bias = layer.w13_bias.new_zeros(num_experts, 2 * padded_size)
|
||||
new_bias[:, :intermediate_size] = layer.w13_bias[:, :intermediate_size]
|
||||
new_bias[:, padded_size : padded_size + intermediate_size] = layer.w13_bias[
|
||||
:, intermediate_size:
|
||||
]
|
||||
# Assign through .data rather than replacing the Parameter: the
|
||||
# quant config is built before this runs and holds a reference to
|
||||
# this very object, which is what feeds self.w1_bias in apply().
|
||||
layer.w13_bias.data = new_bias
|
||||
|
||||
def _select_experts(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
num_expert_group: int | None,
|
||||
topk_group: int | None,
|
||||
e_score_correction_bias: torch.Tensor | None,
|
||||
routed_scaling_factor: float | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
use_grouped_topk=self.use_grouped_topk,
|
||||
renormalize=self.renormalize,
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
custom_routing_function=self.custom_routing_function,
|
||||
scoring_func=self.scoring_func,
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
# grouped topk + fused topk bias parameters
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
topk_weights, topk_ids = self._select_experts(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_expert_group,
|
||||
topk_group,
|
||||
e_score_correction_bias,
|
||||
routed_scaling_factor,
|
||||
)
|
||||
|
||||
if apply_router_weight_on_input:
|
||||
assert topk_ids.size(1) == 1, (
|
||||
"apply_router_weight_on_input is only implemented for topk=1"
|
||||
)
|
||||
hidden_states.mul_(topk_weights.to(hidden_states.dtype))
|
||||
|
||||
return cpu_fused_moe(
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
self.w1_bias,
|
||||
self.w2_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation.value,
|
||||
self.isa,
|
||||
apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
|
||||
class X86CPUUnquantizedExperts(CPUUnquantizedExperts):
|
||||
"""x86 AMX grouped-gemm unquantized MoE experts."""
|
||||
|
||||
isa = "amx"
|
||||
output_alignment = 32
|
||||
reduction_alignment = 32
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return (
|
||||
current_platform.is_cpu()
|
||||
and current_platform.get_cpu_architecture() == CpuArchEnum.X86
|
||||
and torch.cpu._is_amx_tile_supported()
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
supported, reason = mk.FusedMoEExperts.is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if not supported:
|
||||
return supported, reason
|
||||
if moe_config.in_dtype != torch.bfloat16:
|
||||
return False, "kernel requires bfloat16 activations"
|
||||
cpu_cls = cast(type[CPUUnquantizedExperts], cls)
|
||||
return cpu_cls._supports_grouped_gemm(moe_config)
|
||||
|
||||
|
||||
class ArmCPUUnquantizedExperts(CPUUnquantizedExperts):
|
||||
"""Arm NEON grouped-gemm unquantized MoE experts."""
|
||||
|
||||
isa = "neon"
|
||||
output_alignment = 32
|
||||
reduction_alignment = 4
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return (
|
||||
current_platform.is_cpu()
|
||||
and current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
and sys.platform != "darwin"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
supported, reason = mk.FusedMoEExperts.is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if not supported:
|
||||
return supported, reason
|
||||
if moe_config.in_dtype != torch.bfloat16:
|
||||
return False, "kernel requires bfloat16 activations"
|
||||
cpu_cls = cast(type[CPUUnquantizedExperts], cls)
|
||||
return cpu_cls._supports_grouped_gemm(moe_config)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# FP8 W8A16 MoE
|
||||
@@ -131,10 +570,6 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
select_experts,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -289,10 +724,6 @@ class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
select_experts,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -505,10 +936,6 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
"apply_router_weight_on_input=True. "
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
select_experts,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -631,7 +1058,6 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""VNNI-prepack INT8 MoE weights for CPU kernel."""
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
w13 = torch.ops._C.convert_weight_packed(layer.w13_weight)
|
||||
w2 = torch.ops._C.convert_weight_packed(layer.w2_weight)
|
||||
@@ -655,10 +1081,6 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
select_experts,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -796,8 +1218,6 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
return True
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
w13 = cpu_prepack_moe_weight_int8(layer.w13_weight, "neon")
|
||||
w2 = cpu_prepack_moe_weight_int8(layer.w2_weight, "neon")
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
@@ -819,10 +1239,6 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
select_experts,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
|
||||
@@ -143,6 +143,20 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [XPUExperts]
|
||||
|
||||
elif backend == UnquantizedMoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
ArmCPUUnquantizedExperts,
|
||||
CPUUnquantizedExperts,
|
||||
X86CPUUnquantizedExperts,
|
||||
)
|
||||
|
||||
# Prefer architecture-specific kernels before the portable vector path.
|
||||
return [
|
||||
X86CPUUnquantizedExperts,
|
||||
ArmCPUUnquantizedExperts,
|
||||
CPUUnquantizedExperts,
|
||||
]
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown unquantized MoE backend: {backend.value}")
|
||||
|
||||
@@ -199,10 +213,6 @@ def select_unquantized_moe_backend(
|
||||
Note: Shape-specific fallbacks may still occur at runtime.
|
||||
"""
|
||||
|
||||
if current_platform.is_cpu():
|
||||
# TODO: migrate to MK structure.
|
||||
return UnquantizedMoeBackend.CPU, None
|
||||
|
||||
if current_platform.is_tpu():
|
||||
return UnquantizedMoeBackend.TPU, None
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
@@ -33,7 +32,6 @@ from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import CpuArchEnum
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
@@ -54,13 +52,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
moe_config=self.moe,
|
||||
)
|
||||
|
||||
@property
|
||||
def is_monolithic(self) -> bool:
|
||||
# Escape hatch for CPU, which stays on the old monolithic path.
|
||||
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
|
||||
return True
|
||||
return super().is_monolithic
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return True
|
||||
@@ -71,8 +62,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
):
|
||||
raise ValueError(
|
||||
f"{self.__class__.__name__} uses the new modular kernel initialization "
|
||||
"logic for all but the CPU backend. CPU backend is monolithic. "
|
||||
"So this function should not be called."
|
||||
"logic. So this function should not be called."
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
@@ -201,6 +191,14 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
|
||||
# The CPU experts need the layer itself for the setup that
|
||||
# convert_to_unquantized_kernel_format cannot express, since
|
||||
# it only sees the two weight tensors: padding and prepacking
|
||||
# into the grouped-gemm layout (bias included), and capturing
|
||||
# the router config that monolithic apply() cannot carry.
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer: "RoutedExperts") -> None:
|
||||
super().process_weights_after_loading(layer)
|
||||
|
||||
@@ -221,38 +219,6 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
# OOT handles internally.
|
||||
return
|
||||
|
||||
elif self.unquantized_backend == UnquantizedMoeBackend.CPU:
|
||||
# CPU stays on the old path — no oracle, no moe_kernel.
|
||||
from vllm.model_executor.layers.fused_moe import cpu_fused_moe
|
||||
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.X86:
|
||||
from vllm.model_executor.layers.utils import check_cpu_sgl_kernel
|
||||
|
||||
dtype_w13 = layer.w13_weight.dtype
|
||||
_, n_w13, k_w13 = layer.w13_weight.size()
|
||||
dtype_w2 = layer.w2_weight.dtype
|
||||
_, n_w2, k_w2 = layer.w2_weight.size()
|
||||
if (
|
||||
envs.VLLM_CPU_SGL_KERNEL
|
||||
and check_cpu_sgl_kernel(n_w13, k_w13, dtype_w13)
|
||||
and check_cpu_sgl_kernel(n_w2, k_w2, dtype_w2)
|
||||
):
|
||||
packed_w13_weight = torch.ops._C.convert_weight_packed(
|
||||
layer.w13_weight
|
||||
)
|
||||
assert packed_w13_weight.size() == layer.w13_weight.size()
|
||||
layer.w13_weight.copy_(packed_w13_weight)
|
||||
del packed_w13_weight
|
||||
packed_w2_weight = torch.ops._C.convert_weight_packed(
|
||||
layer.w2_weight
|
||||
)
|
||||
assert packed_w2_weight.size() == layer.w2_weight.size()
|
||||
layer.w2_weight.copy_(packed_w2_weight)
|
||||
self.cpu_fused_moe: Callable = cpu_fused_moe.SGLFusedMOE(layer)
|
||||
else:
|
||||
self.cpu_fused_moe = cpu_fused_moe.CPUFusedMOE(layer)
|
||||
else:
|
||||
self.cpu_fused_moe = cpu_fused_moe.CPUFusedMOE(layer)
|
||||
elif self.unquantized_backend == UnquantizedMoeBackend.XPU:
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
@@ -363,39 +329,18 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
|
||||
assert self.moe_kernel is None
|
||||
return self.cpu_fused_moe(
|
||||
layer,
|
||||
x,
|
||||
layer.use_grouped_topk,
|
||||
layer.top_k,
|
||||
router_logits,
|
||||
layer.renormalize,
|
||||
layer.topk_group,
|
||||
layer.num_expert_group,
|
||||
layer.global_num_experts,
|
||||
layer.expert_map,
|
||||
layer.custom_routing_function,
|
||||
layer.scoring_func,
|
||||
layer.routed_scaling_factor,
|
||||
layer.e_score_correction_bias,
|
||||
layer.apply_router_weight_on_input,
|
||||
layer.activation,
|
||||
)
|
||||
else:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
num_expert_group=layer.num_expert_group,
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
)
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
x,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
num_expert_group=layer.num_expert_group,
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user