[Bugfix][CPU] Zero-pad MoE intermediate size for grouped-gemm TP alignment (#49591)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Li, Jiang
2026-07-27 16:32:23 +08:00
committed by GitHub
co-authored by Claude Sonnet 5
parent cbc3a87200
commit eb290ab673
2 changed files with 211 additions and 14 deletions
@@ -20,6 +20,11 @@ from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import direct_register_custom_op
_CPU_MOE_LAYER_CACHE = {}
# The CPU grouped-gemm MoE kernels (AMX and vector) tile the expert
# intermediate ("N") dimension in blocks of this size and have no tail/
# remainder handling, so a shard is only eligible for the fast path when
# its per-partition intermediate size is a multiple of it.
_MOE_GROUPED_GEMM_N_TILE = 32
def _swigluoai_forward_native(
@@ -230,6 +235,7 @@ class CPUFusedMOE:
"""CPU-based fused MoE implementation."""
def __init__(self, layer: torch.nn.Module) -> None:
self._pad_moe_intermediate_for_grouped_gemm(layer)
use_grouped_gemm, isa = self.check_grouped_gemm(layer)
self.isa = isa
if use_grouped_gemm:
@@ -284,6 +290,69 @@ class CPUFusedMOE:
apply_router_weight_on_input,
)
def _pad_moe_intermediate_for_grouped_gemm(self, layer: torch.nn.Module) -> None:
"""Zero-pad the per-partition MoE intermediate dim up to a multiple
of _MOE_GROUPED_GEMM_N_TILE, so the AMX/vector grouped-gemm kernels
can be used even when TP sharding (moe_intermediate_size // tp_size)
lands on an unaligned value (e.g. moe_intermediate_size=704 at tp=4
-> 176). Only applies to the x86 (AMX/vec) kernels and half-split
gate/up activations; interleaved layouts (swigluoai) are left
untouched.
"""
if not hasattr(torch.ops._C, "prepack_moe_weight"):
return
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
return
intermediate_size = layer.w2_weight.size(2)
remainder = intermediate_size % _MOE_GROUPED_GEMM_N_TILE
if remainder == 0:
return
if layer.activation == MoEActivation.SWIGLUOAI:
return
pad = _MOE_GROUPED_GEMM_N_TILE - remainder
padded_size = intermediate_size + pad
num_experts, _, hidden_size = layer.w13_weight.shape
new_w13 = layer.w13_weight.new_zeros(num_experts, 2 * padded_size, hidden_size)
new_w13[:, :intermediate_size] = layer.w13_weight[:, :intermediate_size]
new_w13[:, padded_size : padded_size + intermediate_size] = layer.w13_weight[
:, intermediate_size:
]
replace_parameter(layer, "w13_weight", new_w13)
new_w2 = layer.w2_weight.new_zeros(num_experts, hidden_size, padded_size)
new_w2[:, :, :intermediate_size] = layer.w2_weight
replace_parameter(layer, "w2_weight", new_w2)
if hasattr(layer, "w13_bias"):
new_bias = layer.w13_bias.new_zeros(num_experts, 2 * padded_size)
new_bias[:, :intermediate_size] = layer.w13_bias[:, :intermediate_size]
new_bias[:, padded_size : padded_size + intermediate_size] = layer.w13_bias[
:, intermediate_size:
]
replace_parameter(layer, "w13_bias", new_bias)
def _grouped_gemm_alignment_error(self, layer: torch.nn.Module) -> str:
# w2's input size is the per-partition MoE intermediate size (the
# dimension TP-sharding splits), and it's what most commonly breaks
# alignment, e.g. moe_intermediate_size=704 at tp=4 gives
# 704 // 4 == 176, which isn't a multiple of 32.
intermediate_size_per_partition = layer.w2_weight.size(2)
return (
"CPU fused-MoE AMX grouped-gemm kernel cannot be used for a "
f"layer with w13 shape {tuple(layer.w13_weight.shape)} / w2 "
f"shape {tuple(layer.w2_weight.shape)}: the per-partition MoE "
f"intermediate size ({intermediate_size_per_partition}) is not "
f"a multiple of {_MOE_GROUPED_GEMM_N_TILE}, and automatic "
"zero-padding could not resolve this (typically because the "
"activation uses an interleaved gate/up layout, e.g. "
"swigluoai). vLLM refuses to silently fall back to the much "
"slower per-expert torch loop on AMX-capable CPUs; consider a "
"different --tensor-parallel-size."
)
def check_grouped_gemm(
self,
layer: torch.nn.Module,
@@ -297,20 +366,19 @@ class CPUFusedMOE:
w2_input_size = layer.w2_weight.size(2)
w2_output_size = layer.w2_weight.size(1)
if not (w13_output_size % 32 == 0 and w2_output_size % 32 == 0):
return False, "none"
supports_amx = torch.cpu._is_amx_tile_supported()
if (
supports_amx
and dtype == torch.bfloat16
and w13_input_size % 32 == 0
and w2_input_size % 32 == 0
):
return True, "amx"
if supports_amx:
if (
dtype == torch.bfloat16
and w13_output_size % 32 == 0
and w2_output_size % 32 == 0
and w13_input_size % 32 == 0
and w2_input_size % 32 == 0
):
return True, "amx"
raise RuntimeError(self._grouped_gemm_alignment_error(layer))
if not (w13_output_size % 32 == 0 and w2_output_size % 32 == 0):
return False, "none"
supports_neon = current_platform.get_cpu_architecture() == CpuArchEnum.ARM
@@ -321,8 +389,7 @@ class CPUFusedMOE:
and w2_input_size % 4 == 0
):
return True, "neon"
else:
return False, "none"
return False, "none"
return True, "vec"