[Bugfix][CPU] Zero-pad MoE intermediate size for grouped-gemm TP alignment (#49591)
Signed-off-by: jiang1.li <jiang1.li@intel.com> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
cbc3a87200
commit
eb290ab673
@@ -20,6 +20,11 @@ from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
_CPU_MOE_LAYER_CACHE = {}
|
||||
# The CPU grouped-gemm MoE kernels (AMX and vector) tile the expert
|
||||
# intermediate ("N") dimension in blocks of this size and have no tail/
|
||||
# remainder handling, so a shard is only eligible for the fast path when
|
||||
# its per-partition intermediate size is a multiple of it.
|
||||
_MOE_GROUPED_GEMM_N_TILE = 32
|
||||
|
||||
|
||||
def _swigluoai_forward_native(
|
||||
@@ -230,6 +235,7 @@ class CPUFusedMOE:
|
||||
"""CPU-based fused MoE implementation."""
|
||||
|
||||
def __init__(self, layer: torch.nn.Module) -> None:
|
||||
self._pad_moe_intermediate_for_grouped_gemm(layer)
|
||||
use_grouped_gemm, isa = self.check_grouped_gemm(layer)
|
||||
self.isa = isa
|
||||
if use_grouped_gemm:
|
||||
@@ -284,6 +290,69 @@ class CPUFusedMOE:
|
||||
apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
def _pad_moe_intermediate_for_grouped_gemm(self, layer: torch.nn.Module) -> None:
|
||||
"""Zero-pad the per-partition MoE intermediate dim up to a multiple
|
||||
of _MOE_GROUPED_GEMM_N_TILE, so the AMX/vector grouped-gemm kernels
|
||||
can be used even when TP sharding (moe_intermediate_size // tp_size)
|
||||
lands on an unaligned value (e.g. moe_intermediate_size=704 at tp=4
|
||||
-> 176). Only applies to the x86 (AMX/vec) kernels and half-split
|
||||
gate/up activations; interleaved layouts (swigluoai) are left
|
||||
untouched.
|
||||
"""
|
||||
if not hasattr(torch.ops._C, "prepack_moe_weight"):
|
||||
return
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
return
|
||||
|
||||
intermediate_size = layer.w2_weight.size(2)
|
||||
remainder = intermediate_size % _MOE_GROUPED_GEMM_N_TILE
|
||||
if remainder == 0:
|
||||
return
|
||||
if layer.activation == MoEActivation.SWIGLUOAI:
|
||||
return
|
||||
|
||||
pad = _MOE_GROUPED_GEMM_N_TILE - remainder
|
||||
padded_size = intermediate_size + pad
|
||||
num_experts, _, hidden_size = layer.w13_weight.shape
|
||||
|
||||
new_w13 = layer.w13_weight.new_zeros(num_experts, 2 * padded_size, hidden_size)
|
||||
new_w13[:, :intermediate_size] = layer.w13_weight[:, :intermediate_size]
|
||||
new_w13[:, padded_size : padded_size + intermediate_size] = layer.w13_weight[
|
||||
:, intermediate_size:
|
||||
]
|
||||
replace_parameter(layer, "w13_weight", new_w13)
|
||||
|
||||
new_w2 = layer.w2_weight.new_zeros(num_experts, hidden_size, padded_size)
|
||||
new_w2[:, :, :intermediate_size] = layer.w2_weight
|
||||
replace_parameter(layer, "w2_weight", new_w2)
|
||||
|
||||
if hasattr(layer, "w13_bias"):
|
||||
new_bias = layer.w13_bias.new_zeros(num_experts, 2 * padded_size)
|
||||
new_bias[:, :intermediate_size] = layer.w13_bias[:, :intermediate_size]
|
||||
new_bias[:, padded_size : padded_size + intermediate_size] = layer.w13_bias[
|
||||
:, intermediate_size:
|
||||
]
|
||||
replace_parameter(layer, "w13_bias", new_bias)
|
||||
|
||||
def _grouped_gemm_alignment_error(self, layer: torch.nn.Module) -> str:
|
||||
# w2's input size is the per-partition MoE intermediate size (the
|
||||
# dimension TP-sharding splits), and it's what most commonly breaks
|
||||
# alignment, e.g. moe_intermediate_size=704 at tp=4 gives
|
||||
# 704 // 4 == 176, which isn't a multiple of 32.
|
||||
intermediate_size_per_partition = layer.w2_weight.size(2)
|
||||
return (
|
||||
"CPU fused-MoE AMX grouped-gemm kernel cannot be used for a "
|
||||
f"layer with w13 shape {tuple(layer.w13_weight.shape)} / w2 "
|
||||
f"shape {tuple(layer.w2_weight.shape)}: the per-partition MoE "
|
||||
f"intermediate size ({intermediate_size_per_partition}) is not "
|
||||
f"a multiple of {_MOE_GROUPED_GEMM_N_TILE}, and automatic "
|
||||
"zero-padding could not resolve this (typically because the "
|
||||
"activation uses an interleaved gate/up layout, e.g. "
|
||||
"swigluoai). vLLM refuses to silently fall back to the much "
|
||||
"slower per-expert torch loop on AMX-capable CPUs; consider a "
|
||||
"different --tensor-parallel-size."
|
||||
)
|
||||
|
||||
def check_grouped_gemm(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -297,20 +366,19 @@ class CPUFusedMOE:
|
||||
w2_input_size = layer.w2_weight.size(2)
|
||||
w2_output_size = layer.w2_weight.size(1)
|
||||
|
||||
if not (w13_output_size % 32 == 0 and w2_output_size % 32 == 0):
|
||||
return False, "none"
|
||||
|
||||
supports_amx = torch.cpu._is_amx_tile_supported()
|
||||
|
||||
if (
|
||||
supports_amx
|
||||
and dtype == torch.bfloat16
|
||||
and w13_input_size % 32 == 0
|
||||
and w2_input_size % 32 == 0
|
||||
):
|
||||
return True, "amx"
|
||||
|
||||
if supports_amx:
|
||||
if (
|
||||
dtype == torch.bfloat16
|
||||
and w13_output_size % 32 == 0
|
||||
and w2_output_size % 32 == 0
|
||||
and w13_input_size % 32 == 0
|
||||
and w2_input_size % 32 == 0
|
||||
):
|
||||
return True, "amx"
|
||||
raise RuntimeError(self._grouped_gemm_alignment_error(layer))
|
||||
|
||||
if not (w13_output_size % 32 == 0 and w2_output_size % 32 == 0):
|
||||
return False, "none"
|
||||
|
||||
supports_neon = current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
@@ -321,8 +389,7 @@ class CPUFusedMOE:
|
||||
and w2_input_size % 4 == 0
|
||||
):
|
||||
return True, "neon"
|
||||
else:
|
||||
return False, "none"
|
||||
return False, "none"
|
||||
|
||||
return True, "vec"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user