[CPU] Refine CPU kernel dispatch (#50801)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-08-03 17:42:53 +08:00
committed by GitHub
parent b9d1e2437e
commit c8602c7906
6 changed files with 47 additions and 30 deletions
@@ -508,7 +508,11 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cpu()
return (
current_platform.is_cpu()
and current_platform.get_cpu_architecture() == CpuArchEnum.X86
and torch.cpu._is_amx_tile_supported()
)
@staticmethod
def _supports_no_act_and_mul() -> bool:
@@ -662,7 +666,11 @@ class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cpu()
return (
current_platform.is_cpu()
and current_platform.get_cpu_architecture() == CpuArchEnum.X86
and torch.cpu._is_amx_tile_supported()
)
@staticmethod
def _supports_no_act_and_mul() -> bool:
@@ -865,7 +873,11 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cpu()
return (
current_platform.is_cpu()
and current_platform.get_cpu_architecture() == CpuArchEnum.X86
and torch.cpu._is_amx_tile_supported()
)
@staticmethod
def _supports_no_act_and_mul() -> bool:
@@ -1008,8 +1020,37 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
return (
current_platform.is_cpu()
and current_platform.get_cpu_architecture() == CpuArchEnum.X86
and torch.cpu._is_amx_tile_supported()
)
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
supported, reason = mk.FusedMoEExperts.is_supported_config(
cls,
moe_config,
weight_key,
activation_key,
activation_format,
)
if not supported:
return supported, reason
# convert_weight_packed (shared VNNI prepack) requires the w13
# OC/IC and w2 OC/IC to be multiples of TILE_N=16/TILE_K=32; the
# w1 gate-up kernel additionally requires the intermediate size
# itself (not 2x) to be a multiple of 32 (moe_int8.cpp), which
# dominates. Net effect: both dims must be multiples of 32.
if moe_config.hidden_dim % 32 != 0:
return False, "kernel requires hidden dim divisible by 32"
if moe_config.intermediate_size_per_partition % 32 != 0:
return False, "kernel requires intermediate dim divisible by 32"
return True, None
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
+2 -20
View File
@@ -225,7 +225,7 @@ direct_register_custom_op(
def check_cpu_sgl_kernel(n: int, k: int, dtype: torch.dtype) -> bool:
return (
torch.cpu._is_amx_tile_supported()
and (dtype in (torch.bfloat16, torch.int8))
and (dtype in (torch.bfloat16, torch.float16, torch.int8))
and k % 32 == 0
and n % 16 == 0
)
@@ -263,9 +263,6 @@ def dispatch_cpu_unquantized_gemm(
layer.weight.data = ops.causal_conv1d_weight_pack(unpacked)
return
N, K = layer.weight.size()
dtype = layer.weight.dtype
# Zen CPU path: zentorch_linear_unary with optional eager weight prepacking.
if current_platform.is_zen_cpu() and hasattr(
torch.ops.zentorch, "zentorch_linear_unary"
@@ -294,22 +291,7 @@ def dispatch_cpu_unquantized_gemm(
)
return
if envs.VLLM_CPU_SGL_KERNEL and check_cpu_sgl_kernel(N, K, dtype):
packed_weight = torch.ops._C.convert_weight_packed(layer.weight)
if getattr(layer, "bias", None) is not None:
bias_f32 = layer.bias.to(torch.float32)
else:
bias_f32 = None
layer.cpu_linear = lambda x, weight, bias: torch.ops._C.weight_packed_linear(
x, packed_weight, bias_f32 if bias is not None else None, True
)
if remove_weight:
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
logger.debug_once(
"CPU unquantized GEMM dispatch: using sgl-kernel weight_packed_linear"
)
return
elif (
if (
ops._supports_onednn
and current_platform.get_cpu_architecture() != CpuArchEnum.POWERPC
):