[CPU][Perf] Accelerate unquantized MoE for AArch64 (#46353)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
@@ -16,6 +16,7 @@ from vllm._custom_ops import (
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.quantization.utils.layer_utils import replace_parameter
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
_CPU_MOE_LAYER_CACHE = {}
|
||||
@@ -312,6 +313,17 @@ class CPUFusedMOE:
|
||||
if supports_amx:
|
||||
return False, "none"
|
||||
|
||||
supports_neon = current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
if supports_neon:
|
||||
if (
|
||||
dtype == torch.bfloat16
|
||||
and w13_input_size % 4 == 0
|
||||
and w2_input_size % 4 == 0
|
||||
):
|
||||
return True, "neon"
|
||||
else:
|
||||
return False, "none"
|
||||
|
||||
return True, "vec"
|
||||
|
||||
def init_moe_grouped_gemm(
|
||||
|
||||
Reference in New Issue
Block a user