[CPU][Perf] INT8 Fused MoE Kernel for Arm CPUs (#48637)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
f19ee27e39
commit
c314af1abf
@@ -9,6 +9,8 @@ from vllm._custom_ops import (
|
||||
CPUQuantAlgo,
|
||||
CPUQuantMethod,
|
||||
convert_weight_packed_scale_zp,
|
||||
cpu_fused_moe_int8,
|
||||
cpu_prepack_moe_weight_int8,
|
||||
fused_experts_cpu,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
@@ -27,7 +29,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
# ===========================================================================
|
||||
# FP8 W8A16 MoE
|
||||
@@ -556,7 +558,14 @@ def prepare_int8_moe_layer_for_cpu(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""VNNI-prepack INT8 MoE weights for CPU kernel."""
|
||||
"""Prepack INT8 MoE weights for the current CPU architecture."""
|
||||
# SMMLA packing for AArch64
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
return (
|
||||
cpu_prepack_moe_weight_int8(w13, "neon"),
|
||||
cpu_prepack_moe_weight_int8(w2, "neon"),
|
||||
)
|
||||
# VNNI packing for x86
|
||||
packed_w13 = torch.ops._C.convert_weight_packed(w13)
|
||||
packed_w2 = torch.ops._C.convert_weight_packed(w2)
|
||||
return packed_w13, packed_w2
|
||||
@@ -586,7 +595,10 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cpu()
|
||||
return (
|
||||
current_platform.is_cpu()
|
||||
and current_platform.get_cpu_architecture() == CpuArchEnum.X86
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
@@ -701,3 +713,167 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
None, # limit
|
||||
True, # is_vnni
|
||||
)
|
||||
|
||||
|
||||
class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
"""Arm INT8 MoE with per-token activation and channelwise weight quantization."""
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
supported, reason = mk.FusedMoEExperts.is_supported_config(
|
||||
cls,
|
||||
moe_config,
|
||||
weight_key,
|
||||
activation_key,
|
||||
activation_format,
|
||||
)
|
||||
if not supported:
|
||||
return supported, reason
|
||||
if moe_config.in_dtype not in (
|
||||
torch.float32,
|
||||
torch.float16,
|
||||
torch.bfloat16,
|
||||
):
|
||||
return False, "kernel requires float32, float16, or bfloat16 activations"
|
||||
if moe_config.hidden_dim % 32 != 0:
|
||||
return False, "kernel requires hidden dim divisible by 32"
|
||||
if moe_config.intermediate_size_per_partition % 32 != 0:
|
||||
return False, "kernel requires intermediate dim divisible by 32"
|
||||
return True, None
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return (
|
||||
current_platform.is_cpu()
|
||||
and current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
and hasattr(torch.ops._C, "cpu_fused_moe_int8")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in (
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return not moe_parallel_config.use_ep
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return (weight_key, activation_key) == (
|
||||
kInt8StaticChannelSym,
|
||||
kInt8DynamicTokenSym,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
routing_method: RoutingMethodType,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return routing_method in [
|
||||
RoutingMethodType.Default,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
w13, w2 = prepare_int8_moe_layer_for_cpu(layer.w13_weight, layer.w2_weight)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
select_experts,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.moe_config.routing_method
|
||||
in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
),
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
if apply_router_weight_on_input:
|
||||
assert topk_ids.size(1) == 1
|
||||
hidden_states.mul_(topk_weights.to(hidden_states.dtype))
|
||||
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
return cpu_fused_moe_int8(
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
self.w1_scale,
|
||||
self.w2_scale,
|
||||
self.w1_bias,
|
||||
self.w2_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation.value,
|
||||
"neon",
|
||||
skip_weighted=apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
@@ -24,7 +24,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8StaticChannelSym,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -41,20 +40,12 @@ def _get_priority_backends(
|
||||
"""
|
||||
Get available backends in priority order based on platform and config.
|
||||
"""
|
||||
_AVAILABLE_BACKENDS = [
|
||||
return [
|
||||
Int8MoeBackend.TRITON,
|
||||
Int8MoeBackend.HUMMING,
|
||||
Int8MoeBackend.CPU,
|
||||
]
|
||||
|
||||
def _move_to_front(backends: list[Int8MoeBackend], backend: Int8MoeBackend) -> None:
|
||||
backends.insert(0, backends.pop(backends.index(backend)))
|
||||
|
||||
if current_platform.is_cpu():
|
||||
_move_to_front(_AVAILABLE_BACKENDS, Int8MoeBackend.CPU)
|
||||
|
||||
return _AVAILABLE_BACKENDS
|
||||
|
||||
|
||||
def backend_to_kernel_cls(
|
||||
backend: Int8MoeBackend,
|
||||
@@ -78,14 +69,13 @@ def backend_to_kernel_cls(
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
]
|
||||
|
||||
elif backend == Int8MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
ArmCPUExpertsInt8,
|
||||
CPUExpertsInt8,
|
||||
)
|
||||
|
||||
return [CPUExpertsInt8]
|
||||
|
||||
return [ArmCPUExpertsInt8, CPUExpertsInt8]
|
||||
else:
|
||||
raise ValueError(f"Unknown Int8 MoE backend: {backend.value}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user