[CPU][Perf] INT8 Fused MoE Kernel for Arm CPUs (#48637)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
f19ee27e39
commit
c314af1abf
@@ -7,10 +7,14 @@ import torch
|
||||
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
|
||||
from vllm._custom_ops import (
|
||||
cpu_fused_moe,
|
||||
cpu_fused_moe_int8,
|
||||
cpu_prepack_moe_weight,
|
||||
cpu_prepack_moe_weight_int8,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import _CPU_MOE_ACT_FN
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
_CPU_MOE_ACT_FN,
|
||||
)
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -101,6 +105,90 @@ def ref_fused_moe(
|
||||
return final_out
|
||||
|
||||
|
||||
def quantize_per_channel(
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Symmetrically quantize each weight output channel."""
|
||||
weight_fp32 = weight.float()
|
||||
scale = weight_fp32.abs().amax(dim=-1).clamp_min(1e-12) / 127.0
|
||||
quantized = (
|
||||
(weight_fp32 / scale.unsqueeze(-1)).round().clamp(-127, 127).to(torch.int8)
|
||||
)
|
||||
return quantized, scale
|
||||
|
||||
|
||||
def quantize_per_token(
|
||||
input: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Symmetrically quantize each input token."""
|
||||
input_fp32 = input.float()
|
||||
scale = input_fp32.abs().amax(dim=-1, keepdim=True).clamp_min(1e-7) / 127.0
|
||||
quantized = (input_fp32 / scale).round().clamp(-127, 127).to(torch.int8)
|
||||
return quantized, scale
|
||||
|
||||
|
||||
def ref_fused_moe_int8(
|
||||
input: torch.Tensor,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w13_bias: torch.Tensor | None,
|
||||
w2_bias: torch.Tensor | None,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
) -> torch.Tensor:
|
||||
"""Reference the two dynamically quantized INT8 GEMMs."""
|
||||
input_int8, input_scale = quantize_per_token(input)
|
||||
expert_num = w13.size(0)
|
||||
|
||||
counts = topk_ids.new_zeros((topk_ids.size(0), expert_num))
|
||||
counts.scatter_(1, topk_ids.to(torch.int64), 1)
|
||||
tokens_per_expert = counts.sum(dim=0).tolist()
|
||||
sorted_route_ids = topk_ids.view(-1).argsort()
|
||||
sorted_token_ids = sorted_route_ids // topk_ids.size(1)
|
||||
|
||||
outputs = []
|
||||
start_idx = 0
|
||||
for expert_idx, token_count in enumerate(tokens_per_expert):
|
||||
end_idx = start_idx + token_count
|
||||
if token_count == 0:
|
||||
continue
|
||||
|
||||
token_ids = sorted_token_ids[start_idx:end_idx]
|
||||
gate_up = torch.matmul(
|
||||
input_int8[token_ids].float(),
|
||||
w13[expert_idx].float().T,
|
||||
)
|
||||
gate_up *= input_scale[token_ids] * w13_scale[expert_idx]
|
||||
if w13_bias is not None:
|
||||
gate_up += w13_bias[expert_idx].float()
|
||||
|
||||
intermediate = _CPU_MOE_ACT_FN[activation](gate_up).to(input.dtype)
|
||||
intermediate_int8, intermediate_scale = quantize_per_token(intermediate)
|
||||
output = torch.matmul(
|
||||
intermediate_int8.float(),
|
||||
w2[expert_idx].float().T,
|
||||
)
|
||||
output *= intermediate_scale * w2_scale[expert_idx]
|
||||
if w2_bias is not None:
|
||||
output += w2_bias[expert_idx].float()
|
||||
|
||||
outputs.append(output)
|
||||
start_idx = end_idx
|
||||
|
||||
sorted_output = torch.cat(outputs, dim=0)
|
||||
routed_output = torch.empty_like(sorted_output)
|
||||
routed_output[sorted_route_ids] = sorted_output
|
||||
return (
|
||||
routed_output.view(*topk_ids.shape, -1)
|
||||
.mul_(topk_weights.unsqueeze(-1))
|
||||
.sum(dim=1)
|
||||
.to(input.dtype)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZE)
|
||||
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
|
||||
@@ -176,3 +264,85 @@ def test_cpu_fused_moe(
|
||||
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.get_cpu_architecture() != CpuArchEnum.ARM,
|
||||
reason="Requires Arm CPU",
|
||||
)
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZE)
|
||||
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
|
||||
@pytest.mark.parametrize("intermediate_size", INTERMEDIATE_DIM)
|
||||
@pytest.mark.parametrize("use_bias", USE_BIAS)
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("act", ACT)
|
||||
@pytest.mark.parametrize("isa", ["neon"])
|
||||
def test_cpu_fused_moe_int8(
|
||||
batch_size: int,
|
||||
expert_num: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
use_bias: bool,
|
||||
dtype: torch.dtype,
|
||||
act: MoEActivation,
|
||||
isa: str,
|
||||
):
|
||||
set_random_seed(0)
|
||||
topk_num = max(expert_num // 2, 1)
|
||||
up_dim = 2 * intermediate_size
|
||||
|
||||
input = torch.randn((batch_size, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
w13_fp = torch.randn((expert_num, up_dim, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
w2_fp = torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype) / (
|
||||
0.5 * intermediate_size**0.5
|
||||
)
|
||||
w13, w13_scale = quantize_per_channel(w13_fp)
|
||||
w2, w2_scale = quantize_per_channel(w2_fp)
|
||||
|
||||
w13_bias = None
|
||||
w2_bias = None
|
||||
if use_bias:
|
||||
w13_bias = torch.randn((expert_num, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
|
||||
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
|
||||
router_logits = torch.randn((batch_size, expert_num), dtype=dtype)
|
||||
score = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(score, topk_num)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
ref_output = ref_fused_moe_int8(
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
act,
|
||||
)
|
||||
packed_w13 = cpu_prepack_moe_weight_int8(w13, isa)
|
||||
packed_w2 = cpu_prepack_moe_weight_int8(w2, isa)
|
||||
output = cpu_fused_moe_int8(
|
||||
input,
|
||||
packed_w13,
|
||||
packed_w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
act.value,
|
||||
isa,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=2e-2)
|
||||
|
||||
Reference in New Issue
Block a user