[perf][cpu] Accelerate BF16 GELU with LUT impl on Arm CPUs (#37469)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
18013df6ae
commit
445b7093fd
@@ -3300,6 +3300,12 @@ def cpu_gemm_wna16(
|
||||
return output
|
||||
|
||||
|
||||
def cpu_activation_lut_bf16(input: torch.Tensor, activation: str) -> torch.Tensor:
|
||||
out = torch.empty_like(input)
|
||||
torch.ops._C.activation_lut_bf16(out, input, activation)
|
||||
return out
|
||||
|
||||
|
||||
def cpu_prepack_moe_weight(
|
||||
weight: torch.Tensor,
|
||||
isa: str,
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.distributed import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.collection_utils import LazyDict
|
||||
|
||||
@@ -247,6 +247,34 @@ class GeluAndMulSparse(CustomOp):
|
||||
return self.forward_native(x)
|
||||
|
||||
|
||||
# --8<-- [start:gelu]
|
||||
@CustomOp.register("gelu")
|
||||
class GELU(CustomOp):
|
||||
# --8<-- [end:gelu]
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM and hasattr(
|
||||
torch.ops._C, "activation_lut_bf16"
|
||||
):
|
||||
self.op = torch.ops._C.activation_lut_bf16
|
||||
else:
|
||||
self.op = None
|
||||
|
||||
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return F.gelu(x, approximate="none")
|
||||
|
||||
def forward_cpu(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.op and x.dtype == torch.bfloat16 and x.is_contiguous():
|
||||
out = torch.empty_like(x)
|
||||
self.op(out, x, "gelu")
|
||||
return out
|
||||
return self.forward_native(x)
|
||||
|
||||
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.forward_native(x)
|
||||
|
||||
|
||||
# --8<-- [start:gelu_and_mul]
|
||||
@CustomOp.register("gelu_and_mul")
|
||||
class GeluAndMul(CustomOp):
|
||||
@@ -635,7 +663,7 @@ class ScaledActivation(nn.Module):
|
||||
|
||||
_ACTIVATION_REGISTRY = LazyDict(
|
||||
{
|
||||
"gelu": lambda: nn.GELU(),
|
||||
"gelu": lambda: GELU(),
|
||||
"gelu_fast": lambda: FastGELU(),
|
||||
"gelu_new": lambda: NewGELU(),
|
||||
"gelu_pytorch_tanh": lambda: (
|
||||
|
||||
@@ -246,6 +246,13 @@ class CpuPlatform(Platform):
|
||||
if vllm_config.lora_config is not None:
|
||||
compilation_config.mode = CompilationMode.NONE
|
||||
|
||||
if (
|
||||
cls.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
and "+gelu" not in compilation_config.custom_ops
|
||||
and "-gelu" not in compilation_config.custom_ops
|
||||
):
|
||||
compilation_config.custom_ops.append("+gelu")
|
||||
|
||||
vllm_config.profiler_config.torch_profiler_dump_cuda_time_total = False
|
||||
|
||||
assert vllm_config.device_config.device_type == "cpu"
|
||||
|
||||
Reference in New Issue
Block a user