[CPU][Perf]Added tanh AOR for faster gelu activations. (#44639)
Signed-off-by: Anna Mayne <anna.mayne@arm.com> Signed-off-by: almayne <anna.mayne@arm.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Li, Jiang <jiang1.li@intel.com>
parent
b446792306
commit
89e99202f2
@@ -313,8 +313,10 @@ class GELU(CustomOp):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM and hasattr(
|
||||
torch.ops._C, "activation_lut_bf16"
|
||||
if (
|
||||
current_platform.is_cpu()
|
||||
and current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
and hasattr(torch.ops._C, "activation_lut_bf16")
|
||||
):
|
||||
self.op = torch.ops._C.activation_lut_bf16
|
||||
else:
|
||||
@@ -334,6 +336,36 @@ class GELU(CustomOp):
|
||||
return self.forward_native(x)
|
||||
|
||||
|
||||
# --8<-- [start:gelu_tanh]
|
||||
@CustomOp.register("gelu_tanh")
|
||||
class GELUTanh(CustomOp):
|
||||
# --8<-- [end:gelu_tanh]
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
if (
|
||||
current_platform.is_cpu()
|
||||
and current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
and hasattr(torch.ops._C, "gelu_tanh")
|
||||
):
|
||||
self.op = torch.ops._C.gelu_tanh
|
||||
else:
|
||||
self.op = None
|
||||
|
||||
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return F.gelu(x, approximate="tanh")
|
||||
|
||||
def forward_cpu(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.op:
|
||||
out = torch.empty_like(x)
|
||||
self.op(out, x)
|
||||
return out
|
||||
return self.forward_native(x)
|
||||
|
||||
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.forward_native(x)
|
||||
|
||||
|
||||
# --8<-- [start:gelu_and_mul]
|
||||
@CustomOp.register("gelu_and_mul")
|
||||
class GeluAndMul(CustomOp):
|
||||
@@ -385,6 +417,11 @@ class GeluAndMul(CustomOp):
|
||||
self.op(out, x)
|
||||
return out
|
||||
|
||||
def forward_cpu(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.op:
|
||||
return self.forward_cuda(x)
|
||||
return self.native(x)
|
||||
|
||||
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.forward_cuda(x)
|
||||
|
||||
@@ -739,7 +776,8 @@ _ACTIVATION_REGISTRY = LazyDict(
|
||||
|
||||
|
||||
def _get_gelu_pytorch_tanh() -> nn.Module:
|
||||
"""Get PyTorch GELU with tanh approximation, with ROCm fallback."""
|
||||
"""Get PyTorch GELU with tanh approximation, with ROCm fallback
|
||||
and fast GELU for ARM."""
|
||||
if current_platform.is_rocm():
|
||||
# TODO:[ROCm] PyTorch native GELU with tanh is unstable with torch.compile
|
||||
logger.warning_once(
|
||||
@@ -747,6 +785,11 @@ def _get_gelu_pytorch_tanh() -> nn.Module:
|
||||
"Falling back to GELU(approximate='none')."
|
||||
)
|
||||
return nn.GELU(approximate="none")
|
||||
if (
|
||||
current_platform.is_cpu()
|
||||
and current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
):
|
||||
return GELUTanh()
|
||||
return nn.GELU(approximate="tanh")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user