[XPU] Dispatch nn.LayerNorm to fused SYCL kernel via CustomOp (#57172)
Signed-off-by: Michal Ganczarenko <michal.ganczarenko@intel.com> Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
Harry Mellor
parent
a4c4bb418e
commit
df5668e5c9
@@ -211,6 +211,79 @@ class GemmaRMSNorm(CustomOp):
|
||||
return out
|
||||
|
||||
|
||||
@CustomOp.register("layer_norm")
|
||||
class LayerNorm(CustomOp):
|
||||
"""Standard (mean-centered) LayerNorm.
|
||||
|
||||
Drop-in for a bare `nn.LayerNorm` that dispatches to a fused XPU kernel
|
||||
when one is available, and to the native implementation otherwise.
|
||||
|
||||
Normalization runs in the wider of the input and parameter dtypes and the
|
||||
result is cast back to the input dtype, so `dtype=torch.float32` gives an
|
||||
fp32 reduction for a lower-precision input.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
eps: float = 1e-5,
|
||||
elementwise_affine: bool = True,
|
||||
bias: bool = True,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.normalized_shape = (hidden_size,)
|
||||
self.eps = eps
|
||||
weight_dtype = dtype or torch.get_default_dtype()
|
||||
self.weight: nn.Parameter | None = None
|
||||
self.bias: nn.Parameter | None = None
|
||||
if elementwise_affine:
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=weight_dtype))
|
||||
if bias:
|
||||
self.bias = nn.Parameter(torch.zeros(hidden_size, dtype=weight_dtype))
|
||||
|
||||
def _compute_dtype(self, x: torch.Tensor) -> torch.dtype:
|
||||
if self.weight is None:
|
||||
return x.dtype
|
||||
return torch.promote_types(x.dtype, self.weight.dtype)
|
||||
|
||||
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
|
||||
compute_dtype = self._compute_dtype(x)
|
||||
if compute_dtype != x.dtype:
|
||||
return F.layer_norm(
|
||||
x.to(compute_dtype),
|
||||
self.normalized_shape,
|
||||
self.weight,
|
||||
self.bias,
|
||||
self.eps,
|
||||
).type_as(x)
|
||||
return F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)
|
||||
|
||||
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.forward_native(x)
|
||||
|
||||
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
|
||||
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_layer_norm
|
||||
|
||||
if (
|
||||
self.weight is None
|
||||
or self.bias is None
|
||||
or not hasattr(torch.ops._C, "layer_norm")
|
||||
# The kernel reduces in the input dtype, so a widened reduction
|
||||
# takes the native path.
|
||||
or self._compute_dtype(x) != x.dtype
|
||||
):
|
||||
return self.forward_native(x)
|
||||
# empty_like preserves x's strides, but the kernel requires a
|
||||
# contiguous out (unlike x, which it can handle non-contiguous).
|
||||
out = torch.empty(x.shape, device=x.device, dtype=x.dtype)
|
||||
torch.ops.vllm.xpu_layer_norm(out, x, self.weight, self.bias, self.eps)
|
||||
return out
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"hidden_size={self.normalized_shape[0]}, eps={self.eps}"
|
||||
|
||||
|
||||
# --8<-- [start:rms_norm_gated]
|
||||
@CustomOp.register("rms_norm_gated")
|
||||
class RMSNormGated(CustomOp):
|
||||
@@ -350,19 +423,3 @@ class RMSNormGated(CustomOp):
|
||||
self, x: torch.Tensor, z: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
return self.forward_cuda(x, z)
|
||||
|
||||
|
||||
class LayerNorm(nn.Module):
|
||||
"""Layer Normalization."""
|
||||
|
||||
def __init__(self, dim: int, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.eps = eps
|
||||
self.weight = nn.Parameter(torch.ones(dim, dtype=torch.float32))
|
||||
self.bias = nn.Parameter(torch.zeros(dim, dtype=torch.float32))
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
return F.layer_norm(
|
||||
x.float(), (self.dim,), self.weight, self.bias, self.eps
|
||||
).type_as(x)
|
||||
|
||||
Reference in New Issue
Block a user