[XPU] Dispatch nn.LayerNorm to fused SYCL kernel via CustomOp (#57172)

Signed-off-by: Michal Ganczarenko <michal.ganczarenko@intel.com>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Michał Ganczarenko
2026-09-30 12:19:33 +00:00
committed by GitHub
co-authored by Claude Sonnet 5 Harry Mellor
parent a4c4bb418e
commit df5668e5c9
13 changed files with 237 additions and 46 deletions
+73 -16
View File
@@ -211,6 +211,79 @@ class GemmaRMSNorm(CustomOp):
return out
@CustomOp.register("layer_norm")
class LayerNorm(CustomOp):
"""Standard (mean-centered) LayerNorm.
Drop-in for a bare `nn.LayerNorm` that dispatches to a fused XPU kernel
when one is available, and to the native implementation otherwise.
Normalization runs in the wider of the input and parameter dtypes and the
result is cast back to the input dtype, so `dtype=torch.float32` gives an
fp32 reduction for a lower-precision input.
"""
def __init__(
self,
hidden_size: int,
eps: float = 1e-5,
elementwise_affine: bool = True,
bias: bool = True,
dtype: torch.dtype | None = None,
) -> None:
super().__init__()
self.normalized_shape = (hidden_size,)
self.eps = eps
weight_dtype = dtype or torch.get_default_dtype()
self.weight: nn.Parameter | None = None
self.bias: nn.Parameter | None = None
if elementwise_affine:
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=weight_dtype))
if bias:
self.bias = nn.Parameter(torch.zeros(hidden_size, dtype=weight_dtype))
def _compute_dtype(self, x: torch.Tensor) -> torch.dtype:
if self.weight is None:
return x.dtype
return torch.promote_types(x.dtype, self.weight.dtype)
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
compute_dtype = self._compute_dtype(x)
if compute_dtype != x.dtype:
return F.layer_norm(
x.to(compute_dtype),
self.normalized_shape,
self.weight,
self.bias,
self.eps,
).type_as(x)
return F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps)
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_layer_norm
if (
self.weight is None
or self.bias is None
or not hasattr(torch.ops._C, "layer_norm")
# The kernel reduces in the input dtype, so a widened reduction
# takes the native path.
or self._compute_dtype(x) != x.dtype
):
return self.forward_native(x)
# empty_like preserves x's strides, but the kernel requires a
# contiguous out (unlike x, which it can handle non-contiguous).
out = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops.vllm.xpu_layer_norm(out, x, self.weight, self.bias, self.eps)
return out
def extra_repr(self) -> str:
return f"hidden_size={self.normalized_shape[0]}, eps={self.eps}"
# --8<-- [start:rms_norm_gated]
@CustomOp.register("rms_norm_gated")
class RMSNormGated(CustomOp):
@@ -350,19 +423,3 @@ class RMSNormGated(CustomOp):
self, x: torch.Tensor, z: torch.Tensor | None = None
) -> torch.Tensor:
return self.forward_cuda(x, z)
class LayerNorm(nn.Module):
"""Layer Normalization."""
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.dim = dim
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim, dtype=torch.float32))
self.bias = nn.Parameter(torch.zeros(dim, dtype=torch.float32))
def forward(self, x: torch.Tensor):
return F.layer_norm(
x.float(), (self.dim,), self.weight, self.bias, self.eps
).type_as(x)