[XPU] Dispatch nn.LayerNorm to fused SYCL kernel via CustomOp (#57172)

Signed-off-by: Michal Ganczarenko <michal.ganczarenko@intel.com>
Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Michał Ganczarenko
2026-09-30 12:19:33 +00:00
committed by GitHub
co-authored by Claude Sonnet 5 Harry Mellor
parent a4c4bb418e
commit df5668e5c9
13 changed files with 237 additions and 46 deletions
+1 -1
View File
@@ -715,7 +715,7 @@ class Indexer(nn.Module):
disable_tp=True,
prefix=f"{prefix}.wk_weights_proj",
)
self.k_norm = LayerNorm(self.head_dim, eps=1e-6)
self.k_norm = LayerNorm(self.head_dim, eps=1e-6, dtype=torch.float32)
self.softmax_scale = self.head_dim**-0.5
self.scale_fmt = "ue8m0"
+4 -3
View File
@@ -35,6 +35,7 @@ from vllm.distributed.parallel_state import (
)
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -156,11 +157,11 @@ class GPT2Block(nn.Module):
hidden_size = config.hidden_size
inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.ln_1 = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.attn = GPT2Attention(
config, cache_config, quant_config, prefix=f"{prefix}.attn"
)
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.ln_2 = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
self.mlp = GPT2MLP(inner_dim, config, quant_config, prefix=f"{prefix}.mlp")
def forward(
@@ -212,7 +213,7 @@ class GPT2Model(nn.Module):
lambda prefix: GPT2Block(config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.h",
)
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.n_embd
)
+3 -2
View File
@@ -31,6 +31,7 @@ from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -163,7 +164,7 @@ class GPTJBlock(nn.Module):
):
super().__init__()
inner_dim = 4 * config.n_embd if config.n_inner is None else config.n_inner
self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
self.ln_1 = LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
self.attn = GPTJAttention(
config, cache_config, quant_config, prefix=f"{prefix}.attn"
)
@@ -206,7 +207,7 @@ class GPTJModel(nn.Module):
lambda prefix: GPTJBlock(config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.h",
)
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.n_embd
)
+42 -13
View File
@@ -33,6 +33,7 @@ from torch import nn
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
@@ -77,23 +78,22 @@ def _cast_if_autocast_enabled(device_type: str, *args):
)
class NemotronLayerNorm1P(nn.LayerNorm):
def __init__(
self,
normalized_shape: int | list[int] | torch.Size,
eps: float = 1e-5,
elementwise_affine: bool = True,
bias: bool = True,
device=None,
dtype=None,
):
super().__init__(normalized_shape, eps, elementwise_affine, bias, device, dtype)
@CustomOp.register("nemotron_layer_norm")
class NemotronLayerNorm1P(CustomOp):
"""LayerNorm variant used by Nemotron: `x * (1 + w)` instead of `x * w`."""
def forward(
def __init__(self, hidden_size: int, eps: float = 1e-5) -> None:
super().__init__()
self.normalized_shape = (hidden_size,)
self.eps = eps
self.weight = nn.Parameter(torch.zeros(hidden_size))
self.bias = nn.Parameter(torch.zeros(hidden_size))
def forward_native(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if residual is not None:
x = x + residual
residual = x
@@ -105,6 +105,35 @@ class NemotronLayerNorm1P(nn.LayerNorm):
x = torch.nn.functional.layer_norm(*args)
return x if residual is None else (x, residual)
def forward_cuda(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
return self.forward_native(x, residual)
def forward_xpu(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_nemotron_layer_norm
if not hasattr(torch.ops._C, "nemotron_layer_norm"):
return self.forward_native(x, residual)
if residual is not None:
if not hasattr(torch.ops._C, "fused_add_nemotron_layer_norm"):
return self.forward_native(x, residual)
torch.ops.vllm.xpu_fused_add_nemotron_layer_norm(
x, residual, self.weight, self.bias, self.eps
)
return x, residual
# empty_like preserves x's strides, but the kernel requires a
# contiguous out (unlike x, which it can handle non-contiguous).
out = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops.vllm.xpu_nemotron_layer_norm(out, x, self.weight, self.bias, self.eps)
return out
class NemotronMLP(nn.Module):
def __init__(
+5 -3
View File
@@ -32,6 +32,7 @@ from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -143,7 +144,7 @@ class OPTDecoderLayer(nn.Module):
)
self.do_layer_norm_before = config.do_layer_norm_before
self.self_attn_layer_norm = nn.LayerNorm(
self.self_attn_layer_norm = LayerNorm(
self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine
)
self.fc1 = ColumnParallelLinear(
@@ -161,7 +162,7 @@ class OPTDecoderLayer(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.fc2",
)
self.final_layer_norm = nn.LayerNorm(
self.final_layer_norm = LayerNorm(
self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine
)
@@ -246,8 +247,9 @@ class OPTDecoder(nn.Module):
# keep backward compatibility with checkpoints that have been fine-tuned
# before transformers v4.20.1
# see https://github.com/facebookresearch/metaseq/pull/164
self.final_layer_norm: LayerNorm | None
if config.do_layer_norm_before and not config._remove_final_layer_norm:
self.final_layer_norm = nn.LayerNorm(
self.final_layer_norm = LayerNorm(
config.hidden_size,
elementwise_affine=config.layer_norm_elementwise_affine,
)
+7 -5
View File
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
@@ -467,13 +468,13 @@ class SiglipEncoderLayer(nn.Module):
prefix=f"{prefix}.self_attn",
attn_cls=attn_cls,
)
self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.layer_norm1 = LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.mlp = SiglipMLP(
config,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
self.layer_norm2 = LayerNorm(self.embed_dim, eps=config.layer_norm_eps)
def forward(
self,
@@ -573,7 +574,7 @@ class SiglipTextTransformer(nn.Module):
attn_cls=EncoderOnlyAttention,
)
self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.final_layer_norm = LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.head = nn.Linear(embed_dim, config.projection_size)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
@@ -616,7 +617,7 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
self.attention = torch.nn.MultiheadAttention(
config.hidden_size, config.num_attention_heads, batch_first=True
)
self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
self.layernorm = LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
self.mlp = SiglipMLP(
config=config,
quant_config=quant_config,
@@ -685,8 +686,9 @@ class SiglipVisionTransformer(nn.Module):
if require_post_norm is None:
require_post_norm = len(self.encoder.layers) == num_hidden_layers
self.post_layernorm: LayerNorm | None
if require_post_norm:
self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps)
self.post_layernorm = LayerNorm(embed_dim, eps=config.layer_norm_eps)
else:
self.post_layernorm = None
@@ -71,6 +71,7 @@ from vllm.model_executor.models.transformers.utils import (
named_state,
replace_conv_class,
replace_embedding_class,
replace_layernorm_class,
replace_linear_class,
)
from vllm.model_executor.models.utils import (
@@ -466,6 +467,7 @@ class Base(
- `nn.Conv2d` / `nn.Conv3d` with vLLM's `Conv2d` / `Conv3d`
- Vocab `nn.Embedding`s with vLLM's `VocabParallelEmbedding`
- RMSNorm (detected from their dataflow) with vLLM's `RMSNorm`or `GemmaRMSNorm`
- `nn.LayerNorm` with vLLM's `LayerNorm`
"""
tp_plan = self.model.tp_plan or {}
@@ -556,6 +558,8 @@ class Base(
new_module = replace_embedding_class(
child_module, self.quant_config, prefix=qual_name
)
elif isinstance(child_module, nn.LayerNorm):
new_module = replace_layernorm_class(child_module)
elif child_module_fusers := fusers[child_module]:
for fuser in child_module_fusers:
register_fusion(fuser, qual_name, child_module)
@@ -28,6 +28,7 @@ from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
@@ -187,6 +188,34 @@ def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
)
def replace_layernorm_class(layernorm: nn.LayerNorm) -> nn.Module:
"""Replace a standard (mean-centered) `nn.LayerNorm` with vLLM's
`LayerNorm`.
Args:
layernorm: `nn.LayerNorm` to be replaced.
Returns:
The new `LayerNorm`. If the layernorm is not supported (a subclass with
its own behavior, multi-dim `normalized_shape`, or not both
elementwise-affine and biased), returns the original module unchanged.
"""
if (
type(layernorm) is not nn.LayerNorm
or len(layernorm.normalized_shape) != 1
or not layernorm.elementwise_affine
or layernorm.bias is None
):
return layernorm
return LayerNorm(
layernorm.normalized_shape[0],
eps=layernorm.eps,
dtype=layernorm.weight.dtype,
)
def attrsetter(attr: str) -> Callable[[object, object], None]:
"""Set a possibly nested attribute, like the inverse of attrgetter."""
parent, _, name = attr.rpartition(".")