[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)

Signed-off-by: Luka Govedič <lgovedic@redhat.com>
Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
Luka Govedič
2026-05-01 23:41:15 -04:00
committed by GitHub
parent 3e49479c4b
commit d58c42e19c
44 changed files with 2833 additions and 779 deletions
+69
View File
@@ -75,3 +75,72 @@ def _rms_norm_fake(x: Tensor, weight: Tensor, variance_epsilon: float) -> Tensor
direct_register_aiter_op(
op_name="rms_norm", op_func=_rms_norm_impl, fake_impl=_rms_norm_fake
)
rms_add_no_var_16bit_only = (
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
and x.dtype in (torch.float16, torch.bfloat16)
and (weight is None or weight.dtype == x.dtype)
)
"""
AITER fused_add_rms_norm only supports 16-bit activations and no var_size override.
Requires weight dtype to match x dtype.
"""
@ir.ops.fused_add_rms_norm.register_impl(
"aiter", supports_args=rms_add_no_var_16bit_only, supported=AITER_SUPPORTED
)
def fused_add_rms_norm(
x: Tensor,
x_residual: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> tuple[Tensor, Tensor]:
assert variance_size is None
assert x.dtype in (torch.float16, torch.bfloat16)
if weight is None:
weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype)
return torch.ops.vllm_aiter.fused_add_rms_norm(x, x_residual, weight, epsilon)
def _rocm_aiter_rmsnorm2d_fwd_with_add_impl(
x: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
variance_epsilon: float,
) -> tuple[torch.Tensor, torch.Tensor]:
from aiter import rmsnorm2d_fwd_with_add
# TODO can out = x and residual_out = residual to save memory?
# Need to check if the kernel supports in-place residual output
# (if yes set mutates_args and inplace)
residual_out = torch.empty_like(residual)
out = torch.empty_like(x)
rmsnorm2d_fwd_with_add(
out, # output
x, # input
residual, # residual input
residual_out, # residual output
weight,
variance_epsilon,
)
return out, residual_out
def _rocm_aiter_rmsnorm2d_fwd_with_add_fake(
x: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
variance_epsilon: float,
) -> tuple[torch.Tensor, torch.Tensor]:
residual_out = torch.empty_like(residual)
out = torch.empty_like(x)
return out, residual_out
direct_register_aiter_op(
op_name="fused_add_rms_norm",
op_func=_rocm_aiter_rmsnorm2d_fwd_with_add_impl,
fake_impl=_rocm_aiter_rmsnorm2d_fwd_with_add_fake,
)
+55 -5
View File
@@ -1,6 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""This file registers Oink implementations for vLLM IR ops.
vLLM does not depend on the external Oink repository/package. When an external
plugin registers torch.library.custom_op entrypoints under the `oink::`
namespace (e.g. via vLLM's general_plugins mechanism), these ops will be marked
as supported. To dispatch to those ops, set kernel_config.ir_op_priority.<op> to oink.
Alternatively, `VLLM_USE_OINK_OPS=1` will add this to priority by default.
"""
import torch
from torch import Tensor
from vllm import ir
from vllm.platforms import current_platform
@@ -15,7 +25,7 @@ def has_oink_op(name: str) -> bool:
return OINK_AVAILABLE and hasattr(torch.ops.oink, name)
def _can_view_as_2d(x: torch.Tensor) -> bool:
def _can_view_as_2d(x: Tensor) -> bool:
"""Return True if x.view(-1, x.shape[-1]) is viewable (no copy)."""
if x.dim() < 2:
return False
@@ -32,7 +42,7 @@ def _can_view_as_2d(x: torch.Tensor) -> bool:
return True
def _is_oink_stride_compatible_2d(x_2d: torch.Tensor) -> bool:
def _is_oink_stride_compatible_2d(x_2d: Tensor) -> bool:
"""Return True if x_2d meets Oink's pointer-path stride constraints."""
if x_2d.dim() != 2:
return False
@@ -67,11 +77,51 @@ and no variance_size override.
"oink", supports_args=oink_rms_supported, supported=has_oink_op("rmsnorm")
)
def rms_norm(
x: torch.Tensor,
weight: torch.Tensor | None,
x: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> torch.Tensor:
) -> Tensor:
assert variance_size is None
x_2d = x.view(-1, x.shape[-1])
return torch.ops.oink.rmsnorm(x_2d, weight, epsilon).view_as(x)
oink_add_rms_supported = (
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
and weight is not None
and x.dim() >= 2
and x.dtype == weight.dtype
and weight.is_contiguous()
and _can_view_as_2d(x)
and _is_oink_stride_compatible_2d(x.view(-1, x.shape[-1]))
# residual must have 2d-compatible strides and match x shape/dtype
and x.dtype == x_residual.dtype
and x.shape == x_residual.shape
and _can_view_as_2d(x_residual)
and _is_oink_stride_compatible_2d(x_residual.view(-1, x_residual.shape[-1]))
)
"""
Oink fused_add_rms_norm has the same constraints as rms_norm,
and residual must be 2d-like with compatible strides.
"""
@ir.ops.fused_add_rms_norm.register_impl(
"oink",
supports_args=oink_add_rms_supported,
supported=has_oink_op("fused_add_rms_norm"),
inplace=True,
)
def fused_add_rms_norm(
x: Tensor,
x_residual: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> tuple[Tensor, Tensor]:
assert variance_size is None
x_2d = x.view(-1, x.shape[-1])
residual_2d = x_residual.view(-1, x_residual.shape[-1])
torch.ops.oink.fused_add_rms_norm(x_2d, residual_2d, weight, epsilon)
return x, x_residual
+30
View File
@@ -31,3 +31,33 @@ def rms_norm(
output = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops._C.rms_norm(output, x, weight, epsilon)
return output
rms_add_no_var_size = (
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
and (weight is None or weight.dtype == x.dtype)
)
"""vLLM Kernel does not support variance_size parameter and requires
matching input/weight dtype."""
@ir.ops.fused_add_rms_norm.register_impl(
"vllm_c",
supports_args=rms_add_no_var_size,
supported=CUDA_ALIKE,
inplace=True,
)
def fused_add_rms_norm(
x: Tensor,
x_residual: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> tuple[Tensor, Tensor]:
if weight is None:
# Kernel requires weight tensor, pass ones
weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype)
assert variance_size is None
torch.ops._C.fused_add_rms_norm(x, x_residual, weight, epsilon)
return x, x_residual
+28
View File
@@ -36,3 +36,31 @@ def rms_norm(
output = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops._C.rms_norm(output, x, weight, epsilon)
return output
rms_add_no_var_size = (
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
and (weight is None or weight.dtype == x.dtype)
)
@ir.ops.fused_add_rms_norm.register_impl(
"xpu_kernels",
supports_args=rms_add_no_var_size,
supported=XPU_KERNELS_SUPPORTED,
inplace=True,
)
def fused_add_rms_norm(
x: Tensor,
x_residual: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> tuple[Tensor, Tensor]:
if weight is None:
# Kernel requires weight tensor, pass ones
weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype)
assert variance_size is None
torch.ops._C.fused_add_rms_norm(x, x_residual, weight, epsilon)
return x, x_residual