[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)

Signed-off-by: Luka Govedič <lgovedic@redhat.com>
Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
Luka Govedič
2026-05-01 23:41:15 -04:00
committed by GitHub
parent 3e49479c4b
commit d58c42e19c
44 changed files with 2833 additions and 779 deletions
+69
View File
@@ -75,3 +75,72 @@ def _rms_norm_fake(x: Tensor, weight: Tensor, variance_epsilon: float) -> Tensor
direct_register_aiter_op(
op_name="rms_norm", op_func=_rms_norm_impl, fake_impl=_rms_norm_fake
)
rms_add_no_var_16bit_only = (
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
and x.dtype in (torch.float16, torch.bfloat16)
and (weight is None or weight.dtype == x.dtype)
)
"""
AITER fused_add_rms_norm only supports 16-bit activations and no var_size override.
Requires weight dtype to match x dtype.
"""
@ir.ops.fused_add_rms_norm.register_impl(
"aiter", supports_args=rms_add_no_var_16bit_only, supported=AITER_SUPPORTED
)
def fused_add_rms_norm(
x: Tensor,
x_residual: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> tuple[Tensor, Tensor]:
assert variance_size is None
assert x.dtype in (torch.float16, torch.bfloat16)
if weight is None:
weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype)
return torch.ops.vllm_aiter.fused_add_rms_norm(x, x_residual, weight, epsilon)
def _rocm_aiter_rmsnorm2d_fwd_with_add_impl(
x: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
variance_epsilon: float,
) -> tuple[torch.Tensor, torch.Tensor]:
from aiter import rmsnorm2d_fwd_with_add
# TODO can out = x and residual_out = residual to save memory?
# Need to check if the kernel supports in-place residual output
# (if yes set mutates_args and inplace)
residual_out = torch.empty_like(residual)
out = torch.empty_like(x)
rmsnorm2d_fwd_with_add(
out, # output
x, # input
residual, # residual input
residual_out, # residual output
weight,
variance_epsilon,
)
return out, residual_out
def _rocm_aiter_rmsnorm2d_fwd_with_add_fake(
x: torch.Tensor,
residual: torch.Tensor,
weight: torch.Tensor,
variance_epsilon: float,
) -> tuple[torch.Tensor, torch.Tensor]:
residual_out = torch.empty_like(residual)
out = torch.empty_like(x)
return out, residual_out
direct_register_aiter_op(
op_name="fused_add_rms_norm",
op_func=_rocm_aiter_rmsnorm2d_fwd_with_add_impl,
fake_impl=_rocm_aiter_rmsnorm2d_fwd_with_add_fake,
)