[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
@@ -75,3 +75,72 @@ def _rms_norm_fake(x: Tensor, weight: Tensor, variance_epsilon: float) -> Tensor
|
||||
direct_register_aiter_op(
|
||||
op_name="rms_norm", op_func=_rms_norm_impl, fake_impl=_rms_norm_fake
|
||||
)
|
||||
|
||||
rms_add_no_var_16bit_only = (
|
||||
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
|
||||
and x.dtype in (torch.float16, torch.bfloat16)
|
||||
and (weight is None or weight.dtype == x.dtype)
|
||||
)
|
||||
"""
|
||||
AITER fused_add_rms_norm only supports 16-bit activations and no var_size override.
|
||||
Requires weight dtype to match x dtype.
|
||||
"""
|
||||
|
||||
|
||||
@ir.ops.fused_add_rms_norm.register_impl(
|
||||
"aiter", supports_args=rms_add_no_var_16bit_only, supported=AITER_SUPPORTED
|
||||
)
|
||||
def fused_add_rms_norm(
|
||||
x: Tensor,
|
||||
x_residual: Tensor,
|
||||
weight: Tensor | None,
|
||||
epsilon: float,
|
||||
variance_size: int | None = None,
|
||||
) -> tuple[Tensor, Tensor]:
|
||||
assert variance_size is None
|
||||
assert x.dtype in (torch.float16, torch.bfloat16)
|
||||
if weight is None:
|
||||
weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype)
|
||||
return torch.ops.vllm_aiter.fused_add_rms_norm(x, x_residual, weight, epsilon)
|
||||
|
||||
|
||||
def _rocm_aiter_rmsnorm2d_fwd_with_add_impl(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
variance_epsilon: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from aiter import rmsnorm2d_fwd_with_add
|
||||
|
||||
# TODO can out = x and residual_out = residual to save memory?
|
||||
# Need to check if the kernel supports in-place residual output
|
||||
# (if yes set mutates_args and inplace)
|
||||
residual_out = torch.empty_like(residual)
|
||||
out = torch.empty_like(x)
|
||||
rmsnorm2d_fwd_with_add(
|
||||
out, # output
|
||||
x, # input
|
||||
residual, # residual input
|
||||
residual_out, # residual output
|
||||
weight,
|
||||
variance_epsilon,
|
||||
)
|
||||
return out, residual_out
|
||||
|
||||
|
||||
def _rocm_aiter_rmsnorm2d_fwd_with_add_fake(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
variance_epsilon: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
residual_out = torch.empty_like(residual)
|
||||
out = torch.empty_like(x)
|
||||
return out, residual_out
|
||||
|
||||
|
||||
direct_register_aiter_op(
|
||||
op_name="fused_add_rms_norm",
|
||||
op_func=_rocm_aiter_rmsnorm2d_fwd_with_add_impl,
|
||||
fake_impl=_rocm_aiter_rmsnorm2d_fwd_with_add_fake,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user