[vLLM IR] 1/N Implement IR skeleton and rms_norm op (#33825)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Xinyu Chen <xinyu1.chen@intel.com> Signed-off-by: chzhang <chaojun.zhang@intel.com> Signed-off-by: Luka Govedic <luka.govedic@gmail.com> Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com> Co-authored-by: Chaojun Zhang <chaojun.zhang@intel.com> Co-authored-by: Luka Govedič <ProExpertProg@h100-01.nemg-001.lab.rdu2.dc.redhat.com>
This commit is contained in:
co-authored by
Xinyu Chen
Chaojun Zhang
Luka Govedič
parent
0fab52f0aa
commit
40bb175027
@@ -6,7 +6,9 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import _oink_ops, envs
|
||||
# Import kernels
|
||||
import vllm.kernels # noqa: F401
|
||||
from vllm import _oink_ops, envs, ir
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
@@ -51,23 +53,6 @@ def _is_oink_stride_compatible_2d(x_2d: torch.Tensor) -> bool:
|
||||
return (x_2d.stride(0) % divby) == 0
|
||||
|
||||
|
||||
def rms_norm(
|
||||
x: torch.Tensor, weight: torch.Tensor, variance_epsilon: float
|
||||
) -> torch.Tensor:
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
return rms_norm_batch_invariant(x, weight, variance_epsilon)
|
||||
out = torch.empty_like(x)
|
||||
ops.rms_norm(
|
||||
out,
|
||||
x,
|
||||
weight,
|
||||
variance_epsilon,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def fused_add_rms_norm(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -105,23 +90,16 @@ def poly_norm(
|
||||
return out
|
||||
|
||||
|
||||
def dispatch_rocm_rmsnorm_func(
|
||||
with_fused_add: bool, dtype: torch.dtype, use_aiter: bool = False
|
||||
):
|
||||
def dispatch_rocm_rmsnorm_func(dtype: torch.dtype, use_aiter: bool = False):
|
||||
use_aiter = use_aiter and dtype in [
|
||||
torch.float16,
|
||||
torch.bfloat16,
|
||||
]
|
||||
|
||||
if use_aiter and with_fused_add:
|
||||
return rocm_aiter_ops.rms_norm2d_with_add
|
||||
if use_aiter:
|
||||
return rocm_aiter_ops.rms_norm
|
||||
|
||||
# fall back to CUDA implementation
|
||||
if with_fused_add:
|
||||
return rocm_aiter_ops.rms_norm2d_with_add
|
||||
else:
|
||||
return fused_add_rms_norm
|
||||
return rms_norm
|
||||
|
||||
|
||||
# --8<-- [start:rms_norm]
|
||||
@@ -158,20 +136,14 @@ class RMSNorm(CustomOp):
|
||||
|
||||
if current_platform.is_rocm():
|
||||
aiter_rmsnorm_enabled = rocm_aiter_ops.is_rmsnorm_enabled()
|
||||
self.rocm_norm_func = dispatch_rocm_rmsnorm_func(
|
||||
with_fused_add=False,
|
||||
dtype=weight_dtype,
|
||||
use_aiter=aiter_rmsnorm_enabled,
|
||||
)
|
||||
self.rocm_norm_func_with_add = dispatch_rocm_rmsnorm_func(
|
||||
with_fused_add=True, dtype=weight_dtype, use_aiter=aiter_rmsnorm_enabled
|
||||
dtype=weight_dtype, use_aiter=aiter_rmsnorm_enabled
|
||||
)
|
||||
|
||||
# Optional: enable Oink Blackwell RMSNorm custom-op fast path on
|
||||
# compatible CUDA devices (e.g., SM100) when the external Oink
|
||||
# package is available. This is detected once at construction time
|
||||
# to avoid per-call device queries in the hot path.
|
||||
self._use_oink_rmsnorm = False
|
||||
self._use_oink_fused_add_rmsnorm = False
|
||||
if (
|
||||
not current_platform.is_rocm()
|
||||
@@ -203,7 +175,6 @@ class RMSNorm(CustomOp):
|
||||
try:
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
if _oink_ops.is_oink_available_for_device(device_index):
|
||||
self._use_oink_rmsnorm = True
|
||||
self._use_oink_fused_add_rmsnorm = (
|
||||
_oink_ops.has_fused_add_rms_norm()
|
||||
)
|
||||
@@ -215,7 +186,6 @@ class RMSNorm(CustomOp):
|
||||
"RMSNorm; falling back to vLLM RMSNorm. Error: %s",
|
||||
e,
|
||||
)
|
||||
self._use_oink_rmsnorm = False
|
||||
self._use_oink_fused_add_rmsnorm = False
|
||||
|
||||
@staticmethod
|
||||
@@ -270,6 +240,10 @@ class RMSNorm(CustomOp):
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""PyTorch-native implementation equivalent to forward()."""
|
||||
if residual is None:
|
||||
return ir.ops.rms_norm(
|
||||
x, self.weight.data, self.variance_epsilon, self.variance_size_override
|
||||
)
|
||||
|
||||
return self.forward_static(
|
||||
x,
|
||||
@@ -286,35 +260,14 @@ class RMSNorm(CustomOp):
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if residual is None and not envs.VLLM_BATCH_INVARIANT:
|
||||
return ir.ops.rms_norm(
|
||||
x, self.weight.data, self.variance_epsilon, self.variance_size_override
|
||||
)
|
||||
|
||||
if self.variance_size_override is not None:
|
||||
return self.forward_native(x, residual)
|
||||
|
||||
# Optional Oink SM100 fast path (no residual). This path is
|
||||
# torch.compile-friendly via torch.ops.oink.rmsnorm and preserves
|
||||
# 2D layouts (including padded rows) when using the Oink
|
||||
# pointer-based kernel.
|
||||
if (
|
||||
residual is None
|
||||
and getattr(self, "_use_oink_rmsnorm", False)
|
||||
and x.is_cuda
|
||||
and x.dim() >= 2
|
||||
and self.has_weight
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
and self.weight.data.dtype == x.dtype
|
||||
and self.weight.data.is_contiguous()
|
||||
):
|
||||
orig_shape = x.shape
|
||||
hidden_size = orig_shape[-1]
|
||||
if _can_view_as_2d(x):
|
||||
x_2d = x.view(-1, hidden_size)
|
||||
if _is_oink_stride_compatible_2d(x_2d):
|
||||
y_2d = _oink_ops.rmsnorm(
|
||||
x_2d,
|
||||
self.weight.data,
|
||||
self.variance_epsilon,
|
||||
)
|
||||
return y_2d.view(orig_shape)
|
||||
|
||||
# Optional Oink SM100 fast path (fused residual-add + RMSNorm, in-place).
|
||||
# This mirrors vLLM's fused_add_rms_norm semantics by mutating both
|
||||
# `x` (normalized output) and `residual` (residual-out buffer).
|
||||
@@ -356,29 +309,34 @@ class RMSNorm(CustomOp):
|
||||
)
|
||||
return x, residual
|
||||
|
||||
add_residual = residual is not None
|
||||
if add_residual:
|
||||
if residual is not None:
|
||||
return fused_add_rms_norm(
|
||||
x, residual, self.weight.data, self.variance_epsilon
|
||||
)
|
||||
else:
|
||||
return rms_norm(x, self.weight.data, self.variance_epsilon)
|
||||
assert envs.VLLM_BATCH_INVARIANT
|
||||
return rms_norm_batch_invariant(x, self.weight.data, self.variance_epsilon)
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if residual is None and not envs.VLLM_BATCH_INVARIANT:
|
||||
return ir.ops.rms_norm(
|
||||
x, self.weight.data, self.variance_epsilon, self.variance_size_override
|
||||
)
|
||||
|
||||
if self.variance_size_override is not None:
|
||||
return self.forward_native(x, residual)
|
||||
|
||||
add_residual = residual is not None
|
||||
if add_residual:
|
||||
if residual is not None:
|
||||
return self.rocm_norm_func_with_add(
|
||||
x, residual, self.weight.data, self.variance_epsilon
|
||||
)
|
||||
else:
|
||||
return self.rocm_norm_func(x, self.weight.data, self.variance_epsilon)
|
||||
assert envs.VLLM_BATCH_INVARIANT
|
||||
return rms_norm_batch_invariant(x, self.weight.data, self.variance_epsilon)
|
||||
|
||||
def forward_xpu(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user