[vLLM IR] 1/N Implement IR skeleton and rms_norm op (#33825)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Xinyu Chen <xinyu1.chen@intel.com> Signed-off-by: chzhang <chaojun.zhang@intel.com> Signed-off-by: Luka Govedic <luka.govedic@gmail.com> Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com> Co-authored-by: Chaojun Zhang <chaojun.zhang@intel.com> Co-authored-by: Luka Govedič <ProExpertProg@h100-01.nemg-001.lab.rdu2.dc.redhat.com>
This commit is contained in:
co-authored by
Xinyu Chen
Chaojun Zhang
Luka Govedič
parent
0fab52f0aa
commit
40bb175027
@@ -1,3 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kernel implementations for vLLM."""
|
||||
|
||||
from . import aiter_ops, oink_ops, vllm_c, xpu_ops
|
||||
|
||||
__all__ = ["vllm_c", "aiter_ops", "oink_ops", "xpu_ops"]
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import functools
|
||||
|
||||
import torch
|
||||
from torch import Tensor
|
||||
from torch.library import Library
|
||||
|
||||
from vllm import ir
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
current_platform.import_kernels()
|
||||
|
||||
|
||||
def is_aiter_found() -> bool:
|
||||
from importlib.util import find_spec
|
||||
|
||||
return find_spec("aiter") is not None
|
||||
|
||||
|
||||
aiter_lib = Library("vllm_aiter", "FRAGMENT")
|
||||
"""
|
||||
This library holds torch custom ops for wrapped AITER ops.
|
||||
Many AITER ops want to remain invisible to torch.compile even after lowering.
|
||||
They are thus wrapped into torch custom ops inside the IR op implementations.
|
||||
"""
|
||||
|
||||
direct_register_aiter_op = functools.partial(
|
||||
direct_register_custom_op, target_lib=aiter_lib
|
||||
)
|
||||
"""Syntactic sugar for registering AITER custom ops."""
|
||||
|
||||
AITER_SUPPORTED = is_aiter_found()
|
||||
"""Most kernels in this file are supported if AITER is installed."""
|
||||
|
||||
rms_no_var_16bit_only = (
|
||||
lambda x, weight, epsilon, variance_size=None: variance_size is None
|
||||
and x.dtype
|
||||
in (
|
||||
torch.float16,
|
||||
torch.bfloat16,
|
||||
)
|
||||
)
|
||||
"""AITER rms_norm only supports float16 and bfloat16 acts and no var_size override."""
|
||||
|
||||
|
||||
@ir.ops.rms_norm.register_impl(
|
||||
"aiter", supports_args=rms_no_var_16bit_only, supported=AITER_SUPPORTED
|
||||
)
|
||||
def rms_norm(
|
||||
x: Tensor, weight: Tensor | None, epsilon: float, variance_size: int | None = None
|
||||
) -> Tensor:
|
||||
assert variance_size is None
|
||||
assert x.dtype in (torch.float16, torch.bfloat16)
|
||||
if weight is None:
|
||||
weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype)
|
||||
return torch.ops.vllm_aiter.rms_norm(x, weight, epsilon)
|
||||
|
||||
|
||||
def _rms_norm_impl(x: Tensor, weight: Tensor, variance_epsilon: float) -> Tensor:
|
||||
from aiter import rms_norm
|
||||
|
||||
if x.dim() > 2:
|
||||
x_original_shape = x.shape
|
||||
x = x.reshape(-1, x_original_shape[-1])
|
||||
x = rms_norm(x, weight, variance_epsilon)
|
||||
return x.reshape(x_original_shape)
|
||||
|
||||
return rms_norm(x, weight, variance_epsilon)
|
||||
|
||||
|
||||
def _rms_norm_fake(x: Tensor, weight: Tensor, variance_epsilon: float) -> Tensor:
|
||||
return torch.empty_like(x)
|
||||
|
||||
|
||||
direct_register_aiter_op(
|
||||
op_name="rms_norm", op_func=_rms_norm_impl, fake_impl=_rms_norm_fake
|
||||
)
|
||||
@@ -0,0 +1,77 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm import ir
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
OINK_AVAILABLE = current_platform.has_device_capability(100) and hasattr(
|
||||
torch.ops, "oink"
|
||||
)
|
||||
|
||||
|
||||
def has_oink_op(name: str) -> bool:
|
||||
"""Check if a specific oink op is registered."""
|
||||
return OINK_AVAILABLE and hasattr(torch.ops.oink, name)
|
||||
|
||||
|
||||
def _can_view_as_2d(x: torch.Tensor) -> bool:
|
||||
"""Return True if x.view(-1, x.shape[-1]) is viewable (no copy)."""
|
||||
if x.dim() < 2:
|
||||
return False
|
||||
if x.dim() == 2:
|
||||
return True
|
||||
# For a view(-1, N) to be valid, all leading dims must be contiguous with
|
||||
# respect to each other (size-1 dims are ignored).
|
||||
for dim in range(x.dim() - 1):
|
||||
# Strides for size-1 dims are irrelevant and can be arbitrary.
|
||||
if x.size(dim + 1) != 1 and x.stride(dim) != x.stride(dim + 1) * x.size(
|
||||
dim + 1
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _is_oink_stride_compatible_2d(x_2d: torch.Tensor) -> bool:
|
||||
"""Return True if x_2d meets Oink's pointer-path stride constraints."""
|
||||
if x_2d.dim() != 2:
|
||||
return False
|
||||
if x_2d.stride(1) != 1:
|
||||
return False
|
||||
# Match Oink's vectorization constraint: stride(0) divisible by 256b.
|
||||
if x_2d.dtype in (torch.float16, torch.bfloat16):
|
||||
divby = 16
|
||||
elif x_2d.dtype == torch.float32:
|
||||
divby = 8
|
||||
else:
|
||||
return False
|
||||
return (x_2d.stride(0) % divby) == 0
|
||||
|
||||
|
||||
oink_rms_supported = (
|
||||
lambda x, weight, epsilon, variance_size=None: variance_size is None
|
||||
and weight is not None
|
||||
and x.dim() >= 2
|
||||
and x.dtype == weight.dtype
|
||||
and weight.is_contiguous()
|
||||
and _can_view_as_2d(x)
|
||||
and _is_oink_stride_compatible_2d(x.view(-1, x.shape[-1]))
|
||||
)
|
||||
"""
|
||||
Oink rms only supports 2d-like inputs with contiguous weight
|
||||
and no variance_size override.
|
||||
"""
|
||||
|
||||
|
||||
@ir.ops.rms_norm.register_impl(
|
||||
"oink", supports_args=oink_rms_supported, supported=has_oink_op("rmsnorm")
|
||||
)
|
||||
def rms_norm(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor | None,
|
||||
epsilon: float,
|
||||
variance_size: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert variance_size is None
|
||||
x_2d = x.view(-1, x.shape[-1])
|
||||
return torch.ops.oink.rmsnorm(x_2d, weight, epsilon).view_as(x)
|
||||
@@ -0,0 +1,30 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from torch import Tensor
|
||||
|
||||
from vllm import ir
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
current_platform.import_kernels()
|
||||
|
||||
CUDA_ALIKE = current_platform.is_cuda_alike()
|
||||
"""Most kernels in this file are supported on all CUDA-alike platforms."""
|
||||
|
||||
rms_no_var_size = lambda x, weight, epsilon, variance_size=None: variance_size is None
|
||||
"""vLLM kernel does not support variance_size parameter."""
|
||||
|
||||
|
||||
@ir.ops.rms_norm.register_impl(
|
||||
"vllm_c", supports_args=rms_no_var_size, supported=CUDA_ALIKE
|
||||
)
|
||||
def rms_norm(
|
||||
x: Tensor, weight: Tensor | None, epsilon: float, variance_size: int | None = None
|
||||
) -> Tensor:
|
||||
if weight is None:
|
||||
# Kernel requires weight tensor, pass ones
|
||||
weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype)
|
||||
assert variance_size is None
|
||||
output = torch.empty(x.shape, device=x.device, dtype=x.dtype)
|
||||
torch.ops._C.rms_norm(output, x, weight, epsilon)
|
||||
return output
|
||||
@@ -0,0 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from torch import Tensor
|
||||
|
||||
from vllm import ir
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
current_platform.import_kernels()
|
||||
|
||||
|
||||
def is_xpu_kernels_found() -> bool:
|
||||
from importlib.util import find_spec
|
||||
|
||||
return find_spec("vllm_xpu_kernels") is not None
|
||||
|
||||
|
||||
XPU_KERNELS_SUPPORTED = is_xpu_kernels_found()
|
||||
"""Kernels in this file are supported if vLLM XPU kernels are installed."""
|
||||
|
||||
rms_no_var = lambda x, weight, epsilon, variance_size=None: variance_size is None
|
||||
|
||||
|
||||
@ir.ops.rms_norm.register_impl(
|
||||
"xpu_kernels", supports_args=rms_no_var, supported=XPU_KERNELS_SUPPORTED
|
||||
)
|
||||
def rms_norm(
|
||||
x: Tensor, weight: Tensor | None, epsilon: float, variance_size: int | None = None
|
||||
) -> Tensor:
|
||||
if weight is None:
|
||||
# Kernel requires weight tensor, pass ones
|
||||
weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype)
|
||||
assert variance_size is None
|
||||
output = torch.empty(x.shape, device=x.device, dtype=x.dtype)
|
||||
torch.ops._C.rms_norm(output, x, weight, epsilon)
|
||||
return output
|
||||
Reference in New Issue
Block a user