[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
@@ -23,11 +23,7 @@ from vllm.model_executor.layers.fused_moe.router.fused_topk_router import (
|
||||
vllm_topk_sigmoid,
|
||||
vllm_topk_softmax,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import (
|
||||
RMSNorm,
|
||||
dispatch_rocm_rmsnorm_func,
|
||||
fused_add_rms_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
RMS_NORM_SUPPORTED_DTYPES = [torch.float16, torch.bfloat16]
|
||||
@@ -153,26 +149,3 @@ def test_topk_sigmoid_dispatch(use_rocm_aiter: bool):
|
||||
assert topk_func == rocm_aiter_ops.topk_sigmoid
|
||||
else:
|
||||
assert topk_func == vllm_topk_sigmoid
|
||||
|
||||
|
||||
@pytest.mark.parametrize("add_residual", [False])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("use_rocm_aiter", [True, False])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="AITER is a feature exclusive for ROCm"
|
||||
)
|
||||
def test_rms_norm_dispatch(
|
||||
add_residual: bool, dtype: torch.dtype, use_rocm_aiter: bool
|
||||
):
|
||||
rms_norm_func = dispatch_rocm_rmsnorm_func(dtype, use_rocm_aiter)
|
||||
|
||||
should_use_rocm_aiter = (
|
||||
current_platform.is_rocm()
|
||||
and use_rocm_aiter
|
||||
and dtype in RMS_NORM_SUPPORTED_DTYPES
|
||||
)
|
||||
|
||||
if should_use_rocm_aiter:
|
||||
assert rms_norm_func == rocm_aiter_ops.rms_norm2d_with_add
|
||||
else:
|
||||
assert rms_norm_func == fused_add_rms_norm
|
||||
|
||||
@@ -1,60 +1,97 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import multiprocessing
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _load_oink_ops_module():
|
||||
# Import the module normally (vllm is installed as an editable package in CI).
|
||||
from vllm import _oink_ops
|
||||
def _test_oink_availability_impl(
|
||||
device_capability: tuple[int, int],
|
||||
has_rmsnorm: bool,
|
||||
has_fused_add_rms_norm: bool,
|
||||
expected_available: bool,
|
||||
expected_fused: bool,
|
||||
) -> None:
|
||||
"""Test OINK support detection with mocked state."""
|
||||
import torch
|
||||
|
||||
return _oink_ops
|
||||
from vllm import platforms
|
||||
|
||||
# Mock device capability (class method, override on class)
|
||||
dc = platforms.interface.DeviceCapability(*device_capability)
|
||||
platforms.current_platform.__class__.get_device_capability = lambda device_id=0: dc
|
||||
|
||||
# Mock oink ops
|
||||
oink_ops = types.SimpleNamespace()
|
||||
if has_rmsnorm:
|
||||
oink_ops.rmsnorm = lambda x, w, eps: x
|
||||
if has_fused_add_rms_norm:
|
||||
oink_ops.fused_add_rms_norm = lambda x, residual, w, eps: None
|
||||
|
||||
torch.ops.oink = oink_ops
|
||||
|
||||
# Now import vllm modules with mocks in place (fresh import with mocked platform)
|
||||
import vllm.kernels.oink_ops # noqa: F401
|
||||
from vllm.ir.ops import fused_add_rms_norm, rms_norm
|
||||
|
||||
# Verify support checks
|
||||
assert rms_norm.impls["oink"].supported is expected_available
|
||||
assert fused_add_rms_norm.impls["oink"].supported is expected_fused
|
||||
|
||||
|
||||
def test_oink_availability_checks(monkeypatch: pytest.MonkeyPatch):
|
||||
_oink_ops = _load_oink_ops_module()
|
||||
@pytest.mark.parametrize(
|
||||
"device_capability,has_rmsnorm,has_fused_add_rms_norm,expected_available,expected_fused",
|
||||
[
|
||||
# Case 1: < SM100, ops not supported
|
||||
((9, 0), True, False, False, False),
|
||||
# Case 2: CUDA available and SM100, rmsnorm op registered
|
||||
((10, 0), True, False, True, False),
|
||||
# Case 3: SM100 with both rmsnorm and fused_add_rms_norm
|
||||
((10, 0), True, True, True, True),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test on CUDA")
|
||||
def test_oink_availability_checks(
|
||||
device_capability: tuple[int, int],
|
||||
has_rmsnorm: bool,
|
||||
has_fused_add_rms_norm: bool,
|
||||
expected_available: bool,
|
||||
expected_fused: bool,
|
||||
):
|
||||
"""Test OINK support detection with clean import state for each parameter set."""
|
||||
|
||||
# Ensure the ops namespace exists and is mutable for tests.
|
||||
monkeypatch.setattr(
|
||||
torch.ops,
|
||||
"oink",
|
||||
types.SimpleNamespace(rmsnorm=lambda x, w, eps: x),
|
||||
raising=False,
|
||||
)
|
||||
|
||||
# Case 1: CUDA not available.
|
||||
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
|
||||
assert _oink_ops.is_oink_available_for_device(0) is False
|
||||
|
||||
# Case 2: CUDA available but < SM100.
|
||||
monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
|
||||
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda idx: (9, 0))
|
||||
assert _oink_ops.is_oink_available_for_device(0) is False
|
||||
|
||||
# Case 3: CUDA available and SM100, rmsnorm op registered.
|
||||
monkeypatch.setattr(torch.cuda, "get_device_capability", lambda idx: (10, 0))
|
||||
assert _oink_ops.is_oink_available_for_device(0) is True
|
||||
|
||||
# fused op presence probe
|
||||
assert _oink_ops.has_fused_add_rms_norm() is False
|
||||
monkeypatch.setattr(
|
||||
torch.ops,
|
||||
"oink",
|
||||
types.SimpleNamespace(
|
||||
rmsnorm=lambda x, w, eps: x,
|
||||
fused_add_rms_norm=lambda x, residual, w, eps: None,
|
||||
# Use spawn to run function in fresh process with clean imports
|
||||
# TODO migrate to spawn utility:
|
||||
# https://github.com/vllm-project/vllm/issues/41415
|
||||
ctx = multiprocessing.get_context("spawn")
|
||||
process = ctx.Process(
|
||||
target=_test_oink_availability_impl,
|
||||
args=(
|
||||
device_capability,
|
||||
has_rmsnorm,
|
||||
has_fused_add_rms_norm,
|
||||
expected_available,
|
||||
expected_fused,
|
||||
),
|
||||
raising=False,
|
||||
)
|
||||
assert _oink_ops.has_fused_add_rms_norm() is True
|
||||
process.start()
|
||||
process.join()
|
||||
|
||||
if process.exitcode != 0:
|
||||
raise AssertionError(
|
||||
f"Subprocess test failed with exit code {process.exitcode}"
|
||||
)
|
||||
|
||||
|
||||
def test_can_view_as_2d_stride_guard():
|
||||
# Import the helper from the layernorm module.
|
||||
from vllm.model_executor.layers.layernorm import _can_view_as_2d
|
||||
# No global import
|
||||
import torch
|
||||
|
||||
# Import the helper from the kernels module.
|
||||
from vllm.kernels.oink_ops import _can_view_as_2d
|
||||
|
||||
x = torch.zeros((2, 3, 4))
|
||||
assert _can_view_as_2d(x) is True
|
||||
|
||||
Reference in New Issue
Block a user