[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
-183
View File
@@ -1,183 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.kernels # noqa: F401
from tests.ir.ir_test_utils import assert_close, clone_args
from vllm import ir
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
gelu_and_mul_sparse_native = ir.ops.gelu_and_mul_sparse.impls["native"].impl_fn
def test_gelu_and_mul_sparse_registration():
expected = {
"native": True,
"triton": HAS_TRITON and current_platform.is_cuda(),
}
actual = {
provider: impl.supported
for provider, impl in ir.ops.gelu_and_mul_sparse.impls.items()
}
assert actual == expected
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("num_tokens", [1, 8, 64])
@pytest.mark.parametrize("intermediate_size", [2048, 4096, 8192, 13824, 16384, 32768])
def test_gelu_and_mul_sparse_triton(
dtype: torch.dtype, num_tokens: int, intermediate_size: int
):
torch.manual_seed(0)
torch.set_default_device("cuda")
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
num_tokens=num_tokens,
hidden_size=intermediate_size,
dtype=dtype,
)
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
assert impl.supports_args(*args)
expected = gelu_and_mul_sparse_native(*clone_args(args))
actual = impl.impl_fn(*clone_args(args))
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
mask_mismatch = torch.count_nonzero((actual == 0) != (expected == 0))
assert mask_mismatch / actual.numel() < 1e-3
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_gelu_and_mul_sparse_triton_second_seed():
torch.manual_seed(1)
torch.set_default_device("cuda")
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
num_tokens=32,
hidden_size=8192,
dtype=torch.bfloat16,
)
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*clone_args(args))
expected = gelu_and_mul_sparse_native(*clone_args(args))
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("intermediate_size", [1, 31, 128])
def test_gelu_and_mul_sparse_triton_small_sizes(
dtype: torch.dtype, intermediate_size: int
):
torch.manual_seed(0)
torch.set_default_device("cuda")
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
num_tokens=5,
hidden_size=intermediate_size,
dtype=dtype,
)
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*clone_args(args))
expected = gelu_and_mul_sparse_native(*clone_args(args))
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
assert torch.equal(actual == 0, expected == 0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_gelu_and_mul_sparse_triton_nd_and_empty():
torch.set_default_device("cuda")
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
x = torch.randn(2, 3, 16384, dtype=torch.bfloat16)
args = (x, 1.6448536269514722, "tanh")
actual = impl.impl_fn(*args)
expected = gelu_and_mul_sparse_native(*args)
assert actual.shape == (2, 3, 8192)
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
empty = torch.empty(0, 16384, dtype=torch.bfloat16)
empty_out = impl.impl_fn(empty, 1.6448536269514722, "tanh")
assert empty_out.shape == (0, 8192)
assert empty_out.dtype == empty.dtype
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
def test_gelu_and_mul_sparse_triton_constant_rows(dtype: torch.dtype):
torch.set_default_device("cuda")
gate = torch.full((4, 8192), 3.0, dtype=dtype)
up = torch.randn_like(gate)
x = torch.cat((gate, up), dim=-1)
args = (x, 1.6448536269514722, "tanh")
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*args)
expected = gelu_and_mul_sparse_native(*args)
torch.testing.assert_close(actual, expected, rtol=0.0, atol=0.0)
assert torch.count_nonzero(actual) == 0
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
@pytest.mark.parametrize("nonfinite", [float("nan"), float("inf")])
def test_gelu_and_mul_sparse_triton_nonfinite_gate(nonfinite: float):
torch.set_default_device("cuda")
x = torch.randn(2, 16384, dtype=torch.bfloat16)
x[0, 0] = nonfinite
args = (x, 1.6448536269514722, "tanh")
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*args)
expected = gelu_and_mul_sparse_native(*args)
torch.testing.assert_close(actual, expected, equal_nan=True)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_gelu_and_mul_sparse_triton_fallback_conditions():
torch.set_default_device("cuda")
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
x = torch.randn(4, 16384, dtype=torch.bfloat16)
assert impl.supports_args(x, 1.0, "tanh")
assert not impl.supports_args(x, 1.0, "none")
assert not impl.supports_args(x[:, ::2], 1.0, "tanh")
assert not impl.supports_args(x[:, :-1], 1.0, "tanh")
assert not impl.supports_args(
torch.randn(1, 65538, dtype=torch.bfloat16), 1.0, "tanh"
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_gelu_and_mul_sparse_triton_dispatch_and_opcheck():
torch.set_default_device("cuda")
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
num_tokens=8,
hidden_size=4096,
dtype=torch.bfloat16,
)
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
with ir.ops.gelu_and_mul_sparse.set_priority(["triton", "native"]):
dispatched = ir.ops.gelu_and_mul_sparse(*args)
torch.library.opcheck(torch.ops.vllm_ir.gelu_and_mul_sparse, args)
direct = impl.impl_fn(*args)
torch.testing.assert_close(dispatched, direct, rtol=0.0, atol=0.0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_gelu_and_mul_sparse_triton_cudagraph():
torch.set_default_device("cuda")
x = torch.randn(16, 32768, dtype=torch.bfloat16)
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
for _ in range(3):
expected = impl.impl_fn(x, 1.6448536269514722, "tanh")
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = impl.impl_fn(x, 1.6448536269514722, "tanh")
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(actual, expected, rtol=0.0, atol=0.0)
-23
View File
@@ -1,23 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Meta-tests for vLLM IR op infrastructure.
Ensures all registered ops have input generators defined.
Per-op correctness tests live alongside their op definitions
(e.g. tests/kernels/ir/test_layernorm.py).
"""
import pytest
import vllm.kernels # noqa: F401 — registers provider implementations
from vllm.ir.op import IrOp
pytestmark = pytest.mark.skip_global_cleanup
def test_all_ops_have_input_generator():
missing = [name for name, op in IrOp.registry.items() if not op.has_input_generator]
assert not missing, (
f"IR ops without input generators: {missing}. "
f"Register one with @ir.ops.<name>.register_input_generator"
)
-444
View File
@@ -1,444 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
# This registers op implementations
import vllm.kernels # noqa: F401
from tests.ir.ir_test_utils import (
COMMON_HIDDEN_SIZES,
NUM_TOKENS,
assert_close,
clone_args,
supported_providers,
)
from tests.utils import set_random_seed
from vllm import ir
from vllm.platforms import current_platform
pytestmark = pytest.mark.skip_global_cleanup
rms_norm_native = ir.ops.rms_norm.impls["native"].impl_fn
IS_GPGPU_DEVICE = current_platform.is_cuda_alike() or current_platform.is_xpu()
@pytest.mark.skipif(
not IS_GPGPU_DEVICE,
reason="Currently only kernels on CUDA, ROCm and XPU",
)
def test_rms_norm_registration():
expected = {
"native": True,
"vllm_c": IS_GPGPU_DEVICE,
"aiter": current_platform.is_rocm(),
"oink": current_platform.has_device_capability(100)
and hasattr(torch.ops, "oink")
and hasattr(torch.ops.oink, "rmsnorm"),
}
actual = {
provider: impl.supported for provider, impl in ir.ops.rms_norm.impls.items()
}
assert actual == expected
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("n_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", COMMON_HIDDEN_SIZES)
@pytest.mark.parametrize("epsilon", [1e-6, 1e-5])
@pytest.mark.skipif(
not IS_GPGPU_DEVICE,
reason="Currently only kernels on CUDA, ROCm and XPU",
)
class TestRMSNorm:
def test_native_semantics(self, dtype, n_tokens, hidden_size, epsilon):
set_random_seed(0)
x, weight, epsilon = ir.ops.rms_norm.generate_inputs(
num_tokens=4,
hidden_size=8,
dtype=dtype,
epsilon=epsilon,
device=current_platform.device_type,
)
out = rms_norm_native(x, weight, epsilon=epsilon)
# Check shape, dtype, device
assert out.shape == x.shape
assert out.dtype == x.dtype
assert out.device == x.device
# Check the scaling property of rms norm. This holds only
# approximately: epsilon does not scale with x, so
# rms_norm(2x) = x / sqrt(mean(x^2) + epsilon/4), which differs from
# rms_norm(x) by the epsilon term. Use the op's declared tolerance
# rather than a tighter hard-coded one.
out2 = rms_norm_native(x * 2.0, weight, epsilon=epsilon)
assert_close(ir.ops.rms_norm, out2, out)
# Mean square should be approximately 1 (ignoring epsilon and weight scaling)
combined_norm = out.float() / weight.float()
variance = combined_norm.pow(2).mean(dim=-1)
# After RMS normalization, variance should be close to 1
torch.testing.assert_close(
variance, torch.ones_like(variance), rtol=1e-2, atol=1e-2
)
# Check behavior with and without weight
weight1 = torch.ones_like(weight)
out3 = rms_norm_native(x, weight1, epsilon=epsilon)
out4 = rms_norm_native(x, None, epsilon=epsilon)
torch.testing.assert_close(out3, out4)
@pytest.mark.parametrize("provider", supported_providers(ir.ops.rms_norm))
def test_impls(self, dtype, n_tokens, hidden_size, epsilon, provider):
impl = ir.ops.rms_norm.impls[provider]
x, weight, eps = ir.ops.rms_norm.generate_inputs(
num_tokens=n_tokens,
hidden_size=hidden_size,
dtype=dtype,
epsilon=epsilon,
device=current_platform.device_type,
)
args = (x, weight, eps)
if not impl.supports_args(*args):
pytest.skip(f"{provider} does not support args")
ref_output = rms_norm_native(*clone_args(args))
output = impl.impl_fn(*clone_args(args))
assert_close(ir.ops.rms_norm, output, ref_output)
# check that dispatched call matches direct call
with ir.ops.rms_norm.set_priority([provider, "native"]):
out_dispatched = ir.ops.rms_norm(*args)
out_direct = impl.impl_fn(*args)
torch.testing.assert_close(out_dispatched, out_direct, rtol=0.0, atol=0.0)
# none of these support variance_size override
assert not impl.supports_args(x, weight, eps, 4)
assert not impl.supports_args(x, weight, eps, variance_size=4)
# test weight=None behavior
out_no_weight = impl.impl_fn(x, None, eps)
out_unit_weight = impl.impl_fn(x, torch.ones_like(weight), eps)
assert_close(ir.ops.rms_norm, out_no_weight, out_unit_weight)
@pytest.mark.parametrize("provider", ["vllm_c", "aiter", "native"])
def test_torch_opcheck(self, dtype, n_tokens, hidden_size, epsilon, provider):
if not ir.ops.rms_norm.impls[provider].supported:
pytest.skip(f"{provider} impl not supported on this platform")
args = ir.ops.rms_norm.generate_inputs(
num_tokens=n_tokens,
hidden_size=hidden_size,
dtype=dtype,
epsilon=epsilon,
device=current_platform.device_type,
)
# When checking the torch op, we have to set priority and use dispatch
with ir.ops.rms_norm.set_priority([provider, "native"]):
torch.library.opcheck(torch.ops.vllm_ir.rms_norm, args)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="aiter is only supported on ROCm",
)
def test_aiter_rejects_unsupported_dtypes():
impl = ir.ops.rms_norm.impls["aiter"]
for dtype in [torch.float32, torch.float64]:
args = ir.ops.rms_norm.generate_inputs(
num_tokens=8,
hidden_size=4096,
dtype=dtype,
epsilon=1e-5,
device=current_platform.device_type,
)
assert not impl.supports_args(*args), f"aiter should reject dtype={dtype}"
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm vllm_c RMSNorm needs explicit ND input handling",
)
def test_vllm_c_rms_norm_accepts_nd_input():
impl = ir.ops.rms_norm.impls["vllm_c"]
if not impl.supported:
pytest.skip("vllm_c impl not supported on this platform")
base = torch.randn(
3, 8, 192, dtype=torch.float16, device=current_platform.device_type
)
x = base.split(64, dim=-1)[0].view(3, 8, 4, 16)
assert not x.is_contiguous()
weight = torch.randn(16, dtype=torch.float16, device=current_platform.device_type)
epsilon = 1e-5
output = impl.impl_fn(x, weight, epsilon)
ref_output = rms_norm_native(x, weight, epsilon)
assert output.shape == x.shape
assert_close(ir.ops.rms_norm, output, ref_output)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm vllm_c RMSNorm needs a contiguous output for strided inputs",
)
def test_vllm_c_rms_norm_accepts_transposed_input():
impl = ir.ops.rms_norm.impls["vllm_c"]
if not impl.supported:
pytest.skip("vllm_c impl not supported on this platform")
x = torch.randn(
1, 320, 120, dtype=torch.float16, device=current_platform.device_type
).transpose(1, 2)
assert x.reshape(-1, x.shape[-1]).stride(-1) != 1
weight = torch.randn(320, dtype=torch.float16, device=current_platform.device_type)
epsilon = 1e-5
output = impl.impl_fn(x, weight, epsilon)
ref_output = rms_norm_native(x, weight, epsilon)
assert output.shape == x.shape
assert_close(ir.ops.rms_norm, output, ref_output)
fused_add_rms_norm_native = ir.ops.fused_add_rms_norm.impls["native"].impl_fn
@pytest.mark.skipif(
not IS_GPGPU_DEVICE,
reason="Currently only kernels on CUDA, ROCm and XPU",
)
def test_fused_add_rms_norm_registration():
expected = {
"native": True,
"vllm_c": IS_GPGPU_DEVICE,
"aiter": current_platform.is_rocm(),
"oink": current_platform.has_device_capability(100)
and hasattr(torch.ops, "oink")
and hasattr(torch.ops.oink, "fused_add_rms_norm"),
}
actual = {
provider: impl.supported
for provider, impl in ir.ops.fused_add_rms_norm.impls.items()
}
assert actual == expected
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm vllm_c fused_add_rms_norm needs explicit ND input handling",
)
def test_vllm_c_fused_add_rms_norm_accepts_nd_input():
impl = ir.ops.fused_add_rms_norm.impls["vllm_c"]
if not impl.supported:
pytest.skip("vllm_c impl not supported on this platform")
base = torch.randn(
3, 8, 192, dtype=torch.float16, device=current_platform.device_type
)
residual_base = torch.randn(
3, 8, 192, dtype=torch.float16, device=current_platform.device_type
)
x = base.split(64, dim=-1)[0].view(3, 8, 4, 16)
x_residual = residual_base.split(64, dim=-1)[0].view(3, 8, 4, 16)
assert not x.is_contiguous()
assert not x_residual.is_contiguous()
weight = torch.randn(16, dtype=torch.float16, device=current_platform.device_type)
epsilon = 1e-5
output, residual = impl.impl_fn(x.clone(), x_residual.clone(), weight, epsilon)
ref_output, ref_residual = fused_add_rms_norm_native(x, x_residual, weight, epsilon)
assert output.shape == x.shape
assert residual.shape == x_residual.shape
assert_close(ir.ops.fused_add_rms_norm, output, ref_output)
assert_close(ir.ops.fused_add_rms_norm, residual, ref_residual)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("n_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", COMMON_HIDDEN_SIZES)
@pytest.mark.parametrize("epsilon", [1e-6, 1e-5])
@pytest.mark.skipif(
not IS_GPGPU_DEVICE,
reason="Currently only kernels on CUDA, ROCm and XPU",
)
class TestFusedAddRMSNorm:
def test_native_semantics(self, dtype, n_tokens, hidden_size, epsilon):
set_random_seed(0)
x, x_residual, weight, eps = ir.ops.fused_add_rms_norm.generate_inputs(
num_tokens=4,
hidden_size=8,
dtype=dtype,
epsilon=epsilon,
device=current_platform.device_type,
)
out, residual_out = fused_add_rms_norm_native(x, x_residual, weight, eps)
# Check shape, dtype, device
assert out.shape == x.shape
assert out.dtype == x.dtype
assert out.device == x.device
assert residual_out.shape == x_residual.shape
assert residual_out.dtype == x_residual.dtype
assert residual_out.device == x_residual.device
# Check that residual_out = x + x_residual
expected_residual = (x.float() + x_residual.float()).to(dtype)
torch.testing.assert_close(
residual_out, expected_residual, rtol=1e-3, atol=1e-3
)
# Verify that the output is RMS normalized version of (x + x_residual)
expected_out = rms_norm_native(expected_residual, weight, epsilon)
assert_close(
ir.ops.fused_add_rms_norm,
(out, residual_out),
(expected_out, expected_residual),
)
# Check the scaling property of rms norm
out1, _ = fused_add_rms_norm_native(
x, torch.zeros_like(x), weight, epsilon=epsilon
)
out2, _ = fused_add_rms_norm_native(
x * 2.0, torch.zeros_like(x), weight, epsilon=epsilon
)
assert_close(ir.ops.fused_add_rms_norm, out2, out1)
# Check behavior with and without weight
weight1 = torch.ones_like(weight)
out3, _ = fused_add_rms_norm_native(x, x_residual, weight1, eps)
out4, _ = fused_add_rms_norm_native(x, x_residual, None, eps)
torch.testing.assert_close(out3, out4)
@pytest.mark.parametrize("provider", supported_providers(ir.ops.fused_add_rms_norm))
def test_impls(self, dtype, n_tokens, hidden_size, epsilon, provider):
impl = ir.ops.fused_add_rms_norm.impls[provider]
x, x_residual, weight, eps = ir.ops.fused_add_rms_norm.generate_inputs(
num_tokens=n_tokens,
hidden_size=hidden_size,
dtype=dtype,
epsilon=epsilon,
device=current_platform.device_type,
)
args = (x, x_residual, weight, eps, None)
if not impl.supports_args(*args):
pytest.skip(f"{provider} does not support args")
ref_output, ref_residual = fused_add_rms_norm_native(*clone_args(args))
output, residual = impl.impl_fn(*clone_args(args))
assert_close(ir.ops.fused_add_rms_norm, output, ref_output)
assert_close(ir.ops.fused_add_rms_norm, residual, ref_residual)
# check that dispatched call matches direct call
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
out_dispatched, residual_dispatched = ir.ops.fused_add_rms_norm(*args[:4])
out_direct, residual_direct = impl.impl_fn(*clone_args(args))
torch.testing.assert_close(out_dispatched, out_direct, rtol=0.0, atol=0.0)
torch.testing.assert_close(
residual_dispatched, residual_direct, rtol=0.0, atol=0.0
)
# none of these support variance_size override
assert not impl.supports_args(x, x_residual, weight, epsilon, 4)
assert not impl.supports_args(x, x_residual, weight, epsilon, variance_size=4)
# test weight=None behavior
out_no_weight, residual_no_weight = impl.impl_fn(
x.clone(), x_residual.clone(), None, epsilon
)
out_unit_weight, residual_unit_weight = impl.impl_fn(
x.clone(), x_residual.clone(), torch.ones_like(weight), epsilon
)
assert_close(ir.ops.fused_add_rms_norm, out_no_weight, out_unit_weight)
assert_close(
ir.ops.fused_add_rms_norm, residual_no_weight, residual_unit_weight
)
@pytest.mark.parametrize("provider", ["vllm_c"])
def test_inplace_semantics(self, dtype, n_tokens, hidden_size, epsilon, provider):
"""Test that inplace implementations reuse inputs,
for maybe_inplace overload but not for default overload."""
impl = ir.ops.fused_add_rms_norm.impls[provider]
if not impl.supported:
pytest.skip(f"{provider} impl not supported on this platform")
x, x_residual, weight, eps = ir.ops.fused_add_rms_norm.generate_inputs(
num_tokens=n_tokens,
hidden_size=hidden_size,
dtype=dtype,
epsilon=epsilon,
device=current_platform.device_type,
)
# Test default overload - should NOT modify inputs even with inplace impl
x_default = x.clone()
x_residual_default = x_residual.clone()
x_default_ptr = x_default.data_ptr()
x_residual_default_ptr = x_residual_default.data_ptr()
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
out_default, residual_default = ir.ops.fused_add_rms_norm(
x_default, x_residual_default, weight, eps
)
# Default should NOT be inplace (even with inplace implementation)
assert out_default.data_ptr() != x_default_ptr
assert residual_default.data_ptr() != x_residual_default_ptr
torch.testing.assert_close(x, x_default, rtol=0.0, atol=0.0)
torch.testing.assert_close(x_residual, x_residual_default, rtol=0.0, atol=0.0)
# Test maybe_inplace overload - should modify inputs with inplace impl
x_inplace = x.clone()
x_residual_inplace = x_residual.clone()
x_inplace_ptr = x_inplace.data_ptr()
x_residual_inplace_ptr = x_residual_inplace.data_ptr()
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
out_inplace, residual_inplace = ir.ops.fused_add_rms_norm.maybe_inplace(
x_inplace, x_residual_inplace, weight, eps
)
# maybe_inplace should be inplace
assert out_inplace.data_ptr() == x_inplace_ptr
assert residual_inplace.data_ptr() == x_residual_inplace_ptr
# Both should produce same results
torch.testing.assert_close(out_default, out_inplace, atol=0.0, rtol=0.0)
torch.testing.assert_close(
residual_default, residual_inplace, atol=0.0, rtol=0.0
)
@pytest.mark.parametrize("provider", supported_providers(ir.ops.fused_add_rms_norm))
def test_torch_opcheck(self, dtype, n_tokens, hidden_size, epsilon, provider):
args = ir.ops.fused_add_rms_norm.generate_inputs(
num_tokens=n_tokens,
hidden_size=hidden_size,
dtype=dtype,
epsilon=epsilon,
device=current_platform.device_type,
)
args = args + (None,) # Add variance_size parameter
# When checking the torch op, we have to set priority and use dispatch
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
torch.library.opcheck(torch.ops.vllm_ir.fused_add_rms_norm.default, args)
# Only test maybe_inplace with non-inplace implementations
# Inplace implementations return aliases of inputs which is not allowed.
# We break this invariant, but we also convert maybe_inplace to the default
# overload during compilation, so maybe_inplace never reaches Inductor.
if not ir.ops.fused_add_rms_norm.impls[provider].inplace:
torch.library.opcheck(
torch.ops.vllm_ir.fused_add_rms_norm.maybe_inplace, args
)