[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,183 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.kernels # noqa: F401
|
||||
from tests.ir.ir_test_utils import assert_close, clone_args
|
||||
from vllm import ir
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
gelu_and_mul_sparse_native = ir.ops.gelu_and_mul_sparse.impls["native"].impl_fn
|
||||
|
||||
|
||||
def test_gelu_and_mul_sparse_registration():
|
||||
expected = {
|
||||
"native": True,
|
||||
"triton": HAS_TRITON and current_platform.is_cuda(),
|
||||
}
|
||||
actual = {
|
||||
provider: impl.supported
|
||||
for provider, impl in ir.ops.gelu_and_mul_sparse.impls.items()
|
||||
}
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 8, 64])
|
||||
@pytest.mark.parametrize("intermediate_size", [2048, 4096, 8192, 13824, 16384, 32768])
|
||||
def test_gelu_and_mul_sparse_triton(
|
||||
dtype: torch.dtype, num_tokens: int, intermediate_size: int
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
torch.set_default_device("cuda")
|
||||
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
|
||||
num_tokens=num_tokens,
|
||||
hidden_size=intermediate_size,
|
||||
dtype=dtype,
|
||||
)
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
assert impl.supports_args(*args)
|
||||
|
||||
expected = gelu_and_mul_sparse_native(*clone_args(args))
|
||||
actual = impl.impl_fn(*clone_args(args))
|
||||
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
|
||||
|
||||
mask_mismatch = torch.count_nonzero((actual == 0) != (expected == 0))
|
||||
assert mask_mismatch / actual.numel() < 1e-3
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_second_seed():
|
||||
torch.manual_seed(1)
|
||||
torch.set_default_device("cuda")
|
||||
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
|
||||
num_tokens=32,
|
||||
hidden_size=8192,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*clone_args(args))
|
||||
expected = gelu_and_mul_sparse_native(*clone_args(args))
|
||||
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("intermediate_size", [1, 31, 128])
|
||||
def test_gelu_and_mul_sparse_triton_small_sizes(
|
||||
dtype: torch.dtype, intermediate_size: int
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
torch.set_default_device("cuda")
|
||||
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
|
||||
num_tokens=5,
|
||||
hidden_size=intermediate_size,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*clone_args(args))
|
||||
expected = gelu_and_mul_sparse_native(*clone_args(args))
|
||||
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
|
||||
assert torch.equal(actual == 0, expected == 0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_nd_and_empty():
|
||||
torch.set_default_device("cuda")
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
|
||||
x = torch.randn(2, 3, 16384, dtype=torch.bfloat16)
|
||||
args = (x, 1.6448536269514722, "tanh")
|
||||
actual = impl.impl_fn(*args)
|
||||
expected = gelu_and_mul_sparse_native(*args)
|
||||
assert actual.shape == (2, 3, 8192)
|
||||
assert_close(ir.ops.gelu_and_mul_sparse, actual, expected)
|
||||
|
||||
empty = torch.empty(0, 16384, dtype=torch.bfloat16)
|
||||
empty_out = impl.impl_fn(empty, 1.6448536269514722, "tanh")
|
||||
assert empty_out.shape == (0, 8192)
|
||||
assert empty_out.dtype == empty.dtype
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
def test_gelu_and_mul_sparse_triton_constant_rows(dtype: torch.dtype):
|
||||
torch.set_default_device("cuda")
|
||||
gate = torch.full((4, 8192), 3.0, dtype=dtype)
|
||||
up = torch.randn_like(gate)
|
||||
x = torch.cat((gate, up), dim=-1)
|
||||
args = (x, 1.6448536269514722, "tanh")
|
||||
|
||||
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*args)
|
||||
expected = gelu_and_mul_sparse_native(*args)
|
||||
torch.testing.assert_close(actual, expected, rtol=0.0, atol=0.0)
|
||||
assert torch.count_nonzero(actual) == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
@pytest.mark.parametrize("nonfinite", [float("nan"), float("inf")])
|
||||
def test_gelu_and_mul_sparse_triton_nonfinite_gate(nonfinite: float):
|
||||
torch.set_default_device("cuda")
|
||||
x = torch.randn(2, 16384, dtype=torch.bfloat16)
|
||||
x[0, 0] = nonfinite
|
||||
args = (x, 1.6448536269514722, "tanh")
|
||||
|
||||
actual = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn(*args)
|
||||
expected = gelu_and_mul_sparse_native(*args)
|
||||
torch.testing.assert_close(actual, expected, equal_nan=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_fallback_conditions():
|
||||
torch.set_default_device("cuda")
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
x = torch.randn(4, 16384, dtype=torch.bfloat16)
|
||||
|
||||
assert impl.supports_args(x, 1.0, "tanh")
|
||||
assert not impl.supports_args(x, 1.0, "none")
|
||||
assert not impl.supports_args(x[:, ::2], 1.0, "tanh")
|
||||
assert not impl.supports_args(x[:, :-1], 1.0, "tanh")
|
||||
assert not impl.supports_args(
|
||||
torch.randn(1, 65538, dtype=torch.bfloat16), 1.0, "tanh"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_dispatch_and_opcheck():
|
||||
torch.set_default_device("cuda")
|
||||
args = ir.ops.gelu_and_mul_sparse.generate_inputs(
|
||||
num_tokens=8,
|
||||
hidden_size=4096,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
|
||||
with ir.ops.gelu_and_mul_sparse.set_priority(["triton", "native"]):
|
||||
dispatched = ir.ops.gelu_and_mul_sparse(*args)
|
||||
torch.library.opcheck(torch.ops.vllm_ir.gelu_and_mul_sparse, args)
|
||||
|
||||
direct = impl.impl_fn(*args)
|
||||
torch.testing.assert_close(dispatched, direct, rtol=0.0, atol=0.0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_gelu_and_mul_sparse_triton_cudagraph():
|
||||
torch.set_default_device("cuda")
|
||||
x = torch.randn(16, 32768, dtype=torch.bfloat16)
|
||||
impl = ir.ops.gelu_and_mul_sparse.impls["triton"]
|
||||
|
||||
for _ in range(3):
|
||||
expected = impl.impl_fn(x, 1.6448536269514722, "tanh")
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = impl.impl_fn(x, 1.6448536269514722, "tanh")
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=0.0, atol=0.0)
|
||||
@@ -1,23 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Meta-tests for vLLM IR op infrastructure.
|
||||
|
||||
Ensures all registered ops have input generators defined.
|
||||
Per-op correctness tests live alongside their op definitions
|
||||
(e.g. tests/kernels/ir/test_layernorm.py).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.kernels # noqa: F401 — registers provider implementations
|
||||
from vllm.ir.op import IrOp
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
def test_all_ops_have_input_generator():
|
||||
missing = [name for name, op in IrOp.registry.items() if not op.has_input_generator]
|
||||
assert not missing, (
|
||||
f"IR ops without input generators: {missing}. "
|
||||
f"Register one with @ir.ops.<name>.register_input_generator"
|
||||
)
|
||||
@@ -1,444 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# This registers op implementations
|
||||
import vllm.kernels # noqa: F401
|
||||
from tests.ir.ir_test_utils import (
|
||||
COMMON_HIDDEN_SIZES,
|
||||
NUM_TOKENS,
|
||||
assert_close,
|
||||
clone_args,
|
||||
supported_providers,
|
||||
)
|
||||
from tests.utils import set_random_seed
|
||||
from vllm import ir
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
rms_norm_native = ir.ops.rms_norm.impls["native"].impl_fn
|
||||
|
||||
IS_GPGPU_DEVICE = current_platform.is_cuda_alike() or current_platform.is_xpu()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not IS_GPGPU_DEVICE,
|
||||
reason="Currently only kernels on CUDA, ROCm and XPU",
|
||||
)
|
||||
def test_rms_norm_registration():
|
||||
expected = {
|
||||
"native": True,
|
||||
"vllm_c": IS_GPGPU_DEVICE,
|
||||
"aiter": current_platform.is_rocm(),
|
||||
"oink": current_platform.has_device_capability(100)
|
||||
and hasattr(torch.ops, "oink")
|
||||
and hasattr(torch.ops.oink, "rmsnorm"),
|
||||
}
|
||||
|
||||
actual = {
|
||||
provider: impl.supported for provider, impl in ir.ops.rms_norm.impls.items()
|
||||
}
|
||||
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("n_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", COMMON_HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("epsilon", [1e-6, 1e-5])
|
||||
@pytest.mark.skipif(
|
||||
not IS_GPGPU_DEVICE,
|
||||
reason="Currently only kernels on CUDA, ROCm and XPU",
|
||||
)
|
||||
class TestRMSNorm:
|
||||
def test_native_semantics(self, dtype, n_tokens, hidden_size, epsilon):
|
||||
set_random_seed(0)
|
||||
x, weight, epsilon = ir.ops.rms_norm.generate_inputs(
|
||||
num_tokens=4,
|
||||
hidden_size=8,
|
||||
dtype=dtype,
|
||||
epsilon=epsilon,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
out = rms_norm_native(x, weight, epsilon=epsilon)
|
||||
|
||||
# Check shape, dtype, device
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == x.dtype
|
||||
assert out.device == x.device
|
||||
|
||||
# Check the scaling property of rms norm. This holds only
|
||||
# approximately: epsilon does not scale with x, so
|
||||
# rms_norm(2x) = x / sqrt(mean(x^2) + epsilon/4), which differs from
|
||||
# rms_norm(x) by the epsilon term. Use the op's declared tolerance
|
||||
# rather than a tighter hard-coded one.
|
||||
out2 = rms_norm_native(x * 2.0, weight, epsilon=epsilon)
|
||||
assert_close(ir.ops.rms_norm, out2, out)
|
||||
|
||||
# Mean square should be approximately 1 (ignoring epsilon and weight scaling)
|
||||
combined_norm = out.float() / weight.float()
|
||||
variance = combined_norm.pow(2).mean(dim=-1)
|
||||
# After RMS normalization, variance should be close to 1
|
||||
torch.testing.assert_close(
|
||||
variance, torch.ones_like(variance), rtol=1e-2, atol=1e-2
|
||||
)
|
||||
|
||||
# Check behavior with and without weight
|
||||
weight1 = torch.ones_like(weight)
|
||||
out3 = rms_norm_native(x, weight1, epsilon=epsilon)
|
||||
out4 = rms_norm_native(x, None, epsilon=epsilon)
|
||||
torch.testing.assert_close(out3, out4)
|
||||
|
||||
@pytest.mark.parametrize("provider", supported_providers(ir.ops.rms_norm))
|
||||
def test_impls(self, dtype, n_tokens, hidden_size, epsilon, provider):
|
||||
impl = ir.ops.rms_norm.impls[provider]
|
||||
x, weight, eps = ir.ops.rms_norm.generate_inputs(
|
||||
num_tokens=n_tokens,
|
||||
hidden_size=hidden_size,
|
||||
dtype=dtype,
|
||||
epsilon=epsilon,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
args = (x, weight, eps)
|
||||
|
||||
if not impl.supports_args(*args):
|
||||
pytest.skip(f"{provider} does not support args")
|
||||
|
||||
ref_output = rms_norm_native(*clone_args(args))
|
||||
output = impl.impl_fn(*clone_args(args))
|
||||
assert_close(ir.ops.rms_norm, output, ref_output)
|
||||
|
||||
# check that dispatched call matches direct call
|
||||
with ir.ops.rms_norm.set_priority([provider, "native"]):
|
||||
out_dispatched = ir.ops.rms_norm(*args)
|
||||
out_direct = impl.impl_fn(*args)
|
||||
torch.testing.assert_close(out_dispatched, out_direct, rtol=0.0, atol=0.0)
|
||||
|
||||
# none of these support variance_size override
|
||||
assert not impl.supports_args(x, weight, eps, 4)
|
||||
assert not impl.supports_args(x, weight, eps, variance_size=4)
|
||||
|
||||
# test weight=None behavior
|
||||
out_no_weight = impl.impl_fn(x, None, eps)
|
||||
out_unit_weight = impl.impl_fn(x, torch.ones_like(weight), eps)
|
||||
assert_close(ir.ops.rms_norm, out_no_weight, out_unit_weight)
|
||||
|
||||
@pytest.mark.parametrize("provider", ["vllm_c", "aiter", "native"])
|
||||
def test_torch_opcheck(self, dtype, n_tokens, hidden_size, epsilon, provider):
|
||||
if not ir.ops.rms_norm.impls[provider].supported:
|
||||
pytest.skip(f"{provider} impl not supported on this platform")
|
||||
|
||||
args = ir.ops.rms_norm.generate_inputs(
|
||||
num_tokens=n_tokens,
|
||||
hidden_size=hidden_size,
|
||||
dtype=dtype,
|
||||
epsilon=epsilon,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
|
||||
# When checking the torch op, we have to set priority and use dispatch
|
||||
with ir.ops.rms_norm.set_priority([provider, "native"]):
|
||||
torch.library.opcheck(torch.ops.vllm_ir.rms_norm, args)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="aiter is only supported on ROCm",
|
||||
)
|
||||
def test_aiter_rejects_unsupported_dtypes():
|
||||
impl = ir.ops.rms_norm.impls["aiter"]
|
||||
for dtype in [torch.float32, torch.float64]:
|
||||
args = ir.ops.rms_norm.generate_inputs(
|
||||
num_tokens=8,
|
||||
hidden_size=4096,
|
||||
dtype=dtype,
|
||||
epsilon=1e-5,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
assert not impl.supports_args(*args), f"aiter should reject dtype={dtype}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm vllm_c RMSNorm needs explicit ND input handling",
|
||||
)
|
||||
def test_vllm_c_rms_norm_accepts_nd_input():
|
||||
impl = ir.ops.rms_norm.impls["vllm_c"]
|
||||
if not impl.supported:
|
||||
pytest.skip("vllm_c impl not supported on this platform")
|
||||
|
||||
base = torch.randn(
|
||||
3, 8, 192, dtype=torch.float16, device=current_platform.device_type
|
||||
)
|
||||
x = base.split(64, dim=-1)[0].view(3, 8, 4, 16)
|
||||
assert not x.is_contiguous()
|
||||
weight = torch.randn(16, dtype=torch.float16, device=current_platform.device_type)
|
||||
epsilon = 1e-5
|
||||
|
||||
output = impl.impl_fn(x, weight, epsilon)
|
||||
ref_output = rms_norm_native(x, weight, epsilon)
|
||||
|
||||
assert output.shape == x.shape
|
||||
assert_close(ir.ops.rms_norm, output, ref_output)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm vllm_c RMSNorm needs a contiguous output for strided inputs",
|
||||
)
|
||||
def test_vllm_c_rms_norm_accepts_transposed_input():
|
||||
impl = ir.ops.rms_norm.impls["vllm_c"]
|
||||
if not impl.supported:
|
||||
pytest.skip("vllm_c impl not supported on this platform")
|
||||
|
||||
x = torch.randn(
|
||||
1, 320, 120, dtype=torch.float16, device=current_platform.device_type
|
||||
).transpose(1, 2)
|
||||
assert x.reshape(-1, x.shape[-1]).stride(-1) != 1
|
||||
weight = torch.randn(320, dtype=torch.float16, device=current_platform.device_type)
|
||||
epsilon = 1e-5
|
||||
|
||||
output = impl.impl_fn(x, weight, epsilon)
|
||||
ref_output = rms_norm_native(x, weight, epsilon)
|
||||
|
||||
assert output.shape == x.shape
|
||||
assert_close(ir.ops.rms_norm, output, ref_output)
|
||||
|
||||
|
||||
fused_add_rms_norm_native = ir.ops.fused_add_rms_norm.impls["native"].impl_fn
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not IS_GPGPU_DEVICE,
|
||||
reason="Currently only kernels on CUDA, ROCm and XPU",
|
||||
)
|
||||
def test_fused_add_rms_norm_registration():
|
||||
expected = {
|
||||
"native": True,
|
||||
"vllm_c": IS_GPGPU_DEVICE,
|
||||
"aiter": current_platform.is_rocm(),
|
||||
"oink": current_platform.has_device_capability(100)
|
||||
and hasattr(torch.ops, "oink")
|
||||
and hasattr(torch.ops.oink, "fused_add_rms_norm"),
|
||||
}
|
||||
|
||||
actual = {
|
||||
provider: impl.supported
|
||||
for provider, impl in ir.ops.fused_add_rms_norm.impls.items()
|
||||
}
|
||||
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm vllm_c fused_add_rms_norm needs explicit ND input handling",
|
||||
)
|
||||
def test_vllm_c_fused_add_rms_norm_accepts_nd_input():
|
||||
impl = ir.ops.fused_add_rms_norm.impls["vllm_c"]
|
||||
if not impl.supported:
|
||||
pytest.skip("vllm_c impl not supported on this platform")
|
||||
|
||||
base = torch.randn(
|
||||
3, 8, 192, dtype=torch.float16, device=current_platform.device_type
|
||||
)
|
||||
residual_base = torch.randn(
|
||||
3, 8, 192, dtype=torch.float16, device=current_platform.device_type
|
||||
)
|
||||
x = base.split(64, dim=-1)[0].view(3, 8, 4, 16)
|
||||
x_residual = residual_base.split(64, dim=-1)[0].view(3, 8, 4, 16)
|
||||
assert not x.is_contiguous()
|
||||
assert not x_residual.is_contiguous()
|
||||
weight = torch.randn(16, dtype=torch.float16, device=current_platform.device_type)
|
||||
epsilon = 1e-5
|
||||
|
||||
output, residual = impl.impl_fn(x.clone(), x_residual.clone(), weight, epsilon)
|
||||
ref_output, ref_residual = fused_add_rms_norm_native(x, x_residual, weight, epsilon)
|
||||
|
||||
assert output.shape == x.shape
|
||||
assert residual.shape == x_residual.shape
|
||||
assert_close(ir.ops.fused_add_rms_norm, output, ref_output)
|
||||
assert_close(ir.ops.fused_add_rms_norm, residual, ref_residual)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("n_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", COMMON_HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("epsilon", [1e-6, 1e-5])
|
||||
@pytest.mark.skipif(
|
||||
not IS_GPGPU_DEVICE,
|
||||
reason="Currently only kernels on CUDA, ROCm and XPU",
|
||||
)
|
||||
class TestFusedAddRMSNorm:
|
||||
def test_native_semantics(self, dtype, n_tokens, hidden_size, epsilon):
|
||||
set_random_seed(0)
|
||||
x, x_residual, weight, eps = ir.ops.fused_add_rms_norm.generate_inputs(
|
||||
num_tokens=4,
|
||||
hidden_size=8,
|
||||
dtype=dtype,
|
||||
epsilon=epsilon,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
out, residual_out = fused_add_rms_norm_native(x, x_residual, weight, eps)
|
||||
|
||||
# Check shape, dtype, device
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == x.dtype
|
||||
assert out.device == x.device
|
||||
assert residual_out.shape == x_residual.shape
|
||||
assert residual_out.dtype == x_residual.dtype
|
||||
assert residual_out.device == x_residual.device
|
||||
|
||||
# Check that residual_out = x + x_residual
|
||||
expected_residual = (x.float() + x_residual.float()).to(dtype)
|
||||
torch.testing.assert_close(
|
||||
residual_out, expected_residual, rtol=1e-3, atol=1e-3
|
||||
)
|
||||
|
||||
# Verify that the output is RMS normalized version of (x + x_residual)
|
||||
expected_out = rms_norm_native(expected_residual, weight, epsilon)
|
||||
assert_close(
|
||||
ir.ops.fused_add_rms_norm,
|
||||
(out, residual_out),
|
||||
(expected_out, expected_residual),
|
||||
)
|
||||
|
||||
# Check the scaling property of rms norm
|
||||
out1, _ = fused_add_rms_norm_native(
|
||||
x, torch.zeros_like(x), weight, epsilon=epsilon
|
||||
)
|
||||
out2, _ = fused_add_rms_norm_native(
|
||||
x * 2.0, torch.zeros_like(x), weight, epsilon=epsilon
|
||||
)
|
||||
assert_close(ir.ops.fused_add_rms_norm, out2, out1)
|
||||
|
||||
# Check behavior with and without weight
|
||||
weight1 = torch.ones_like(weight)
|
||||
out3, _ = fused_add_rms_norm_native(x, x_residual, weight1, eps)
|
||||
out4, _ = fused_add_rms_norm_native(x, x_residual, None, eps)
|
||||
torch.testing.assert_close(out3, out4)
|
||||
|
||||
@pytest.mark.parametrize("provider", supported_providers(ir.ops.fused_add_rms_norm))
|
||||
def test_impls(self, dtype, n_tokens, hidden_size, epsilon, provider):
|
||||
impl = ir.ops.fused_add_rms_norm.impls[provider]
|
||||
x, x_residual, weight, eps = ir.ops.fused_add_rms_norm.generate_inputs(
|
||||
num_tokens=n_tokens,
|
||||
hidden_size=hidden_size,
|
||||
dtype=dtype,
|
||||
epsilon=epsilon,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
args = (x, x_residual, weight, eps, None)
|
||||
|
||||
if not impl.supports_args(*args):
|
||||
pytest.skip(f"{provider} does not support args")
|
||||
|
||||
ref_output, ref_residual = fused_add_rms_norm_native(*clone_args(args))
|
||||
output, residual = impl.impl_fn(*clone_args(args))
|
||||
assert_close(ir.ops.fused_add_rms_norm, output, ref_output)
|
||||
assert_close(ir.ops.fused_add_rms_norm, residual, ref_residual)
|
||||
|
||||
# check that dispatched call matches direct call
|
||||
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
|
||||
out_dispatched, residual_dispatched = ir.ops.fused_add_rms_norm(*args[:4])
|
||||
out_direct, residual_direct = impl.impl_fn(*clone_args(args))
|
||||
torch.testing.assert_close(out_dispatched, out_direct, rtol=0.0, atol=0.0)
|
||||
torch.testing.assert_close(
|
||||
residual_dispatched, residual_direct, rtol=0.0, atol=0.0
|
||||
)
|
||||
|
||||
# none of these support variance_size override
|
||||
assert not impl.supports_args(x, x_residual, weight, epsilon, 4)
|
||||
assert not impl.supports_args(x, x_residual, weight, epsilon, variance_size=4)
|
||||
|
||||
# test weight=None behavior
|
||||
out_no_weight, residual_no_weight = impl.impl_fn(
|
||||
x.clone(), x_residual.clone(), None, epsilon
|
||||
)
|
||||
out_unit_weight, residual_unit_weight = impl.impl_fn(
|
||||
x.clone(), x_residual.clone(), torch.ones_like(weight), epsilon
|
||||
)
|
||||
assert_close(ir.ops.fused_add_rms_norm, out_no_weight, out_unit_weight)
|
||||
assert_close(
|
||||
ir.ops.fused_add_rms_norm, residual_no_weight, residual_unit_weight
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("provider", ["vllm_c"])
|
||||
def test_inplace_semantics(self, dtype, n_tokens, hidden_size, epsilon, provider):
|
||||
"""Test that inplace implementations reuse inputs,
|
||||
for maybe_inplace overload but not for default overload."""
|
||||
impl = ir.ops.fused_add_rms_norm.impls[provider]
|
||||
if not impl.supported:
|
||||
pytest.skip(f"{provider} impl not supported on this platform")
|
||||
|
||||
x, x_residual, weight, eps = ir.ops.fused_add_rms_norm.generate_inputs(
|
||||
num_tokens=n_tokens,
|
||||
hidden_size=hidden_size,
|
||||
dtype=dtype,
|
||||
epsilon=epsilon,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
|
||||
# Test default overload - should NOT modify inputs even with inplace impl
|
||||
x_default = x.clone()
|
||||
x_residual_default = x_residual.clone()
|
||||
x_default_ptr = x_default.data_ptr()
|
||||
x_residual_default_ptr = x_residual_default.data_ptr()
|
||||
|
||||
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
|
||||
out_default, residual_default = ir.ops.fused_add_rms_norm(
|
||||
x_default, x_residual_default, weight, eps
|
||||
)
|
||||
|
||||
# Default should NOT be inplace (even with inplace implementation)
|
||||
assert out_default.data_ptr() != x_default_ptr
|
||||
assert residual_default.data_ptr() != x_residual_default_ptr
|
||||
torch.testing.assert_close(x, x_default, rtol=0.0, atol=0.0)
|
||||
torch.testing.assert_close(x_residual, x_residual_default, rtol=0.0, atol=0.0)
|
||||
|
||||
# Test maybe_inplace overload - should modify inputs with inplace impl
|
||||
x_inplace = x.clone()
|
||||
x_residual_inplace = x_residual.clone()
|
||||
x_inplace_ptr = x_inplace.data_ptr()
|
||||
x_residual_inplace_ptr = x_residual_inplace.data_ptr()
|
||||
|
||||
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
|
||||
out_inplace, residual_inplace = ir.ops.fused_add_rms_norm.maybe_inplace(
|
||||
x_inplace, x_residual_inplace, weight, eps
|
||||
)
|
||||
|
||||
# maybe_inplace should be inplace
|
||||
assert out_inplace.data_ptr() == x_inplace_ptr
|
||||
assert residual_inplace.data_ptr() == x_residual_inplace_ptr
|
||||
|
||||
# Both should produce same results
|
||||
torch.testing.assert_close(out_default, out_inplace, atol=0.0, rtol=0.0)
|
||||
torch.testing.assert_close(
|
||||
residual_default, residual_inplace, atol=0.0, rtol=0.0
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("provider", supported_providers(ir.ops.fused_add_rms_norm))
|
||||
def test_torch_opcheck(self, dtype, n_tokens, hidden_size, epsilon, provider):
|
||||
args = ir.ops.fused_add_rms_norm.generate_inputs(
|
||||
num_tokens=n_tokens,
|
||||
hidden_size=hidden_size,
|
||||
dtype=dtype,
|
||||
epsilon=epsilon,
|
||||
device=current_platform.device_type,
|
||||
)
|
||||
args = args + (None,) # Add variance_size parameter
|
||||
|
||||
# When checking the torch op, we have to set priority and use dispatch
|
||||
with ir.ops.fused_add_rms_norm.set_priority([provider, "native"]):
|
||||
torch.library.opcheck(torch.ops.vllm_ir.fused_add_rms_norm.default, args)
|
||||
|
||||
# Only test maybe_inplace with non-inplace implementations
|
||||
# Inplace implementations return aliases of inputs which is not allowed.
|
||||
# We break this invariant, but we also convert maybe_inplace to the default
|
||||
# overload during compilation, so maybe_inplace never reaches Inductor.
|
||||
if not ir.ops.fused_add_rms_norm.impls[provider].inplace:
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm_ir.fused_add_rms_norm.maybe_inplace, args
|
||||
)
|
||||
Reference in New Issue
Block a user