[vLLM IR] Add IR op testing and benchmarking infrastructure (#40167)
Signed-off-by: Yanan Cao <gmagogsfm@gmail.com> Co-authored-by: Theresa Shan <Theresa.Shan@amd.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Theresa Shan
Claude Opus 4.6
parent
6867bcd076
commit
fe5c115ee4
@@ -0,0 +1,46 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Shared test utilities for vLLM IR op correctness tests.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.ir.op import IrOp
|
||||
|
||||
NUM_TOKENS = [1, 8, 17, 32, 512, 2048]
|
||||
COMMON_HIDDEN_SIZES = [
|
||||
2048, # Llama 3.2 1B, Qwen 3 MoE 30B-A3B, Gemma 3n
|
||||
4096, # Llama 3 8B, Qwen 3 8B
|
||||
5120, # Llama 4 Scout 17B-16E
|
||||
7168, # DeepSeek V3
|
||||
8192, # Llama 3 70B
|
||||
]
|
||||
|
||||
|
||||
def clone_args(args: tuple) -> tuple:
|
||||
return tuple(a.clone() if isinstance(a, torch.Tensor) else a for a in args)
|
||||
|
||||
|
||||
def supported_providers(op: IrOp) -> list[str]:
|
||||
return [
|
||||
name for name, impl in op.impls.items() if name != "native" and impl.supported
|
||||
]
|
||||
|
||||
|
||||
def assert_close(op: IrOp, actual, expected):
|
||||
if isinstance(actual, torch.Tensor):
|
||||
tol = op.get_tolerance(actual.dtype)
|
||||
try:
|
||||
torch.testing.assert_close(actual, expected, **tol)
|
||||
except AssertionError as e:
|
||||
raise AssertionError(
|
||||
f"{e}\n\nTo adjust tolerance, use:\n"
|
||||
f" ir.ops.{op.name}.override_tolerance("
|
||||
f"{actual.dtype}, atol=..., rtol=...)"
|
||||
) from None
|
||||
elif isinstance(actual, (tuple, list)):
|
||||
for a, ex in zip(actual, expected):
|
||||
assert_close(op, a, ex)
|
||||
else:
|
||||
assert actual == expected
|
||||
@@ -495,3 +495,68 @@ def test_uuid_and_oot(tmp_path: Path):
|
||||
assert uuid2 == uuid
|
||||
assert uuid2 != uuid1
|
||||
del _custom_mm.impls["impl_mm_oot"]
|
||||
|
||||
|
||||
def _test_native(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
return x + y
|
||||
|
||||
|
||||
def _make_op_with_generator(name: str = "_ig_test"):
|
||||
op = IrOp(name, _test_native)
|
||||
|
||||
@op.register_input_generator
|
||||
def _gen(n: int = 4):
|
||||
x = torch.randn(n, 3)
|
||||
y = torch.randn(n, 3)
|
||||
return x, y
|
||||
|
||||
return op
|
||||
|
||||
|
||||
def _test_native_single(x: torch.Tensor) -> torch.Tensor:
|
||||
return x
|
||||
|
||||
|
||||
class TestInputGenerator:
|
||||
def test_no_input_generator_by_default(self):
|
||||
op = IrOp("_ig_test_no_gen", _test_native_single)
|
||||
assert not op.has_input_generator
|
||||
|
||||
def test_register_input_generator(self):
|
||||
op = _make_op_with_generator("_ig_test_reg")
|
||||
assert op.has_input_generator
|
||||
|
||||
def test_generate_inputs_returns_tuple(self):
|
||||
op = _make_op_with_generator("_ig_test_tuple")
|
||||
result = op.generate_inputs(n=2)
|
||||
assert isinstance(result, tuple)
|
||||
assert len(result) == 2
|
||||
assert result[0].shape == (2, 3)
|
||||
assert result[1].shape == (2, 3)
|
||||
|
||||
def test_generate_inputs_default_kwargs(self):
|
||||
op = _make_op_with_generator("_ig_test_default")
|
||||
result = op.generate_inputs()
|
||||
assert result[0].shape == (4, 3)
|
||||
|
||||
def test_generate_inputs_without_registration_raises(self):
|
||||
op = IrOp("_ig_test_no_gen_raises", _test_native_single)
|
||||
with pytest.raises(RuntimeError, match="No input generator"):
|
||||
op.generate_inputs()
|
||||
|
||||
|
||||
class TestTolerance:
|
||||
def test_override_and_get_tolerance(self):
|
||||
op = IrOp("_tol_test", _test_native)
|
||||
|
||||
tol = op.get_tolerance(torch.float32)
|
||||
assert tol == {"atol": 1e-5, "rtol": 1.3e-6}
|
||||
|
||||
op.override_tolerance(torch.float32, atol=0.1, rtol=0.2)
|
||||
assert op.get_tolerance(torch.float32) == {"atol": 0.1, "rtol": 0.2}
|
||||
assert op.get_tolerance(torch.float16) == {"atol": 1e-3, "rtol": 1e-3}
|
||||
|
||||
def test_get_tolerance_raises_for_unknown_dtype(self):
|
||||
op = IrOp("_tol_test_unknown", _test_native)
|
||||
with pytest.raises(ValueError, match="No tolerance defined"):
|
||||
op.get_tolerance(torch.complex64)
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Meta-tests for vLLM IR op infrastructure.
|
||||
|
||||
Ensures all registered ops have input generators defined.
|
||||
Per-op correctness tests live alongside their op definitions
|
||||
(e.g. tests/kernels/ir/test_layernorm.py).
|
||||
"""
|
||||
|
||||
import vllm.kernels # noqa: F401 — registers provider implementations
|
||||
from vllm.ir.op import IrOp
|
||||
|
||||
|
||||
def test_all_ops_have_input_generator():
|
||||
missing = [name for name, op in IrOp.registry.items() if not op.has_input_generator]
|
||||
assert not missing, (
|
||||
f"IR ops without input generators: {missing}. "
|
||||
f"Register one with @ir.ops.<name>.register_input_generator"
|
||||
)
|
||||
@@ -5,17 +5,17 @@ import torch
|
||||
|
||||
# This registers op implementations
|
||||
import vllm.kernels # noqa: F401
|
||||
from tests.ir.ir_test_utils import (
|
||||
COMMON_HIDDEN_SIZES,
|
||||
NUM_TOKENS,
|
||||
assert_close,
|
||||
clone_args,
|
||||
supported_providers,
|
||||
)
|
||||
from tests.kernels.allclose_default import get_default_rtol
|
||||
from vllm import ir
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def rms_norm_inputs(n_tokens: int, hidden_size: int, dtype: torch.dtype):
|
||||
x = torch.randn(n_tokens, hidden_size, dtype=dtype)
|
||||
weight = torch.rand(hidden_size, dtype=dtype)
|
||||
return x, weight
|
||||
|
||||
|
||||
rms_norm_native = ir.ops.rms_norm.impls["native"].impl_fn
|
||||
|
||||
|
||||
@@ -40,8 +40,8 @@ def test_rms_norm_registration():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("n_tokens", [1, 8, 17])
|
||||
@pytest.mark.parametrize("hidden_size", [16, 4096, 8192])
|
||||
@pytest.mark.parametrize("n_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", COMMON_HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("epsilon", [1e-6, 1e-5])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike() and not current_platform.is_xpu(),
|
||||
@@ -53,7 +53,9 @@ class TestRMSNorm:
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
def test_native_semantics(self, dtype, n_tokens, hidden_size, epsilon):
|
||||
x, weight = rms_norm_inputs(4, 8, dtype)
|
||||
x, weight, epsilon = ir.ops.rms_norm.generate_inputs(
|
||||
num_tokens=4, hidden_size=8, dtype=dtype, epsilon=epsilon
|
||||
)
|
||||
out = rms_norm_native(x, weight, epsilon=epsilon)
|
||||
|
||||
# Check shape, dtype, device
|
||||
@@ -71,59 +73,59 @@ class TestRMSNorm:
|
||||
out4 = rms_norm_native(x, None, epsilon=epsilon)
|
||||
torch.testing.assert_close(out3, out4)
|
||||
|
||||
@pytest.mark.parametrize("provider", ["vllm_c", "aiter", "xpu_kernels"])
|
||||
@pytest.mark.parametrize("provider", supported_providers(ir.ops.rms_norm))
|
||||
def test_impls(self, dtype, n_tokens, hidden_size, epsilon, provider):
|
||||
impl = ir.ops.rms_norm.impls[provider]
|
||||
if not impl.supported:
|
||||
pytest.skip(f"{provider} impl not supported on this platform")
|
||||
|
||||
x, weight = rms_norm_inputs(n_tokens, hidden_size, dtype)
|
||||
args = (x, weight, epsilon, None)
|
||||
|
||||
assert impl.supported
|
||||
|
||||
if provider == "aiter" and dtype not in [torch.float16, torch.bfloat16]:
|
||||
assert not impl.supports_args(*args)
|
||||
return
|
||||
|
||||
assert impl.supports_args(*args)
|
||||
|
||||
out_impl = impl.impl_fn(*args)
|
||||
out_native = rms_norm_native(*args)
|
||||
|
||||
torch.testing.assert_close(
|
||||
out_impl, out_native, rtol=get_default_rtol(out_impl), atol=1e-3
|
||||
x, weight, eps = ir.ops.rms_norm.generate_inputs(
|
||||
num_tokens=n_tokens, hidden_size=hidden_size, dtype=dtype, epsilon=epsilon
|
||||
)
|
||||
args = (x, weight, eps)
|
||||
|
||||
if not impl.supports_args(*args):
|
||||
pytest.skip(f"{provider} does not support args")
|
||||
|
||||
ref_output = rms_norm_native(*clone_args(args))
|
||||
output = impl.impl_fn(*clone_args(args))
|
||||
assert_close(ir.ops.rms_norm, output, ref_output)
|
||||
|
||||
# check that dispatched call matches direct call
|
||||
with ir.ops.rms_norm.set_priority([provider, "native"]):
|
||||
out_impl2 = ir.ops.rms_norm(*args)
|
||||
|
||||
# exact match
|
||||
torch.testing.assert_close(out_impl2, out_impl, rtol=0.0, atol=0.0)
|
||||
out_dispatched = ir.ops.rms_norm(*args)
|
||||
out_direct = impl.impl_fn(*args)
|
||||
torch.testing.assert_close(out_dispatched, out_direct, rtol=0.0, atol=0.0)
|
||||
|
||||
# none of these support variance_size override
|
||||
assert not impl.supports_args(x, weight, epsilon, 4)
|
||||
assert not impl.supports_args(x, weight, epsilon, variance_size=4)
|
||||
assert not impl.supports_args(x, weight, eps, 4)
|
||||
assert not impl.supports_args(x, weight, eps, variance_size=4)
|
||||
|
||||
# test weight=None behavior
|
||||
out_impl_no_weight = impl.impl_fn(x, None, epsilon)
|
||||
out_impl_unit_weight = impl.impl_fn(x, torch.ones_like(weight), epsilon)
|
||||
torch.testing.assert_close(
|
||||
out_impl_no_weight,
|
||||
out_impl_unit_weight,
|
||||
rtol=get_default_rtol(out_impl_no_weight),
|
||||
atol=2e-4,
|
||||
)
|
||||
out_no_weight = impl.impl_fn(x, None, eps)
|
||||
out_unit_weight = impl.impl_fn(x, torch.ones_like(weight), eps)
|
||||
assert_close(ir.ops.rms_norm, out_no_weight, out_unit_weight)
|
||||
|
||||
@pytest.mark.parametrize("provider", ["vllm_c", "aiter", "xpu_kernels", "native"])
|
||||
def test_torch_opcheck(self, dtype, n_tokens, hidden_size, epsilon, provider):
|
||||
if not ir.ops.rms_norm.impls[provider].supported:
|
||||
pytest.skip(f"{provider} impl not supported on this platform")
|
||||
|
||||
x, weight = rms_norm_inputs(n_tokens, hidden_size, dtype)
|
||||
args = (x, weight, epsilon, None)
|
||||
args = ir.ops.rms_norm.generate_inputs(
|
||||
num_tokens=n_tokens, hidden_size=hidden_size, dtype=dtype, epsilon=epsilon
|
||||
)
|
||||
|
||||
# When checking the torch op, we have to set priority and use dispatch
|
||||
with ir.ops.rms_norm.set_priority([provider, "native"]):
|
||||
torch.library.opcheck(torch.ops.vllm_ir.rms_norm, args)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="aiter is only supported on ROCm",
|
||||
)
|
||||
def test_aiter_rejects_unsupported_dtypes():
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
impl = ir.ops.rms_norm.impls["aiter"]
|
||||
for dtype in [torch.float32, torch.float64]:
|
||||
args = ir.ops.rms_norm.generate_inputs(
|
||||
num_tokens=8, hidden_size=4096, dtype=dtype, epsilon=1e-5
|
||||
)
|
||||
assert not impl.supports_args(*args), f"aiter should reject dtype={dtype}"
|
||||
|
||||
Reference in New Issue
Block a user