[vLLM IR] Add IR op testing and benchmarking infrastructure (#40167)

Signed-off-by: Yanan Cao <gmagogsfm@gmail.com>
Co-authored-by: Theresa Shan <Theresa.Shan@amd.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yanan Cao
2026-04-21 00:23:03 +00:00
committed by GitHub
co-authored by Theresa Shan Claude Opus 4.6
parent 6867bcd076
commit fe5c115ee4
12 changed files with 674 additions and 46 deletions
+37
View File
@@ -9,10 +9,13 @@ from typing import Any, ClassVar, overload
import torch
from torch.library import Library, infer_schema
from vllm.ir.tolerances import DEFAULT_TOLERANCES, ToleranceSpec
from vllm.ir.util import hash_source, weak_cache
from vllm.logger import init_logger
from vllm.logging_utils import lazy, tensors_str_no_data
InputGenerator = Callable[..., tuple[Any, ...]]
vllm_ir_lib = Library("vllm_ir", "FRAGMENT")
logger = init_logger(__name__)
@@ -113,6 +116,8 @@ class IrOp:
self.impls: dict[str, IrOpImpl] = {}
self._priority_impls: list[IrOpImpl] = []
self._schema_str = infer_schema(native_impl, mutates_args=[])
self._input_generator: InputGenerator | None = None
self._tolerance_overrides: ToleranceSpec = {}
# native implementation
self.impls["native"] = IrOpImpl(
@@ -316,6 +321,38 @@ class IrOp:
def supported_providers(self) -> list[str]:
return [p.provider for p in self.impls.values() if p.supported]
@property
def has_input_generator(self) -> bool:
return self._input_generator is not None
def register_input_generator(self, fn: InputGenerator) -> InputGenerator:
self._input_generator = fn
return fn
def generate_inputs(self, **kwargs: Any) -> tuple[Any, ...]:
if self._input_generator is None:
raise RuntimeError(
f"No input generator registered for op '{self.name}'. "
f"Use @ir.ops.{self.name}.register_input_generator"
)
return self._input_generator(**kwargs)
def override_tolerance(
self, dtype: torch.dtype, *, atol: float, rtol: float
) -> None:
self._tolerance_overrides[dtype] = {"atol": atol, "rtol": rtol}
def get_tolerance(self, dtype: torch.dtype) -> dict[str, float]:
if dtype in self._tolerance_overrides:
return self._tolerance_overrides[dtype]
if dtype in DEFAULT_TOLERANCES:
return DEFAULT_TOLERANCES[dtype]
raise ValueError(
f"No tolerance defined for dtype {dtype} in op '{self.name}'. "
f"Use op.override_tolerance({dtype}, atol=..., rtol=...) "
f"or add {dtype} to DEFAULT_TOLERANCES."
)
class IrOpImpl:
def __init__(
+15
View File
@@ -19,3 +19,18 @@ def rms_norm(
if weight is not None:
x = x.to(weight.dtype) * weight
return x.to(orig_dtype)
@rms_norm.register_input_generator
def _rms_norm_input_generator(
num_tokens: int, hidden_size: int, dtype: torch.dtype, epsilon: float = 1e-5
) -> tuple:
x = torch.randn(num_tokens, hidden_size, dtype=dtype)
weight = torch.randn(hidden_size, dtype=dtype)
return (x, weight, epsilon)
# Reductions in rms_norm accumulate rounding error at large shapes
# (e.g. 32768x16384), causing a few elements out of millions to exceed
# the default float16 tolerance.
rms_norm.override_tolerance(torch.float16, atol=1e-2, rtol=2e-3)
+36
View File
@@ -0,0 +1,36 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
ToleranceSpec = dict[torch.dtype, dict[str, float]]
# Default tolerances for comparing IR op implementations against native.
# These are intentionally conservative (permissive) to avoid false failures
# across different hardware and kernel implementations. Ops that need tighter
# or looser bounds should use override_tolerance.
DEFAULT_TOLERANCES: ToleranceSpec = {
# 52-bit mantissa; machine epsilon ~1.1e-16
torch.float64: {"atol": 1e-8, "rtol": 1e-8},
# 23-bit mantissa; machine epsilon ~1.2e-7.
# Values from PyTorch test_transformers.py reference defaults.
torch.float32: {"atol": 1e-5, "rtol": 1.3e-6},
# 10-bit mantissa; machine epsilon ~9.8e-4.
# Standard tolerance used across vLLM kernel tests.
torch.float16: {"atol": 1e-3, "rtol": 1e-3},
# 7-bit mantissa; machine epsilon ~7.8e-3.
# Wider rtol than float16 to account for the coarser mantissa.
torch.bfloat16: {"atol": 1e-3, "rtol": 1.6e-2},
# 3-bit mantissa; machine epsilon ~6.25e-2.
# Derived from vLLM fp8 kernel tests (merge_attn_states, silu_mul_fp8).
torch.float8_e4m3fn: {"atol": 1e-1, "rtol": 1e-1},
# 2-bit mantissa; machine epsilon ~1.25e-1.
# Wider than e4m3fn due to the smaller mantissa.
torch.float8_e5m2: {"atol": 2e-1, "rtol": 2e-1},
# 1-bit mantissa; machine epsilon ~2.5e-1. Packed pair format (x2).
# Derived from vLLM fp4 tests (test_silu_mul_nvfp4_quant: atol=3e-1).
torch.float4_e2m1fn_x2: {"atol": 3e-1, "rtol": 3e-1},
# Integer quantized; off-by-one from rounding is expected.
# rtol=0 because relative error is meaningless for small integers.
torch.int8: {"atol": 1, "rtol": 0},
}