[vLLM IR] Add IR op testing and benchmarking infrastructure (#40167)
Signed-off-by: Yanan Cao <gmagogsfm@gmail.com> Co-authored-by: Theresa Shan <Theresa.Shan@amd.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Theresa Shan
Claude Opus 4.6
parent
6867bcd076
commit
fe5c115ee4
@@ -9,10 +9,13 @@ from typing import Any, ClassVar, overload
|
||||
import torch
|
||||
from torch.library import Library, infer_schema
|
||||
|
||||
from vllm.ir.tolerances import DEFAULT_TOLERANCES, ToleranceSpec
|
||||
from vllm.ir.util import hash_source, weak_cache
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logging_utils import lazy, tensors_str_no_data
|
||||
|
||||
InputGenerator = Callable[..., tuple[Any, ...]]
|
||||
|
||||
vllm_ir_lib = Library("vllm_ir", "FRAGMENT")
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -113,6 +116,8 @@ class IrOp:
|
||||
self.impls: dict[str, IrOpImpl] = {}
|
||||
self._priority_impls: list[IrOpImpl] = []
|
||||
self._schema_str = infer_schema(native_impl, mutates_args=[])
|
||||
self._input_generator: InputGenerator | None = None
|
||||
self._tolerance_overrides: ToleranceSpec = {}
|
||||
|
||||
# native implementation
|
||||
self.impls["native"] = IrOpImpl(
|
||||
@@ -316,6 +321,38 @@ class IrOp:
|
||||
def supported_providers(self) -> list[str]:
|
||||
return [p.provider for p in self.impls.values() if p.supported]
|
||||
|
||||
@property
|
||||
def has_input_generator(self) -> bool:
|
||||
return self._input_generator is not None
|
||||
|
||||
def register_input_generator(self, fn: InputGenerator) -> InputGenerator:
|
||||
self._input_generator = fn
|
||||
return fn
|
||||
|
||||
def generate_inputs(self, **kwargs: Any) -> tuple[Any, ...]:
|
||||
if self._input_generator is None:
|
||||
raise RuntimeError(
|
||||
f"No input generator registered for op '{self.name}'. "
|
||||
f"Use @ir.ops.{self.name}.register_input_generator"
|
||||
)
|
||||
return self._input_generator(**kwargs)
|
||||
|
||||
def override_tolerance(
|
||||
self, dtype: torch.dtype, *, atol: float, rtol: float
|
||||
) -> None:
|
||||
self._tolerance_overrides[dtype] = {"atol": atol, "rtol": rtol}
|
||||
|
||||
def get_tolerance(self, dtype: torch.dtype) -> dict[str, float]:
|
||||
if dtype in self._tolerance_overrides:
|
||||
return self._tolerance_overrides[dtype]
|
||||
if dtype in DEFAULT_TOLERANCES:
|
||||
return DEFAULT_TOLERANCES[dtype]
|
||||
raise ValueError(
|
||||
f"No tolerance defined for dtype {dtype} in op '{self.name}'. "
|
||||
f"Use op.override_tolerance({dtype}, atol=..., rtol=...) "
|
||||
f"or add {dtype} to DEFAULT_TOLERANCES."
|
||||
)
|
||||
|
||||
|
||||
class IrOpImpl:
|
||||
def __init__(
|
||||
|
||||
@@ -19,3 +19,18 @@ def rms_norm(
|
||||
if weight is not None:
|
||||
x = x.to(weight.dtype) * weight
|
||||
return x.to(orig_dtype)
|
||||
|
||||
|
||||
@rms_norm.register_input_generator
|
||||
def _rms_norm_input_generator(
|
||||
num_tokens: int, hidden_size: int, dtype: torch.dtype, epsilon: float = 1e-5
|
||||
) -> tuple:
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype)
|
||||
weight = torch.randn(hidden_size, dtype=dtype)
|
||||
return (x, weight, epsilon)
|
||||
|
||||
|
||||
# Reductions in rms_norm accumulate rounding error at large shapes
|
||||
# (e.g. 32768x16384), causing a few elements out of millions to exceed
|
||||
# the default float16 tolerance.
|
||||
rms_norm.override_tolerance(torch.float16, atol=1e-2, rtol=2e-3)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
ToleranceSpec = dict[torch.dtype, dict[str, float]]
|
||||
|
||||
# Default tolerances for comparing IR op implementations against native.
|
||||
# These are intentionally conservative (permissive) to avoid false failures
|
||||
# across different hardware and kernel implementations. Ops that need tighter
|
||||
# or looser bounds should use override_tolerance.
|
||||
DEFAULT_TOLERANCES: ToleranceSpec = {
|
||||
# 52-bit mantissa; machine epsilon ~1.1e-16
|
||||
torch.float64: {"atol": 1e-8, "rtol": 1e-8},
|
||||
# 23-bit mantissa; machine epsilon ~1.2e-7.
|
||||
# Values from PyTorch test_transformers.py reference defaults.
|
||||
torch.float32: {"atol": 1e-5, "rtol": 1.3e-6},
|
||||
# 10-bit mantissa; machine epsilon ~9.8e-4.
|
||||
# Standard tolerance used across vLLM kernel tests.
|
||||
torch.float16: {"atol": 1e-3, "rtol": 1e-3},
|
||||
# 7-bit mantissa; machine epsilon ~7.8e-3.
|
||||
# Wider rtol than float16 to account for the coarser mantissa.
|
||||
torch.bfloat16: {"atol": 1e-3, "rtol": 1.6e-2},
|
||||
# 3-bit mantissa; machine epsilon ~6.25e-2.
|
||||
# Derived from vLLM fp8 kernel tests (merge_attn_states, silu_mul_fp8).
|
||||
torch.float8_e4m3fn: {"atol": 1e-1, "rtol": 1e-1},
|
||||
# 2-bit mantissa; machine epsilon ~1.25e-1.
|
||||
# Wider than e4m3fn due to the smaller mantissa.
|
||||
torch.float8_e5m2: {"atol": 2e-1, "rtol": 2e-1},
|
||||
# 1-bit mantissa; machine epsilon ~2.5e-1. Packed pair format (x2).
|
||||
# Derived from vLLM fp4 tests (test_silu_mul_nvfp4_quant: atol=3e-1).
|
||||
torch.float4_e2m1fn_x2: {"atol": 3e-1, "rtol": 3e-1},
|
||||
# Integer quantized; off-by-one from rounding is expected.
|
||||
# rtol=0 because relative error is meaningless for small integers.
|
||||
torch.int8: {"atol": 1, "rtol": 0},
|
||||
}
|
||||
Reference in New Issue
Block a user