[vLLM IR] Minor improvements (#39362) (#39558)

Signed-off-by: Avishek Goswami <avishek.goswami@ibm.com>
Co-authored-by: Avishek Goswami <avishek.goswami@ibm.com>
This commit is contained in:
Avishek Goswami
2026-05-12 10:58:36 -04:00
committed by GitHub
co-authored by Avishek Goswami
parent 6ccb10d794
commit 289cee0473
3 changed files with 328 additions and 89 deletions
+73 -23
View File
@@ -2,10 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import inspect
import traceback
from collections.abc import Callable
from pathlib import Path
from typing import Any, ClassVar, Literal, overload
import regex as re
import torch
from torch.library import Library, infer_schema
@@ -16,13 +18,35 @@ from vllm.logging_utils import lazy, tensors_str_no_data
InputGenerator = Callable[..., tuple[Any, ...]]
vllm_ir_lib = Library("vllm_ir", "FRAGMENT")
vllm_ir_torch_lib = Library("vllm_ir", "FRAGMENT") # IR op lib; monkeypatch in tests.
logger = init_logger(__name__)
def _torch_ops_subtree(lib: Any) -> Any:
"""``torch.ops`` subtree for ``lib.ns``; fall back if doc mocks replace ``ns``."""
ns = getattr(lib, "ns", None)
if isinstance(ns, str):
return getattr(torch.ops, ns)
return torch.ops.vllm_ir
_NAME_PATTERN = re.compile(r"^[a-z_][a-z_0-9]*$")
RESERVED_PROVIDERS = ["native", "unfused"]
"""Providers that are reserved and cannot be used for custom implementations."""
def _validate_name(name: str, entity_type: str) -> None:
"""Validate that a name matches the required pattern `[a-z_][a-z_0-9]*`."""
if not _NAME_PATTERN.match(name):
raise ValueError(
f"{entity_type} name '{name}' is invalid. "
f"Names must start with a letter or underscore, "
f"followed by lowercase letters, underscores, or digits only."
)
_ENABLE_TORCH_WRAP: bool = True
"""Global override flag to control torch op layer wrapping."""
@@ -100,11 +124,14 @@ def register_op(
def decorator(_f: Callable):
op_name: str = _f.__name__ if name is None else name
assert op_name not in IrOp.registry
_validate_name(op_name, "Op")
assert op_name not in IrOp.registry, f"Op '{op_name}' is already registered."
# Slice out the decorator function frames from the stack
stack = traceback.format_stack()[:-2]
if allow_inplace:
op: IrOp = IrOpInplace(op_name, _f, activations)
op: IrOp = IrOpInplace(op_name, _f, activations, stack)
else:
op = IrOp(op_name, _f, activations)
op = IrOp(op_name, _f, activations, stack)
IrOp.registry[op_name] = op
return op
@@ -126,6 +153,7 @@ class IrOp:
name: str,
native_impl: Callable,
activations: list[str] | None = None,
registration_stack: list[str] | None = None,
):
self._py_signature = inspect.signature(native_impl)
if any(
@@ -146,6 +174,8 @@ class IrOp:
]
self.name = name
self._docstring = inspect.getdoc(native_impl) or ""
self._registration_stack = registration_stack or []
self.impls: dict[str, IrOpImpl] = {}
self.activations = activations
self.activation_indices = [
@@ -166,22 +196,23 @@ class IrOp:
# always supported
supported=True,
supports_args=None,
registration_stack=self._registration_stack,
)
# By default, fake routes directly to native,
# can be overridden by register_fake
self._fake_fn = native_impl
# torch registration
vllm_ir_lib.define(self.name + self._schema_str)
# torch registration (resolve ``torch.ops`` subtree from ``lib.ns``)
lib = vllm_ir_torch_lib
lib.define(self.name + self._schema_str)
# CompositeExplicitAutograd is not decomposed
# by ATen IR normalization in AOTAutograd
vllm_ir_lib.impl(
self.name, self._inner_call, dispatch_key="CompositeExplicitAutograd"
)
vllm_ir_lib._register_fake(self.name, self._fake_call)
assert hasattr(torch.ops.vllm_ir, name)
self.torch_op: torch._ops.OpOverload = getattr(torch.ops.vllm_ir, name).default
lib.impl(self.name, self._inner_call, dispatch_key="CompositeExplicitAutograd")
lib._register_fake(self.name, self._fake_call)
torch_ops = _torch_ops_subtree(lib)
assert hasattr(torch_ops, name)
self.torch_op: torch._ops.OpOverload = getattr(torch_ops, name).default
def register_fake(self, fn: Callable) -> Callable:
"""
@@ -237,9 +268,12 @@ class IrOp:
assert provider not in RESERVED_PROVIDERS, (
f"Provider name {provider} is reserved."
)
_validate_name(provider, "Provider")
def _register_impl(f: Callable):
impl = IrOpImpl(self, provider, f, supported, supports_args, inplace)
# Slice out the decorator function from the stack
stack = traceback.format_stack()[:-1]
impl = IrOpImpl(self, provider, f, supported, supports_args, inplace, stack)
self.impls[provider] = impl
if self.get_priority():
@@ -323,6 +357,17 @@ class IrOp:
return self.torch_op(*args, **kwargs)
def __repr__(self) -> str:
"""Return unambiguous string representation."""
return f"IrOp('{self.name}')"
def __str__(self) -> str:
"""Return human-readable string representation using docstring."""
if not self._docstring:
return f"IrOp('{self.name}')"
first_line = self._docstring.split("\n")[0].strip()
return f"IrOp('{self.name}') - {first_line}"
def get_priority(self) -> list[str]:
"""Get the current dispatch priority for implementations for this op."""
return [p.provider for p in self._priority_impls]
@@ -419,8 +464,9 @@ class IrOpInplace(IrOp):
name: str,
native_impl: Callable,
activations: list[str] | None = None,
registration_stack: list[str] | None = None,
):
super().__init__(name, native_impl, activations)
super().__init__(name, native_impl, activations, registration_stack)
# Create the inplace overload
self.maybe_inplace = IrOpInplaceOverload(self)
@@ -445,16 +491,16 @@ class IrOpInplaceOverload:
op.impls["native"].impl_fn, mutates_args=op.activations
)
# torch registration
vllm_ir_lib.define(self.name + self._schema_str)
vllm_ir_lib.impl(
self.name, self._inner_call, dispatch_key="CompositeExplicitAutograd"
)
# torch registration (resolve ``torch.ops`` subtree from ``lib.ns``)
lib = vllm_ir_torch_lib
lib.define(self.name + self._schema_str)
lib.impl(self.name, self._inner_call, dispatch_key="CompositeExplicitAutograd")
# fake goes to default overload for now
vllm_ir_lib._register_fake(self.name, self.op._fake_call)
lib._register_fake(self.name, self.op._fake_call)
assert hasattr(getattr(torch.ops.vllm_ir, self.op.name), "maybe_inplace")
self.torch_op = getattr(torch.ops.vllm_ir, self.op.name).maybe_inplace
torch_ops = _torch_ops_subtree(lib)
assert hasattr(getattr(torch_ops, self.op.name), "maybe_inplace")
self.torch_op = getattr(torch_ops, self.op.name).maybe_inplace
def __call__(self, *args, **kwargs) -> Any:
if not _ENABLE_TORCH_WRAP:
@@ -477,12 +523,15 @@ class IrOpImpl:
supported: bool,
supports_args: Callable[..., bool] | None,
inplace: bool = False,
registration_stack: list[str] | None = None,
):
assert provider not in op.impls, (
f"Implementation for provider {provider} already registered."
)
# Native also uses this path, so we allow it here.
assert provider == "native" or provider not in RESERVED_PROVIDERS
assert provider == "native" or provider not in RESERVED_PROVIDERS, (
f"Provider name {provider} is reserved."
)
# Enforce the exact same schema as the native implementation.
# This takes care of names, types, and defaults.
@@ -547,6 +596,7 @@ class IrOpImpl:
self.supported = supported
self._supports_args = supports_args
self.inplace = inplace
self._registration_stack = registration_stack or []
@property
def supports_all_args(self) -> bool: