[Transformers backend] Find attention with a fuser and attach vLLM's layer to it (#54941)

Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Thomas Ortner
2026-09-04 16:26:01 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 99a1ab8e4c
commit 8ad2076c4a
14 changed files with 521 additions and 150 deletions
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Concrete fusers for the Transformers modeling backend."""
from vllm.model_executor.models.transformers.fusers.attention import AttentionFuser
from vllm.model_executor.models.transformers.fusers.base import (
BaseFuser,
RewriteFuser,
@@ -15,6 +16,7 @@ from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
__all__ = [
"AttentionFuser",
"BaseFuser",
"RewriteFuser",
"StackedFuser",
@@ -0,0 +1,129 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Attention fuser: the module that dispatches to the attention interface."""
import ast
import inspect
import textwrap
from dataclasses import dataclass
from functools import cache
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
if TYPE_CHECKING:
from vllm.config import VllmConfig
VLLM_ATTN_IMPL = "vllm"
VLLM_MLA_ATTN_IMPL = "vllm_mla"
def _is_interface_lookup(node: ast.expr | None) -> bool:
"""Whether `node` reads an entry out of `ALL_ATTENTION_FUNCTIONS`."""
# ALL_ATTENTION_FUNCTIONS.get_interface(...) or ALL_ATTENTION_FUNCTIONS[...]
if isinstance(node, ast.Call):
node = node.func
if not isinstance(node, (ast.Attribute, ast.Subscript)):
return False
return (
isinstance(node.value, ast.Name) and node.value.id == "ALL_ATTENTION_FUNCTIONS"
)
@cache
def interface_call(cls: type[nn.Module]) -> ast.Call | None:
"""The attention interface call in `cls.forward`, if it makes exactly one."""
try:
source = inspect.getsource(inspect.unwrap(cls.forward))
tree = ast.parse(textwrap.dedent(source))
except (AttributeError, OSError, SyntaxError, TypeError):
return None
# Find the names of the local variables that read from the interface lookup
names = set()
for node in ast.walk(tree):
if not isinstance(node, (ast.Assign, ast.AnnAssign)):
continue
if not _is_interface_lookup(node.value):
continue
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
names.update(t.id for t in targets if isinstance(t, ast.Name))
# Find the calls to those names, and return the one if there is exactly one
calls = [
node
for node in ast.walk(tree)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id in names
]
return calls[0] if len(calls) == 1 else None
def _resolve(node: ast.expr, module: nn.Module) -> object:
"""The value of `node` on `module`, for literals and `self.<attr>`."""
if isinstance(node, ast.Constant):
return node.value
if (
isinstance(node, ast.Attribute)
and isinstance(node.value, ast.Name)
and node.value.id == "self"
):
return getattr(module, node.attr, None)
return None
@dataclass
class AttentionFuser(BaseFuser):
"""A module that dispatches through the Transformers attention interface."""
redefines_forward: ClassVar[bool] = False
source_cls: str
"""Class of the HF module that dispatches (for logging)."""
scale_expr: ast.expr | None
"""Source of the `scaling=` the module hands the interface, if it hands one."""
def info(self, name: str) -> str:
return f"Found: {name} ({self.source_cls}) -> attention interface"
@classmethod
def match(
cls, graph: fx.Graph | None, module: nn.Module
) -> "AttentionFuser | None":
if (call := interface_call(type(module))) is None:
return None
scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"]
scale_expr = scaling[0] if len(scaling) == 1 else None
return cls(source_cls=type(module).__name__, scale_expr=scale_expr)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Whether `module` will actually dispatch to vLLM."""
config = getattr(module, "config", None)
# Only patched in the text config, this excludes attention based mm encoders
vllm_attn_impls = {VLLM_ATTN_IMPL, VLLM_MLA_ATTN_IMPL}
return getattr(config, "_attn_implementation", None) in vllm_attn_impls
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
return module
def layer_index(self, module: nn.Module) -> int | None:
"""The layer `module` computes attention for, if it declares one."""
layer_idx = getattr(module, "layer_idx", None)
return layer_idx if isinstance(layer_idx, int) else None
def scale(self, module: nn.Module) -> float | None:
"""The softmax scale `module` passes to the interface, or `None`."""
if self.scale_expr is None:
return None
scale = _resolve(self.scale_expr, module)
if not isinstance(scale, (int, float)) or isinstance(scale, bool):
expression = ast.unparse(self.scale_expr)
raise ValueError(
f"Cannot resolve attention scaling expression {expression!r} in "
f"{type(module).__name__}."
)
return float(scale)
@@ -20,11 +20,15 @@ if TYPE_CHECKING:
class BaseFuser(ABC):
"""A detected fusion and how to apply it.
`match` analyses the module *class* once (cached, see `get_fuser`); `fuse`
`match` analyses the module *class* once (cached, see `get_fusers`); `fuse`
then applies the fusion to an instance in `recursive_replace`, returning the
module to install in its place.
"""
redefines_forward: ClassVar[bool] = True
"""Whether `fuse` gives the module a different forward,
by rewriting its source or by returning a different module in its place."""
@abstractmethod
def info(self, name: str) -> str:
"""A human-readable description of the fusion at `name`, for logging."""
@@ -59,7 +59,7 @@ def _is_rms_norm(module: nn.Module) -> bool:
# Go via `get_fuser` because it caches
from vllm.model_executor.models.transformers.fuser import get_fuser
return isinstance(get_fuser(module), RMSNormFuser)
return get_fuser(module, RMSNormFuser) is not None
def _top_level_index(funcdef: ast.FunctionDef, node: ast.AST) -> int: