[Transformers backend] Find attention with a fuser and attach vLLM's layer to it (#54941)

Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Thomas Ortner
2026-09-04 16:26:01 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 99a1ab8e4c
commit 8ad2076c4a
14 changed files with 521 additions and 150 deletions
+4 -1
View File
@@ -1253,7 +1253,10 @@ class _ModelRegistry:
"'auto_map' (relevant if the model is custom)."
)
if not model_module.is_backend_compatible():
if not (
model_module.is_backend_compatible()
or model_module._can_set_attn_implementation()
):
if model_config.model_impl != "transformers":
return None
@@ -21,7 +21,7 @@ from typing import TYPE_CHECKING
import torch.nn.functional as F
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from vllm.model_executor.models.transformers.base import Base
from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR, Base
from vllm.model_executor.models.transformers.causal import CausalMixin
from vllm.model_executor.models.transformers.legacy import LegacyMixin
from vllm.model_executor.models.transformers.moe import MoEMixin
@@ -40,8 +40,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
if TYPE_CHECKING:
import torch
from vllm.model_executor.layers.attention import Attention, MLAAttention
def vllm_attention_forward(
# Transformers args
@@ -50,15 +48,9 @@ def vllm_attention_forward(
key: "torch.Tensor",
value: "torch.Tensor",
attention_mask: "torch.Tensor",
# Transformers kwargs
scaling: float | None = None,
# vLLM kwargs
attention_instances: "dict[int, Attention] | None" = None,
**kwargs,
):
self_attn = attention_instances[module.layer_idx]
if scaling is not None:
self_attn.impl.scale = float(scaling)
self_attn = getattr(module, VLLM_ATTN_ATTR)
hidden = query.shape[-2]
head_dim_qk = query.shape[-1]
head_dim_v = value.shape[-1]
@@ -85,13 +77,9 @@ def vllm_mla_attention_forward(
kv_c_normed: "torch.Tensor",
k_pe: "torch.Tensor",
attention_mask: "torch.Tensor",
# Transformers kwargs
scaling: float | None = None,
# vLLM kwargs
attention_instances: "dict[int, MLAAttention] | None" = None,
**kwargs,
):
self_attn = attention_instances[module.layer_idx]
self_attn = getattr(module, VLLM_ATTN_ATTR)
# [batch=1, heads, num_tokens, qk_head_dim] -> [num_tokens, heads, qk_head_dim]
query = query.transpose(1, 2).flatten(0, 1)
num_tokens, num_heads = query.shape[:2]
+78 -34
View File
@@ -57,7 +57,11 @@ from vllm.model_executor.models.interfaces import (
)
from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
from vllm.model_executor.models.transformers.fusers import MLAFuser
from vllm.model_executor.models.transformers.fusers import AttentionFuser, MLAFuser
from vllm.model_executor.models.transformers.fusers.attention import (
VLLM_ATTN_IMPL,
VLLM_MLA_ATTN_IMPL,
)
from vllm.model_executor.models.transformers.utils import (
attrsetter,
can_enable_torch_compile,
@@ -73,7 +77,6 @@ from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
@@ -86,6 +89,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
VLLM_ATTN_ATTR = "attn"
"""Attribute in which to store the vLLM `Attention` instance that serves a Transformers
attention module. `"attn"` is chosen deliberately to match the mappings generated by
`maybe_remap_kv_scale_name`."""
class PreTrainedModelClasses(NamedTuple):
decoder: type["PreTrainedModel"]
@@ -129,9 +137,13 @@ class Base(
self.packed_modules_mapping: dict[str, list[str]] = {}
"""Fused module -> constituent projections, populated by `recursive_replace`
for the quantization machinery and loaders (e.g. bitsandbytes)."""
self.fusers: dict[str, BaseFuser] = {}
"""Module qualname -> the fuser applied to it, populated
self.fusers: dict[str, list[BaseFuser]] = {}
"""Module qualname -> the fusers applied to it, populated
by `recursive_replace` for `create_attention_instances`."""
self.attention_fusers: dict[int, tuple[str, AttentionFuser]] = {}
"""`layer_idx` -> the qualname and fuser of the module computing that
layer's attention, populated by `recursive_replace` for
`create_attention_instances`."""
# Attrs for Eagle3 (see self.set_aux_hidden_state_layers)
self._target_class: type[nn.Module] = nn.Module
@@ -189,7 +201,7 @@ class Base(
- Sets the dtype to the default torch dtype set by vLLM because Transformers
uses the config dtype when creating the model
"""
self.text_config._attn_implementation = "vllm"
self.text_config._attn_implementation = VLLM_ATTN_IMPL
self.config.dtype = torch.get_default_dtype()
@contextmanager
@@ -475,9 +487,22 @@ class Base(
vocab_embeddings = self._vocab_embeddings()
def register_fusion(fuser: BaseFuser, prefix: str):
def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module):
"""Register a fused layer's mappings just before it is built."""
self.fusers[prefix] = fuser
self.fusers.setdefault(prefix, []).append(fuser)
if (
isinstance(fuser, AttentionFuser)
and (index := fuser.layer_index(module)) is not None
):
if index in self.attention_fusers:
other, _ = self.attention_fusers[index]
raise ValueError(
f"Layer {index} dispatches attention from more than one "
f"module ({other} and {prefix}), which one vLLM attention "
"layer cannot serve."
)
self.attention_fusers[index] = (prefix, fuser)
orig_to_new_stacked = fuser.orig_to_new_stacked(prefix)
self.hf_to_vllm_mapper.orig_to_new_stacked.update(orig_to_new_stacked)
@@ -528,10 +553,11 @@ class Base(
new_module = replace_embedding_class(
child_module, self.quant_config, prefix=qual_name
)
elif (fuser := fusers[child_module]) is not None:
register_fusion(fuser, qual_name)
new_module = fuser.fuse(child_module, qual_name, self.vllm_config)
logger.info_once(fuser.info(child_name))
elif child_module_fusers := fusers[child_module]:
for fuser in child_module_fusers:
register_fusion(fuser, qual_name, child_module)
new_module = fuser.fuse(new_module, qual_name, self.vllm_config)
logger.info_once(fuser.info(child_name))
_recursive_replace(new_module, prefix=qual_name)
elif not isinstance(child_module, MoERunner):
# MoERunner can contain aliases of shared experts and gates,
@@ -548,20 +574,14 @@ class Base(
"""
Create `Attention` instances to inform KV cache allocation.
"""
mla_fusers = {}
attention_instances = {}
text_config = self.text_config
attn_cls = self._get_attn_cls()
# kv_lora_rank indicates that this is an MLA model
if getattr(text_config, "kv_lora_rank", None) is not None:
mla_fusers = {
extract_layer_index(prefix): (prefix, fuser)
for prefix, fuser in self.fusers.items()
if isinstance(fuser, MLAFuser)
}
if attn_cls is MLAAttention:
text_config._attn_implementation = "vllm_mla"
text_config._attn_implementation = VLLM_MLA_ATTN_IMPL
else:
# MLA model not using MLAAttention: recompute head_size for full attn
qk_nope_head_dim = getattr(text_config, "qk_nope_head_dim", 0)
@@ -570,12 +590,23 @@ class Base(
self.model_config.model_arch_config.head_size = qk_head_dim
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
layer_types = getattr(text_config, "layer_types", None)
pp_rank = self.pp_group.rank_in_group
pp_size = self.pp_group.world_size
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
for i in range(start, end):
if i not in self.attention_fusers:
in_range = layer_types and i < len(layer_types)
layer = f"{i} ({layer_types[i]})" if in_range else str(i)
raise ValueError(
f"Layer {layer} does not dispatch through the Transformers "
"attention interface and vLLM has no other way to handle it."
)
prefix, attn_fuser = self.attention_fusers[i]
attn_module = self.get_submodule(prefix)
# `[i]` is the whole-model config unless the checkpoint is
# heterogeneous, in which case it is this layer's own geometry.
arch_config = self.model_config.model_arch_config[i]
@@ -583,8 +614,10 @@ class Base(
self.parallel_config, arch_config
)
head_size = arch_config.head_size
# Default to Llama scale, maybe updated in vllm_attention_forward
scale = head_size**-0.5
scale = attn_fuser.scale(attn_module)
# Default to Llama scale if AttentionFuser couldn't identify it
if scale is None:
scale = head_size**-0.5
num_kv_heads = self.model_config.get_num_kv_heads(
self.parallel_config, arch_config
)
@@ -598,17 +631,22 @@ class Base(
)
if attn_cls is MLAAttention:
prefix, fuser = mla_fusers[i]
mla_module = self.get_submodule(prefix)
mla_fuser = next(
(f for f in self.fusers[prefix] if isinstance(f, MLAFuser)), None
)
if mla_fuser is None:
raise ValueError(
f"Layer {i} ({prefix}) did not fuse as MLA, but the rest of "
"the model did, so its latent KV cache cannot be allocated."
)
dims = get_mla_dims(self.model_config)
kwargs.update(
scale=mla_module.scaling,
qk_nope_head_dim=dims.qk_nope_head_dim,
qk_rope_head_dim=dims.qk_rope_head_dim,
v_head_dim=dims.v_head_dim,
q_lora_rank=dims.q_lora_rank,
kv_lora_rank=dims.kv_lora_rank,
kv_b_proj=mla_module.get_submodule(fuser.kv_b_proj_name),
kv_b_proj=attn_module.get_submodule(mla_fuser.kv_b_proj_name),
)
else:
kwargs.update(
@@ -618,17 +656,20 @@ class Base(
)
# Handle interleaved sliding window attention
if (
hasattr(text_config, "layer_types")
and text_config.layer_types[i] == "sliding_attention"
):
if layer_types and layer_types[i] == "sliding_attention":
kwargs["per_layer_sliding_window"] = text_config.sliding_window
attn_instance = attn_cls(**kwargs)
if attn_cls is MLAAttention:
# Attach MLA attn_instance to mla_module so it appears in
# model.named_modules() and runs its process_weights_after_loading
mla_module._vllm_mla_attn = attn_instance
# Attach the instance to the module it serves, which is the only way it
# is in the module tree at all: a plain dict is invisible to
# `nn.Module.__setattr__`. Being in the tree runs its
# `process_weights_after_loading` and moves its KV-scale buffers with
# `model.to(device)`, and lets the attention interface read it off the
# module rather than index by `module.layer_idx`, an `int` that bakes
# layer identity into the traced graph and so costs one compiled
# artifact per layer.
setattr(attn_module, VLLM_ATTN_ATTR, attn_instance)
attention_instances[i] = attn_instance
return attention_instances
@@ -645,7 +686,11 @@ class Base(
return EncoderOnlyAttention
if self.model_config.use_mla:
self.check_version("5.15.0.dev0", "optimized MLA support")
if any(isinstance(fuser, MLAFuser) for fuser in self.fusers.values()):
if any(
isinstance(fuser, MLAFuser)
for fusers in self.fusers.values()
for fuser in fusers
):
return MLAAttention
logger.warning_once(
"This model uses MLA but `MLAFuser` failed to match and/or fuse any "
@@ -731,7 +776,6 @@ class Base(
inputs_embeds=inputs_embeds,
use_cache=False,
position_ids=positions,
attention_instances=self.attention_instances,
return_dict=False,
**self._output_aux_hidden_states_kwargs,
**kwargs,
@@ -2,20 +2,22 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fuser detection for the Transformers modeling backend.
`get_fuser` traces a module class once (see `fx_utils`) and matches it against
each concrete fuser in `fusers`; `Fusers` caches the result per class for a
whole model. `base.recursive_replace` then applies the matched fuser per
`get_fusers` traces a module class once (see `fx_utils`) and matches it against each
concrete fuser in `fusers`; `Fusers` caches the result per class for a whole model.
At most one match may give the module a different forward, and any number of fusers that
leave the forward alone apply alongside. `base.recursive_replace` then applies them per
instance. RMSNorm-shaped modules the tracer cannot match are warned about.
"""
from collections import UserDict
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, TypeVar
from cachetools import cached
from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.models.transformers.fusers import (
AttentionFuser,
BaseFuser,
GLUFuser,
MLAFuser,
@@ -31,58 +33,90 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
F = TypeVar("F", bound=BaseFuser)
def key(module: nn.Module) -> tuple:
"""Cache key for `get_fuser`. Considers module type and its immediate children."""
"""Cache key for `get_fusers`. Considers module type and its immediate children."""
return (type(module), tuple(name for name, _ in module.named_children()))
FUSERS: tuple[type[BaseFuser], ...] = (
MLAFuser,
GLUFuser,
QKVFuser,
PackedQKVFuser,
RMSNormFuser,
AttentionFuser,
)
"""Every fuser, in match order: those that redefine the forward first, then those
that leave it alone. A new fuser is added here."""
@cached(cache={}, key=key)
def get_fuser(module: nn.Module) -> BaseFuser | None:
"""The fuser for `module`'s class and shape (cached), or `None` if no match."""
def get_fusers(module: nn.Module) -> list[BaseFuser]:
"""Every fuser that applies to `module`'s class and shape (cached).
The one that redefines the forward comes first, so applying them in order
fuses before the rest read or adjust the result."""
# Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a
# leaf module (raw tensor math, no submodules). Nothing else can match, and
# tracing is skipped for it.
n_linear = sum(isinstance(c, nn.Linear) for c in module.children())
is_leaf = next(module.children(), None) is None
if n_linear < 2 and not is_leaf:
return None
if (graph := trace(module)) is None:
return None
for fuser_cls in (MLAFuser, GLUFuser, QKVFuser, PackedQKVFuser, RMSNormFuser):
if (fuser := fuser_cls.match(graph, module)) is not None:
if isinstance(fuser, RewriteFuser):
try:
fuser.update_forward(module)
except Exception as exc:
logger.debug(
"Attempted to fuse %s using %s but failed "
"to update its forward method: %s",
type(module),
fuser_cls.__name__,
exc,
)
continue
return fuser
graph = trace(module) if n_linear >= 2 or is_leaf else None
fusers: list[BaseFuser] = []
for fuser_cls in FUSERS:
if fuser_cls.redefines_forward and (
# A forward can only be rewritten from a trace of it, and only once,
# because every rewrite starts from the original source.
graph is None or any(fuser.redefines_forward for fuser in fusers)
):
continue
if (fuser := fuser_cls.match(graph, module)) is None:
continue
if isinstance(fuser, RewriteFuser):
try:
fuser.update_forward(module)
except Exception as exc:
logger.debug(
"Attempted to fuse %s using %s but failed "
"to update its forward method: %s",
type(module),
fuser_cls.__name__,
exc,
)
continue
fusers.append(fuser)
# A norm we could not match structurally is left unfused; flag likely misses.
if module.__class__.__name__.endswith("RMSNorm"):
if (
module.__class__.__name__.endswith("RMSNorm")
and graph is not None
and not any(isinstance(fuser, RMSNormFuser) for fuser in fusers)
):
logger.warning_once(
"%s looks like an RMSNorm but its computation did not match the "
"expected pattern, so it was left unfused.",
module.__class__.__name__,
)
return None
return fusers
def get_fuser(module: nn.Module, fuser_cls: type[F]) -> "F | None":
"""The first `fuser_cls` that applies to `module`, if one does."""
return next((f for f in get_fusers(module) if isinstance(f, fuser_cls)), None)
class Fusers(UserDict):
"""Mapping from module class and shape to fuser, for all fusable modules."""
"""Mapping from module class and shape to fusers, for all fusable modules."""
def __init__(self, model: nn.Module, vllm_config: "VllmConfig"):
self.vllm_config = vllm_config
super().__init__({key(m): get_fuser(m) for m in model.modules()})
super().__init__({key(m): get_fusers(m) for m in model.modules()})
def __getitem__(self, m: nn.Module) -> BaseFuser | None:
fuser = self.data.get(key(m))
if fuser is not None and fuser.validate(m, self.vllm_config):
return fuser
return None
def __getitem__(self, m: nn.Module) -> list[BaseFuser]:
"""The fusers this instance can take, in the order to apply them."""
fusers = self.data.get(key(m), ())
return [fuser for fuser in fusers if fuser.validate(m, self.vllm_config)]
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Concrete fusers for the Transformers modeling backend."""
from vllm.model_executor.models.transformers.fusers.attention import AttentionFuser
from vllm.model_executor.models.transformers.fusers.base import (
BaseFuser,
RewriteFuser,
@@ -15,6 +16,7 @@ from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
__all__ = [
"AttentionFuser",
"BaseFuser",
"RewriteFuser",
"StackedFuser",
@@ -0,0 +1,129 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Attention fuser: the module that dispatches to the attention interface."""
import ast
import inspect
import textwrap
from dataclasses import dataclass
from functools import cache
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
if TYPE_CHECKING:
from vllm.config import VllmConfig
VLLM_ATTN_IMPL = "vllm"
VLLM_MLA_ATTN_IMPL = "vllm_mla"
def _is_interface_lookup(node: ast.expr | None) -> bool:
"""Whether `node` reads an entry out of `ALL_ATTENTION_FUNCTIONS`."""
# ALL_ATTENTION_FUNCTIONS.get_interface(...) or ALL_ATTENTION_FUNCTIONS[...]
if isinstance(node, ast.Call):
node = node.func
if not isinstance(node, (ast.Attribute, ast.Subscript)):
return False
return (
isinstance(node.value, ast.Name) and node.value.id == "ALL_ATTENTION_FUNCTIONS"
)
@cache
def interface_call(cls: type[nn.Module]) -> ast.Call | None:
"""The attention interface call in `cls.forward`, if it makes exactly one."""
try:
source = inspect.getsource(inspect.unwrap(cls.forward))
tree = ast.parse(textwrap.dedent(source))
except (AttributeError, OSError, SyntaxError, TypeError):
return None
# Find the names of the local variables that read from the interface lookup
names = set()
for node in ast.walk(tree):
if not isinstance(node, (ast.Assign, ast.AnnAssign)):
continue
if not _is_interface_lookup(node.value):
continue
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
names.update(t.id for t in targets if isinstance(t, ast.Name))
# Find the calls to those names, and return the one if there is exactly one
calls = [
node
for node in ast.walk(tree)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id in names
]
return calls[0] if len(calls) == 1 else None
def _resolve(node: ast.expr, module: nn.Module) -> object:
"""The value of `node` on `module`, for literals and `self.<attr>`."""
if isinstance(node, ast.Constant):
return node.value
if (
isinstance(node, ast.Attribute)
and isinstance(node.value, ast.Name)
and node.value.id == "self"
):
return getattr(module, node.attr, None)
return None
@dataclass
class AttentionFuser(BaseFuser):
"""A module that dispatches through the Transformers attention interface."""
redefines_forward: ClassVar[bool] = False
source_cls: str
"""Class of the HF module that dispatches (for logging)."""
scale_expr: ast.expr | None
"""Source of the `scaling=` the module hands the interface, if it hands one."""
def info(self, name: str) -> str:
return f"Found: {name} ({self.source_cls}) -> attention interface"
@classmethod
def match(
cls, graph: fx.Graph | None, module: nn.Module
) -> "AttentionFuser | None":
if (call := interface_call(type(module))) is None:
return None
scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"]
scale_expr = scaling[0] if len(scaling) == 1 else None
return cls(source_cls=type(module).__name__, scale_expr=scale_expr)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Whether `module` will actually dispatch to vLLM."""
config = getattr(module, "config", None)
# Only patched in the text config, this excludes attention based mm encoders
vllm_attn_impls = {VLLM_ATTN_IMPL, VLLM_MLA_ATTN_IMPL}
return getattr(config, "_attn_implementation", None) in vllm_attn_impls
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
return module
def layer_index(self, module: nn.Module) -> int | None:
"""The layer `module` computes attention for, if it declares one."""
layer_idx = getattr(module, "layer_idx", None)
return layer_idx if isinstance(layer_idx, int) else None
def scale(self, module: nn.Module) -> float | None:
"""The softmax scale `module` passes to the interface, or `None`."""
if self.scale_expr is None:
return None
scale = _resolve(self.scale_expr, module)
if not isinstance(scale, (int, float)) or isinstance(scale, bool):
expression = ast.unparse(self.scale_expr)
raise ValueError(
f"Cannot resolve attention scaling expression {expression!r} in "
f"{type(module).__name__}."
)
return float(scale)
@@ -20,11 +20,15 @@ if TYPE_CHECKING:
class BaseFuser(ABC):
"""A detected fusion and how to apply it.
`match` analyses the module *class* once (cached, see `get_fuser`); `fuse`
`match` analyses the module *class* once (cached, see `get_fusers`); `fuse`
then applies the fusion to an instance in `recursive_replace`, returning the
module to install in its place.
"""
redefines_forward: ClassVar[bool] = True
"""Whether `fuse` gives the module a different forward,
by rewriting its source or by returning a different module in its place."""
@abstractmethod
def info(self, name: str) -> str:
"""A human-readable description of the fusion at `name`, for logging."""
@@ -59,7 +59,7 @@ def _is_rms_norm(module: nn.Module) -> bool:
# Go via `get_fuser` because it caches
from vllm.model_executor.models.transformers.fuser import get_fuser
return isinstance(get_fuser(module), RMSNormFuser)
return get_fuser(module, RMSNormFuser) is not None
def _top_level_index(funcdef: ast.FunctionDef, node: ast.AST) -> int:
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.fused_moe import (
)
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
@@ -319,8 +320,8 @@ class MoEMixin(MixtureOfExperts):
# Store shared experts for later down projection adjustment
if shared_experts is not None:
hf_shared = shared_experts.shared_experts
glu_fuser = get_fuser(hf_shared)
down_name = getattr(glu_fuser, "down_name", None)
glu_fuser = get_fuser(hf_shared, GLUFuser)
down_name = glu_fuser and glu_fuser.down_name
if down_name is not None:
shared_down_projs.append((hf_shared, down_name))
# Prefer config, otherwise read it from fuser.