[Bugfix] Apply attention sinks in the Transformers backend (#52156)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Thomas Parnell
2026-09-08 13:54:07 +00:00
committed by GitHub
co-authored by Claude Harry Mellor
parent 472d8c9125
commit 414057a3d3
4 changed files with 127 additions and 31 deletions
@@ -40,6 +40,29 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
if TYPE_CHECKING:
import torch
from vllm.model_executor.layers.attention import Attention, MLAAttention
def check_sinks(
module: "torch.nn.Module",
self_attn: "Attention | MLAAttention",
s_aux: "torch.Tensor | None",
):
"""Fail loudly if the model applies a sink the attention layer will not.
Only the attention impl can fold a sink into the softmax denominator, so a sink
that never reached `Attention` is dropped and every softmax is subtly wrong.
"""
if s_aux is None or getattr(self_attn, "has_sink", False):
return
raise ValueError(
f"{type(module).__name__} applies attention sinks, but they were not passed "
f"to {type(self_attn).__name__}, so the output would be wrong. Either the "
"Transformers modeling backend could not find the parameter holding them, or "
"vLLM does not support sinks for this kind of attention. Please open an issue "
"at https://github.com/vllm-project/vllm/issues/new"
)
def vllm_attention_forward(
# Transformers args
@@ -51,6 +74,7 @@ def vllm_attention_forward(
**kwargs,
):
self_attn = getattr(module, VLLM_ATTN_ATTR)
check_sinks(module, self_attn, kwargs.get("s_aux"))
hidden = query.shape[-2]
head_dim_qk = query.shape[-1]
head_dim_v = value.shape[-1]
@@ -80,6 +104,7 @@ def vllm_mla_attention_forward(
**kwargs,
):
self_attn = getattr(module, VLLM_ATTN_ATTR)
check_sinks(module, self_attn, kwargs.get("s_aux"))
# [batch=1, heads, num_tokens, qk_head_dim] -> [num_tokens, heads, qk_head_dim]
query = query.transpose(1, 2).flatten(0, 1)
num_tokens, num_heads = query.shape[:2]
+14 -18
View File
@@ -139,11 +139,11 @@ class Base(
for the quantization machinery and loaders (e.g. bitsandbytes)."""
self.fusers: dict[str, list[BaseFuser]] = {}
"""Module qualname -> the fusers applied to it, populated
by `recursive_replace` for `create_attention_instances`."""
by `recursive_replace` for `_create_attention_instances`."""
self.attention_fusers: dict[int, tuple[str, AttentionFuser]] = {}
"""`layer_idx` -> the qualname and fuser of the module computing that
layer's attention, populated by `recursive_replace` for
`create_attention_instances`."""
`_create_attention_instances`."""
# Attrs for Eagle3 (see self.set_aux_hidden_state_layers)
self._target_class: type[nn.Module] = nn.Module
@@ -179,7 +179,7 @@ class Base(
# Substitute remaining layers with vLLM's layers as needed
self.recursive_replace()
# Create attention instances for KV cache allocation
self.attention_instances = self.create_attention_instances()
self._create_attention_instances()
# Initialize any parameters that have not had their modules replaced
self.init_parameters(self.model)
@@ -197,7 +197,7 @@ class Base(
Patch the config to ensure that the model is created correctly:
- Sets the attention implementation to "vllm" so the attention instances from
`create_attention_instances` are used
`_create_attention_instances` are used
- Sets the dtype to the default torch dtype set by vLLM because Transformers
uses the config dtype when creating the model
"""
@@ -570,11 +570,8 @@ class Base(
_recursive_replace(self.model, prefix="model")
def create_attention_instances(self) -> dict[int, Attention]:
"""
Create `Attention` instances to inform KV cache allocation.
"""
attention_instances = {}
def _create_attention_instances(self):
"""Create `Attention` instances to inform KV cache allocation."""
text_config = self.text_config
attn_cls = self._get_attn_cls()
@@ -614,13 +611,9 @@ class Base(
self.parallel_config, arch_config
)
head_size = arch_config.head_size
scale = attn_fuser.scale(attn_module)
# Default to Llama scale if AttentionFuser couldn't identify it
if scale is None:
if (scale := attn_fuser.scale(attn_module)) is None:
# Default to Llama scale if AttentionFuser couldn't identify it
scale = head_size**-0.5
num_kv_heads = self.model_config.get_num_kv_heads(
self.parallel_config, arch_config
)
kwargs = dict(
num_heads=num_heads,
@@ -651,13 +644,18 @@ class Base(
else:
kwargs.update(
head_size=head_size,
num_kv_heads=num_kv_heads,
num_kv_heads=self.model_config.get_num_kv_heads(
self.parallel_config, arch_config
),
logits_soft_cap=logits_soft_cap,
)
# Handle interleaved sliding window attention
if layer_types and layer_types[i] == "sliding_attention":
kwargs["per_layer_sliding_window"] = text_config.sliding_window
# Handle attention sinks
if (sinks := attn_fuser.sinks(attn_module)) is not None:
kwargs["sinks"] = sinks
attn_instance = attn_cls(**kwargs)
@@ -670,8 +668,6 @@ class Base(
# layer identity into the traced graph and so costs one compiled
# artifact per layer.
setattr(attn_module, VLLM_ATTN_ATTR, attn_instance)
attention_instances[i] = attn_instance
return attention_instances
def _get_attn_cls(self) -> type[AttentionLayerBase]:
"""Return the `Attention` class to use for this model's layers."""
@@ -9,9 +9,12 @@ from dataclasses import dataclass
from functools import cache
from typing import TYPE_CHECKING, ClassVar
import torch
from torch import fx, nn
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
from vllm.model_executor.utils import set_weight_attrs
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -82,8 +85,10 @@ class AttentionFuser(BaseFuser):
source_cls: str
"""Class of the HF module that dispatches (for logging)."""
scale_expr: ast.expr | None
scale_expr: ast.expr | None = None
"""Source of the `scaling=` the module hands the interface, if it hands one."""
s_aux_expr: ast.expr | None = None
"""Source of the `s_aux=` the module hands the interface, if it hands one."""
def info(self, name: str) -> str:
return f"Found: {name} ({self.source_cls}) -> attention interface"
@@ -96,7 +101,13 @@ class AttentionFuser(BaseFuser):
return None
scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"]
scale_expr = scaling[0] if len(scaling) == 1 else None
return cls(source_cls=type(module).__name__, scale_expr=scale_expr)
s_aux = [kw.value for kw in call.keywords if kw.arg == "s_aux"]
s_aux_expr = s_aux[0] if len(s_aux) == 1 else None
return cls(
source_cls=type(module).__name__,
scale_expr=scale_expr,
s_aux_expr=s_aux_expr,
)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Whether `module` will actually dispatch to vLLM."""
@@ -108,6 +119,13 @@ class AttentionFuser(BaseFuser):
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
if (sinks := self.sinks(module)) is not None:
size = sinks.numel() // vllm_config.parallel_config.tensor_parallel_size
device = vllm_config.device_config.device
data = torch.empty(size, dtype=sinks.dtype, device=device)
sinks_param = nn.Parameter(data, requires_grad=False)
set_weight_attrs(sinks_param, {"weight_loader": sharded_weight_loader(0)})
setattr(module, self.s_aux_expr.attr, sinks_param)
return module
def layer_index(self, module: nn.Module) -> int | None:
@@ -127,3 +145,16 @@ class AttentionFuser(BaseFuser):
f"{type(module).__name__}."
)
return float(scale)
def sinks(self, module: nn.Module) -> nn.Parameter | None:
"""The per-head sink tensor `module` passes to the interface, or `None`."""
if self.s_aux_expr is None:
return None
s_aux = _resolve(self.s_aux_expr, module)
if not isinstance(s_aux, (nn.Parameter, type(None))):
expression = ast.unparse(self.s_aux_expr)
raise ValueError(
f"Cannot resolve attention s_aux expression {expression!r} in "
f"{type(module).__name__}."
)
return s_aux