[Bugfix] Apply attention sinks in the Transformers backend (#52156)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Thomas Parnell
2026-09-08 13:54:07 +00:00
committed by GitHub
co-authored by Claude Harry Mellor
parent 472d8c9125
commit 414057a3d3
4 changed files with 127 additions and 31 deletions
@@ -9,9 +9,12 @@ from dataclasses import dataclass
from functools import cache
from typing import TYPE_CHECKING, ClassVar
import torch
from torch import fx, nn
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
from vllm.model_executor.utils import set_weight_attrs
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -82,8 +85,10 @@ class AttentionFuser(BaseFuser):
source_cls: str
"""Class of the HF module that dispatches (for logging)."""
scale_expr: ast.expr | None
scale_expr: ast.expr | None = None
"""Source of the `scaling=` the module hands the interface, if it hands one."""
s_aux_expr: ast.expr | None = None
"""Source of the `s_aux=` the module hands the interface, if it hands one."""
def info(self, name: str) -> str:
return f"Found: {name} ({self.source_cls}) -> attention interface"
@@ -96,7 +101,13 @@ class AttentionFuser(BaseFuser):
return None
scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"]
scale_expr = scaling[0] if len(scaling) == 1 else None
return cls(source_cls=type(module).__name__, scale_expr=scale_expr)
s_aux = [kw.value for kw in call.keywords if kw.arg == "s_aux"]
s_aux_expr = s_aux[0] if len(s_aux) == 1 else None
return cls(
source_cls=type(module).__name__,
scale_expr=scale_expr,
s_aux_expr=s_aux_expr,
)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Whether `module` will actually dispatch to vLLM."""
@@ -108,6 +119,13 @@ class AttentionFuser(BaseFuser):
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
if (sinks := self.sinks(module)) is not None:
size = sinks.numel() // vllm_config.parallel_config.tensor_parallel_size
device = vllm_config.device_config.device
data = torch.empty(size, dtype=sinks.dtype, device=device)
sinks_param = nn.Parameter(data, requires_grad=False)
set_weight_attrs(sinks_param, {"weight_loader": sharded_weight_loader(0)})
setattr(module, self.s_aux_expr.attr, sinks_param)
return module
def layer_index(self, module: nn.Module) -> int | None:
@@ -127,3 +145,16 @@ class AttentionFuser(BaseFuser):
f"{type(module).__name__}."
)
return float(scale)
def sinks(self, module: nn.Module) -> nn.Parameter | None:
"""The per-head sink tensor `module` passes to the interface, or `None`."""
if self.s_aux_expr is None:
return None
s_aux = _resolve(self.s_aux_expr, module)
if not isinstance(s_aux, (nn.Parameter, type(None))):
expression = ast.unparse(self.s_aux_expr)
raise ValueError(
f"Cannot resolve attention s_aux expression {expression!r} in "
f"{type(module).__name__}."
)
return s_aux