From 414057a3d3aa47915b10103e4a181604e0b3562f Mon Sep 17 00:00:00 2001 From: Thomas Parnell Date: Tue, 8 Sep 2026 15:54:07 +0200 Subject: [PATCH] [Bugfix] Apply attention sinks in the Transformers backend (#52156) Signed-off-by: Thomas Parnell Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- tests/models/transformers/test_backend.py | 66 +++++++++++++++---- .../models/transformers/__init__.py | 25 +++++++ .../models/transformers/base.py | 32 ++++----- .../models/transformers/fusers/attention.py | 35 +++++++++- 4 files changed, 127 insertions(+), 31 deletions(-) diff --git a/tests/models/transformers/test_backend.py b/tests/models/transformers/test_backend.py index 40c0434c92..3014bf05c8 100644 --- a/tests/models/transformers/test_backend.py +++ b/tests/models/transformers/test_backend.py @@ -6,6 +6,7 @@ import ast import contextlib import os import tempfile +from functools import partial from types import SimpleNamespace from typing import Any @@ -40,23 +41,21 @@ def get_model(arch: str) -> str: return model_info.default +def count_modules(model, cls: type) -> int: + return sum(isinstance(m, cls) for m in model.modules()) + + def get_num_fused(model) -> tuple[int, int]: from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, ) - glu = sum(isinstance(m, MergedColumnParallelLinear) for m in model.modules()) - qkv = sum(isinstance(m, QKVParallelLinear) for m in model.modules()) + glu = count_modules(model, MergedColumnParallelLinear) + qkv = count_modules(model, QKVParallelLinear) return glu, qkv -def count_mla_layers(model) -> int: - from vllm.model_executor.layers.attention import MLAAttention - - return sum(isinstance(m, MLAAttention) for m in model.attention_instances.values()) - - def check_implementation( runner_ref: type[HfRunner | VllmRunner], runner_test: type[VllmRunner], @@ -87,6 +86,12 @@ def check_implementation( assert num_glu == expected_glu * num_layers assert num_qkv == expected_qkv * num_layers + tp_size = kwargs_test.get("tensor_parallel_size", 1) + from vllm.model_executor.layers.attention import Attention + + counts = model_test.apply_model(partial(count_modules, cls=Attention)) + assert counts == [num_layers] * tp_size + outputs_test = model_test.generate_greedy_logprobs(*args) with runner_ref(model, **kwargs_ref) as model_ref: @@ -143,6 +148,42 @@ def test_hybrid_attention(vllm_runner: type[VllmRunner]) -> None: ) +def get_sinks(model) -> dict[int, torch.Tensor]: + """The sink tensor each attention layer was handed, keyed by layer index.""" + sinks = {} + for i, (prefix, fuser) in model.attention_fusers.items(): + if (sink := fuser.sinks(model.get_submodule(prefix))) is not None: + sinks[i] = sink.float().cpu() + return sinks + + +def test_sinks(hf_runner: type[HfRunner], vllm_runner: type[VllmRunner]) -> None: + """Learnable attention sinks must reach the attention layers. + + Only the attention impl can apply them, so if they are not passed to + `Attention` they are dropped and every softmax is subtly wrong. + """ + from vllm.platforms import current_platform + + if not current_platform.has_device_capability(90): + pytest.skip("Attention sinks need FlashAttention 3 or TRTLLM attention") + + model = "tiny-random/gpt-oss-bf16" + with vllm_runner( + model, model_impl="transformers", max_model_len=1024, enforce_eager=True + ) as model_test: + assert model_test.llm.llm_engine.model_config.using_transformers_backend() + sinks = model_test.apply_model(get_sinks)[0] + + with hf_runner(model, dtype="bfloat16") as model_ref: + layers = model_ref.model.model.get_decoder().layers + expected = {i: layer.self_attn.sinks.float() for i, layer in enumerate(layers)} + + assert sinks.keys() == expected.keys() + for i, sink in sinks.items(): + torch.testing.assert_close(sink, expected[i].cpu()) + + def test_mla(vllm_runner: type[VllmRunner], example_prompts: list[str]) -> None: import transformers from packaging.version import Version @@ -165,7 +206,10 @@ def test_mla(vllm_runner: type[VllmRunner], example_prompts: list[str]) -> None: model_config = model_test.llm.llm_engine.model_config assert model_config.using_transformers_backend() num_layers = model_config.hf_config.get_text_config().num_hidden_layers - assert model_test.apply_model(count_mla_layers) == [num_layers] + from vllm.model_executor.layers.attention import MLAAttention + + counts = model_test.apply_model(partial(count_modules, cls=MLAAttention)) + assert counts == [num_layers] outputs_test = model_test.generate_greedy_logprobs(*args) with vllm_runner(model, model_impl="auto") as model_ref: @@ -627,7 +671,7 @@ def test_attention_dispatch_is_matched(model_type: str): """Exactly the decoder layers' attention modules match an `AttentionFuser`. That match is what `recursive_replace` records, and so what - `create_attention_instances` attaches vLLM's attention layer to. + `_create_attention_instances` attaches vLLM's attention layer to. """ model = build_model(model_type) matched = { @@ -682,7 +726,7 @@ def test_attention_dispatch_is_required(monkeypatch: pytest.MonkeyPatch): ValueError, match="Layer 0 does not dispatch through the Transformers attention interface", ): - Base.create_attention_instances(model) + Base._create_attention_instances(model) @pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES) diff --git a/vllm/model_executor/models/transformers/__init__.py b/vllm/model_executor/models/transformers/__init__.py index 1ec5849408..cbfe551952 100644 --- a/vllm/model_executor/models/transformers/__init__.py +++ b/vllm/model_executor/models/transformers/__init__.py @@ -40,6 +40,29 @@ from vllm.multimodal import MULTIMODAL_REGISTRY if TYPE_CHECKING: import torch + from vllm.model_executor.layers.attention import Attention, MLAAttention + + +def check_sinks( + module: "torch.nn.Module", + self_attn: "Attention | MLAAttention", + s_aux: "torch.Tensor | None", +): + """Fail loudly if the model applies a sink the attention layer will not. + + Only the attention impl can fold a sink into the softmax denominator, so a sink + that never reached `Attention` is dropped and every softmax is subtly wrong. + """ + if s_aux is None or getattr(self_attn, "has_sink", False): + return + raise ValueError( + f"{type(module).__name__} applies attention sinks, but they were not passed " + f"to {type(self_attn).__name__}, so the output would be wrong. Either the " + "Transformers modeling backend could not find the parameter holding them, or " + "vLLM does not support sinks for this kind of attention. Please open an issue " + "at https://github.com/vllm-project/vllm/issues/new" + ) + def vllm_attention_forward( # Transformers args @@ -51,6 +74,7 @@ def vllm_attention_forward( **kwargs, ): self_attn = getattr(module, VLLM_ATTN_ATTR) + check_sinks(module, self_attn, kwargs.get("s_aux")) hidden = query.shape[-2] head_dim_qk = query.shape[-1] head_dim_v = value.shape[-1] @@ -80,6 +104,7 @@ def vllm_mla_attention_forward( **kwargs, ): self_attn = getattr(module, VLLM_ATTN_ATTR) + check_sinks(module, self_attn, kwargs.get("s_aux")) # [batch=1, heads, num_tokens, qk_head_dim] -> [num_tokens, heads, qk_head_dim] query = query.transpose(1, 2).flatten(0, 1) num_tokens, num_heads = query.shape[:2] diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py index 5ee2751a4f..89c6486e73 100644 --- a/vllm/model_executor/models/transformers/base.py +++ b/vllm/model_executor/models/transformers/base.py @@ -139,11 +139,11 @@ class Base( for the quantization machinery and loaders (e.g. bitsandbytes).""" self.fusers: dict[str, list[BaseFuser]] = {} """Module qualname -> the fusers applied to it, populated - by `recursive_replace` for `create_attention_instances`.""" + by `recursive_replace` for `_create_attention_instances`.""" self.attention_fusers: dict[int, tuple[str, AttentionFuser]] = {} """`layer_idx` -> the qualname and fuser of the module computing that layer's attention, populated by `recursive_replace` for - `create_attention_instances`.""" + `_create_attention_instances`.""" # Attrs for Eagle3 (see self.set_aux_hidden_state_layers) self._target_class: type[nn.Module] = nn.Module @@ -179,7 +179,7 @@ class Base( # Substitute remaining layers with vLLM's layers as needed self.recursive_replace() # Create attention instances for KV cache allocation - self.attention_instances = self.create_attention_instances() + self._create_attention_instances() # Initialize any parameters that have not had their modules replaced self.init_parameters(self.model) @@ -197,7 +197,7 @@ class Base( Patch the config to ensure that the model is created correctly: - Sets the attention implementation to "vllm" so the attention instances from - `create_attention_instances` are used + `_create_attention_instances` are used - Sets the dtype to the default torch dtype set by vLLM because Transformers uses the config dtype when creating the model """ @@ -570,11 +570,8 @@ class Base( _recursive_replace(self.model, prefix="model") - def create_attention_instances(self) -> dict[int, Attention]: - """ - Create `Attention` instances to inform KV cache allocation. - """ - attention_instances = {} + def _create_attention_instances(self): + """Create `Attention` instances to inform KV cache allocation.""" text_config = self.text_config attn_cls = self._get_attn_cls() @@ -614,13 +611,9 @@ class Base( self.parallel_config, arch_config ) head_size = arch_config.head_size - scale = attn_fuser.scale(attn_module) - # Default to Llama scale if AttentionFuser couldn't identify it - if scale is None: + if (scale := attn_fuser.scale(attn_module)) is None: + # Default to Llama scale if AttentionFuser couldn't identify it scale = head_size**-0.5 - num_kv_heads = self.model_config.get_num_kv_heads( - self.parallel_config, arch_config - ) kwargs = dict( num_heads=num_heads, @@ -651,13 +644,18 @@ class Base( else: kwargs.update( head_size=head_size, - num_kv_heads=num_kv_heads, + num_kv_heads=self.model_config.get_num_kv_heads( + self.parallel_config, arch_config + ), logits_soft_cap=logits_soft_cap, ) # Handle interleaved sliding window attention if layer_types and layer_types[i] == "sliding_attention": kwargs["per_layer_sliding_window"] = text_config.sliding_window + # Handle attention sinks + if (sinks := attn_fuser.sinks(attn_module)) is not None: + kwargs["sinks"] = sinks attn_instance = attn_cls(**kwargs) @@ -670,8 +668,6 @@ class Base( # layer identity into the traced graph and so costs one compiled # artifact per layer. setattr(attn_module, VLLM_ATTN_ATTR, attn_instance) - attention_instances[i] = attn_instance - return attention_instances def _get_attn_cls(self) -> type[AttentionLayerBase]: """Return the `Attention` class to use for this model's layers.""" diff --git a/vllm/model_executor/models/transformers/fusers/attention.py b/vllm/model_executor/models/transformers/fusers/attention.py index 1c6edd5e94..fc16527f15 100644 --- a/vllm/model_executor/models/transformers/fusers/attention.py +++ b/vllm/model_executor/models/transformers/fusers/attention.py @@ -9,9 +9,12 @@ from dataclasses import dataclass from functools import cache from typing import TYPE_CHECKING, ClassVar +import torch from torch import fx, nn +from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader from vllm.model_executor.models.transformers.fusers.base import BaseFuser +from vllm.model_executor.utils import set_weight_attrs if TYPE_CHECKING: from vllm.config import VllmConfig @@ -82,8 +85,10 @@ class AttentionFuser(BaseFuser): source_cls: str """Class of the HF module that dispatches (for logging).""" - scale_expr: ast.expr | None + scale_expr: ast.expr | None = None """Source of the `scaling=` the module hands the interface, if it hands one.""" + s_aux_expr: ast.expr | None = None + """Source of the `s_aux=` the module hands the interface, if it hands one.""" def info(self, name: str) -> str: return f"Found: {name} ({self.source_cls}) -> attention interface" @@ -96,7 +101,13 @@ class AttentionFuser(BaseFuser): return None scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"] scale_expr = scaling[0] if len(scaling) == 1 else None - return cls(source_cls=type(module).__name__, scale_expr=scale_expr) + s_aux = [kw.value for kw in call.keywords if kw.arg == "s_aux"] + s_aux_expr = s_aux[0] if len(s_aux) == 1 else None + return cls( + source_cls=type(module).__name__, + scale_expr=scale_expr, + s_aux_expr=s_aux_expr, + ) def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: """Whether `module` will actually dispatch to vLLM.""" @@ -108,6 +119,13 @@ class AttentionFuser(BaseFuser): def fuse( self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" ) -> nn.Module: + if (sinks := self.sinks(module)) is not None: + size = sinks.numel() // vllm_config.parallel_config.tensor_parallel_size + device = vllm_config.device_config.device + data = torch.empty(size, dtype=sinks.dtype, device=device) + sinks_param = nn.Parameter(data, requires_grad=False) + set_weight_attrs(sinks_param, {"weight_loader": sharded_weight_loader(0)}) + setattr(module, self.s_aux_expr.attr, sinks_param) return module def layer_index(self, module: nn.Module) -> int | None: @@ -127,3 +145,16 @@ class AttentionFuser(BaseFuser): f"{type(module).__name__}." ) return float(scale) + + def sinks(self, module: nn.Module) -> nn.Parameter | None: + """The per-head sink tensor `module` passes to the interface, or `None`.""" + if self.s_aux_expr is None: + return None + s_aux = _resolve(self.s_aux_expr, module) + if not isinstance(s_aux, (nn.Parameter, type(None))): + expression = ast.unparse(self.s_aux_expr) + raise ValueError( + f"Cannot resolve attention s_aux expression {expression!r} in " + f"{type(module).__name__}." + ) + return s_aux