[Transformers backend] Find attention with a fuser and attach vLLM's layer to it (#54941)

Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Thomas Ortner
2026-09-04 16:26:01 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 99a1ab8e4c
commit 8ad2076c4a
14 changed files with 521 additions and 150 deletions
+32 -15
View File
@@ -72,19 +72,26 @@ This means that, with the Transformers modeling backend for vLLM, new models can
This section details the necessary modifications to make to a Transformers compatible custom model that make it compatible with the Transformers modeling backend for vLLM. (We assume that a Transformers compatible custom model has already been created, see [Transformers - Customizing models](https://huggingface.co/docs/transformers/en/custom_models)).
To make your model compatible with the Transformers modeling backend, it needs:
To make your model compatible with the Transformers modeling backend:
1. `kwargs` passed down through all modules from `MyModel` to `MyAttention`.
- If your model is encoder-only:
1. Add `is_causal = False` to `MyAttention`.
- If your model is mixture-of-experts (MoE):
1. Your sparse MoE block must have an attribute called `experts`.
2. The class of `experts` (`MyExperts`) must either:
- Inherit from `nn.ModuleList` (naive).
- Or contain all 3D `nn.Parameters` (packed).
3. `MyExperts.forward` must accept `hidden_states`, `top_k_index`, `top_k_weights`.
2. `MyAttention` must use `ALL_ATTENTION_FUNCTIONS` to call attention.
3. `MyModel` must contain `_supports_attention_backend = True`.
1. `MyAttention` must use `ALL_ATTENTION_FUNCTIONS` to call attention.
- It must make exactly one such call. vLLM attaches one `Attention` layer per `MyAttention` module.
- `MyAttention` must contain a unique `layer_idx`. vLLM keys its KV cache using this index.
- Pass `scaling=` to the interface if your scale is not `head_size**-0.5`. vLLM reads it from the call to the attention interface.
2. If your model is encoder-only:
1. Add `is_causal = False` to `MyAttention`.
3. If your model is mixture-of-experts (MoE):
1. Your sparse MoE block must have an attribute called `experts`.
2. The class of `experts` (`MyExperts`) must either:
- Inherit from `nn.ModuleList` (naive).
- Or contain all 3D `nn.Parameters` (packed).
3. `MyExperts.forward` must accept `hidden_states`, `top_k_index`, `top_k_weights`.
!!! note
`MyModel` no longer needs `_supports_attention_backend = True`, and `kwargs` no
longer need to be passed down through every module from `MyModel` to
`MyAttention`. vLLM reaches its attention layer through `MyAttention` itself, so
all it asks is that `MyAttention` dispatches through the interface.
<details class="code">
<summary>modeling_my_model.py</summary>
@@ -97,14 +104,24 @@ from torch import nn
class MyAttention(nn.Module):
is_causal = False # Only do this for encoder-only models
def __init__(self, config, layer_idx):
...
self.config = config
self.layer_idx = layer_idx
self.scaling = self.head_dim**-0.5
...
def forward(self, hidden_states, **kwargs):
...
attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface(
self.config._attn_implementation, eager_attention_forward
)
attn_output, attn_weights = attention_interface(
self,
query_states,
key_states,
value_states,
scaling=self.scaling,
**kwargs,
)
...
@@ -127,7 +144,7 @@ class MySparseMoEBlock(nn.Module):
...
class MyModel(PreTrainedModel):
_supports_attention_backend = True
...
```
</details>
@@ -135,7 +152,7 @@ class MyModel(PreTrainedModel):
Here is what happens in the background when this model is loaded:
1. The config is loaded.
2. `MyModel` Python class is loaded from the `auto_map` in config, and we check that the model `is_backend_compatible()`.
2. `MyModel` Python class is loaded from the `auto_map` in config, and we check that the model `_can_set_attn_implementation()`.
3. `MyModel` is loaded into one of the Transformers modeling backend classes in [vllm/model_executor/models/transformers](../../vllm/model_executor/models/transformers) which sets `self.config._attn_implementation = "vllm"` so that vLLM's attention layer is used.
That's it!
+41 -35
View File
@@ -10,7 +10,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fuser import get_fuser, get_fusers
from vllm.model_executor.models.transformers.fusers import (
GLUFuser,
PackedQKVFuser,
@@ -307,7 +307,7 @@ class PerHeadSplitAttention(nn.Module):
class FakeSelfAttn(nn.Module):
"""Stand-in for the vLLM `Attention` looked up in `attention_instances`."""
"""Stand-in for the vLLM `Attention` attached to the dispatching module."""
def __init__(self):
super().__init__()
@@ -328,9 +328,9 @@ class FakeMQASelfAttn(FakeSelfAttn):
@pytest.fixture(autouse=True)
def _clear_fuser_cache():
get_fuser.cache_clear()
get_fusers.cache_clear()
yield
get_fuser.cache_clear()
get_fusers.cache_clear()
def _apply_glu_fuser_with_stubs(module: nn.Module, fuser: GLUFuser):
@@ -392,7 +392,7 @@ def _apply_packed_qkv_fuser_with_stubs(module: nn.Module, fuser: PackedQKVFuser)
def test_detects_and_rewrites_glu(mlp_cls, bias):
with torch.device("meta"):
meta = mlp_cls(bias=bias)
fuser = get_fuser(meta)
fuser = get_fuser(meta, GLUFuser)
assert isinstance(fuser, GLUFuser)
assert (
fuser.gate_name,
@@ -427,9 +427,9 @@ def test_glu_identifies_down_projection():
matches the column-parallel merged gate/up; `None` when there is no such
projection to force (fusion of gate/up still applies)."""
with torch.device("meta"):
assert get_fuser(GLUMLP()).down_name == "down_proj"
assert get_fuser(ReversedGLUMLP()).down_name == "down_proj"
assert get_fuser(NoDownGLU()).down_name is None
assert get_fuser(GLUMLP(), GLUFuser).down_name == "down_proj"
assert get_fuser(ReversedGLUMLP(), GLUFuser).down_name == "down_proj"
assert get_fuser(NoDownGLU(), GLUFuser).down_name is None
@pytest.mark.parametrize("attn_cls", [FakeAttention, ReversedFakeAttention])
@@ -439,7 +439,7 @@ def test_detects_and_rewrites_qkv(attn_cls, kv_heads):
pytest.skip("MHA q/k/v assignment is order-based by design")
with torch.device("meta"):
meta = attn_cls(kv_heads=kv_heads)
fuser = get_fuser(meta)
fuser = get_fuser(meta, QKVFuser)
assert isinstance(fuser, QKVFuser)
# q (sharded differently under TP) must be identified exactly; k/v may be
# swapped for non-canonical compute order, which is numerically consistent
@@ -467,27 +467,27 @@ def test_detects_and_rewrites_qkv(attn_cls, kv_heads):
for p in real.parameters():
nn.init.normal_(p, std=0.05)
x = torch.randn(1, 5, 32)
attention_instances = {3: FakeSelfAttn()}
expected, _ = real(x, attention_instances=attention_instances)
real.attn = FakeSelfAttn()
expected, _ = real(x)
fused = _apply_qkv_fuser_with_stubs(real, fuser)
# Fusion is in place: the module keeps its class and other attributes
assert fused is real and type(fused) is attn_cls
assert fused.layer_idx == 3 and fused.is_causal and fused.config is not None
out, _ = fused(x, attention_instances=attention_instances)
out, _ = fused(x)
torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5)
def test_qkv_identifies_output_projection():
with torch.device("meta"):
assert get_fuser(FakeAttention()).o_name == "o_proj"
assert get_fuser(ReversedFakeAttention()).o_name == "o_proj"
assert get_fuser(ExtraProjAttention()).o_name == "o_proj"
assert get_fuser(FakeAttention(), QKVFuser).o_name == "o_proj"
assert get_fuser(ReversedFakeAttention(), QKVFuser).o_name == "o_proj"
assert get_fuser(ExtraProjAttention(), QKVFuser).o_name == "o_proj"
# Norm children (q_norm/k_norm) must not disturb o_proj identification.
assert get_fuser(QKNormAttention()).o_name == "o_proj"
assert get_fuser(PerHeadQKNormAttention()).o_name == "o_proj"
assert get_fuser(QKNormAttention(), QKVFuser).o_name == "o_proj"
assert get_fuser(PerHeadQKNormAttention(), QKVFuser).o_name == "o_proj"
# A module between o_proj and the return is transparent.
assert get_fuser(ResidDropoutAttention()).o_name == "o_proj"
assert get_fuser(ResidDropoutAttention(), QKVFuser).o_name == "o_proj"
@pytest.mark.parametrize("kv_heads", [1, 2])
@@ -498,7 +498,7 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
checkpoint weight as-is, and shards q by heads while replicating k/v."""
with torch.device("meta"):
meta = PackedQKVAttention(kv_heads=kv_heads)
fuser = get_fuser(meta)
fuser = get_fuser(meta, PackedQKVFuser)
assert isinstance(fuser, PackedQKVFuser)
assert (fuser.qkv_name, fuser.o_name) == ("c_attn", "c_proj")
assert (fuser.q_size, fuser.kv_size) == (32, 8 * kv_heads)
@@ -513,14 +513,14 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
for p in real.parameters():
nn.init.normal_(p, std=0.05)
x = torch.randn(1, 5, 32)
attention_instances = {3: FakeMQASelfAttn()}
expected, _ = real(x, attention_instances=attention_instances)
real.attn = FakeMQASelfAttn()
expected, _ = real(x)
fused = _apply_packed_qkv_fuser_with_stubs(real, fuser)
# Fusion is in place: the module keeps its class and other attributes
assert fused is real and type(fused) is PackedQKVAttention
assert fused.layer_idx == 3 and fused.is_causal
out, _ = fused(x, attention_instances=attention_instances)
out, _ = fused(x)
torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5)
@@ -528,28 +528,28 @@ def test_per_head_split_is_not_packed_qkv():
"""The split must consume the whole projection, else its sizes are head
widths and re-sharding by them would be wrong."""
with torch.device("meta"):
assert get_fuser(PerHeadSplitAttention()) is None
assert get_fuser(PerHeadSplitAttention(), PackedQKVFuser) is None
def test_fuser_is_cached_per_class_and_structure():
with torch.device("meta"):
fuser_a = get_fuser(GLUMLP())
fuser_b = get_fuser(GLUMLP())
fuser_a = get_fuser(GLUMLP(), GLUFuser)
fuser_b = get_fuser(GLUMLP(), GLUFuser)
assert fuser_a is fuser_b
assert any(key[0] is GLUMLP for key in get_fuser.cache)
assert any(key[0] is GLUMLP for key in get_fusers.cache)
@pytest.mark.parametrize("cls", [NotAnMLP, UntraceableMLP])
def test_non_matching_modules_return_none(cls):
with torch.device("meta"):
module = cls()
assert get_fuser(module) is None
assert get_fuser(module, GLUFuser) is None
def test_untraceable_tail_still_fuses():
with torch.device("meta"):
meta = UntraceableTailGLUMLP()
fuser = get_fuser(meta)
fuser = get_fuser(meta, GLUFuser)
assert isinstance(fuser, GLUFuser)
# Numerics: the live tail must survive the rewrite
@@ -566,8 +566,8 @@ def test_weight_mappings_are_scoped_to_fused_prefixes():
from vllm.model_executor.models.utils import WeightsMapper
with torch.device("meta"):
glu_fuser = get_fuser(GLUMLP())
qkv_fuser = get_fuser(FakeAttention())
glu_fuser = get_fuser(GLUMLP(), GLUFuser)
qkv_fuser = get_fuser(FakeAttention(), QKVFuser)
mapper = WeightsMapper()
for prefix in ("model.layers.0.mlp", "model.layers.1.mlp"):
@@ -620,7 +620,7 @@ def test_weight_mappings_are_scoped_to_fused_prefixes():
def test_unfusable_modules_are_not_fused(cls, default_vllm_config):
with torch.device("meta"):
module = cls()
fuser = get_fuser(module)
fuser = get_fuser(module, GLUFuser)
# Either no pattern matches the class, or this instance fails validation
# (`recursive_replace` gates fusion and its weight mappings on `validate`)
assert fuser is None or not fuser.validate(module, default_vllm_config)
@@ -654,9 +654,15 @@ def _wider_model_config(head_dim: int) -> SimpleNamespace:
@pytest.mark.parametrize(
"cls, fuser_module", [(FakeAttention, qkv), (PackedQKVAttention, packed_qkv)]
"cls, fuser_module, fuser_cls",
[
(FakeAttention, qkv, QKVFuser),
(PackedQKVAttention, packed_qkv, PackedQKVFuser),
],
)
def test_head_counts_come_from_the_module_not_the_model(cls, fuser_module, monkeypatch):
def test_head_counts_come_from_the_module_not_the_model(
cls, fuser_module, fuser_cls, monkeypatch
):
"""A layer narrower than the model-wide head size must not be miscounted.
On a heterogeneous checkpoint (Gemma 4) the model-wide head size is the
@@ -680,7 +686,7 @@ def test_head_counts_come_from_the_module_not_the_model(cls, fuser_module, monke
monkeypatch.setattr(
fuser_module, "replace_linear_class", lambda *a, **kw: nn.Identity()
)
fuser = get_fuser(module)
fuser = get_fuser(module, fuser_cls)
assert fuser is not None and fuser.validate(module, vllm_config)
fuser.update_attrs(module, "model.layers.0.self_attn", vllm_config)
@@ -700,5 +706,5 @@ def test_validate_accepts_a_layer_the_model_wide_head_size_would_reject():
# kv width is 24, not a multiple of the model-wide 16, but is of this
# layer's 8.
fuser = get_fuser(module)
fuser = get_fuser(module, QKVFuser)
assert fuser is not None and fuser.validate(module, vllm_config)
+1 -1
View File
@@ -158,4 +158,4 @@ class GLU(nn.Module):
def test_non_mla_is_not_matched():
with torch.device("meta"):
assert not isinstance(get_fuser(GLU()), MLAFuser)
assert get_fuser(GLU(), MLAFuser) is None
@@ -162,7 +162,7 @@ class UntraceableGatedRMSNorm(RMSNorm):
)
def test_detects_rms_norm_variants(cls, eps, zero_centered):
with torch.device("meta"):
fuser = get_fuser(cls(16, eps=eps))
fuser = get_fuser(cls(16, eps=eps), RMSNormFuser)
assert isinstance(fuser, RMSNormFuser)
assert fuser.zero_centered == zero_centered
@@ -171,7 +171,7 @@ def test_detects_rms_norm_variants(cls, eps, zero_centered):
def test_non_rms_norms_are_not_matched(cls):
with torch.device("meta"):
module = cls(16) if cls is nn.LayerNorm else cls()
assert not isinstance(get_fuser(module), RMSNormFuser)
assert get_fuser(module, RMSNormFuser) is None
@pytest.mark.parametrize(
@@ -179,7 +179,7 @@ def test_non_rms_norms_are_not_matched(cls):
)
def test_gated_rms_norm_is_not_fused(cls):
with torch.device("meta"):
assert not isinstance(get_fuser(cls()), RMSNormFuser)
assert get_fuser(cls(), RMSNormFuser) is None
@pytest.mark.parametrize(
@@ -196,7 +196,7 @@ def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_c
with torch.device("meta"):
module = cls()
fuser = get_fuser(module)
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
from vllm.model_executor.models.transformers.fusers.rms_norm import (
TPAwareNormMixin,
@@ -221,7 +221,7 @@ def test_weightless_norm_has_no_hidden_size(default_vllm_config):
matching the unfused norm and vLLM's native `RMSNorm(hidden_size=head_dim)`.
"""
module = WeightlessRMSNorm(72)
built = get_fuser(module).fuse(module, "norm", default_vllm_config)
built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
assert built.hidden_size == 0
built.tp_size = 2 # emulate TP=2 without a real process group
@@ -236,7 +236,7 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config):
with torch.device("meta"):
module = torch.nn.RMSNorm(16) # forward is a single `F.rms_norm` call
fuser = get_fuser(module)
fuser = get_fuser(module, RMSNormFuser)
assert isinstance(fuser, RMSNormFuser)
assert not fuser.zero_centered
vllm_config = SimpleNamespace(model_config=SimpleNamespace(dtype=torch.float32))
@@ -251,7 +251,8 @@ def test_eps_is_derived_per_instance(default_vllm_config):
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = RMSNorm(16, eps=eps)
built = get_fuser(module).fuse(module, "norm", default_vllm_config)
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps
@@ -262,7 +263,7 @@ def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = cls(16, eps=eps)
fuser = get_fuser(module)
fuser = get_fuser(module, RMSNormFuser)
assert fuser.eps_attr == cls.attr
built = fuser.fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps
@@ -274,7 +275,7 @@ def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplo
logger = "vllm.model_executor.models.transformers.fusers.rms_norm"
with caplog.at_level("DEBUG", logger=logger), torch.device("meta"):
module = LiteralEpsRMSNorm()
fuser = get_fuser(module)
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
assert fuser.eps_attr is None
assert built.variance_epsilon == 1e-4
@@ -287,7 +288,7 @@ def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
with torch.device("meta"):
module = AmbiguousEpsRMSNorm(16, eps=1e-6)
before = dict(vars(module))
fuser = get_fuser(module)
fuser = get_fuser(module, RMSNormFuser)
assert fuser.eps_attr == "variance_epsilon"
assert vars(module) == before
+142
View File
@@ -2,9 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test the functionality of the Transformers modeling backend."""
import ast
import contextlib
import os
import tempfile
from types import SimpleNamespace
from typing import Any
import pytest
@@ -15,6 +17,8 @@ from transformers import AutoConfig, AutoModel, PretrainedConfig
from vllm.config import ModelConfig, VllmConfig
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.transformers.base import Base
from vllm.model_executor.models.transformers.fusers import AttentionFuser
from vllm.model_executor.models.transformers.fusers.attention import VLLM_ATTN_IMPL
from vllm.model_executor.models.transformers.multimodal import MultiModalMixin
from vllm.model_executor.models.utils import StageMissingLayer
@@ -583,3 +587,141 @@ def test_marking_skipped_without_tokenizer():
"""
vision_tower = build_marked_model(0, skip_tokenizer_init=True).vision_tower
assert not isinstance(vision_tower, StageMissingLayer)
NUM_LAYERS = 4
def build_model(model_type: str, **overrides) -> nn.Module:
"""A tiny HF model of `NUM_LAYERS` layers, built on the meta device."""
try:
config = AutoConfig.for_model(
model_type,
num_hidden_layers=NUM_LAYERS,
hidden_size=64,
intermediate_size=128,
num_attention_heads=4,
num_key_value_heads=2,
head_dim=16,
vocab_size=VOCAB_SIZE,
**overrides,
)
except ValueError:
pytest.skip(f"The installed transformers has no {model_type!r} model")
with torch.device("meta"):
return AutoModel.from_config(config)
ATTENTION_MODEL_TYPES = [
# The default `head_size**-0.5` happens to be right here
"llama",
# Scales the query by a learnable per-dim weight, so it declares 1.0
"gemma4_text",
# MLA, and declares yarn's `mscale`
"deepseek_v3",
]
@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
def test_attention_dispatch_is_matched(model_type: str):
"""Exactly the decoder layers' attention modules match an `AttentionFuser`.
That match is what `recursive_replace` records, and so what
`create_attention_instances` attaches vLLM's attention layer to.
"""
model = build_model(model_type)
matched = {
name
for name, module in model.named_modules()
if AttentionFuser.match(None, module) is not None
}
assert matched == {f"layers.{i}.self_attn" for i in range(NUM_LAYERS)}
def test_attention_layer_index_is_the_modules_own():
"""The layer served comes from the module, not from its position in the stack.
LongCat Flash gives each decoder layer two attention sublayers numbered
`2i` and `2i + 1`, so `num_hidden_layers` is twice the length of the stack
and the enclosing layer's position is not the index the KV cache is keyed by.
A vision tower is excluded separately, by `validate`: only the text config is
patched to dispatch to vLLM, so a tower's attention is left to Transformers.
"""
model = build_model("longcat_flash", num_layers=2)
text_config = model.config.get_text_config()
assert len(model.layers) * 2 == text_config.num_hidden_layers
attentions = [
model.get_submodule(f"layers.{i}.self_attn.{j}") for i in (0, 1) for j in (0, 1)
]
fusers = [AttentionFuser.match(None, attn) for attn in attentions]
assert all(fuser is not None for fuser in fusers)
assert [f.layer_index(a) for f, a in zip(fusers, attentions)] == [0, 1, 2, 3]
# Not dispatching to vLLM yet, as a vision tower never would be. The vLLM
# config is not consulted, only the one the module was built with.
assert not any(f.validate(a, None) for f, a in zip(fusers, attentions))
text_config._attn_implementation = VLLM_ATTN_IMPL
assert all(f.validate(a, None) for f, a in zip(fusers, attentions))
def test_attention_dispatch_is_required(monkeypatch: pytest.MonkeyPatch):
"""Models selected by the permissive registry fail clearly without dispatch."""
monkeypatch.setattr(
"vllm.model_executor.models.transformers.base.get_pp_indices",
lambda *_: (0, 1),
)
model = SimpleNamespace(
text_config=SimpleNamespace(num_hidden_layers=1),
pp_group=SimpleNamespace(rank_in_group=0, world_size=1),
attention_fusers={},
_get_attn_cls=lambda: None,
)
with pytest.raises(
ValueError,
match="Layer 0 does not dispatch through the Transformers attention interface",
):
Base.create_attention_instances(model)
@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
def test_attention_scale_is_the_declared_one(model_type: str):
"""The scale comes off the module, not from the `head_size**-0.5` default.
Two of the three model types declare something the default would silently
replace, so they are what makes this more than a tautology.
"""
attention = build_model(model_type).layers[0].self_attn
fuser = AttentionFuser.match(None, attention)
assert fuser is not None
assert fuser.scale(attention) == attention.scaling
def test_attention_scale_is_the_argument_not_the_attribute():
"""OPT applies the scale to the query itself, then declares a literal 1.0.
Its `self.scaling` is the `head_size**-0.5` it has already applied, so
reading the attribute rather than the argument the module hands the
interface would scale twice.
"""
model = build_model("opt", ffn_dim=128, word_embed_proj_dim=64)
attention = model.get_submodule("decoder.layers.0.self_attn")
fuser = AttentionFuser.match(None, attention)
assert fuser is not None
assert fuser.scale(attention) == 1.0 != attention.scaling
def test_attention_scale_rejects_unresolvable_expression():
"""Unresolvable declared scales must not silently use the Llama default."""
class Attention(nn.Module):
head_dim = 16
fuser = AttentionFuser(
source_cls="Attention",
scale_expr=ast.parse("self.head_dim**-0.5 * self.factor", mode="eval").body,
)
with pytest.raises(ValueError, match="Cannot resolve attention scaling expression"):
fuser.scale(Attention())
+4 -1
View File
@@ -1253,7 +1253,10 @@ class _ModelRegistry:
"'auto_map' (relevant if the model is custom)."
)
if not model_module.is_backend_compatible():
if not (
model_module.is_backend_compatible()
or model_module._can_set_attn_implementation()
):
if model_config.model_impl != "transformers":
return None
@@ -21,7 +21,7 @@ from typing import TYPE_CHECKING
import torch.nn.functional as F
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from vllm.model_executor.models.transformers.base import Base
from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR, Base
from vllm.model_executor.models.transformers.causal import CausalMixin
from vllm.model_executor.models.transformers.legacy import LegacyMixin
from vllm.model_executor.models.transformers.moe import MoEMixin
@@ -40,8 +40,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
if TYPE_CHECKING:
import torch
from vllm.model_executor.layers.attention import Attention, MLAAttention
def vllm_attention_forward(
# Transformers args
@@ -50,15 +48,9 @@ def vllm_attention_forward(
key: "torch.Tensor",
value: "torch.Tensor",
attention_mask: "torch.Tensor",
# Transformers kwargs
scaling: float | None = None,
# vLLM kwargs
attention_instances: "dict[int, Attention] | None" = None,
**kwargs,
):
self_attn = attention_instances[module.layer_idx]
if scaling is not None:
self_attn.impl.scale = float(scaling)
self_attn = getattr(module, VLLM_ATTN_ATTR)
hidden = query.shape[-2]
head_dim_qk = query.shape[-1]
head_dim_v = value.shape[-1]
@@ -85,13 +77,9 @@ def vllm_mla_attention_forward(
kv_c_normed: "torch.Tensor",
k_pe: "torch.Tensor",
attention_mask: "torch.Tensor",
# Transformers kwargs
scaling: float | None = None,
# vLLM kwargs
attention_instances: "dict[int, MLAAttention] | None" = None,
**kwargs,
):
self_attn = attention_instances[module.layer_idx]
self_attn = getattr(module, VLLM_ATTN_ATTR)
# [batch=1, heads, num_tokens, qk_head_dim] -> [num_tokens, heads, qk_head_dim]
query = query.transpose(1, 2).flatten(0, 1)
num_tokens, num_heads = query.shape[:2]
+78 -34
View File
@@ -57,7 +57,11 @@ from vllm.model_executor.models.interfaces import (
)
from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
from vllm.model_executor.models.transformers.fusers import MLAFuser
from vllm.model_executor.models.transformers.fusers import AttentionFuser, MLAFuser
from vllm.model_executor.models.transformers.fusers.attention import (
VLLM_ATTN_IMPL,
VLLM_MLA_ATTN_IMPL,
)
from vllm.model_executor.models.transformers.utils import (
attrsetter,
can_enable_torch_compile,
@@ -73,7 +77,6 @@ from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
@@ -86,6 +89,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
VLLM_ATTN_ATTR = "attn"
"""Attribute in which to store the vLLM `Attention` instance that serves a Transformers
attention module. `"attn"` is chosen deliberately to match the mappings generated by
`maybe_remap_kv_scale_name`."""
class PreTrainedModelClasses(NamedTuple):
decoder: type["PreTrainedModel"]
@@ -129,9 +137,13 @@ class Base(
self.packed_modules_mapping: dict[str, list[str]] = {}
"""Fused module -> constituent projections, populated by `recursive_replace`
for the quantization machinery and loaders (e.g. bitsandbytes)."""
self.fusers: dict[str, BaseFuser] = {}
"""Module qualname -> the fuser applied to it, populated
self.fusers: dict[str, list[BaseFuser]] = {}
"""Module qualname -> the fusers applied to it, populated
by `recursive_replace` for `create_attention_instances`."""
self.attention_fusers: dict[int, tuple[str, AttentionFuser]] = {}
"""`layer_idx` -> the qualname and fuser of the module computing that
layer's attention, populated by `recursive_replace` for
`create_attention_instances`."""
# Attrs for Eagle3 (see self.set_aux_hidden_state_layers)
self._target_class: type[nn.Module] = nn.Module
@@ -189,7 +201,7 @@ class Base(
- Sets the dtype to the default torch dtype set by vLLM because Transformers
uses the config dtype when creating the model
"""
self.text_config._attn_implementation = "vllm"
self.text_config._attn_implementation = VLLM_ATTN_IMPL
self.config.dtype = torch.get_default_dtype()
@contextmanager
@@ -475,9 +487,22 @@ class Base(
vocab_embeddings = self._vocab_embeddings()
def register_fusion(fuser: BaseFuser, prefix: str):
def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module):
"""Register a fused layer's mappings just before it is built."""
self.fusers[prefix] = fuser
self.fusers.setdefault(prefix, []).append(fuser)
if (
isinstance(fuser, AttentionFuser)
and (index := fuser.layer_index(module)) is not None
):
if index in self.attention_fusers:
other, _ = self.attention_fusers[index]
raise ValueError(
f"Layer {index} dispatches attention from more than one "
f"module ({other} and {prefix}), which one vLLM attention "
"layer cannot serve."
)
self.attention_fusers[index] = (prefix, fuser)
orig_to_new_stacked = fuser.orig_to_new_stacked(prefix)
self.hf_to_vllm_mapper.orig_to_new_stacked.update(orig_to_new_stacked)
@@ -528,10 +553,11 @@ class Base(
new_module = replace_embedding_class(
child_module, self.quant_config, prefix=qual_name
)
elif (fuser := fusers[child_module]) is not None:
register_fusion(fuser, qual_name)
new_module = fuser.fuse(child_module, qual_name, self.vllm_config)
logger.info_once(fuser.info(child_name))
elif child_module_fusers := fusers[child_module]:
for fuser in child_module_fusers:
register_fusion(fuser, qual_name, child_module)
new_module = fuser.fuse(new_module, qual_name, self.vllm_config)
logger.info_once(fuser.info(child_name))
_recursive_replace(new_module, prefix=qual_name)
elif not isinstance(child_module, MoERunner):
# MoERunner can contain aliases of shared experts and gates,
@@ -548,20 +574,14 @@ class Base(
"""
Create `Attention` instances to inform KV cache allocation.
"""
mla_fusers = {}
attention_instances = {}
text_config = self.text_config
attn_cls = self._get_attn_cls()
# kv_lora_rank indicates that this is an MLA model
if getattr(text_config, "kv_lora_rank", None) is not None:
mla_fusers = {
extract_layer_index(prefix): (prefix, fuser)
for prefix, fuser in self.fusers.items()
if isinstance(fuser, MLAFuser)
}
if attn_cls is MLAAttention:
text_config._attn_implementation = "vllm_mla"
text_config._attn_implementation = VLLM_MLA_ATTN_IMPL
else:
# MLA model not using MLAAttention: recompute head_size for full attn
qk_nope_head_dim = getattr(text_config, "qk_nope_head_dim", 0)
@@ -570,12 +590,23 @@ class Base(
self.model_config.model_arch_config.head_size = qk_head_dim
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
layer_types = getattr(text_config, "layer_types", None)
pp_rank = self.pp_group.rank_in_group
pp_size = self.pp_group.world_size
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
for i in range(start, end):
if i not in self.attention_fusers:
in_range = layer_types and i < len(layer_types)
layer = f"{i} ({layer_types[i]})" if in_range else str(i)
raise ValueError(
f"Layer {layer} does not dispatch through the Transformers "
"attention interface and vLLM has no other way to handle it."
)
prefix, attn_fuser = self.attention_fusers[i]
attn_module = self.get_submodule(prefix)
# `[i]` is the whole-model config unless the checkpoint is
# heterogeneous, in which case it is this layer's own geometry.
arch_config = self.model_config.model_arch_config[i]
@@ -583,8 +614,10 @@ class Base(
self.parallel_config, arch_config
)
head_size = arch_config.head_size
# Default to Llama scale, maybe updated in vllm_attention_forward
scale = head_size**-0.5
scale = attn_fuser.scale(attn_module)
# Default to Llama scale if AttentionFuser couldn't identify it
if scale is None:
scale = head_size**-0.5
num_kv_heads = self.model_config.get_num_kv_heads(
self.parallel_config, arch_config
)
@@ -598,17 +631,22 @@ class Base(
)
if attn_cls is MLAAttention:
prefix, fuser = mla_fusers[i]
mla_module = self.get_submodule(prefix)
mla_fuser = next(
(f for f in self.fusers[prefix] if isinstance(f, MLAFuser)), None
)
if mla_fuser is None:
raise ValueError(
f"Layer {i} ({prefix}) did not fuse as MLA, but the rest of "
"the model did, so its latent KV cache cannot be allocated."
)
dims = get_mla_dims(self.model_config)
kwargs.update(
scale=mla_module.scaling,
qk_nope_head_dim=dims.qk_nope_head_dim,
qk_rope_head_dim=dims.qk_rope_head_dim,
v_head_dim=dims.v_head_dim,
q_lora_rank=dims.q_lora_rank,
kv_lora_rank=dims.kv_lora_rank,
kv_b_proj=mla_module.get_submodule(fuser.kv_b_proj_name),
kv_b_proj=attn_module.get_submodule(mla_fuser.kv_b_proj_name),
)
else:
kwargs.update(
@@ -618,17 +656,20 @@ class Base(
)
# Handle interleaved sliding window attention
if (
hasattr(text_config, "layer_types")
and text_config.layer_types[i] == "sliding_attention"
):
if layer_types and layer_types[i] == "sliding_attention":
kwargs["per_layer_sliding_window"] = text_config.sliding_window
attn_instance = attn_cls(**kwargs)
if attn_cls is MLAAttention:
# Attach MLA attn_instance to mla_module so it appears in
# model.named_modules() and runs its process_weights_after_loading
mla_module._vllm_mla_attn = attn_instance
# Attach the instance to the module it serves, which is the only way it
# is in the module tree at all: a plain dict is invisible to
# `nn.Module.__setattr__`. Being in the tree runs its
# `process_weights_after_loading` and moves its KV-scale buffers with
# `model.to(device)`, and lets the attention interface read it off the
# module rather than index by `module.layer_idx`, an `int` that bakes
# layer identity into the traced graph and so costs one compiled
# artifact per layer.
setattr(attn_module, VLLM_ATTN_ATTR, attn_instance)
attention_instances[i] = attn_instance
return attention_instances
@@ -645,7 +686,11 @@ class Base(
return EncoderOnlyAttention
if self.model_config.use_mla:
self.check_version("5.15.0.dev0", "optimized MLA support")
if any(isinstance(fuser, MLAFuser) for fuser in self.fusers.values()):
if any(
isinstance(fuser, MLAFuser)
for fusers in self.fusers.values()
for fuser in fusers
):
return MLAAttention
logger.warning_once(
"This model uses MLA but `MLAFuser` failed to match and/or fuse any "
@@ -731,7 +776,6 @@ class Base(
inputs_embeds=inputs_embeds,
use_cache=False,
position_ids=positions,
attention_instances=self.attention_instances,
return_dict=False,
**self._output_aux_hidden_states_kwargs,
**kwargs,
@@ -2,20 +2,22 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fuser detection for the Transformers modeling backend.
`get_fuser` traces a module class once (see `fx_utils`) and matches it against
each concrete fuser in `fusers`; `Fusers` caches the result per class for a
whole model. `base.recursive_replace` then applies the matched fuser per
`get_fusers` traces a module class once (see `fx_utils`) and matches it against each
concrete fuser in `fusers`; `Fusers` caches the result per class for a whole model.
At most one match may give the module a different forward, and any number of fusers that
leave the forward alone apply alongside. `base.recursive_replace` then applies them per
instance. RMSNorm-shaped modules the tracer cannot match are warned about.
"""
from collections import UserDict
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, TypeVar
from cachetools import cached
from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.models.transformers.fusers import (
AttentionFuser,
BaseFuser,
GLUFuser,
MLAFuser,
@@ -31,58 +33,90 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
F = TypeVar("F", bound=BaseFuser)
def key(module: nn.Module) -> tuple:
"""Cache key for `get_fuser`. Considers module type and its immediate children."""
"""Cache key for `get_fusers`. Considers module type and its immediate children."""
return (type(module), tuple(name for name, _ in module.named_children()))
FUSERS: tuple[type[BaseFuser], ...] = (
MLAFuser,
GLUFuser,
QKVFuser,
PackedQKVFuser,
RMSNormFuser,
AttentionFuser,
)
"""Every fuser, in match order: those that redefine the forward first, then those
that leave it alone. A new fuser is added here."""
@cached(cache={}, key=key)
def get_fuser(module: nn.Module) -> BaseFuser | None:
"""The fuser for `module`'s class and shape (cached), or `None` if no match."""
def get_fusers(module: nn.Module) -> list[BaseFuser]:
"""Every fuser that applies to `module`'s class and shape (cached).
The one that redefines the forward comes first, so applying them in order
fuses before the rest read or adjust the result."""
# Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a
# leaf module (raw tensor math, no submodules). Nothing else can match, and
# tracing is skipped for it.
n_linear = sum(isinstance(c, nn.Linear) for c in module.children())
is_leaf = next(module.children(), None) is None
if n_linear < 2 and not is_leaf:
return None
if (graph := trace(module)) is None:
return None
for fuser_cls in (MLAFuser, GLUFuser, QKVFuser, PackedQKVFuser, RMSNormFuser):
if (fuser := fuser_cls.match(graph, module)) is not None:
if isinstance(fuser, RewriteFuser):
try:
fuser.update_forward(module)
except Exception as exc:
logger.debug(
"Attempted to fuse %s using %s but failed "
"to update its forward method: %s",
type(module),
fuser_cls.__name__,
exc,
)
continue
return fuser
graph = trace(module) if n_linear >= 2 or is_leaf else None
fusers: list[BaseFuser] = []
for fuser_cls in FUSERS:
if fuser_cls.redefines_forward and (
# A forward can only be rewritten from a trace of it, and only once,
# because every rewrite starts from the original source.
graph is None or any(fuser.redefines_forward for fuser in fusers)
):
continue
if (fuser := fuser_cls.match(graph, module)) is None:
continue
if isinstance(fuser, RewriteFuser):
try:
fuser.update_forward(module)
except Exception as exc:
logger.debug(
"Attempted to fuse %s using %s but failed "
"to update its forward method: %s",
type(module),
fuser_cls.__name__,
exc,
)
continue
fusers.append(fuser)
# A norm we could not match structurally is left unfused; flag likely misses.
if module.__class__.__name__.endswith("RMSNorm"):
if (
module.__class__.__name__.endswith("RMSNorm")
and graph is not None
and not any(isinstance(fuser, RMSNormFuser) for fuser in fusers)
):
logger.warning_once(
"%s looks like an RMSNorm but its computation did not match the "
"expected pattern, so it was left unfused.",
module.__class__.__name__,
)
return None
return fusers
def get_fuser(module: nn.Module, fuser_cls: type[F]) -> "F | None":
"""The first `fuser_cls` that applies to `module`, if one does."""
return next((f for f in get_fusers(module) if isinstance(f, fuser_cls)), None)
class Fusers(UserDict):
"""Mapping from module class and shape to fuser, for all fusable modules."""
"""Mapping from module class and shape to fusers, for all fusable modules."""
def __init__(self, model: nn.Module, vllm_config: "VllmConfig"):
self.vllm_config = vllm_config
super().__init__({key(m): get_fuser(m) for m in model.modules()})
super().__init__({key(m): get_fusers(m) for m in model.modules()})
def __getitem__(self, m: nn.Module) -> BaseFuser | None:
fuser = self.data.get(key(m))
if fuser is not None and fuser.validate(m, self.vllm_config):
return fuser
return None
def __getitem__(self, m: nn.Module) -> list[BaseFuser]:
"""The fusers this instance can take, in the order to apply them."""
fusers = self.data.get(key(m), ())
return [fuser for fuser in fusers if fuser.validate(m, self.vllm_config)]
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Concrete fusers for the Transformers modeling backend."""
from vllm.model_executor.models.transformers.fusers.attention import AttentionFuser
from vllm.model_executor.models.transformers.fusers.base import (
BaseFuser,
RewriteFuser,
@@ -15,6 +16,7 @@ from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
__all__ = [
"AttentionFuser",
"BaseFuser",
"RewriteFuser",
"StackedFuser",
@@ -0,0 +1,129 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Attention fuser: the module that dispatches to the attention interface."""
import ast
import inspect
import textwrap
from dataclasses import dataclass
from functools import cache
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
if TYPE_CHECKING:
from vllm.config import VllmConfig
VLLM_ATTN_IMPL = "vllm"
VLLM_MLA_ATTN_IMPL = "vllm_mla"
def _is_interface_lookup(node: ast.expr | None) -> bool:
"""Whether `node` reads an entry out of `ALL_ATTENTION_FUNCTIONS`."""
# ALL_ATTENTION_FUNCTIONS.get_interface(...) or ALL_ATTENTION_FUNCTIONS[...]
if isinstance(node, ast.Call):
node = node.func
if not isinstance(node, (ast.Attribute, ast.Subscript)):
return False
return (
isinstance(node.value, ast.Name) and node.value.id == "ALL_ATTENTION_FUNCTIONS"
)
@cache
def interface_call(cls: type[nn.Module]) -> ast.Call | None:
"""The attention interface call in `cls.forward`, if it makes exactly one."""
try:
source = inspect.getsource(inspect.unwrap(cls.forward))
tree = ast.parse(textwrap.dedent(source))
except (AttributeError, OSError, SyntaxError, TypeError):
return None
# Find the names of the local variables that read from the interface lookup
names = set()
for node in ast.walk(tree):
if not isinstance(node, (ast.Assign, ast.AnnAssign)):
continue
if not _is_interface_lookup(node.value):
continue
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
names.update(t.id for t in targets if isinstance(t, ast.Name))
# Find the calls to those names, and return the one if there is exactly one
calls = [
node
for node in ast.walk(tree)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id in names
]
return calls[0] if len(calls) == 1 else None
def _resolve(node: ast.expr, module: nn.Module) -> object:
"""The value of `node` on `module`, for literals and `self.<attr>`."""
if isinstance(node, ast.Constant):
return node.value
if (
isinstance(node, ast.Attribute)
and isinstance(node.value, ast.Name)
and node.value.id == "self"
):
return getattr(module, node.attr, None)
return None
@dataclass
class AttentionFuser(BaseFuser):
"""A module that dispatches through the Transformers attention interface."""
redefines_forward: ClassVar[bool] = False
source_cls: str
"""Class of the HF module that dispatches (for logging)."""
scale_expr: ast.expr | None
"""Source of the `scaling=` the module hands the interface, if it hands one."""
def info(self, name: str) -> str:
return f"Found: {name} ({self.source_cls}) -> attention interface"
@classmethod
def match(
cls, graph: fx.Graph | None, module: nn.Module
) -> "AttentionFuser | None":
if (call := interface_call(type(module))) is None:
return None
scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"]
scale_expr = scaling[0] if len(scaling) == 1 else None
return cls(source_cls=type(module).__name__, scale_expr=scale_expr)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Whether `module` will actually dispatch to vLLM."""
config = getattr(module, "config", None)
# Only patched in the text config, this excludes attention based mm encoders
vllm_attn_impls = {VLLM_ATTN_IMPL, VLLM_MLA_ATTN_IMPL}
return getattr(config, "_attn_implementation", None) in vllm_attn_impls
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
return module
def layer_index(self, module: nn.Module) -> int | None:
"""The layer `module` computes attention for, if it declares one."""
layer_idx = getattr(module, "layer_idx", None)
return layer_idx if isinstance(layer_idx, int) else None
def scale(self, module: nn.Module) -> float | None:
"""The softmax scale `module` passes to the interface, or `None`."""
if self.scale_expr is None:
return None
scale = _resolve(self.scale_expr, module)
if not isinstance(scale, (int, float)) or isinstance(scale, bool):
expression = ast.unparse(self.scale_expr)
raise ValueError(
f"Cannot resolve attention scaling expression {expression!r} in "
f"{type(module).__name__}."
)
return float(scale)
@@ -20,11 +20,15 @@ if TYPE_CHECKING:
class BaseFuser(ABC):
"""A detected fusion and how to apply it.
`match` analyses the module *class* once (cached, see `get_fuser`); `fuse`
`match` analyses the module *class* once (cached, see `get_fusers`); `fuse`
then applies the fusion to an instance in `recursive_replace`, returning the
module to install in its place.
"""
redefines_forward: ClassVar[bool] = True
"""Whether `fuse` gives the module a different forward,
by rewriting its source or by returning a different module in its place."""
@abstractmethod
def info(self, name: str) -> str:
"""A human-readable description of the fusion at `name`, for logging."""
@@ -59,7 +59,7 @@ def _is_rms_norm(module: nn.Module) -> bool:
# Go via `get_fuser` because it caches
from vllm.model_executor.models.transformers.fuser import get_fuser
return isinstance(get_fuser(module), RMSNormFuser)
return get_fuser(module, RMSNormFuser) is not None
def _top_level_index(funcdef: ast.FunctionDef, node: ast.AST) -> int:
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.fused_moe import (
)
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
@@ -319,8 +320,8 @@ class MoEMixin(MixtureOfExperts):
# Store shared experts for later down projection adjustment
if shared_experts is not None:
hf_shared = shared_experts.shared_experts
glu_fuser = get_fuser(hf_shared)
down_name = getattr(glu_fuser, "down_name", None)
glu_fuser = get_fuser(hf_shared, GLUFuser)
down_name = glu_fuser and glu_fuser.down_name
if down_name is not None:
shared_down_projs.append((hf_shared, down_name))
# Prefer config, otherwise read it from fuser.