diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index de896a6fc8..125754420e 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -72,19 +72,26 @@ This means that, with the Transformers modeling backend for vLLM, new models can
This section details the necessary modifications to make to a Transformers compatible custom model that make it compatible with the Transformers modeling backend for vLLM. (We assume that a Transformers compatible custom model has already been created, see [Transformers - Customizing models](https://huggingface.co/docs/transformers/en/custom_models)).
-To make your model compatible with the Transformers modeling backend, it needs:
+To make your model compatible with the Transformers modeling backend:
-1. `kwargs` passed down through all modules from `MyModel` to `MyAttention`.
- - If your model is encoder-only:
- 1. Add `is_causal = False` to `MyAttention`.
- - If your model is mixture-of-experts (MoE):
- 1. Your sparse MoE block must have an attribute called `experts`.
- 2. The class of `experts` (`MyExperts`) must either:
- - Inherit from `nn.ModuleList` (naive).
- - Or contain all 3D `nn.Parameters` (packed).
- 3. `MyExperts.forward` must accept `hidden_states`, `top_k_index`, `top_k_weights`.
-2. `MyAttention` must use `ALL_ATTENTION_FUNCTIONS` to call attention.
-3. `MyModel` must contain `_supports_attention_backend = True`.
+1. `MyAttention` must use `ALL_ATTENTION_FUNCTIONS` to call attention.
+ - It must make exactly one such call. vLLM attaches one `Attention` layer per `MyAttention` module.
+ - `MyAttention` must contain a unique `layer_idx`. vLLM keys its KV cache using this index.
+ - Pass `scaling=` to the interface if your scale is not `head_size**-0.5`. vLLM reads it from the call to the attention interface.
+2. If your model is encoder-only:
+ 1. Add `is_causal = False` to `MyAttention`.
+3. If your model is mixture-of-experts (MoE):
+ 1. Your sparse MoE block must have an attribute called `experts`.
+ 2. The class of `experts` (`MyExperts`) must either:
+ - Inherit from `nn.ModuleList` (naive).
+ - Or contain all 3D `nn.Parameters` (packed).
+ 3. `MyExperts.forward` must accept `hidden_states`, `top_k_index`, `top_k_weights`.
+
+!!! note
+ `MyModel` no longer needs `_supports_attention_backend = True`, and `kwargs` no
+ longer need to be passed down through every module from `MyModel` to
+ `MyAttention`. vLLM reaches its attention layer through `MyAttention` itself, so
+ all it asks is that `MyAttention` dispatches through the interface.
modeling_my_model.py
@@ -97,14 +104,24 @@ from torch import nn
class MyAttention(nn.Module):
is_causal = False # Only do this for encoder-only models
+ def __init__(self, config, layer_idx):
+ ...
+ self.config = config
+ self.layer_idx = layer_idx
+ self.scaling = self.head_dim**-0.5
+ ...
+
def forward(self, hidden_states, **kwargs):
...
- attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
+ attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface(
+ self.config._attn_implementation, eager_attention_forward
+ )
attn_output, attn_weights = attention_interface(
self,
query_states,
key_states,
value_states,
+ scaling=self.scaling,
**kwargs,
)
...
@@ -127,7 +144,7 @@ class MySparseMoEBlock(nn.Module):
...
class MyModel(PreTrainedModel):
- _supports_attention_backend = True
+ ...
```
@@ -135,7 +152,7 @@ class MyModel(PreTrainedModel):
Here is what happens in the background when this model is loaded:
1. The config is loaded.
-2. `MyModel` Python class is loaded from the `auto_map` in config, and we check that the model `is_backend_compatible()`.
+2. `MyModel` Python class is loaded from the `auto_map` in config, and we check that the model `_can_set_attn_implementation()`.
3. `MyModel` is loaded into one of the Transformers modeling backend classes in [vllm/model_executor/models/transformers](../../vllm/model_executor/models/transformers) which sets `self.config._attn_implementation = "vllm"` so that vLLM's attention layer is used.
That's it!
diff --git a/tests/models/transformers/fusers/test_linear.py b/tests/models/transformers/fusers/test_linear.py
index ba6fc483ac..020459d7d9 100644
--- a/tests/models/transformers/fusers/test_linear.py
+++ b/tests/models/transformers/fusers/test_linear.py
@@ -10,7 +10,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
-from vllm.model_executor.models.transformers.fuser import get_fuser
+from vllm.model_executor.models.transformers.fuser import get_fuser, get_fusers
from vllm.model_executor.models.transformers.fusers import (
GLUFuser,
PackedQKVFuser,
@@ -307,7 +307,7 @@ class PerHeadSplitAttention(nn.Module):
class FakeSelfAttn(nn.Module):
- """Stand-in for the vLLM `Attention` looked up in `attention_instances`."""
+ """Stand-in for the vLLM `Attention` attached to the dispatching module."""
def __init__(self):
super().__init__()
@@ -328,9 +328,9 @@ class FakeMQASelfAttn(FakeSelfAttn):
@pytest.fixture(autouse=True)
def _clear_fuser_cache():
- get_fuser.cache_clear()
+ get_fusers.cache_clear()
yield
- get_fuser.cache_clear()
+ get_fusers.cache_clear()
def _apply_glu_fuser_with_stubs(module: nn.Module, fuser: GLUFuser):
@@ -392,7 +392,7 @@ def _apply_packed_qkv_fuser_with_stubs(module: nn.Module, fuser: PackedQKVFuser)
def test_detects_and_rewrites_glu(mlp_cls, bias):
with torch.device("meta"):
meta = mlp_cls(bias=bias)
- fuser = get_fuser(meta)
+ fuser = get_fuser(meta, GLUFuser)
assert isinstance(fuser, GLUFuser)
assert (
fuser.gate_name,
@@ -427,9 +427,9 @@ def test_glu_identifies_down_projection():
matches the column-parallel merged gate/up; `None` when there is no such
projection to force (fusion of gate/up still applies)."""
with torch.device("meta"):
- assert get_fuser(GLUMLP()).down_name == "down_proj"
- assert get_fuser(ReversedGLUMLP()).down_name == "down_proj"
- assert get_fuser(NoDownGLU()).down_name is None
+ assert get_fuser(GLUMLP(), GLUFuser).down_name == "down_proj"
+ assert get_fuser(ReversedGLUMLP(), GLUFuser).down_name == "down_proj"
+ assert get_fuser(NoDownGLU(), GLUFuser).down_name is None
@pytest.mark.parametrize("attn_cls", [FakeAttention, ReversedFakeAttention])
@@ -439,7 +439,7 @@ def test_detects_and_rewrites_qkv(attn_cls, kv_heads):
pytest.skip("MHA q/k/v assignment is order-based by design")
with torch.device("meta"):
meta = attn_cls(kv_heads=kv_heads)
- fuser = get_fuser(meta)
+ fuser = get_fuser(meta, QKVFuser)
assert isinstance(fuser, QKVFuser)
# q (sharded differently under TP) must be identified exactly; k/v may be
# swapped for non-canonical compute order, which is numerically consistent
@@ -467,27 +467,27 @@ def test_detects_and_rewrites_qkv(attn_cls, kv_heads):
for p in real.parameters():
nn.init.normal_(p, std=0.05)
x = torch.randn(1, 5, 32)
- attention_instances = {3: FakeSelfAttn()}
- expected, _ = real(x, attention_instances=attention_instances)
+ real.attn = FakeSelfAttn()
+ expected, _ = real(x)
fused = _apply_qkv_fuser_with_stubs(real, fuser)
# Fusion is in place: the module keeps its class and other attributes
assert fused is real and type(fused) is attn_cls
assert fused.layer_idx == 3 and fused.is_causal and fused.config is not None
- out, _ = fused(x, attention_instances=attention_instances)
+ out, _ = fused(x)
torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5)
def test_qkv_identifies_output_projection():
with torch.device("meta"):
- assert get_fuser(FakeAttention()).o_name == "o_proj"
- assert get_fuser(ReversedFakeAttention()).o_name == "o_proj"
- assert get_fuser(ExtraProjAttention()).o_name == "o_proj"
+ assert get_fuser(FakeAttention(), QKVFuser).o_name == "o_proj"
+ assert get_fuser(ReversedFakeAttention(), QKVFuser).o_name == "o_proj"
+ assert get_fuser(ExtraProjAttention(), QKVFuser).o_name == "o_proj"
# Norm children (q_norm/k_norm) must not disturb o_proj identification.
- assert get_fuser(QKNormAttention()).o_name == "o_proj"
- assert get_fuser(PerHeadQKNormAttention()).o_name == "o_proj"
+ assert get_fuser(QKNormAttention(), QKVFuser).o_name == "o_proj"
+ assert get_fuser(PerHeadQKNormAttention(), QKVFuser).o_name == "o_proj"
# A module between o_proj and the return is transparent.
- assert get_fuser(ResidDropoutAttention()).o_name == "o_proj"
+ assert get_fuser(ResidDropoutAttention(), QKVFuser).o_name == "o_proj"
@pytest.mark.parametrize("kv_heads", [1, 2])
@@ -498,7 +498,7 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
checkpoint weight as-is, and shards q by heads while replicating k/v."""
with torch.device("meta"):
meta = PackedQKVAttention(kv_heads=kv_heads)
- fuser = get_fuser(meta)
+ fuser = get_fuser(meta, PackedQKVFuser)
assert isinstance(fuser, PackedQKVFuser)
assert (fuser.qkv_name, fuser.o_name) == ("c_attn", "c_proj")
assert (fuser.q_size, fuser.kv_size) == (32, 8 * kv_heads)
@@ -513,14 +513,14 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
for p in real.parameters():
nn.init.normal_(p, std=0.05)
x = torch.randn(1, 5, 32)
- attention_instances = {3: FakeMQASelfAttn()}
- expected, _ = real(x, attention_instances=attention_instances)
+ real.attn = FakeMQASelfAttn()
+ expected, _ = real(x)
fused = _apply_packed_qkv_fuser_with_stubs(real, fuser)
# Fusion is in place: the module keeps its class and other attributes
assert fused is real and type(fused) is PackedQKVAttention
assert fused.layer_idx == 3 and fused.is_causal
- out, _ = fused(x, attention_instances=attention_instances)
+ out, _ = fused(x)
torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5)
@@ -528,28 +528,28 @@ def test_per_head_split_is_not_packed_qkv():
"""The split must consume the whole projection, else its sizes are head
widths and re-sharding by them would be wrong."""
with torch.device("meta"):
- assert get_fuser(PerHeadSplitAttention()) is None
+ assert get_fuser(PerHeadSplitAttention(), PackedQKVFuser) is None
def test_fuser_is_cached_per_class_and_structure():
with torch.device("meta"):
- fuser_a = get_fuser(GLUMLP())
- fuser_b = get_fuser(GLUMLP())
+ fuser_a = get_fuser(GLUMLP(), GLUFuser)
+ fuser_b = get_fuser(GLUMLP(), GLUFuser)
assert fuser_a is fuser_b
- assert any(key[0] is GLUMLP for key in get_fuser.cache)
+ assert any(key[0] is GLUMLP for key in get_fusers.cache)
@pytest.mark.parametrize("cls", [NotAnMLP, UntraceableMLP])
def test_non_matching_modules_return_none(cls):
with torch.device("meta"):
module = cls()
- assert get_fuser(module) is None
+ assert get_fuser(module, GLUFuser) is None
def test_untraceable_tail_still_fuses():
with torch.device("meta"):
meta = UntraceableTailGLUMLP()
- fuser = get_fuser(meta)
+ fuser = get_fuser(meta, GLUFuser)
assert isinstance(fuser, GLUFuser)
# Numerics: the live tail must survive the rewrite
@@ -566,8 +566,8 @@ def test_weight_mappings_are_scoped_to_fused_prefixes():
from vllm.model_executor.models.utils import WeightsMapper
with torch.device("meta"):
- glu_fuser = get_fuser(GLUMLP())
- qkv_fuser = get_fuser(FakeAttention())
+ glu_fuser = get_fuser(GLUMLP(), GLUFuser)
+ qkv_fuser = get_fuser(FakeAttention(), QKVFuser)
mapper = WeightsMapper()
for prefix in ("model.layers.0.mlp", "model.layers.1.mlp"):
@@ -620,7 +620,7 @@ def test_weight_mappings_are_scoped_to_fused_prefixes():
def test_unfusable_modules_are_not_fused(cls, default_vllm_config):
with torch.device("meta"):
module = cls()
- fuser = get_fuser(module)
+ fuser = get_fuser(module, GLUFuser)
# Either no pattern matches the class, or this instance fails validation
# (`recursive_replace` gates fusion and its weight mappings on `validate`)
assert fuser is None or not fuser.validate(module, default_vllm_config)
@@ -654,9 +654,15 @@ def _wider_model_config(head_dim: int) -> SimpleNamespace:
@pytest.mark.parametrize(
- "cls, fuser_module", [(FakeAttention, qkv), (PackedQKVAttention, packed_qkv)]
+ "cls, fuser_module, fuser_cls",
+ [
+ (FakeAttention, qkv, QKVFuser),
+ (PackedQKVAttention, packed_qkv, PackedQKVFuser),
+ ],
)
-def test_head_counts_come_from_the_module_not_the_model(cls, fuser_module, monkeypatch):
+def test_head_counts_come_from_the_module_not_the_model(
+ cls, fuser_module, fuser_cls, monkeypatch
+):
"""A layer narrower than the model-wide head size must not be miscounted.
On a heterogeneous checkpoint (Gemma 4) the model-wide head size is the
@@ -680,7 +686,7 @@ def test_head_counts_come_from_the_module_not_the_model(cls, fuser_module, monke
monkeypatch.setattr(
fuser_module, "replace_linear_class", lambda *a, **kw: nn.Identity()
)
- fuser = get_fuser(module)
+ fuser = get_fuser(module, fuser_cls)
assert fuser is not None and fuser.validate(module, vllm_config)
fuser.update_attrs(module, "model.layers.0.self_attn", vllm_config)
@@ -700,5 +706,5 @@ def test_validate_accepts_a_layer_the_model_wide_head_size_would_reject():
# kv width is 24, not a multiple of the model-wide 16, but is of this
# layer's 8.
- fuser = get_fuser(module)
+ fuser = get_fuser(module, QKVFuser)
assert fuser is not None and fuser.validate(module, vllm_config)
diff --git a/tests/models/transformers/fusers/test_mla.py b/tests/models/transformers/fusers/test_mla.py
index 2358b5c59a..d61e54e923 100644
--- a/tests/models/transformers/fusers/test_mla.py
+++ b/tests/models/transformers/fusers/test_mla.py
@@ -158,4 +158,4 @@ class GLU(nn.Module):
def test_non_mla_is_not_matched():
with torch.device("meta"):
- assert not isinstance(get_fuser(GLU()), MLAFuser)
+ assert get_fuser(GLU(), MLAFuser) is None
diff --git a/tests/models/transformers/fusers/test_rms_norm.py b/tests/models/transformers/fusers/test_rms_norm.py
index ac259b66b4..153f7863bd 100644
--- a/tests/models/transformers/fusers/test_rms_norm.py
+++ b/tests/models/transformers/fusers/test_rms_norm.py
@@ -162,7 +162,7 @@ class UntraceableGatedRMSNorm(RMSNorm):
)
def test_detects_rms_norm_variants(cls, eps, zero_centered):
with torch.device("meta"):
- fuser = get_fuser(cls(16, eps=eps))
+ fuser = get_fuser(cls(16, eps=eps), RMSNormFuser)
assert isinstance(fuser, RMSNormFuser)
assert fuser.zero_centered == zero_centered
@@ -171,7 +171,7 @@ def test_detects_rms_norm_variants(cls, eps, zero_centered):
def test_non_rms_norms_are_not_matched(cls):
with torch.device("meta"):
module = cls(16) if cls is nn.LayerNorm else cls()
- assert not isinstance(get_fuser(module), RMSNormFuser)
+ assert get_fuser(module, RMSNormFuser) is None
@pytest.mark.parametrize(
@@ -179,7 +179,7 @@ def test_non_rms_norms_are_not_matched(cls):
)
def test_gated_rms_norm_is_not_fused(cls):
with torch.device("meta"):
- assert not isinstance(get_fuser(cls()), RMSNormFuser)
+ assert get_fuser(cls(), RMSNormFuser) is None
@pytest.mark.parametrize(
@@ -196,7 +196,7 @@ def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_c
with torch.device("meta"):
module = cls()
- fuser = get_fuser(module)
+ fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
from vllm.model_executor.models.transformers.fusers.rms_norm import (
TPAwareNormMixin,
@@ -221,7 +221,7 @@ def test_weightless_norm_has_no_hidden_size(default_vllm_config):
matching the unfused norm and vLLM's native `RMSNorm(hidden_size=head_dim)`.
"""
module = WeightlessRMSNorm(72)
- built = get_fuser(module).fuse(module, "norm", default_vllm_config)
+ built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
assert built.hidden_size == 0
built.tp_size = 2 # emulate TP=2 without a real process group
@@ -236,7 +236,7 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config):
with torch.device("meta"):
module = torch.nn.RMSNorm(16) # forward is a single `F.rms_norm` call
- fuser = get_fuser(module)
+ fuser = get_fuser(module, RMSNormFuser)
assert isinstance(fuser, RMSNormFuser)
assert not fuser.zero_centered
vllm_config = SimpleNamespace(model_config=SimpleNamespace(dtype=torch.float32))
@@ -251,7 +251,8 @@ def test_eps_is_derived_per_instance(default_vllm_config):
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = RMSNorm(16, eps=eps)
- built = get_fuser(module).fuse(module, "norm", default_vllm_config)
+ fuser = get_fuser(module, RMSNormFuser)
+ built = fuser.fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps
@@ -262,7 +263,7 @@ def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = cls(16, eps=eps)
- fuser = get_fuser(module)
+ fuser = get_fuser(module, RMSNormFuser)
assert fuser.eps_attr == cls.attr
built = fuser.fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps
@@ -274,7 +275,7 @@ def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplo
logger = "vllm.model_executor.models.transformers.fusers.rms_norm"
with caplog.at_level("DEBUG", logger=logger), torch.device("meta"):
module = LiteralEpsRMSNorm()
- fuser = get_fuser(module)
+ fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
assert fuser.eps_attr is None
assert built.variance_epsilon == 1e-4
@@ -287,7 +288,7 @@ def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
with torch.device("meta"):
module = AmbiguousEpsRMSNorm(16, eps=1e-6)
before = dict(vars(module))
- fuser = get_fuser(module)
+ fuser = get_fuser(module, RMSNormFuser)
assert fuser.eps_attr == "variance_epsilon"
assert vars(module) == before
diff --git a/tests/models/transformers/test_backend.py b/tests/models/transformers/test_backend.py
index 0429f3da2c..40c0434c92 100644
--- a/tests/models/transformers/test_backend.py
+++ b/tests/models/transformers/test_backend.py
@@ -2,9 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test the functionality of the Transformers modeling backend."""
+import ast
import contextlib
import os
import tempfile
+from types import SimpleNamespace
from typing import Any
import pytest
@@ -15,6 +17,8 @@ from transformers import AutoConfig, AutoModel, PretrainedConfig
from vllm.config import ModelConfig, VllmConfig
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.transformers.base import Base
+from vllm.model_executor.models.transformers.fusers import AttentionFuser
+from vllm.model_executor.models.transformers.fusers.attention import VLLM_ATTN_IMPL
from vllm.model_executor.models.transformers.multimodal import MultiModalMixin
from vllm.model_executor.models.utils import StageMissingLayer
@@ -583,3 +587,141 @@ def test_marking_skipped_without_tokenizer():
"""
vision_tower = build_marked_model(0, skip_tokenizer_init=True).vision_tower
assert not isinstance(vision_tower, StageMissingLayer)
+
+
+NUM_LAYERS = 4
+
+
+def build_model(model_type: str, **overrides) -> nn.Module:
+ """A tiny HF model of `NUM_LAYERS` layers, built on the meta device."""
+ try:
+ config = AutoConfig.for_model(
+ model_type,
+ num_hidden_layers=NUM_LAYERS,
+ hidden_size=64,
+ intermediate_size=128,
+ num_attention_heads=4,
+ num_key_value_heads=2,
+ head_dim=16,
+ vocab_size=VOCAB_SIZE,
+ **overrides,
+ )
+ except ValueError:
+ pytest.skip(f"The installed transformers has no {model_type!r} model")
+ with torch.device("meta"):
+ return AutoModel.from_config(config)
+
+
+ATTENTION_MODEL_TYPES = [
+ # The default `head_size**-0.5` happens to be right here
+ "llama",
+ # Scales the query by a learnable per-dim weight, so it declares 1.0
+ "gemma4_text",
+ # MLA, and declares yarn's `mscale`
+ "deepseek_v3",
+]
+
+
+@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
+def test_attention_dispatch_is_matched(model_type: str):
+ """Exactly the decoder layers' attention modules match an `AttentionFuser`.
+
+ That match is what `recursive_replace` records, and so what
+ `create_attention_instances` attaches vLLM's attention layer to.
+ """
+ model = build_model(model_type)
+ matched = {
+ name
+ for name, module in model.named_modules()
+ if AttentionFuser.match(None, module) is not None
+ }
+ assert matched == {f"layers.{i}.self_attn" for i in range(NUM_LAYERS)}
+
+
+def test_attention_layer_index_is_the_modules_own():
+ """The layer served comes from the module, not from its position in the stack.
+
+ LongCat Flash gives each decoder layer two attention sublayers numbered
+ `2i` and `2i + 1`, so `num_hidden_layers` is twice the length of the stack
+ and the enclosing layer's position is not the index the KV cache is keyed by.
+ A vision tower is excluded separately, by `validate`: only the text config is
+ patched to dispatch to vLLM, so a tower's attention is left to Transformers.
+ """
+ model = build_model("longcat_flash", num_layers=2)
+ text_config = model.config.get_text_config()
+ assert len(model.layers) * 2 == text_config.num_hidden_layers
+
+ attentions = [
+ model.get_submodule(f"layers.{i}.self_attn.{j}") for i in (0, 1) for j in (0, 1)
+ ]
+ fusers = [AttentionFuser.match(None, attn) for attn in attentions]
+ assert all(fuser is not None for fuser in fusers)
+ assert [f.layer_index(a) for f, a in zip(fusers, attentions)] == [0, 1, 2, 3]
+
+ # Not dispatching to vLLM yet, as a vision tower never would be. The vLLM
+ # config is not consulted, only the one the module was built with.
+ assert not any(f.validate(a, None) for f, a in zip(fusers, attentions))
+ text_config._attn_implementation = VLLM_ATTN_IMPL
+ assert all(f.validate(a, None) for f, a in zip(fusers, attentions))
+
+
+def test_attention_dispatch_is_required(monkeypatch: pytest.MonkeyPatch):
+ """Models selected by the permissive registry fail clearly without dispatch."""
+ monkeypatch.setattr(
+ "vllm.model_executor.models.transformers.base.get_pp_indices",
+ lambda *_: (0, 1),
+ )
+ model = SimpleNamespace(
+ text_config=SimpleNamespace(num_hidden_layers=1),
+ pp_group=SimpleNamespace(rank_in_group=0, world_size=1),
+ attention_fusers={},
+ _get_attn_cls=lambda: None,
+ )
+
+ with pytest.raises(
+ ValueError,
+ match="Layer 0 does not dispatch through the Transformers attention interface",
+ ):
+ Base.create_attention_instances(model)
+
+
+@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
+def test_attention_scale_is_the_declared_one(model_type: str):
+ """The scale comes off the module, not from the `head_size**-0.5` default.
+
+ Two of the three model types declare something the default would silently
+ replace, so they are what makes this more than a tautology.
+ """
+ attention = build_model(model_type).layers[0].self_attn
+ fuser = AttentionFuser.match(None, attention)
+ assert fuser is not None
+ assert fuser.scale(attention) == attention.scaling
+
+
+def test_attention_scale_is_the_argument_not_the_attribute():
+ """OPT applies the scale to the query itself, then declares a literal 1.0.
+
+ Its `self.scaling` is the `head_size**-0.5` it has already applied, so
+ reading the attribute rather than the argument the module hands the
+ interface would scale twice.
+ """
+ model = build_model("opt", ffn_dim=128, word_embed_proj_dim=64)
+ attention = model.get_submodule("decoder.layers.0.self_attn")
+ fuser = AttentionFuser.match(None, attention)
+ assert fuser is not None
+ assert fuser.scale(attention) == 1.0 != attention.scaling
+
+
+def test_attention_scale_rejects_unresolvable_expression():
+ """Unresolvable declared scales must not silently use the Llama default."""
+
+ class Attention(nn.Module):
+ head_dim = 16
+
+ fuser = AttentionFuser(
+ source_cls="Attention",
+ scale_expr=ast.parse("self.head_dim**-0.5 * self.factor", mode="eval").body,
+ )
+
+ with pytest.raises(ValueError, match="Cannot resolve attention scaling expression"):
+ fuser.scale(Attention())
diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py
index dabd1e4460..2f030ec1b7 100644
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -1253,7 +1253,10 @@ class _ModelRegistry:
"'auto_map' (relevant if the model is custom)."
)
- if not model_module.is_backend_compatible():
+ if not (
+ model_module.is_backend_compatible()
+ or model_module._can_set_attn_implementation()
+ ):
if model_config.model_impl != "transformers":
return None
diff --git a/vllm/model_executor/models/transformers/__init__.py b/vllm/model_executor/models/transformers/__init__.py
index 7a7aab7266..1ec5849408 100644
--- a/vllm/model_executor/models/transformers/__init__.py
+++ b/vllm/model_executor/models/transformers/__init__.py
@@ -21,7 +21,7 @@ from typing import TYPE_CHECKING
import torch.nn.functional as F
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
-from vllm.model_executor.models.transformers.base import Base
+from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR, Base
from vllm.model_executor.models.transformers.causal import CausalMixin
from vllm.model_executor.models.transformers.legacy import LegacyMixin
from vllm.model_executor.models.transformers.moe import MoEMixin
@@ -40,8 +40,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
if TYPE_CHECKING:
import torch
- from vllm.model_executor.layers.attention import Attention, MLAAttention
-
def vllm_attention_forward(
# Transformers args
@@ -50,15 +48,9 @@ def vllm_attention_forward(
key: "torch.Tensor",
value: "torch.Tensor",
attention_mask: "torch.Tensor",
- # Transformers kwargs
- scaling: float | None = None,
- # vLLM kwargs
- attention_instances: "dict[int, Attention] | None" = None,
**kwargs,
):
- self_attn = attention_instances[module.layer_idx]
- if scaling is not None:
- self_attn.impl.scale = float(scaling)
+ self_attn = getattr(module, VLLM_ATTN_ATTR)
hidden = query.shape[-2]
head_dim_qk = query.shape[-1]
head_dim_v = value.shape[-1]
@@ -85,13 +77,9 @@ def vllm_mla_attention_forward(
kv_c_normed: "torch.Tensor",
k_pe: "torch.Tensor",
attention_mask: "torch.Tensor",
- # Transformers kwargs
- scaling: float | None = None,
- # vLLM kwargs
- attention_instances: "dict[int, MLAAttention] | None" = None,
**kwargs,
):
- self_attn = attention_instances[module.layer_idx]
+ self_attn = getattr(module, VLLM_ATTN_ATTR)
# [batch=1, heads, num_tokens, qk_head_dim] -> [num_tokens, heads, qk_head_dim]
query = query.transpose(1, 2).flatten(0, 1)
num_tokens, num_heads = query.shape[:2]
diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py
index 11b7c8466c..5ee2751a4f 100644
--- a/vllm/model_executor/models/transformers/base.py
+++ b/vllm/model_executor/models/transformers/base.py
@@ -57,7 +57,11 @@ from vllm.model_executor.models.interfaces import (
)
from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
-from vllm.model_executor.models.transformers.fusers import MLAFuser
+from vllm.model_executor.models.transformers.fusers import AttentionFuser, MLAFuser
+from vllm.model_executor.models.transformers.fusers.attention import (
+ VLLM_ATTN_IMPL,
+ VLLM_MLA_ATTN_IMPL,
+)
from vllm.model_executor.models.transformers.utils import (
attrsetter,
can_enable_torch_compile,
@@ -73,7 +77,6 @@ from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
- extract_layer_index,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
@@ -86,6 +89,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
+VLLM_ATTN_ATTR = "attn"
+"""Attribute in which to store the vLLM `Attention` instance that serves a Transformers
+attention module. `"attn"` is chosen deliberately to match the mappings generated by
+`maybe_remap_kv_scale_name`."""
+
class PreTrainedModelClasses(NamedTuple):
decoder: type["PreTrainedModel"]
@@ -129,9 +137,13 @@ class Base(
self.packed_modules_mapping: dict[str, list[str]] = {}
"""Fused module -> constituent projections, populated by `recursive_replace`
for the quantization machinery and loaders (e.g. bitsandbytes)."""
- self.fusers: dict[str, BaseFuser] = {}
- """Module qualname -> the fuser applied to it, populated
+ self.fusers: dict[str, list[BaseFuser]] = {}
+ """Module qualname -> the fusers applied to it, populated
by `recursive_replace` for `create_attention_instances`."""
+ self.attention_fusers: dict[int, tuple[str, AttentionFuser]] = {}
+ """`layer_idx` -> the qualname and fuser of the module computing that
+ layer's attention, populated by `recursive_replace` for
+ `create_attention_instances`."""
# Attrs for Eagle3 (see self.set_aux_hidden_state_layers)
self._target_class: type[nn.Module] = nn.Module
@@ -189,7 +201,7 @@ class Base(
- Sets the dtype to the default torch dtype set by vLLM because Transformers
uses the config dtype when creating the model
"""
- self.text_config._attn_implementation = "vllm"
+ self.text_config._attn_implementation = VLLM_ATTN_IMPL
self.config.dtype = torch.get_default_dtype()
@contextmanager
@@ -475,9 +487,22 @@ class Base(
vocab_embeddings = self._vocab_embeddings()
- def register_fusion(fuser: BaseFuser, prefix: str):
+ def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module):
"""Register a fused layer's mappings just before it is built."""
- self.fusers[prefix] = fuser
+ self.fusers.setdefault(prefix, []).append(fuser)
+
+ if (
+ isinstance(fuser, AttentionFuser)
+ and (index := fuser.layer_index(module)) is not None
+ ):
+ if index in self.attention_fusers:
+ other, _ = self.attention_fusers[index]
+ raise ValueError(
+ f"Layer {index} dispatches attention from more than one "
+ f"module ({other} and {prefix}), which one vLLM attention "
+ "layer cannot serve."
+ )
+ self.attention_fusers[index] = (prefix, fuser)
orig_to_new_stacked = fuser.orig_to_new_stacked(prefix)
self.hf_to_vllm_mapper.orig_to_new_stacked.update(orig_to_new_stacked)
@@ -528,10 +553,11 @@ class Base(
new_module = replace_embedding_class(
child_module, self.quant_config, prefix=qual_name
)
- elif (fuser := fusers[child_module]) is not None:
- register_fusion(fuser, qual_name)
- new_module = fuser.fuse(child_module, qual_name, self.vllm_config)
- logger.info_once(fuser.info(child_name))
+ elif child_module_fusers := fusers[child_module]:
+ for fuser in child_module_fusers:
+ register_fusion(fuser, qual_name, child_module)
+ new_module = fuser.fuse(new_module, qual_name, self.vllm_config)
+ logger.info_once(fuser.info(child_name))
_recursive_replace(new_module, prefix=qual_name)
elif not isinstance(child_module, MoERunner):
# MoERunner can contain aliases of shared experts and gates,
@@ -548,20 +574,14 @@ class Base(
"""
Create `Attention` instances to inform KV cache allocation.
"""
- mla_fusers = {}
attention_instances = {}
text_config = self.text_config
attn_cls = self._get_attn_cls()
# kv_lora_rank indicates that this is an MLA model
if getattr(text_config, "kv_lora_rank", None) is not None:
- mla_fusers = {
- extract_layer_index(prefix): (prefix, fuser)
- for prefix, fuser in self.fusers.items()
- if isinstance(fuser, MLAFuser)
- }
if attn_cls is MLAAttention:
- text_config._attn_implementation = "vllm_mla"
+ text_config._attn_implementation = VLLM_MLA_ATTN_IMPL
else:
# MLA model not using MLAAttention: recompute head_size for full attn
qk_nope_head_dim = getattr(text_config, "qk_nope_head_dim", 0)
@@ -570,12 +590,23 @@ class Base(
self.model_config.model_arch_config.head_size = qk_head_dim
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
+ layer_types = getattr(text_config, "layer_types", None)
pp_rank = self.pp_group.rank_in_group
pp_size = self.pp_group.world_size
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
for i in range(start, end):
+ if i not in self.attention_fusers:
+ in_range = layer_types and i < len(layer_types)
+ layer = f"{i} ({layer_types[i]})" if in_range else str(i)
+ raise ValueError(
+ f"Layer {layer} does not dispatch through the Transformers "
+ "attention interface and vLLM has no other way to handle it."
+ )
+ prefix, attn_fuser = self.attention_fusers[i]
+ attn_module = self.get_submodule(prefix)
+
# `[i]` is the whole-model config unless the checkpoint is
# heterogeneous, in which case it is this layer's own geometry.
arch_config = self.model_config.model_arch_config[i]
@@ -583,8 +614,10 @@ class Base(
self.parallel_config, arch_config
)
head_size = arch_config.head_size
- # Default to Llama scale, maybe updated in vllm_attention_forward
- scale = head_size**-0.5
+ scale = attn_fuser.scale(attn_module)
+ # Default to Llama scale if AttentionFuser couldn't identify it
+ if scale is None:
+ scale = head_size**-0.5
num_kv_heads = self.model_config.get_num_kv_heads(
self.parallel_config, arch_config
)
@@ -598,17 +631,22 @@ class Base(
)
if attn_cls is MLAAttention:
- prefix, fuser = mla_fusers[i]
- mla_module = self.get_submodule(prefix)
+ mla_fuser = next(
+ (f for f in self.fusers[prefix] if isinstance(f, MLAFuser)), None
+ )
+ if mla_fuser is None:
+ raise ValueError(
+ f"Layer {i} ({prefix}) did not fuse as MLA, but the rest of "
+ "the model did, so its latent KV cache cannot be allocated."
+ )
dims = get_mla_dims(self.model_config)
kwargs.update(
- scale=mla_module.scaling,
qk_nope_head_dim=dims.qk_nope_head_dim,
qk_rope_head_dim=dims.qk_rope_head_dim,
v_head_dim=dims.v_head_dim,
q_lora_rank=dims.q_lora_rank,
kv_lora_rank=dims.kv_lora_rank,
- kv_b_proj=mla_module.get_submodule(fuser.kv_b_proj_name),
+ kv_b_proj=attn_module.get_submodule(mla_fuser.kv_b_proj_name),
)
else:
kwargs.update(
@@ -618,17 +656,20 @@ class Base(
)
# Handle interleaved sliding window attention
- if (
- hasattr(text_config, "layer_types")
- and text_config.layer_types[i] == "sliding_attention"
- ):
+ if layer_types and layer_types[i] == "sliding_attention":
kwargs["per_layer_sliding_window"] = text_config.sliding_window
attn_instance = attn_cls(**kwargs)
- if attn_cls is MLAAttention:
- # Attach MLA attn_instance to mla_module so it appears in
- # model.named_modules() and runs its process_weights_after_loading
- mla_module._vllm_mla_attn = attn_instance
+
+ # Attach the instance to the module it serves, which is the only way it
+ # is in the module tree at all: a plain dict is invisible to
+ # `nn.Module.__setattr__`. Being in the tree runs its
+ # `process_weights_after_loading` and moves its KV-scale buffers with
+ # `model.to(device)`, and lets the attention interface read it off the
+ # module rather than index by `module.layer_idx`, an `int` that bakes
+ # layer identity into the traced graph and so costs one compiled
+ # artifact per layer.
+ setattr(attn_module, VLLM_ATTN_ATTR, attn_instance)
attention_instances[i] = attn_instance
return attention_instances
@@ -645,7 +686,11 @@ class Base(
return EncoderOnlyAttention
if self.model_config.use_mla:
self.check_version("5.15.0.dev0", "optimized MLA support")
- if any(isinstance(fuser, MLAFuser) for fuser in self.fusers.values()):
+ if any(
+ isinstance(fuser, MLAFuser)
+ for fusers in self.fusers.values()
+ for fuser in fusers
+ ):
return MLAAttention
logger.warning_once(
"This model uses MLA but `MLAFuser` failed to match and/or fuse any "
@@ -731,7 +776,6 @@ class Base(
inputs_embeds=inputs_embeds,
use_cache=False,
position_ids=positions,
- attention_instances=self.attention_instances,
return_dict=False,
**self._output_aux_hidden_states_kwargs,
**kwargs,
diff --git a/vllm/model_executor/models/transformers/fuser.py b/vllm/model_executor/models/transformers/fuser.py
index 52a08ac90b..6f211af0d3 100644
--- a/vllm/model_executor/models/transformers/fuser.py
+++ b/vllm/model_executor/models/transformers/fuser.py
@@ -2,20 +2,22 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fuser detection for the Transformers modeling backend.
-`get_fuser` traces a module class once (see `fx_utils`) and matches it against
-each concrete fuser in `fusers`; `Fusers` caches the result per class for a
-whole model. `base.recursive_replace` then applies the matched fuser per
+`get_fusers` traces a module class once (see `fx_utils`) and matches it against each
+concrete fuser in `fusers`; `Fusers` caches the result per class for a whole model.
+At most one match may give the module a different forward, and any number of fusers that
+leave the forward alone apply alongside. `base.recursive_replace` then applies them per
instance. RMSNorm-shaped modules the tracer cannot match are warned about.
"""
from collections import UserDict
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, TypeVar
from cachetools import cached
from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.models.transformers.fusers import (
+ AttentionFuser,
BaseFuser,
GLUFuser,
MLAFuser,
@@ -31,58 +33,90 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
+F = TypeVar("F", bound=BaseFuser)
+
def key(module: nn.Module) -> tuple:
- """Cache key for `get_fuser`. Considers module type and its immediate children."""
+ """Cache key for `get_fusers`. Considers module type and its immediate children."""
return (type(module), tuple(name for name, _ in module.named_children()))
+FUSERS: tuple[type[BaseFuser], ...] = (
+ MLAFuser,
+ GLUFuser,
+ QKVFuser,
+ PackedQKVFuser,
+ RMSNormFuser,
+ AttentionFuser,
+)
+"""Every fuser, in match order: those that redefine the forward first, then those
+that leave it alone. A new fuser is added here."""
+
+
@cached(cache={}, key=key)
-def get_fuser(module: nn.Module) -> BaseFuser | None:
- """The fuser for `module`'s class and shape (cached), or `None` if no match."""
+def get_fusers(module: nn.Module) -> list[BaseFuser]:
+ """Every fuser that applies to `module`'s class and shape (cached).
+
+ The one that redefines the forward comes first, so applying them in order
+ fuses before the rest read or adjust the result."""
# Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a
# leaf module (raw tensor math, no submodules). Nothing else can match, and
# tracing is skipped for it.
n_linear = sum(isinstance(c, nn.Linear) for c in module.children())
is_leaf = next(module.children(), None) is None
- if n_linear < 2 and not is_leaf:
- return None
- if (graph := trace(module)) is None:
- return None
- for fuser_cls in (MLAFuser, GLUFuser, QKVFuser, PackedQKVFuser, RMSNormFuser):
- if (fuser := fuser_cls.match(graph, module)) is not None:
- if isinstance(fuser, RewriteFuser):
- try:
- fuser.update_forward(module)
- except Exception as exc:
- logger.debug(
- "Attempted to fuse %s using %s but failed "
- "to update its forward method: %s",
- type(module),
- fuser_cls.__name__,
- exc,
- )
- continue
- return fuser
+ graph = trace(module) if n_linear >= 2 or is_leaf else None
+
+ fusers: list[BaseFuser] = []
+ for fuser_cls in FUSERS:
+ if fuser_cls.redefines_forward and (
+ # A forward can only be rewritten from a trace of it, and only once,
+ # because every rewrite starts from the original source.
+ graph is None or any(fuser.redefines_forward for fuser in fusers)
+ ):
+ continue
+ if (fuser := fuser_cls.match(graph, module)) is None:
+ continue
+ if isinstance(fuser, RewriteFuser):
+ try:
+ fuser.update_forward(module)
+ except Exception as exc:
+ logger.debug(
+ "Attempted to fuse %s using %s but failed "
+ "to update its forward method: %s",
+ type(module),
+ fuser_cls.__name__,
+ exc,
+ )
+ continue
+ fusers.append(fuser)
+
# A norm we could not match structurally is left unfused; flag likely misses.
- if module.__class__.__name__.endswith("RMSNorm"):
+ if (
+ module.__class__.__name__.endswith("RMSNorm")
+ and graph is not None
+ and not any(isinstance(fuser, RMSNormFuser) for fuser in fusers)
+ ):
logger.warning_once(
"%s looks like an RMSNorm but its computation did not match the "
"expected pattern, so it was left unfused.",
module.__class__.__name__,
)
- return None
+ return fusers
+
+
+def get_fuser(module: nn.Module, fuser_cls: type[F]) -> "F | None":
+ """The first `fuser_cls` that applies to `module`, if one does."""
+ return next((f for f in get_fusers(module) if isinstance(f, fuser_cls)), None)
class Fusers(UserDict):
- """Mapping from module class and shape to fuser, for all fusable modules."""
+ """Mapping from module class and shape to fusers, for all fusable modules."""
def __init__(self, model: nn.Module, vllm_config: "VllmConfig"):
self.vllm_config = vllm_config
- super().__init__({key(m): get_fuser(m) for m in model.modules()})
+ super().__init__({key(m): get_fusers(m) for m in model.modules()})
- def __getitem__(self, m: nn.Module) -> BaseFuser | None:
- fuser = self.data.get(key(m))
- if fuser is not None and fuser.validate(m, self.vllm_config):
- return fuser
- return None
+ def __getitem__(self, m: nn.Module) -> list[BaseFuser]:
+ """The fusers this instance can take, in the order to apply them."""
+ fusers = self.data.get(key(m), ())
+ return [fuser for fuser in fusers if fuser.validate(m, self.vllm_config)]
diff --git a/vllm/model_executor/models/transformers/fusers/__init__.py b/vllm/model_executor/models/transformers/fusers/__init__.py
index a0772ed4a2..735392a3ab 100644
--- a/vllm/model_executor/models/transformers/fusers/__init__.py
+++ b/vllm/model_executor/models/transformers/fusers/__init__.py
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Concrete fusers for the Transformers modeling backend."""
+from vllm.model_executor.models.transformers.fusers.attention import AttentionFuser
from vllm.model_executor.models.transformers.fusers.base import (
BaseFuser,
RewriteFuser,
@@ -15,6 +16,7 @@ from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
__all__ = [
+ "AttentionFuser",
"BaseFuser",
"RewriteFuser",
"StackedFuser",
diff --git a/vllm/model_executor/models/transformers/fusers/attention.py b/vllm/model_executor/models/transformers/fusers/attention.py
new file mode 100644
index 0000000000..1c6edd5e94
--- /dev/null
+++ b/vllm/model_executor/models/transformers/fusers/attention.py
@@ -0,0 +1,129 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Attention fuser: the module that dispatches to the attention interface."""
+
+import ast
+import inspect
+import textwrap
+from dataclasses import dataclass
+from functools import cache
+from typing import TYPE_CHECKING, ClassVar
+
+from torch import fx, nn
+
+from vllm.model_executor.models.transformers.fusers.base import BaseFuser
+
+if TYPE_CHECKING:
+ from vllm.config import VllmConfig
+
+VLLM_ATTN_IMPL = "vllm"
+VLLM_MLA_ATTN_IMPL = "vllm_mla"
+
+
+def _is_interface_lookup(node: ast.expr | None) -> bool:
+ """Whether `node` reads an entry out of `ALL_ATTENTION_FUNCTIONS`."""
+ # ALL_ATTENTION_FUNCTIONS.get_interface(...) or ALL_ATTENTION_FUNCTIONS[...]
+ if isinstance(node, ast.Call):
+ node = node.func
+ if not isinstance(node, (ast.Attribute, ast.Subscript)):
+ return False
+ return (
+ isinstance(node.value, ast.Name) and node.value.id == "ALL_ATTENTION_FUNCTIONS"
+ )
+
+
+@cache
+def interface_call(cls: type[nn.Module]) -> ast.Call | None:
+ """The attention interface call in `cls.forward`, if it makes exactly one."""
+ try:
+ source = inspect.getsource(inspect.unwrap(cls.forward))
+ tree = ast.parse(textwrap.dedent(source))
+ except (AttributeError, OSError, SyntaxError, TypeError):
+ return None
+
+ # Find the names of the local variables that read from the interface lookup
+ names = set()
+ for node in ast.walk(tree):
+ if not isinstance(node, (ast.Assign, ast.AnnAssign)):
+ continue
+ if not _is_interface_lookup(node.value):
+ continue
+ targets = node.targets if isinstance(node, ast.Assign) else [node.target]
+ names.update(t.id for t in targets if isinstance(t, ast.Name))
+ # Find the calls to those names, and return the one if there is exactly one
+ calls = [
+ node
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id in names
+ ]
+ return calls[0] if len(calls) == 1 else None
+
+
+def _resolve(node: ast.expr, module: nn.Module) -> object:
+ """The value of `node` on `module`, for literals and `self.`."""
+ if isinstance(node, ast.Constant):
+ return node.value
+ if (
+ isinstance(node, ast.Attribute)
+ and isinstance(node.value, ast.Name)
+ and node.value.id == "self"
+ ):
+ return getattr(module, node.attr, None)
+ return None
+
+
+@dataclass
+class AttentionFuser(BaseFuser):
+ """A module that dispatches through the Transformers attention interface."""
+
+ redefines_forward: ClassVar[bool] = False
+
+ source_cls: str
+ """Class of the HF module that dispatches (for logging)."""
+ scale_expr: ast.expr | None
+ """Source of the `scaling=` the module hands the interface, if it hands one."""
+
+ def info(self, name: str) -> str:
+ return f"Found: {name} ({self.source_cls}) -> attention interface"
+
+ @classmethod
+ def match(
+ cls, graph: fx.Graph | None, module: nn.Module
+ ) -> "AttentionFuser | None":
+ if (call := interface_call(type(module))) is None:
+ return None
+ scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"]
+ scale_expr = scaling[0] if len(scaling) == 1 else None
+ return cls(source_cls=type(module).__name__, scale_expr=scale_expr)
+
+ def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
+ """Whether `module` will actually dispatch to vLLM."""
+ config = getattr(module, "config", None)
+ # Only patched in the text config, this excludes attention based mm encoders
+ vllm_attn_impls = {VLLM_ATTN_IMPL, VLLM_MLA_ATTN_IMPL}
+ return getattr(config, "_attn_implementation", None) in vllm_attn_impls
+
+ def fuse(
+ self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
+ ) -> nn.Module:
+ return module
+
+ def layer_index(self, module: nn.Module) -> int | None:
+ """The layer `module` computes attention for, if it declares one."""
+ layer_idx = getattr(module, "layer_idx", None)
+ return layer_idx if isinstance(layer_idx, int) else None
+
+ def scale(self, module: nn.Module) -> float | None:
+ """The softmax scale `module` passes to the interface, or `None`."""
+ if self.scale_expr is None:
+ return None
+ scale = _resolve(self.scale_expr, module)
+ if not isinstance(scale, (int, float)) or isinstance(scale, bool):
+ expression = ast.unparse(self.scale_expr)
+ raise ValueError(
+ f"Cannot resolve attention scaling expression {expression!r} in "
+ f"{type(module).__name__}."
+ )
+ return float(scale)
diff --git a/vllm/model_executor/models/transformers/fusers/base.py b/vllm/model_executor/models/transformers/fusers/base.py
index e15b442d07..e36b355739 100644
--- a/vllm/model_executor/models/transformers/fusers/base.py
+++ b/vllm/model_executor/models/transformers/fusers/base.py
@@ -20,11 +20,15 @@ if TYPE_CHECKING:
class BaseFuser(ABC):
"""A detected fusion and how to apply it.
- `match` analyses the module *class* once (cached, see `get_fuser`); `fuse`
+ `match` analyses the module *class* once (cached, see `get_fusers`); `fuse`
then applies the fusion to an instance in `recursive_replace`, returning the
module to install in its place.
"""
+ redefines_forward: ClassVar[bool] = True
+ """Whether `fuse` gives the module a different forward,
+ by rewriting its source or by returning a different module in its place."""
+
@abstractmethod
def info(self, name: str) -> str:
"""A human-readable description of the fusion at `name`, for logging."""
diff --git a/vllm/model_executor/models/transformers/fusers/mla.py b/vllm/model_executor/models/transformers/fusers/mla.py
index db29ded911..31bee9dc63 100644
--- a/vllm/model_executor/models/transformers/fusers/mla.py
+++ b/vllm/model_executor/models/transformers/fusers/mla.py
@@ -59,7 +59,7 @@ def _is_rms_norm(module: nn.Module) -> bool:
# Go via `get_fuser` because it caches
from vllm.model_executor.models.transformers.fuser import get_fuser
- return isinstance(get_fuser(module), RMSNormFuser)
+ return get_fuser(module, RMSNormFuser) is not None
def _top_level_index(funcdef: ast.FunctionDef, node: ast.AST) -> int:
diff --git a/vllm/model_executor/models/transformers/moe.py b/vllm/model_executor/models/transformers/moe.py
index d59fb2b258..480bc7f66e 100644
--- a/vllm/model_executor/models/transformers/moe.py
+++ b/vllm/model_executor/models/transformers/moe.py
@@ -36,6 +36,7 @@ from vllm.model_executor.layers.fused_moe import (
)
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.transformers.fuser import get_fuser
+from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
@@ -319,8 +320,8 @@ class MoEMixin(MixtureOfExperts):
# Store shared experts for later down projection adjustment
if shared_experts is not None:
hf_shared = shared_experts.shared_experts
- glu_fuser = get_fuser(hf_shared)
- down_name = getattr(glu_fuser, "down_name", None)
+ glu_fuser = get_fuser(hf_shared, GLUFuser)
+ down_name = glu_fuser and glu_fuser.down_name
if down_name is not None:
shared_down_projs.append((hf_shared, down_name))
# Prefer config, otherwise read it from fuser.