From 8ad2076c4a0e2a7e5ccf5b11455f737430ad12db Mon Sep 17 00:00:00 2001 From: Thomas Ortner Date: Fri, 4 Sep 2026 18:26:01 +0200 Subject: [PATCH] [Transformers backend] Find attention with a fuser and attach vLLM's layer to it (#54941) Signed-off-by: Thomas Ortner Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- docs/models/supported_models.md | 47 ++++-- .../models/transformers/fusers/test_linear.py | 76 +++++----- tests/models/transformers/fusers/test_mla.py | 2 +- .../transformers/fusers/test_rms_norm.py | 21 +-- tests/models/transformers/test_backend.py | 142 ++++++++++++++++++ vllm/model_executor/models/registry.py | 5 +- .../models/transformers/__init__.py | 18 +-- .../models/transformers/base.py | 112 +++++++++----- .../models/transformers/fuser.py | 104 ++++++++----- .../models/transformers/fusers/__init__.py | 2 + .../models/transformers/fusers/attention.py | 129 ++++++++++++++++ .../models/transformers/fusers/base.py | 6 +- .../models/transformers/fusers/mla.py | 2 +- .../model_executor/models/transformers/moe.py | 5 +- 14 files changed, 521 insertions(+), 150 deletions(-) create mode 100644 vllm/model_executor/models/transformers/fusers/attention.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index de896a6fc8..125754420e 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -72,19 +72,26 @@ This means that, with the Transformers modeling backend for vLLM, new models can This section details the necessary modifications to make to a Transformers compatible custom model that make it compatible with the Transformers modeling backend for vLLM. (We assume that a Transformers compatible custom model has already been created, see [Transformers - Customizing models](https://huggingface.co/docs/transformers/en/custom_models)). -To make your model compatible with the Transformers modeling backend, it needs: +To make your model compatible with the Transformers modeling backend: -1. `kwargs` passed down through all modules from `MyModel` to `MyAttention`. - - If your model is encoder-only: - 1. Add `is_causal = False` to `MyAttention`. - - If your model is mixture-of-experts (MoE): - 1. Your sparse MoE block must have an attribute called `experts`. - 2. The class of `experts` (`MyExperts`) must either: - - Inherit from `nn.ModuleList` (naive). - - Or contain all 3D `nn.Parameters` (packed). - 3. `MyExperts.forward` must accept `hidden_states`, `top_k_index`, `top_k_weights`. -2. `MyAttention` must use `ALL_ATTENTION_FUNCTIONS` to call attention. -3. `MyModel` must contain `_supports_attention_backend = True`. +1. `MyAttention` must use `ALL_ATTENTION_FUNCTIONS` to call attention. + - It must make exactly one such call. vLLM attaches one `Attention` layer per `MyAttention` module. + - `MyAttention` must contain a unique `layer_idx`. vLLM keys its KV cache using this index. + - Pass `scaling=` to the interface if your scale is not `head_size**-0.5`. vLLM reads it from the call to the attention interface. +2. If your model is encoder-only: + 1. Add `is_causal = False` to `MyAttention`. +3. If your model is mixture-of-experts (MoE): + 1. Your sparse MoE block must have an attribute called `experts`. + 2. The class of `experts` (`MyExperts`) must either: + - Inherit from `nn.ModuleList` (naive). + - Or contain all 3D `nn.Parameters` (packed). + 3. `MyExperts.forward` must accept `hidden_states`, `top_k_index`, `top_k_weights`. + +!!! note + `MyModel` no longer needs `_supports_attention_backend = True`, and `kwargs` no + longer need to be passed down through every module from `MyModel` to + `MyAttention`. vLLM reaches its attention layer through `MyAttention` itself, so + all it asks is that `MyAttention` dispatches through the interface.
modeling_my_model.py @@ -97,14 +104,24 @@ from torch import nn class MyAttention(nn.Module): is_causal = False # Only do this for encoder-only models + def __init__(self, config, layer_idx): + ... + self.config = config + self.layer_idx = layer_idx + self.scaling = self.head_dim**-0.5 + ... + def forward(self, hidden_states, **kwargs): ... - attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, eager_attention_forward + ) attn_output, attn_weights = attention_interface( self, query_states, key_states, value_states, + scaling=self.scaling, **kwargs, ) ... @@ -127,7 +144,7 @@ class MySparseMoEBlock(nn.Module): ... class MyModel(PreTrainedModel): - _supports_attention_backend = True + ... ```
@@ -135,7 +152,7 @@ class MyModel(PreTrainedModel): Here is what happens in the background when this model is loaded: 1. The config is loaded. -2. `MyModel` Python class is loaded from the `auto_map` in config, and we check that the model `is_backend_compatible()`. +2. `MyModel` Python class is loaded from the `auto_map` in config, and we check that the model `_can_set_attn_implementation()`. 3. `MyModel` is loaded into one of the Transformers modeling backend classes in [vllm/model_executor/models/transformers](../../vllm/model_executor/models/transformers) which sets `self.config._attn_implementation = "vllm"` so that vLLM's attention layer is used. That's it! diff --git a/tests/models/transformers/fusers/test_linear.py b/tests/models/transformers/fusers/test_linear.py index ba6fc483ac..020459d7d9 100644 --- a/tests/models/transformers/fusers/test_linear.py +++ b/tests/models/transformers/fusers/test_linear.py @@ -10,7 +10,7 @@ import torch import torch.nn as nn import torch.nn.functional as F -from vllm.model_executor.models.transformers.fuser import get_fuser +from vllm.model_executor.models.transformers.fuser import get_fuser, get_fusers from vllm.model_executor.models.transformers.fusers import ( GLUFuser, PackedQKVFuser, @@ -307,7 +307,7 @@ class PerHeadSplitAttention(nn.Module): class FakeSelfAttn(nn.Module): - """Stand-in for the vLLM `Attention` looked up in `attention_instances`.""" + """Stand-in for the vLLM `Attention` attached to the dispatching module.""" def __init__(self): super().__init__() @@ -328,9 +328,9 @@ class FakeMQASelfAttn(FakeSelfAttn): @pytest.fixture(autouse=True) def _clear_fuser_cache(): - get_fuser.cache_clear() + get_fusers.cache_clear() yield - get_fuser.cache_clear() + get_fusers.cache_clear() def _apply_glu_fuser_with_stubs(module: nn.Module, fuser: GLUFuser): @@ -392,7 +392,7 @@ def _apply_packed_qkv_fuser_with_stubs(module: nn.Module, fuser: PackedQKVFuser) def test_detects_and_rewrites_glu(mlp_cls, bias): with torch.device("meta"): meta = mlp_cls(bias=bias) - fuser = get_fuser(meta) + fuser = get_fuser(meta, GLUFuser) assert isinstance(fuser, GLUFuser) assert ( fuser.gate_name, @@ -427,9 +427,9 @@ def test_glu_identifies_down_projection(): matches the column-parallel merged gate/up; `None` when there is no such projection to force (fusion of gate/up still applies).""" with torch.device("meta"): - assert get_fuser(GLUMLP()).down_name == "down_proj" - assert get_fuser(ReversedGLUMLP()).down_name == "down_proj" - assert get_fuser(NoDownGLU()).down_name is None + assert get_fuser(GLUMLP(), GLUFuser).down_name == "down_proj" + assert get_fuser(ReversedGLUMLP(), GLUFuser).down_name == "down_proj" + assert get_fuser(NoDownGLU(), GLUFuser).down_name is None @pytest.mark.parametrize("attn_cls", [FakeAttention, ReversedFakeAttention]) @@ -439,7 +439,7 @@ def test_detects_and_rewrites_qkv(attn_cls, kv_heads): pytest.skip("MHA q/k/v assignment is order-based by design") with torch.device("meta"): meta = attn_cls(kv_heads=kv_heads) - fuser = get_fuser(meta) + fuser = get_fuser(meta, QKVFuser) assert isinstance(fuser, QKVFuser) # q (sharded differently under TP) must be identified exactly; k/v may be # swapped for non-canonical compute order, which is numerically consistent @@ -467,27 +467,27 @@ def test_detects_and_rewrites_qkv(attn_cls, kv_heads): for p in real.parameters(): nn.init.normal_(p, std=0.05) x = torch.randn(1, 5, 32) - attention_instances = {3: FakeSelfAttn()} - expected, _ = real(x, attention_instances=attention_instances) + real.attn = FakeSelfAttn() + expected, _ = real(x) fused = _apply_qkv_fuser_with_stubs(real, fuser) # Fusion is in place: the module keeps its class and other attributes assert fused is real and type(fused) is attn_cls assert fused.layer_idx == 3 and fused.is_causal and fused.config is not None - out, _ = fused(x, attention_instances=attention_instances) + out, _ = fused(x) torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5) def test_qkv_identifies_output_projection(): with torch.device("meta"): - assert get_fuser(FakeAttention()).o_name == "o_proj" - assert get_fuser(ReversedFakeAttention()).o_name == "o_proj" - assert get_fuser(ExtraProjAttention()).o_name == "o_proj" + assert get_fuser(FakeAttention(), QKVFuser).o_name == "o_proj" + assert get_fuser(ReversedFakeAttention(), QKVFuser).o_name == "o_proj" + assert get_fuser(ExtraProjAttention(), QKVFuser).o_name == "o_proj" # Norm children (q_norm/k_norm) must not disturb o_proj identification. - assert get_fuser(QKNormAttention()).o_name == "o_proj" - assert get_fuser(PerHeadQKNormAttention()).o_name == "o_proj" + assert get_fuser(QKNormAttention(), QKVFuser).o_name == "o_proj" + assert get_fuser(PerHeadQKNormAttention(), QKVFuser).o_name == "o_proj" # A module between o_proj and the return is transparent. - assert get_fuser(ResidDropoutAttention()).o_name == "o_proj" + assert get_fuser(ResidDropoutAttention(), QKVFuser).o_name == "o_proj" @pytest.mark.parametrize("kv_heads", [1, 2]) @@ -498,7 +498,7 @@ def test_detects_and_rewrites_packed_qkv(kv_heads): checkpoint weight as-is, and shards q by heads while replicating k/v.""" with torch.device("meta"): meta = PackedQKVAttention(kv_heads=kv_heads) - fuser = get_fuser(meta) + fuser = get_fuser(meta, PackedQKVFuser) assert isinstance(fuser, PackedQKVFuser) assert (fuser.qkv_name, fuser.o_name) == ("c_attn", "c_proj") assert (fuser.q_size, fuser.kv_size) == (32, 8 * kv_heads) @@ -513,14 +513,14 @@ def test_detects_and_rewrites_packed_qkv(kv_heads): for p in real.parameters(): nn.init.normal_(p, std=0.05) x = torch.randn(1, 5, 32) - attention_instances = {3: FakeMQASelfAttn()} - expected, _ = real(x, attention_instances=attention_instances) + real.attn = FakeMQASelfAttn() + expected, _ = real(x) fused = _apply_packed_qkv_fuser_with_stubs(real, fuser) # Fusion is in place: the module keeps its class and other attributes assert fused is real and type(fused) is PackedQKVAttention assert fused.layer_idx == 3 and fused.is_causal - out, _ = fused(x, attention_instances=attention_instances) + out, _ = fused(x) torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5) @@ -528,28 +528,28 @@ def test_per_head_split_is_not_packed_qkv(): """The split must consume the whole projection, else its sizes are head widths and re-sharding by them would be wrong.""" with torch.device("meta"): - assert get_fuser(PerHeadSplitAttention()) is None + assert get_fuser(PerHeadSplitAttention(), PackedQKVFuser) is None def test_fuser_is_cached_per_class_and_structure(): with torch.device("meta"): - fuser_a = get_fuser(GLUMLP()) - fuser_b = get_fuser(GLUMLP()) + fuser_a = get_fuser(GLUMLP(), GLUFuser) + fuser_b = get_fuser(GLUMLP(), GLUFuser) assert fuser_a is fuser_b - assert any(key[0] is GLUMLP for key in get_fuser.cache) + assert any(key[0] is GLUMLP for key in get_fusers.cache) @pytest.mark.parametrize("cls", [NotAnMLP, UntraceableMLP]) def test_non_matching_modules_return_none(cls): with torch.device("meta"): module = cls() - assert get_fuser(module) is None + assert get_fuser(module, GLUFuser) is None def test_untraceable_tail_still_fuses(): with torch.device("meta"): meta = UntraceableTailGLUMLP() - fuser = get_fuser(meta) + fuser = get_fuser(meta, GLUFuser) assert isinstance(fuser, GLUFuser) # Numerics: the live tail must survive the rewrite @@ -566,8 +566,8 @@ def test_weight_mappings_are_scoped_to_fused_prefixes(): from vllm.model_executor.models.utils import WeightsMapper with torch.device("meta"): - glu_fuser = get_fuser(GLUMLP()) - qkv_fuser = get_fuser(FakeAttention()) + glu_fuser = get_fuser(GLUMLP(), GLUFuser) + qkv_fuser = get_fuser(FakeAttention(), QKVFuser) mapper = WeightsMapper() for prefix in ("model.layers.0.mlp", "model.layers.1.mlp"): @@ -620,7 +620,7 @@ def test_weight_mappings_are_scoped_to_fused_prefixes(): def test_unfusable_modules_are_not_fused(cls, default_vllm_config): with torch.device("meta"): module = cls() - fuser = get_fuser(module) + fuser = get_fuser(module, GLUFuser) # Either no pattern matches the class, or this instance fails validation # (`recursive_replace` gates fusion and its weight mappings on `validate`) assert fuser is None or not fuser.validate(module, default_vllm_config) @@ -654,9 +654,15 @@ def _wider_model_config(head_dim: int) -> SimpleNamespace: @pytest.mark.parametrize( - "cls, fuser_module", [(FakeAttention, qkv), (PackedQKVAttention, packed_qkv)] + "cls, fuser_module, fuser_cls", + [ + (FakeAttention, qkv, QKVFuser), + (PackedQKVAttention, packed_qkv, PackedQKVFuser), + ], ) -def test_head_counts_come_from_the_module_not_the_model(cls, fuser_module, monkeypatch): +def test_head_counts_come_from_the_module_not_the_model( + cls, fuser_module, fuser_cls, monkeypatch +): """A layer narrower than the model-wide head size must not be miscounted. On a heterogeneous checkpoint (Gemma 4) the model-wide head size is the @@ -680,7 +686,7 @@ def test_head_counts_come_from_the_module_not_the_model(cls, fuser_module, monke monkeypatch.setattr( fuser_module, "replace_linear_class", lambda *a, **kw: nn.Identity() ) - fuser = get_fuser(module) + fuser = get_fuser(module, fuser_cls) assert fuser is not None and fuser.validate(module, vllm_config) fuser.update_attrs(module, "model.layers.0.self_attn", vllm_config) @@ -700,5 +706,5 @@ def test_validate_accepts_a_layer_the_model_wide_head_size_would_reject(): # kv width is 24, not a multiple of the model-wide 16, but is of this # layer's 8. - fuser = get_fuser(module) + fuser = get_fuser(module, QKVFuser) assert fuser is not None and fuser.validate(module, vllm_config) diff --git a/tests/models/transformers/fusers/test_mla.py b/tests/models/transformers/fusers/test_mla.py index 2358b5c59a..d61e54e923 100644 --- a/tests/models/transformers/fusers/test_mla.py +++ b/tests/models/transformers/fusers/test_mla.py @@ -158,4 +158,4 @@ class GLU(nn.Module): def test_non_mla_is_not_matched(): with torch.device("meta"): - assert not isinstance(get_fuser(GLU()), MLAFuser) + assert get_fuser(GLU(), MLAFuser) is None diff --git a/tests/models/transformers/fusers/test_rms_norm.py b/tests/models/transformers/fusers/test_rms_norm.py index ac259b66b4..153f7863bd 100644 --- a/tests/models/transformers/fusers/test_rms_norm.py +++ b/tests/models/transformers/fusers/test_rms_norm.py @@ -162,7 +162,7 @@ class UntraceableGatedRMSNorm(RMSNorm): ) def test_detects_rms_norm_variants(cls, eps, zero_centered): with torch.device("meta"): - fuser = get_fuser(cls(16, eps=eps)) + fuser = get_fuser(cls(16, eps=eps), RMSNormFuser) assert isinstance(fuser, RMSNormFuser) assert fuser.zero_centered == zero_centered @@ -171,7 +171,7 @@ def test_detects_rms_norm_variants(cls, eps, zero_centered): def test_non_rms_norms_are_not_matched(cls): with torch.device("meta"): module = cls(16) if cls is nn.LayerNorm else cls() - assert not isinstance(get_fuser(module), RMSNormFuser) + assert get_fuser(module, RMSNormFuser) is None @pytest.mark.parametrize( @@ -179,7 +179,7 @@ def test_non_rms_norms_are_not_matched(cls): ) def test_gated_rms_norm_is_not_fused(cls): with torch.device("meta"): - assert not isinstance(get_fuser(cls()), RMSNormFuser) + assert get_fuser(cls(), RMSNormFuser) is None @pytest.mark.parametrize( @@ -196,7 +196,7 @@ def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_c with torch.device("meta"): module = cls() - fuser = get_fuser(module) + fuser = get_fuser(module, RMSNormFuser) built = fuser.fuse(module, "norm", default_vllm_config) from vllm.model_executor.models.transformers.fusers.rms_norm import ( TPAwareNormMixin, @@ -221,7 +221,7 @@ def test_weightless_norm_has_no_hidden_size(default_vllm_config): matching the unfused norm and vLLM's native `RMSNorm(hidden_size=head_dim)`. """ module = WeightlessRMSNorm(72) - built = get_fuser(module).fuse(module, "norm", default_vllm_config) + built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config) assert built.hidden_size == 0 built.tp_size = 2 # emulate TP=2 without a real process group @@ -236,7 +236,7 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config): with torch.device("meta"): module = torch.nn.RMSNorm(16) # forward is a single `F.rms_norm` call - fuser = get_fuser(module) + fuser = get_fuser(module, RMSNormFuser) assert isinstance(fuser, RMSNormFuser) assert not fuser.zero_centered vllm_config = SimpleNamespace(model_config=SimpleNamespace(dtype=torch.float32)) @@ -251,7 +251,8 @@ def test_eps_is_derived_per_instance(default_vllm_config): with torch.device("meta"): for eps in (1e-5, 1e-6): module = RMSNorm(16, eps=eps) - built = get_fuser(module).fuse(module, "norm", default_vllm_config) + fuser = get_fuser(module, RMSNormFuser) + built = fuser.fuse(module, "norm", default_vllm_config) assert built.variance_epsilon == eps @@ -262,7 +263,7 @@ def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config): with torch.device("meta"): for eps in (1e-5, 1e-6): module = cls(16, eps=eps) - fuser = get_fuser(module) + fuser = get_fuser(module, RMSNormFuser) assert fuser.eps_attr == cls.attr built = fuser.fuse(module, "norm", default_vllm_config) assert built.variance_epsilon == eps @@ -274,7 +275,7 @@ def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplo logger = "vllm.model_executor.models.transformers.fusers.rms_norm" with caplog.at_level("DEBUG", logger=logger), torch.device("meta"): module = LiteralEpsRMSNorm() - fuser = get_fuser(module) + fuser = get_fuser(module, RMSNormFuser) built = fuser.fuse(module, "norm", default_vllm_config) assert fuser.eps_attr is None assert built.variance_epsilon == 1e-4 @@ -287,7 +288,7 @@ def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config): with torch.device("meta"): module = AmbiguousEpsRMSNorm(16, eps=1e-6) before = dict(vars(module)) - fuser = get_fuser(module) + fuser = get_fuser(module, RMSNormFuser) assert fuser.eps_attr == "variance_epsilon" assert vars(module) == before diff --git a/tests/models/transformers/test_backend.py b/tests/models/transformers/test_backend.py index 0429f3da2c..40c0434c92 100644 --- a/tests/models/transformers/test_backend.py +++ b/tests/models/transformers/test_backend.py @@ -2,9 +2,11 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Test the functionality of the Transformers modeling backend.""" +import ast import contextlib import os import tempfile +from types import SimpleNamespace from typing import Any import pytest @@ -15,6 +17,8 @@ from transformers import AutoConfig, AutoModel, PretrainedConfig from vllm.config import ModelConfig, VllmConfig from vllm.model_executor.models.interfaces import SupportsMultiModal from vllm.model_executor.models.transformers.base import Base +from vllm.model_executor.models.transformers.fusers import AttentionFuser +from vllm.model_executor.models.transformers.fusers.attention import VLLM_ATTN_IMPL from vllm.model_executor.models.transformers.multimodal import MultiModalMixin from vllm.model_executor.models.utils import StageMissingLayer @@ -583,3 +587,141 @@ def test_marking_skipped_without_tokenizer(): """ vision_tower = build_marked_model(0, skip_tokenizer_init=True).vision_tower assert not isinstance(vision_tower, StageMissingLayer) + + +NUM_LAYERS = 4 + + +def build_model(model_type: str, **overrides) -> nn.Module: + """A tiny HF model of `NUM_LAYERS` layers, built on the meta device.""" + try: + config = AutoConfig.for_model( + model_type, + num_hidden_layers=NUM_LAYERS, + hidden_size=64, + intermediate_size=128, + num_attention_heads=4, + num_key_value_heads=2, + head_dim=16, + vocab_size=VOCAB_SIZE, + **overrides, + ) + except ValueError: + pytest.skip(f"The installed transformers has no {model_type!r} model") + with torch.device("meta"): + return AutoModel.from_config(config) + + +ATTENTION_MODEL_TYPES = [ + # The default `head_size**-0.5` happens to be right here + "llama", + # Scales the query by a learnable per-dim weight, so it declares 1.0 + "gemma4_text", + # MLA, and declares yarn's `mscale` + "deepseek_v3", +] + + +@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES) +def test_attention_dispatch_is_matched(model_type: str): + """Exactly the decoder layers' attention modules match an `AttentionFuser`. + + That match is what `recursive_replace` records, and so what + `create_attention_instances` attaches vLLM's attention layer to. + """ + model = build_model(model_type) + matched = { + name + for name, module in model.named_modules() + if AttentionFuser.match(None, module) is not None + } + assert matched == {f"layers.{i}.self_attn" for i in range(NUM_LAYERS)} + + +def test_attention_layer_index_is_the_modules_own(): + """The layer served comes from the module, not from its position in the stack. + + LongCat Flash gives each decoder layer two attention sublayers numbered + `2i` and `2i + 1`, so `num_hidden_layers` is twice the length of the stack + and the enclosing layer's position is not the index the KV cache is keyed by. + A vision tower is excluded separately, by `validate`: only the text config is + patched to dispatch to vLLM, so a tower's attention is left to Transformers. + """ + model = build_model("longcat_flash", num_layers=2) + text_config = model.config.get_text_config() + assert len(model.layers) * 2 == text_config.num_hidden_layers + + attentions = [ + model.get_submodule(f"layers.{i}.self_attn.{j}") for i in (0, 1) for j in (0, 1) + ] + fusers = [AttentionFuser.match(None, attn) for attn in attentions] + assert all(fuser is not None for fuser in fusers) + assert [f.layer_index(a) for f, a in zip(fusers, attentions)] == [0, 1, 2, 3] + + # Not dispatching to vLLM yet, as a vision tower never would be. The vLLM + # config is not consulted, only the one the module was built with. + assert not any(f.validate(a, None) for f, a in zip(fusers, attentions)) + text_config._attn_implementation = VLLM_ATTN_IMPL + assert all(f.validate(a, None) for f, a in zip(fusers, attentions)) + + +def test_attention_dispatch_is_required(monkeypatch: pytest.MonkeyPatch): + """Models selected by the permissive registry fail clearly without dispatch.""" + monkeypatch.setattr( + "vllm.model_executor.models.transformers.base.get_pp_indices", + lambda *_: (0, 1), + ) + model = SimpleNamespace( + text_config=SimpleNamespace(num_hidden_layers=1), + pp_group=SimpleNamespace(rank_in_group=0, world_size=1), + attention_fusers={}, + _get_attn_cls=lambda: None, + ) + + with pytest.raises( + ValueError, + match="Layer 0 does not dispatch through the Transformers attention interface", + ): + Base.create_attention_instances(model) + + +@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES) +def test_attention_scale_is_the_declared_one(model_type: str): + """The scale comes off the module, not from the `head_size**-0.5` default. + + Two of the three model types declare something the default would silently + replace, so they are what makes this more than a tautology. + """ + attention = build_model(model_type).layers[0].self_attn + fuser = AttentionFuser.match(None, attention) + assert fuser is not None + assert fuser.scale(attention) == attention.scaling + + +def test_attention_scale_is_the_argument_not_the_attribute(): + """OPT applies the scale to the query itself, then declares a literal 1.0. + + Its `self.scaling` is the `head_size**-0.5` it has already applied, so + reading the attribute rather than the argument the module hands the + interface would scale twice. + """ + model = build_model("opt", ffn_dim=128, word_embed_proj_dim=64) + attention = model.get_submodule("decoder.layers.0.self_attn") + fuser = AttentionFuser.match(None, attention) + assert fuser is not None + assert fuser.scale(attention) == 1.0 != attention.scaling + + +def test_attention_scale_rejects_unresolvable_expression(): + """Unresolvable declared scales must not silently use the Llama default.""" + + class Attention(nn.Module): + head_dim = 16 + + fuser = AttentionFuser( + source_cls="Attention", + scale_expr=ast.parse("self.head_dim**-0.5 * self.factor", mode="eval").body, + ) + + with pytest.raises(ValueError, match="Cannot resolve attention scaling expression"): + fuser.scale(Attention()) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index dabd1e4460..2f030ec1b7 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -1253,7 +1253,10 @@ class _ModelRegistry: "'auto_map' (relevant if the model is custom)." ) - if not model_module.is_backend_compatible(): + if not ( + model_module.is_backend_compatible() + or model_module._can_set_attn_implementation() + ): if model_config.model_impl != "transformers": return None diff --git a/vllm/model_executor/models/transformers/__init__.py b/vllm/model_executor/models/transformers/__init__.py index 7a7aab7266..1ec5849408 100644 --- a/vllm/model_executor/models/transformers/__init__.py +++ b/vllm/model_executor/models/transformers/__init__.py @@ -21,7 +21,7 @@ from typing import TYPE_CHECKING import torch.nn.functional as F from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS -from vllm.model_executor.models.transformers.base import Base +from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR, Base from vllm.model_executor.models.transformers.causal import CausalMixin from vllm.model_executor.models.transformers.legacy import LegacyMixin from vllm.model_executor.models.transformers.moe import MoEMixin @@ -40,8 +40,6 @@ from vllm.multimodal import MULTIMODAL_REGISTRY if TYPE_CHECKING: import torch - from vllm.model_executor.layers.attention import Attention, MLAAttention - def vllm_attention_forward( # Transformers args @@ -50,15 +48,9 @@ def vllm_attention_forward( key: "torch.Tensor", value: "torch.Tensor", attention_mask: "torch.Tensor", - # Transformers kwargs - scaling: float | None = None, - # vLLM kwargs - attention_instances: "dict[int, Attention] | None" = None, **kwargs, ): - self_attn = attention_instances[module.layer_idx] - if scaling is not None: - self_attn.impl.scale = float(scaling) + self_attn = getattr(module, VLLM_ATTN_ATTR) hidden = query.shape[-2] head_dim_qk = query.shape[-1] head_dim_v = value.shape[-1] @@ -85,13 +77,9 @@ def vllm_mla_attention_forward( kv_c_normed: "torch.Tensor", k_pe: "torch.Tensor", attention_mask: "torch.Tensor", - # Transformers kwargs - scaling: float | None = None, - # vLLM kwargs - attention_instances: "dict[int, MLAAttention] | None" = None, **kwargs, ): - self_attn = attention_instances[module.layer_idx] + self_attn = getattr(module, VLLM_ATTN_ATTR) # [batch=1, heads, num_tokens, qk_head_dim] -> [num_tokens, heads, qk_head_dim] query = query.transpose(1, 2).flatten(0, 1) num_tokens, num_heads = query.shape[:2] diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py index 11b7c8466c..5ee2751a4f 100644 --- a/vllm/model_executor/models/transformers/base.py +++ b/vllm/model_executor/models/transformers/base.py @@ -57,7 +57,11 @@ from vllm.model_executor.models.interfaces import ( ) from vllm.model_executor.models.interfaces_base import VllmModel from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers -from vllm.model_executor.models.transformers.fusers import MLAFuser +from vllm.model_executor.models.transformers.fusers import AttentionFuser, MLAFuser +from vllm.model_executor.models.transformers.fusers.attention import ( + VLLM_ATTN_IMPL, + VLLM_MLA_ATTN_IMPL, +) from vllm.model_executor.models.transformers.utils import ( attrsetter, can_enable_torch_compile, @@ -73,7 +77,6 @@ from vllm.model_executor.models.utils import ( AutoWeightsLoader, PPMissingLayer, WeightsMapper, - extract_layer_index, make_empty_intermediate_tensors_factory, maybe_prefix, ) @@ -86,6 +89,11 @@ if TYPE_CHECKING: logger = init_logger(__name__) +VLLM_ATTN_ATTR = "attn" +"""Attribute in which to store the vLLM `Attention` instance that serves a Transformers +attention module. `"attn"` is chosen deliberately to match the mappings generated by +`maybe_remap_kv_scale_name`.""" + class PreTrainedModelClasses(NamedTuple): decoder: type["PreTrainedModel"] @@ -129,9 +137,13 @@ class Base( self.packed_modules_mapping: dict[str, list[str]] = {} """Fused module -> constituent projections, populated by `recursive_replace` for the quantization machinery and loaders (e.g. bitsandbytes).""" - self.fusers: dict[str, BaseFuser] = {} - """Module qualname -> the fuser applied to it, populated + self.fusers: dict[str, list[BaseFuser]] = {} + """Module qualname -> the fusers applied to it, populated by `recursive_replace` for `create_attention_instances`.""" + self.attention_fusers: dict[int, tuple[str, AttentionFuser]] = {} + """`layer_idx` -> the qualname and fuser of the module computing that + layer's attention, populated by `recursive_replace` for + `create_attention_instances`.""" # Attrs for Eagle3 (see self.set_aux_hidden_state_layers) self._target_class: type[nn.Module] = nn.Module @@ -189,7 +201,7 @@ class Base( - Sets the dtype to the default torch dtype set by vLLM because Transformers uses the config dtype when creating the model """ - self.text_config._attn_implementation = "vllm" + self.text_config._attn_implementation = VLLM_ATTN_IMPL self.config.dtype = torch.get_default_dtype() @contextmanager @@ -475,9 +487,22 @@ class Base( vocab_embeddings = self._vocab_embeddings() - def register_fusion(fuser: BaseFuser, prefix: str): + def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module): """Register a fused layer's mappings just before it is built.""" - self.fusers[prefix] = fuser + self.fusers.setdefault(prefix, []).append(fuser) + + if ( + isinstance(fuser, AttentionFuser) + and (index := fuser.layer_index(module)) is not None + ): + if index in self.attention_fusers: + other, _ = self.attention_fusers[index] + raise ValueError( + f"Layer {index} dispatches attention from more than one " + f"module ({other} and {prefix}), which one vLLM attention " + "layer cannot serve." + ) + self.attention_fusers[index] = (prefix, fuser) orig_to_new_stacked = fuser.orig_to_new_stacked(prefix) self.hf_to_vllm_mapper.orig_to_new_stacked.update(orig_to_new_stacked) @@ -528,10 +553,11 @@ class Base( new_module = replace_embedding_class( child_module, self.quant_config, prefix=qual_name ) - elif (fuser := fusers[child_module]) is not None: - register_fusion(fuser, qual_name) - new_module = fuser.fuse(child_module, qual_name, self.vllm_config) - logger.info_once(fuser.info(child_name)) + elif child_module_fusers := fusers[child_module]: + for fuser in child_module_fusers: + register_fusion(fuser, qual_name, child_module) + new_module = fuser.fuse(new_module, qual_name, self.vllm_config) + logger.info_once(fuser.info(child_name)) _recursive_replace(new_module, prefix=qual_name) elif not isinstance(child_module, MoERunner): # MoERunner can contain aliases of shared experts and gates, @@ -548,20 +574,14 @@ class Base( """ Create `Attention` instances to inform KV cache allocation. """ - mla_fusers = {} attention_instances = {} text_config = self.text_config attn_cls = self._get_attn_cls() # kv_lora_rank indicates that this is an MLA model if getattr(text_config, "kv_lora_rank", None) is not None: - mla_fusers = { - extract_layer_index(prefix): (prefix, fuser) - for prefix, fuser in self.fusers.items() - if isinstance(fuser, MLAFuser) - } if attn_cls is MLAAttention: - text_config._attn_implementation = "vllm_mla" + text_config._attn_implementation = VLLM_MLA_ATTN_IMPL else: # MLA model not using MLAAttention: recompute head_size for full attn qk_nope_head_dim = getattr(text_config, "qk_nope_head_dim", 0) @@ -570,12 +590,23 @@ class Base( self.model_config.model_arch_config.head_size = qk_head_dim logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None) + layer_types = getattr(text_config, "layer_types", None) pp_rank = self.pp_group.rank_in_group pp_size = self.pp_group.world_size start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size) for i in range(start, end): + if i not in self.attention_fusers: + in_range = layer_types and i < len(layer_types) + layer = f"{i} ({layer_types[i]})" if in_range else str(i) + raise ValueError( + f"Layer {layer} does not dispatch through the Transformers " + "attention interface and vLLM has no other way to handle it." + ) + prefix, attn_fuser = self.attention_fusers[i] + attn_module = self.get_submodule(prefix) + # `[i]` is the whole-model config unless the checkpoint is # heterogeneous, in which case it is this layer's own geometry. arch_config = self.model_config.model_arch_config[i] @@ -583,8 +614,10 @@ class Base( self.parallel_config, arch_config ) head_size = arch_config.head_size - # Default to Llama scale, maybe updated in vllm_attention_forward - scale = head_size**-0.5 + scale = attn_fuser.scale(attn_module) + # Default to Llama scale if AttentionFuser couldn't identify it + if scale is None: + scale = head_size**-0.5 num_kv_heads = self.model_config.get_num_kv_heads( self.parallel_config, arch_config ) @@ -598,17 +631,22 @@ class Base( ) if attn_cls is MLAAttention: - prefix, fuser = mla_fusers[i] - mla_module = self.get_submodule(prefix) + mla_fuser = next( + (f for f in self.fusers[prefix] if isinstance(f, MLAFuser)), None + ) + if mla_fuser is None: + raise ValueError( + f"Layer {i} ({prefix}) did not fuse as MLA, but the rest of " + "the model did, so its latent KV cache cannot be allocated." + ) dims = get_mla_dims(self.model_config) kwargs.update( - scale=mla_module.scaling, qk_nope_head_dim=dims.qk_nope_head_dim, qk_rope_head_dim=dims.qk_rope_head_dim, v_head_dim=dims.v_head_dim, q_lora_rank=dims.q_lora_rank, kv_lora_rank=dims.kv_lora_rank, - kv_b_proj=mla_module.get_submodule(fuser.kv_b_proj_name), + kv_b_proj=attn_module.get_submodule(mla_fuser.kv_b_proj_name), ) else: kwargs.update( @@ -618,17 +656,20 @@ class Base( ) # Handle interleaved sliding window attention - if ( - hasattr(text_config, "layer_types") - and text_config.layer_types[i] == "sliding_attention" - ): + if layer_types and layer_types[i] == "sliding_attention": kwargs["per_layer_sliding_window"] = text_config.sliding_window attn_instance = attn_cls(**kwargs) - if attn_cls is MLAAttention: - # Attach MLA attn_instance to mla_module so it appears in - # model.named_modules() and runs its process_weights_after_loading - mla_module._vllm_mla_attn = attn_instance + + # Attach the instance to the module it serves, which is the only way it + # is in the module tree at all: a plain dict is invisible to + # `nn.Module.__setattr__`. Being in the tree runs its + # `process_weights_after_loading` and moves its KV-scale buffers with + # `model.to(device)`, and lets the attention interface read it off the + # module rather than index by `module.layer_idx`, an `int` that bakes + # layer identity into the traced graph and so costs one compiled + # artifact per layer. + setattr(attn_module, VLLM_ATTN_ATTR, attn_instance) attention_instances[i] = attn_instance return attention_instances @@ -645,7 +686,11 @@ class Base( return EncoderOnlyAttention if self.model_config.use_mla: self.check_version("5.15.0.dev0", "optimized MLA support") - if any(isinstance(fuser, MLAFuser) for fuser in self.fusers.values()): + if any( + isinstance(fuser, MLAFuser) + for fusers in self.fusers.values() + for fuser in fusers + ): return MLAAttention logger.warning_once( "This model uses MLA but `MLAFuser` failed to match and/or fuse any " @@ -731,7 +776,6 @@ class Base( inputs_embeds=inputs_embeds, use_cache=False, position_ids=positions, - attention_instances=self.attention_instances, return_dict=False, **self._output_aux_hidden_states_kwargs, **kwargs, diff --git a/vllm/model_executor/models/transformers/fuser.py b/vllm/model_executor/models/transformers/fuser.py index 52a08ac90b..6f211af0d3 100644 --- a/vllm/model_executor/models/transformers/fuser.py +++ b/vllm/model_executor/models/transformers/fuser.py @@ -2,20 +2,22 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Fuser detection for the Transformers modeling backend. -`get_fuser` traces a module class once (see `fx_utils`) and matches it against -each concrete fuser in `fusers`; `Fusers` caches the result per class for a -whole model. `base.recursive_replace` then applies the matched fuser per +`get_fusers` traces a module class once (see `fx_utils`) and matches it against each +concrete fuser in `fusers`; `Fusers` caches the result per class for a whole model. +At most one match may give the module a different forward, and any number of fusers that +leave the forward alone apply alongside. `base.recursive_replace` then applies them per instance. RMSNorm-shaped modules the tracer cannot match are warned about. """ from collections import UserDict -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypeVar from cachetools import cached from torch import nn from vllm.logger import init_logger from vllm.model_executor.models.transformers.fusers import ( + AttentionFuser, BaseFuser, GLUFuser, MLAFuser, @@ -31,58 +33,90 @@ if TYPE_CHECKING: logger = init_logger(__name__) +F = TypeVar("F", bound=BaseFuser) + def key(module: nn.Module) -> tuple: - """Cache key for `get_fuser`. Considers module type and its immediate children.""" + """Cache key for `get_fusers`. Considers module type and its immediate children.""" return (type(module), tuple(name for name, _ in module.named_children())) +FUSERS: tuple[type[BaseFuser], ...] = ( + MLAFuser, + GLUFuser, + QKVFuser, + PackedQKVFuser, + RMSNormFuser, + AttentionFuser, +) +"""Every fuser, in match order: those that redefine the forward first, then those +that leave it alone. A new fuser is added here.""" + + @cached(cache={}, key=key) -def get_fuser(module: nn.Module) -> BaseFuser | None: - """The fuser for `module`'s class and shape (cached), or `None` if no match.""" +def get_fusers(module: nn.Module) -> list[BaseFuser]: + """Every fuser that applies to `module`'s class and shape (cached). + + The one that redefines the forward comes first, so applying them in order + fuses before the rest read or adjust the result.""" # Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a # leaf module (raw tensor math, no submodules). Nothing else can match, and # tracing is skipped for it. n_linear = sum(isinstance(c, nn.Linear) for c in module.children()) is_leaf = next(module.children(), None) is None - if n_linear < 2 and not is_leaf: - return None - if (graph := trace(module)) is None: - return None - for fuser_cls in (MLAFuser, GLUFuser, QKVFuser, PackedQKVFuser, RMSNormFuser): - if (fuser := fuser_cls.match(graph, module)) is not None: - if isinstance(fuser, RewriteFuser): - try: - fuser.update_forward(module) - except Exception as exc: - logger.debug( - "Attempted to fuse %s using %s but failed " - "to update its forward method: %s", - type(module), - fuser_cls.__name__, - exc, - ) - continue - return fuser + graph = trace(module) if n_linear >= 2 or is_leaf else None + + fusers: list[BaseFuser] = [] + for fuser_cls in FUSERS: + if fuser_cls.redefines_forward and ( + # A forward can only be rewritten from a trace of it, and only once, + # because every rewrite starts from the original source. + graph is None or any(fuser.redefines_forward for fuser in fusers) + ): + continue + if (fuser := fuser_cls.match(graph, module)) is None: + continue + if isinstance(fuser, RewriteFuser): + try: + fuser.update_forward(module) + except Exception as exc: + logger.debug( + "Attempted to fuse %s using %s but failed " + "to update its forward method: %s", + type(module), + fuser_cls.__name__, + exc, + ) + continue + fusers.append(fuser) + # A norm we could not match structurally is left unfused; flag likely misses. - if module.__class__.__name__.endswith("RMSNorm"): + if ( + module.__class__.__name__.endswith("RMSNorm") + and graph is not None + and not any(isinstance(fuser, RMSNormFuser) for fuser in fusers) + ): logger.warning_once( "%s looks like an RMSNorm but its computation did not match the " "expected pattern, so it was left unfused.", module.__class__.__name__, ) - return None + return fusers + + +def get_fuser(module: nn.Module, fuser_cls: type[F]) -> "F | None": + """The first `fuser_cls` that applies to `module`, if one does.""" + return next((f for f in get_fusers(module) if isinstance(f, fuser_cls)), None) class Fusers(UserDict): - """Mapping from module class and shape to fuser, for all fusable modules.""" + """Mapping from module class and shape to fusers, for all fusable modules.""" def __init__(self, model: nn.Module, vllm_config: "VllmConfig"): self.vllm_config = vllm_config - super().__init__({key(m): get_fuser(m) for m in model.modules()}) + super().__init__({key(m): get_fusers(m) for m in model.modules()}) - def __getitem__(self, m: nn.Module) -> BaseFuser | None: - fuser = self.data.get(key(m)) - if fuser is not None and fuser.validate(m, self.vllm_config): - return fuser - return None + def __getitem__(self, m: nn.Module) -> list[BaseFuser]: + """The fusers this instance can take, in the order to apply them.""" + fusers = self.data.get(key(m), ()) + return [fuser for fuser in fusers if fuser.validate(m, self.vllm_config)] diff --git a/vllm/model_executor/models/transformers/fusers/__init__.py b/vllm/model_executor/models/transformers/fusers/__init__.py index a0772ed4a2..735392a3ab 100644 --- a/vllm/model_executor/models/transformers/fusers/__init__.py +++ b/vllm/model_executor/models/transformers/fusers/__init__.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Concrete fusers for the Transformers modeling backend.""" +from vllm.model_executor.models.transformers.fusers.attention import AttentionFuser from vllm.model_executor.models.transformers.fusers.base import ( BaseFuser, RewriteFuser, @@ -15,6 +16,7 @@ from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser __all__ = [ + "AttentionFuser", "BaseFuser", "RewriteFuser", "StackedFuser", diff --git a/vllm/model_executor/models/transformers/fusers/attention.py b/vllm/model_executor/models/transformers/fusers/attention.py new file mode 100644 index 0000000000..1c6edd5e94 --- /dev/null +++ b/vllm/model_executor/models/transformers/fusers/attention.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Attention fuser: the module that dispatches to the attention interface.""" + +import ast +import inspect +import textwrap +from dataclasses import dataclass +from functools import cache +from typing import TYPE_CHECKING, ClassVar + +from torch import fx, nn + +from vllm.model_executor.models.transformers.fusers.base import BaseFuser + +if TYPE_CHECKING: + from vllm.config import VllmConfig + +VLLM_ATTN_IMPL = "vllm" +VLLM_MLA_ATTN_IMPL = "vllm_mla" + + +def _is_interface_lookup(node: ast.expr | None) -> bool: + """Whether `node` reads an entry out of `ALL_ATTENTION_FUNCTIONS`.""" + # ALL_ATTENTION_FUNCTIONS.get_interface(...) or ALL_ATTENTION_FUNCTIONS[...] + if isinstance(node, ast.Call): + node = node.func + if not isinstance(node, (ast.Attribute, ast.Subscript)): + return False + return ( + isinstance(node.value, ast.Name) and node.value.id == "ALL_ATTENTION_FUNCTIONS" + ) + + +@cache +def interface_call(cls: type[nn.Module]) -> ast.Call | None: + """The attention interface call in `cls.forward`, if it makes exactly one.""" + try: + source = inspect.getsource(inspect.unwrap(cls.forward)) + tree = ast.parse(textwrap.dedent(source)) + except (AttributeError, OSError, SyntaxError, TypeError): + return None + + # Find the names of the local variables that read from the interface lookup + names = set() + for node in ast.walk(tree): + if not isinstance(node, (ast.Assign, ast.AnnAssign)): + continue + if not _is_interface_lookup(node.value): + continue + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + names.update(t.id for t in targets if isinstance(t, ast.Name)) + # Find the calls to those names, and return the one if there is exactly one + calls = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in names + ] + return calls[0] if len(calls) == 1 else None + + +def _resolve(node: ast.expr, module: nn.Module) -> object: + """The value of `node` on `module`, for literals and `self.`.""" + if isinstance(node, ast.Constant): + return node.value + if ( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "self" + ): + return getattr(module, node.attr, None) + return None + + +@dataclass +class AttentionFuser(BaseFuser): + """A module that dispatches through the Transformers attention interface.""" + + redefines_forward: ClassVar[bool] = False + + source_cls: str + """Class of the HF module that dispatches (for logging).""" + scale_expr: ast.expr | None + """Source of the `scaling=` the module hands the interface, if it hands one.""" + + def info(self, name: str) -> str: + return f"Found: {name} ({self.source_cls}) -> attention interface" + + @classmethod + def match( + cls, graph: fx.Graph | None, module: nn.Module + ) -> "AttentionFuser | None": + if (call := interface_call(type(module))) is None: + return None + scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"] + scale_expr = scaling[0] if len(scaling) == 1 else None + return cls(source_cls=type(module).__name__, scale_expr=scale_expr) + + def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: + """Whether `module` will actually dispatch to vLLM.""" + config = getattr(module, "config", None) + # Only patched in the text config, this excludes attention based mm encoders + vllm_attn_impls = {VLLM_ATTN_IMPL, VLLM_MLA_ATTN_IMPL} + return getattr(config, "_attn_implementation", None) in vllm_attn_impls + + def fuse( + self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" + ) -> nn.Module: + return module + + def layer_index(self, module: nn.Module) -> int | None: + """The layer `module` computes attention for, if it declares one.""" + layer_idx = getattr(module, "layer_idx", None) + return layer_idx if isinstance(layer_idx, int) else None + + def scale(self, module: nn.Module) -> float | None: + """The softmax scale `module` passes to the interface, or `None`.""" + if self.scale_expr is None: + return None + scale = _resolve(self.scale_expr, module) + if not isinstance(scale, (int, float)) or isinstance(scale, bool): + expression = ast.unparse(self.scale_expr) + raise ValueError( + f"Cannot resolve attention scaling expression {expression!r} in " + f"{type(module).__name__}." + ) + return float(scale) diff --git a/vllm/model_executor/models/transformers/fusers/base.py b/vllm/model_executor/models/transformers/fusers/base.py index e15b442d07..e36b355739 100644 --- a/vllm/model_executor/models/transformers/fusers/base.py +++ b/vllm/model_executor/models/transformers/fusers/base.py @@ -20,11 +20,15 @@ if TYPE_CHECKING: class BaseFuser(ABC): """A detected fusion and how to apply it. - `match` analyses the module *class* once (cached, see `get_fuser`); `fuse` + `match` analyses the module *class* once (cached, see `get_fusers`); `fuse` then applies the fusion to an instance in `recursive_replace`, returning the module to install in its place. """ + redefines_forward: ClassVar[bool] = True + """Whether `fuse` gives the module a different forward, + by rewriting its source or by returning a different module in its place.""" + @abstractmethod def info(self, name: str) -> str: """A human-readable description of the fusion at `name`, for logging.""" diff --git a/vllm/model_executor/models/transformers/fusers/mla.py b/vllm/model_executor/models/transformers/fusers/mla.py index db29ded911..31bee9dc63 100644 --- a/vllm/model_executor/models/transformers/fusers/mla.py +++ b/vllm/model_executor/models/transformers/fusers/mla.py @@ -59,7 +59,7 @@ def _is_rms_norm(module: nn.Module) -> bool: # Go via `get_fuser` because it caches from vllm.model_executor.models.transformers.fuser import get_fuser - return isinstance(get_fuser(module), RMSNormFuser) + return get_fuser(module, RMSNormFuser) is not None def _top_level_index(funcdef: ast.FunctionDef, node: ast.AST) -> int: diff --git a/vllm/model_executor/models/transformers/moe.py b/vllm/model_executor/models/transformers/moe.py index d59fb2b258..480bc7f66e 100644 --- a/vllm/model_executor/models/transformers/moe.py +++ b/vllm/model_executor/models/transformers/moe.py @@ -36,6 +36,7 @@ from vllm.model_executor.layers.fused_moe import ( ) from vllm.model_executor.models.interfaces import MixtureOfExperts from vllm.model_executor.models.transformers.fuser import get_fuser +from vllm.model_executor.models.transformers.fusers.glu import GLUFuser from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op @@ -319,8 +320,8 @@ class MoEMixin(MixtureOfExperts): # Store shared experts for later down projection adjustment if shared_experts is not None: hf_shared = shared_experts.shared_experts - glu_fuser = get_fuser(hf_shared) - down_name = getattr(glu_fuser, "down_name", None) + glu_fuser = get_fuser(hf_shared, GLUFuser) + down_name = glu_fuser and glu_fuser.down_name if down_name is not None: shared_down_projs.append((hf_shared, down_name)) # Prefer config, otherwise read it from fuser.