[Transformers backend] Find attention with a fuser and attach vLLM's layer to it (#54941)

Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Thomas Ortner
2026-09-04 16:26:01 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 99a1ab8e4c
commit 8ad2076c4a
14 changed files with 521 additions and 150 deletions
+41 -35
View File
@@ -10,7 +10,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fuser import get_fuser, get_fusers
from vllm.model_executor.models.transformers.fusers import (
GLUFuser,
PackedQKVFuser,
@@ -307,7 +307,7 @@ class PerHeadSplitAttention(nn.Module):
class FakeSelfAttn(nn.Module):
"""Stand-in for the vLLM `Attention` looked up in `attention_instances`."""
"""Stand-in for the vLLM `Attention` attached to the dispatching module."""
def __init__(self):
super().__init__()
@@ -328,9 +328,9 @@ class FakeMQASelfAttn(FakeSelfAttn):
@pytest.fixture(autouse=True)
def _clear_fuser_cache():
get_fuser.cache_clear()
get_fusers.cache_clear()
yield
get_fuser.cache_clear()
get_fusers.cache_clear()
def _apply_glu_fuser_with_stubs(module: nn.Module, fuser: GLUFuser):
@@ -392,7 +392,7 @@ def _apply_packed_qkv_fuser_with_stubs(module: nn.Module, fuser: PackedQKVFuser)
def test_detects_and_rewrites_glu(mlp_cls, bias):
with torch.device("meta"):
meta = mlp_cls(bias=bias)
fuser = get_fuser(meta)
fuser = get_fuser(meta, GLUFuser)
assert isinstance(fuser, GLUFuser)
assert (
fuser.gate_name,
@@ -427,9 +427,9 @@ def test_glu_identifies_down_projection():
matches the column-parallel merged gate/up; `None` when there is no such
projection to force (fusion of gate/up still applies)."""
with torch.device("meta"):
assert get_fuser(GLUMLP()).down_name == "down_proj"
assert get_fuser(ReversedGLUMLP()).down_name == "down_proj"
assert get_fuser(NoDownGLU()).down_name is None
assert get_fuser(GLUMLP(), GLUFuser).down_name == "down_proj"
assert get_fuser(ReversedGLUMLP(), GLUFuser).down_name == "down_proj"
assert get_fuser(NoDownGLU(), GLUFuser).down_name is None
@pytest.mark.parametrize("attn_cls", [FakeAttention, ReversedFakeAttention])
@@ -439,7 +439,7 @@ def test_detects_and_rewrites_qkv(attn_cls, kv_heads):
pytest.skip("MHA q/k/v assignment is order-based by design")
with torch.device("meta"):
meta = attn_cls(kv_heads=kv_heads)
fuser = get_fuser(meta)
fuser = get_fuser(meta, QKVFuser)
assert isinstance(fuser, QKVFuser)
# q (sharded differently under TP) must be identified exactly; k/v may be
# swapped for non-canonical compute order, which is numerically consistent
@@ -467,27 +467,27 @@ def test_detects_and_rewrites_qkv(attn_cls, kv_heads):
for p in real.parameters():
nn.init.normal_(p, std=0.05)
x = torch.randn(1, 5, 32)
attention_instances = {3: FakeSelfAttn()}
expected, _ = real(x, attention_instances=attention_instances)
real.attn = FakeSelfAttn()
expected, _ = real(x)
fused = _apply_qkv_fuser_with_stubs(real, fuser)
# Fusion is in place: the module keeps its class and other attributes
assert fused is real and type(fused) is attn_cls
assert fused.layer_idx == 3 and fused.is_causal and fused.config is not None
out, _ = fused(x, attention_instances=attention_instances)
out, _ = fused(x)
torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5)
def test_qkv_identifies_output_projection():
with torch.device("meta"):
assert get_fuser(FakeAttention()).o_name == "o_proj"
assert get_fuser(ReversedFakeAttention()).o_name == "o_proj"
assert get_fuser(ExtraProjAttention()).o_name == "o_proj"
assert get_fuser(FakeAttention(), QKVFuser).o_name == "o_proj"
assert get_fuser(ReversedFakeAttention(), QKVFuser).o_name == "o_proj"
assert get_fuser(ExtraProjAttention(), QKVFuser).o_name == "o_proj"
# Norm children (q_norm/k_norm) must not disturb o_proj identification.
assert get_fuser(QKNormAttention()).o_name == "o_proj"
assert get_fuser(PerHeadQKNormAttention()).o_name == "o_proj"
assert get_fuser(QKNormAttention(), QKVFuser).o_name == "o_proj"
assert get_fuser(PerHeadQKNormAttention(), QKVFuser).o_name == "o_proj"
# A module between o_proj and the return is transparent.
assert get_fuser(ResidDropoutAttention()).o_name == "o_proj"
assert get_fuser(ResidDropoutAttention(), QKVFuser).o_name == "o_proj"
@pytest.mark.parametrize("kv_heads", [1, 2])
@@ -498,7 +498,7 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
checkpoint weight as-is, and shards q by heads while replicating k/v."""
with torch.device("meta"):
meta = PackedQKVAttention(kv_heads=kv_heads)
fuser = get_fuser(meta)
fuser = get_fuser(meta, PackedQKVFuser)
assert isinstance(fuser, PackedQKVFuser)
assert (fuser.qkv_name, fuser.o_name) == ("c_attn", "c_proj")
assert (fuser.q_size, fuser.kv_size) == (32, 8 * kv_heads)
@@ -513,14 +513,14 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
for p in real.parameters():
nn.init.normal_(p, std=0.05)
x = torch.randn(1, 5, 32)
attention_instances = {3: FakeMQASelfAttn()}
expected, _ = real(x, attention_instances=attention_instances)
real.attn = FakeMQASelfAttn()
expected, _ = real(x)
fused = _apply_packed_qkv_fuser_with_stubs(real, fuser)
# Fusion is in place: the module keeps its class and other attributes
assert fused is real and type(fused) is PackedQKVAttention
assert fused.layer_idx == 3 and fused.is_causal
out, _ = fused(x, attention_instances=attention_instances)
out, _ = fused(x)
torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5)
@@ -528,28 +528,28 @@ def test_per_head_split_is_not_packed_qkv():
"""The split must consume the whole projection, else its sizes are head
widths and re-sharding by them would be wrong."""
with torch.device("meta"):
assert get_fuser(PerHeadSplitAttention()) is None
assert get_fuser(PerHeadSplitAttention(), PackedQKVFuser) is None
def test_fuser_is_cached_per_class_and_structure():
with torch.device("meta"):
fuser_a = get_fuser(GLUMLP())
fuser_b = get_fuser(GLUMLP())
fuser_a = get_fuser(GLUMLP(), GLUFuser)
fuser_b = get_fuser(GLUMLP(), GLUFuser)
assert fuser_a is fuser_b
assert any(key[0] is GLUMLP for key in get_fuser.cache)
assert any(key[0] is GLUMLP for key in get_fusers.cache)
@pytest.mark.parametrize("cls", [NotAnMLP, UntraceableMLP])
def test_non_matching_modules_return_none(cls):
with torch.device("meta"):
module = cls()
assert get_fuser(module) is None
assert get_fuser(module, GLUFuser) is None
def test_untraceable_tail_still_fuses():
with torch.device("meta"):
meta = UntraceableTailGLUMLP()
fuser = get_fuser(meta)
fuser = get_fuser(meta, GLUFuser)
assert isinstance(fuser, GLUFuser)
# Numerics: the live tail must survive the rewrite
@@ -566,8 +566,8 @@ def test_weight_mappings_are_scoped_to_fused_prefixes():
from vllm.model_executor.models.utils import WeightsMapper
with torch.device("meta"):
glu_fuser = get_fuser(GLUMLP())
qkv_fuser = get_fuser(FakeAttention())
glu_fuser = get_fuser(GLUMLP(), GLUFuser)
qkv_fuser = get_fuser(FakeAttention(), QKVFuser)
mapper = WeightsMapper()
for prefix in ("model.layers.0.mlp", "model.layers.1.mlp"):
@@ -620,7 +620,7 @@ def test_weight_mappings_are_scoped_to_fused_prefixes():
def test_unfusable_modules_are_not_fused(cls, default_vllm_config):
with torch.device("meta"):
module = cls()
fuser = get_fuser(module)
fuser = get_fuser(module, GLUFuser)
# Either no pattern matches the class, or this instance fails validation
# (`recursive_replace` gates fusion and its weight mappings on `validate`)
assert fuser is None or not fuser.validate(module, default_vllm_config)
@@ -654,9 +654,15 @@ def _wider_model_config(head_dim: int) -> SimpleNamespace:
@pytest.mark.parametrize(
"cls, fuser_module", [(FakeAttention, qkv), (PackedQKVAttention, packed_qkv)]
"cls, fuser_module, fuser_cls",
[
(FakeAttention, qkv, QKVFuser),
(PackedQKVAttention, packed_qkv, PackedQKVFuser),
],
)
def test_head_counts_come_from_the_module_not_the_model(cls, fuser_module, monkeypatch):
def test_head_counts_come_from_the_module_not_the_model(
cls, fuser_module, fuser_cls, monkeypatch
):
"""A layer narrower than the model-wide head size must not be miscounted.
On a heterogeneous checkpoint (Gemma 4) the model-wide head size is the
@@ -680,7 +686,7 @@ def test_head_counts_come_from_the_module_not_the_model(cls, fuser_module, monke
monkeypatch.setattr(
fuser_module, "replace_linear_class", lambda *a, **kw: nn.Identity()
)
fuser = get_fuser(module)
fuser = get_fuser(module, fuser_cls)
assert fuser is not None and fuser.validate(module, vllm_config)
fuser.update_attrs(module, "model.layers.0.self_attn", vllm_config)
@@ -700,5 +706,5 @@ def test_validate_accepts_a_layer_the_model_wide_head_size_would_reject():
# kv width is 24, not a multiple of the model-wide 16, but is of this
# layer's 8.
fuser = get_fuser(module)
fuser = get_fuser(module, QKVFuser)
assert fuser is not None and fuser.validate(module, vllm_config)
+1 -1
View File
@@ -158,4 +158,4 @@ class GLU(nn.Module):
def test_non_mla_is_not_matched():
with torch.device("meta"):
assert not isinstance(get_fuser(GLU()), MLAFuser)
assert get_fuser(GLU(), MLAFuser) is None
@@ -162,7 +162,7 @@ class UntraceableGatedRMSNorm(RMSNorm):
)
def test_detects_rms_norm_variants(cls, eps, zero_centered):
with torch.device("meta"):
fuser = get_fuser(cls(16, eps=eps))
fuser = get_fuser(cls(16, eps=eps), RMSNormFuser)
assert isinstance(fuser, RMSNormFuser)
assert fuser.zero_centered == zero_centered
@@ -171,7 +171,7 @@ def test_detects_rms_norm_variants(cls, eps, zero_centered):
def test_non_rms_norms_are_not_matched(cls):
with torch.device("meta"):
module = cls(16) if cls is nn.LayerNorm else cls()
assert not isinstance(get_fuser(module), RMSNormFuser)
assert get_fuser(module, RMSNormFuser) is None
@pytest.mark.parametrize(
@@ -179,7 +179,7 @@ def test_non_rms_norms_are_not_matched(cls):
)
def test_gated_rms_norm_is_not_fused(cls):
with torch.device("meta"):
assert not isinstance(get_fuser(cls()), RMSNormFuser)
assert get_fuser(cls(), RMSNormFuser) is None
@pytest.mark.parametrize(
@@ -196,7 +196,7 @@ def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_c
with torch.device("meta"):
module = cls()
fuser = get_fuser(module)
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
from vllm.model_executor.models.transformers.fusers.rms_norm import (
TPAwareNormMixin,
@@ -221,7 +221,7 @@ def test_weightless_norm_has_no_hidden_size(default_vllm_config):
matching the unfused norm and vLLM's native `RMSNorm(hidden_size=head_dim)`.
"""
module = WeightlessRMSNorm(72)
built = get_fuser(module).fuse(module, "norm", default_vllm_config)
built = get_fuser(module, RMSNormFuser).fuse(module, "norm", default_vllm_config)
assert built.hidden_size == 0
built.tp_size = 2 # emulate TP=2 without a real process group
@@ -236,7 +236,7 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config):
with torch.device("meta"):
module = torch.nn.RMSNorm(16) # forward is a single `F.rms_norm` call
fuser = get_fuser(module)
fuser = get_fuser(module, RMSNormFuser)
assert isinstance(fuser, RMSNormFuser)
assert not fuser.zero_centered
vllm_config = SimpleNamespace(model_config=SimpleNamespace(dtype=torch.float32))
@@ -251,7 +251,8 @@ def test_eps_is_derived_per_instance(default_vllm_config):
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = RMSNorm(16, eps=eps)
built = get_fuser(module).fuse(module, "norm", default_vllm_config)
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps
@@ -262,7 +263,7 @@ def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = cls(16, eps=eps)
fuser = get_fuser(module)
fuser = get_fuser(module, RMSNormFuser)
assert fuser.eps_attr == cls.attr
built = fuser.fuse(module, "norm", default_vllm_config)
assert built.variance_epsilon == eps
@@ -274,7 +275,7 @@ def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplo
logger = "vllm.model_executor.models.transformers.fusers.rms_norm"
with caplog.at_level("DEBUG", logger=logger), torch.device("meta"):
module = LiteralEpsRMSNorm()
fuser = get_fuser(module)
fuser = get_fuser(module, RMSNormFuser)
built = fuser.fuse(module, "norm", default_vllm_config)
assert fuser.eps_attr is None
assert built.variance_epsilon == 1e-4
@@ -287,7 +288,7 @@ def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
with torch.device("meta"):
module = AmbiguousEpsRMSNorm(16, eps=1e-6)
before = dict(vars(module))
fuser = get_fuser(module)
fuser = get_fuser(module, RMSNormFuser)
assert fuser.eps_attr == "variance_epsilon"
assert vars(module) == before
+142
View File
@@ -2,9 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test the functionality of the Transformers modeling backend."""
import ast
import contextlib
import os
import tempfile
from types import SimpleNamespace
from typing import Any
import pytest
@@ -15,6 +17,8 @@ from transformers import AutoConfig, AutoModel, PretrainedConfig
from vllm.config import ModelConfig, VllmConfig
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.transformers.base import Base
from vllm.model_executor.models.transformers.fusers import AttentionFuser
from vllm.model_executor.models.transformers.fusers.attention import VLLM_ATTN_IMPL
from vllm.model_executor.models.transformers.multimodal import MultiModalMixin
from vllm.model_executor.models.utils import StageMissingLayer
@@ -583,3 +587,141 @@ def test_marking_skipped_without_tokenizer():
"""
vision_tower = build_marked_model(0, skip_tokenizer_init=True).vision_tower
assert not isinstance(vision_tower, StageMissingLayer)
NUM_LAYERS = 4
def build_model(model_type: str, **overrides) -> nn.Module:
"""A tiny HF model of `NUM_LAYERS` layers, built on the meta device."""
try:
config = AutoConfig.for_model(
model_type,
num_hidden_layers=NUM_LAYERS,
hidden_size=64,
intermediate_size=128,
num_attention_heads=4,
num_key_value_heads=2,
head_dim=16,
vocab_size=VOCAB_SIZE,
**overrides,
)
except ValueError:
pytest.skip(f"The installed transformers has no {model_type!r} model")
with torch.device("meta"):
return AutoModel.from_config(config)
ATTENTION_MODEL_TYPES = [
# The default `head_size**-0.5` happens to be right here
"llama",
# Scales the query by a learnable per-dim weight, so it declares 1.0
"gemma4_text",
# MLA, and declares yarn's `mscale`
"deepseek_v3",
]
@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
def test_attention_dispatch_is_matched(model_type: str):
"""Exactly the decoder layers' attention modules match an `AttentionFuser`.
That match is what `recursive_replace` records, and so what
`create_attention_instances` attaches vLLM's attention layer to.
"""
model = build_model(model_type)
matched = {
name
for name, module in model.named_modules()
if AttentionFuser.match(None, module) is not None
}
assert matched == {f"layers.{i}.self_attn" for i in range(NUM_LAYERS)}
def test_attention_layer_index_is_the_modules_own():
"""The layer served comes from the module, not from its position in the stack.
LongCat Flash gives each decoder layer two attention sublayers numbered
`2i` and `2i + 1`, so `num_hidden_layers` is twice the length of the stack
and the enclosing layer's position is not the index the KV cache is keyed by.
A vision tower is excluded separately, by `validate`: only the text config is
patched to dispatch to vLLM, so a tower's attention is left to Transformers.
"""
model = build_model("longcat_flash", num_layers=2)
text_config = model.config.get_text_config()
assert len(model.layers) * 2 == text_config.num_hidden_layers
attentions = [
model.get_submodule(f"layers.{i}.self_attn.{j}") for i in (0, 1) for j in (0, 1)
]
fusers = [AttentionFuser.match(None, attn) for attn in attentions]
assert all(fuser is not None for fuser in fusers)
assert [f.layer_index(a) for f, a in zip(fusers, attentions)] == [0, 1, 2, 3]
# Not dispatching to vLLM yet, as a vision tower never would be. The vLLM
# config is not consulted, only the one the module was built with.
assert not any(f.validate(a, None) for f, a in zip(fusers, attentions))
text_config._attn_implementation = VLLM_ATTN_IMPL
assert all(f.validate(a, None) for f, a in zip(fusers, attentions))
def test_attention_dispatch_is_required(monkeypatch: pytest.MonkeyPatch):
"""Models selected by the permissive registry fail clearly without dispatch."""
monkeypatch.setattr(
"vllm.model_executor.models.transformers.base.get_pp_indices",
lambda *_: (0, 1),
)
model = SimpleNamespace(
text_config=SimpleNamespace(num_hidden_layers=1),
pp_group=SimpleNamespace(rank_in_group=0, world_size=1),
attention_fusers={},
_get_attn_cls=lambda: None,
)
with pytest.raises(
ValueError,
match="Layer 0 does not dispatch through the Transformers attention interface",
):
Base.create_attention_instances(model)
@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
def test_attention_scale_is_the_declared_one(model_type: str):
"""The scale comes off the module, not from the `head_size**-0.5` default.
Two of the three model types declare something the default would silently
replace, so they are what makes this more than a tautology.
"""
attention = build_model(model_type).layers[0].self_attn
fuser = AttentionFuser.match(None, attention)
assert fuser is not None
assert fuser.scale(attention) == attention.scaling
def test_attention_scale_is_the_argument_not_the_attribute():
"""OPT applies the scale to the query itself, then declares a literal 1.0.
Its `self.scaling` is the `head_size**-0.5` it has already applied, so
reading the attribute rather than the argument the module hands the
interface would scale twice.
"""
model = build_model("opt", ffn_dim=128, word_embed_proj_dim=64)
attention = model.get_submodule("decoder.layers.0.self_attn")
fuser = AttentionFuser.match(None, attention)
assert fuser is not None
assert fuser.scale(attention) == 1.0 != attention.scaling
def test_attention_scale_rejects_unresolvable_expression():
"""Unresolvable declared scales must not silently use the Llama default."""
class Attention(nn.Module):
head_dim = 16
fuser = AttentionFuser(
source_cls="Attention",
scale_expr=ast.parse("self.head_dim**-0.5 * self.factor", mode="eval").body,
)
with pytest.raises(ValueError, match="Cannot resolve attention scaling expression"):
fuser.scale(Attention())