[Transformers backend] Find attention with a fuser and attach vLLM's layer to it (#54941)

Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Thomas Ortner
2026-09-04 16:26:01 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 99a1ab8e4c
commit 8ad2076c4a
14 changed files with 521 additions and 150 deletions
+142
View File
@@ -2,9 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test the functionality of the Transformers modeling backend."""
import ast
import contextlib
import os
import tempfile
from types import SimpleNamespace
from typing import Any
import pytest
@@ -15,6 +17,8 @@ from transformers import AutoConfig, AutoModel, PretrainedConfig
from vllm.config import ModelConfig, VllmConfig
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.transformers.base import Base
from vllm.model_executor.models.transformers.fusers import AttentionFuser
from vllm.model_executor.models.transformers.fusers.attention import VLLM_ATTN_IMPL
from vllm.model_executor.models.transformers.multimodal import MultiModalMixin
from vllm.model_executor.models.utils import StageMissingLayer
@@ -583,3 +587,141 @@ def test_marking_skipped_without_tokenizer():
"""
vision_tower = build_marked_model(0, skip_tokenizer_init=True).vision_tower
assert not isinstance(vision_tower, StageMissingLayer)
NUM_LAYERS = 4
def build_model(model_type: str, **overrides) -> nn.Module:
"""A tiny HF model of `NUM_LAYERS` layers, built on the meta device."""
try:
config = AutoConfig.for_model(
model_type,
num_hidden_layers=NUM_LAYERS,
hidden_size=64,
intermediate_size=128,
num_attention_heads=4,
num_key_value_heads=2,
head_dim=16,
vocab_size=VOCAB_SIZE,
**overrides,
)
except ValueError:
pytest.skip(f"The installed transformers has no {model_type!r} model")
with torch.device("meta"):
return AutoModel.from_config(config)
ATTENTION_MODEL_TYPES = [
# The default `head_size**-0.5` happens to be right here
"llama",
# Scales the query by a learnable per-dim weight, so it declares 1.0
"gemma4_text",
# MLA, and declares yarn's `mscale`
"deepseek_v3",
]
@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
def test_attention_dispatch_is_matched(model_type: str):
"""Exactly the decoder layers' attention modules match an `AttentionFuser`.
That match is what `recursive_replace` records, and so what
`create_attention_instances` attaches vLLM's attention layer to.
"""
model = build_model(model_type)
matched = {
name
for name, module in model.named_modules()
if AttentionFuser.match(None, module) is not None
}
assert matched == {f"layers.{i}.self_attn" for i in range(NUM_LAYERS)}
def test_attention_layer_index_is_the_modules_own():
"""The layer served comes from the module, not from its position in the stack.
LongCat Flash gives each decoder layer two attention sublayers numbered
`2i` and `2i + 1`, so `num_hidden_layers` is twice the length of the stack
and the enclosing layer's position is not the index the KV cache is keyed by.
A vision tower is excluded separately, by `validate`: only the text config is
patched to dispatch to vLLM, so a tower's attention is left to Transformers.
"""
model = build_model("longcat_flash", num_layers=2)
text_config = model.config.get_text_config()
assert len(model.layers) * 2 == text_config.num_hidden_layers
attentions = [
model.get_submodule(f"layers.{i}.self_attn.{j}") for i in (0, 1) for j in (0, 1)
]
fusers = [AttentionFuser.match(None, attn) for attn in attentions]
assert all(fuser is not None for fuser in fusers)
assert [f.layer_index(a) for f, a in zip(fusers, attentions)] == [0, 1, 2, 3]
# Not dispatching to vLLM yet, as a vision tower never would be. The vLLM
# config is not consulted, only the one the module was built with.
assert not any(f.validate(a, None) for f, a in zip(fusers, attentions))
text_config._attn_implementation = VLLM_ATTN_IMPL
assert all(f.validate(a, None) for f, a in zip(fusers, attentions))
def test_attention_dispatch_is_required(monkeypatch: pytest.MonkeyPatch):
"""Models selected by the permissive registry fail clearly without dispatch."""
monkeypatch.setattr(
"vllm.model_executor.models.transformers.base.get_pp_indices",
lambda *_: (0, 1),
)
model = SimpleNamespace(
text_config=SimpleNamespace(num_hidden_layers=1),
pp_group=SimpleNamespace(rank_in_group=0, world_size=1),
attention_fusers={},
_get_attn_cls=lambda: None,
)
with pytest.raises(
ValueError,
match="Layer 0 does not dispatch through the Transformers attention interface",
):
Base.create_attention_instances(model)
@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
def test_attention_scale_is_the_declared_one(model_type: str):
"""The scale comes off the module, not from the `head_size**-0.5` default.
Two of the three model types declare something the default would silently
replace, so they are what makes this more than a tautology.
"""
attention = build_model(model_type).layers[0].self_attn
fuser = AttentionFuser.match(None, attention)
assert fuser is not None
assert fuser.scale(attention) == attention.scaling
def test_attention_scale_is_the_argument_not_the_attribute():
"""OPT applies the scale to the query itself, then declares a literal 1.0.
Its `self.scaling` is the `head_size**-0.5` it has already applied, so
reading the attribute rather than the argument the module hands the
interface would scale twice.
"""
model = build_model("opt", ffn_dim=128, word_embed_proj_dim=64)
attention = model.get_submodule("decoder.layers.0.self_attn")
fuser = AttentionFuser.match(None, attention)
assert fuser is not None
assert fuser.scale(attention) == 1.0 != attention.scaling
def test_attention_scale_rejects_unresolvable_expression():
"""Unresolvable declared scales must not silently use the Llama default."""
class Attention(nn.Module):
head_dim = 16
fuser = AttentionFuser(
source_cls="Attention",
scale_expr=ast.parse("self.head_dim**-0.5 * self.factor", mode="eval").body,
)
with pytest.raises(ValueError, match="Cannot resolve attention scaling expression"):
fuser.scale(Attention())