[Transformers backend] Find attention with a fuser and attach vLLM's layer to it (#54941)
Signed-off-by: Thomas Ortner <boh@zurich.ibm.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
99a1ab8e4c
commit
8ad2076c4a
@@ -2,9 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test the functionality of the Transformers modeling backend."""
|
||||
|
||||
import ast
|
||||
import contextlib
|
||||
import os
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -15,6 +17,8 @@ from transformers import AutoConfig, AutoModel, PretrainedConfig
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
from vllm.model_executor.models.transformers.base import Base
|
||||
from vllm.model_executor.models.transformers.fusers import AttentionFuser
|
||||
from vllm.model_executor.models.transformers.fusers.attention import VLLM_ATTN_IMPL
|
||||
from vllm.model_executor.models.transformers.multimodal import MultiModalMixin
|
||||
from vllm.model_executor.models.utils import StageMissingLayer
|
||||
|
||||
@@ -583,3 +587,141 @@ def test_marking_skipped_without_tokenizer():
|
||||
"""
|
||||
vision_tower = build_marked_model(0, skip_tokenizer_init=True).vision_tower
|
||||
assert not isinstance(vision_tower, StageMissingLayer)
|
||||
|
||||
|
||||
NUM_LAYERS = 4
|
||||
|
||||
|
||||
def build_model(model_type: str, **overrides) -> nn.Module:
|
||||
"""A tiny HF model of `NUM_LAYERS` layers, built on the meta device."""
|
||||
try:
|
||||
config = AutoConfig.for_model(
|
||||
model_type,
|
||||
num_hidden_layers=NUM_LAYERS,
|
||||
hidden_size=64,
|
||||
intermediate_size=128,
|
||||
num_attention_heads=4,
|
||||
num_key_value_heads=2,
|
||||
head_dim=16,
|
||||
vocab_size=VOCAB_SIZE,
|
||||
**overrides,
|
||||
)
|
||||
except ValueError:
|
||||
pytest.skip(f"The installed transformers has no {model_type!r} model")
|
||||
with torch.device("meta"):
|
||||
return AutoModel.from_config(config)
|
||||
|
||||
|
||||
ATTENTION_MODEL_TYPES = [
|
||||
# The default `head_size**-0.5` happens to be right here
|
||||
"llama",
|
||||
# Scales the query by a learnable per-dim weight, so it declares 1.0
|
||||
"gemma4_text",
|
||||
# MLA, and declares yarn's `mscale`
|
||||
"deepseek_v3",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
|
||||
def test_attention_dispatch_is_matched(model_type: str):
|
||||
"""Exactly the decoder layers' attention modules match an `AttentionFuser`.
|
||||
|
||||
That match is what `recursive_replace` records, and so what
|
||||
`create_attention_instances` attaches vLLM's attention layer to.
|
||||
"""
|
||||
model = build_model(model_type)
|
||||
matched = {
|
||||
name
|
||||
for name, module in model.named_modules()
|
||||
if AttentionFuser.match(None, module) is not None
|
||||
}
|
||||
assert matched == {f"layers.{i}.self_attn" for i in range(NUM_LAYERS)}
|
||||
|
||||
|
||||
def test_attention_layer_index_is_the_modules_own():
|
||||
"""The layer served comes from the module, not from its position in the stack.
|
||||
|
||||
LongCat Flash gives each decoder layer two attention sublayers numbered
|
||||
`2i` and `2i + 1`, so `num_hidden_layers` is twice the length of the stack
|
||||
and the enclosing layer's position is not the index the KV cache is keyed by.
|
||||
A vision tower is excluded separately, by `validate`: only the text config is
|
||||
patched to dispatch to vLLM, so a tower's attention is left to Transformers.
|
||||
"""
|
||||
model = build_model("longcat_flash", num_layers=2)
|
||||
text_config = model.config.get_text_config()
|
||||
assert len(model.layers) * 2 == text_config.num_hidden_layers
|
||||
|
||||
attentions = [
|
||||
model.get_submodule(f"layers.{i}.self_attn.{j}") for i in (0, 1) for j in (0, 1)
|
||||
]
|
||||
fusers = [AttentionFuser.match(None, attn) for attn in attentions]
|
||||
assert all(fuser is not None for fuser in fusers)
|
||||
assert [f.layer_index(a) for f, a in zip(fusers, attentions)] == [0, 1, 2, 3]
|
||||
|
||||
# Not dispatching to vLLM yet, as a vision tower never would be. The vLLM
|
||||
# config is not consulted, only the one the module was built with.
|
||||
assert not any(f.validate(a, None) for f, a in zip(fusers, attentions))
|
||||
text_config._attn_implementation = VLLM_ATTN_IMPL
|
||||
assert all(f.validate(a, None) for f, a in zip(fusers, attentions))
|
||||
|
||||
|
||||
def test_attention_dispatch_is_required(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Models selected by the permissive registry fail clearly without dispatch."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.models.transformers.base.get_pp_indices",
|
||||
lambda *_: (0, 1),
|
||||
)
|
||||
model = SimpleNamespace(
|
||||
text_config=SimpleNamespace(num_hidden_layers=1),
|
||||
pp_group=SimpleNamespace(rank_in_group=0, world_size=1),
|
||||
attention_fusers={},
|
||||
_get_attn_cls=lambda: None,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Layer 0 does not dispatch through the Transformers attention interface",
|
||||
):
|
||||
Base.create_attention_instances(model)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
|
||||
def test_attention_scale_is_the_declared_one(model_type: str):
|
||||
"""The scale comes off the module, not from the `head_size**-0.5` default.
|
||||
|
||||
Two of the three model types declare something the default would silently
|
||||
replace, so they are what makes this more than a tautology.
|
||||
"""
|
||||
attention = build_model(model_type).layers[0].self_attn
|
||||
fuser = AttentionFuser.match(None, attention)
|
||||
assert fuser is not None
|
||||
assert fuser.scale(attention) == attention.scaling
|
||||
|
||||
|
||||
def test_attention_scale_is_the_argument_not_the_attribute():
|
||||
"""OPT applies the scale to the query itself, then declares a literal 1.0.
|
||||
|
||||
Its `self.scaling` is the `head_size**-0.5` it has already applied, so
|
||||
reading the attribute rather than the argument the module hands the
|
||||
interface would scale twice.
|
||||
"""
|
||||
model = build_model("opt", ffn_dim=128, word_embed_proj_dim=64)
|
||||
attention = model.get_submodule("decoder.layers.0.self_attn")
|
||||
fuser = AttentionFuser.match(None, attention)
|
||||
assert fuser is not None
|
||||
assert fuser.scale(attention) == 1.0 != attention.scaling
|
||||
|
||||
|
||||
def test_attention_scale_rejects_unresolvable_expression():
|
||||
"""Unresolvable declared scales must not silently use the Llama default."""
|
||||
|
||||
class Attention(nn.Module):
|
||||
head_dim = 16
|
||||
|
||||
fuser = AttentionFuser(
|
||||
source_cls="Attention",
|
||||
scale_expr=ast.parse("self.head_dim**-0.5 * self.factor", mode="eval").body,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Cannot resolve attention scaling expression"):
|
||||
fuser.scale(Attention())
|
||||
|
||||
Reference in New Issue
Block a user