[Transformers backend] Find attention with a fuser and attach vLLM's layer to it (#54941)

Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Thomas Ortner
2026-09-04 16:26:01 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 99a1ab8e4c
commit 8ad2076c4a
14 changed files with 521 additions and 150 deletions
+41 -35
View File
@@ -10,7 +10,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fuser import get_fuser, get_fusers
from vllm.model_executor.models.transformers.fusers import (
GLUFuser,
PackedQKVFuser,
@@ -307,7 +307,7 @@ class PerHeadSplitAttention(nn.Module):
class FakeSelfAttn(nn.Module):
"""Stand-in for the vLLM `Attention` looked up in `attention_instances`."""
"""Stand-in for the vLLM `Attention` attached to the dispatching module."""
def __init__(self):
super().__init__()
@@ -328,9 +328,9 @@ class FakeMQASelfAttn(FakeSelfAttn):
@pytest.fixture(autouse=True)
def _clear_fuser_cache():
get_fuser.cache_clear()
get_fusers.cache_clear()
yield
get_fuser.cache_clear()
get_fusers.cache_clear()
def _apply_glu_fuser_with_stubs(module: nn.Module, fuser: GLUFuser):
@@ -392,7 +392,7 @@ def _apply_packed_qkv_fuser_with_stubs(module: nn.Module, fuser: PackedQKVFuser)
def test_detects_and_rewrites_glu(mlp_cls, bias):
with torch.device("meta"):
meta = mlp_cls(bias=bias)
fuser = get_fuser(meta)
fuser = get_fuser(meta, GLUFuser)
assert isinstance(fuser, GLUFuser)
assert (
fuser.gate_name,
@@ -427,9 +427,9 @@ def test_glu_identifies_down_projection():
matches the column-parallel merged gate/up; `None` when there is no such
projection to force (fusion of gate/up still applies)."""
with torch.device("meta"):
assert get_fuser(GLUMLP()).down_name == "down_proj"
assert get_fuser(ReversedGLUMLP()).down_name == "down_proj"
assert get_fuser(NoDownGLU()).down_name is None
assert get_fuser(GLUMLP(), GLUFuser).down_name == "down_proj"
assert get_fuser(ReversedGLUMLP(), GLUFuser).down_name == "down_proj"
assert get_fuser(NoDownGLU(), GLUFuser).down_name is None
@pytest.mark.parametrize("attn_cls", [FakeAttention, ReversedFakeAttention])
@@ -439,7 +439,7 @@ def test_detects_and_rewrites_qkv(attn_cls, kv_heads):
pytest.skip("MHA q/k/v assignment is order-based by design")
with torch.device("meta"):
meta = attn_cls(kv_heads=kv_heads)
fuser = get_fuser(meta)
fuser = get_fuser(meta, QKVFuser)
assert isinstance(fuser, QKVFuser)
# q (sharded differently under TP) must be identified exactly; k/v may be
# swapped for non-canonical compute order, which is numerically consistent
@@ -467,27 +467,27 @@ def test_detects_and_rewrites_qkv(attn_cls, kv_heads):
for p in real.parameters():
nn.init.normal_(p, std=0.05)
x = torch.randn(1, 5, 32)
attention_instances = {3: FakeSelfAttn()}
expected, _ = real(x, attention_instances=attention_instances)
real.attn = FakeSelfAttn()
expected, _ = real(x)
fused = _apply_qkv_fuser_with_stubs(real, fuser)
# Fusion is in place: the module keeps its class and other attributes
assert fused is real and type(fused) is attn_cls
assert fused.layer_idx == 3 and fused.is_causal and fused.config is not None
out, _ = fused(x, attention_instances=attention_instances)
out, _ = fused(x)
torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5)
def test_qkv_identifies_output_projection():
with torch.device("meta"):
assert get_fuser(FakeAttention()).o_name == "o_proj"
assert get_fuser(ReversedFakeAttention()).o_name == "o_proj"
assert get_fuser(ExtraProjAttention()).o_name == "o_proj"
assert get_fuser(FakeAttention(), QKVFuser).o_name == "o_proj"
assert get_fuser(ReversedFakeAttention(), QKVFuser).o_name == "o_proj"
assert get_fuser(ExtraProjAttention(), QKVFuser).o_name == "o_proj"
# Norm children (q_norm/k_norm) must not disturb o_proj identification.
assert get_fuser(QKNormAttention()).o_name == "o_proj"
assert get_fuser(PerHeadQKNormAttention()).o_name == "o_proj"
assert get_fuser(QKNormAttention(), QKVFuser).o_name == "o_proj"
assert get_fuser(PerHeadQKNormAttention(), QKVFuser).o_name == "o_proj"
# A module between o_proj and the return is transparent.
assert get_fuser(ResidDropoutAttention()).o_name == "o_proj"
assert get_fuser(ResidDropoutAttention(), QKVFuser).o_name == "o_proj"
@pytest.mark.parametrize("kv_heads", [1, 2])
@@ -498,7 +498,7 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
checkpoint weight as-is, and shards q by heads while replicating k/v."""
with torch.device("meta"):
meta = PackedQKVAttention(kv_heads=kv_heads)
fuser = get_fuser(meta)
fuser = get_fuser(meta, PackedQKVFuser)
assert isinstance(fuser, PackedQKVFuser)
assert (fuser.qkv_name, fuser.o_name) == ("c_attn", "c_proj")
assert (fuser.q_size, fuser.kv_size) == (32, 8 * kv_heads)
@@ -513,14 +513,14 @@ def test_detects_and_rewrites_packed_qkv(kv_heads):
for p in real.parameters():
nn.init.normal_(p, std=0.05)
x = torch.randn(1, 5, 32)
attention_instances = {3: FakeMQASelfAttn()}
expected, _ = real(x, attention_instances=attention_instances)
real.attn = FakeMQASelfAttn()
expected, _ = real(x)
fused = _apply_packed_qkv_fuser_with_stubs(real, fuser)
# Fusion is in place: the module keeps its class and other attributes
assert fused is real and type(fused) is PackedQKVAttention
assert fused.layer_idx == 3 and fused.is_causal
out, _ = fused(x, attention_instances=attention_instances)
out, _ = fused(x)
torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5)
@@ -528,28 +528,28 @@ def test_per_head_split_is_not_packed_qkv():
"""The split must consume the whole projection, else its sizes are head
widths and re-sharding by them would be wrong."""
with torch.device("meta"):
assert get_fuser(PerHeadSplitAttention()) is None
assert get_fuser(PerHeadSplitAttention(), PackedQKVFuser) is None
def test_fuser_is_cached_per_class_and_structure():
with torch.device("meta"):
fuser_a = get_fuser(GLUMLP())
fuser_b = get_fuser(GLUMLP())
fuser_a = get_fuser(GLUMLP(), GLUFuser)
fuser_b = get_fuser(GLUMLP(), GLUFuser)
assert fuser_a is fuser_b
assert any(key[0] is GLUMLP for key in get_fuser.cache)
assert any(key[0] is GLUMLP for key in get_fusers.cache)
@pytest.mark.parametrize("cls", [NotAnMLP, UntraceableMLP])
def test_non_matching_modules_return_none(cls):
with torch.device("meta"):
module = cls()
assert get_fuser(module) is None
assert get_fuser(module, GLUFuser) is None
def test_untraceable_tail_still_fuses():
with torch.device("meta"):
meta = UntraceableTailGLUMLP()
fuser = get_fuser(meta)
fuser = get_fuser(meta, GLUFuser)
assert isinstance(fuser, GLUFuser)
# Numerics: the live tail must survive the rewrite
@@ -566,8 +566,8 @@ def test_weight_mappings_are_scoped_to_fused_prefixes():
from vllm.model_executor.models.utils import WeightsMapper
with torch.device("meta"):
glu_fuser = get_fuser(GLUMLP())
qkv_fuser = get_fuser(FakeAttention())
glu_fuser = get_fuser(GLUMLP(), GLUFuser)
qkv_fuser = get_fuser(FakeAttention(), QKVFuser)
mapper = WeightsMapper()
for prefix in ("model.layers.0.mlp", "model.layers.1.mlp"):
@@ -620,7 +620,7 @@ def test_weight_mappings_are_scoped_to_fused_prefixes():
def test_unfusable_modules_are_not_fused(cls, default_vllm_config):
with torch.device("meta"):
module = cls()
fuser = get_fuser(module)
fuser = get_fuser(module, GLUFuser)
# Either no pattern matches the class, or this instance fails validation
# (`recursive_replace` gates fusion and its weight mappings on `validate`)
assert fuser is None or not fuser.validate(module, default_vllm_config)
@@ -654,9 +654,15 @@ def _wider_model_config(head_dim: int) -> SimpleNamespace:
@pytest.mark.parametrize(
"cls, fuser_module", [(FakeAttention, qkv), (PackedQKVAttention, packed_qkv)]
"cls, fuser_module, fuser_cls",
[
(FakeAttention, qkv, QKVFuser),
(PackedQKVAttention, packed_qkv, PackedQKVFuser),
],
)
def test_head_counts_come_from_the_module_not_the_model(cls, fuser_module, monkeypatch):
def test_head_counts_come_from_the_module_not_the_model(
cls, fuser_module, fuser_cls, monkeypatch
):
"""A layer narrower than the model-wide head size must not be miscounted.
On a heterogeneous checkpoint (Gemma 4) the model-wide head size is the
@@ -680,7 +686,7 @@ def test_head_counts_come_from_the_module_not_the_model(cls, fuser_module, monke
monkeypatch.setattr(
fuser_module, "replace_linear_class", lambda *a, **kw: nn.Identity()
)
fuser = get_fuser(module)
fuser = get_fuser(module, fuser_cls)
assert fuser is not None and fuser.validate(module, vllm_config)
fuser.update_attrs(module, "model.layers.0.self_attn", vllm_config)
@@ -700,5 +706,5 @@ def test_validate_accepts_a_layer_the_model_wide_head_size_would_reject():
# kv width is 24, not a multiple of the model-wide 16, but is of this
# layer's 8.
fuser = get_fuser(module)
fuser = get_fuser(module, QKVFuser)
assert fuser is not None and fuser.validate(module, vllm_config)