Fix MQA with tensor parallelism on transformers modeling backend (#49987)
Signed-off-by: microslaw <milosz.grunwald@intel.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
28158b2fc3
commit
ebcef33766
@@ -11,7 +11,11 @@ import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers import GLUFuser, QKVFuser
|
||||
from vllm.model_executor.models.transformers.fusers import (
|
||||
GLUFuser,
|
||||
PackedQKVFuser,
|
||||
QKVFuser,
|
||||
)
|
||||
|
||||
|
||||
class SiluAndMulStub(nn.Module):
|
||||
@@ -203,6 +207,103 @@ class PerHeadQKNormAttention(FakeAttention):
|
||||
return self.o_proj((q + k + v).flatten(-2)), None
|
||||
|
||||
|
||||
class ResidDropoutAttention(FakeAttention):
|
||||
"""GPT-style dropout after `o_proj` -> the output projection is still found."""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.resid_dropout = nn.Dropout(0.0)
|
||||
|
||||
def forward(
|
||||
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
|
||||
):
|
||||
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
|
||||
|
||||
input_shape = hidden_states.shape[:-1]
|
||||
hidden_shape = (*input_shape, -1, self.head_dim)
|
||||
q = self.q_proj(hidden_states).view(hidden_shape).transpose(1, 2)
|
||||
k = self.k_proj(hidden_states).view(hidden_shape).transpose(1, 2)
|
||||
v = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
|
||||
attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface(
|
||||
self.config._attn_implementation, None
|
||||
)
|
||||
attn_output, _ = attention_interface(
|
||||
self, q, k, v, attention_mask, scaling=self.scaling, **kwargs
|
||||
)
|
||||
attn_output = attn_output.reshape(*input_shape, -1).contiguous()
|
||||
return self.resid_dropout(self.o_proj(attn_output)), None
|
||||
|
||||
|
||||
class PackedQKVAttention(nn.Module):
|
||||
"""GPTBigCode-style: one packed projection split into q/k/v in the forward."""
|
||||
|
||||
is_causal = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden: int = 32,
|
||||
head_dim: int = 8,
|
||||
heads: int = 4,
|
||||
kv_heads: int = 1,
|
||||
bias: bool = False,
|
||||
layer_idx: int = 0,
|
||||
):
|
||||
super().__init__()
|
||||
self.config = SimpleNamespace(_attn_implementation="vllm")
|
||||
self.layer_idx = layer_idx
|
||||
self.head_dim = head_dim
|
||||
self.scaling = head_dim**-0.5
|
||||
self.embed_dim = heads * head_dim
|
||||
self.kv_dim = kv_heads * head_dim
|
||||
self.c_attn = nn.Linear(hidden, self.embed_dim + 2 * self.kv_dim, bias=bias)
|
||||
self.c_proj = nn.Linear(self.embed_dim, hidden, bias=bias)
|
||||
self.resid_dropout = nn.Dropout(0.0)
|
||||
|
||||
def forward(
|
||||
self, hidden_states, attention_mask=None, past_key_values=None, **kwargs
|
||||
):
|
||||
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
|
||||
|
||||
input_shape = hidden_states.shape[:-1]
|
||||
q, k, v = (
|
||||
self.c_attn(hidden_states)
|
||||
.unsqueeze(1)
|
||||
.split((self.embed_dim, self.kv_dim, self.kv_dim), dim=3)
|
||||
)
|
||||
q = q.view(*input_shape, -1, self.head_dim).transpose(1, 2)
|
||||
if past_key_values is not None:
|
||||
k, v = past_key_values.update(k, v, self.layer_idx)
|
||||
attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface(
|
||||
self.config._attn_implementation, None
|
||||
)
|
||||
attn_output, attn_weights = attention_interface(
|
||||
self, q, k, v, attention_mask, scaling=self.scaling, **kwargs
|
||||
)
|
||||
attn_output = attn_output.reshape(*input_shape, -1).contiguous()
|
||||
return self.resid_dropout(self.c_proj(attn_output)), attn_weights
|
||||
|
||||
|
||||
class PerHeadSplitAttention(nn.Module):
|
||||
"""A packed projection reshaped and split *per head* -> not a q/k/v split."""
|
||||
|
||||
def __init__(self, hidden: int = 32, head_dim: int = 8, heads: int = 4):
|
||||
super().__init__()
|
||||
self.head_dim = head_dim
|
||||
self.heads = heads
|
||||
self.c_attn = nn.Linear(hidden, 3 * heads * head_dim)
|
||||
self.c_proj = nn.Linear(heads * head_dim, hidden)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
shape = (*hidden_states.shape[:2], self.heads, 3 * self.head_dim)
|
||||
q, k, v = (
|
||||
self.c_attn(hidden_states)
|
||||
.view(shape)
|
||||
.transpose(1, 2)
|
||||
.split((self.head_dim, self.head_dim, self.head_dim), dim=3)
|
||||
)
|
||||
return self.c_proj((q + k + v).transpose(1, 2).flatten(-2))
|
||||
|
||||
|
||||
class FakeSelfAttn(nn.Module):
|
||||
"""Stand-in for the vLLM `Attention` looked up in `attention_instances`."""
|
||||
|
||||
@@ -215,6 +316,14 @@ class FakeSelfAttn(nn.Module):
|
||||
return q + 2 * k + 3 * v
|
||||
|
||||
|
||||
class FakeMQASelfAttn(FakeSelfAttn):
|
||||
"""Stand-in for grouped/multi-query layouts, where `k`/`v` are narrower."""
|
||||
|
||||
def forward(self, q, k, v):
|
||||
groups = q.shape[-1] // k.shape[-1]
|
||||
return q + (2 * k + 3 * v).repeat(1, groups)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clear_fuser_cache():
|
||||
get_fuser.cache_clear()
|
||||
@@ -267,6 +376,15 @@ def _apply_qkv_fuser_with_stubs(module: nn.Module, fuser: QKVFuser):
|
||||
return module
|
||||
|
||||
|
||||
def _apply_packed_qkv_fuser_with_stubs(module: nn.Module, fuser: PackedQKVFuser):
|
||||
"""Apply a fuser at `tp_size == 1`, where the rewritten split is unchanged."""
|
||||
qkv = module.get_submodule(fuser.qkv_name)
|
||||
qkv.output_sizes = [fuser.q_size, fuser.kv_size, fuser.kv_size]
|
||||
qkv.tp_size = 1
|
||||
module.forward = MethodType(fuser.fused_forward, module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mlp_cls", [GLUMLP, ReversedGLUMLP])
|
||||
@pytest.mark.parametrize("bias", [False, True])
|
||||
def test_detects_and_rewrites_glu(mlp_cls, bias):
|
||||
@@ -366,6 +484,49 @@ def test_qkv_identifies_output_projection():
|
||||
# Norm children (q_norm/k_norm) must not disturb o_proj identification.
|
||||
assert get_fuser(QKNormAttention()).o_name == "o_proj"
|
||||
assert get_fuser(PerHeadQKNormAttention()).o_name == "o_proj"
|
||||
# A module between o_proj and the return is transparent.
|
||||
assert get_fuser(ResidDropoutAttention()).o_name == "o_proj"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_heads", [1, 2])
|
||||
def test_detects_and_rewrites_packed_qkv(kv_heads):
|
||||
"""A single projection split into q/k/v must be re-sharded, not merged.
|
||||
|
||||
Only the split sizes change: `QKVParallelLinear` loads the packed
|
||||
checkpoint weight as-is, and shards q by heads while replicating k/v."""
|
||||
with torch.device("meta"):
|
||||
meta = PackedQKVAttention(kv_heads=kv_heads)
|
||||
fuser = get_fuser(meta)
|
||||
assert isinstance(fuser, PackedQKVFuser)
|
||||
assert (fuser.qkv_name, fuser.o_name) == ("c_attn", "c_proj")
|
||||
assert (fuser.q_size, fuser.kv_size) == (32, 8 * kv_heads)
|
||||
|
||||
# The hard-coded widths become the per-rank widths of the sharded linear
|
||||
names = fuser.fused_forward.__code__.co_names
|
||||
assert "output_sizes" in names and "tp_size" in names
|
||||
assert "kv_dim" not in names and "embed_dim" not in names
|
||||
|
||||
# Numerics: the rewritten forward must match the original on a real instance
|
||||
real = PackedQKVAttention(kv_heads=kv_heads, layer_idx=3)
|
||||
for p in real.parameters():
|
||||
nn.init.normal_(p, std=0.05)
|
||||
x = torch.randn(1, 5, 32)
|
||||
attention_instances = {3: FakeMQASelfAttn()}
|
||||
expected, _ = real(x, attention_instances=attention_instances)
|
||||
fused = _apply_packed_qkv_fuser_with_stubs(real, fuser)
|
||||
|
||||
# Fusion is in place: the module keeps its class and other attributes
|
||||
assert fused is real and type(fused) is PackedQKVAttention
|
||||
assert fused.layer_idx == 3 and fused.is_causal
|
||||
out, _ = fused(x, attention_instances=attention_instances)
|
||||
torch.testing.assert_close(out, expected, atol=1e-5, rtol=1e-5)
|
||||
|
||||
|
||||
def test_per_head_split_is_not_packed_qkv():
|
||||
"""The split must consume the whole projection, else its sizes are head
|
||||
widths and re-sharding by them would be wrong."""
|
||||
with torch.device("meta"):
|
||||
assert get_fuser(PerHeadSplitAttention()) is None
|
||||
|
||||
|
||||
def test_fuser_is_cached_per_class_and_structure():
|
||||
|
||||
Reference in New Issue
Block a user