[Core] Optimizing cross-attention QKVParallelLinear computation (#12325)
Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: NickLucche <nick@nlucches-4xa100.c.openshift-330514.internal> Co-authored-by: NickLucche <nick@nlucches-4xa100.c.openshift-330514.internal>
This commit is contained in:
co-authored by
NickLucche
parent
5d802522a7
commit
69ff99fdcd
@@ -31,6 +31,7 @@ from vllm.config import CacheConfig, LoRAConfig, VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
QKVCrossParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
@@ -169,7 +170,7 @@ class BartEncoderAttention(nn.Module):
|
||||
# Number of KV heads is less than TP size, so we replicate
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert tp_world_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
|
||||
self.num_kv_heads = self.num_heads
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
|
||||
@@ -248,7 +249,7 @@ class BartDecoderSelfAttention(nn.Module):
|
||||
# Number of KV heads is less than TP size, so we replicate
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert tp_world_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
|
||||
self.num_kv_heads = self.num_heads
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
|
||||
@@ -299,14 +300,14 @@ class BartCrossAttention(nn.Module):
|
||||
f" and `num_heads`: {num_heads}).")
|
||||
self.scaling = self.head_dim**-0.5
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
self.d_model,
|
||||
self.d_model // self.total_num_heads,
|
||||
self.total_num_heads,
|
||||
self.total_num_kv_heads,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
# TP sharding sizes is accounted for within "*Parallel" layers.
|
||||
self.qkv_proj = QKVCrossParallelLinear(self.d_model,
|
||||
self.d_model //
|
||||
self.total_num_heads,
|
||||
self.total_num_heads,
|
||||
self.total_num_kv_heads,
|
||||
bias,
|
||||
quant_config=quant_config)
|
||||
|
||||
self.out_proj = RowParallelLinear(
|
||||
embed_dim,
|
||||
@@ -327,10 +328,7 @@ class BartCrossAttention(nn.Module):
|
||||
# Number of KV heads is less than TP size, so we replicate
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert tp_world_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
|
||||
self.num_kv_heads = self.num_heads # No GQA in bart
|
||||
self.attn = Attention(self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
@@ -347,18 +345,7 @@ class BartCrossAttention(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
"""Input shape: Batch x Time x Channel"""
|
||||
|
||||
# (afeldman-nm 2024/07/22) TODO:
|
||||
# Need a more efficient solution for q/k/v
|
||||
qkv_dec, _ = self.qkv_proj(decoder_hidden_states)
|
||||
q, _, _ = qkv_dec.split([self.q_size, self.kv_size, self.kv_size],
|
||||
dim=-1)
|
||||
if encoder_hidden_states is None:
|
||||
k = None
|
||||
v = None
|
||||
else:
|
||||
qkv_enc, _ = self.qkv_proj(encoder_hidden_states)
|
||||
_, k, v = qkv_enc.split([self.q_size, self.kv_size, self.kv_size],
|
||||
dim=-1)
|
||||
q, k, v = self.qkv_proj(decoder_hidden_states, encoder_hidden_states)
|
||||
|
||||
attn_output = self.attn(q, k, v)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user