[Core] Optimizing cross-attention QKVParallelLinear computation (#12325)
Signed-off-by: NickLucche <nlucches@redhat.com> Signed-off-by: NickLucche <nick@nlucches-4xa100.c.openshift-330514.internal> Co-authored-by: NickLucche <nick@nlucches-4xa100.c.openshift-330514.internal>
This commit is contained in:
co-authored by
NickLucche
parent
5d802522a7
commit
69ff99fdcd
@@ -43,6 +43,7 @@ from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
QKVCrossParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
@@ -798,21 +799,22 @@ class MllamaTextCrossAttention(nn.Module):
|
||||
self.config = config
|
||||
self.pipeline_parallel_rank = get_pp_group().rank_in_group
|
||||
self.tensor_parallel_size = get_tp_group().world_size
|
||||
self.num_heads = self.config.num_attention_heads
|
||||
self.num_heads = config.num_attention_heads
|
||||
self.num_key_value_heads = config.num_key_value_heads
|
||||
|
||||
self.num_local_heads = self.num_heads // self.tensor_parallel_size
|
||||
self.num_key_value_heads = self.config.num_key_value_heads
|
||||
self.num_local_key_value_heads = \
|
||||
self.num_key_value_heads // self.tensor_parallel_size
|
||||
self.dropout = config.dropout
|
||||
self.hidden_size = config.hidden_size
|
||||
self.head_dim = config.hidden_size // self.num_heads
|
||||
self.num_key_value_heads = config.num_key_value_heads
|
||||
|
||||
self.layer_idx = layer_idx
|
||||
self.num_key_value_groups = self.num_heads // self.num_key_value_heads
|
||||
self.q_local_size = self.num_local_heads * self.head_dim
|
||||
self.kv_local_size = self.num_local_key_value_heads * self.head_dim
|
||||
|
||||
# TODO: change to Q/KV separate linear after #7448 is merged
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
self.qkv_proj = QKVCrossParallelLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
self.num_heads,
|
||||
@@ -821,6 +823,7 @@ class MllamaTextCrossAttention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.num_heads * self.head_dim,
|
||||
self.hidden_size,
|
||||
@@ -851,21 +854,12 @@ class MllamaTextCrossAttention(nn.Module):
|
||||
kv_range_for_decode: Optional[List[Tuple[int, int]]],
|
||||
cross_attention_states: Optional[torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
qkv_dec, _ = self.qkv_proj(hidden_states)
|
||||
q, _, _ = qkv_dec.split(
|
||||
[self.q_local_size, self.kv_local_size, self.kv_local_size],
|
||||
dim=-1)
|
||||
if cross_attention_states is None:
|
||||
k = None
|
||||
v = None
|
||||
else:
|
||||
qkv_enc, _ = self.qkv_proj(cross_attention_states)
|
||||
_, k, v = qkv_enc.split(
|
||||
[self.q_local_size, self.kv_local_size, self.kv_local_size],
|
||||
dim=-1)
|
||||
q, k, v = self.qkv_proj(hidden_states, cross_attention_states)
|
||||
if cross_attention_states is not None:
|
||||
k = k.view(-1, self.num_local_key_value_heads, self.head_dim)
|
||||
v = v.view(-1, self.num_local_key_value_heads, self.head_dim)
|
||||
k = self.k_norm(k)
|
||||
|
||||
q = q.view(-1, self.num_local_heads, self.head_dim)
|
||||
q = self.q_norm(q)
|
||||
|
||||
@@ -889,6 +883,7 @@ class MllamaTextCrossAttention(nn.Module):
|
||||
kv_cache = self.attn.kv_cache[self.pipeline_parallel_rank]
|
||||
attn_metadata: AttentionMetadata = get_forward_context().attn_metadata
|
||||
# Skip writing kv-cache for the initial profiling run.
|
||||
# TODO (NickLucche) replace with custom attn bias and use standard attn
|
||||
if len(kv_cache.shape) > 1:
|
||||
i = torch.ones(1, dtype=torch.float32)
|
||||
if self.attn.backend in (_Backend.FLASH_ATTN,
|
||||
|
||||
Reference in New Issue
Block a user