[Perf] Use SDPA for BLIP-2 Q-Former attention (#55285)
Signed-off-by: levius <2114377220@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -6,6 +6,7 @@ from typing import Annotated, Literal, TypeAlias
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers import (
|
||||
BatchFeature,
|
||||
Blip2Config,
|
||||
@@ -141,14 +142,13 @@ class Blip2QFormerMultiHeadAttention(nn.Module):
|
||||
|
||||
query_layer = self.transpose_for_scores(mixed_query_layer)
|
||||
|
||||
attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
|
||||
attention_probs = torch.softmax(attention_scores * self.scaling, dim=-1)
|
||||
|
||||
# This is actually dropping out entire tokens to attend to, which might
|
||||
# seem a bit unusual, but is taken from the original Transformer paper.
|
||||
attention_probs_dropped = self.dropout(attention_probs)
|
||||
|
||||
context_layer = torch.matmul(attention_probs_dropped, value_layer)
|
||||
context_layer = F.scaled_dot_product_attention(
|
||||
query_layer,
|
||||
key_layer,
|
||||
value_layer,
|
||||
dropout_p=self.dropout.p if self.training else 0.0,
|
||||
scale=self.scaling,
|
||||
)
|
||||
|
||||
context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
|
||||
context_layer = context_layer.view(
|
||||
|
||||
Reference in New Issue
Block a user