From a1541f5742a29864a80087af313ad460066a1524 Mon Sep 17 00:00:00 2001 From: Sun <2114377220@qq.com> Date: Sun, 6 Sep 2026 15:04:19 +0800 Subject: [PATCH] [Perf] Use SDPA for BLIP-2 Q-Former attention (#55285) Signed-off-by: levius <2114377220@qq.com> Signed-off-by: Isotr0py Co-authored-by: Isotr0py --- vllm/model_executor/models/blip2.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vllm/model_executor/models/blip2.py b/vllm/model_executor/models/blip2.py index cc7fcd335d..091001fc01 100644 --- a/vllm/model_executor/models/blip2.py +++ b/vllm/model_executor/models/blip2.py @@ -6,6 +6,7 @@ from typing import Annotated, Literal, TypeAlias import torch import torch.nn as nn +import torch.nn.functional as F from transformers import ( BatchFeature, Blip2Config, @@ -141,14 +142,13 @@ class Blip2QFormerMultiHeadAttention(nn.Module): query_layer = self.transpose_for_scores(mixed_query_layer) - attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) - attention_probs = torch.softmax(attention_scores * self.scaling, dim=-1) - - # This is actually dropping out entire tokens to attend to, which might - # seem a bit unusual, but is taken from the original Transformer paper. - attention_probs_dropped = self.dropout(attention_probs) - - context_layer = torch.matmul(attention_probs_dropped, value_layer) + context_layer = F.scaled_dot_product_attention( + query_layer, + key_layer, + value_layer, + dropout_p=self.dropout.p if self.training else 0.0, + scale=self.scaling, + ) context_layer = context_layer.permute(0, 2, 1, 3).contiguous() context_layer = context_layer.view(