[Bugfix][VLM] Add fallback to SDPA for ViT model running on CPU backend (#8061)

This commit is contained in:
Isotr0py
2024-09-03 21:37:52 +08:00
committed by GitHub
parent 0fbc6696c2
commit ec266536b7
5 changed files with 157 additions and 44 deletions
+20 -5
View File
@@ -7,7 +7,7 @@ import torch
import torch.nn as nn
from PIL import Image
from transformers import Blip2VisionConfig, BlipVisionConfig
from xformers import ops as xops
from transformers.models.blip.modeling_blip import BlipAttention
from vllm.config import ModelConfig
from vllm.distributed import divide, get_tensor_model_parallel_world_size
@@ -21,6 +21,12 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
repeat_and_pad_placeholder_tokens)
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
try:
from xformers import ops as xops
USE_XFORMERS_OPS = True
except ImportError:
USE_XFORMERS_OPS = False
def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
assert image_size % patch_size == 0
@@ -156,7 +162,7 @@ class BlipVisionEmbeddings(nn.Module):
return embeddings
class BlipAttention(nn.Module):
class BlipParallelAttention(nn.Module):
"""Multi-headed attention from 'Attention Is All You Need' paper"""
def __init__(
@@ -224,7 +230,7 @@ class BlipAttention(nn.Module):
out = out.view(bsz, tgt_len, -1)
attn_output, _ = self.projection(out)
return attn_output
return attn_output, None
class BlipMLP(nn.Module):
@@ -261,7 +267,16 @@ class BlipEncoderLayer(nn.Module):
quant_config: Optional[QuantizationConfig] = None):
super().__init__()
self.self_attn = BlipAttention(config, quant_config=quant_config)
# fallback to sdpa attention if tp unavailable
num_heads = config.num_attention_heads
tp_size = get_tensor_model_parallel_world_size()
if USE_XFORMERS_OPS and num_heads % tp_size == 0:
self.self_attn = BlipParallelAttention(config,
quant_config=quant_config)
else:
# Blip doesn't have SDPA attention implemented in transformers
# use eager attention instead for cpu backend
self.self_attn = BlipAttention(config)
self.layer_norm1 = nn.LayerNorm(config.hidden_size,
eps=config.layer_norm_eps)
self.mlp = BlipMLP(config, quant_config=quant_config)
@@ -272,7 +287,7 @@ class BlipEncoderLayer(nn.Module):
residual = hidden_states
hidden_states = self.layer_norm1(hidden_states)
hidden_states = self.self_attn(hidden_states=hidden_states)
hidden_states, _ = self.self_attn(hidden_states=hidden_states)
hidden_states = residual + hidden_states
residual = hidden_states