[Model] Support SDPA attention for Molmo vision backbone (#9410)
This commit is contained in:
@@ -1,4 +1,3 @@
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from array import array
|
||||
@@ -14,10 +13,8 @@ from torch import nn
|
||||
from torch.nn import functional as F
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.attention import Attention, AttentionMetadata
|
||||
from vllm.attention.selector import (_Backend, backend_name_to_enum,
|
||||
get_global_forced_attn_backend)
|
||||
from vllm.attention.selector import _Backend
|
||||
from vllm.config import CacheConfig, MultiModalConfig
|
||||
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -43,12 +40,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
from vllm.model_executor.models.utils import make_layers
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
|
||||
SequenceData)
|
||||
from vllm.transformers_utils.processor import get_processor
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
from .utils import get_vit_attn_backend
|
||||
|
||||
# TODO: hard-coded for now. Consider making it configurable.
|
||||
VIT_LAYERS = [-2, -9]
|
||||
@@ -190,35 +186,12 @@ class MultiHeadDotProductAttention(nn.Module):
|
||||
)
|
||||
|
||||
# Detect attention implementation.
|
||||
selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
|
||||
if selected_backend is None:
|
||||
backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
|
||||
if backend_by_env_var is not None:
|
||||
selected_backend = backend_name_to_enum(backend_by_env_var)
|
||||
if selected_backend is None:
|
||||
# For Volta and Turing GPUs, use xformers instead.
|
||||
device_available = current_platform.get_device_capability()[0] >= 8
|
||||
if device_available:
|
||||
from transformers.utils import is_flash_attn_2_available
|
||||
if is_flash_attn_2_available():
|
||||
self._use_flash_attn = True
|
||||
else:
|
||||
log.warning(
|
||||
"Current Molmo implementation has a bug with "
|
||||
"`vllm-flash-attn` inside vision module, so we use "
|
||||
"xformers backend instead. You can run `pip install "
|
||||
"flash-attn to use flash-attention backend.")
|
||||
self._use_flash_attn = False
|
||||
else:
|
||||
self._use_flash_attn = False
|
||||
else:
|
||||
if selected_backend == _Backend.FLASH_ATTN:
|
||||
self._use_flash_attn = True
|
||||
elif selected_backend == _Backend.XFORMERS:
|
||||
self._use_flash_attn = False
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Molmo does not support {selected_backend} backend now.")
|
||||
self.attn_backend: _Backend = get_vit_attn_backend()
|
||||
if self.attn_backend not in {
|
||||
_Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
|
||||
}:
|
||||
raise RuntimeError(
|
||||
f"Molmo does not support {self.attn_backend} backend now.")
|
||||
|
||||
def forward(self,
|
||||
inputs_q: torch.Tensor,
|
||||
@@ -240,10 +213,15 @@ class MultiHeadDotProductAttention(nn.Module):
|
||||
xk = xk.view(*kv_shape)
|
||||
xv = xv.view(*kv_shape)
|
||||
|
||||
if self._use_flash_attn:
|
||||
if self.attn_backend == _Backend.FLASH_ATTN:
|
||||
from flash_attn import flash_attn_func
|
||||
output = flash_attn_func(xq, xk, xv, dropout_p=0.0, causal=False)
|
||||
else:
|
||||
elif self.attn_backend == _Backend.TORCH_SDPA:
|
||||
xq, xk, xv = (rearrange(x, "b s h d -> b h s d")
|
||||
for x in (xq, xk, xv))
|
||||
output = F.scaled_dot_product_attention(xq, xk, xv)
|
||||
output = rearrange(output, "b h s d -> b s h d ")
|
||||
elif self.attn_backend == _Backend.XFORMERS:
|
||||
from xformers import ops as xops
|
||||
output = xops.memory_efficient_attention_forward(xq, xk, xv, p=0)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user