[Multi Modal] Add FA3 in VIT (#24347)

Signed-off-by: wwl2755 <wangwenlong2755@gmail.com>
This commit is contained in:
Wenlong Wang
2025-09-12 21:27:24 +08:00
committed by GitHub
parent fdb09c77d6
commit 72fc8aa412
13 changed files with 247 additions and 66 deletions
+17 -11
View File
@@ -209,18 +209,24 @@ class CudaPlatformBase(Platform):
return torch.cuda.max_memory_allocated(device)
@classmethod
def get_vit_attn_backend(cls, support_fa: bool = False) -> _Backend:
if cls.has_device_capability(80) and support_fa:
from transformers.utils import is_flash_attn_2_available
if is_flash_attn_2_available():
def get_vit_attn_backend(cls, head_size: int,
dtype: torch.dtype) -> _Backend:
if dtype not in (torch.float16, torch.bfloat16):
return _Backend.XFORMERS
if cls.has_device_capability(80):
FLASH_ATTN_V1 = "vllm.v1.attention.backends.flash_attn.FlashAttentionBackend" # noqa: E501
from vllm.attention.selector import is_attn_backend_supported
is_default_fa_supported = is_attn_backend_supported(
FLASH_ATTN_V1, head_size, dtype, allow_import_error=False)
if is_default_fa_supported:
return _Backend.FLASH_ATTN
logger.warning_once(
"Current `vllm-flash-attn` has a bug inside vision "
"module, so we use xformers backend instead. You can "
"run `pip install flash-attn` to use flash-attention "
"backend.")
# Fallback for Volta/Turing GPUs or FA not supported
return _Backend.XFORMERS
else:
# Fallback to XFORMERS
return _Backend.XFORMERS
else:
# Fallback for Volta/Turing GPUs or FA not supported
return _Backend.XFORMERS
@classmethod
def get_attn_backend_cls(cls, selected_backend, head_size, dtype,
+2 -1
View File
@@ -192,7 +192,8 @@ class Platform:
return device_id
@classmethod
def get_vit_attn_backend(cls, support_fa: bool = False) -> _Backend:
def get_vit_attn_backend(cls, head_size: int,
dtype: torch.dtype) -> _Backend:
return _Backend.TORCH_SDPA
@classmethod
+9 -9
View File
@@ -175,15 +175,15 @@ class RocmPlatform(Platform):
]
@classmethod
def get_vit_attn_backend(cls, support_fa: bool = False) -> _Backend:
if support_fa:
if (envs.VLLM_ROCM_USE_AITER and envs.VLLM_ROCM_USE_AITER_MHA
and on_gfx9()):
# Note: AITER FA is only supported for Qwen-VL models.
# TODO: Add support for other VL models in their model class.
return _Backend.ROCM_AITER_FA
if on_gfx9():
return _Backend.FLASH_ATTN
def get_vit_attn_backend(cls, head_size: int,
dtype: torch.dtype) -> _Backend:
if (envs.VLLM_ROCM_USE_AITER and envs.VLLM_ROCM_USE_AITER_MHA
and on_gfx9()):
# Note: AITER FA is only supported for Qwen-VL models.
# TODO: Add support for other VL models in their model class.
return _Backend.ROCM_AITER_FA
if on_gfx9():
return _Backend.FLASH_ATTN
return _Backend.TORCH_SDPA
@classmethod