[MM Encoder] Add Triton ViT attention backend (#32183)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
+19
-7
@@ -411,8 +411,9 @@ class CudaPlatformBase(Platform):
|
||||
@classmethod
|
||||
def get_supported_vit_attn_backends(cls) -> list["AttentionBackendEnum"]:
|
||||
return [
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
]
|
||||
|
||||
@classmethod
|
||||
@@ -430,14 +431,25 @@ class CudaPlatformBase(Platform):
|
||||
logger.info_once(f"Using backend {backend} for vit attention")
|
||||
return backend
|
||||
|
||||
# Try FlashAttention first
|
||||
if (cc := cls.get_device_capability()) and cc.major >= 8:
|
||||
cc = cls.get_device_capability()
|
||||
for vit_attn_backend in cls.get_supported_vit_attn_backends():
|
||||
if vit_attn_backend == AttentionBackendEnum.TORCH_SDPA:
|
||||
continue
|
||||
try:
|
||||
backend_class = AttentionBackendEnum.FLASH_ATTN.get_class()
|
||||
if backend_class.supports_head_size(
|
||||
backend_class = vit_attn_backend.get_class()
|
||||
is_backend_supported = backend_class.supports_head_size(
|
||||
head_size
|
||||
) and backend_class.supports_dtype(dtype):
|
||||
return AttentionBackendEnum.FLASH_ATTN
|
||||
) and backend_class.supports_dtype(dtype)
|
||||
if cc is not None:
|
||||
is_backend_supported = (
|
||||
is_backend_supported
|
||||
and backend_class.supports_compute_capability(cc)
|
||||
)
|
||||
if is_backend_supported:
|
||||
logger.info_once(
|
||||
f"Using backend {vit_attn_backend} for vit attention"
|
||||
)
|
||||
return vit_attn_backend
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
@@ -384,6 +384,7 @@ class RocmPlatform(Platform):
|
||||
return [
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user