[MM Encoder] Default to use TORCH_SDPA backend for ViT on Volta/Turing GPU (#36472)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
+15
-7
@@ -413,12 +413,20 @@ class CudaPlatformBase(Platform):
|
||||
|
||||
@classmethod
|
||||
def get_supported_vit_attn_backends(cls) -> list["AttentionBackendEnum"]:
|
||||
return [
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
]
|
||||
if cls.has_device_capability(80):
|
||||
return [
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
]
|
||||
else:
|
||||
return [
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def get_vit_attn_backend(
|
||||
@@ -438,7 +446,7 @@ class CudaPlatformBase(Platform):
|
||||
cc = cls.get_device_capability()
|
||||
for vit_attn_backend in cls.get_supported_vit_attn_backends():
|
||||
if vit_attn_backend == AttentionBackendEnum.TORCH_SDPA:
|
||||
continue
|
||||
return vit_attn_backend
|
||||
try:
|
||||
backend_class = vit_attn_backend.get_class()
|
||||
is_backend_supported = backend_class.supports_head_size(
|
||||
|
||||
Reference in New Issue
Block a user