[Attention] Add Triton/FlashInfer composite for multimodal prefix attention (#56305)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
Lucas Wilkinson
Lucas Wilkinson
OpenAI Codex
parent
8cf9de9080
commit
82a85dc1d2
@@ -87,6 +87,7 @@ def _get_backend_priorities(
|
||||
kv_cache_dtype: CacheDType | None = None,
|
||||
use_non_causal: bool = False,
|
||||
head_size: int | None = None,
|
||||
use_mm_prefix: bool = False,
|
||||
) -> list[AttentionBackendEnum]:
|
||||
"""Get backend priorities with lazy import to avoid circular dependency."""
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
@@ -156,6 +157,7 @@ def _get_backend_priorities(
|
||||
# So prefer FlashAttention when non-causal on SM100f.
|
||||
if device_capability.major == 10 and not use_non_causal:
|
||||
return [
|
||||
*([AttentionBackendEnum.TRITON_FLASHINFER] if use_mm_prefix else []),
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
@@ -164,6 +166,11 @@ def _get_backend_priorities(
|
||||
]
|
||||
else:
|
||||
return [
|
||||
*(
|
||||
[AttentionBackendEnum.TRITON_FLASH_ATTN]
|
||||
if device_capability.major == 9 and use_mm_prefix
|
||||
else []
|
||||
),
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
@@ -394,6 +401,7 @@ class CudaPlatformBase(Platform):
|
||||
kv_cache_dtype=attn_selector_config.kv_cache_dtype,
|
||||
use_non_causal=attn_selector_config.use_non_causal,
|
||||
head_size=attn_selector_config.head_size,
|
||||
use_mm_prefix=attn_selector_config.use_mm_prefix,
|
||||
)
|
||||
for priority, backend in enumerate(backend_priorities):
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user