[Attention][AMD] Standardize kv layout to blocks first for AMD (#43660)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -362,6 +362,7 @@ def flash_attn_triton_available() -> bool:
|
||||
def _get_backend_priorities(
|
||||
use_mla: bool,
|
||||
use_sparse: bool,
|
||||
use_kv_connector: bool = False,
|
||||
) -> list[AttentionBackendEnum]:
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
|
||||
@@ -380,9 +381,11 @@ def _get_backend_priorities(
|
||||
AttentionBackendEnum.TRITON_MLA,
|
||||
]
|
||||
|
||||
backends = [
|
||||
AttentionBackendEnum.ROCM_ATTN,
|
||||
]
|
||||
backends = []
|
||||
# ROCM_ATTN uses (2, num_blocks, ...) KV cache layout which is
|
||||
# incompatible with KV connectors that require blocks-first layout.
|
||||
if not use_kv_connector:
|
||||
backends.append(AttentionBackendEnum.ROCM_ATTN)
|
||||
if rocm_aiter_ops.is_mha_enabled():
|
||||
backends.append(AttentionBackendEnum.ROCM_AITER_FA)
|
||||
if is_aiter_found_and_supported():
|
||||
@@ -461,6 +464,7 @@ class RocmPlatform(Platform):
|
||||
backend_priorities = _get_backend_priorities(
|
||||
attn_selector_config.use_mla,
|
||||
attn_selector_config.use_sparse,
|
||||
attn_selector_config.use_kv_connector,
|
||||
)
|
||||
for priority, backend in enumerate(backend_priorities):
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user