Fix TURBOQUANT backend selection in cuda.py (#40060)
Signed-off-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
@@ -131,6 +131,7 @@ def _get_backend_priorities(
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
AttentionBackendEnum.FLEX_ATTENTION,
|
||||
AttentionBackendEnum.TURBOQUANT,
|
||||
]
|
||||
else:
|
||||
return [
|
||||
@@ -138,6 +139,7 @@ def _get_backend_priorities(
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
AttentionBackendEnum.FLEX_ATTENTION,
|
||||
AttentionBackendEnum.TURBOQUANT,
|
||||
]
|
||||
|
||||
|
||||
@@ -255,11 +257,6 @@ class CudaPlatformBase(Platform):
|
||||
valid_backends_priorities = []
|
||||
invalid_reasons: dict[AttentionBackendEnum, tuple[int, list[str]]] = {}
|
||||
|
||||
# TurboQuant KV cache: route directly to TQ backend
|
||||
kv_cache_dtype = attn_selector_config.kv_cache_dtype
|
||||
if kv_cache_dtype is not None and kv_cache_dtype.startswith("turboquant_"):
|
||||
return [(AttentionBackendEnum.TURBOQUANT, 0)], {}
|
||||
|
||||
backend_priorities = _get_backend_priorities(
|
||||
attn_selector_config.use_mla,
|
||||
device_capability,
|
||||
|
||||
Reference in New Issue
Block a user