[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity (#38479)

Signed-off-by: vibhavagarwal5 <vibhavagarwal5@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
Vibhav Agarwal
2026-04-14 19:57:13 -07:00
committed by GitHub
co-authored by Xinyu Chen Michael Goin
parent 3bfe55a037
commit f4b42df048
27 changed files with 2963 additions and 3 deletions
+5
View File
@@ -255,6 +255,11 @@ class CudaPlatformBase(Platform):
valid_backends_priorities = []
invalid_reasons: dict[AttentionBackendEnum, tuple[int, list[str]]] = {}
# TurboQuant KV cache: route directly to TQ backend
kv_cache_dtype = attn_selector_config.kv_cache_dtype
if kv_cache_dtype is not None and kv_cache_dtype.startswith("turboquant_"):
return [(AttentionBackendEnum.TURBOQUANT, 0)], {}
backend_priorities = _get_backend_priorities(
attn_selector_config.use_mla,
device_capability,