[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity (#38479)
Signed-off-by: vibhavagarwal5 <vibhavagarwal5@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Xinyu Chen
Michael Goin
parent
3bfe55a037
commit
f4b42df048
@@ -255,6 +255,11 @@ class CudaPlatformBase(Platform):
|
||||
valid_backends_priorities = []
|
||||
invalid_reasons: dict[AttentionBackendEnum, tuple[int, list[str]]] = {}
|
||||
|
||||
# TurboQuant KV cache: route directly to TQ backend
|
||||
kv_cache_dtype = attn_selector_config.kv_cache_dtype
|
||||
if kv_cache_dtype is not None and kv_cache_dtype.startswith("turboquant_"):
|
||||
return [(AttentionBackendEnum.TURBOQUANT, 0)], {}
|
||||
|
||||
backend_priorities = _get_backend_priorities(
|
||||
attn_selector_config.use_mla,
|
||||
device_capability,
|
||||
|
||||
@@ -61,6 +61,12 @@ class XPUPlatform(Platform):
|
||||
"only NHD layout is supported by XPU attention kernels."
|
||||
)
|
||||
|
||||
# TurboQuant KV cache: route directly to TQ backend
|
||||
kv_cache_dtype = attn_selector_config.kv_cache_dtype
|
||||
if kv_cache_dtype is not None and kv_cache_dtype.startswith("turboquant_"):
|
||||
logger.info_once("Using TurboQuant attention backend.")
|
||||
return AttentionBackendEnum.TURBOQUANT.get_path()
|
||||
|
||||
dtype = attn_selector_config.dtype
|
||||
if attn_selector_config.use_sparse:
|
||||
logger.info_once("Using XPU MLA Sparse backend.")
|
||||
|
||||
Reference in New Issue
Block a user