[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity (#38479)
Signed-off-by: vibhavagarwal5 <vibhavagarwal5@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Xinyu Chen
Michael Goin
parent
3bfe55a037
commit
f4b42df048
@@ -27,6 +27,11 @@ class AttentionConfig:
|
||||
flash_attn_max_num_splits_for_cuda_graph: int = 32
|
||||
"""Flash Attention max number splits for cuda graph decode."""
|
||||
|
||||
tq_max_kv_splits_for_cuda_graph: int = 32
|
||||
"""TurboQuant max NUM_KV_SPLITS for cuda graph decode.
|
||||
Fixes the split count so grid dimensions are constant across captures,
|
||||
and buffers can be pre-allocated to avoid inflating the memory estimate."""
|
||||
|
||||
use_cudnn_prefill: bool = False
|
||||
"""Whether to use cudnn prefill."""
|
||||
|
||||
|
||||
@@ -24,6 +24,10 @@ CacheDType = Literal[
|
||||
"fp8_e5m2",
|
||||
"fp8_inc",
|
||||
"fp8_ds_mla",
|
||||
"turboquant_k8v4",
|
||||
"turboquant_4bit_nc",
|
||||
"turboquant_k3v4_nc",
|
||||
"turboquant_3bit_nc",
|
||||
"int8_per_token_head",
|
||||
"fp8_per_token_head",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user