[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity (#38479)

Signed-off-by: vibhavagarwal5 <vibhavagarwal5@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
Vibhav Agarwal
2026-04-14 19:57:13 -07:00
committed by GitHub
co-authored by Xinyu Chen Michael Goin
parent 3bfe55a037
commit f4b42df048
27 changed files with 2963 additions and 3 deletions
+4 -2
View File
@@ -21,6 +21,7 @@ from vllm.v1.kv_cache_interface import (
MLAAttentionSpec,
SinkFullAttentionSpec,
SlidingWindowSpec,
TQFullAttentionSpec,
)
from vllm.v1.request import Request
@@ -209,7 +210,7 @@ class SingleTypeKVCacheManager(ABC):
cdiv(num_total_computed_tokens, self.block_size) - len(req_blocks)
)
req_blocks.extend(allocated_blocks)
if type(self.kv_cache_spec) is FullAttentionSpec:
if type(self.kv_cache_spec) in (FullAttentionSpec, TQFullAttentionSpec):
self.new_block_ids.extend(b.block_id for b in allocated_blocks)
def allocate_new_blocks(
@@ -237,7 +238,7 @@ class SingleTypeKVCacheManager(ABC):
else:
new_blocks = self.block_pool.get_new_blocks(num_new_blocks)
req_blocks.extend(new_blocks)
if type(self.kv_cache_spec) is FullAttentionSpec:
if type(self.kv_cache_spec) in (FullAttentionSpec, TQFullAttentionSpec):
self.new_block_ids.extend(b.block_id for b in new_blocks)
return new_blocks
@@ -1114,6 +1115,7 @@ class SinkFullAttentionManager(FullAttentionManager):
spec_manager_map: dict[type[KVCacheSpec], type[SingleTypeKVCacheManager]] = {
FullAttentionSpec: FullAttentionManager,
TQFullAttentionSpec: FullAttentionManager,
MLAAttentionSpec: FullAttentionManager,
SlidingWindowSpec: SlidingWindowManager,
ChunkedLocalAttentionSpec: ChunkedLocalAttentionManager,