[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity (#38479)

Signed-off-by: vibhavagarwal5 <vibhavagarwal5@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
Vibhav Agarwal
2026-04-14 19:57:13 -07:00
committed by GitHub
co-authored by Xinyu Chen Michael Goin
parent 3bfe55a037
commit f4b42df048
27 changed files with 2963 additions and 3 deletions
+26
View File
@@ -245,6 +245,32 @@ class FullAttentionSpec(AttentionSpec):
)
@dataclass(frozen=True, kw_only=True)
class TQFullAttentionSpec(FullAttentionSpec):
"""FullAttentionSpec with TQ-aware page size.
Python equivalent of the C++ TQ4FullAttentionSpec. Overrides
real_page_size_bytes to use TQ slot bytes instead of the raw
head_size * dtype formula.
"""
tq_slot_size: int = 0
@property
def real_page_size_bytes(self) -> int:
if self.tq_slot_size > 0:
return self.block_size * self.num_kv_heads * self.tq_slot_size
return super().real_page_size_bytes
@classmethod
def merge(cls, specs: list[Self]) -> Self:
merged = super().merge(specs)
assert all(s.tq_slot_size == specs[0].tq_slot_size for s in specs), (
"All TQ layers in the same KV cache group must use the same tq_slot_size."
)
return replace(merged, tq_slot_size=specs[0].tq_slot_size)
@dataclass(frozen=True, kw_only=True)
class MLAAttentionSpec(FullAttentionSpec):
# TODO(Lucas/Chen): less hacky way to do this