[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity (#38479)
Signed-off-by: vibhavagarwal5 <vibhavagarwal5@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Xinyu Chen
Michael Goin
parent
3bfe55a037
commit
f4b42df048
@@ -245,6 +245,32 @@ class FullAttentionSpec(AttentionSpec):
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class TQFullAttentionSpec(FullAttentionSpec):
|
||||
"""FullAttentionSpec with TQ-aware page size.
|
||||
|
||||
Python equivalent of the C++ TQ4FullAttentionSpec. Overrides
|
||||
real_page_size_bytes to use TQ slot bytes instead of the raw
|
||||
head_size * dtype formula.
|
||||
"""
|
||||
|
||||
tq_slot_size: int = 0
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
if self.tq_slot_size > 0:
|
||||
return self.block_size * self.num_kv_heads * self.tq_slot_size
|
||||
return super().real_page_size_bytes
|
||||
|
||||
@classmethod
|
||||
def merge(cls, specs: list[Self]) -> Self:
|
||||
merged = super().merge(specs)
|
||||
assert all(s.tq_slot_size == specs[0].tq_slot_size for s in specs), (
|
||||
"All TQ layers in the same KV cache group must use the same tq_slot_size."
|
||||
)
|
||||
return replace(merged, tq_slot_size=specs[0].tq_slot_size)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class MLAAttentionSpec(FullAttentionSpec):
|
||||
# TODO(Lucas/Chen): less hacky way to do this
|
||||
|
||||
Reference in New Issue
Block a user