[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity (#38479)

Signed-off-by: vibhavagarwal5 <vibhavagarwal5@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
Vibhav Agarwal
2026-04-14 19:57:13 -07:00
committed by GitHub
co-authored by Xinyu Chen Michael Goin
parent 3bfe55a037
commit f4b42df048
27 changed files with 2963 additions and 3 deletions
@@ -379,6 +379,10 @@ class Attention(nn.Module, AttentionLayerBase):
# Initialize KV cache quantization attributes
_init_kv_cache_quant(self, quant_config, prefix)
# Initialize TurboQuant buffers (Pi, S, centroids) if tq cache dtype
if kv_cache_dtype.startswith("turboquant_"):
self._init_turboquant_buffers(kv_cache_dtype, head_size, prefix)
# for attn backends supporting query quantization
self.query_quant = None
if (
@@ -397,6 +401,67 @@ class Attention(nn.Module, AttentionLayerBase):
else GroupShape.PER_TENSOR,
)
def _init_turboquant_buffers(
self, cache_dtype: str, head_size: int, prefix: str
) -> None:
"""Initialize TurboQuant rotation/projection matrices and centroids."""
from vllm.model_executor.layers.quantization.turboquant.centroids import (
get_centroids,
)
from vllm.model_executor.layers.quantization.turboquant.config import (
TurboQuantConfig,
)
from vllm.model_executor.layers.quantization.turboquant.quantizer import (
generate_wht_signs,
)
tq_config = TurboQuantConfig.from_cache_dtype(cache_dtype, head_size)
# Each layer needs a unique rotation matrix so quantization errors
# don't correlate across layers. Stride must exceed max head_dim to
# ensure non-overlapping RNG streams between adjacent layers.
_TQ_LAYER_SEED_STRIDE = 1337
from vllm.model_executor.models.utils import extract_layer_index
layer_idx = extract_layer_index(prefix)
seed = tq_config.seed + layer_idx * _TQ_LAYER_SEED_STRIDE
self.register_buffer(
"_tq_signs",
generate_wht_signs(head_size, seed=seed),
)
self.register_buffer(
"_tq_centroids",
get_centroids(head_size, tq_config.centroid_bits),
)
self._tq_config = tq_config
# Pre-allocate decode intermediate buffers so model.to(device) moves
# them to GPU *before* the memory profiler runs. Without this the
# profiler gives all free memory to KV cache blocks and the first
# decode OOMs when these buffers are lazily allocated.
_vllm_cfg = get_current_vllm_config()
B = _vllm_cfg.scheduler_config.max_num_seqs
Hq = self.num_heads
S = _vllm_cfg.attention_config.tq_max_kv_splits_for_cuda_graph
D = head_size
self.register_buffer(
"_tq_mid_o_buf",
torch.empty(B, Hq, S, D + 1, dtype=torch.float32),
persistent=False,
)
self.register_buffer(
"_tq_output_buf",
torch.empty(B, Hq, D, dtype=torch.float32),
persistent=False,
)
self.register_buffer(
"_tq_lse_buf",
torch.empty(B, Hq, dtype=torch.float32),
persistent=False,
)
def forward(
self,
query: torch.Tensor,
@@ -544,6 +609,23 @@ class Attention(nn.Module, AttentionLayerBase):
kv_quant_mode=quant_mode,
sliding_window=self.sliding_window,
)
elif self.kv_cache_dtype.startswith("turboquant_"):
from vllm.model_executor.layers.quantization.turboquant.config import (
TurboQuantConfig,
)
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
tq_config = TurboQuantConfig.from_cache_dtype(
self.kv_cache_dtype, self.head_size
)
return TQFullAttentionSpec(
block_size=block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
head_size_v=self.head_size,
dtype=self.kv_cache_torch_dtype,
tq_slot_size=tq_config.slot_size_aligned,
)
else:
return FullAttentionSpec(
block_size=block_size,