[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity (#38479)
Signed-off-by: vibhavagarwal5 <vibhavagarwal5@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Xinyu Chen
Michael Goin
parent
3bfe55a037
commit
f4b42df048
@@ -379,6 +379,10 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
# Initialize KV cache quantization attributes
|
||||
_init_kv_cache_quant(self, quant_config, prefix)
|
||||
|
||||
# Initialize TurboQuant buffers (Pi, S, centroids) if tq cache dtype
|
||||
if kv_cache_dtype.startswith("turboquant_"):
|
||||
self._init_turboquant_buffers(kv_cache_dtype, head_size, prefix)
|
||||
|
||||
# for attn backends supporting query quantization
|
||||
self.query_quant = None
|
||||
if (
|
||||
@@ -397,6 +401,67 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
else GroupShape.PER_TENSOR,
|
||||
)
|
||||
|
||||
def _init_turboquant_buffers(
|
||||
self, cache_dtype: str, head_size: int, prefix: str
|
||||
) -> None:
|
||||
"""Initialize TurboQuant rotation/projection matrices and centroids."""
|
||||
from vllm.model_executor.layers.quantization.turboquant.centroids import (
|
||||
get_centroids,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
TurboQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.turboquant.quantizer import (
|
||||
generate_wht_signs,
|
||||
)
|
||||
|
||||
tq_config = TurboQuantConfig.from_cache_dtype(cache_dtype, head_size)
|
||||
|
||||
# Each layer needs a unique rotation matrix so quantization errors
|
||||
# don't correlate across layers. Stride must exceed max head_dim to
|
||||
# ensure non-overlapping RNG streams between adjacent layers.
|
||||
_TQ_LAYER_SEED_STRIDE = 1337
|
||||
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
seed = tq_config.seed + layer_idx * _TQ_LAYER_SEED_STRIDE
|
||||
|
||||
self.register_buffer(
|
||||
"_tq_signs",
|
||||
generate_wht_signs(head_size, seed=seed),
|
||||
)
|
||||
self.register_buffer(
|
||||
"_tq_centroids",
|
||||
get_centroids(head_size, tq_config.centroid_bits),
|
||||
)
|
||||
self._tq_config = tq_config
|
||||
|
||||
# Pre-allocate decode intermediate buffers so model.to(device) moves
|
||||
# them to GPU *before* the memory profiler runs. Without this the
|
||||
# profiler gives all free memory to KV cache blocks and the first
|
||||
# decode OOMs when these buffers are lazily allocated.
|
||||
_vllm_cfg = get_current_vllm_config()
|
||||
B = _vllm_cfg.scheduler_config.max_num_seqs
|
||||
Hq = self.num_heads
|
||||
S = _vllm_cfg.attention_config.tq_max_kv_splits_for_cuda_graph
|
||||
D = head_size
|
||||
self.register_buffer(
|
||||
"_tq_mid_o_buf",
|
||||
torch.empty(B, Hq, S, D + 1, dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
self.register_buffer(
|
||||
"_tq_output_buf",
|
||||
torch.empty(B, Hq, D, dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
self.register_buffer(
|
||||
"_tq_lse_buf",
|
||||
torch.empty(B, Hq, dtype=torch.float32),
|
||||
persistent=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
@@ -544,6 +609,23 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
kv_quant_mode=quant_mode,
|
||||
sliding_window=self.sliding_window,
|
||||
)
|
||||
elif self.kv_cache_dtype.startswith("turboquant_"):
|
||||
from vllm.model_executor.layers.quantization.turboquant.config import (
|
||||
TurboQuantConfig,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import TQFullAttentionSpec
|
||||
|
||||
tq_config = TurboQuantConfig.from_cache_dtype(
|
||||
self.kv_cache_dtype, self.head_size
|
||||
)
|
||||
return TQFullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
head_size_v=self.head_size,
|
||||
dtype=self.kv_cache_torch_dtype,
|
||||
tq_slot_size=tq_config.slot_size_aligned,
|
||||
)
|
||||
else:
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
|
||||
Reference in New Issue
Block a user