[Attention] TurboQuant: remove redundant random signs, add prior art attribution (#40194)

Signed-off-by: Dan Alistarh <d.alistarh@gmail.com>
This commit is contained in:
Dan Alistarh
2026-04-18 14:31:59 -04:00
committed by GitHub
parent b5f6c5f834
commit ed0622e3a8
6 changed files with 53 additions and 103 deletions
@@ -1,12 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TurboQuant: Near-optimal KV-cache quantization for vLLM.
"""TurboQuant: KV-cache quantization for vLLM.
PolarQuant compression: random rotation + per-coordinate Lloyd-Max
scalar quantization for keys, uniform quantization for values.
Hadamard rotation + per-coordinate Lloyd-Max scalar quantization for
keys, uniform quantization for values.
Reference: "TurboQuant: Online Vector Quantization with Near-optimal
Distortion Rate" (ICLR 2026), Zandieh et al.
The technique implemented here consists of the scalar case of the HIGGS
quantization method (Malinovskii et al., "Pushing the Limits of Large
Language Model Quantization via the Linearity Theorem", NAACL 2025;
preprint arXiv:2411.17525): rotation + optimized grid + optional
re-normalization, applied to KV cache compression. A first application
of this approach to KV-cache compression is in "Cache Me If You Must:
Adaptive Key-Value Quantization for Large Language Models" (Shutova
et al., ICML 2025; preprint arXiv:2501.19392). Both these references
pre-date the TurboQuant paper (Zandieh et al., ICLR 2026).
"""
from vllm.model_executor.layers.quantization.turboquant.config import TurboQuantConfig
@@ -36,10 +36,22 @@ TQ_PRESETS: dict[str, dict] = {
class TurboQuantConfig:
"""Configuration for TurboQuant KV-cache quantization.
Uses PolarQuant (WHT rotation + Lloyd-Max scalar quantization) for keys
and uniform quantization for values. QJL is intentionally omitted —
community consensus (5+ independent groups) found it hurts attention
quality by amplifying variance through softmax.
Applies Hadamard rotation followed by per-coordinate Lloyd-Max scalar
quantization for keys, and uniform quantization for values.
Historical note: this is the scalar case of the HIGGS quantization
method (Malinovskii et al., "Pushing the Limits of Large Language Model
Quantization via the Linearity Theorem", NAACL 2025; preprint
arXiv:2411.17525): rotation + optimized grid + optional re-normalization,
applied to KV cache compression. A first application of this approach to
KV-cache compression is in "Cache Me If You Must: Adaptive Key-Value
Quantization for Large Language Models" (Shutova et al., ICML 2025;
preprint arXiv:2501.19392). Both these references pre-date the
TurboQuant paper.
QJL is intentionally omitted — community consensus (5+ independent
groups) found it hurts attention quality by amplifying variance through
softmax.
Named presets (use via --kv-cache-dtype):
turboquant_k8v4: FP8 keys + 4-bit values, 2.6x, +1.17% PPL
@@ -53,8 +65,6 @@ class TurboQuantConfig:
rotation/MSE). 3-4 = Lloyd-Max MSE quantized keys.
value_quant_bits: Bits per value dimension for uniform quantization.
3 = 8 levels, 4 = 16 levels (default).
seed: Base seed for deterministic random matrix generation.
Actual seed per layer = seed + layer_idx * 1337.
norm_correction: Re-normalize centroid vectors to unit norm before
inverse rotation during dequant. Fixes quantization-induced norm
distortion, improving PPL by ~0.8% at 4-bit.
@@ -63,7 +73,7 @@ class TurboQuantConfig:
head_dim: int = 128
key_quant_bits: int = 3 # 3-4 = MSE keys, 8 = FP8 keys
value_quant_bits: int = 4 # 3-4 = uniform quantized values
seed: int = 42
seed: int = 42 # kept for backward compatibility; no longer used internally
norm_correction: bool = False
@property
@@ -2,23 +2,5 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TurboQuant quantizer utilities.
Serving path uses generate_wht_signs() for WHT rotation sign buffers.
Triton kernels handle all quantization, packing, and dequantization on GPU.
"""
import torch
_CPU = torch.device("cpu")
def generate_wht_signs(d: int, seed: int, device: torch.device = _CPU) -> torch.Tensor:
"""Generate deterministic random ±1 signs for WHT rotation.
Used with Walsh-Hadamard Transform for per-layer rotation randomization.
Same seed derivation as QR (per-layer via seed + layer_idx * stride).
"""
gen = torch.Generator(device="cpu")
gen.manual_seed(seed)
bits = torch.randint(0, 2, (d,), generator=gen, device="cpu")
signs = bits.float() * 2 - 1
return signs.to(device)