[Attention Backend] TurboQuant: 2-bit KV cache compression with 4x capacity (#38479)

Signed-off-by: vibhavagarwal5 <vibhavagarwal5@gmail.com>
Signed-off-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: Xinyu Chen <xinyu1.chen@intel.com>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
Vibhav Agarwal
2026-04-14 19:57:13 -07:00
committed by GitHub
co-authored by Xinyu Chen Michael Goin
parent 3bfe55a037
commit f4b42df048
27 changed files with 2963 additions and 3 deletions
+38
View File
@@ -1642,6 +1642,31 @@ class EngineArgs:
kv_offloading_backend=self.kv_offloading_backend,
)
# TurboQuant: auto-skip first/last 2 layers (boundary protection).
# These layers are most sensitive to quantization error.
# Users can add extra layers via --kv-cache-dtype-skip-layers.
if resolved_cache_dtype.startswith("turboquant_"):
if model_config.is_hybrid:
raise NotImplementedError(
"TurboQuant KV cache is not supported for hybrid "
"(attention + Mamba) models. Boundary layer protection "
"requires uniform attention layers."
)
from vllm.model_executor.layers.quantization.turboquant.config import (
TurboQuantConfig,
)
num_layers = model_config.hf_text_config.num_hidden_layers
boundary = TurboQuantConfig.get_boundary_skip_layers(num_layers)
existing = set(cache_config.kv_cache_dtype_skip_layers)
merged = sorted(existing | set(boundary), key=lambda x: int(x))
cache_config.kv_cache_dtype_skip_layers = merged
logger.info(
"TQ: skipping layers %s for boundary protection (num_layers=%d)",
merged,
num_layers,
)
ray_runtime_env = None
if is_ray_initialized():
# Ray Serve LLM calls `create_engine_config` in the context
@@ -1948,6 +1973,19 @@ class EngineArgs:
self.attention_backend
)
# TurboQuant requires FlashAttention 2 — FA3 boundary layers assert
# FlashAttentionImpl which fails with TurboQuantAttentionImpl.
if resolved_cache_dtype.startswith("turboquant_") and (
attention_config.flash_attn_version is None
or attention_config.flash_attn_version >= 3
):
logger.warning(
"TurboQuant is not yet compatible with FlashAttention >= 3. "
"Overriding flash_attn_version to 2. To silence this "
"warning, pass --attention-config.flash_attn_version=2"
)
attention_config.flash_attn_version = 2
# Mamba config overrides
mamba_config = copy.deepcopy(self.mamba_config)
# Convert string to enum if needed (CLI parsing returns a string)