[Feature] KV cache per-token-head INT8/FP8 quantization (#38378)

Signed-off-by: JartX <sagformas@epdcenter.es>
Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: yangyang4991 <yangyang4991@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
JartX
2026-04-02 08:13:26 -04:00
committed by GitHub
co-authored by yangyang4991 Wentao Ye Isotr0py
parent 4eefbf9609
commit 2ce3d0ce36
16 changed files with 1308 additions and 66 deletions
@@ -38,6 +38,7 @@ from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheSpec,
SlidingWindowSpec,
get_kv_quant_mode,
)
if TYPE_CHECKING:
@@ -381,8 +382,10 @@ class Attention(nn.Module, AttentionLayerBase):
# for attn backends supporting query quantization
self.query_quant = None
if self.impl.supports_quant_query_input and self.kv_cache_dtype.startswith(
"fp8"
if (
self.impl.supports_quant_query_input
and self.kv_cache_dtype.startswith("fp8")
and not self.kv_cache_dtype.endswith("per_token_head")
):
is_per_head = (
hasattr(self, "q_scale") and self.q_scale.numel() == self.num_kv_heads
@@ -539,6 +542,7 @@ class Attention(nn.Module, AttentionLayerBase):
block_size = vllm_config.cache_config.block_size
# Should not be called for enc-dec or encoder-only attention.
assert self.attn_type == AttentionType.DECODER
quant_mode = get_kv_quant_mode(self.kv_cache_dtype)
if self.sliding_window is not None:
assert not vllm_config.model_config.use_mla, (
"MLA is not supported for slidingwindow"
@@ -548,6 +552,7 @@ class Attention(nn.Module, AttentionLayerBase):
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=quant_mode,
sliding_window=self.sliding_window,
)
else:
@@ -557,6 +562,7 @@ class Attention(nn.Module, AttentionLayerBase):
head_size=self.head_size,
head_size_v=self.head_size_v,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=quant_mode,
)
@@ -23,6 +23,7 @@ from vllm.v1.kv_cache_interface import (
AttentionSpec,
ChunkedLocalAttentionSpec,
KVCacheSpec,
get_kv_quant_mode,
)
@@ -123,5 +124,6 @@ class ChunkedLocalAttention(Attention):
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
attention_chunk_size=self.attention_chunk_size,
)
@@ -18,7 +18,11 @@ from vllm.v1.attention.backend import (
subclass_attention_backend_with_overrides,
)
from vllm.v1.attention.selector import get_attn_backend
from vllm.v1.kv_cache_interface import CrossAttentionSpec, KVCacheSpec
from vllm.v1.kv_cache_interface import (
CrossAttentionSpec,
KVCacheSpec,
get_kv_quant_mode,
)
logger = init_logger(__name__)
@@ -220,4 +224,5 @@ class CrossAttention(Attention):
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
)
@@ -26,6 +26,7 @@ from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheSpec,
SinkFullAttentionSpec,
get_kv_quant_mode,
)
logger = init_logger(__name__)
@@ -217,6 +218,7 @@ class StaticSinkAttention(Attention, CustomOp):
head_size_v=self.head_size_v,
sink_len=self.sink_len,
dtype=self.kv_cache_torch_dtype,
kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype),
)
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.base_config import (
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.kv_cache_interface import kv_cache_uses_per_token_head_scales
logger = init_logger(__name__)
@@ -53,6 +54,20 @@ class BaseKVCacheMethod(QuantizeMethodBase):
assert not hasattr(layer, "prob_scale")
return
# Per-token-head quantized KV cache: scales are computed dynamically
# per (token, head) in the kernel at cache-write time. Checkpoint
# scales are never used regardless of calculate_kv_scales.
if kv_cache_uses_per_token_head_scales(layer.kv_cache_dtype):
layer._k_scale.copy_(1.0)
layer._v_scale.copy_(1.0)
layer._k_scale_float = 1.0
layer._v_scale_float = 1.0
del layer.k_scale
del layer.v_scale
del layer.q_scale
del layer.prob_scale
return
# If the kv-cache is not quantized, we enforce the k/v_scale to be 1.0
# regardless whether the kv-scale is available in the checkpoint.
# No need to process kv scales after loading if we are going to