[Feature] KV cache per-token-head INT8/FP8 quantization (#38378)

Signed-off-by: JartX <sagformas@epdcenter.es>
Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: yangyang4991 <yangyang4991@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
JartX
2026-04-02 08:13:26 -04:00
committed by GitHub
co-authored by yangyang4991 Wentao Ye Isotr0py
parent 4eefbf9609
commit 2ce3d0ce36
16 changed files with 1308 additions and 66 deletions
+61 -1
View File
@@ -1,21 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import copy
from dataclasses import dataclass, fields, replace
from enum import IntEnum
from math import prod
from typing import TYPE_CHECKING
import torch
from typing_extensions import Self
from vllm.config import VllmConfig
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import get_dtype_size
logger = init_logger(__name__)
# ---------------------------------------------------------------------------
# KV cache quantization mode
# ---------------------------------------------------------------------------
class KVQuantMode(IntEnum):
"""KV cache quantization mode.
Used by attention backends and kernels to dispatch quantization logic
without string matching on ``kv_cache_dtype``.
"""
NONE = 0
FP8_PER_TENSOR = 1 # per-tensor scales (current fp8 path)
INT8_PER_TOKEN_HEAD = 2 # per-token-head dynamic scales for int8
FP8_PER_TOKEN_HEAD = 3 # per-token-head dynamic scales for fp8
@property
def is_per_token_head(self) -> bool:
"""True for any per-token-head quantization mode."""
return self >= 2
def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
"""Map a ``kv_cache_dtype`` string to a :class:`KVQuantMode`."""
if kv_cache_dtype == "int8_per_token_head":
return KVQuantMode.INT8_PER_TOKEN_HEAD
if kv_cache_dtype == "fp8_per_token_head":
return KVQuantMode.FP8_PER_TOKEN_HEAD
if kv_cache_dtype.startswith("fp8"):
return KVQuantMode.FP8_PER_TENSOR
return KVQuantMode.NONE
def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
return get_kv_quant_mode(kv_cache_dtype) != KVQuantMode.NONE
def kv_cache_uses_per_token_head_scales(kv_cache_dtype: str) -> bool:
"""Return True if *kv_cache_dtype* needs per-token-head scales."""
return get_kv_quant_mode(kv_cache_dtype).is_per_token_head
@dataclass(frozen=True)
class KVCacheSpec:
"""
@@ -66,11 +115,19 @@ class AttentionSpec(KVCacheSpec):
num_kv_heads: int
head_size: int
dtype: torch.dtype
kv_quant_mode: KVQuantMode = KVQuantMode.NONE
page_size_padded: int | None = None
@property
def page_size_bytes(self) -> int:
real_page_size = self.real_page_size_bytes
# Per-token-head scales are stored in separate tensors managed
# by the attention backend, but the memory is carved from the
# raw KV cache allocation so it must be budgeted here.
if self.kv_quant_mode.is_per_token_head:
real_page_size += (
2 * self.block_size * self.num_kv_heads * get_dtype_size(torch.float32)
)
if self.page_size_padded is not None:
assert self.page_size_padded >= real_page_size
return self.page_size_padded
@@ -159,6 +216,7 @@ class FullAttentionSpec(AttentionSpec):
head_size=specs[0].head_size,
head_size_v=specs[0].head_size_v,
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
sliding_window=cls.merge_window_sizes(sliding_window),
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),
@@ -220,6 +278,7 @@ class MLAAttentionSpec(FullAttentionSpec):
num_kv_heads=specs[0].num_kv_heads,
head_size=specs[0].head_size,
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
cache_dtype_str=cache_dtype_str_set.pop(),
)
@@ -352,6 +411,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
head_size_v=specs[0].head_size_v,
sink_len=specs[0].sink_len,
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
sliding_window=cls.merge_window_sizes(sliding_window),
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),