[Feature] KV cache per-token-head INT8/FP8 quantization (#38378)
Signed-off-by: JartX <sagformas@epdcenter.es> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: yangyang4991 <yangyang4991@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
co-authored by
yangyang4991
Wentao Ye
Isotr0py
parent
4eefbf9609
commit
2ce3d0ce36
+17
-4
@@ -8,7 +8,10 @@ from pydantic import Field, SkipValidation, field_validator, model_validator
|
||||
|
||||
from vllm.config.utils import config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.utils.torch_utils import (
|
||||
is_quantized_kv_cache,
|
||||
kv_cache_uses_per_token_head_scales,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -21,6 +24,8 @@ CacheDType = Literal[
|
||||
"fp8_e5m2",
|
||||
"fp8_inc",
|
||||
"fp8_ds_mla",
|
||||
"int8_per_token_head",
|
||||
"fp8_per_token_head",
|
||||
]
|
||||
MambaDType = Literal["auto", "float32", "float16"]
|
||||
MambaCacheMode = Literal["all", "align", "none"]
|
||||
@@ -237,12 +242,20 @@ class CacheConfig:
|
||||
@field_validator("cache_dtype", mode="after")
|
||||
@classmethod
|
||||
def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType:
|
||||
if is_quantized_kv_cache(cache_dtype):
|
||||
if kv_cache_uses_per_token_head_scales(cache_dtype):
|
||||
logger.info(
|
||||
"Using fp8 data type to store kv cache. It reduces the GPU "
|
||||
"Using %s data type to store kv cache. It reduces the GPU "
|
||||
"memory footprint and boosts the performance. "
|
||||
"Dynamic per-token-head scales will be computed at runtime.",
|
||||
str(cache_dtype),
|
||||
)
|
||||
elif is_quantized_kv_cache(cache_dtype):
|
||||
logger.info(
|
||||
"Using %s data type to store kv cache. It reduces the GPU "
|
||||
"memory footprint and boosts the performance. "
|
||||
"Meanwhile, it may cause accuracy drop without a proper "
|
||||
"scaling factor."
|
||||
"scaling factor",
|
||||
str(cache_dtype),
|
||||
)
|
||||
return cache_dtype
|
||||
|
||||
|
||||
Reference in New Issue
Block a user