[Feature] KV cache per-token-head INT8/FP8 quantization (#38378)
Signed-off-by: JartX <sagformas@epdcenter.es> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: yangyang4991 <yangyang4991@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
co-authored by
yangyang4991
Wentao Ye
Isotr0py
parent
4eefbf9609
commit
2ce3d0ce36
@@ -10,6 +10,7 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.kv_cache_interface import kv_cache_uses_per_token_head_scales
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -53,6 +54,20 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
assert not hasattr(layer, "prob_scale")
|
||||
return
|
||||
|
||||
# Per-token-head quantized KV cache: scales are computed dynamically
|
||||
# per (token, head) in the kernel at cache-write time. Checkpoint
|
||||
# scales are never used regardless of calculate_kv_scales.
|
||||
if kv_cache_uses_per_token_head_scales(layer.kv_cache_dtype):
|
||||
layer._k_scale.copy_(1.0)
|
||||
layer._v_scale.copy_(1.0)
|
||||
layer._k_scale_float = 1.0
|
||||
layer._v_scale_float = 1.0
|
||||
del layer.k_scale
|
||||
del layer.v_scale
|
||||
del layer.q_scale
|
||||
del layer.prob_scale
|
||||
return
|
||||
|
||||
# If the kv-cache is not quantized, we enforce the k/v_scale to be 1.0
|
||||
# regardless whether the kv-scale is available in the checkpoint.
|
||||
# No need to process kv scales after loading if we are going to
|
||||
|
||||
Reference in New Issue
Block a user