[Feature] KV cache per-token-head INT8/FP8 quantization (#38378)
Signed-off-by: JartX <sagformas@epdcenter.es> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: yangyang4991 <yangyang4991@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
co-authored by
yangyang4991
Wentao Ye
Isotr0py
parent
4eefbf9609
commit
2ce3d0ce36
@@ -505,6 +505,7 @@ class Platform:
|
||||
FullAttentionSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
@@ -516,6 +517,8 @@ class Platform:
|
||||
else:
|
||||
kv_cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
|
||||
|
||||
kv_quant_mode = get_kv_quant_mode(cache_config.cache_dtype)
|
||||
|
||||
# Compute attention page size for 1 token
|
||||
if model_config.use_mla:
|
||||
attn_page_size_1_token = MLAAttentionSpec(
|
||||
@@ -523,6 +526,7 @@ class Platform:
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=kv_cache_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
).page_size_bytes
|
||||
else:
|
||||
attn_page_size_1_token = FullAttentionSpec(
|
||||
@@ -530,6 +534,7 @@ class Platform:
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=kv_cache_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
).page_size_bytes
|
||||
|
||||
# Compute mamba page size
|
||||
|
||||
Reference in New Issue
Block a user