[Feature] KV cache per-token-head INT8/FP8 quantization (#38378)

Signed-off-by: JartX <sagformas@epdcenter.es>
Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: yangyang4991 <yangyang4991@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
JartX
2026-04-02 08:13:26 -04:00
committed by GitHub
co-authored by yangyang4991 Wentao Ye Isotr0py
parent 4eefbf9609
commit 2ce3d0ce36
16 changed files with 1308 additions and 66 deletions
+5
View File
@@ -505,6 +505,7 @@ class Platform:
FullAttentionSpec,
MambaSpec,
MLAAttentionSpec,
get_kv_quant_mode,
)
cache_config = vllm_config.cache_config
@@ -516,6 +517,8 @@ class Platform:
else:
kv_cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
kv_quant_mode = get_kv_quant_mode(cache_config.cache_dtype)
# Compute attention page size for 1 token
if model_config.use_mla:
attn_page_size_1_token = MLAAttentionSpec(
@@ -523,6 +526,7 @@ class Platform:
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
head_size=model_config.get_head_size(),
dtype=kv_cache_dtype,
kv_quant_mode=kv_quant_mode,
).page_size_bytes
else:
attn_page_size_1_token = FullAttentionSpec(
@@ -530,6 +534,7 @@ class Platform:
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
head_size=model_config.get_head_size(),
dtype=kv_cache_dtype,
kv_quant_mode=kv_quant_mode,
).page_size_bytes
# Compute mamba page size