[Attention][MLA] Support fp8_ds_mla KV cache for NoPE-512 models on SM90 (#59246)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: Leoyzen <leoyzen@gmail.com>
This commit is contained in:
Nicolò Lucchesi
2026-10-05 13:12:00 +00:00
committed by GitHub
co-authored by Leoyzen
parent d0d6e5f3a2
commit ff53f32409
5 changed files with 82 additions and 44 deletions
+2 -3
View File
@@ -538,10 +538,9 @@ def create_and_prepopulate_kv_cache(
if fp8_attention:
if use_fp8_ds_mla:
kv_lora_rank = kv_c_contexts[0].shape[-1]
rope_dim = k_pe_contexts[0].shape[-1]
# 4 * 4: 4 float32 scale values for 128-element tiles
# 2 * rope_dim: 16-bit RoPE values
kv_entry_size = kv_lora_rank + 4 * 4 + 2 * rope_dim
# 2 * 64: 16-bit RoPE values (zero-filled for NoPE models)
kv_entry_size = kv_lora_rank + 4 * 4 + 2 * 64
elif use_nvfp4_ds_mla:
kv_lora_rank = kv_c_contexts[0].shape[-1]
rope_dim = k_pe_contexts[0].shape[-1]