[Attention][MLA] Support fp8_ds_mla KV cache for NoPE-512 models on SM90 (#59246)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: Leoyzen <leoyzen@gmail.com>
This commit is contained in:
co-authored by
Leoyzen
parent
d0d6e5f3a2
commit
ff53f32409
@@ -538,10 +538,9 @@ def create_and_prepopulate_kv_cache(
|
||||
if fp8_attention:
|
||||
if use_fp8_ds_mla:
|
||||
kv_lora_rank = kv_c_contexts[0].shape[-1]
|
||||
rope_dim = k_pe_contexts[0].shape[-1]
|
||||
# 4 * 4: 4 float32 scale values for 128-element tiles
|
||||
# 2 * rope_dim: 16-bit RoPE values
|
||||
kv_entry_size = kv_lora_rank + 4 * 4 + 2 * rope_dim
|
||||
# 2 * 64: 16-bit RoPE values (zero-filled for NoPE models)
|
||||
kv_entry_size = kv_lora_rank + 4 * 4 + 2 * 64
|
||||
elif use_nvfp4_ds_mla:
|
||||
kv_lora_rank = kv_c_contexts[0].shape[-1]
|
||||
rope_dim = k_pe_contexts[0].shape[-1]
|
||||
|
||||
Reference in New Issue
Block a user