[Bugfix] Enable FlashAttention MLA prefill for Mistral Small 4 head dims (#48855)
Signed-off-by: juliendenize <julien.denize@mistral.ai> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
d5b1ec2684
commit
ce4bdcbda4
@@ -63,11 +63,16 @@ class FlashAttnPrefillBackend(MLAPrefillBackend):
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=256,
|
||||
)
|
||||
dims_mistral_s4 = MLADimensions(
|
||||
qk_nope_head_dim=64,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
)
|
||||
fa_version = get_flash_attn_version()
|
||||
if fa_version == 4:
|
||||
return mla_dimensions == dims_deepseek
|
||||
return mla_dimensions in [dims_deepseek, dims_mistral_s4]
|
||||
else:
|
||||
return mla_dimensions in [dims_deepseek, dims_glm]
|
||||
return mla_dimensions in [dims_deepseek, dims_glm, dims_mistral_s4]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user