[Bugfix] Enable FlashAttention MLA prefill for Mistral Small 4 head dims (#48855)
Signed-off-by: juliendenize <julien.denize@mistral.ai> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
d5b1ec2684
commit
ce4bdcbda4
@@ -205,7 +205,7 @@ hardware and configuration.
|
|||||||
|
|
||||||
| Backend | Description | Dtypes | Compute Cap. | Notes |
|
| Backend | Description | Dtypes | Compute Cap. | Notes |
|
||||||
| ------- | ----------- | ------ | ------------ | ----- |
|
| ------- | ----------- | ------ | ------------ | ----- |
|
||||||
| `FLASH_ATTN`‡ | FlashAttention varlen (FA2/FA3/FA4) | fp16, bf16 | Any | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) (FA2/FA3/FA4) or (qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256) (FA2/FA3 only) |
|
| `FLASH_ATTN`‡ | FlashAttention varlen (FA2/FA3/FA4) | fp16, bf16 | Any | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) (FA2/FA3/FA4) or (qk_nope_head_dim=64, qk_rope_head_dim=64, v_head_dim=128) (FA2/FA3/FA4) or (qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256) (FA2/FA3 only) |
|
||||||
| `TRTLLM_RAGGED` | TensorRT-LLM ragged attention | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) or (qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256) only |
|
| `TRTLLM_RAGGED` | TensorRT-LLM ragged attention | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) or (qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256) only |
|
||||||
| `FLASHINFER` | FlashInfer CUTLASS backend | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
|
| `FLASHINFER` | FlashInfer CUTLASS backend | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
|
||||||
| `TOKENSPEED_MLA` | | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
|
| `TOKENSPEED_MLA` | | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
|
||||||
|
|||||||
@@ -125,8 +125,8 @@ class TestGetMLAPrefillBackend:
|
|||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("qk_nope_head_dim", "v_head_dim"),
|
("qk_nope_head_dim", "v_head_dim"),
|
||||||
[(128, 128), (192, 256)],
|
[(128, 128), (192, 256), (64, 128)],
|
||||||
ids=["deepseek", "glm"],
|
ids=["deepseek", "glm", "mistral_s4"],
|
||||||
)
|
)
|
||||||
def test_auto_selection_on_hopper(self, qk_nope_head_dim: int, v_head_dim: int):
|
def test_auto_selection_on_hopper(self, qk_nope_head_dim: int, v_head_dim: int):
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -63,11 +63,16 @@ class FlashAttnPrefillBackend(MLAPrefillBackend):
|
|||||||
qk_rope_head_dim=64,
|
qk_rope_head_dim=64,
|
||||||
v_head_dim=256,
|
v_head_dim=256,
|
||||||
)
|
)
|
||||||
|
dims_mistral_s4 = MLADimensions(
|
||||||
|
qk_nope_head_dim=64,
|
||||||
|
qk_rope_head_dim=64,
|
||||||
|
v_head_dim=128,
|
||||||
|
)
|
||||||
fa_version = get_flash_attn_version()
|
fa_version = get_flash_attn_version()
|
||||||
if fa_version == 4:
|
if fa_version == 4:
|
||||||
return mla_dimensions == dims_deepseek
|
return mla_dimensions in [dims_deepseek, dims_mistral_s4]
|
||||||
else:
|
else:
|
||||||
return mla_dimensions in [dims_deepseek, dims_glm]
|
return mla_dimensions in [dims_deepseek, dims_glm, dims_mistral_s4]
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
|
|||||||
Reference in New Issue
Block a user