Fix FlashAttention MLA prefill V unpadding (#42642)

Signed-off-by: Martin Vit <martin@voipmonitor.org>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Martin Vit
2026-07-08 15:22:20 +00:00
committed by GitHub
co-authored by Matthew Bonanni
parent 0d12618e98
commit c2ecd0f888
3 changed files with 12 additions and 4 deletions
@@ -2409,6 +2409,9 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
q, kv_c_and_k_pe_cache, attn_metadata, k_scale
)
context_output = context_output[..., : self.v_head_dim]
suffix_output = suffix_output[..., : self.v_head_dim]
output = output.view(-1, self.num_heads, self.v_head_dim)
merge_attn_states(
output=output,
@@ -2421,6 +2424,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
elif output_scale is None:
# With output_scale set, backend already wrote into `output` in place.
assert isinstance(output_prefill, torch.Tensor)
output_prefill = output_prefill[..., : self.v_head_dim]
output_prefill = output_prefill.flatten(start_dim=-2)
output.copy_(output_prefill)
@@ -192,10 +192,6 @@ class FlashAttnPrefillBackend(MLAPrefillBackend):
if isinstance(attn_out, tuple):
attn_out, lse = attn_out[0], attn_out[1]
# Unpad output back to v_head_dim if we padded V
if self.requires_v_padding:
attn_out = attn_out[..., : v.shape[-1]]
# Remain consistent with old `flash_attn_varlen_func` where there
# is only one output tensor if `return_softmax_lse` is False.
if return_softmax_lse:
@@ -46,6 +46,14 @@ def merge_attn_states(
When provided, output must be FP8 dtype.
"""
# Both the CUDA and Triton kernels derive the suffix head stride from
# prefix_output, so suffix_output must share the same head stride.
assert prefix_output.stride(1) == suffix_output.stride(1), (
"merge_attn_states requires prefix_output and suffix_output to have "
f"matching head strides, got {prefix_output.stride(1)} and "
f"{suffix_output.stride(1)}"
)
# NOTE(DefTruth): Currently, custom merge_attn_states CUDA kernel
# does not support FP8 dtype for inputs, fallback to use Triton kernel.
# However, when output_scale is provided, the inputs are still BF16/FP16