Fix FlashAttention MLA prefill V unpadding (#42642)
Signed-off-by: Martin Vit <martin@voipmonitor.org> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
parent
0d12618e98
commit
c2ecd0f888
@@ -2409,6 +2409,9 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
q, kv_c_and_k_pe_cache, attn_metadata, k_scale
|
||||
)
|
||||
|
||||
context_output = context_output[..., : self.v_head_dim]
|
||||
suffix_output = suffix_output[..., : self.v_head_dim]
|
||||
|
||||
output = output.view(-1, self.num_heads, self.v_head_dim)
|
||||
merge_attn_states(
|
||||
output=output,
|
||||
@@ -2421,6 +2424,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]):
|
||||
elif output_scale is None:
|
||||
# With output_scale set, backend already wrote into `output` in place.
|
||||
assert isinstance(output_prefill, torch.Tensor)
|
||||
output_prefill = output_prefill[..., : self.v_head_dim]
|
||||
output_prefill = output_prefill.flatten(start_dim=-2)
|
||||
output.copy_(output_prefill)
|
||||
|
||||
|
||||
@@ -192,10 +192,6 @@ class FlashAttnPrefillBackend(MLAPrefillBackend):
|
||||
if isinstance(attn_out, tuple):
|
||||
attn_out, lse = attn_out[0], attn_out[1]
|
||||
|
||||
# Unpad output back to v_head_dim if we padded V
|
||||
if self.requires_v_padding:
|
||||
attn_out = attn_out[..., : v.shape[-1]]
|
||||
|
||||
# Remain consistent with old `flash_attn_varlen_func` where there
|
||||
# is only one output tensor if `return_softmax_lse` is False.
|
||||
if return_softmax_lse:
|
||||
|
||||
@@ -46,6 +46,14 @@ def merge_attn_states(
|
||||
When provided, output must be FP8 dtype.
|
||||
"""
|
||||
|
||||
# Both the CUDA and Triton kernels derive the suffix head stride from
|
||||
# prefix_output, so suffix_output must share the same head stride.
|
||||
assert prefix_output.stride(1) == suffix_output.stride(1), (
|
||||
"merge_attn_states requires prefix_output and suffix_output to have "
|
||||
f"matching head strides, got {prefix_output.stride(1)} and "
|
||||
f"{suffix_output.stride(1)}"
|
||||
)
|
||||
|
||||
# NOTE(DefTruth): Currently, custom merge_attn_states CUDA kernel
|
||||
# does not support FP8 dtype for inputs, fallback to use Triton kernel.
|
||||
# However, when output_scale is provided, the inputs are still BF16/FP16
|
||||
|
||||
Reference in New Issue
Block a user