diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 027f19a90b..b8f197b316 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -2409,6 +2409,9 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]): q, kv_c_and_k_pe_cache, attn_metadata, k_scale ) + context_output = context_output[..., : self.v_head_dim] + suffix_output = suffix_output[..., : self.v_head_dim] + output = output.view(-1, self.num_heads, self.v_head_dim) merge_attn_states( output=output, @@ -2421,6 +2424,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]): elif output_scale is None: # With output_scale set, backend already wrote into `output` in place. assert isinstance(output_prefill, torch.Tensor) + output_prefill = output_prefill[..., : self.v_head_dim] output_prefill = output_prefill.flatten(start_dim=-2) output.copy_(output_prefill) diff --git a/vllm/v1/attention/backends/mla/prefill/flash_attn.py b/vllm/v1/attention/backends/mla/prefill/flash_attn.py index 470197876e..70e1f41c81 100644 --- a/vllm/v1/attention/backends/mla/prefill/flash_attn.py +++ b/vllm/v1/attention/backends/mla/prefill/flash_attn.py @@ -192,10 +192,6 @@ class FlashAttnPrefillBackend(MLAPrefillBackend): if isinstance(attn_out, tuple): attn_out, lse = attn_out[0], attn_out[1] - # Unpad output back to v_head_dim if we padded V - if self.requires_v_padding: - attn_out = attn_out[..., : v.shape[-1]] - # Remain consistent with old `flash_attn_varlen_func` where there # is only one output tensor if `return_softmax_lse` is False. if return_softmax_lse: diff --git a/vllm/v1/attention/ops/merge_attn_states.py b/vllm/v1/attention/ops/merge_attn_states.py index cf4338fb18..20c7503e9d 100644 --- a/vllm/v1/attention/ops/merge_attn_states.py +++ b/vllm/v1/attention/ops/merge_attn_states.py @@ -46,6 +46,14 @@ def merge_attn_states( When provided, output must be FP8 dtype. """ + # Both the CUDA and Triton kernels derive the suffix head stride from + # prefix_output, so suffix_output must share the same head stride. + assert prefix_output.stride(1) == suffix_output.stride(1), ( + "merge_attn_states requires prefix_output and suffix_output to have " + f"matching head strides, got {prefix_output.stride(1)} and " + f"{suffix_output.stride(1)}" + ) + # NOTE(DefTruth): Currently, custom merge_attn_states CUDA kernel # does not support FP8 dtype for inputs, fallback to use Triton kernel. # However, when output_scale is provided, the inputs are still BF16/FP16