From bf5149b51606dc567503684905705cc26427f099 Mon Sep 17 00:00:00 2001 From: Ajay Anubolu <124525760+AjAnubolu@users.noreply.github.com> Date: Tue, 16 Jun 2026 07:09:00 -0700 Subject: [PATCH] [Bugfix] Fix FlashMLA sparse accuracy with topk_length and zero-init padding (#36616) Signed-off-by: AjAnubolu Signed-off-by: Matthew Bonanni Co-authored-by: Matthew Bonanni --- .../v1/attention/backends/mla/flashmla_sparse.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 353900f38f..2da71f9d2c 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -612,18 +612,20 @@ class FlashMLASparseImpl(SparseMLAAttentionImpl[FlashMLASparseMetadata]): ) -> torch.Tensor: # Convert per-request indices to global slots (decode) or workspace # offsets (prefill). - topk_indices = triton_convert_req_index_to_global_index( + topk_indices, topk_length = triton_convert_req_index_to_global_index( attn_metadata.req_id_per_token, attn_metadata.block_table, topk_indices, BLOCK_SIZE=attn_metadata.block_size, NUM_TOPK_TOKENS=topk_indices.shape[1], + return_valid_counts=True, ) return self._bf16_flash_mla_kernel( q, kv_c_and_k_pe_cache, topk_indices, + topk_length, ) def _forward_fp8_kv_separate_prefill_decode( @@ -651,7 +653,7 @@ class FlashMLASparseImpl(SparseMLAAttentionImpl[FlashMLASparseMetadata]): # For BF16 cache: always use global cache slots (no workspace) # prefill_workspace_starts has been adjusted in-place per chunk so # prefill indices automatically come out chunk-local - topk_indices = triton_convert_req_index_to_global_index( + topk_indices, topk_length = triton_convert_req_index_to_global_index( attn_metadata.req_id_per_token, attn_metadata.block_table, topk_indices, @@ -660,6 +662,7 @@ class FlashMLASparseImpl(SparseMLAAttentionImpl[FlashMLASparseMetadata]): HAS_PREFILL_WORKSPACE=has_prefill_workspace, prefill_workspace_request_ids=prefill_request_ids, prefill_workspace_starts=prefill_workspace_starts, + return_valid_counts=True, ) fp8_metadata = attn_metadata.fp8_extra_metadata @@ -722,11 +725,13 @@ class FlashMLASparseImpl(SparseMLAAttentionImpl[FlashMLASparseMetadata]): chunk_q = q[chunk.tokens_slice] chunk_topk_indices_workspace = topk_indices[chunk.tokens_slice] + chunk_topk_length = topk_length[chunk.tokens_slice] attn_out[chunk.tokens_slice] = self._bf16_flash_mla_kernel( chunk_q, chunk_workspace, chunk_topk_indices_workspace, + chunk_topk_length, ) return attn_out @@ -814,6 +819,7 @@ class FlashMLASparseImpl(SparseMLAAttentionImpl[FlashMLASparseMetadata]): q: torch.Tensor, kv_c_and_k_pe_cache: torch.Tensor, topk_indices: torch.Tensor, + topk_length: torch.Tensor | None = None, ) -> torch.Tensor: num_tokens = q.shape[0] kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view( @@ -834,7 +840,11 @@ class FlashMLASparseImpl(SparseMLAAttentionImpl[FlashMLASparseMetadata]): topk_indices = topk_indices.view(num_tokens, 1, -1) output = flash_mla_sparse_fwd( - q, kv_c_and_k_pe_cache, topk_indices, self.softmax_scale + q, + kv_c_and_k_pe_cache, + topk_indices, + self.softmax_scale, + topk_length=topk_length, )[0] output = output[:, : self.num_heads, :]