From d21c5b50a1d3db5950ab905fcf78b713c81e15e1 Mon Sep 17 00:00:00 2001 From: Chauncey Date: Thu, 27 Aug 2026 23:54:32 +0800 Subject: [PATCH] [Perf][GLM5.2] Fuse sparse MLA Q concatenation with head padding (#53878) Signed-off-by: chaunceyjiang --- tests/kernels/test_concat_mla_q.py | 11 ++++++++ .../v1/attention/test_sparse_mla_backends.py | 6 +++-- .../attention/backends/mla/flashmla_sparse.py | 26 ++++++++++++++++--- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/tests/kernels/test_concat_mla_q.py b/tests/kernels/test_concat_mla_q.py index fec5c063c7..6ace34e503 100644 --- a/tests/kernels/test_concat_mla_q.py +++ b/tests/kernels/test_concat_mla_q.py @@ -34,6 +34,17 @@ def test_concat_mla_q_contiguous(num_tokens, num_heads, nope_dim, rope_dim, dtyp torch.testing.assert_close(q_out, ref, atol=0, rtol=0) +def test_concat_mla_q_padded_output_heads(): + """The fused concat can write directly into a head-padded workspace.""" + ql_nope = torch.randn(4, 8, 512, dtype=torch.bfloat16, device="cuda") + q_pe = torch.randn(4, 8, 64, dtype=torch.bfloat16, device="cuda") + q_out = torch.empty(4, 64, 576, dtype=torch.bfloat16, device="cuda") + + ops.concat_mla_q(ql_nope, q_pe, q_out) + + torch.testing.assert_close(q_out[:, :8], torch.cat((ql_nope, q_pe), dim=-1)) + + @pytest.mark.parametrize("num_tokens", [t for t in NUM_TOKENS if t > 1]) @pytest.mark.parametrize("num_heads", NUM_HEADS) @pytest.mark.parametrize("nope_dim", NOPE_DIM) diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index be58cf8b46..b65fe24243 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -824,18 +824,20 @@ def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens(): captured = {} - def _stub_kernel(q, kv, indices, lengths): + def _stub_kernel(q, kv, indices, lengths, actual_num_heads): captured["indices"] = indices + captured["actual_num_heads"] = actual_num_heads return torch.zeros(q.shape[0], q.shape[1], 512, dtype=q.dtype, device=q.device) stub_impl = SimpleNamespace(_bf16_flash_mla_kernel=_stub_kernel) out = FlashMLASparseImpl._forward_bf16_kv( - stub_impl, q, kv_cache, topk_indices, attn_metadata + stub_impl, q, kv_cache, topk_indices, attn_metadata, q.shape[1] ) assert out.shape[0] == num_mqa_tokens assert captured["indices"].shape[0] == num_mqa_tokens + assert captured["actual_num_heads"] == q.shape[1] reference = _triton_convert_reference_impl( attn_metadata.req_id_per_token[:num_mqa_tokens], attn_metadata.block_table, diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 2ff9a3013d..e134cd1754 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -587,7 +587,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): vllm_config = get_current_vllm_config() max_tokens = vllm_config.scheduler_config.max_num_batched_tokens - q_concat_shape = (max_tokens, num_heads, head_size) + q_concat_heads = num_heads + if not is_quantized_kv_cache(kv_cache_dtype): + q_concat_heads = ( + (num_heads + self.prefill_padding - 1) + // self.prefill_padding + * self.prefill_padding + ) + q_concat_shape = (max_tokens, q_concat_heads, head_size) if is_quantized_kv_cache(kv_cache_dtype): assert kv_cache_dtype == "fp8_ds_mla", ( "FlashMLA Sparse Attention backend fp8 only supports " @@ -626,6 +633,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): kv_c_and_k_pe_cache: torch.Tensor, topk_indices: torch.Tensor, attn_metadata: FlashMLASparseMetadata, + actual_num_heads: int, ) -> tuple[torch.Tensor, torch.Tensor]: # Convert per-request indices to global slots (decode) or workspace # offsets (prefill). req_id_per_token covers the whole batch; slice it @@ -648,6 +656,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): kv_rows, topk_indices, topk_length, + actual_num_heads, ) def _forward_fp8_kv_separate_prefill_decode( @@ -880,6 +889,7 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): kv_c_and_k_pe_cache: torch.Tensor, topk_indices: torch.Tensor, topk_length: torch.Tensor | None = None, + actual_num_heads: int | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: num_tokens = q.shape[0] kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view( @@ -889,13 +899,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): # NOTE(Chen): kernel requires num_local_head to be a multiple of # 64 on hopper and 128 on blackwell. Pad from q's head count, not # self.num_heads: under DCP the heads are all-gathered before this. - actual_num_heads = q.shape[1] + if actual_num_heads is None: + actual_num_heads = q.shape[1] padded_num_heads = ( (actual_num_heads + self.prefill_padding - 1) // self.prefill_padding * self.prefill_padding ) - if actual_num_heads < padded_num_heads: + if q.shape[1] < padded_num_heads: logger.warning_once( f"Padding num_heads from {actual_num_heads} to " f"{padded_num_heads} for BF16 sparse prefill kernel" @@ -928,10 +939,13 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): # MQA 576/512 approach for both prefill and decode # Concatenate q if it's a tuple (ql_nope, q_pe) + actual_num_heads = self.num_heads if isinstance(q, tuple): ql_nope, q_pe = q q = self.q_concat_buffer[: ql_nope.shape[0]] ops.concat_mla_q(ql_nope, q_pe, q) + else: + actual_num_heads = q.shape[1] num_actual_toks = q.shape[0] @@ -945,7 +959,11 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): if not use_fp8_cache: attn_out, bf16_lse = self._forward_bf16_kv( - q, kv_c_and_k_pe_cache, topk_indices, attn_metadata + q, + kv_c_and_k_pe_cache, + topk_indices, + attn_metadata, + actual_num_heads, ) if self.need_to_return_lse_for_decode: lse = bf16_lse