From 837eae64580c885101ee95b073aafb27a485e7ce Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Thu, 30 Jul 2026 11:39:07 -0400 Subject: [PATCH] [DSv4 Perf] Remove redundant full kernel for dsv4, 1.88x kernel performance improvement (#50298) Signed-off-by: yewentao256 --- .../kernels/attention/test_flashmla_sparse.py | 26 +++++++++++-------- .../deepseek_v4/common/ops/cache_utils.py | 22 +++++++++------- vllm/models/deepseek_v4/nvidia/flashmla.py | 17 ++++++++++-- 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/tests/kernels/attention/test_flashmla_sparse.py b/tests/kernels/attention/test_flashmla_sparse.py index fa5602fdf9..040ad50a5b 100644 --- a/tests/kernels/attention/test_flashmla_sparse.py +++ b/tests/kernels/attention/test_flashmla_sparse.py @@ -135,7 +135,8 @@ def test_sparse_flashmla_decode_smoke(): assert lse.shape[0] == batch_size -def test_sparse_flashmla_prefill_smoke(): +@pytest.mark.parametrize("h_q", [64, 128]) +def test_sparse_flashmla_prefill_smoke(h_q: int): import vllm.v1.attention.ops.flashmla as fm ok, reason = fm.is_flashmla_sparse_supported() @@ -143,22 +144,25 @@ def test_sparse_flashmla_prefill_smoke(): pytest.skip(reason) device = torch.device("cuda") + torch.manual_seed(0) s_q = 1 - s_kv = 1 - h_q = 64 # kernel expects multiple of 64 + s_kv = 8 h_kv = 1 d_qk = 576 d_v = 512 topk = 128 + q = torch.randn((s_q, h_q, d_qk), dtype=torch.bfloat16, device=device) + kv = torch.randn((s_kv, h_kv, d_qk), dtype=torch.bfloat16, device=device) + indices = torch.randint(s_kv, (s_q, h_kv, topk), dtype=torch.int32, device=device) + reference_indices = indices.clone() + reference_indices[..., 1:] = -1 + kwargs = {"topk_length": torch.ones(1, dtype=torch.int32, device=device)} + reference = fm.flash_mla_sparse_fwd(q, kv, reference_indices, 1.0, d_v, **kwargs) + actual = fm.flash_mla_sparse_fwd(q, kv, indices, 1.0, d_v, **kwargs) - q = torch.zeros((s_q, h_q, d_qk), dtype=torch.bfloat16, device=device) - kv = torch.zeros((s_kv, h_kv, d_qk), dtype=torch.bfloat16, device=device) - indices = torch.zeros((s_q, h_kv, topk), dtype=torch.int32, device=device) - - out, max_logits, lse = fm.flash_mla_sparse_fwd(q, kv, indices, 1.0, d_v) - assert out.shape == (s_q, h_q, d_v) - assert max_logits.shape == (s_q, h_q) - assert lse.shape == (s_q, h_q) + for actual_tensor, reference_tensor in zip(actual, reference): + torch.testing.assert_close(actual_tensor, reference_tensor, rtol=0, atol=0) + assert actual[0].shape == (s_q, h_q, d_v) def test_deepseek_v4_prefill_chunk_planning_expands_for_short_sequences(): diff --git a/vllm/models/deepseek_v4/common/ops/cache_utils.py b/vllm/models/deepseek_v4/common/ops/cache_utils.py index fc6e4ee330..c73d39499e 100644 --- a/vllm/models/deepseek_v4/common/ops/cache_utils.py +++ b/vllm/models/deepseek_v4/common/ops/cache_utils.py @@ -535,6 +535,7 @@ def combine_topk_swa_indices( topk: int, M: int, N: int, + out: tuple[torch.Tensor, torch.Tensor] | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: num_tokens = topk_indices.shape[0] combined_topk = ( @@ -542,15 +543,18 @@ def combine_topk_swa_indices( // _SPARSE_PREFILL_TOPK_ALIGNMENT * _SPARSE_PREFILL_TOPK_ALIGNMENT ) - combined_indices = torch.full( - (num_tokens, combined_topk), - fill_value=-1, - dtype=torch.int32, - device=topk_indices.device, - ) - combined_lens = torch.empty( - num_tokens, dtype=torch.int32, device=topk_indices.device - ) + if out is None: + combined_indices = torch.full( + (num_tokens, combined_topk), + fill_value=-1, + dtype=torch.int32, + device=topk_indices.device, + ) + combined_lens = torch.empty( + num_tokens, dtype=torch.int32, device=topk_indices.device + ) + else: + combined_indices, combined_lens = out _COMBINE_TOPK_SWA_INDICES_KERNEL( combined_indices, diff --git a/vllm/models/deepseek_v4/nvidia/flashmla.py b/vllm/models/deepseek_v4/nvidia/flashmla.py index 9fa4e1c11b..3169ccfce4 100644 --- a/vllm/models/deepseek_v4/nvidia/flashmla.py +++ b/vllm/models/deepseek_v4/nvidia/flashmla.py @@ -20,6 +20,7 @@ from vllm.models.deepseek_v4.sparse_mla import ( DeepseekV4FlashMLABackend, DeepseekV4FlashMLAMetadata, ) +from vllm.utils.math_utils import round_up from vllm.v1.attention.ops.flashmla import ( flash_mla_sparse_fwd, flash_mla_with_kvcache, @@ -95,8 +96,13 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention): // self.compress_ratio ) M = N + self.window_size + self.max_num_batched_tokens + assert self.topk_indices_buffer is not None + top_k = 0 if swa_only else self.topk_indices_buffer.shape[-1] + combined_topk = round_up(top_k + self.window_size, 128) current_workspace_manager().get_simultaneous( ((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16), + ((self.max_num_batched_tokens, combined_topk), torch.int32), + ((self.max_num_batched_tokens,), torch.int32), ) output.zero_() return @@ -284,11 +290,15 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention): ) assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0" workspace_manager = current_workspace_manager() + combined_topk = round_up(top_k + self.window_size, 128) for chunk_start, chunk_end, chunk_N, chunk_M in chunk_plan: chunk_size = chunk_end - chunk_start - kv = workspace_manager.get_simultaneous( + workspace = workspace_manager.get_simultaneous( ((chunk_size, chunk_M, q.shape[-1]), torch.bfloat16), - )[0] + ((self.max_num_batched_tokens, combined_topk), torch.int32), + ((self.max_num_batched_tokens,), torch.int32), + ) + kv, combined_indices_out, combined_lens_out = workspace if not swa_only: # Gather compressed KV assert attn_metadata is not None @@ -322,6 +332,8 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention): query_end = ( query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base ) + combined_indices_out = combined_indices_out[: query_end - query_start] + combined_lens_out = combined_lens_out[: query_end - query_start] combined_indices, combined_lens = combine_topk_swa_indices( topk_indices[query_start:query_end], @@ -335,6 +347,7 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention): top_k, chunk_M, chunk_N, + out=(combined_indices_out, combined_lens_out), ) flash_mla_sparse_fwd( q=q[query_start:query_end],