[DSv4 Perf] Remove redundant full kernel for dsv4, 1.88x kernel performance improvement (#50298)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-07-30 11:39:07 -04:00
committed by GitHub
parent 904fae8be1
commit 837eae6458
3 changed files with 43 additions and 22 deletions
+15 -11
View File
@@ -135,7 +135,8 @@ def test_sparse_flashmla_decode_smoke():
assert lse.shape[0] == batch_size
def test_sparse_flashmla_prefill_smoke():
@pytest.mark.parametrize("h_q", [64, 128])
def test_sparse_flashmla_prefill_smoke(h_q: int):
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_sparse_supported()
@@ -143,22 +144,25 @@ def test_sparse_flashmla_prefill_smoke():
pytest.skip(reason)
device = torch.device("cuda")
torch.manual_seed(0)
s_q = 1
s_kv = 1
h_q = 64 # kernel expects multiple of 64
s_kv = 8
h_kv = 1
d_qk = 576
d_v = 512
topk = 128
q = torch.randn((s_q, h_q, d_qk), dtype=torch.bfloat16, device=device)
kv = torch.randn((s_kv, h_kv, d_qk), dtype=torch.bfloat16, device=device)
indices = torch.randint(s_kv, (s_q, h_kv, topk), dtype=torch.int32, device=device)
reference_indices = indices.clone()
reference_indices[..., 1:] = -1
kwargs = {"topk_length": torch.ones(1, dtype=torch.int32, device=device)}
reference = fm.flash_mla_sparse_fwd(q, kv, reference_indices, 1.0, d_v, **kwargs)
actual = fm.flash_mla_sparse_fwd(q, kv, indices, 1.0, d_v, **kwargs)
q = torch.zeros((s_q, h_q, d_qk), dtype=torch.bfloat16, device=device)
kv = torch.zeros((s_kv, h_kv, d_qk), dtype=torch.bfloat16, device=device)
indices = torch.zeros((s_q, h_kv, topk), dtype=torch.int32, device=device)
out, max_logits, lse = fm.flash_mla_sparse_fwd(q, kv, indices, 1.0, d_v)
assert out.shape == (s_q, h_q, d_v)
assert max_logits.shape == (s_q, h_q)
assert lse.shape == (s_q, h_q)
for actual_tensor, reference_tensor in zip(actual, reference):
torch.testing.assert_close(actual_tensor, reference_tensor, rtol=0, atol=0)
assert actual[0].shape == (s_q, h_q, d_v)
def test_deepseek_v4_prefill_chunk_planning_expands_for_short_sequences():
@@ -535,6 +535,7 @@ def combine_topk_swa_indices(
topk: int,
M: int,
N: int,
out: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
num_tokens = topk_indices.shape[0]
combined_topk = (
@@ -542,15 +543,18 @@ def combine_topk_swa_indices(
// _SPARSE_PREFILL_TOPK_ALIGNMENT
* _SPARSE_PREFILL_TOPK_ALIGNMENT
)
combined_indices = torch.full(
(num_tokens, combined_topk),
fill_value=-1,
dtype=torch.int32,
device=topk_indices.device,
)
combined_lens = torch.empty(
num_tokens, dtype=torch.int32, device=topk_indices.device
)
if out is None:
combined_indices = torch.full(
(num_tokens, combined_topk),
fill_value=-1,
dtype=torch.int32,
device=topk_indices.device,
)
combined_lens = torch.empty(
num_tokens, dtype=torch.int32, device=topk_indices.device
)
else:
combined_indices, combined_lens = out
_COMBINE_TOPK_SWA_INDICES_KERNEL(
combined_indices,
+15 -2
View File
@@ -20,6 +20,7 @@ from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4FlashMLABackend,
DeepseekV4FlashMLAMetadata,
)
from vllm.utils.math_utils import round_up
from vllm.v1.attention.ops.flashmla import (
flash_mla_sparse_fwd,
flash_mla_with_kvcache,
@@ -95,8 +96,13 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
// self.compress_ratio
)
M = N + self.window_size + self.max_num_batched_tokens
assert self.topk_indices_buffer is not None
top_k = 0 if swa_only else self.topk_indices_buffer.shape[-1]
combined_topk = round_up(top_k + self.window_size, 128)
current_workspace_manager().get_simultaneous(
((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16),
((self.max_num_batched_tokens, combined_topk), torch.int32),
((self.max_num_batched_tokens,), torch.int32),
)
output.zero_()
return
@@ -284,11 +290,15 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
)
assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0"
workspace_manager = current_workspace_manager()
combined_topk = round_up(top_k + self.window_size, 128)
for chunk_start, chunk_end, chunk_N, chunk_M in chunk_plan:
chunk_size = chunk_end - chunk_start
kv = workspace_manager.get_simultaneous(
workspace = workspace_manager.get_simultaneous(
((chunk_size, chunk_M, q.shape[-1]), torch.bfloat16),
)[0]
((self.max_num_batched_tokens, combined_topk), torch.int32),
((self.max_num_batched_tokens,), torch.int32),
)
kv, combined_indices_out, combined_lens_out = workspace
if not swa_only:
# Gather compressed KV
assert attn_metadata is not None
@@ -322,6 +332,8 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
query_end = (
query_start_loc_cpu[num_decodes + chunk_end] - prefill_token_base
)
combined_indices_out = combined_indices_out[: query_end - query_start]
combined_lens_out = combined_lens_out[: query_end - query_start]
combined_indices, combined_lens = combine_topk_swa_indices(
topk_indices[query_start:query_end],
@@ -335,6 +347,7 @@ class DeepseekV4FlashMLAAttention(DeepseekV4Attention):
top_k,
chunk_M,
chunk_N,
out=(combined_indices_out, combined_lens_out),
)
flash_mla_sparse_fwd(
q=q[query_start:query_end],