[Bugfix][DSv4.1] Fix FlashInfer DSpark non-causal attention (#57432)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Woosuk Kwon
2026-09-17 15:14:37 -07:00
committed by GitHub
co-authored by Codex
parent d3074acdda
commit 80447d2765
3 changed files with 214 additions and 6 deletions
@@ -26,7 +26,11 @@ from vllm.models.deepseek_v41.sparse_mla import (
)
from vllm.platforms.interface import DeviceCapability
from vllm.utils.flashinfer import flashinfer_trtllm_batch_decode_sparse_mla_dsv4
from vllm.v1.attention.backend import AttentionCGSupport, MultipleOf
from vllm.v1.attention.backend import (
AttentionCGSupport,
CommonAttentionMetadata,
MultipleOf,
)
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
@@ -183,6 +187,39 @@ class DeepseekSparseSWAFlashInferMetadataBuilder(DeepseekV41SparseSWAMetadataBui
_cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Graphs retain these addresses while each build refreshes their contents.
self._decode_topk_lens = torch.empty(
self._max_tokens, dtype=torch.int32, device=self.device
)
self._decode_seq_lens = torch.empty_like(self._decode_topk_lens)
def build(
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
fast_build: bool = False,
) -> "DeepseekSparseSWAMetadata":
metadata = super().build(common_prefix_len, common_attn_metadata, fast_build)
num_tokens = metadata.num_decode_tokens
if not common_attn_metadata.causal and num_tokens > 0:
assert metadata.decode_swa_lens is not None
assert metadata.seq_lens is not None
assert metadata.token_to_req_indices is not None
topk_lens = self._decode_topk_lens[:num_tokens]
seq_lens = self._decode_seq_lens[:num_tokens]
torch.clamp(metadata.decode_swa_lens, min=self.window_size, out=topk_lens)
torch.index_select(
metadata.seq_lens,
0,
metadata.token_to_req_indices[:num_tokens],
out=seq_lens,
)
metadata.flashinfer_decode_topk_lens = topk_lens
metadata.flashinfer_decode_seq_lens = seq_lens
return metadata
class DeepseekSparseSWAFlashInferBackend(DeepseekSparseSWABackend):
@staticmethod
@@ -493,21 +530,40 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
# Keep the TRTLLM-gen decode/prefill split: the launcher is tuned for
# uniform-q batches, and this avoids flattening mixed batches into one call.
if num_decode_tokens > 0:
decode_query = query[:num_decode_tokens]
decode_output = output[:num_decode_tokens]
decode_cu = query_start_loc[: num_decodes + 1]
decode_seq_lens = seq_lens[:num_decodes]
decode_topk_lens = sparse_topk_lens[:num_decode_tokens]
max_decode_query_len = swa_metadata.max_decode_query_len
if swa_metadata.decode_swa_width > self.window_size:
# DSpark's non-causal window extends past the fixed 128 SWA
# columns into the aliased compressed pool. Exclude padding,
# and expose the full block to each query instead of letting
# TRTLLM derive a causal SWA length from its query position.
assert swa_only
assert swa_metadata.flashinfer_decode_topk_lens is not None
assert swa_metadata.flashinfer_decode_seq_lens is not None
decode_topk_lens = swa_metadata.flashinfer_decode_topk_lens
decode_seq_lens = swa_metadata.flashinfer_decode_seq_lens
decode_query = decode_query.unsqueeze(1)
decode_output = decode_output.unsqueeze(1)
decode_cu = None
max_decode_query_len = 1
flashinfer_trtllm_batch_decode_sparse_mla_dsv4(
query=query[:num_decode_tokens],
query=decode_query,
swa_kv_cache=swa_k_cache,
workspace_buffer=workspace,
sparse_indices=sparse_indices[:num_decode_tokens],
compressed_kv_cache=compressed_kv_cache,
sparse_topk_lens=sparse_topk_lens[:num_decode_tokens],
seq_lens=seq_lens[:num_decodes],
out=output[:num_decode_tokens],
sparse_topk_lens=decode_topk_lens,
seq_lens=decode_seq_lens,
out=decode_output,
bmm1_scale=bmm1_scale,
bmm2_scale=bmm2_scale,
sinks=self.attn_sink,
cum_seq_lens_q=decode_cu,
max_q_len=swa_metadata.max_decode_query_len,
max_q_len=max_decode_query_len,
)
if num_prefill_tokens > 0:
@@ -205,6 +205,8 @@ class DeepseekSparseSWAMetadata:
num_decode_tokens: int = 0
num_prefill_tokens: int = 0
max_decode_query_len: int = 1
flashinfer_decode_topk_lens: torch.Tensor | None = None
flashinfer_decode_seq_lens: torch.Tensor | None = None
# Pre-computed prefill metadata shared across all DeepseekV4 attention layers.
prefill_seq_lens: torch.Tensor | None = None