[Attention][DSA] support dcp for FLASHINFER_MLA_SPARSE (#46076)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Jingyi Yang <girasoleyang@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: GirasoleY <girasoleyang@gmail.com>
Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jiangyun Zhu
2026-07-01 00:32:20 -04:00
committed by GitHub
co-authored by Jingyi Yang Lucas Wilkinson Claude Opus 4.8
parent 3c1396bab6
commit c5200d3565
12 changed files with 1995 additions and 97 deletions
+167 -44
View File
@@ -8,6 +8,8 @@ import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config
from vllm.distributed import get_dcp_group
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
@@ -21,6 +23,7 @@ from vllm.utils.deep_gemm import (
fp8_fp4_paged_mqa_logits,
has_deep_gemm,
)
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
@@ -41,6 +44,85 @@ RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
MXFP4_BLOCK_SIZE = 32
def _assert_cutedsl_dcp_merge_supported(
logits: torch.Tensor,
topk_indices: torch.Tensor,
k: int,
) -> None:
# The DCP merge only supports the CuteDSL path (Triton pack kernel + CuteDSL
# stable-topk selector); there is no PyTorch fallback. The first cut targets
# Blackwell/Hopper with index_topk in (512, 1024, 2048) (the selector's radix
# sizing); the Triton pack itself has no shape/topk constraints.
if not has_cutedsl():
raise RuntimeError(
"DCP sparse-indexer merge requires CuteDSL; install it or disable DCP."
)
if logits.device.type != "cuda":
raise RuntimeError("DCP sparse-indexer merge requires CUDA tensors.")
if logits.dtype != torch.float32 or topk_indices.dtype != torch.int32:
raise RuntimeError(
"DCP sparse-indexer merge requires fp32 logits and int32 indices."
)
if k not in (512, 1024, 2048):
raise RuntimeError(
f"DCP sparse-indexer merge requires index_topk in (512, 1024, 2048); "
f"got {k}."
)
def _merge_dcp_topk_global(
logits: torch.Tensor,
topk_indices: torch.Tensor,
topk_tokens: int,
dcp_rank: int,
dcp_world_size: int,
cp_interleave: int,
row_starts: torch.Tensor | None = None,
) -> None:
"""Merge each DCP rank's local top-K into the global top-K.
``topk_indices`` are this rank's local top-K positions into its 1/N KV
shard. A token in the global top-K must also be in its owning rank's local
top-K (at most ``topk_tokens - 1`` tokens rank globally above it, hence at
most that many on its own rank), so exchanging only the per-rank local
candidates is exact -- equivalent to all-gathering the full logit matrix,
but it ships ``dcp_world_size * topk_tokens`` candidates instead of the whole
score row. Overwrites ``topk_indices`` with global token ids (``-1`` for
padding); the attention backend localizes them back to physical slots per
rank.
"""
if dcp_world_size <= 1:
return
# CuteDSL-only path (no PyTorch fallback): Triton-pack each rank's
# (score, global_id) candidates on-device, all-gather, then the CuteDSL
# stable-topk selector.
_assert_cutedsl_dcp_merge_supported(logits, topk_indices, topk_tokens)
from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import (
pack_dcp_topk_candidates_cutedsl,
stable_topk_from_gathered_candidates_cutedsl,
)
packed = torch.empty(
(*topk_indices.shape, 2),
dtype=torch.float32,
device=topk_indices.device,
)
pack_dcp_topk_candidates_cutedsl(
logits,
topk_indices,
packed,
dcp_rank,
dcp_world_size,
cp_interleave,
row_starts,
)
gathered = get_dcp_group().all_gather(packed, dim=1)
stable_topk_from_gathered_candidates_cutedsl(
gathered, topk_tokens, out=topk_indices
)
@triton.jit
def _fused_indexer_q_rope_quant_kernel(
positions,
@@ -227,6 +309,9 @@ def sparse_attn_indexer(
topk_indices_buffer: torch.Tensor,
skip_k_cache_insert: bool,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
dcp_world_size: int = 1,
cp_kv_cache_interleave_size: int = 1,
skip_topk_buffer_clear: bool = False,
) -> torch.Tensor:
# careful! this will be None in dummy run
@@ -328,16 +413,18 @@ def sparse_attn_indexer(
scales_spec,
)
for chunk in prefill_metadata.chunks:
k_quant = k_quant_full[: chunk.total_seq_lens]
k_scale = k_scale_full[: chunk.total_seq_lens]
if not chunk.skip_kv_gather:
cu_seqlen_ks = chunk.cu_seqlen_ks
cu_seqlen_ke = chunk.cu_seqlen_ke
assert chunk.local_cu_seq_lens is not None
k_quant = k_quant_full[: chunk.max_local_total_seq_lens]
k_scale = k_scale_full[: chunk.max_local_total_seq_lens]
if not chunk.skip_kv_gather and chunk.local_total_seq_lens > 0:
ops.cp_gather_indexer_k_quant_cache(
kv_cache,
k_quant,
k_scale,
chunk.block_table,
chunk.cu_seq_lens,
chunk.local_cu_seq_lens,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
@@ -346,51 +433,64 @@ def sparse_attn_indexer(
if q_scale is not None
else None
)
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
if use_fp4_cache:
q_slice_cast = q_slice.view(torch.int8)
k_quant_cast = k_quant.view(torch.int8)
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
else:
q_slice_cast = q_slice
k_quant_cast = k_quant
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
if current_platform.is_xpu():
if q_scale_slice is not None:
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
logits = torch.ops.vllm.xpu_fp8_mqa_logits(
q_slice_cast,
k_quant_cast,
k_scale_cast,
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
)
else:
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
clean_logits=False,
)
num_rows = logits.shape[0]
topk_indices = topk_indices_buffer[
chunk.token_start : chunk.token_end, :topk_tokens
]
ops.top_k_per_row_prefill(
if chunk.local_total_seq_lens == 0:
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
topk_indices.fill_(-1)
else:
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
if use_fp4_cache:
q_slice_cast = q_slice.view(torch.int8)
k_quant_cast = k_quant.view(torch.int8)
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
else:
q_slice_cast = q_slice
k_quant_cast = k_quant
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
if current_platform.is_xpu():
if q_scale_slice is not None:
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
logits = torch.ops.vllm.xpu_fp8_mqa_logits(
q_slice_cast,
k_quant_cast,
k_scale_cast,
weights[chunk.token_start : chunk.token_end],
cu_seqlen_ks,
cu_seqlen_ke,
)
else:
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
cu_seqlen_ks,
cu_seqlen_ke,
clean_logits=False,
)
num_rows = logits.shape[0]
ops.top_k_per_row_prefill(
logits,
cu_seqlen_ks,
cu_seqlen_ke,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
)
_merge_dcp_topk_global(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
row_starts=chunk.cu_seqlen_ks,
)
if has_decode:
@@ -508,7 +608,7 @@ def sparse_attn_indexer(
topk_indices,
topk_workspace,
topk_tokens,
attn_metadata_narrowed.max_seq_len,
logits.shape[1],
)
else:
ops.top_k_per_row_decode(
@@ -522,6 +622,16 @@ def sparse_attn_indexer(
topk_tokens,
)
if decode_metadata.global_seq_lens is not None:
_merge_dcp_topk_global(
logits,
topk_indices,
topk_tokens,
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
)
if decode_metadata.requires_padding:
# if padded, we need to unpack
# the topk indices removing padded tokens
@@ -553,6 +663,9 @@ def sparse_attn_indexer_fake(
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
dcp_world_size: int = 1,
cp_kv_cache_interleave_size: int = 1,
skip_topk_buffer_clear: bool = False,
) -> torch.Tensor:
return topk_indices_buffer
@@ -604,6 +717,13 @@ class SparseAttnIndexer(CustomOp):
self.topk_indices_buffer = topk_indices_buffer
self.skip_k_cache_insert = skip_k_cache_insert
self.use_fp4_cache = use_fp4_cache
# DCP scalars are constant for the run; resolve them here (config is set
# during model construction) and pass them into the custom op, rather
# than threading them through per-step metadata.
parallel_config = get_current_vllm_config().parallel_config
self.dcp_world_size = parallel_config.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
if current_platform.is_cuda() and not has_deep_gemm():
raise RuntimeError(
"Sparse Attention Indexer CUDA op requires DeepGEMM support in "
@@ -657,6 +777,9 @@ class SparseAttnIndexer(CustomOp):
self.topk_indices_buffer,
self.skip_k_cache_insert,
self.use_fp4_cache,
self.dcp_rank,
self.dcp_world_size,
self.cp_kv_cache_interleave_size,
)
def forward_xpu(