[Attention][DSA] Take the native decode path for MTP=3 on SM90 (#52164)

Signed-off-by: zobinHuang <zobin1999@gmail.com>
Co-authored-by: nodeeeeee <zhangkai.nodeee@gmail.com>
This commit is contained in:
Zhuobin Huang
2026-08-14 10:09:51 +00:00
committed by GitHub
co-authored by nodeeeeee
parent aa31003573
commit 63a9a5010a
4 changed files with 229 additions and 104 deletions
+31 -5
View File
@@ -552,6 +552,29 @@ def fp8_fp4_mqa_logits(
)
def native_next_n_supported(next_n: int) -> bool:
"""Whether the paged MQA logits kernel takes `next_n` Q rows per request.
SM90 implements only {1, 2, 4}; SM100 and SM120 schedule any `next_n` via
multi-atom tiles. Unsupported values must be flattened to one row per query.
"""
if current_platform.is_device_capability_family(90):
return next_n in (1, 2, 4)
return True
def _paged_mqa_logits_schedule_slots(num_sms: int, next_n: int) -> int:
"""Scheduler tasks the paged MQA logits kernel launches.
SM90 `next_n=4` runs one task per 2-CTA multicast cluster rather than per
SM, and `fp8_fp4_paged_mqa_logits` asserts its metadata is sized to match.
"""
num_kv_multicast = (
2 if next_n == 4 and current_platform.is_device_capability_family(90) else 1
)
return num_sms // num_kv_multicast
def get_paged_mqa_logits_metadata(
context_lens: torch.Tensor,
block_size: int,
@@ -561,22 +584,24 @@ def get_paged_mqa_logits_metadata(
"""Build scheduling metadata for paged MQA logits.
Args:
context_lens: Tensor of shape [B], dtype int32; effective context length
per batch element.
context_lens: Tensor of shape [B, next_n], dtype int32; effective
context length per Q row.
block_size: KV-cache block size in tokens (e.g., 64).
num_sms: Number of SMs available. 132 for Hopper
indices: Optional request index for each varlen row.
Returns:
Backend-specific tensor consumed by `fp8_fp4_paged_mqa_logits` to
schedule work across SMs.
Tensor of shape [slots + 1, 2] consumed by `fp8_fp4_paged_mqa_logits`
to schedule work across SMs.
"""
_lazy_init()
if _get_paged_mqa_logits_metadata_impl is None:
return _missing()
next_n = context_lens.shape[1] if context_lens.dim() == 2 else 1
num_slots = _paged_mqa_logits_schedule_slots(num_sms, next_n)
kwargs = {} if indices is None else {"indices": indices}
return _get_paged_mqa_logits_metadata_impl(
context_lens, block_size, num_sms, **kwargs
context_lens, block_size, num_slots, **kwargs
)
@@ -752,6 +777,7 @@ __all__ = [
"fp8_fp4_mqa_logits",
"fp8_fp4_paged_mqa_logits",
"get_paged_mqa_logits_metadata",
"native_next_n_supported",
"per_block_cast_to_fp8",
"is_deep_gemm_e8m0_used",
"is_deep_gemm_supported",
+30 -11
View File
@@ -21,6 +21,7 @@ from vllm.triton_utils import tl, triton
from vllm.utils.deep_gemm import (
get_paged_mqa_logits_metadata,
has_deep_gemm,
native_next_n_supported,
)
from vllm.utils.platform_utils import num_compute_units
from vllm.v1.attention.backend import (
@@ -467,6 +468,20 @@ def _supports_varlen_paged_mqa_logits() -> bool:
)
def _supports_native_decode(next_n: int) -> bool:
"""Whether decode can pass `next_n` Q rows per request to the kernel
instead of flattening to one single-token row per query, which re-reads
the KV tile once per row.
"""
if not (current_platform.is_cuda() and has_deep_gemm()):
return next_n in (1, 2)
if current_platform.is_device_capability_family(100):
return True
if current_platform.is_device_capability_family(90):
return native_next_n_supported(next_n)
return next_n in (1, 2)
class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
# The indexer opts out of the shared reorder-threshold vote (see __init__),
# so this is None; its own split uses self.decode_threshold.
@@ -524,14 +539,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
next_n = self.num_speculative_tokens + 1
self.decode_threshold = next_n
self.reorder_batch_threshold = None
# NOTE: SM100 datacenter GPUs support any next_n natively via the
# multi-atom paged MQA logits kernels (FP8 and FP4 indexer
# caches). Outside the SM100 family the FP8
# paged MQA logits kernel only supports next_n in (1, 2)
# (deepgemm smxx_fp8_fp4_paged_mqa_logits.hpp:233), so flatten there.
self.use_flattening = not current_platform.is_device_capability_family(
100
) and next_n not in (1, 2)
self.use_flattening = not _supports_native_decode(next_n)
self.supports_varlen = _supports_varlen_paged_mqa_logits()
logger.info_once(
"DSA indexer decode path: use_flattening=%s supports_varlen=%s "
@@ -585,7 +593,8 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
device=self.device,
)
# See: DeepGMM/csrc/apis/attention.hpp
# See: DeepGMM/csrc/apis/attention.hpp. Sized for one slot per SM;
# build() narrows it to whatever the kernel actually schedules.
self.scheduler_metadata_buffer = torch.empty(
(self.num_sms + 1, 2), dtype=torch.int32, device=self.device
)
@@ -924,9 +933,16 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
max_decode_len = int(decode_lens_cpu.max().item())
next_n = 1 + self.num_speculative_tokens
# The kernel sees max_decode_len Q rows, not the configured next_n,
# so legality is per-step: on SM90 a uniformly 3-deep batch has no
# native kernel. max_decode_len <= 1 always has one.
step_next_n_ok = max_decode_len <= 1 or _supports_native_decode(
max_decode_len
)
use_native = (
not (self.use_flattening or self.supports_varlen)
and max_decode_len <= next_n
and step_next_n_ok
)
global_seq_lens_for_decode = self._prepare_global_decode_seq_lens(
@@ -992,20 +1008,23 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
seq_lens = seq_lens.unsqueeze(-1)
# DeepGEMM is required for the paged MQA logits on CUDA devices
schedule_metadata = self.scheduler_metadata_buffer
if current_platform.is_cuda() and has_deep_gemm():
self.scheduler_metadata_buffer[:] = get_paged_mqa_logits_metadata(
metadata = get_paged_mqa_logits_metadata(
seq_lens,
self.kv_cache_spec.storage_block_size,
self.num_sms,
indices=decode_indices,
)
schedule_metadata = self.scheduler_metadata_buffer[: metadata.shape[0]]
schedule_metadata[:] = metadata
decode_metadata = DeepSeekV32IndexerDecodeMetadata(
block_table=block_table,
seq_lens=seq_lens,
decode_lens=decode_lens,
requires_padding=requires_padding,
schedule_metadata=self.scheduler_metadata_buffer,
schedule_metadata=schedule_metadata,
indices=decode_indices,
global_seq_lens=global_seq_lens_for_decode,
)