[Attention][DSA] Take the native decode path for MTP=3 on SM90 (#52164)
Signed-off-by: zobinHuang <zobin1999@gmail.com> Co-authored-by: nodeeeeee <zhangkai.nodeee@gmail.com>
This commit is contained in:
co-authored by
nodeeeeee
parent
aa31003573
commit
63a9a5010a
+31
-5
@@ -552,6 +552,29 @@ def fp8_fp4_mqa_logits(
|
||||
)
|
||||
|
||||
|
||||
def native_next_n_supported(next_n: int) -> bool:
|
||||
"""Whether the paged MQA logits kernel takes `next_n` Q rows per request.
|
||||
|
||||
SM90 implements only {1, 2, 4}; SM100 and SM120 schedule any `next_n` via
|
||||
multi-atom tiles. Unsupported values must be flattened to one row per query.
|
||||
"""
|
||||
if current_platform.is_device_capability_family(90):
|
||||
return next_n in (1, 2, 4)
|
||||
return True
|
||||
|
||||
|
||||
def _paged_mqa_logits_schedule_slots(num_sms: int, next_n: int) -> int:
|
||||
"""Scheduler tasks the paged MQA logits kernel launches.
|
||||
|
||||
SM90 `next_n=4` runs one task per 2-CTA multicast cluster rather than per
|
||||
SM, and `fp8_fp4_paged_mqa_logits` asserts its metadata is sized to match.
|
||||
"""
|
||||
num_kv_multicast = (
|
||||
2 if next_n == 4 and current_platform.is_device_capability_family(90) else 1
|
||||
)
|
||||
return num_sms // num_kv_multicast
|
||||
|
||||
|
||||
def get_paged_mqa_logits_metadata(
|
||||
context_lens: torch.Tensor,
|
||||
block_size: int,
|
||||
@@ -561,22 +584,24 @@ def get_paged_mqa_logits_metadata(
|
||||
"""Build scheduling metadata for paged MQA logits.
|
||||
|
||||
Args:
|
||||
context_lens: Tensor of shape [B], dtype int32; effective context length
|
||||
per batch element.
|
||||
context_lens: Tensor of shape [B, next_n], dtype int32; effective
|
||||
context length per Q row.
|
||||
block_size: KV-cache block size in tokens (e.g., 64).
|
||||
num_sms: Number of SMs available. 132 for Hopper
|
||||
indices: Optional request index for each varlen row.
|
||||
|
||||
Returns:
|
||||
Backend-specific tensor consumed by `fp8_fp4_paged_mqa_logits` to
|
||||
schedule work across SMs.
|
||||
Tensor of shape [slots + 1, 2] consumed by `fp8_fp4_paged_mqa_logits`
|
||||
to schedule work across SMs.
|
||||
"""
|
||||
_lazy_init()
|
||||
if _get_paged_mqa_logits_metadata_impl is None:
|
||||
return _missing()
|
||||
next_n = context_lens.shape[1] if context_lens.dim() == 2 else 1
|
||||
num_slots = _paged_mqa_logits_schedule_slots(num_sms, next_n)
|
||||
kwargs = {} if indices is None else {"indices": indices}
|
||||
return _get_paged_mqa_logits_metadata_impl(
|
||||
context_lens, block_size, num_sms, **kwargs
|
||||
context_lens, block_size, num_slots, **kwargs
|
||||
)
|
||||
|
||||
|
||||
@@ -752,6 +777,7 @@ __all__ = [
|
||||
"fp8_fp4_mqa_logits",
|
||||
"fp8_fp4_paged_mqa_logits",
|
||||
"get_paged_mqa_logits_metadata",
|
||||
"native_next_n_supported",
|
||||
"per_block_cast_to_fp8",
|
||||
"is_deep_gemm_e8m0_used",
|
||||
"is_deep_gemm_supported",
|
||||
|
||||
Reference in New Issue
Block a user