From 63a9a5010a6d1539c52957646ef9d6bbcf7a4deb Mon Sep 17 00:00:00 2001 From: Zhuobin Huang Date: Fri, 14 Aug 2026 18:09:51 +0800 Subject: [PATCH] [Attention][DSA] Take the native decode path for MTP=3 on SM90 (#52164) Signed-off-by: zobinHuang Co-authored-by: nodeeeeee --- .../attention/test_deepgemm_attention.py | 181 +++++++++--------- .../attention/test_indexer_native_next_n.py | 75 ++++++++ vllm/utils/deep_gemm.py | 36 +++- vllm/v1/attention/backends/mla/indexer.py | 41 ++-- 4 files changed, 229 insertions(+), 104 deletions(-) create mode 100644 tests/v1/attention/test_indexer_native_next_n.py diff --git a/tests/kernels/attention/test_deepgemm_attention.py b/tests/kernels/attention/test_deepgemm_attention.py index 0cea46d628..5d11063c7a 100644 --- a/tests/kernels/attention/test_deepgemm_attention.py +++ b/tests/kernels/attention/test_deepgemm_attention.py @@ -13,6 +13,7 @@ from vllm.utils.deep_gemm import ( fp8_fp4_paged_mqa_logits, get_num_sms, get_paged_mqa_logits_metadata, + native_next_n_supported, ) from vllm.utils.import_utils import has_deep_gemm from vllm.utils.math_utils import cdiv @@ -205,7 +206,12 @@ def _ref_fp8_fp4_paged_mqa_logits( @pytest.mark.skipif( not current_platform.has_device_capability(90), reason="SM90 and SM100 only" ) -def test_deepgemm_fp8_fp4_paged_mqa_logits(): +# next_n = 1 + num_speculative_tokens, so next_n=4 is MTP=3 (issue #35878). +@pytest.mark.parametrize("batch_size,next_n", [(4, 1), (2, 2), (2, 4)]) +def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int): + if not native_next_n_supported(next_n): + pytest.skip(f"next_n={next_n} has no native kernel on this architecture") + # NOTE: clean_logits=True is incompatible with the 2D context_lens # required by csrc/apis/attention.hpp; only the False path is exercised. clean_logits = False @@ -213,98 +219,97 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(): random.seed(0) max_model_len = 4096 - for batch_size, next_n in [(4, 1), (2, 2)]: - for heads, index_dim in [(32, 128)]: - for avg_kv in (2048,): - num_blocks, blocksize = max_model_len * 2, 64 + for heads, index_dim in [(32, 128)]: + for avg_kv in (2048,): + num_blocks, blocksize = max_model_len * 2, 64 - q = torch.randn( - (batch_size, next_n, heads, index_dim), - device="cuda", - dtype=torch.bfloat16, - ) - kv_cache = torch.randn( - (num_blocks, blocksize, 1, index_dim), - device="cuda", - dtype=torch.bfloat16, - ) - weights = torch.randn( - (batch_size * next_n, heads), - device="cuda", - dtype=torch.float32, - ) + q = torch.randn( + (batch_size, next_n, heads, index_dim), + device="cuda", + dtype=torch.bfloat16, + ) + kv_cache = torch.randn( + (num_blocks, blocksize, 1, index_dim), + device="cuda", + dtype=torch.bfloat16, + ) + weights = torch.randn( + (batch_size * next_n, heads), + device="cuda", + dtype=torch.float32, + ) - context_lens = ( - torch.randint(int(0.8 * avg_kv), int(1.2 * avg_kv), (batch_size,)) - .cuda() - .to(torch.int32) - ) - max_block_len = ( - (context_lens.max().item() + blocksize - 1) // blocksize * blocksize - ) - block_tables = torch.zeros( - (batch_size, max_block_len), - device="cuda", - dtype=torch.int32, - ) + context_lens = ( + torch.randint(int(0.8 * avg_kv), int(1.2 * avg_kv), (batch_size,)) + .cuda() + .to(torch.int32) + ) + max_block_len = ( + (context_lens.max().item() + blocksize - 1) // blocksize * blocksize + ) + block_tables = torch.zeros( + (batch_size, max_block_len), + device="cuda", + dtype=torch.int32, + ) - counter = 0 - block_idx_pool = list(range(num_blocks)) - random.shuffle(block_idx_pool) - for i in range(batch_size): - ctx_len = int(context_lens[i].item()) - for j in range((ctx_len + blocksize - 1) // blocksize): - block_tables[i][j] = block_idx_pool[counter] - counter += 1 + counter = 0 + block_idx_pool = list(range(num_blocks)) + random.shuffle(block_idx_pool) + for i in range(batch_size): + ctx_len = int(context_lens[i].item()) + for j in range((ctx_len + blocksize - 1) // blocksize): + block_tables[i][j] = block_idx_pool[counter] + counter += 1 - q_fp8 = q.to(torch.float8_e4m3fn) - kv_cache_fp8 = kv_cache_cast_to_fp8(kv_cache) + q_fp8 = q.to(torch.float8_e4m3fn) + kv_cache_fp8 = kv_cache_cast_to_fp8(kv_cache) - # deep_gemm paged MQA logits requires 2D context_lens of - # shape (B, next_n) (csrc/apis/attention.hpp:332-335); - # see indexer.py:607-608. For each batch/next_n token, the - # effective context length is context_lens[b] - next_n + j + 1. - next_n_arange = torch.arange(next_n, device="cuda", dtype=torch.int32) - context_lens_2d = ( - context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange - ).contiguous() - schedule_metadata = get_paged_mqa_logits_metadata( - context_lens_2d, blocksize, get_num_sms() - ) - logits = fp8_fp4_paged_mqa_logits( - (q_fp8, None), - kv_cache_fp8, - weights, - context_lens_2d, - block_tables, - schedule_metadata, - max_model_len, - clean_logits=clean_logits, - ) + # deep_gemm paged MQA logits requires 2D context_lens of + # shape (B, next_n) (csrc/apis/attention.hpp:332-335); + # see indexer.py:607-608. For each batch/next_n token, the + # effective context length is context_lens[b] - next_n + j + 1. + next_n_arange = torch.arange(next_n, device="cuda", dtype=torch.int32) + context_lens_2d = ( + context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange + ).contiguous() + schedule_metadata = get_paged_mqa_logits_metadata( + context_lens_2d, + blocksize, + get_num_sms(), + ) + logits = fp8_fp4_paged_mqa_logits( + (q_fp8, None), + kv_cache_fp8, + weights, + context_lens_2d, + block_tables, + schedule_metadata, + max_model_len, + clean_logits=clean_logits, + ) - ref_logits = _ref_fp8_fp4_paged_mqa_logits( - q, - kv_cache, - weights, - context_lens, - block_tables, - max_model_len, - ) + ref_logits = _ref_fp8_fp4_paged_mqa_logits( + q, + kv_cache, + weights, + context_lens, + block_tables, + max_model_len, + ) - positions = ( - torch.arange(max_model_len, device="cuda") - .unsqueeze(0) - .expand(batch_size * next_n, -1) - ) - row_indices = torch.arange(batch_size * next_n, device="cuda") // next_n - next_n_offset = ( - torch.arange(batch_size * next_n, device="cuda") % next_n - ) - mask = positions <= ( - context_lens[row_indices] - next_n + next_n_offset - ).unsqueeze(1) + positions = ( + torch.arange(max_model_len, device="cuda") + .unsqueeze(0) + .expand(batch_size * next_n, -1) + ) + row_indices = torch.arange(batch_size * next_n, device="cuda") // next_n + next_n_offset = torch.arange(batch_size * next_n, device="cuda") % next_n + mask = positions <= ( + context_lens[row_indices] - next_n + next_n_offset + ).unsqueeze(1) - logits = logits.masked_fill(~mask, 0) - ref_logits = ref_logits.masked_fill(~mask, 0) - diff = calc_diff(logits, ref_logits) - assert diff < 1e-3, f"{diff=}" + logits = logits.masked_fill(~mask, 0) + ref_logits = ref_logits.masked_fill(~mask, 0) + diff = calc_diff(logits, ref_logits) + assert diff < 1e-3, f"{diff=}" diff --git a/tests/v1/attention/test_indexer_native_next_n.py b/tests/v1/attention/test_indexer_native_next_n.py new file mode 100644 index 0000000000..3ec14fe62e --- /dev/null +++ b/tests/v1/attention/test_indexer_native_next_n.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Which next_n the DSA indexer decode path may hand to DeepGEMM unflattened. + +Getting this wrong is not a slow path but a crash: `fp8_fp4_paged_mqa_logits` +asserts both that the architecture implements the requested `next_n` and that +the schedule metadata was sized for the matching slot count. +""" + +import pytest + +from vllm.platforms import current_platform +from vllm.utils.deep_gemm import _paged_mqa_logits_schedule_slots +from vllm.v1.attention.backends.mla import indexer + +NUM_SMS = 114 # H100 PCIe + + +def _set_arch(monkeypatch, family: int, *, cuda: bool = True, deep_gemm: bool = True): + monkeypatch.setattr(current_platform, "is_cuda", lambda: cuda) + monkeypatch.setattr( + current_platform, + "is_device_capability_family", + lambda capability, device_id=0: capability // 10 == family, + ) + monkeypatch.setattr(indexer, "has_deep_gemm", lambda: deep_gemm) + + +@pytest.mark.parametrize( + "family,expected_native", + [ + # SM90 gained next_n=4 (MTP=3) via 2-CTA multicast, but never 3. + (9, {1, 2, 4}), + # SM100 schedules any next_n with multi-atom tiles. + (10, {1, 2, 3, 4, 5, 8}), + # SM120 advertises multi-atom too but is unvalidated on hardware, so + # it stays on the conservative gate. Loosen it only with measurements. + (12, {1, 2}), + ], +) +def test_native_decode_gate_per_architecture(monkeypatch, family, expected_native): + _set_arch(monkeypatch, family) + for next_n in (1, 2, 3, 4, 5, 8): + assert indexer._supports_native_decode(next_n) == (next_n in expected_native), ( + f"family={family} next_n={next_n}" + ) + + +@pytest.mark.parametrize( + "cuda,deep_gemm", [(False, True), (True, False), (False, False)] +) +def test_native_decode_gate_without_deepgemm(monkeypatch, cuda, deep_gemm): + """Without the DeepGEMM kernels only the shapes every backend handles.""" + _set_arch(monkeypatch, 9, cuda=cuda, deep_gemm=deep_gemm) + assert [indexer._supports_native_decode(n) for n in (1, 2, 3, 4)] == [ + True, + True, + False, + False, + ] + + +def test_sm90_next_n_4_halves_the_schedule_slots(monkeypatch): + """SM90 next_n=4 runs one scheduler task per 2-CTA cluster, not per SM.""" + _set_arch(monkeypatch, 9) + assert _paged_mqa_logits_schedule_slots(NUM_SMS, 4) == NUM_SMS // 2 + for next_n in (1, 2, 3): + assert _paged_mqa_logits_schedule_slots(NUM_SMS, next_n) == NUM_SMS + + +@pytest.mark.parametrize("family", [10, 12]) +def test_multicast_is_sm90_only(monkeypatch, family): + _set_arch(monkeypatch, family) + for next_n in (1, 2, 3, 4): + assert _paged_mqa_logits_schedule_slots(NUM_SMS, next_n) == NUM_SMS diff --git a/vllm/utils/deep_gemm.py b/vllm/utils/deep_gemm.py index 29adb2a6f3..4b78142c6f 100644 --- a/vllm/utils/deep_gemm.py +++ b/vllm/utils/deep_gemm.py @@ -552,6 +552,29 @@ def fp8_fp4_mqa_logits( ) +def native_next_n_supported(next_n: int) -> bool: + """Whether the paged MQA logits kernel takes `next_n` Q rows per request. + + SM90 implements only {1, 2, 4}; SM100 and SM120 schedule any `next_n` via + multi-atom tiles. Unsupported values must be flattened to one row per query. + """ + if current_platform.is_device_capability_family(90): + return next_n in (1, 2, 4) + return True + + +def _paged_mqa_logits_schedule_slots(num_sms: int, next_n: int) -> int: + """Scheduler tasks the paged MQA logits kernel launches. + + SM90 `next_n=4` runs one task per 2-CTA multicast cluster rather than per + SM, and `fp8_fp4_paged_mqa_logits` asserts its metadata is sized to match. + """ + num_kv_multicast = ( + 2 if next_n == 4 and current_platform.is_device_capability_family(90) else 1 + ) + return num_sms // num_kv_multicast + + def get_paged_mqa_logits_metadata( context_lens: torch.Tensor, block_size: int, @@ -561,22 +584,24 @@ def get_paged_mqa_logits_metadata( """Build scheduling metadata for paged MQA logits. Args: - context_lens: Tensor of shape [B], dtype int32; effective context length - per batch element. + context_lens: Tensor of shape [B, next_n], dtype int32; effective + context length per Q row. block_size: KV-cache block size in tokens (e.g., 64). num_sms: Number of SMs available. 132 for Hopper indices: Optional request index for each varlen row. Returns: - Backend-specific tensor consumed by `fp8_fp4_paged_mqa_logits` to - schedule work across SMs. + Tensor of shape [slots + 1, 2] consumed by `fp8_fp4_paged_mqa_logits` + to schedule work across SMs. """ _lazy_init() if _get_paged_mqa_logits_metadata_impl is None: return _missing() + next_n = context_lens.shape[1] if context_lens.dim() == 2 else 1 + num_slots = _paged_mqa_logits_schedule_slots(num_sms, next_n) kwargs = {} if indices is None else {"indices": indices} return _get_paged_mqa_logits_metadata_impl( - context_lens, block_size, num_sms, **kwargs + context_lens, block_size, num_slots, **kwargs ) @@ -752,6 +777,7 @@ __all__ = [ "fp8_fp4_mqa_logits", "fp8_fp4_paged_mqa_logits", "get_paged_mqa_logits_metadata", + "native_next_n_supported", "per_block_cast_to_fp8", "is_deep_gemm_e8m0_used", "is_deep_gemm_supported", diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 30951ca4c1..2b3e400de3 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -21,6 +21,7 @@ from vllm.triton_utils import tl, triton from vllm.utils.deep_gemm import ( get_paged_mqa_logits_metadata, has_deep_gemm, + native_next_n_supported, ) from vllm.utils.platform_utils import num_compute_units from vllm.v1.attention.backend import ( @@ -467,6 +468,20 @@ def _supports_varlen_paged_mqa_logits() -> bool: ) +def _supports_native_decode(next_n: int) -> bool: + """Whether decode can pass `next_n` Q rows per request to the kernel + instead of flattening to one single-token row per query, which re-reads + the KV tile once per row. + """ + if not (current_platform.is_cuda() and has_deep_gemm()): + return next_n in (1, 2) + if current_platform.is_device_capability_family(100): + return True + if current_platform.is_device_capability_family(90): + return native_next_n_supported(next_n) + return next_n in (1, 2) + + class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): # The indexer opts out of the shared reorder-threshold vote (see __init__), # so this is None; its own split uses self.decode_threshold. @@ -524,14 +539,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): next_n = self.num_speculative_tokens + 1 self.decode_threshold = next_n self.reorder_batch_threshold = None - # NOTE: SM100 datacenter GPUs support any next_n natively via the - # multi-atom paged MQA logits kernels (FP8 and FP4 indexer - # caches). Outside the SM100 family the FP8 - # paged MQA logits kernel only supports next_n in (1, 2) - # (deepgemm smxx_fp8_fp4_paged_mqa_logits.hpp:233), so flatten there. - self.use_flattening = not current_platform.is_device_capability_family( - 100 - ) and next_n not in (1, 2) + self.use_flattening = not _supports_native_decode(next_n) self.supports_varlen = _supports_varlen_paged_mqa_logits() logger.info_once( "DSA indexer decode path: use_flattening=%s supports_varlen=%s " @@ -585,7 +593,8 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): device=self.device, ) - # See: DeepGMM/csrc/apis/attention.hpp + # See: DeepGMM/csrc/apis/attention.hpp. Sized for one slot per SM; + # build() narrows it to whatever the kernel actually schedules. self.scheduler_metadata_buffer = torch.empty( (self.num_sms + 1, 2), dtype=torch.int32, device=self.device ) @@ -924,9 +933,16 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): max_decode_len = int(decode_lens_cpu.max().item()) next_n = 1 + self.num_speculative_tokens + # The kernel sees max_decode_len Q rows, not the configured next_n, + # so legality is per-step: on SM90 a uniformly 3-deep batch has no + # native kernel. max_decode_len <= 1 always has one. + step_next_n_ok = max_decode_len <= 1 or _supports_native_decode( + max_decode_len + ) use_native = ( not (self.use_flattening or self.supports_varlen) and max_decode_len <= next_n + and step_next_n_ok ) global_seq_lens_for_decode = self._prepare_global_decode_seq_lens( @@ -992,20 +1008,23 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): seq_lens = seq_lens.unsqueeze(-1) # DeepGEMM is required for the paged MQA logits on CUDA devices + schedule_metadata = self.scheduler_metadata_buffer if current_platform.is_cuda() and has_deep_gemm(): - self.scheduler_metadata_buffer[:] = get_paged_mqa_logits_metadata( + metadata = get_paged_mqa_logits_metadata( seq_lens, self.kv_cache_spec.storage_block_size, self.num_sms, indices=decode_indices, ) + schedule_metadata = self.scheduler_metadata_buffer[: metadata.shape[0]] + schedule_metadata[:] = metadata decode_metadata = DeepSeekV32IndexerDecodeMetadata( block_table=block_table, seq_lens=seq_lens, decode_lens=decode_lens, requires_padding=requires_padding, - schedule_metadata=self.scheduler_metadata_buffer, + schedule_metadata=schedule_metadata, indices=decode_indices, global_seq_lens=global_seq_lens_for_decode, )