diff --git a/tests/kernels/attention/test_rocm_aiter_mla_causal_verify_mask.py b/tests/kernels/attention/test_rocm_aiter_mla_causal_verify_mask.py index 5755580142..dab49c33fa 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_causal_verify_mask.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_causal_verify_mask.py @@ -1,21 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Regression test for causal masking in the AITER MLA verify flatten. +"""Regression test for causal masking in the AITER MLA verify MTP path. When ``num_heads < 16`` the ROCm AITER MLA backend cannot use the ASM decode -kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` flattens a -``1 + num_speculative_tokens`` verify block into that many single-token Gluon -decodes. Every resulting row used to be handed its request's entire paged-KV -range. ``paged_kv_indptr`` is the cumulative sum of ``seq_lens``, which counts -the tokens scheduled in the current step, so that range already spans the -verify block: each verify position could attend to the draft tokens after it, -which are the tokens it is supposed to be checking. Nothing crashes and nothing -warns, so the only way to catch it is to look at the KV window per row. +kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` routes verify +through ``mla_gluon``'s native 4-D MTP entry (``q`` shaped +``[batch, qlen, nhead, dim]`` with ``use_2d_view=False``). aiter applies the +causal tail internally: position ``t`` attends KV ``[0, seq_len - qlen + t]``. -The test drives the real builder and the real ``forward_mqa`` over a -multi-token decode batch, intercepts the ``(page_table, seq_info)`` actually -handed to the Gluon kernel, and pins verify row ``t`` of request ``r`` to the -``context_r + t + 1`` entries of that request's ascending slice. +The test drives the real builder and ``forward_mqa`` over a multi-token decode +batch, intercepts the arguments actually handed to the Gluon kernel, and pins +that the 4-D MTP contract is used with per-request ``block_table`` and +``cache_seqlens`` rather than a flattened per-token paged-KV view. """ import types @@ -32,15 +28,15 @@ def _on_rocm_with_aiter() -> bool: return current_platform.is_rocm() and is_aiter_found() -# The production verify flatten is gfx950-only. This test replaces the Gluon +# The production verify MTP path is gfx950-only. This test replaces the Gluon # kernel with a spy and forces the architecture feature probe so the metadata # regression remains testable on every ROCm AITER CI runner. pytestmark = pytest.mark.skipif( not _on_rocm_with_aiter(), - reason="ROCM_AITER_MLA verify flatten requires ROCm and AITER", + reason="ROCM_AITER_MLA verify MTP requires ROCm and AITER", ) -# Below the 16-head threshold that selects the flatten. DeepSeek-V3 / R1 at +# Below the 16-head threshold that selects the MTP path. DeepSeek-V3 / R1 at # TP=16 and Kimi-K3 at TP=8 both land here. NUM_QUERY_HEADS = 12 KV_LORA_RANK = 512 @@ -54,8 +50,7 @@ QLEN = 8 # Committed context per request, including a fresh request at zero context # where the whole KV range is the verify block itself. CONTEXT_LENS = [0, 1, 37, 512] -# A cudagraph padding request: seq_len 0, so every one of its rows must clamp -# to an empty window. +# A cudagraph padding request: seq_len 0. PADDING_ROWS = 1 # The Gluon path flattens the KV cache to one page per token. @@ -68,16 +63,10 @@ def _seq_lens() -> list[int]: return [c + QLEN for c in CONTEXT_LENS] + [0] * PADDING_ROWS -def _expected_row_lens() -> list[int]: - """Causal row lengths: row r*QLEN + t sees seq_len_r - (QLEN - 1) + t.""" - return [max(0, s - (QLEN - 1) + t) for s in _seq_lens() for t in range(QLEN)] - - def _run_verify_block(): """Drive the real builder + forward_mqa over one multi-token verify block. - Returns ``(metadata, captured)`` where ``captured`` holds the ``page_table``, - ``seq_info`` and ``min_kv_seq_len`` the backend passed to the Gluon kernel. + Returns ``(metadata, captured)`` where ``captured`` holds the Gluon kwargs. """ from tests.v1.attention.utils import ( BatchSpec, @@ -144,9 +133,10 @@ def _run_verify_block(): captured: dict = {} def spy(**kwargs): + captured.update(kwargs) + captured["q_nope"] = kwargs["q_nope"].detach().clone() captured["page_table"] = kwargs["page_table"].detach().clone() captured["seq_info"] = kwargs["seq_info"].detach().clone() - captured["min_kv_seq_len"] = kwargs["min_kv_seq_len"] with set_current_vllm_config(vllm_config): builder = builder_cls(spec, [layer_name], vllm_config, device) @@ -206,12 +196,12 @@ def _run_verify_block(): return metadata, captured -def test_verify_flatten_rows_are_causal(): - """Verify row t must see the committed prefix plus tokens 0..t, and no more. +def test_verify_mtp_uses_native_4d_gluon_entry(): + """Verify uses 4-D q with per-request paged_kv metadata. - Regression guard: before the fix every row of a request got that request's - whole paged-KV range, so verify position t attended to the draft tokens - after it. Fails on unmodified upstream. + Regression guard: before the fix every flattened row of a request got that + request's whole paged-KV range, so verify position t attended to the draft + tokens after it. The native MTP path delegates causal masking to aiter. """ metadata, captured = _run_verify_block() @@ -219,50 +209,37 @@ def test_verify_flatten_rows_are_causal(): assert decode is not None, "batch was not classified as a decode" assert decode.max_qo_len == QLEN, ( f"expected a {QLEN}-token verify block, got max_qo_len=" - f"{decode.max_qo_len}; the flatten under test was not reached" + f"{decode.max_qo_len}; the MTP path under test was not reached" ) assert captured, "forward_mqa did not reach the Gluon kernel" seq_lens = _seq_lens() - indptr = captured["seq_info"].tolist() + num_reqs = len(seq_lens) + + q_nope = captured["q_nope"] + assert q_nope.dim() == 4, ( + f"verify must pass 4-D q to mla_gluon, got {q_nope.dim()}-D shape " + f"{tuple(q_nope.shape)}" + ) + assert q_nope.shape[:2] == (num_reqs, QLEN), ( + f"expected q shape [num_reqs={num_reqs}, qlen={QLEN}, ...], " + f"got {tuple(q_nope.shape)}" + ) + assert q_nope.shape[2] == NUM_QUERY_HEADS + + assert captured["use_2d_view"] is False page_table = captured["page_table"] - got_row_lens = [indptr[i + 1] - indptr[i] for i in range(len(indptr) - 1)] - want_row_lens = _expected_row_lens() - - assert got_row_lens == want_row_lens, ( - "AITER MLA verify flatten is not causal: verify row r*qlen+t must get " - f"seq_len_r - {QLEN - 1} + t KV entries.\n" - f" seq_lens {seq_lens}\n" - f" got {got_row_lens}\n" - f" expected {want_row_lens}\n" - "Rows longer than expected let a verify position attend to the draft " - "tokens it is supposed to be checking." + seq_info = captured["seq_info"] + assert page_table.dim() == 1, ( + f"verify MTP passes the 1-D paged_kv_indices buffer, got {page_table.dim()}-D" ) - - # arange_block_indices lays request r's pages out ascending from - # r * max_blocks, so each causal window must be that slice's prefix. - max_blocks = max(seq_lens) - for r, seq_len in enumerate(seq_lens): - for t in range(QLEN): - row = r * QLEN + t - window = page_table[indptr[row] : indptr[row + 1]].tolist() - n = max(0, seq_len - (QLEN - 1) + t) - want = [r * max_blocks + j for j in range(n)] - assert window == want, ( - f"request {r} (seq_len {seq_len}) verify token {t} reads KV " - f"pages {window}, expected the ascending causal prefix {want}" - ) - - padding_rows = got_row_lens[len(CONTEXT_LENS) * QLEN :] - assert padding_rows == [0] * (PADDING_ROWS * QLEN), ( - f"cudagraph padding requests (seq_len 0) must clamp to empty windows, " - f"got {padding_rows}" + assert seq_info.dim() == 1 and seq_info.numel() == num_reqs + 1, ( + f"verify MTP passes paged_kv_indptr [num_reqs + 1], got shape " + f"{tuple(seq_info.shape)}" ) - - # min_kv_seq_len tells Gluon how short the shortest row is, so it must be - # the minimum over the causal rows actually submitted, not over the - # per-request lengths they were cut from. - assert captured["min_kv_seq_len"] == min(want_row_lens), ( - f"min_kv_seq_len={captured['min_kv_seq_len']} does not match the " - f"shortest submitted row ({min(want_row_lens)})" + assert decode is not None and decode.paged_kv_indptr is not None + assert torch.equal(seq_info, decode.paged_kv_indptr[: num_reqs + 1]) + assert torch.equal( + page_table[: int(seq_info[-1].item())], + decode.paged_kv_indices[: int(seq_info[-1].item())], ) diff --git a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py index 1894d8aa9c..604e7b84f7 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py +++ b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py @@ -40,7 +40,6 @@ class _ExpandPageIndicesKernel: block_table_tensor, stride, paged_kv_indptr, - seq_lens_for_kernel, *, KERNEL_BLOCK_SIZE, BLOCK_SIZE, @@ -48,7 +47,9 @@ class _ExpandPageIndicesKernel: self.kernel_block_size = KERNEL_BLOCK_SIZE for req_idx in range(self.grid[0]): out_start = int(paged_kv_indptr[req_idx].item()) - seq_len = int(seq_lens_for_kernel[req_idx].item()) + seq_len = int( + (paged_kv_indptr[req_idx + 1] - paged_kv_indptr[req_idx]).item() + ) for token_idx in range(seq_len): block_id = int( block_table_tensor[req_idx, token_idx // KERNEL_BLOCK_SIZE].item() @@ -249,6 +250,51 @@ def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch): assert get_mla_metadata_v1.call_args.kwargs["uni_seqlen_qo"] == 4 +def test_min_kv_seq_len_ignores_cudagraph_padding_rows(monkeypatch): + """min_kv_seq_len must not be driven by padded dummy requests. + + Full-CG verify pins zero-qo rows to max_qo_len for paged_kv metadata; taking + per_req_len.min() over all rows then reports qlen instead of a real KV length + and collapses Gluon KV-split parallelism on single-request decode. + """ + monkeypatch.setitem( + sys.modules, + "aiter", + SimpleNamespace(get_mla_metadata_v1=mock.MagicMock()), + ) + monkeypatch.setattr( + rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel() + ) + monkeypatch.setattr(rocm_aiter_mla, "_gluon_mla_decode_supported", lambda: True) + monkeypatch.setattr(rocm_aiter_mla, "_aiter_mla_small_head_mode", lambda: "auto") + + mtp_qlen = 8 + num_reqs = 8 + active_seq_len = 1032 + seq_lens = torch.tensor([active_seq_len] + [0] * (num_reqs - 1), dtype=torch.int32) + query_start_loc = torch.tensor( + [0, mtp_qlen] + [mtp_qlen] * (num_reqs - 1), dtype=torch.int32 + ) + + metadata = AiterMLAMetadataBuilder._build_decode( + _builder( + mtp_decode_qlen=mtp_qlen, + has_full_cudagraphs=True, + max_decode_rows=num_reqs, + num_heads=12, + ), + block_table_tensor=torch.zeros(num_reqs, active_seq_len, dtype=torch.int32), + seq_lens_device=seq_lens, + max_seq_len=active_seq_len, + query_start_loc_cpu=query_start_loc, + query_start_loc_device=query_start_loc, + num_decode_tokens=num_reqs * mtp_qlen, + dcp_tot_seq_lens_device=None, + ) + + assert metadata.min_kv_seq_len == active_seq_len + + def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr( monkeypatch, ): diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index 0454502dab..7b1263dedd 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -729,7 +729,6 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): block_table_tensor, block_table_tensor.stride(0), paged_kv_indptr, - seq_lens_for_kernel, KERNEL_BLOCK_SIZE=self.kernel_block_size, BLOCK_SIZE=1024, ) @@ -834,6 +833,26 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): ) has_persistent_metadata = True + # Small-head multi-token verify uses mla_gluon's 4-D MTP entry over the + # ordinary per-request paged-KV view, so there is no expanded per-token + # buffer to build here. mla_gluon still wants a lower bound on the KV + # length it is asked to split; on the verify path that bound is over + # active requests, not cudagraph padding rows pinned to max_qo_len. + # The .item() below runs in the builder, outside the captured region, + # so it does not abort HIP graph capture the way the per-layer syncs + # in forward_mqa did. + min_kv_seq_len = 1 + if AiterMLAHelper.use_gluon_verify( + self.num_heads, int(max_qo_len), self._kv_cache_dtype_str + ): + per_req_len = paged_kv_indptr[1:] - paged_kv_indptr[:-1] + if pad_uniform_mtp: + active = qo_lens_device > 0 + if active.any(): + min_kv_seq_len = int(per_req_len[active].min().item()) + else: + min_kv_seq_len = int(per_req_len.min().item()) + attn_metadata = AiterMLADecodeMetadata( block_table=block_table_tensor, seq_lens=seq_lens_for_kernel, @@ -843,6 +862,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): qo_indptr=qo_indptr, dcp_tot_seq_lens=dcp_tot_seq_lens_device, max_qo_len=max_qo_len, + min_kv_seq_len=min_kv_seq_len, use_gluon_decode=use_gluon_decode, attn_out_dtype=self.decode_attn_out_dtype, has_persistent_metadata=has_persistent_metadata, @@ -884,7 +904,6 @@ def _expand_page_indices_kernel( block_table, block_table_stride, cu_num_tokens, - seq_lens, KERNEL_BLOCK_SIZE: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): @@ -904,7 +923,7 @@ def _expand_page_indices_kernel( req_idx = tl.program_id(0) row_ptr = block_table + req_idx * block_table_stride start_idx = tl.load(cu_num_tokens + req_idx) - num_tokens = tl.load(seq_lens + req_idx) + num_tokens = tl.load(cu_num_tokens + req_idx + 1) - start_idx offset = tl.arange(0, BLOCK_SIZE) for i in tl.range(0, num_tokens, BLOCK_SIZE): @@ -1056,11 +1075,12 @@ class AiterMLAHelper: @staticmethod def use_gluon_verify(num_heads: int, max_qo_len: int, kv_cache_dtype: str) -> bool: - """Whether a small-head multi-token verify is flattened onto Gluon. + """Whether a small-head multi-token verify uses native Gluon MTP. - bf16 has no gqa<16, qseqlen>1 asm kernel, so the verify is flattened - into per-token Gluon decodes. fp8 has one via the q-row fold and must - not come here: the flatten hands Gluon the batch size its fp8 regime + bf16 has no gqa<16, qseqlen>1 asm kernel, so verify goes through + ``mla_gluon``'s 4-D MTP entry (``q`` shaped ``[batch, qlen, nhead, dim]``) + with ``use_2d_view=False``. fp8 has one via the q-row fold and must not + come here: the MTP path hands Gluon the batch size its fp8 regime asserts against. A predicate rather than inline in forward_mqa so the builder sees the same answer the impl acts on. """ @@ -1391,19 +1411,17 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): return o, None # 12-head (<16) multi-token verify (DSpark): the asm path has no - # gqa<16, qseqlen>1 kernel. Flatten each verify token to its own - # qseqlen=1 gluon decode, mirroring the TRITON_MLA / sparse-backend - # flatten but on the fast gluon kernel. The block is causal -- the - # target is checking draft tokens, so position t must not see t+1 -- - # and attention rows are independent, so giving row t the KV range - # [0, context + t] is exactly causal multi-token attention. + # gqa<16, qseqlen>1 kernel, so the block goes to the gluon kernel's 4-D + # MTP entry, which serves a whole (1 + num_spec) block in one launch. + # The block is causal -- the target is checking draft tokens, so + # position t must not see t+1 -- and the kernel bounds each query + # position's scores itself. # Arch, mode and dtype gating all live in use_gluon_verify, so that the # builder -- which has to know whether the asm decode will run -- sees # the same answer as this branch. if AiterMLAHelper.use_gluon_verify( self.num_heads, int(decode.max_qo_len), self.kv_cache_dtype ): - qlen = int(decode.max_qo_len) if type(q) is tuple: q_nope, q_pe = q else: @@ -1419,56 +1437,39 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): device=q_nope.device, ) kv_buffer = kv_c_and_k_pe_cache.reshape(-1, kv_c_and_k_pe_cache.shape[-1]) - # Expand per-request paged-KV to per-verify-token. Row r*qlen+t is - # request r's verify token t, and seq_lens counts the tokens - # scheduled in this step, so a request's KV range already spans its - # whole verify block and context_r = seq_len_r - qlen. Token t may - # attend to [0, context_r + t], i.e. seq_len_r - (qlen - 1) + t - # entries. paged_kv_indices lists a request's pages in ascending - # position order, so each row's causal window is a prefix of that - # request's slice and only the row length changes. Rows clamp to - # zero for cudagraph padding requests, whose seq_len is 0. Fully - # vectorized (no host loop). - old_indptr = decode.paged_kv_indptr - per_req_len = old_indptr[1:] - old_indptr[:-1] - dev = q_nope.device - row_req = torch.arange(per_req_len.shape[0], device=dev).repeat_interleave( - qlen + assert attn_metadata.causal, ( + "AITER MLA small-head verify MTP is causal-only" ) - row_len = ( - ( - per_req_len.unsqueeze(1) - - (qlen - 1) - + torch.arange(qlen, device=dev, dtype=per_req_len.dtype) + # Hand mla_gluon its 4-D MTP entry instead of an expanded + # per-verify-token paged-KV view. The flat query layout is already + # row-major (r * qlen + t), so unflatten is a free view. mla_gluon + # applies the per-position causal bound + # score_end = min(split_kv_end, seq_len - qlen + q_pos + 1) + # in-kernel, and seq_lens already spans the verify block, so that + # bound is context_r + q_pos + 1 -- the same window the expanded + # view supplied by truncating each row's page list. + qlen = int(decode.max_qo_len) + num_reqs = B // qlen + if num_reqs * qlen != B: + raise ValueError( + f"verify block {B} rows is not a multiple of qlen {qlen}" ) - .clamp_(min=0) - .flatten() - ) - new_indptr = torch.cat([old_indptr.new_zeros(1), row_len.cumsum(0)]).to( - torch.int32 - ) - total = int(new_indptr[-1].item()) - within = torch.arange(total, device=dev, dtype=torch.int64) - new_indptr[ - :-1 - ].to(torch.int64).repeat_interleave(row_len) - src = ( - old_indptr[row_req].to(torch.int64).repeat_interleave(row_len) + within - ) - new_indices = decode.paged_kv_indices[src] + assert decode.paged_kv_indptr is not None + assert decode.paged_kv_indices is not None mla_gluon = _get_mla_gluon() mla_gluon( - q_nope=q_nope, - q_pe=q_pe, + q_nope=q_nope.unflatten(0, (num_reqs, qlen)), + q_pe=q_pe.unflatten(0, (num_reqs, qlen)), kv_c=kv_buffer, - o=o, - page_table=new_indices, - seq_info=new_indptr, + o=o.unflatten(0, (num_reqs, qlen)), + page_table=decode.paged_kv_indices, + seq_info=decode.paged_kv_indptr, sm_scale=self.scale, k_pe=None, kv_pe_offset=self.kv_lora_rank, use_2d_view=False, kv_scale=1.0, - min_kv_seq_len=int(row_len.min()), + min_kv_seq_len=decode.min_kv_seq_len, ) return o, None diff --git a/vllm/v1/attention/backends/mla/triton_mla.py b/vllm/v1/attention/backends/mla/triton_mla.py index 6275778902..f28d8c78b6 100644 --- a/vllm/v1/attention/backends/mla/triton_mla.py +++ b/vllm/v1/attention/backends/mla/triton_mla.py @@ -6,6 +6,7 @@ from typing import ClassVar import torch import vllm.envs as envs +from vllm.config import VllmConfig from vllm.config.cache import CacheDType from vllm.logger import init_logger from vllm.model_executor.layers.attention.mla_attention import ( @@ -25,6 +26,7 @@ from vllm.v1.attention.backend import ( MultipleOf, ) from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd +from vllm.v1.kv_cache_interface import KVCacheSpec from vllm.v1.worker.workspace import ( current_workspace_manager, is_workspace_manager_initialized, @@ -55,6 +57,34 @@ class TritonMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): # forward_mqa, so no intra-block causal masking is required. supports_non_causal_multi_token_decode: ClassVar[bool] = True + @classmethod + def get_cudagraph_support( + cls, + vllm_config: VllmConfig, + kv_cache_spec: KVCacheSpec, + ) -> AttentionCGSupport: + """Report UNIFORM_BATCH where a non-causal multi-token block is served. + + ``_cudagraph_support`` is a class constant, so serving the DSpark + draft's (1 + num_spec) block through the decode path reports + UNIFORM_SINGLE_TOKEN_DECODE and, because the engine takes the minimum + over all attention groups, downgrades the *whole* engine off full + cudagraphs. ``forward_mqa`` flattens that block with + ``repeat_interleave`` on a Python int and performs no device->host + sync, so it does satisfy the UNIFORM_BATCH contract. + + ``non_causal_multi_token_decode`` is a KV-cache-group property, not a + per-layer one: ``MLAAttentionSpec.merge`` ORs it over every layer in + the group, so a group holding both a draft and its target reports it + for both. That is the same predicate ``__init__`` below already uses to + raise ``reorder_batch_threshold``, so the two stay consistent, but it + does mean this lifts a causal target sharing the draft's KV cache group + as well. + """ + if getattr(kv_cache_spec, "non_causal_multi_token_decode", False): + return AttentionCGSupport.UNIFORM_BATCH + return cls._cudagraph_support + def __init__(self, kv_cache_spec, layer_names, vllm_config, device): super().__init__(kv_cache_spec, layer_names, vllm_config, device) # DCP local sequence lengths are not advanced between draft steps.