From 32636580a6f1c3bc41deefc4bf7800f850c24034 Mon Sep 17 00:00:00 2001 From: olka-amd Date: Fri, 18 Sep 2026 11:49:53 +0300 Subject: [PATCH] [Bugfix][MLA] TritonMLA: fix illegal memory access on causal multi-token decode (#51065) Signed-off-by: Oleksandr Kachur Co-authored-by: Claude Opus 5 --- .../test_triton_mla_causal_verify_flatten.py | 215 ++++++++++++++++++ vllm/v1/attention/backends/mla/triton_mla.py | 91 +++----- 2 files changed, 252 insertions(+), 54 deletions(-) create mode 100644 tests/kernels/attention/test_triton_mla_causal_verify_flatten.py diff --git a/tests/kernels/attention/test_triton_mla_causal_verify_flatten.py b/tests/kernels/attention/test_triton_mla_causal_verify_flatten.py new file mode 100644 index 0000000000..a27552c90e --- /dev/null +++ b/tests/kernels/attention/test_triton_mla_causal_verify_flatten.py @@ -0,0 +1,215 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the multi-token decode flatten in ``TritonMLAImpl.forward_mqa``. + +``decode_attention_fwd`` has no causal flag: it launches one program per row of +``q`` and reads that row's KV extent from ``B_Seqlen``. So a ``query_len``-token +decode block is flattened to one row per query token, and intra-block causality +has to be expressed through the per-row sequence lengths -- row ``t`` gets +``seq_len - (query_len - 1) + t``, i.e. the committed prefix plus block tokens +``0..t``. A non-causal draft block instead gives every row the same full extent. + +The kernel is replaced by a spy, so these tests assert on the ``block_table`` +and ``seq_lens`` the backend actually submits and need no GPU or Triton runtime. +""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from vllm.model_executor.layers.attention.mla_attention import QueryLenSupport +from vllm.v1.attention.backend import AttentionCGSupport +from vllm.v1.attention.backends.mla.triton_mla import ( + TritonMLAImpl, + TritonMLAMetadataBuilder, +) + +NUM_HEADS = 16 +KV_LORA_RANK = 512 +QK_ROPE_HEAD_DIM = 64 +HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM +PAGE_SIZE = 16 + +# Committed context per request, including a fresh request at zero context +# where the whole KV extent is the block itself. A decode block's seq_len +# already counts the block, so seq_len = context + query_len. +CONTEXT_LENS = [0, 1, 37, 512] + + +def _seq_lens(query_len: int) -> list[int]: + return [c + query_len for c in CONTEXT_LENS] + + +def _make_impl(dcp_world_size: int = 1) -> TritonMLAImpl: + """Only the attributes ``forward_mqa`` reads; __init__ wants a full config.""" + impl = object.__new__(TritonMLAImpl) + impl.kv_lora_rank = KV_LORA_RANK + impl.scale = HEAD_SIZE**-0.5 + impl._sm_count = 304 + impl.dcp_world_size = dcp_world_size + return impl + + +def _make_metadata(seq_lens: list[int], query_len: int, causal: bool): + num_decodes = len(seq_lens) + return SimpleNamespace( + num_decodes=num_decodes, + num_decode_tokens=num_decodes * query_len, + max_seq_len=max(seq_lens), + causal=causal, + decode=SimpleNamespace( + # arange page ids so each request's window is an identifiable slice. + block_table=torch.arange(num_decodes * 64, dtype=torch.int32).reshape( + num_decodes, 64 + ), + seq_lens=torch.tensor(seq_lens, dtype=torch.int32), + ), + ) + + +def _run_forward_mqa( + seq_lens: list[int], + query_len: int, + causal: bool, + q_rows=None, + dcp_world_size: int = 1, +): + """Drive forward_mqa with the decode kernel spied out. + + Returns the ``(block_table, seq_lens)`` handed to ``decode_attention_fwd``. + """ + captured: dict = {} + + def spy(q, kv_cache, kv_c_cache, o, lse, block_table, b_seq_len, *args, **kwargs): + captured["block_table"] = block_table.detach().clone() + captured["seq_lens"] = b_seq_len.detach().clone() + + metadata = _make_metadata(seq_lens, query_len, causal) + rows = metadata.num_decode_tokens if q_rows is None else q_rows + q = torch.zeros(rows, NUM_HEADS, HEAD_SIZE, dtype=torch.bfloat16) + kv_cache = torch.zeros( + len(seq_lens) * 64, PAGE_SIZE, HEAD_SIZE, dtype=torch.bfloat16 + ) + layer = SimpleNamespace(_k_scale=torch.ones(1), layer_name="test") + + with patch("vllm.v1.attention.backends.mla.triton_mla.decode_attention_fwd", spy): + _make_impl(dcp_world_size).forward_mqa(q, kv_cache, metadata, layer) + + assert captured, "forward_mqa did not reach the decode kernel" + return captured + + +def test_multi_token_decode_flags_are_declared_together(): + """Capture probes max_query_len > 1, which only clears the assert in + build_for_cudagraph_capture once query_len_support raised the threshold.""" + assert ( + TritonMLAMetadataBuilder._cudagraph_support == AttentionCGSupport.UNIFORM_BATCH + ) + assert TritonMLAMetadataBuilder.query_len_support == QueryLenSupport.UNIFORM + + +@pytest.mark.parametrize("query_len", [2, 3, 4, 5, 8]) +def test_causal_block_rows_see_prefix_plus_own_position(query_len): + """Causal verify: row t sees seq_len - (query_len - 1) + t KV entries. + + Regression guard: the causal branch used to be skipped entirely, so a verify + token could attend the draft siblings it is supposed to be checking. Fails + on unmodified upstream, where no flatten happens at all for causal blocks. + """ + seq_lens = _seq_lens(query_len) + captured = _run_forward_mqa(seq_lens, query_len, causal=True) + + got = captured["seq_lens"].tolist() + want = [s - (query_len - 1) + t for s in seq_lens for t in range(query_len)] + assert got == want, ( + "causal flatten is not causal: verify row r*query_len+t must get " + f"seq_len_r - {query_len - 1} + t entries.\n" + f" seq_lens {seq_lens}\n got {got}\n expected {want}" + ) + + # Per-request invariants the arithmetic must hold regardless of query_len. + for r, seq_len in enumerate(seq_lens): + rows = got[r * query_len : (r + 1) * query_len] + assert rows == sorted(rows) and len(set(rows)) == query_len, ( + f"request {r} rows must strictly increase, got {rows}" + ) + assert rows[-1] == seq_len, "last row must see the full sequence" + assert rows[0] == seq_len - (query_len - 1) + assert all(n > 0 for n in rows), f"non-positive KV extent in {rows}" + + +def test_cudagraph_padding_rows_present_no_kv_extent(): + """Causal flattening pins cudagraph padding rows (seq_len 0) to exactly + zero KV extent, keeping them in the kernel's skipped branch.""" + query_len = 4 + seq_lens = _seq_lens(query_len) + [0] + captured = _run_forward_mqa(seq_lens, query_len, causal=True) + + padding = captured["seq_lens"].tolist()[-query_len:] + assert padding == [0] * query_len, ( + f"padding rows must present exactly zero KV extent, got {padding}" + ) + + +@pytest.mark.parametrize("query_len", [2, 4]) +def test_non_causal_block_rows_all_see_the_full_prefix(query_len): + """The DSpark draft block is generated in one pass, so no row is masked.""" + seq_lens = _seq_lens(query_len) + captured = _run_forward_mqa(seq_lens, query_len, causal=False) + + want = [s for s in seq_lens for _ in range(query_len)] + assert captured["seq_lens"].tolist() == want + + +@pytest.mark.parametrize("causal", [True, False]) +def test_block_table_is_expanded_to_match_q_rows(causal): + """The kernel indexes block_table by program_id, so it needs one row per + query token -- a short block_table is read out of bounds.""" + query_len = 4 + captured = _run_forward_mqa(_seq_lens(query_len), query_len, causal=causal) + + num_rows = len(CONTEXT_LENS) * query_len + assert captured["block_table"].shape[0] == num_rows + assert captured["seq_lens"].shape[0] == num_rows + # repeat_interleave, not repeat: a request's rows must stay adjacent. + for r in range(len(CONTEXT_LENS)): + block = captured["block_table"][r * query_len : (r + 1) * query_len] + assert torch.equal(block, block[:1].expand_as(block)) + + +@pytest.mark.parametrize("causal", [True, False]) +def test_single_token_decode_is_untouched(causal): + """query_len == 1 must short-circuit: ordinary decode sees no expansion.""" + seq_lens = _seq_lens(1) + captured = _run_forward_mqa(seq_lens, query_len=1, causal=causal) + + assert captured["seq_lens"].tolist() == seq_lens + assert captured["block_table"].shape[0] == len(seq_lens) + + +def test_expansion_factor_follows_the_q_row_count(): + """The kernel indexes block_table and seq_lens by program_id, so the + expansion must follow q rows, not the token/request counts.""" + num_rows = len(CONTEXT_LENS) * 4 + captured = _run_forward_mqa(_seq_lens(1), query_len=1, causal=True, q_rows=num_rows) + + assert captured["block_table"].shape[0] == num_rows + assert captured["seq_lens"].shape[0] == num_rows + + +def test_non_uniform_block_is_rejected(): + """Q rows must divide evenly across the decode requests.""" + with pytest.raises(AssertionError, match="non-uniform decode block"): + _run_forward_mqa(_seq_lens(1), query_len=1, causal=True, q_rows=6) + + +def test_causal_multi_token_decode_is_rejected_under_dcp(): + """Per-row extents offset the global seq_len, but DCP passes the rank-local + slice, so the causal arithmetic would silently address the wrong KV.""" + with pytest.raises(AssertionError, match="not supported with DCP"): + _run_forward_mqa(_seq_lens(4), query_len=4, causal=True, dcp_world_size=2) + + # Non-causal rows all take the same extent, which stays correct when local. + _run_forward_mqa(_seq_lens(4), query_len=4, causal=False, dcp_world_size=2) diff --git a/vllm/v1/attention/backends/mla/triton_mla.py b/vllm/v1/attention/backends/mla/triton_mla.py index 78787d5c26..3deb996a5b 100644 --- a/vllm/v1/attention/backends/mla/triton_mla.py +++ b/vllm/v1/attention/backends/mla/triton_mla.py @@ -6,7 +6,6 @@ from typing import ClassVar import torch import vllm.envs as envs -from vllm.config import VllmConfig from vllm.config.cache import CacheDType from vllm.logger import init_logger from vllm.model_executor.layers.attention.mla_attention import ( @@ -14,6 +13,7 @@ from vllm.model_executor.layers.attention.mla_attention import ( MLACommonImpl, MLACommonMetadata, MLACommonMetadataBuilder, + QueryLenSupport, ) from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability @@ -26,7 +26,6 @@ from vllm.v1.attention.backend import ( MultipleOf, ) from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd -from vllm.v1.kv_cache_interface import KVCacheSpec from vllm.v1.worker.workspace import ( current_workspace_manager, is_workspace_manager_initialized, @@ -50,50 +49,16 @@ def _compute_num_kv_splits(max_seq_len: int, sm_count: int) -> int: class TritonMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): - _cudagraph_support: ClassVar[AttentionCGSupport] = ( - AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE - ) - # Non-causal DSpark block is flattened to one decode row per query token in - # forward_mqa, so no intra-block causal masking is required. + # forward_mqa flattens a uniform multi-token block to one decode row per + # query token, so causal and non-causal blocks both take the decode path. + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + query_len_support: ClassVar[QueryLenSupport] = QueryLenSupport.UNIFORM supports_non_causal_multi_token_decode: ClassVar[bool] = True - @classmethod - def get_cudagraph_support( - cls, - vllm_config: VllmConfig, - kv_cache_spec: KVCacheSpec, - ) -> AttentionCGSupport: - """Report UNIFORM_BATCH where a non-causal multi-token block is served. - - ``_cudagraph_support`` is a class constant, so serving the DSpark - draft's (1 + num_spec) block through the decode path reports - UNIFORM_SINGLE_TOKEN_DECODE and, because the engine takes the minimum - over all attention groups, downgrades the *whole* engine off full - cudagraphs. ``forward_mqa`` flattens that block with - ``repeat_interleave`` on a Python int and performs no device->host - sync, so it does satisfy the UNIFORM_BATCH contract. - - ``non_causal_multi_token_decode`` is a KV-cache-group property, not a - per-layer one: ``MLAAttentionSpec.merge`` ORs it over every layer in - the group, so a group holding both a draft and its target reports it - for both. That is the same predicate ``__init__`` below already uses to - raise ``reorder_batch_threshold``, so the two stay consistent, but it - does mean this lifts a causal target sharing the draft's KV cache group - as well. - """ - if getattr(kv_cache_spec, "non_causal_multi_token_decode", False): - return AttentionCGSupport.UNIFORM_BATCH - return cls._cudagraph_support - def __init__(self, kv_cache_spec, layer_names, vllm_config, device): super().__init__(kv_cache_spec, layer_names, vllm_config, device) # DCP local sequence lengths are not advanced between draft steps. self.supports_draft_decode_metadata_update = self.dcp_world_size == 1 - # Only the non-causal DSpark draft group serves multi-token blocks via - # the decode path; raise its reorder threshold to the spec block length - # so full-cudagraph capture admits it. Causal usage stays single-token. - if getattr(self, "non_causal_multi_token_decode", False): - self._init_reorder_batch_threshold(1, supports_spec_as_decode=True) self._reserve_attn_logits_workspace() def update_draft_decode_metadata(self, _metadata: MLACommonMetadata) -> None: @@ -109,12 +74,12 @@ class TritonMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]): """ if not is_workspace_manager_initialized(): return - # Decode reorder threshold is 1, so decode tokens <= max_num_seqs. - B = self.vllm_config.scheduler_config.max_num_seqs - # Non-causal DSpark draft flattens each request's block to query_len - # decode rows; cover max_num_seqs * block_len rows. - if getattr(self, "non_causal_multi_token_decode", False): - B *= self.reorder_batch_threshold + # forward_mqa flattens each request's block to query_len decode rows, + # and query_len is bounded by the reorder threshold. + B = ( + self.vllm_config.scheduler_config.max_num_seqs + * self.reorder_batch_threshold + ) # DCP all-gathers the query heads before forward_mqa. q_num_heads = self.num_heads * self.dcp_world_size max_splits = _compute_num_kv_splits( @@ -313,14 +278,32 @@ class TritonMLAImpl(MLACommonImpl[MLACommonMetadata]): block_table = attn_metadata.decode.block_table seq_lens = attn_metadata.decode.seq_lens - if not attn_metadata.causal: - # Non-causal DSpark block: flatten to one decode row per query token. - # Each row attends to the same committed KV prefix (per-row seq_lens) - # and never to sibling block tokens = non-causal block semantics. - # Mirrors FlashInferMLA's non-causal path. - query_len = attn_metadata.num_decode_tokens // attn_metadata.num_decodes - if query_len > 1: - block_table = block_table.repeat_interleave(query_len, dim=0) + # decode_attention_fwd has no causal flag: it launches one program per + # row of q and reads that row's KV extent from seq_lens, so intra-block + # causality is expressed as per-row extents. Deriving query_len from the + # tensors the kernel indexes keeps the three row counts in step. + num_decodes = seq_lens.shape[0] + query_len, remainder = divmod(B, num_decodes) if num_decodes else (1, 0) + assert remainder == 0, ( + f"non-uniform decode block: {B} query rows over {num_decodes} requests" + ) + if query_len > 1: + block_table = block_table.repeat_interleave(query_len, dim=0) + if attn_metadata.causal: + # Per-row extents are offsets off the global sequence length; + # under DCP seq_lens is this rank's local slice instead. + assert self.dcp_world_size == 1, ( + "causal multi-token decode is not supported with DCP" + ) + # Row t attends the prefix plus block tokens 0..t. Clamp holds + # padding rows at extent 0; the kernel skips them either way + # (split_kv_end == split_kv_start). + offsets = torch.arange( + 1 - query_len, 1, device=seq_lens.device, dtype=seq_lens.dtype + ) + seq_lens = (seq_lens.unsqueeze(1) + offsets).flatten().clamp(min=0) + else: + # Non-causal draft block: every row sees the same prefix. seq_lens = seq_lens.repeat_interleave(query_len) # Run MQA — always pass layer scales. When KV cache is