[ROCm][MLA] Reach FULL cudagraphs for AITER MLA speculative decoding (#51171)

Signed-off-by: yudigege86 <naqin@amd.com>
Co-authored-by: yudigege86 <naqin@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Matt Qin
2026-08-29 19:49:55 -07:00
committed by GitHub
co-authored by yudigege86 Cursor Andreas Karatzas
parent b5707bf994
commit dbf662c9e8
4 changed files with 180 additions and 126 deletions
@@ -1,21 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for causal masking in the AITER MLA verify flatten.
"""Regression test for causal masking in the AITER MLA verify MTP path.
When ``num_heads < 16`` the ROCm AITER MLA backend cannot use the ASM decode
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` flattens a
``1 + num_speculative_tokens`` verify block into that many single-token Gluon
decodes. Every resulting row used to be handed its request's entire paged-KV
range. ``paged_kv_indptr`` is the cumulative sum of ``seq_lens``, which counts
the tokens scheduled in the current step, so that range already spans the
verify block: each verify position could attend to the draft tokens after it,
which are the tokens it is supposed to be checking. Nothing crashes and nothing
warns, so the only way to catch it is to look at the KV window per row.
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` routes verify
through ``mla_gluon``'s native 4-D MTP entry (``q`` shaped
``[batch, qlen, nhead, dim]`` with ``use_2d_view=False``). aiter applies the
causal tail internally: position ``t`` attends KV ``[0, seq_len - qlen + t]``.
The test drives the real builder and the real ``forward_mqa`` over a
multi-token decode batch, intercepts the ``(page_table, seq_info)`` actually
handed to the Gluon kernel, and pins verify row ``t`` of request ``r`` to the
``context_r + t + 1`` entries of that request's ascending slice.
The test drives the real builder and ``forward_mqa`` over a multi-token decode
batch, intercepts the arguments actually handed to the Gluon kernel, and pins
that the 4-D MTP contract is used with per-request ``block_table`` and
``cache_seqlens`` rather than a flattened per-token paged-KV view.
"""
import types
@@ -32,15 +28,15 @@ def _on_rocm_with_aiter() -> bool:
return current_platform.is_rocm() and is_aiter_found()
# The production verify flatten is gfx950-only. This test replaces the Gluon
# The production verify MTP path is gfx950-only. This test replaces the Gluon
# kernel with a spy and forces the architecture feature probe so the metadata
# regression remains testable on every ROCm AITER CI runner.
pytestmark = pytest.mark.skipif(
not _on_rocm_with_aiter(),
reason="ROCM_AITER_MLA verify flatten requires ROCm and AITER",
reason="ROCM_AITER_MLA verify MTP requires ROCm and AITER",
)
# Below the 16-head threshold that selects the flatten. DeepSeek-V3 / R1 at
# Below the 16-head threshold that selects the MTP path. DeepSeek-V3 / R1 at
# TP=16 and Kimi-K3 at TP=8 both land here.
NUM_QUERY_HEADS = 12
KV_LORA_RANK = 512
@@ -54,8 +50,7 @@ QLEN = 8
# Committed context per request, including a fresh request at zero context
# where the whole KV range is the verify block itself.
CONTEXT_LENS = [0, 1, 37, 512]
# A cudagraph padding request: seq_len 0, so every one of its rows must clamp
# to an empty window.
# A cudagraph padding request: seq_len 0.
PADDING_ROWS = 1
# The Gluon path flattens the KV cache to one page per token.
@@ -68,16 +63,10 @@ def _seq_lens() -> list[int]:
return [c + QLEN for c in CONTEXT_LENS] + [0] * PADDING_ROWS
def _expected_row_lens() -> list[int]:
"""Causal row lengths: row r*QLEN + t sees seq_len_r - (QLEN - 1) + t."""
return [max(0, s - (QLEN - 1) + t) for s in _seq_lens() for t in range(QLEN)]
def _run_verify_block():
"""Drive the real builder + forward_mqa over one multi-token verify block.
Returns ``(metadata, captured)`` where ``captured`` holds the ``page_table``,
``seq_info`` and ``min_kv_seq_len`` the backend passed to the Gluon kernel.
Returns ``(metadata, captured)`` where ``captured`` holds the Gluon kwargs.
"""
from tests.v1.attention.utils import (
BatchSpec,
@@ -144,9 +133,10 @@ def _run_verify_block():
captured: dict = {}
def spy(**kwargs):
captured.update(kwargs)
captured["q_nope"] = kwargs["q_nope"].detach().clone()
captured["page_table"] = kwargs["page_table"].detach().clone()
captured["seq_info"] = kwargs["seq_info"].detach().clone()
captured["min_kv_seq_len"] = kwargs["min_kv_seq_len"]
with set_current_vllm_config(vllm_config):
builder = builder_cls(spec, [layer_name], vllm_config, device)
@@ -206,12 +196,12 @@ def _run_verify_block():
return metadata, captured
def test_verify_flatten_rows_are_causal():
"""Verify row t must see the committed prefix plus tokens 0..t, and no more.
def test_verify_mtp_uses_native_4d_gluon_entry():
"""Verify uses 4-D q with per-request paged_kv metadata.
Regression guard: before the fix every row of a request got that request's
whole paged-KV range, so verify position t attended to the draft tokens
after it. Fails on unmodified upstream.
Regression guard: before the fix every flattened row of a request got that
request's whole paged-KV range, so verify position t attended to the draft
tokens after it. The native MTP path delegates causal masking to aiter.
"""
metadata, captured = _run_verify_block()
@@ -219,50 +209,37 @@ def test_verify_flatten_rows_are_causal():
assert decode is not None, "batch was not classified as a decode"
assert decode.max_qo_len == QLEN, (
f"expected a {QLEN}-token verify block, got max_qo_len="
f"{decode.max_qo_len}; the flatten under test was not reached"
f"{decode.max_qo_len}; the MTP path under test was not reached"
)
assert captured, "forward_mqa did not reach the Gluon kernel"
seq_lens = _seq_lens()
indptr = captured["seq_info"].tolist()
num_reqs = len(seq_lens)
q_nope = captured["q_nope"]
assert q_nope.dim() == 4, (
f"verify must pass 4-D q to mla_gluon, got {q_nope.dim()}-D shape "
f"{tuple(q_nope.shape)}"
)
assert q_nope.shape[:2] == (num_reqs, QLEN), (
f"expected q shape [num_reqs={num_reqs}, qlen={QLEN}, ...], "
f"got {tuple(q_nope.shape)}"
)
assert q_nope.shape[2] == NUM_QUERY_HEADS
assert captured["use_2d_view"] is False
page_table = captured["page_table"]
got_row_lens = [indptr[i + 1] - indptr[i] for i in range(len(indptr) - 1)]
want_row_lens = _expected_row_lens()
assert got_row_lens == want_row_lens, (
"AITER MLA verify flatten is not causal: verify row r*qlen+t must get "
f"seq_len_r - {QLEN - 1} + t KV entries.\n"
f" seq_lens {seq_lens}\n"
f" got {got_row_lens}\n"
f" expected {want_row_lens}\n"
"Rows longer than expected let a verify position attend to the draft "
"tokens it is supposed to be checking."
seq_info = captured["seq_info"]
assert page_table.dim() == 1, (
f"verify MTP passes the 1-D paged_kv_indices buffer, got {page_table.dim()}-D"
)
# arange_block_indices lays request r's pages out ascending from
# r * max_blocks, so each causal window must be that slice's prefix.
max_blocks = max(seq_lens)
for r, seq_len in enumerate(seq_lens):
for t in range(QLEN):
row = r * QLEN + t
window = page_table[indptr[row] : indptr[row + 1]].tolist()
n = max(0, seq_len - (QLEN - 1) + t)
want = [r * max_blocks + j for j in range(n)]
assert window == want, (
f"request {r} (seq_len {seq_len}) verify token {t} reads KV "
f"pages {window}, expected the ascending causal prefix {want}"
)
padding_rows = got_row_lens[len(CONTEXT_LENS) * QLEN :]
assert padding_rows == [0] * (PADDING_ROWS * QLEN), (
f"cudagraph padding requests (seq_len 0) must clamp to empty windows, "
f"got {padding_rows}"
assert seq_info.dim() == 1 and seq_info.numel() == num_reqs + 1, (
f"verify MTP passes paged_kv_indptr [num_reqs + 1], got shape "
f"{tuple(seq_info.shape)}"
)
# min_kv_seq_len tells Gluon how short the shortest row is, so it must be
# the minimum over the causal rows actually submitted, not over the
# per-request lengths they were cut from.
assert captured["min_kv_seq_len"] == min(want_row_lens), (
f"min_kv_seq_len={captured['min_kv_seq_len']} does not match the "
f"shortest submitted row ({min(want_row_lens)})"
assert decode is not None and decode.paged_kv_indptr is not None
assert torch.equal(seq_info, decode.paged_kv_indptr[: num_reqs + 1])
assert torch.equal(
page_table[: int(seq_info[-1].item())],
decode.paged_kv_indices[: int(seq_info[-1].item())],
)
@@ -40,7 +40,6 @@ class _ExpandPageIndicesKernel:
block_table_tensor,
stride,
paged_kv_indptr,
seq_lens_for_kernel,
*,
KERNEL_BLOCK_SIZE,
BLOCK_SIZE,
@@ -48,7 +47,9 @@ class _ExpandPageIndicesKernel:
self.kernel_block_size = KERNEL_BLOCK_SIZE
for req_idx in range(self.grid[0]):
out_start = int(paged_kv_indptr[req_idx].item())
seq_len = int(seq_lens_for_kernel[req_idx].item())
seq_len = int(
(paged_kv_indptr[req_idx + 1] - paged_kv_indptr[req_idx]).item()
)
for token_idx in range(seq_len):
block_id = int(
block_table_tensor[req_idx, token_idx // KERNEL_BLOCK_SIZE].item()
@@ -249,6 +250,51 @@ def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch):
assert get_mla_metadata_v1.call_args.kwargs["uni_seqlen_qo"] == 4
def test_min_kv_seq_len_ignores_cudagraph_padding_rows(monkeypatch):
"""min_kv_seq_len must not be driven by padded dummy requests.
Full-CG verify pins zero-qo rows to max_qo_len for paged_kv metadata; taking
per_req_len.min() over all rows then reports qlen instead of a real KV length
and collapses Gluon KV-split parallelism on single-request decode.
"""
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=mock.MagicMock()),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
monkeypatch.setattr(rocm_aiter_mla, "_gluon_mla_decode_supported", lambda: True)
monkeypatch.setattr(rocm_aiter_mla, "_aiter_mla_small_head_mode", lambda: "auto")
mtp_qlen = 8
num_reqs = 8
active_seq_len = 1032
seq_lens = torch.tensor([active_seq_len] + [0] * (num_reqs - 1), dtype=torch.int32)
query_start_loc = torch.tensor(
[0, mtp_qlen] + [mtp_qlen] * (num_reqs - 1), dtype=torch.int32
)
metadata = AiterMLAMetadataBuilder._build_decode(
_builder(
mtp_decode_qlen=mtp_qlen,
has_full_cudagraphs=True,
max_decode_rows=num_reqs,
num_heads=12,
),
block_table_tensor=torch.zeros(num_reqs, active_seq_len, dtype=torch.int32),
seq_lens_device=seq_lens,
max_seq_len=active_seq_len,
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=num_reqs * mtp_qlen,
dcp_tot_seq_lens_device=None,
)
assert metadata.min_kv_seq_len == active_seq_len
def test_full_cudagraph_padded_uniform_mtp_synthesizes_decode_indptr(
monkeypatch,
):
@@ -729,7 +729,6 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
block_table_tensor,
block_table_tensor.stride(0),
paged_kv_indptr,
seq_lens_for_kernel,
KERNEL_BLOCK_SIZE=self.kernel_block_size,
BLOCK_SIZE=1024,
)
@@ -834,6 +833,26 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
)
has_persistent_metadata = True
# Small-head multi-token verify uses mla_gluon's 4-D MTP entry over the
# ordinary per-request paged-KV view, so there is no expanded per-token
# buffer to build here. mla_gluon still wants a lower bound on the KV
# length it is asked to split; on the verify path that bound is over
# active requests, not cudagraph padding rows pinned to max_qo_len.
# The .item() below runs in the builder, outside the captured region,
# so it does not abort HIP graph capture the way the per-layer syncs
# in forward_mqa did.
min_kv_seq_len = 1
if AiterMLAHelper.use_gluon_verify(
self.num_heads, int(max_qo_len), self._kv_cache_dtype_str
):
per_req_len = paged_kv_indptr[1:] - paged_kv_indptr[:-1]
if pad_uniform_mtp:
active = qo_lens_device > 0
if active.any():
min_kv_seq_len = int(per_req_len[active].min().item())
else:
min_kv_seq_len = int(per_req_len.min().item())
attn_metadata = AiterMLADecodeMetadata(
block_table=block_table_tensor,
seq_lens=seq_lens_for_kernel,
@@ -843,6 +862,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
qo_indptr=qo_indptr,
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
max_qo_len=max_qo_len,
min_kv_seq_len=min_kv_seq_len,
use_gluon_decode=use_gluon_decode,
attn_out_dtype=self.decode_attn_out_dtype,
has_persistent_metadata=has_persistent_metadata,
@@ -884,7 +904,6 @@ def _expand_page_indices_kernel(
block_table,
block_table_stride,
cu_num_tokens,
seq_lens,
KERNEL_BLOCK_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
@@ -904,7 +923,7 @@ def _expand_page_indices_kernel(
req_idx = tl.program_id(0)
row_ptr = block_table + req_idx * block_table_stride
start_idx = tl.load(cu_num_tokens + req_idx)
num_tokens = tl.load(seq_lens + req_idx)
num_tokens = tl.load(cu_num_tokens + req_idx + 1) - start_idx
offset = tl.arange(0, BLOCK_SIZE)
for i in tl.range(0, num_tokens, BLOCK_SIZE):
@@ -1056,11 +1075,12 @@ class AiterMLAHelper:
@staticmethod
def use_gluon_verify(num_heads: int, max_qo_len: int, kv_cache_dtype: str) -> bool:
"""Whether a small-head multi-token verify is flattened onto Gluon.
"""Whether a small-head multi-token verify uses native Gluon MTP.
bf16 has no gqa<16, qseqlen>1 asm kernel, so the verify is flattened
into per-token Gluon decodes. fp8 has one via the q-row fold and must
not come here: the flatten hands Gluon the batch size its fp8 regime
bf16 has no gqa<16, qseqlen>1 asm kernel, so verify goes through
``mla_gluon``'s 4-D MTP entry (``q`` shaped ``[batch, qlen, nhead, dim]``)
with ``use_2d_view=False``. fp8 has one via the q-row fold and must not
come here: the MTP path hands Gluon the batch size its fp8 regime
asserts against. A predicate rather than inline in forward_mqa so the
builder sees the same answer the impl acts on.
"""
@@ -1391,19 +1411,17 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
return o, None
# 12-head (<16) multi-token verify (DSpark): the asm path has no
# gqa<16, qseqlen>1 kernel. Flatten each verify token to its own
# qseqlen=1 gluon decode, mirroring the TRITON_MLA / sparse-backend
# flatten but on the fast gluon kernel. The block is causal -- the
# target is checking draft tokens, so position t must not see t+1 --
# and attention rows are independent, so giving row t the KV range
# [0, context + t] is exactly causal multi-token attention.
# gqa<16, qseqlen>1 kernel, so the block goes to the gluon kernel's 4-D
# MTP entry, which serves a whole (1 + num_spec) block in one launch.
# The block is causal -- the target is checking draft tokens, so
# position t must not see t+1 -- and the kernel bounds each query
# position's scores itself.
# Arch, mode and dtype gating all live in use_gluon_verify, so that the
# builder -- which has to know whether the asm decode will run -- sees
# the same answer as this branch.
if AiterMLAHelper.use_gluon_verify(
self.num_heads, int(decode.max_qo_len), self.kv_cache_dtype
):
qlen = int(decode.max_qo_len)
if type(q) is tuple:
q_nope, q_pe = q
else:
@@ -1419,56 +1437,39 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
device=q_nope.device,
)
kv_buffer = kv_c_and_k_pe_cache.reshape(-1, kv_c_and_k_pe_cache.shape[-1])
# Expand per-request paged-KV to per-verify-token. Row r*qlen+t is
# request r's verify token t, and seq_lens counts the tokens
# scheduled in this step, so a request's KV range already spans its
# whole verify block and context_r = seq_len_r - qlen. Token t may
# attend to [0, context_r + t], i.e. seq_len_r - (qlen - 1) + t
# entries. paged_kv_indices lists a request's pages in ascending
# position order, so each row's causal window is a prefix of that
# request's slice and only the row length changes. Rows clamp to
# zero for cudagraph padding requests, whose seq_len is 0. Fully
# vectorized (no host loop).
old_indptr = decode.paged_kv_indptr
per_req_len = old_indptr[1:] - old_indptr[:-1]
dev = q_nope.device
row_req = torch.arange(per_req_len.shape[0], device=dev).repeat_interleave(
qlen
assert attn_metadata.causal, (
"AITER MLA small-head verify MTP is causal-only"
)
row_len = (
(
per_req_len.unsqueeze(1)
- (qlen - 1)
+ torch.arange(qlen, device=dev, dtype=per_req_len.dtype)
# Hand mla_gluon its 4-D MTP entry instead of an expanded
# per-verify-token paged-KV view. The flat query layout is already
# row-major (r * qlen + t), so unflatten is a free view. mla_gluon
# applies the per-position causal bound
# score_end = min(split_kv_end, seq_len - qlen + q_pos + 1)
# in-kernel, and seq_lens already spans the verify block, so that
# bound is context_r + q_pos + 1 -- the same window the expanded
# view supplied by truncating each row's page list.
qlen = int(decode.max_qo_len)
num_reqs = B // qlen
if num_reqs * qlen != B:
raise ValueError(
f"verify block {B} rows is not a multiple of qlen {qlen}"
)
.clamp_(min=0)
.flatten()
)
new_indptr = torch.cat([old_indptr.new_zeros(1), row_len.cumsum(0)]).to(
torch.int32
)
total = int(new_indptr[-1].item())
within = torch.arange(total, device=dev, dtype=torch.int64) - new_indptr[
:-1
].to(torch.int64).repeat_interleave(row_len)
src = (
old_indptr[row_req].to(torch.int64).repeat_interleave(row_len) + within
)
new_indices = decode.paged_kv_indices[src]
assert decode.paged_kv_indptr is not None
assert decode.paged_kv_indices is not None
mla_gluon = _get_mla_gluon()
mla_gluon(
q_nope=q_nope,
q_pe=q_pe,
q_nope=q_nope.unflatten(0, (num_reqs, qlen)),
q_pe=q_pe.unflatten(0, (num_reqs, qlen)),
kv_c=kv_buffer,
o=o,
page_table=new_indices,
seq_info=new_indptr,
o=o.unflatten(0, (num_reqs, qlen)),
page_table=decode.paged_kv_indices,
seq_info=decode.paged_kv_indptr,
sm_scale=self.scale,
k_pe=None,
kv_pe_offset=self.kv_lora_rank,
use_2d_view=False,
kv_scale=1.0,
min_kv_seq_len=int(row_len.min()),
min_kv_seq_len=decode.min_kv_seq_len,
)
return o, None
@@ -6,6 +6,7 @@ from typing import ClassVar
import torch
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import (
@@ -25,6 +26,7 @@ from vllm.v1.attention.backend import (
MultipleOf,
)
from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd
from vllm.v1.kv_cache_interface import KVCacheSpec
from vllm.v1.worker.workspace import (
current_workspace_manager,
is_workspace_manager_initialized,
@@ -55,6 +57,34 @@ class TritonMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
# forward_mqa, so no intra-block causal masking is required.
supports_non_causal_multi_token_decode: ClassVar[bool] = True
@classmethod
def get_cudagraph_support(
cls,
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> AttentionCGSupport:
"""Report UNIFORM_BATCH where a non-causal multi-token block is served.
``_cudagraph_support`` is a class constant, so serving the DSpark
draft's (1 + num_spec) block through the decode path reports
UNIFORM_SINGLE_TOKEN_DECODE and, because the engine takes the minimum
over all attention groups, downgrades the *whole* engine off full
cudagraphs. ``forward_mqa`` flattens that block with
``repeat_interleave`` on a Python int and performs no device->host
sync, so it does satisfy the UNIFORM_BATCH contract.
``non_causal_multi_token_decode`` is a KV-cache-group property, not a
per-layer one: ``MLAAttentionSpec.merge`` ORs it over every layer in
the group, so a group holding both a draft and its target reports it
for both. That is the same predicate ``__init__`` below already uses to
raise ``reorder_batch_threshold``, so the two stay consistent, but it
does mean this lifts a causal target sharing the draft's KV cache group
as well.
"""
if getattr(kv_cache_spec, "non_causal_multi_token_decode", False):
return AttentionCGSupport.UNIFORM_BATCH
return cls._cudagraph_support
def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
# DCP local sequence lengths are not advanced between draft steps.