[ROCm][MLA] Reach FULL cudagraphs for AITER MLA speculative decoding (#51171)

Signed-off-by: yudigege86 <naqin@amd.com>
Co-authored-by: yudigege86 <naqin@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Matt Qin
2026-08-29 19:49:55 -07:00
committed by GitHub
co-authored by yudigege86 Cursor Andreas Karatzas
parent b5707bf994
commit dbf662c9e8
4 changed files with 180 additions and 126 deletions
@@ -1,21 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for causal masking in the AITER MLA verify flatten.
"""Regression test for causal masking in the AITER MLA verify MTP path.
When ``num_heads < 16`` the ROCm AITER MLA backend cannot use the ASM decode
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` flattens a
``1 + num_speculative_tokens`` verify block into that many single-token Gluon
decodes. Every resulting row used to be handed its request's entire paged-KV
range. ``paged_kv_indptr`` is the cumulative sum of ``seq_lens``, which counts
the tokens scheduled in the current step, so that range already spans the
verify block: each verify position could attend to the draft tokens after it,
which are the tokens it is supposed to be checking. Nothing crashes and nothing
warns, so the only way to catch it is to look at the KV window per row.
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` routes verify
through ``mla_gluon``'s native 4-D MTP entry (``q`` shaped
``[batch, qlen, nhead, dim]`` with ``use_2d_view=False``). aiter applies the
causal tail internally: position ``t`` attends KV ``[0, seq_len - qlen + t]``.
The test drives the real builder and the real ``forward_mqa`` over a
multi-token decode batch, intercepts the ``(page_table, seq_info)`` actually
handed to the Gluon kernel, and pins verify row ``t`` of request ``r`` to the
``context_r + t + 1`` entries of that request's ascending slice.
The test drives the real builder and ``forward_mqa`` over a multi-token decode
batch, intercepts the arguments actually handed to the Gluon kernel, and pins
that the 4-D MTP contract is used with per-request ``block_table`` and
``cache_seqlens`` rather than a flattened per-token paged-KV view.
"""
import types
@@ -32,15 +28,15 @@ def _on_rocm_with_aiter() -> bool:
return current_platform.is_rocm() and is_aiter_found()
# The production verify flatten is gfx950-only. This test replaces the Gluon
# The production verify MTP path is gfx950-only. This test replaces the Gluon
# kernel with a spy and forces the architecture feature probe so the metadata
# regression remains testable on every ROCm AITER CI runner.
pytestmark = pytest.mark.skipif(
not _on_rocm_with_aiter(),
reason="ROCM_AITER_MLA verify flatten requires ROCm and AITER",
reason="ROCM_AITER_MLA verify MTP requires ROCm and AITER",
)
# Below the 16-head threshold that selects the flatten. DeepSeek-V3 / R1 at
# Below the 16-head threshold that selects the MTP path. DeepSeek-V3 / R1 at
# TP=16 and Kimi-K3 at TP=8 both land here.
NUM_QUERY_HEADS = 12
KV_LORA_RANK = 512
@@ -54,8 +50,7 @@ QLEN = 8
# Committed context per request, including a fresh request at zero context
# where the whole KV range is the verify block itself.
CONTEXT_LENS = [0, 1, 37, 512]
# A cudagraph padding request: seq_len 0, so every one of its rows must clamp
# to an empty window.
# A cudagraph padding request: seq_len 0.
PADDING_ROWS = 1
# The Gluon path flattens the KV cache to one page per token.
@@ -68,16 +63,10 @@ def _seq_lens() -> list[int]:
return [c + QLEN for c in CONTEXT_LENS] + [0] * PADDING_ROWS
def _expected_row_lens() -> list[int]:
"""Causal row lengths: row r*QLEN + t sees seq_len_r - (QLEN - 1) + t."""
return [max(0, s - (QLEN - 1) + t) for s in _seq_lens() for t in range(QLEN)]
def _run_verify_block():
"""Drive the real builder + forward_mqa over one multi-token verify block.
Returns ``(metadata, captured)`` where ``captured`` holds the ``page_table``,
``seq_info`` and ``min_kv_seq_len`` the backend passed to the Gluon kernel.
Returns ``(metadata, captured)`` where ``captured`` holds the Gluon kwargs.
"""
from tests.v1.attention.utils import (
BatchSpec,
@@ -144,9 +133,10 @@ def _run_verify_block():
captured: dict = {}
def spy(**kwargs):
captured.update(kwargs)
captured["q_nope"] = kwargs["q_nope"].detach().clone()
captured["page_table"] = kwargs["page_table"].detach().clone()
captured["seq_info"] = kwargs["seq_info"].detach().clone()
captured["min_kv_seq_len"] = kwargs["min_kv_seq_len"]
with set_current_vllm_config(vllm_config):
builder = builder_cls(spec, [layer_name], vllm_config, device)
@@ -206,12 +196,12 @@ def _run_verify_block():
return metadata, captured
def test_verify_flatten_rows_are_causal():
"""Verify row t must see the committed prefix plus tokens 0..t, and no more.
def test_verify_mtp_uses_native_4d_gluon_entry():
"""Verify uses 4-D q with per-request paged_kv metadata.
Regression guard: before the fix every row of a request got that request's
whole paged-KV range, so verify position t attended to the draft tokens
after it. Fails on unmodified upstream.
Regression guard: before the fix every flattened row of a request got that
request's whole paged-KV range, so verify position t attended to the draft
tokens after it. The native MTP path delegates causal masking to aiter.
"""
metadata, captured = _run_verify_block()
@@ -219,50 +209,37 @@ def test_verify_flatten_rows_are_causal():
assert decode is not None, "batch was not classified as a decode"
assert decode.max_qo_len == QLEN, (
f"expected a {QLEN}-token verify block, got max_qo_len="
f"{decode.max_qo_len}; the flatten under test was not reached"
f"{decode.max_qo_len}; the MTP path under test was not reached"
)
assert captured, "forward_mqa did not reach the Gluon kernel"
seq_lens = _seq_lens()
indptr = captured["seq_info"].tolist()
num_reqs = len(seq_lens)
q_nope = captured["q_nope"]
assert q_nope.dim() == 4, (
f"verify must pass 4-D q to mla_gluon, got {q_nope.dim()}-D shape "
f"{tuple(q_nope.shape)}"
)
assert q_nope.shape[:2] == (num_reqs, QLEN), (
f"expected q shape [num_reqs={num_reqs}, qlen={QLEN}, ...], "
f"got {tuple(q_nope.shape)}"
)
assert q_nope.shape[2] == NUM_QUERY_HEADS
assert captured["use_2d_view"] is False
page_table = captured["page_table"]
got_row_lens = [indptr[i + 1] - indptr[i] for i in range(len(indptr) - 1)]
want_row_lens = _expected_row_lens()
assert got_row_lens == want_row_lens, (
"AITER MLA verify flatten is not causal: verify row r*qlen+t must get "
f"seq_len_r - {QLEN - 1} + t KV entries.\n"
f" seq_lens {seq_lens}\n"
f" got {got_row_lens}\n"
f" expected {want_row_lens}\n"
"Rows longer than expected let a verify position attend to the draft "
"tokens it is supposed to be checking."
seq_info = captured["seq_info"]
assert page_table.dim() == 1, (
f"verify MTP passes the 1-D paged_kv_indices buffer, got {page_table.dim()}-D"
)
# arange_block_indices lays request r's pages out ascending from
# r * max_blocks, so each causal window must be that slice's prefix.
max_blocks = max(seq_lens)
for r, seq_len in enumerate(seq_lens):
for t in range(QLEN):
row = r * QLEN + t
window = page_table[indptr[row] : indptr[row + 1]].tolist()
n = max(0, seq_len - (QLEN - 1) + t)
want = [r * max_blocks + j for j in range(n)]
assert window == want, (
f"request {r} (seq_len {seq_len}) verify token {t} reads KV "
f"pages {window}, expected the ascending causal prefix {want}"
)
padding_rows = got_row_lens[len(CONTEXT_LENS) * QLEN :]
assert padding_rows == [0] * (PADDING_ROWS * QLEN), (
f"cudagraph padding requests (seq_len 0) must clamp to empty windows, "
f"got {padding_rows}"
assert seq_info.dim() == 1 and seq_info.numel() == num_reqs + 1, (
f"verify MTP passes paged_kv_indptr [num_reqs + 1], got shape "
f"{tuple(seq_info.shape)}"
)
# min_kv_seq_len tells Gluon how short the shortest row is, so it must be
# the minimum over the causal rows actually submitted, not over the
# per-request lengths they were cut from.
assert captured["min_kv_seq_len"] == min(want_row_lens), (
f"min_kv_seq_len={captured['min_kv_seq_len']} does not match the "
f"shortest submitted row ({min(want_row_lens)})"
assert decode is not None and decode.paged_kv_indptr is not None
assert torch.equal(seq_info, decode.paged_kv_indptr[: num_reqs + 1])
assert torch.equal(
page_table[: int(seq_info[-1].item())],
decode.paged_kv_indices[: int(seq_info[-1].item())],
)