[ROCm][MLA] Reach FULL cudagraphs for AITER MLA speculative decoding (#51171)
Signed-off-by: yudigege86 <naqin@amd.com> Co-authored-by: yudigege86 <naqin@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
yudigege86
Cursor
Andreas Karatzas
parent
b5707bf994
commit
dbf662c9e8
@@ -1,21 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for causal masking in the AITER MLA verify flatten.
|
||||
"""Regression test for causal masking in the AITER MLA verify MTP path.
|
||||
|
||||
When ``num_heads < 16`` the ROCm AITER MLA backend cannot use the ASM decode
|
||||
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` flattens a
|
||||
``1 + num_speculative_tokens`` verify block into that many single-token Gluon
|
||||
decodes. Every resulting row used to be handed its request's entire paged-KV
|
||||
range. ``paged_kv_indptr`` is the cumulative sum of ``seq_lens``, which counts
|
||||
the tokens scheduled in the current step, so that range already spans the
|
||||
verify block: each verify position could attend to the draft tokens after it,
|
||||
which are the tokens it is supposed to be checking. Nothing crashes and nothing
|
||||
warns, so the only way to catch it is to look at the KV window per row.
|
||||
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` routes verify
|
||||
through ``mla_gluon``'s native 4-D MTP entry (``q`` shaped
|
||||
``[batch, qlen, nhead, dim]`` with ``use_2d_view=False``). aiter applies the
|
||||
causal tail internally: position ``t`` attends KV ``[0, seq_len - qlen + t]``.
|
||||
|
||||
The test drives the real builder and the real ``forward_mqa`` over a
|
||||
multi-token decode batch, intercepts the ``(page_table, seq_info)`` actually
|
||||
handed to the Gluon kernel, and pins verify row ``t`` of request ``r`` to the
|
||||
``context_r + t + 1`` entries of that request's ascending slice.
|
||||
The test drives the real builder and ``forward_mqa`` over a multi-token decode
|
||||
batch, intercepts the arguments actually handed to the Gluon kernel, and pins
|
||||
that the 4-D MTP contract is used with per-request ``block_table`` and
|
||||
``cache_seqlens`` rather than a flattened per-token paged-KV view.
|
||||
"""
|
||||
|
||||
import types
|
||||
@@ -32,15 +28,15 @@ def _on_rocm_with_aiter() -> bool:
|
||||
return current_platform.is_rocm() and is_aiter_found()
|
||||
|
||||
|
||||
# The production verify flatten is gfx950-only. This test replaces the Gluon
|
||||
# The production verify MTP path is gfx950-only. This test replaces the Gluon
|
||||
# kernel with a spy and forces the architecture feature probe so the metadata
|
||||
# regression remains testable on every ROCm AITER CI runner.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _on_rocm_with_aiter(),
|
||||
reason="ROCM_AITER_MLA verify flatten requires ROCm and AITER",
|
||||
reason="ROCM_AITER_MLA verify MTP requires ROCm and AITER",
|
||||
)
|
||||
|
||||
# Below the 16-head threshold that selects the flatten. DeepSeek-V3 / R1 at
|
||||
# Below the 16-head threshold that selects the MTP path. DeepSeek-V3 / R1 at
|
||||
# TP=16 and Kimi-K3 at TP=8 both land here.
|
||||
NUM_QUERY_HEADS = 12
|
||||
KV_LORA_RANK = 512
|
||||
@@ -54,8 +50,7 @@ QLEN = 8
|
||||
# Committed context per request, including a fresh request at zero context
|
||||
# where the whole KV range is the verify block itself.
|
||||
CONTEXT_LENS = [0, 1, 37, 512]
|
||||
# A cudagraph padding request: seq_len 0, so every one of its rows must clamp
|
||||
# to an empty window.
|
||||
# A cudagraph padding request: seq_len 0.
|
||||
PADDING_ROWS = 1
|
||||
|
||||
# The Gluon path flattens the KV cache to one page per token.
|
||||
@@ -68,16 +63,10 @@ def _seq_lens() -> list[int]:
|
||||
return [c + QLEN for c in CONTEXT_LENS] + [0] * PADDING_ROWS
|
||||
|
||||
|
||||
def _expected_row_lens() -> list[int]:
|
||||
"""Causal row lengths: row r*QLEN + t sees seq_len_r - (QLEN - 1) + t."""
|
||||
return [max(0, s - (QLEN - 1) + t) for s in _seq_lens() for t in range(QLEN)]
|
||||
|
||||
|
||||
def _run_verify_block():
|
||||
"""Drive the real builder + forward_mqa over one multi-token verify block.
|
||||
|
||||
Returns ``(metadata, captured)`` where ``captured`` holds the ``page_table``,
|
||||
``seq_info`` and ``min_kv_seq_len`` the backend passed to the Gluon kernel.
|
||||
Returns ``(metadata, captured)`` where ``captured`` holds the Gluon kwargs.
|
||||
"""
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
@@ -144,9 +133,10 @@ def _run_verify_block():
|
||||
captured: dict = {}
|
||||
|
||||
def spy(**kwargs):
|
||||
captured.update(kwargs)
|
||||
captured["q_nope"] = kwargs["q_nope"].detach().clone()
|
||||
captured["page_table"] = kwargs["page_table"].detach().clone()
|
||||
captured["seq_info"] = kwargs["seq_info"].detach().clone()
|
||||
captured["min_kv_seq_len"] = kwargs["min_kv_seq_len"]
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
builder = builder_cls(spec, [layer_name], vllm_config, device)
|
||||
@@ -206,12 +196,12 @@ def _run_verify_block():
|
||||
return metadata, captured
|
||||
|
||||
|
||||
def test_verify_flatten_rows_are_causal():
|
||||
"""Verify row t must see the committed prefix plus tokens 0..t, and no more.
|
||||
def test_verify_mtp_uses_native_4d_gluon_entry():
|
||||
"""Verify uses 4-D q with per-request paged_kv metadata.
|
||||
|
||||
Regression guard: before the fix every row of a request got that request's
|
||||
whole paged-KV range, so verify position t attended to the draft tokens
|
||||
after it. Fails on unmodified upstream.
|
||||
Regression guard: before the fix every flattened row of a request got that
|
||||
request's whole paged-KV range, so verify position t attended to the draft
|
||||
tokens after it. The native MTP path delegates causal masking to aiter.
|
||||
"""
|
||||
metadata, captured = _run_verify_block()
|
||||
|
||||
@@ -219,50 +209,37 @@ def test_verify_flatten_rows_are_causal():
|
||||
assert decode is not None, "batch was not classified as a decode"
|
||||
assert decode.max_qo_len == QLEN, (
|
||||
f"expected a {QLEN}-token verify block, got max_qo_len="
|
||||
f"{decode.max_qo_len}; the flatten under test was not reached"
|
||||
f"{decode.max_qo_len}; the MTP path under test was not reached"
|
||||
)
|
||||
assert captured, "forward_mqa did not reach the Gluon kernel"
|
||||
|
||||
seq_lens = _seq_lens()
|
||||
indptr = captured["seq_info"].tolist()
|
||||
num_reqs = len(seq_lens)
|
||||
|
||||
q_nope = captured["q_nope"]
|
||||
assert q_nope.dim() == 4, (
|
||||
f"verify must pass 4-D q to mla_gluon, got {q_nope.dim()}-D shape "
|
||||
f"{tuple(q_nope.shape)}"
|
||||
)
|
||||
assert q_nope.shape[:2] == (num_reqs, QLEN), (
|
||||
f"expected q shape [num_reqs={num_reqs}, qlen={QLEN}, ...], "
|
||||
f"got {tuple(q_nope.shape)}"
|
||||
)
|
||||
assert q_nope.shape[2] == NUM_QUERY_HEADS
|
||||
|
||||
assert captured["use_2d_view"] is False
|
||||
page_table = captured["page_table"]
|
||||
got_row_lens = [indptr[i + 1] - indptr[i] for i in range(len(indptr) - 1)]
|
||||
want_row_lens = _expected_row_lens()
|
||||
|
||||
assert got_row_lens == want_row_lens, (
|
||||
"AITER MLA verify flatten is not causal: verify row r*qlen+t must get "
|
||||
f"seq_len_r - {QLEN - 1} + t KV entries.\n"
|
||||
f" seq_lens {seq_lens}\n"
|
||||
f" got {got_row_lens}\n"
|
||||
f" expected {want_row_lens}\n"
|
||||
"Rows longer than expected let a verify position attend to the draft "
|
||||
"tokens it is supposed to be checking."
|
||||
seq_info = captured["seq_info"]
|
||||
assert page_table.dim() == 1, (
|
||||
f"verify MTP passes the 1-D paged_kv_indices buffer, got {page_table.dim()}-D"
|
||||
)
|
||||
|
||||
# arange_block_indices lays request r's pages out ascending from
|
||||
# r * max_blocks, so each causal window must be that slice's prefix.
|
||||
max_blocks = max(seq_lens)
|
||||
for r, seq_len in enumerate(seq_lens):
|
||||
for t in range(QLEN):
|
||||
row = r * QLEN + t
|
||||
window = page_table[indptr[row] : indptr[row + 1]].tolist()
|
||||
n = max(0, seq_len - (QLEN - 1) + t)
|
||||
want = [r * max_blocks + j for j in range(n)]
|
||||
assert window == want, (
|
||||
f"request {r} (seq_len {seq_len}) verify token {t} reads KV "
|
||||
f"pages {window}, expected the ascending causal prefix {want}"
|
||||
)
|
||||
|
||||
padding_rows = got_row_lens[len(CONTEXT_LENS) * QLEN :]
|
||||
assert padding_rows == [0] * (PADDING_ROWS * QLEN), (
|
||||
f"cudagraph padding requests (seq_len 0) must clamp to empty windows, "
|
||||
f"got {padding_rows}"
|
||||
assert seq_info.dim() == 1 and seq_info.numel() == num_reqs + 1, (
|
||||
f"verify MTP passes paged_kv_indptr [num_reqs + 1], got shape "
|
||||
f"{tuple(seq_info.shape)}"
|
||||
)
|
||||
|
||||
# min_kv_seq_len tells Gluon how short the shortest row is, so it must be
|
||||
# the minimum over the causal rows actually submitted, not over the
|
||||
# per-request lengths they were cut from.
|
||||
assert captured["min_kv_seq_len"] == min(want_row_lens), (
|
||||
f"min_kv_seq_len={captured['min_kv_seq_len']} does not match the "
|
||||
f"shortest submitted row ({min(want_row_lens)})"
|
||||
assert decode is not None and decode.paged_kv_indptr is not None
|
||||
assert torch.equal(seq_info, decode.paged_kv_indptr[: num_reqs + 1])
|
||||
assert torch.equal(
|
||||
page_table[: int(seq_info[-1].item())],
|
||||
decode.paged_kv_indices[: int(seq_info[-1].item())],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user