[ROCm][MLA] Mask the AITER MLA small-head verify flatten causally (#50476)
Signed-off-by: yudigege86 <naqin@amd.com> Co-authored-by: yudigege86 <naqin@amd.com>
This commit is contained in:
@@ -0,0 +1,257 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for causal masking in the AITER MLA verify flatten.
|
||||
|
||||
When ``num_heads < 16`` the ROCm AITER MLA backend cannot use the ASM decode
|
||||
kernel for a multi-token block, so ``AiterMLAImpl.forward_mqa`` flattens a
|
||||
``1 + num_speculative_tokens`` verify block into that many single-token Gluon
|
||||
decodes. Every resulting row used to be handed its request's entire paged-KV
|
||||
range. ``paged_kv_indptr`` is the cumulative sum of ``seq_lens``, which counts
|
||||
the tokens scheduled in the current step, so that range already spans the
|
||||
verify block: each verify position could attend to the draft tokens after it,
|
||||
which are the tokens it is supposed to be checking. Nothing crashes and nothing
|
||||
warns, so the only way to catch it is to look at the KV window per row.
|
||||
|
||||
The test drives the real builder and the real ``forward_mqa`` over a
|
||||
multi-token decode batch, intercepts the ``(page_table, seq_info)`` actually
|
||||
handed to the Gluon kernel, and pins verify row ``t`` of request ``r`` to the
|
||||
``context_r + t + 1`` entries of that request's ascending slice.
|
||||
"""
|
||||
|
||||
import types
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _on_rocm_with_aiter() -> bool:
|
||||
return current_platform.is_rocm() and is_aiter_found()
|
||||
|
||||
|
||||
# Unlike the fp8 persistent-decode metadata this file's neighbour guards, the
|
||||
# verify flatten is not gfx950-only: it is selected by head count alone and is
|
||||
# reached on any ROCm AITER MLA deployment with fewer than 16 heads per rank.
|
||||
# The Gluon kernel is replaced by a spy here, so nothing below needs it to be
|
||||
# runnable either.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _on_rocm_with_aiter(),
|
||||
reason="ROCM_AITER_MLA verify flatten requires ROCm and AITER",
|
||||
)
|
||||
|
||||
# Below the 16-head threshold that selects the flatten. DeepSeek-V3 / R1 at
|
||||
# TP=16 and Kimi-K3 at TP=8 both land here.
|
||||
NUM_QUERY_HEADS = 12
|
||||
KV_LORA_RANK = 512
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
V_HEAD_DIM = 128
|
||||
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM
|
||||
|
||||
# 1 + num_speculative_tokens, i.e. the verify block length.
|
||||
QLEN = 8
|
||||
# Committed context per request, including a fresh request at zero context
|
||||
# where the whole KV range is the verify block itself.
|
||||
CONTEXT_LENS = [0, 1, 37, 512]
|
||||
# A cudagraph padding request: seq_len 0, so every one of its rows must clamp
|
||||
# to an empty window.
|
||||
PADDING_ROWS = 1
|
||||
|
||||
# The Gluon path flattens the KV cache to one page per token.
|
||||
PAGE_SIZE = 1
|
||||
MAX_MODEL_LEN = 1024
|
||||
MAX_NUM_SEQS = 8
|
||||
|
||||
|
||||
def _seq_lens() -> list[int]:
|
||||
return [c + QLEN for c in CONTEXT_LENS] + [0] * PADDING_ROWS
|
||||
|
||||
|
||||
def _expected_row_lens() -> list[int]:
|
||||
"""Causal row lengths: row r*QLEN + t sees seq_len_r - (QLEN - 1) + t."""
|
||||
return [max(0, s - (QLEN - 1) + t) for s in _seq_lens() for t in range(QLEN)]
|
||||
|
||||
|
||||
def _run_verify_block():
|
||||
"""Drive the real builder + forward_mqa over one multi-token verify block.
|
||||
|
||||
Returns ``(metadata, captured)`` where ``captured`` holds the ``page_table``,
|
||||
``seq_info`` and ``min_kv_seq_len`` the backend passed to the Gluon kernel.
|
||||
"""
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
seq_lens = _seq_lens()
|
||||
num_reqs = len(seq_lens)
|
||||
# One flat page per token, arange block indices, plus room to spare.
|
||||
num_gpu_blocks = num_reqs * max(seq_lens) + 256
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="deepseek-ai/DeepSeek-R1",
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
block_size=PAGE_SIZE,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
max_num_batched_tokens=8192,
|
||||
hf_config_override={"num_attention_heads": NUM_QUERY_HEADS},
|
||||
)
|
||||
# What raises reorder_batch_threshold to QLEN, so a QLEN-token block is
|
||||
# classified as a decode instead of a prefill. ngram needs no draft model.
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=QLEN - 1
|
||||
)
|
||||
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=PAGE_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=vllm_config.model_config.get_head_size(),
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
cache_dtype_str="auto",
|
||||
)
|
||||
|
||||
backend_cls = AttentionBackendEnum.ROCM_AITER_MLA.get_class()
|
||||
builder_cls = backend_cls.get_builder_cls()
|
||||
impl_cls = backend_cls.get_impl_cls()
|
||||
|
||||
# The builder reads layer.prefill_backend from static_forward_context; a
|
||||
# stub with the attribute is enough for metadata construction.
|
||||
layer_name = "placeholder"
|
||||
vllm_config.compilation_config.static_forward_context[layer_name] = (
|
||||
types.SimpleNamespace(prefill_backend=torch.empty((1,)))
|
||||
)
|
||||
|
||||
init_workspace_manager(device)
|
||||
|
||||
batch_spec = BatchSpec(seq_lens=seq_lens, query_lens=[QLEN] * num_reqs)
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
def spy(**kwargs):
|
||||
captured["page_table"] = kwargs["page_table"].detach().clone()
|
||||
captured["seq_info"] = kwargs["seq_info"].detach().clone()
|
||||
captured["min_kv_seq_len"] = kwargs["min_kv_seq_len"]
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
builder = builder_cls(spec, [layer_name], vllm_config, device)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, PAGE_SIZE, device, arange_block_indices=True
|
||||
)
|
||||
metadata = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
|
||||
impl = impl_cls(
|
||||
num_heads=NUM_QUERY_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
scale=(QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype="auto",
|
||||
logits_soft_cap=None,
|
||||
attn_type="decoder",
|
||||
kv_sharing_target_layer_name=None,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
qk_head_dim=QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM,
|
||||
v_head_dim=V_HEAD_DIM,
|
||||
kv_b_proj=None,
|
||||
)
|
||||
|
||||
num_rows = num_reqs * QLEN
|
||||
dtype = torch.bfloat16
|
||||
q_nope = torch.zeros(
|
||||
num_rows, NUM_QUERY_HEADS, KV_LORA_RANK, dtype=dtype, device=device
|
||||
)
|
||||
q_pe = torch.zeros(
|
||||
num_rows, NUM_QUERY_HEADS, QK_ROPE_HEAD_DIM, dtype=dtype, device=device
|
||||
)
|
||||
kv_cache = torch.zeros(
|
||||
num_gpu_blocks, PAGE_SIZE, HEAD_SIZE, dtype=dtype, device=device
|
||||
)
|
||||
|
||||
# The Gluon kernel only reads the metadata this test is about, so a spy in
|
||||
# its place keeps the assertions independent of the AITER build.
|
||||
with patch(
|
||||
"vllm.v1.attention.backends.mla.rocm_aiter_mla._get_mla_gluon",
|
||||
lambda: spy,
|
||||
):
|
||||
impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None)
|
||||
|
||||
return metadata, captured
|
||||
|
||||
|
||||
def test_verify_flatten_rows_are_causal():
|
||||
"""Verify row t must see the committed prefix plus tokens 0..t, and no more.
|
||||
|
||||
Regression guard: before the fix every row of a request got that request's
|
||||
whole paged-KV range, so verify position t attended to the draft tokens
|
||||
after it. Fails on unmodified upstream.
|
||||
"""
|
||||
metadata, captured = _run_verify_block()
|
||||
|
||||
decode = metadata.decode
|
||||
assert decode is not None, "batch was not classified as a decode"
|
||||
assert decode.max_qo_len == QLEN, (
|
||||
f"expected a {QLEN}-token verify block, got max_qo_len="
|
||||
f"{decode.max_qo_len}; the flatten under test was not reached"
|
||||
)
|
||||
assert captured, "forward_mqa did not reach the Gluon kernel"
|
||||
|
||||
seq_lens = _seq_lens()
|
||||
indptr = captured["seq_info"].tolist()
|
||||
page_table = captured["page_table"]
|
||||
got_row_lens = [indptr[i + 1] - indptr[i] for i in range(len(indptr) - 1)]
|
||||
want_row_lens = _expected_row_lens()
|
||||
|
||||
assert got_row_lens == want_row_lens, (
|
||||
"AITER MLA verify flatten is not causal: verify row r*qlen+t must get "
|
||||
f"seq_len_r - {QLEN - 1} + t KV entries.\n"
|
||||
f" seq_lens {seq_lens}\n"
|
||||
f" got {got_row_lens}\n"
|
||||
f" expected {want_row_lens}\n"
|
||||
"Rows longer than expected let a verify position attend to the draft "
|
||||
"tokens it is supposed to be checking."
|
||||
)
|
||||
|
||||
# arange_block_indices lays request r's pages out ascending from
|
||||
# r * max_blocks, so each causal window must be that slice's prefix.
|
||||
max_blocks = max(seq_lens)
|
||||
for r, seq_len in enumerate(seq_lens):
|
||||
for t in range(QLEN):
|
||||
row = r * QLEN + t
|
||||
window = page_table[indptr[row] : indptr[row + 1]].tolist()
|
||||
n = max(0, seq_len - (QLEN - 1) + t)
|
||||
want = [r * max_blocks + j for j in range(n)]
|
||||
assert window == want, (
|
||||
f"request {r} (seq_len {seq_len}) verify token {t} reads KV "
|
||||
f"pages {window}, expected the ascending causal prefix {want}"
|
||||
)
|
||||
|
||||
padding_rows = got_row_lens[len(CONTEXT_LENS) * QLEN :]
|
||||
assert padding_rows == [0] * (PADDING_ROWS * QLEN), (
|
||||
f"cudagraph padding requests (seq_len 0) must clamp to empty windows, "
|
||||
f"got {padding_rows}"
|
||||
)
|
||||
|
||||
# min_kv_seq_len tells Gluon how short the shortest row is, so it must be
|
||||
# the minimum over the causal rows actually submitted, not over the
|
||||
# per-request lengths they were cut from.
|
||||
assert captured["min_kv_seq_len"] == min(want_row_lens), (
|
||||
f"min_kv_seq_len={captured['min_kv_seq_len']} does not match the "
|
||||
f"shortest submitted row ({min(want_row_lens)})"
|
||||
)
|
||||
@@ -1106,11 +1106,13 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
)
|
||||
return o, None
|
||||
|
||||
# 12-head (<16) non-causal multi-token verify (DSpark): the asm path has
|
||||
# no gqa<16, qseqlen>1 kernel. Flatten each verify token to a qseqlen=1
|
||||
# gluon decode over the same committed prefix (non-causal), mirroring the
|
||||
# TRITON_MLA / sparse-backend flatten but on the fast gluon kernel. Each
|
||||
# request's KV metadata is repeated max_qo_len times.
|
||||
# 12-head (<16) multi-token verify (DSpark): the asm path has no
|
||||
# gqa<16, qseqlen>1 kernel. Flatten each verify token to its own
|
||||
# qseqlen=1 gluon decode, mirroring the TRITON_MLA / sparse-backend
|
||||
# flatten but on the fast gluon kernel. The block is causal -- the
|
||||
# target is checking draft tokens, so position t must not see t+1 --
|
||||
# and attention rows are independent, so giving row t the KV range
|
||||
# [0, context + t] is exactly causal multi-token attention.
|
||||
if (
|
||||
self.num_heads < AiterMLAHelper._AITER_MIN_MLA_HEADS
|
||||
and int(decode.max_qo_len) > 1
|
||||
@@ -1131,16 +1133,31 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
device=q_nope.device,
|
||||
)
|
||||
kv_buffer = kv_c_and_k_pe_cache.reshape(-1, kv_c_and_k_pe_cache.shape[-1])
|
||||
# Expand per-request paged-KV to per-verify-token: each request's KV
|
||||
# block is repeated qlen times (row r*qlen+t attends request r's
|
||||
# committed prefix). Fully vectorized (no host loop).
|
||||
# Expand per-request paged-KV to per-verify-token. Row r*qlen+t is
|
||||
# request r's verify token t, and seq_lens counts the tokens
|
||||
# scheduled in this step, so a request's KV range already spans its
|
||||
# whole verify block and context_r = seq_len_r - qlen. Token t may
|
||||
# attend to [0, context_r + t], i.e. seq_len_r - (qlen - 1) + t
|
||||
# entries. paged_kv_indices lists a request's pages in ascending
|
||||
# position order, so each row's causal window is a prefix of that
|
||||
# request's slice and only the row length changes. Rows clamp to
|
||||
# zero for cudagraph padding requests, whose seq_len is 0. Fully
|
||||
# vectorized (no host loop).
|
||||
old_indptr = decode.paged_kv_indptr
|
||||
per_req_len = old_indptr[1:] - old_indptr[:-1]
|
||||
dev = q_nope.device
|
||||
row_req = torch.arange(per_req_len.shape[0], device=dev).repeat_interleave(
|
||||
qlen
|
||||
)
|
||||
row_len = per_req_len[row_req]
|
||||
row_len = (
|
||||
(
|
||||
per_req_len.unsqueeze(1)
|
||||
- (qlen - 1)
|
||||
+ torch.arange(qlen, device=dev, dtype=per_req_len.dtype)
|
||||
)
|
||||
.clamp_(min=0)
|
||||
.flatten()
|
||||
)
|
||||
new_indptr = torch.cat([old_indptr.new_zeros(1), row_len.cumsum(0)]).to(
|
||||
torch.int32
|
||||
)
|
||||
@@ -1165,7 +1182,7 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
kv_pe_offset=self.kv_lora_rank,
|
||||
use_2d_view=False,
|
||||
kv_scale=1.0,
|
||||
min_kv_seq_len=int(per_req_len.min()),
|
||||
min_kv_seq_len=int(row_len.min()),
|
||||
)
|
||||
return o, None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user