[ROCm][MLA] Reach FULL cudagraphs for AITER MLA speculative decoding (#51171)

Signed-off-by: yudigege86 <naqin@amd.com>
Co-authored-by: yudigege86 <naqin@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Matt Qin
2026-08-29 19:49:55 -07:00
committed by GitHub
co-authored by yudigege86 Cursor Andreas Karatzas
parent b5707bf994
commit dbf662c9e8
4 changed files with 180 additions and 126 deletions
@@ -729,7 +729,6 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
block_table_tensor,
block_table_tensor.stride(0),
paged_kv_indptr,
seq_lens_for_kernel,
KERNEL_BLOCK_SIZE=self.kernel_block_size,
BLOCK_SIZE=1024,
)
@@ -834,6 +833,26 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
)
has_persistent_metadata = True
# Small-head multi-token verify uses mla_gluon's 4-D MTP entry over the
# ordinary per-request paged-KV view, so there is no expanded per-token
# buffer to build here. mla_gluon still wants a lower bound on the KV
# length it is asked to split; on the verify path that bound is over
# active requests, not cudagraph padding rows pinned to max_qo_len.
# The .item() below runs in the builder, outside the captured region,
# so it does not abort HIP graph capture the way the per-layer syncs
# in forward_mqa did.
min_kv_seq_len = 1
if AiterMLAHelper.use_gluon_verify(
self.num_heads, int(max_qo_len), self._kv_cache_dtype_str
):
per_req_len = paged_kv_indptr[1:] - paged_kv_indptr[:-1]
if pad_uniform_mtp:
active = qo_lens_device > 0
if active.any():
min_kv_seq_len = int(per_req_len[active].min().item())
else:
min_kv_seq_len = int(per_req_len.min().item())
attn_metadata = AiterMLADecodeMetadata(
block_table=block_table_tensor,
seq_lens=seq_lens_for_kernel,
@@ -843,6 +862,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
qo_indptr=qo_indptr,
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
max_qo_len=max_qo_len,
min_kv_seq_len=min_kv_seq_len,
use_gluon_decode=use_gluon_decode,
attn_out_dtype=self.decode_attn_out_dtype,
has_persistent_metadata=has_persistent_metadata,
@@ -884,7 +904,6 @@ def _expand_page_indices_kernel(
block_table,
block_table_stride,
cu_num_tokens,
seq_lens,
KERNEL_BLOCK_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
@@ -904,7 +923,7 @@ def _expand_page_indices_kernel(
req_idx = tl.program_id(0)
row_ptr = block_table + req_idx * block_table_stride
start_idx = tl.load(cu_num_tokens + req_idx)
num_tokens = tl.load(seq_lens + req_idx)
num_tokens = tl.load(cu_num_tokens + req_idx + 1) - start_idx
offset = tl.arange(0, BLOCK_SIZE)
for i in tl.range(0, num_tokens, BLOCK_SIZE):
@@ -1056,11 +1075,12 @@ class AiterMLAHelper:
@staticmethod
def use_gluon_verify(num_heads: int, max_qo_len: int, kv_cache_dtype: str) -> bool:
"""Whether a small-head multi-token verify is flattened onto Gluon.
"""Whether a small-head multi-token verify uses native Gluon MTP.
bf16 has no gqa<16, qseqlen>1 asm kernel, so the verify is flattened
into per-token Gluon decodes. fp8 has one via the q-row fold and must
not come here: the flatten hands Gluon the batch size its fp8 regime
bf16 has no gqa<16, qseqlen>1 asm kernel, so verify goes through
``mla_gluon``'s 4-D MTP entry (``q`` shaped ``[batch, qlen, nhead, dim]``)
with ``use_2d_view=False``. fp8 has one via the q-row fold and must not
come here: the MTP path hands Gluon the batch size its fp8 regime
asserts against. A predicate rather than inline in forward_mqa so the
builder sees the same answer the impl acts on.
"""
@@ -1391,19 +1411,17 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
return o, None
# 12-head (<16) multi-token verify (DSpark): the asm path has no
# gqa<16, qseqlen>1 kernel. Flatten each verify token to its own
# qseqlen=1 gluon decode, mirroring the TRITON_MLA / sparse-backend
# flatten but on the fast gluon kernel. The block is causal -- the
# target is checking draft tokens, so position t must not see t+1 --
# and attention rows are independent, so giving row t the KV range
# [0, context + t] is exactly causal multi-token attention.
# gqa<16, qseqlen>1 kernel, so the block goes to the gluon kernel's 4-D
# MTP entry, which serves a whole (1 + num_spec) block in one launch.
# The block is causal -- the target is checking draft tokens, so
# position t must not see t+1 -- and the kernel bounds each query
# position's scores itself.
# Arch, mode and dtype gating all live in use_gluon_verify, so that the
# builder -- which has to know whether the asm decode will run -- sees
# the same answer as this branch.
if AiterMLAHelper.use_gluon_verify(
self.num_heads, int(decode.max_qo_len), self.kv_cache_dtype
):
qlen = int(decode.max_qo_len)
if type(q) is tuple:
q_nope, q_pe = q
else:
@@ -1419,56 +1437,39 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
device=q_nope.device,
)
kv_buffer = kv_c_and_k_pe_cache.reshape(-1, kv_c_and_k_pe_cache.shape[-1])
# Expand per-request paged-KV to per-verify-token. Row r*qlen+t is
# request r's verify token t, and seq_lens counts the tokens
# scheduled in this step, so a request's KV range already spans its
# whole verify block and context_r = seq_len_r - qlen. Token t may
# attend to [0, context_r + t], i.e. seq_len_r - (qlen - 1) + t
# entries. paged_kv_indices lists a request's pages in ascending
# position order, so each row's causal window is a prefix of that
# request's slice and only the row length changes. Rows clamp to
# zero for cudagraph padding requests, whose seq_len is 0. Fully
# vectorized (no host loop).
old_indptr = decode.paged_kv_indptr
per_req_len = old_indptr[1:] - old_indptr[:-1]
dev = q_nope.device
row_req = torch.arange(per_req_len.shape[0], device=dev).repeat_interleave(
qlen
assert attn_metadata.causal, (
"AITER MLA small-head verify MTP is causal-only"
)
row_len = (
(
per_req_len.unsqueeze(1)
- (qlen - 1)
+ torch.arange(qlen, device=dev, dtype=per_req_len.dtype)
# Hand mla_gluon its 4-D MTP entry instead of an expanded
# per-verify-token paged-KV view. The flat query layout is already
# row-major (r * qlen + t), so unflatten is a free view. mla_gluon
# applies the per-position causal bound
# score_end = min(split_kv_end, seq_len - qlen + q_pos + 1)
# in-kernel, and seq_lens already spans the verify block, so that
# bound is context_r + q_pos + 1 -- the same window the expanded
# view supplied by truncating each row's page list.
qlen = int(decode.max_qo_len)
num_reqs = B // qlen
if num_reqs * qlen != B:
raise ValueError(
f"verify block {B} rows is not a multiple of qlen {qlen}"
)
.clamp_(min=0)
.flatten()
)
new_indptr = torch.cat([old_indptr.new_zeros(1), row_len.cumsum(0)]).to(
torch.int32
)
total = int(new_indptr[-1].item())
within = torch.arange(total, device=dev, dtype=torch.int64) - new_indptr[
:-1
].to(torch.int64).repeat_interleave(row_len)
src = (
old_indptr[row_req].to(torch.int64).repeat_interleave(row_len) + within
)
new_indices = decode.paged_kv_indices[src]
assert decode.paged_kv_indptr is not None
assert decode.paged_kv_indices is not None
mla_gluon = _get_mla_gluon()
mla_gluon(
q_nope=q_nope,
q_pe=q_pe,
q_nope=q_nope.unflatten(0, (num_reqs, qlen)),
q_pe=q_pe.unflatten(0, (num_reqs, qlen)),
kv_c=kv_buffer,
o=o,
page_table=new_indices,
seq_info=new_indptr,
o=o.unflatten(0, (num_reqs, qlen)),
page_table=decode.paged_kv_indices,
seq_info=decode.paged_kv_indptr,
sm_scale=self.scale,
k_pe=None,
kv_pe_offset=self.kv_lora_rank,
use_2d_view=False,
kv_scale=1.0,
min_kv_seq_len=int(row_len.min()),
min_kv_seq_len=decode.min_kv_seq_len,
)
return o, None
@@ -6,6 +6,7 @@ from typing import ClassVar
import torch
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.mla_attention import (
@@ -25,6 +26,7 @@ from vllm.v1.attention.backend import (
MultipleOf,
)
from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd
from vllm.v1.kv_cache_interface import KVCacheSpec
from vllm.v1.worker.workspace import (
current_workspace_manager,
is_workspace_manager_initialized,
@@ -55,6 +57,34 @@ class TritonMLAMetadataBuilder(MLACommonMetadataBuilder[MLACommonMetadata]):
# forward_mqa, so no intra-block causal masking is required.
supports_non_causal_multi_token_decode: ClassVar[bool] = True
@classmethod
def get_cudagraph_support(
cls,
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> AttentionCGSupport:
"""Report UNIFORM_BATCH where a non-causal multi-token block is served.
``_cudagraph_support`` is a class constant, so serving the DSpark
draft's (1 + num_spec) block through the decode path reports
UNIFORM_SINGLE_TOKEN_DECODE and, because the engine takes the minimum
over all attention groups, downgrades the *whole* engine off full
cudagraphs. ``forward_mqa`` flattens that block with
``repeat_interleave`` on a Python int and performs no device->host
sync, so it does satisfy the UNIFORM_BATCH contract.
``non_causal_multi_token_decode`` is a KV-cache-group property, not a
per-layer one: ``MLAAttentionSpec.merge`` ORs it over every layer in
the group, so a group holding both a draft and its target reports it
for both. That is the same predicate ``__init__`` below already uses to
raise ``reorder_batch_threshold``, so the two stay consistent, but it
does mean this lifts a causal target sharing the draft's KV cache group
as well.
"""
if getattr(kv_cache_spec, "non_causal_multi_token_decode", False):
return AttentionCGSupport.UNIFORM_BATCH
return cls._cudagraph_support
def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
# DCP local sequence lengths are not advanced between draft steps.