[ROCm][MLA] Add an AITER ASM round-robin decode route for DCP multi-token verify (#56861)

Signed-off-by: Xiaohu Guo <Xiaohu.Guo@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: seungrokj <144636725+seungrokj@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
xiaohuguo2023
2026-09-28 14:52:21 -04:00
committed by GitHub
co-authored by Claude Opus 5 seungrokj Cursor
parent 5ba2e3090b
commit 25624f65cf
5 changed files with 1202 additions and 30 deletions
+10
View File
@@ -145,6 +145,7 @@ if TYPE_CHECKING:
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
VLLM_ROCM_USE_AITER_MLA: bool = True
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["asm", "segmented"] = "segmented"
VLLM_ROCM_USE_AITER_MHA: bool = True
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False
VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False
@@ -1309,6 +1310,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ROCM_USE_AITER_MLA": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MLA", "True").lower() in ("true", "1")
),
# Kernel for causal multi-token (spec-decode) verify steps under decode
# context parallelism on gfx950: "asm" uses AITER's round-robin ASM
# decode, "segmented" the Triton segmented MLA path.
"VLLM_ROCM_AITER_MLA_DCP_VERIFY": env_with_choices(
"VLLM_ROCM_AITER_MLA_DCP_VERIFY",
"segmented",
["asm", "segmented"],
case_sensitive=False,
),
# Small-head (<16) AITER MLA decode kernel selection. Small head counts
# (e.g. Kimi-K3: 12 heads/rank at TP8, 6 at TP16) can decode either through
# the Gluon small-head kernel or through the padded persistent-scheduling
+258 -23
View File
@@ -3,11 +3,13 @@
import functools
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING, ClassVar, Final
import torch
from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
@@ -239,6 +241,78 @@ def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) ->
)
# DCP decode routing
#
# With decode context parallelism each rank holds a round-robin shard of the KV
# cache (token t on rank t % dcp_world_size; requires interleave size 1) and
# attends over it with all DCP-gathered query heads. Partial outputs are merged
# across ranks by LSE. Each decode batch takes one route:
#
# * plain: qlen == 1 or a non-causal block. Every query sees every local token.
# * cprr: causal qlen >= _MIN_CPRR_QLEN, gfx950, speculative decoding. AITER's
# round-robin ASM decode rebuilds global positions from g_kv_indptr, cp_rank
# and cp_world_size and applies the causal window in-kernel. Kernels exist only
# at _NATIVE_CPRR_HEADS; other gathered head counts are padded up.
# * segmented: causal qlen > 1 when cprr is not selected or qlen is below
# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV length
# encodes its causal window.
#
# The plain kernel must never serve a causal qlen > 1 block: it applies
# causality on local indices, which is wrong over a round-robin shard.
_NATIVE_CPRR_HEADS: Final = (16, 32, 64, 128)
_MIN_CPRR_QLEN: Final = 3
class _DCPDecodeRoute(Enum):
PLAIN = "plain"
SEGMENTED = "segmented"
CPRR = "cprr"
def _asm_dcp_verify_heads(decode_num_heads: int) -> int:
"""Head count the cprr asm decode runs at, or 0 if it cannot serve this shape."""
if decode_num_heads in _NATIVE_CPRR_HEADS:
return decode_num_heads
for nat in _NATIVE_CPRR_HEADS:
if nat >= decode_num_heads:
return nat
return 0
def _select_dcp_decode_route(
supports_segmented: bool,
causal: bool,
max_qo_len: int,
asm_selected: bool,
) -> _DCPDecodeRoute:
"""Select one DCP decode route for this batch; see DCP decode routing."""
if not causal or max_qo_len <= 1:
return _DCPDecodeRoute.PLAIN
if asm_selected and max_qo_len >= _MIN_CPRR_QLEN:
return _DCPDecodeRoute.CPRR
if supports_segmented:
return _DCPDecodeRoute.SEGMENTED
raise RuntimeError(
"ROCM_AITER_MLA DCP multi-token verify requires either segmented "
"MLA or the round-robin asm decode."
)
def _asm_dcp_verify_configured(
dcp_world_size: int, cp_interleave: int, multi_token_decode: bool
) -> bool:
"""Whether asm cprr DCP verify is reachable in this process.
Requires DCP with interleave size 1, speculative decoding, and gfx950 (the
only arch AITER builds cprr kernels for).
"""
if not (dcp_world_size > 1 and cp_interleave == 1 and multi_token_decode):
return False
from vllm.platforms.rocm import on_gfx950
return bool(on_gfx950())
def _aiter_mla_small_head_mode() -> str:
"""Small-head (<16) MLA decode kernel selection.
@@ -410,8 +484,19 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
max_qo_len: int | None = None
# Minimum KV length used by Gluon to choose a safe split count.
min_kv_seq_len: int = 1
# Exactly one DCP decode route is selected by the metadata builder.
dcp_route: _DCPDecodeRoute = _DCPDecodeRoute.PLAIN
# Set exactly when this batch routes to segmented DCP verification.
dcp_verify: AiterMLADCPVerifyMetadata | None = None
# cprr only: global per-request page indptr and this rank's position.
g_kv_indptr: torch.Tensor | None = None
cp_world_size: int = 1
cp_rank: int = 0
# Padded head count cprr runs at; 0 when this batch is not on cprr.
asm_decode_num_heads: int = 0
# Must equal the split cap the reduce scratch was sized with, or the kernel
# writes past it and faults.
mla_num_kv_splits: int = 0
# Small-head decode uses Gluon (avoids padding to 16).
use_gluon_decode: bool = False
# Small-head non-DCP multi-token verify uses Gluon's native MTP entry.
@@ -460,6 +545,9 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
# Set from the common metadata every build; a batch is causal unless the
# drafter says otherwise.
_decode_causal: bool = True
# A non-causal draft needs no cross-shard causal window: the ordinary
# mask0 decode returns per-row LSE for the DCP merge.
supports_non_causal_multi_token_dcp: ClassVar[bool] = True
@staticmethod
def _uniform_padded_mtp_qo_len(
@@ -513,15 +601,48 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
parallel_config.decode_context_parallel_size,
parallel_config.cp_kv_cache_interleave_size,
)
asm_dcp_verify_config = (
_asm_dcp_verify_configured(
parallel_config.decode_context_parallel_size,
parallel_config.cp_kv_cache_interleave_size,
multi_token_decode=vllm_config.speculative_config is not None,
)
and envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "asm"
)
super().__init__(
kv_cache_spec,
layer_names,
vllm_config,
device,
AiterMLAMetadata,
supports_dcp_with_varlen=supports_segmented_dcp_verify,
supports_dcp_with_varlen=(
supports_segmented_dcp_verify or asm_dcp_verify_config
),
)
self._asm_dcp_verify = False
self._asm_dcp_verify_heads = 0
# The head-count half of the route decision needs self.num_heads, so only
# the config half is known before super().__init__().
if asm_dcp_verify_config:
self._asm_dcp_verify_heads = _asm_dcp_verify_heads(
self.num_heads * self.dcp_world_size
)
if not self._asm_dcp_verify_heads:
raise ValueError(
"VLLM_ROCM_AITER_MLA_DCP_VERIFY=asm, but the round-robin asm "
f"decode has no kernel for {self.num_heads * self.dcp_world_size} "
f"DCP-gathered heads (native counts: {_NATIVE_CPRR_HEADS}). "
"Set VLLM_ROCM_AITER_MLA_DCP_VERIFY=segmented."
)
self._asm_dcp_verify = True
self._supports_segmented_dcp_verify = supports_segmented_dcp_verify
self._mla_max_split_per_batch = 0
if self._asm_dcp_verify:
# Cap on KV splits per batch; at low batch the KV axis is the only
# parallelism. Buffer sizing and every runtime call must see this value.
self._mla_max_split_per_batch = torch.cuda.get_device_properties(
device
).multi_processor_count
self.compilation_config = vllm_config.compilation_config
self.decode_attn_out_dtype = vllm_config.model_config.dtype
@@ -576,6 +697,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads(
self._decode_num_heads
)
# the cprr kernel exists only at the native head
# counts, so run (and size the persistent schedule) at the padded count.
# 96 is 16-aligned, so get_actual_mla_num_heads leaves it alone and the
# pad has to be applied here explicitly.
if self._asm_dcp_verify:
self._num_attention_heads = self._asm_dcp_verify_heads
kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")
if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"):
kv_cache_dtype_str = "fp8"
@@ -619,6 +746,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
kv_dtype,
is_sparse=False,
fast_mode=True,
# Must match max_split_per_batch in get_mla_metadata_v1; without it
# aiter sizes the reduce scratch from capacity (~9 GiB).
**(
dict(max_split_per_batch=self._mla_max_split_per_batch)
if self._asm_dcp_verify
else {}
),
)
self._mla_work_meta_data = torch.empty(
work_meta_data_size, dtype=work_meta_data_type, device=device
@@ -664,6 +798,14 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
device,
)
# Full cudagraphs need a stable address for the cprr global page indptr.
# Zero-initialized; element 0 is the indptr base and is never rewritten.
self._g_kv_indptr_buf: torch.Tensor | None = None
if self._asm_dcp_verify:
self._g_kv_indptr_buf = torch.zeros(
max_num_reqs + 1, dtype=torch.int32, device=device
)
# Persistent buffers for segmented DCP verification. Captured graphs
# require stable addresses while row lengths vary between replays.
self._dcp_verify_buffers: AiterMLADCPVerifyMetadata | None = None
@@ -683,8 +825,11 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
)
if self._supports_segmented_dcp_verify and self._mtp_decode_qlen > 1:
# A DCP rank's shard of the longest sequence bounds every verify
# row, and full graphs need that bound to be constant.
# Allocate even when CPRR is the preferred route: a later
# replay can fall below _MIN_CPRR_QLEN and needs these
# addresses. A DCP rank's shard of the longest sequence
# bounds every verify row, and full graphs need that bound
# to be constant.
num_dcp_partitions = (
self.dcp_world_size * self.cp_kv_cache_interleave_size
)
@@ -1063,6 +1208,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
pad_uniform_mtp = padded_mtp_qo_len > 0
seq_lens_for_kernel = seq_lens_device
# the global lengths need the same dummy-row
# treatment as the local ones below, but only for g_kv_indptr -- the
# metadata field keeps the true values, which is what the segmented
# route wants (it derives its own row lens and gives a dummy row
# length 0, correctly).
g_tot_seq_lens = dcp_tot_seq_lens_device
num_kernel_reqs = num_reqs
if pad_uniform_mtp:
qo_lens_device = (
@@ -1074,6 +1225,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
seq_lens_for_kernel,
seq_lens_for_kernel.new_full((), max_qo_len),
)
# A dummy row just got a LOCAL length of max_qo_len, so its GLOBAL
# length has to say the same thing or the cprr mask is degenerate:
# global_len 0 puts the row query positions at {-3,-2,-1} while its
# local shard sits at {rank, W+rank, 2W+rank}, so every entry is
# masked and the row softmaxes over nothing. max_qo_len * W is
# exactly the global length whose round-robin shard is max_qo_len
# on every rank.
if g_tot_seq_lens is not None:
g_tot_seq_lens = torch.where(
qo_lens_device > 0,
g_tot_seq_lens,
g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size),
)
if self._graph_seq_lens is not None:
self._graph_seq_lens[:num_kernel_reqs].copy_(
@@ -1106,16 +1270,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
causal,
self._kv_cache_bytes,
)
use_segmented_dcp_verify = (
self._supports_segmented_dcp_verify and max_qo_len > 1
)
dcp_route = _DCPDecodeRoute.PLAIN
if self.dcp_world_size > 1:
dcp_route = _select_dcp_decode_route(
self._supports_segmented_dcp_verify,
causal,
int(max_qo_len),
self._asm_dcp_verify,
)
# Segmented DCP verify carries its own per-row subpage table, so the
# flat per-token view is dead work for it. Leave the buffer alone and
# hand the metadata None, so a future reader cannot pick up whatever
# the previous batch left behind.
paged_kv_indices = None
if not use_segmented_dcp_verify:
if dcp_route is not _DCPDecodeRoute.SEGMENTED:
if self.compilation_config.cudagraph_mode.has_full_cudagraphs():
self.paged_kv_indices.fill_(-1)
@@ -1169,6 +1338,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
else:
qo_indptr = query_start_loc_device[: 1 + num_kernel_reqs]
g_kv_indptr = None
if dcp_route is _DCPDecodeRoute.CPRR and g_tot_seq_lens is not None:
assert self._g_kv_indptr_buf is not None
_ngk = g_tot_seq_lens.shape[0]
g_kv_indptr = self._g_kv_indptr_buf[: _ngk + 1]
g_kv_indptr[1:].copy_(g_tot_seq_lens.cumsum(dim=0, dtype=torch.int32))
# A FULL cudagraph replays at its capture-time request count, so a
# smaller real batch leaves the tail holding the PREVIOUS step's
# cumsum. Flatten it the way paged_kv_indptr is flattened below, so
# padding rows read global_len == 0 and agree with their zero-length
# local shard instead of a stale global length.
self._g_kv_indptr_buf[_ngk + 1 :].fill_(g_kv_indptr[-1])
has_persistent_metadata = False
# Only the asm decode consumes the schedule, so gate on the routing
# rather than on num_heads >= 16, which denies it to a padded rank
@@ -1178,7 +1360,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
use_persistent_metadata = (
not use_gluon_decode
and not use_gluon_verify
and not use_segmented_dcp_verify
and dcp_route is not _DCPDecodeRoute.SEGMENTED
# A padded rank has no bf16 persistent kernel past qlen 4 where the
# gfx950 fold is absent; the non-persistent entry covers it. fp8
# keeps the schedule -- its fold rejects non-persistent outright.
@@ -1218,13 +1400,28 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
uni_qo_len = (
max_qo_len if pad_uniform_mtp or torch.all(qo_len == max_qo_len) else -1
)
cprr_kwargs: dict = {}
is_causal = causal
if self._asm_dcp_verify:
cprr_kwargs = dict(
max_split_per_batch=self._mla_max_split_per_batch,
intra_batch_mode=False,
)
if dcp_route is _DCPDecodeRoute.CPRR:
is_causal = False
cprr_kwargs["is_cp_round_robin"] = True
metadata_num_heads = (
self._asm_dcp_verify_heads
if dcp_route is _DCPDecodeRoute.CPRR
else AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads)
)
get_mla_metadata_v1(
qo_indptr,
paged_kv_indptr,
paged_kv_last_page_len,
self._num_attention_heads,
metadata_num_heads,
1,
causal,
is_causal,
self._mla_work_meta_data,
self._mla_work_info_set,
self._mla_work_indptr,
@@ -1238,6 +1435,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
fast_mode=True,
dtype_q=self._mla_q_dtype,
dtype_kv=self._mla_kv_dtype,
**cprr_kwargs,
)
has_persistent_metadata = True
@@ -1261,7 +1459,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
min_kv_seq_len = int(per_req_len.min().item())
dcp_verify = None
if use_segmented_dcp_verify:
if dcp_route is _DCPDecodeRoute.SEGMENTED:
assert dcp_tot_seq_lens_device is not None
dcp_verify = self._build_dcp_verify_row_view(
int(max_qo_len),
@@ -1279,7 +1477,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
max_qo_len=max_qo_len,
min_kv_seq_len=min_kv_seq_len,
dcp_route=dcp_route,
dcp_verify=dcp_verify,
g_kv_indptr=g_kv_indptr,
cp_world_size=self.dcp_world_size,
cp_rank=self.dcp_rank,
asm_decode_num_heads=(
self._asm_dcp_verify_heads if dcp_route is _DCPDecodeRoute.CPRR else 0
),
mla_num_kv_splits=(
self._mla_max_split_per_batch
if dcp_route is _DCPDecodeRoute.CPRR
else 0
),
use_gluon_decode=use_gluon_decode,
use_gluon_verify=use_gluon_verify,
attn_out_dtype=self.decode_attn_out_dtype,
@@ -2062,8 +2272,10 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
)
return o, None
verify = decode.dcp_verify
if verify is not None:
dcp_route = decode.dcp_route
if dcp_route is _DCPDecodeRoute.SEGMENTED:
verify = decode.dcp_verify
assert verify is not None
if type(q) is tuple:
q_nope, q_pe = q
else:
@@ -2085,11 +2297,6 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
decode.attn_out_dtype,
)
if self.dcp_world_size > 1 and int(decode.max_qo_len) > 1:
raise RuntimeError(
"ROCM_AITER_MLA DCP multi-token verify requires segmented MLA."
)
if type(q) is tuple:
q = torch.cat(q, dim=-1)
@@ -2102,8 +2309,19 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
"ROCM_AITER_MLA decode expected the DCP-gathered query head count "
f"{self._decode_num_heads}, got {q.shape[1]}"
)
mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q)
mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads)
# cprr has kernels only at native head counts, so pad q up (e.g. 96 ->
# 128). Heads are independent in MLA; the extras are sliced off below.
asm_dcp_heads = decode.asm_decode_num_heads
if asm_dcp_heads > self._decode_num_heads:
mla_num_heads = asm_dcp_heads
mla_padded_q = AiterMLAHelper.get_mla_padded_q(
self._decode_num_heads, q, target_heads=mla_num_heads
)
else:
mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q)
mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads(
self._decode_num_heads
)
o = torch.empty(
B,
mla_num_heads,
@@ -2137,6 +2355,16 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
lse = None
if self.dcp_world_size > 1:
if dcp_route is _DCPDecodeRoute.CPRR:
# cprr only: global positions for the in-kernel causal window.
assert decode.g_kv_indptr is not None
mla_kwargs["g_kv_indptr"] = decode.g_kv_indptr
mla_kwargs["cp_world_size"] = decode.cp_world_size
mla_kwargs["cp_rank"] = decode.cp_rank
# Must match the schedule get_mla_metadata_v1 was built with.
assert decode.mla_num_kv_splits > 0
mla_kwargs["num_kv_splits"] = decode.mla_num_kv_splits
mla_kwargs["intra_batch_mode"] = False
# The vLLM custom-op wrapper exposes only the in-place output and
# drops aiter's final LSE, which the cross-shard merge needs, so go
# through aiter's native entry point on the DCP path.
@@ -2151,6 +2379,7 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
decode.max_qo_len,
sm_scale=self.scale,
return_lse=True,
causal=attn_metadata.causal,
**mla_kwargs,
)
assert lse is not None, (
@@ -2172,7 +2401,13 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
**mla_kwargs,
)
output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o)
if lse is not None:
lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse)
if asm_dcp_heads > self._decode_num_heads:
# Keep a view: a post-kernel copy races the symm-mem a2a combine.
output = o[:, : self._decode_num_heads]
if lse is not None:
lse = lse[:, : self._decode_num_heads, ...]
else:
output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o)
if lse is not None:
lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse)
return output, lse