[ROCm][MLA] Add an AITER ASM round-robin decode route for DCP multi-token verify (#56861)
Signed-off-by: Xiaohu Guo <Xiaohu.Guo@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: seungrokj <144636725+seungrokj@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Claude Opus 5
seungrokj
Cursor
parent
5ba2e3090b
commit
25624f65cf
@@ -145,6 +145,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
|
||||
VLLM_ROCM_USE_AITER_MLA: bool = True
|
||||
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
|
||||
VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["asm", "segmented"] = "segmented"
|
||||
VLLM_ROCM_USE_AITER_MHA: bool = True
|
||||
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False
|
||||
VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False
|
||||
@@ -1309,6 +1310,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ROCM_USE_AITER_MLA": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MLA", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Kernel for causal multi-token (spec-decode) verify steps under decode
|
||||
# context parallelism on gfx950: "asm" uses AITER's round-robin ASM
|
||||
# decode, "segmented" the Triton segmented MLA path.
|
||||
"VLLM_ROCM_AITER_MLA_DCP_VERIFY": env_with_choices(
|
||||
"VLLM_ROCM_AITER_MLA_DCP_VERIFY",
|
||||
"segmented",
|
||||
["asm", "segmented"],
|
||||
case_sensitive=False,
|
||||
),
|
||||
# Small-head (<16) AITER MLA decode kernel selection. Small head counts
|
||||
# (e.g. Kimi-K3: 12 heads/rank at TP8, 6 at TP16) can decode either through
|
||||
# the Gluon small-head kernel or through the padded persistent-scheduling
|
||||
|
||||
@@ -3,11 +3,13 @@
|
||||
|
||||
import functools
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, ClassVar, Final
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
@@ -239,6 +241,78 @@ def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) ->
|
||||
)
|
||||
|
||||
|
||||
# DCP decode routing
|
||||
#
|
||||
# With decode context parallelism each rank holds a round-robin shard of the KV
|
||||
# cache (token t on rank t % dcp_world_size; requires interleave size 1) and
|
||||
# attends over it with all DCP-gathered query heads. Partial outputs are merged
|
||||
# across ranks by LSE. Each decode batch takes one route:
|
||||
#
|
||||
# * plain: qlen == 1 or a non-causal block. Every query sees every local token.
|
||||
# * cprr: causal qlen >= _MIN_CPRR_QLEN, gfx950, speculative decoding. AITER's
|
||||
# round-robin ASM decode rebuilds global positions from g_kv_indptr, cp_rank
|
||||
# and cp_world_size and applies the causal window in-kernel. Kernels exist only
|
||||
# at _NATIVE_CPRR_HEADS; other gathered head counts are padded up.
|
||||
# * segmented: causal qlen > 1 when cprr is not selected or qlen is below
|
||||
# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV length
|
||||
# encodes its causal window.
|
||||
#
|
||||
# The plain kernel must never serve a causal qlen > 1 block: it applies
|
||||
# causality on local indices, which is wrong over a round-robin shard.
|
||||
_NATIVE_CPRR_HEADS: Final = (16, 32, 64, 128)
|
||||
_MIN_CPRR_QLEN: Final = 3
|
||||
|
||||
|
||||
class _DCPDecodeRoute(Enum):
|
||||
PLAIN = "plain"
|
||||
SEGMENTED = "segmented"
|
||||
CPRR = "cprr"
|
||||
|
||||
|
||||
def _asm_dcp_verify_heads(decode_num_heads: int) -> int:
|
||||
"""Head count the cprr asm decode runs at, or 0 if it cannot serve this shape."""
|
||||
if decode_num_heads in _NATIVE_CPRR_HEADS:
|
||||
return decode_num_heads
|
||||
for nat in _NATIVE_CPRR_HEADS:
|
||||
if nat >= decode_num_heads:
|
||||
return nat
|
||||
return 0
|
||||
|
||||
|
||||
def _select_dcp_decode_route(
|
||||
supports_segmented: bool,
|
||||
causal: bool,
|
||||
max_qo_len: int,
|
||||
asm_selected: bool,
|
||||
) -> _DCPDecodeRoute:
|
||||
"""Select one DCP decode route for this batch; see DCP decode routing."""
|
||||
if not causal or max_qo_len <= 1:
|
||||
return _DCPDecodeRoute.PLAIN
|
||||
if asm_selected and max_qo_len >= _MIN_CPRR_QLEN:
|
||||
return _DCPDecodeRoute.CPRR
|
||||
if supports_segmented:
|
||||
return _DCPDecodeRoute.SEGMENTED
|
||||
raise RuntimeError(
|
||||
"ROCM_AITER_MLA DCP multi-token verify requires either segmented "
|
||||
"MLA or the round-robin asm decode."
|
||||
)
|
||||
|
||||
|
||||
def _asm_dcp_verify_configured(
|
||||
dcp_world_size: int, cp_interleave: int, multi_token_decode: bool
|
||||
) -> bool:
|
||||
"""Whether asm cprr DCP verify is reachable in this process.
|
||||
|
||||
Requires DCP with interleave size 1, speculative decoding, and gfx950 (the
|
||||
only arch AITER builds cprr kernels for).
|
||||
"""
|
||||
if not (dcp_world_size > 1 and cp_interleave == 1 and multi_token_decode):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return bool(on_gfx950())
|
||||
|
||||
|
||||
def _aiter_mla_small_head_mode() -> str:
|
||||
"""Small-head (<16) MLA decode kernel selection.
|
||||
|
||||
@@ -410,8 +484,19 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
|
||||
max_qo_len: int | None = None
|
||||
# Minimum KV length used by Gluon to choose a safe split count.
|
||||
min_kv_seq_len: int = 1
|
||||
# Exactly one DCP decode route is selected by the metadata builder.
|
||||
dcp_route: _DCPDecodeRoute = _DCPDecodeRoute.PLAIN
|
||||
# Set exactly when this batch routes to segmented DCP verification.
|
||||
dcp_verify: AiterMLADCPVerifyMetadata | None = None
|
||||
# cprr only: global per-request page indptr and this rank's position.
|
||||
g_kv_indptr: torch.Tensor | None = None
|
||||
cp_world_size: int = 1
|
||||
cp_rank: int = 0
|
||||
# Padded head count cprr runs at; 0 when this batch is not on cprr.
|
||||
asm_decode_num_heads: int = 0
|
||||
# Must equal the split cap the reduce scratch was sized with, or the kernel
|
||||
# writes past it and faults.
|
||||
mla_num_kv_splits: int = 0
|
||||
# Small-head decode uses Gluon (avoids padding to 16).
|
||||
use_gluon_decode: bool = False
|
||||
# Small-head non-DCP multi-token verify uses Gluon's native MTP entry.
|
||||
@@ -460,6 +545,9 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
# Set from the common metadata every build; a batch is causal unless the
|
||||
# drafter says otherwise.
|
||||
_decode_causal: bool = True
|
||||
# A non-causal draft needs no cross-shard causal window: the ordinary
|
||||
# mask0 decode returns per-row LSE for the DCP merge.
|
||||
supports_non_causal_multi_token_dcp: ClassVar[bool] = True
|
||||
|
||||
@staticmethod
|
||||
def _uniform_padded_mtp_qo_len(
|
||||
@@ -513,15 +601,48 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
parallel_config.decode_context_parallel_size,
|
||||
parallel_config.cp_kv_cache_interleave_size,
|
||||
)
|
||||
asm_dcp_verify_config = (
|
||||
_asm_dcp_verify_configured(
|
||||
parallel_config.decode_context_parallel_size,
|
||||
parallel_config.cp_kv_cache_interleave_size,
|
||||
multi_token_decode=vllm_config.speculative_config is not None,
|
||||
)
|
||||
and envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "asm"
|
||||
)
|
||||
super().__init__(
|
||||
kv_cache_spec,
|
||||
layer_names,
|
||||
vllm_config,
|
||||
device,
|
||||
AiterMLAMetadata,
|
||||
supports_dcp_with_varlen=supports_segmented_dcp_verify,
|
||||
supports_dcp_with_varlen=(
|
||||
supports_segmented_dcp_verify or asm_dcp_verify_config
|
||||
),
|
||||
)
|
||||
self._asm_dcp_verify = False
|
||||
self._asm_dcp_verify_heads = 0
|
||||
# The head-count half of the route decision needs self.num_heads, so only
|
||||
# the config half is known before super().__init__().
|
||||
if asm_dcp_verify_config:
|
||||
self._asm_dcp_verify_heads = _asm_dcp_verify_heads(
|
||||
self.num_heads * self.dcp_world_size
|
||||
)
|
||||
if not self._asm_dcp_verify_heads:
|
||||
raise ValueError(
|
||||
"VLLM_ROCM_AITER_MLA_DCP_VERIFY=asm, but the round-robin asm "
|
||||
f"decode has no kernel for {self.num_heads * self.dcp_world_size} "
|
||||
f"DCP-gathered heads (native counts: {_NATIVE_CPRR_HEADS}). "
|
||||
"Set VLLM_ROCM_AITER_MLA_DCP_VERIFY=segmented."
|
||||
)
|
||||
self._asm_dcp_verify = True
|
||||
self._supports_segmented_dcp_verify = supports_segmented_dcp_verify
|
||||
self._mla_max_split_per_batch = 0
|
||||
if self._asm_dcp_verify:
|
||||
# Cap on KV splits per batch; at low batch the KV axis is the only
|
||||
# parallelism. Buffer sizing and every runtime call must see this value.
|
||||
self._mla_max_split_per_batch = torch.cuda.get_device_properties(
|
||||
device
|
||||
).multi_processor_count
|
||||
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
self.decode_attn_out_dtype = vllm_config.model_config.dtype
|
||||
@@ -576,6 +697,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads(
|
||||
self._decode_num_heads
|
||||
)
|
||||
# the cprr kernel exists only at the native head
|
||||
# counts, so run (and size the persistent schedule) at the padded count.
|
||||
# 96 is 16-aligned, so get_actual_mla_num_heads leaves it alone and the
|
||||
# pad has to be applied here explicitly.
|
||||
if self._asm_dcp_verify:
|
||||
self._num_attention_heads = self._asm_dcp_verify_heads
|
||||
kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")
|
||||
if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"):
|
||||
kv_cache_dtype_str = "fp8"
|
||||
@@ -619,6 +746,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
kv_dtype,
|
||||
is_sparse=False,
|
||||
fast_mode=True,
|
||||
# Must match max_split_per_batch in get_mla_metadata_v1; without it
|
||||
# aiter sizes the reduce scratch from capacity (~9 GiB).
|
||||
**(
|
||||
dict(max_split_per_batch=self._mla_max_split_per_batch)
|
||||
if self._asm_dcp_verify
|
||||
else {}
|
||||
),
|
||||
)
|
||||
self._mla_work_meta_data = torch.empty(
|
||||
work_meta_data_size, dtype=work_meta_data_type, device=device
|
||||
@@ -664,6 +798,14 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
device,
|
||||
)
|
||||
|
||||
# Full cudagraphs need a stable address for the cprr global page indptr.
|
||||
# Zero-initialized; element 0 is the indptr base and is never rewritten.
|
||||
self._g_kv_indptr_buf: torch.Tensor | None = None
|
||||
if self._asm_dcp_verify:
|
||||
self._g_kv_indptr_buf = torch.zeros(
|
||||
max_num_reqs + 1, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
# Persistent buffers for segmented DCP verification. Captured graphs
|
||||
# require stable addresses while row lengths vary between replays.
|
||||
self._dcp_verify_buffers: AiterMLADCPVerifyMetadata | None = None
|
||||
@@ -683,8 +825,11 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
)
|
||||
|
||||
if self._supports_segmented_dcp_verify and self._mtp_decode_qlen > 1:
|
||||
# A DCP rank's shard of the longest sequence bounds every verify
|
||||
# row, and full graphs need that bound to be constant.
|
||||
# Allocate even when CPRR is the preferred route: a later
|
||||
# replay can fall below _MIN_CPRR_QLEN and needs these
|
||||
# addresses. A DCP rank's shard of the longest sequence
|
||||
# bounds every verify row, and full graphs need that bound
|
||||
# to be constant.
|
||||
num_dcp_partitions = (
|
||||
self.dcp_world_size * self.cp_kv_cache_interleave_size
|
||||
)
|
||||
@@ -1063,6 +1208,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
pad_uniform_mtp = padded_mtp_qo_len > 0
|
||||
|
||||
seq_lens_for_kernel = seq_lens_device
|
||||
# the global lengths need the same dummy-row
|
||||
# treatment as the local ones below, but only for g_kv_indptr -- the
|
||||
# metadata field keeps the true values, which is what the segmented
|
||||
# route wants (it derives its own row lens and gives a dummy row
|
||||
# length 0, correctly).
|
||||
g_tot_seq_lens = dcp_tot_seq_lens_device
|
||||
num_kernel_reqs = num_reqs
|
||||
if pad_uniform_mtp:
|
||||
qo_lens_device = (
|
||||
@@ -1074,6 +1225,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
seq_lens_for_kernel,
|
||||
seq_lens_for_kernel.new_full((), max_qo_len),
|
||||
)
|
||||
# A dummy row just got a LOCAL length of max_qo_len, so its GLOBAL
|
||||
# length has to say the same thing or the cprr mask is degenerate:
|
||||
# global_len 0 puts the row query positions at {-3,-2,-1} while its
|
||||
# local shard sits at {rank, W+rank, 2W+rank}, so every entry is
|
||||
# masked and the row softmaxes over nothing. max_qo_len * W is
|
||||
# exactly the global length whose round-robin shard is max_qo_len
|
||||
# on every rank.
|
||||
if g_tot_seq_lens is not None:
|
||||
g_tot_seq_lens = torch.where(
|
||||
qo_lens_device > 0,
|
||||
g_tot_seq_lens,
|
||||
g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size),
|
||||
)
|
||||
|
||||
if self._graph_seq_lens is not None:
|
||||
self._graph_seq_lens[:num_kernel_reqs].copy_(
|
||||
@@ -1106,16 +1270,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
causal,
|
||||
self._kv_cache_bytes,
|
||||
)
|
||||
use_segmented_dcp_verify = (
|
||||
self._supports_segmented_dcp_verify and max_qo_len > 1
|
||||
)
|
||||
dcp_route = _DCPDecodeRoute.PLAIN
|
||||
if self.dcp_world_size > 1:
|
||||
dcp_route = _select_dcp_decode_route(
|
||||
self._supports_segmented_dcp_verify,
|
||||
causal,
|
||||
int(max_qo_len),
|
||||
self._asm_dcp_verify,
|
||||
)
|
||||
|
||||
# Segmented DCP verify carries its own per-row subpage table, so the
|
||||
# flat per-token view is dead work for it. Leave the buffer alone and
|
||||
# hand the metadata None, so a future reader cannot pick up whatever
|
||||
# the previous batch left behind.
|
||||
paged_kv_indices = None
|
||||
if not use_segmented_dcp_verify:
|
||||
if dcp_route is not _DCPDecodeRoute.SEGMENTED:
|
||||
if self.compilation_config.cudagraph_mode.has_full_cudagraphs():
|
||||
self.paged_kv_indices.fill_(-1)
|
||||
|
||||
@@ -1169,6 +1338,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
else:
|
||||
qo_indptr = query_start_loc_device[: 1 + num_kernel_reqs]
|
||||
|
||||
g_kv_indptr = None
|
||||
if dcp_route is _DCPDecodeRoute.CPRR and g_tot_seq_lens is not None:
|
||||
assert self._g_kv_indptr_buf is not None
|
||||
_ngk = g_tot_seq_lens.shape[0]
|
||||
g_kv_indptr = self._g_kv_indptr_buf[: _ngk + 1]
|
||||
g_kv_indptr[1:].copy_(g_tot_seq_lens.cumsum(dim=0, dtype=torch.int32))
|
||||
# A FULL cudagraph replays at its capture-time request count, so a
|
||||
# smaller real batch leaves the tail holding the PREVIOUS step's
|
||||
# cumsum. Flatten it the way paged_kv_indptr is flattened below, so
|
||||
# padding rows read global_len == 0 and agree with their zero-length
|
||||
# local shard instead of a stale global length.
|
||||
self._g_kv_indptr_buf[_ngk + 1 :].fill_(g_kv_indptr[-1])
|
||||
|
||||
has_persistent_metadata = False
|
||||
# Only the asm decode consumes the schedule, so gate on the routing
|
||||
# rather than on num_heads >= 16, which denies it to a padded rank
|
||||
@@ -1178,7 +1360,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
use_persistent_metadata = (
|
||||
not use_gluon_decode
|
||||
and not use_gluon_verify
|
||||
and not use_segmented_dcp_verify
|
||||
and dcp_route is not _DCPDecodeRoute.SEGMENTED
|
||||
# A padded rank has no bf16 persistent kernel past qlen 4 where the
|
||||
# gfx950 fold is absent; the non-persistent entry covers it. fp8
|
||||
# keeps the schedule -- its fold rejects non-persistent outright.
|
||||
@@ -1218,13 +1400,28 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
uni_qo_len = (
|
||||
max_qo_len if pad_uniform_mtp or torch.all(qo_len == max_qo_len) else -1
|
||||
)
|
||||
cprr_kwargs: dict = {}
|
||||
is_causal = causal
|
||||
if self._asm_dcp_verify:
|
||||
cprr_kwargs = dict(
|
||||
max_split_per_batch=self._mla_max_split_per_batch,
|
||||
intra_batch_mode=False,
|
||||
)
|
||||
if dcp_route is _DCPDecodeRoute.CPRR:
|
||||
is_causal = False
|
||||
cprr_kwargs["is_cp_round_robin"] = True
|
||||
metadata_num_heads = (
|
||||
self._asm_dcp_verify_heads
|
||||
if dcp_route is _DCPDecodeRoute.CPRR
|
||||
else AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads)
|
||||
)
|
||||
get_mla_metadata_v1(
|
||||
qo_indptr,
|
||||
paged_kv_indptr,
|
||||
paged_kv_last_page_len,
|
||||
self._num_attention_heads,
|
||||
metadata_num_heads,
|
||||
1,
|
||||
causal,
|
||||
is_causal,
|
||||
self._mla_work_meta_data,
|
||||
self._mla_work_info_set,
|
||||
self._mla_work_indptr,
|
||||
@@ -1238,6 +1435,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
fast_mode=True,
|
||||
dtype_q=self._mla_q_dtype,
|
||||
dtype_kv=self._mla_kv_dtype,
|
||||
**cprr_kwargs,
|
||||
)
|
||||
has_persistent_metadata = True
|
||||
|
||||
@@ -1261,7 +1459,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
min_kv_seq_len = int(per_req_len.min().item())
|
||||
|
||||
dcp_verify = None
|
||||
if use_segmented_dcp_verify:
|
||||
if dcp_route is _DCPDecodeRoute.SEGMENTED:
|
||||
assert dcp_tot_seq_lens_device is not None
|
||||
dcp_verify = self._build_dcp_verify_row_view(
|
||||
int(max_qo_len),
|
||||
@@ -1279,7 +1477,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
|
||||
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
|
||||
max_qo_len=max_qo_len,
|
||||
min_kv_seq_len=min_kv_seq_len,
|
||||
dcp_route=dcp_route,
|
||||
dcp_verify=dcp_verify,
|
||||
g_kv_indptr=g_kv_indptr,
|
||||
cp_world_size=self.dcp_world_size,
|
||||
cp_rank=self.dcp_rank,
|
||||
asm_decode_num_heads=(
|
||||
self._asm_dcp_verify_heads if dcp_route is _DCPDecodeRoute.CPRR else 0
|
||||
),
|
||||
mla_num_kv_splits=(
|
||||
self._mla_max_split_per_batch
|
||||
if dcp_route is _DCPDecodeRoute.CPRR
|
||||
else 0
|
||||
),
|
||||
use_gluon_decode=use_gluon_decode,
|
||||
use_gluon_verify=use_gluon_verify,
|
||||
attn_out_dtype=self.decode_attn_out_dtype,
|
||||
@@ -2062,8 +2272,10 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
)
|
||||
return o, None
|
||||
|
||||
verify = decode.dcp_verify
|
||||
if verify is not None:
|
||||
dcp_route = decode.dcp_route
|
||||
if dcp_route is _DCPDecodeRoute.SEGMENTED:
|
||||
verify = decode.dcp_verify
|
||||
assert verify is not None
|
||||
if type(q) is tuple:
|
||||
q_nope, q_pe = q
|
||||
else:
|
||||
@@ -2085,11 +2297,6 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
decode.attn_out_dtype,
|
||||
)
|
||||
|
||||
if self.dcp_world_size > 1 and int(decode.max_qo_len) > 1:
|
||||
raise RuntimeError(
|
||||
"ROCM_AITER_MLA DCP multi-token verify requires segmented MLA."
|
||||
)
|
||||
|
||||
if type(q) is tuple:
|
||||
q = torch.cat(q, dim=-1)
|
||||
|
||||
@@ -2102,8 +2309,19 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
"ROCM_AITER_MLA decode expected the DCP-gathered query head count "
|
||||
f"{self._decode_num_heads}, got {q.shape[1]}"
|
||||
)
|
||||
mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q)
|
||||
mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads)
|
||||
# cprr has kernels only at native head counts, so pad q up (e.g. 96 ->
|
||||
# 128). Heads are independent in MLA; the extras are sliced off below.
|
||||
asm_dcp_heads = decode.asm_decode_num_heads
|
||||
if asm_dcp_heads > self._decode_num_heads:
|
||||
mla_num_heads = asm_dcp_heads
|
||||
mla_padded_q = AiterMLAHelper.get_mla_padded_q(
|
||||
self._decode_num_heads, q, target_heads=mla_num_heads
|
||||
)
|
||||
else:
|
||||
mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q)
|
||||
mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads(
|
||||
self._decode_num_heads
|
||||
)
|
||||
o = torch.empty(
|
||||
B,
|
||||
mla_num_heads,
|
||||
@@ -2137,6 +2355,16 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
|
||||
lse = None
|
||||
if self.dcp_world_size > 1:
|
||||
if dcp_route is _DCPDecodeRoute.CPRR:
|
||||
# cprr only: global positions for the in-kernel causal window.
|
||||
assert decode.g_kv_indptr is not None
|
||||
mla_kwargs["g_kv_indptr"] = decode.g_kv_indptr
|
||||
mla_kwargs["cp_world_size"] = decode.cp_world_size
|
||||
mla_kwargs["cp_rank"] = decode.cp_rank
|
||||
# Must match the schedule get_mla_metadata_v1 was built with.
|
||||
assert decode.mla_num_kv_splits > 0
|
||||
mla_kwargs["num_kv_splits"] = decode.mla_num_kv_splits
|
||||
mla_kwargs["intra_batch_mode"] = False
|
||||
# The vLLM custom-op wrapper exposes only the in-place output and
|
||||
# drops aiter's final LSE, which the cross-shard merge needs, so go
|
||||
# through aiter's native entry point on the DCP path.
|
||||
@@ -2151,6 +2379,7 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
decode.max_qo_len,
|
||||
sm_scale=self.scale,
|
||||
return_lse=True,
|
||||
causal=attn_metadata.causal,
|
||||
**mla_kwargs,
|
||||
)
|
||||
assert lse is not None, (
|
||||
@@ -2172,7 +2401,13 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
|
||||
**mla_kwargs,
|
||||
)
|
||||
|
||||
output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o)
|
||||
if lse is not None:
|
||||
lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse)
|
||||
if asm_dcp_heads > self._decode_num_heads:
|
||||
# Keep a view: a post-kernel copy races the symm-mem a2a combine.
|
||||
output = o[:, : self._decode_num_heads]
|
||||
if lse is not None:
|
||||
lse = lse[:, : self._decode_num_heads, ...]
|
||||
else:
|
||||
output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o)
|
||||
if lse is not None:
|
||||
lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse)
|
||||
return output, lse
|
||||
|
||||
Reference in New Issue
Block a user