[ROCm][MLA] Add an AITER ASM round-robin decode route for DCP multi-token verify (#56861)

Signed-off-by: Xiaohu Guo <Xiaohu.Guo@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: seungrokj <144636725+seungrokj@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
xiaohuguo2023
2026-09-28 14:52:21 -04:00
committed by GitHub
co-authored by Claude Opus 5 seungrokj Cursor
parent 5ba2e3090b
commit 25624f65cf
5 changed files with 1202 additions and 30 deletions
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routing/selection tests for the AITER ASM cprr DCP-verify route.
Companion to the upstream segmented-verify tests. Everything here is pure CPU:
it pins the *decisions* (which route a KV group takes, which head count the
kernel runs at, which configurations are refused) rather than kernel numerics.
The numerics parity test against the segmented route needs a GPU and both
routes built; it is marked and skipped when unavailable.
"""
from __future__ import annotations
import pytest
rocm_aiter_mla = pytest.importorskip(
"vllm.v1.attention.backends.mla.rocm_aiter_mla",
reason="ROCm AITER MLA backend not available",
)
_heads = rocm_aiter_mla._asm_dcp_verify_heads
_configured = rocm_aiter_mla._asm_dcp_verify_configured
_select_route = rocm_aiter_mla._select_dcp_decode_route
Route = rocm_aiter_mla._DCPDecodeRoute
NATIVE = rocm_aiter_mla._NATIVE_CPRR_HEADS
MIN_QLEN = rocm_aiter_mla._MIN_CPRR_QLEN
def test_dcp_verify_env_defaults_to_segmented(monkeypatch):
monkeypatch.delenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", raising=False)
assert rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "segmented"
def test_dcp_verify_env_rejects_unknown_route(monkeypatch):
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "unknown")
with pytest.raises(ValueError, match="Invalid value"):
_ = rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY
# --------------------------------------------------------------------------
# _asm_dcp_verify_heads -- which head count the cprr kernel runs at
# --------------------------------------------------------------------------
@pytest.mark.parametrize("n", NATIVE)
def test_native_head_counts_are_used_as_is(n):
assert _heads(n) == n
def test_non_native_head_count_pads_up_to_the_next_native():
"""K3 at TP8/DCP8 gathers 96 heads for the target; the kernel has no 96
variant, so it must run at 128, not silently fall out of the asm path."""
assert _heads(96) == 128
assert _heads(1) == 16
assert _heads(17) == 32
assert _heads(65) == 128
def test_head_count_above_the_largest_native_is_unservable():
"""0 means 'cannot serve'; anything else would route a shape the kernel
has no build for."""
assert _heads(max(NATIVE) + 1) == 0
assert _heads(256) == 0
# --------------------------------------------------------------------------
# _asm_dcp_verify_configured -- reachability
# --------------------------------------------------------------------------
@pytest.fixture
def on_gfx950(monkeypatch):
"""The cprr kernels are built only for gfx950, so the gate checks the arch.
Patched rather than detected, so both branches run on any runner."""
import vllm.platforms.rocm as rocm
monkeypatch.setattr(rocm, "on_gfx950", lambda: True)
def test_route_needs_dcp(on_gfx950):
assert _configured(dcp_world_size=1, cp_interleave=1, multi_token_decode=True) is (
False
)
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is (
True
)
def test_round_robin_interleave_other_than_one_is_excluded(on_gfx950):
"""The kernel's global-position causal window assumes interleave 1, the
same restriction the segmented gate carries."""
assert _configured(dcp_world_size=8, cp_interleave=2, multi_token_decode=True) is (
False
)
assert _configured(dcp_world_size=8, cp_interleave=16, multi_token_decode=True) is (
False
)
def test_route_needs_multi_token_decode(on_gfx950):
"""Single-token decode is already served. Enabling the route for it would
pad the head count on every step and buy nothing."""
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=False) is (
False
)
def test_route_needs_gfx950(monkeypatch):
"""hsa/gfx942/mla and hsa/gfx1250/mla carry no cprr rows, so off gfx950 the
kernel lookup fails at the first verify step rather than falling back."""
import vllm.platforms.rocm as rocm
monkeypatch.setattr(rocm, "on_gfx950", lambda: False)
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is (
False
)
# --------------------------------------------------------------------------
# qlen floor
# --------------------------------------------------------------------------
def test_min_cprr_qlen_is_above_two():
"""Qlen 2 is below CPRR's floor and must use segmented verification."""
assert MIN_QLEN > 2
@pytest.mark.parametrize(
"supports,causal,qlen,asm,expected",
[
(True, True, 1, True, Route.PLAIN),
(True, True, 2, True, Route.SEGMENTED),
(True, True, 3, True, Route.CPRR),
(True, True, 4, True, Route.CPRR),
(True, True, 4, False, Route.SEGMENTED),
(True, False, 2, True, Route.PLAIN),
(True, False, 4, False, Route.PLAIN),
(False, True, 3, True, Route.CPRR),
],
)
def test_dcp_decode_route(supports, causal, qlen, asm, expected):
assert _select_route(supports, causal, qlen, asm) is expected
def test_causal_multi_token_batch_without_a_valid_route_fails():
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
_select_route(False, True, 2, True)
# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs
# the real builder and the real asm kernel on a gfx950 GPU.
@@ -0,0 +1,540 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GPU numerics for the AITER ASM cprr DCP-verify route.
Runs the REAL metadata builder (``AiterMLAMetadataBuilder.build``, so
``get_mla_metadata_v1`` is exercised with the cprr kwargs and the persistent
schedule buffers are produced) and the REAL asm kernel, on ONE GPU with the DCP
ranks simulated one after another.
That is legitimate because the aiter MLA backend contains no collective on this
path: the query all-gather happens in the *layer*, strictly before
``forward_mqa``, so the head fold, the round-robin kernel call and the un-fold
are pure per-rank local arithmetic in which ``cp_world_size``/``cp_rank`` are
just numbers feeding the causal-mask maths. Restricting a shard to a residue
class of global positions is exactly what DCP rank r sees.
Three properties are checked:
* each shard matches an exact torch reference over the positions it holds,
with causality applied on the GLOBAL position (the thing cprr reconstructs
in-kernel and the segmented route instead carries in per-row lengths);
* the LSE merge of all shards reproduces full-context attention;
* a shard told the wrong ``cp_rank`` fails both -- without this positive
control a passing run proves nothing, since a kernel that ignored global
positions still produces plausible-looking output.
"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("ROCm AITER MLA tests", allow_module_level=True)
from vllm._aiter_ops import is_aiter_found # noqa: E402
from vllm.v1.attention.backends.mla import rocm_aiter_mla # noqa: E402
KV_LORA_RANK = 512
QK_ROPE_HEAD_DIM = 64
QK_NOPE_HEAD_DIM = 128
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM
SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5
DCP = 8
NUM_REQS = 8
CTX = 256
QLEN = 5 # 1 + num_speculative_tokens; must clear _MIN_CPRR_QLEN
# fp8-e4m3 keeps 3 mantissa bits, so the relative error floor is ~6.2e-2. Gates
# are relative for the same reason: an absolute bound would just be a bound on
# the output magnitude.
SHARD_RTOL = 1.0e-1
MERGE_RTOL = 5.0e-2
MODEL = "deepseek-ai/DeepSeek-R1"
def _gpu_available() -> bool:
return is_aiter_found() and torch.accelerator.is_available()
def _on_gfx950() -> bool:
if not is_aiter_found():
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
requires_cprr = pytest.mark.skipif(
not (_gpu_available() and _on_gfx950()),
reason="cprr DCP verify needs ROCm + AITER on gfx950",
)
@pytest.fixture(autouse=True)
def select_asm_route(monkeypatch):
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "asm")
# --------------------------------------------------------------------------
# reference maths
# --------------------------------------------------------------------------
def _torch_reference(q_f32, kv_f32, ctx, qlen, scale, positions):
"""Exact absorbed-MLA decode over an explicit set of GLOBAL KV positions.
A shard sees only ``positions``, but causality still uses the global
position: query token i (global position ctx + i) may attend to any p in
positions with p <= ctx + i. Restricting ``positions`` to a round-robin
residue class is precisely DCP rank r's view.
"""
nr, _, heads, _ = q_f32.shape
dev = q_f32.device
o = torch.zeros(nr, qlen, heads, KV_LORA_RANK, dtype=torch.float32, device=dev)
lse = torch.full((nr, qlen, heads), float("-inf"), dtype=torch.float32, device=dev)
for r in range(nr):
kv_sel = kv_f32[r].index_select(0, positions)
for i in range(qlen):
valid = positions <= (ctx + i)
if not bool(valid.any()):
continue
k = kv_sel[valid]
s = (q_f32[r, i] @ k.T) * scale
m = s.max(dim=-1, keepdim=True).values
p = torch.exp(s - m)
denom = p.sum(dim=-1, keepdim=True)
o[r, i] = (p / denom) @ k[:, :KV_LORA_RANK]
lse[r, i] = m.squeeze(-1) + torch.log(denom.squeeze(-1))
return o, lse
def _merge_partials(os_, lses):
"""Cross-shard softmax merge (natural-log LSE)."""
stack_lse = torch.stack(lses)
m = stack_lse.max(dim=0).values
w = torch.exp(stack_lse - m)
num = (w.unsqueeze(-1) * torch.stack(os_)).sum(dim=0)
den = w.sum(dim=0).unsqueeze(-1)
return num / den, m + torch.log(w.sum(dim=0))
# --------------------------------------------------------------------------
# single-process rank simulation
# --------------------------------------------------------------------------
class _FakeGroup:
def __init__(self, world_size: int, rank: int):
self.world_size = world_size
self.rank_in_group = rank
self.rank = rank
self.local_rank = rank
def all_gather(self, tensor, dim=0):
raise RuntimeError(
"collective called in a single-process test -- the path under test "
"is supposed to be collective-free; this indicates a real change"
)
def _install_fake_groups(dcp_size: int, cp_rank: int, tp_size: int = 1) -> None:
"""Point vLLM's parallel_state at rank-simulating stubs (no NCCL)."""
import vllm.distributed.parallel_state as ps
import vllm.model_executor.layers.attention.mla_attention as mla_attention
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
dcp = _FakeGroup(dcp_size, cp_rank)
tp = _FakeGroup(tp_size, cp_rank)
ps.get_dcp_group = lambda: dcp
ps.get_tp_group = lambda: tp
# Both of these imported the symbol directly at module import time, so
# patching parallel_state alone leaves them resolving the real group (and
# the builder then reports dcp_world_size 1, silently skipping the route).
backend.get_dcp_group = lambda: dcp
mla_attention.get_dcp_group = lambda: dcp
@pytest.fixture(autouse=True)
def _restore_parallel_groups():
"""Undo _install_fake_groups after every test.
It patches module globals in three modules, so without this the fake DCP
group leaks into the rest of the pytest session and silently breaks
unrelated tests that expect a real (or absent) group.
"""
import vllm.distributed.parallel_state as ps
import vllm.model_executor.layers.attention.mla_attention as mla_attention
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
saved = [
(mod, name, getattr(mod, name))
for mod, name in (
(ps, "get_dcp_group"),
(ps, "get_tp_group"),
(backend, "get_dcp_group"),
(mla_attention, "get_dcp_group"),
)
]
yield
for mod, name, value in saved:
setattr(mod, name, value)
class _PrefillBackendStub:
"""The base builder does ``attention_layer.prefill_backend.clone()``.
prefill_backend is consulted only from the chunked-context prefill path,
which a decode-only test never enters.
"""
def clone(self):
return self
def _make_ctx_layer(vllm_config):
"""Stand-in for the registered attention layer in static_forward_context.
``MLACommonMetadataBuilder.__init__`` reads the MLA dims, ``dcp_manager``
and ``prefill_backend`` off this object. ``dcp_manager`` is the Direct-DCP
symmetric-memory workspace, used ONLY by the chunked-prefill KV all-gather;
it is constructed without running __init__ so no peer pointers are exchanged
and no collective is issued. That is what keeps this test single-process.
"""
from vllm.v1.attention.ops.dcp import MLADCPManager
mgr = object.__new__(MLADCPManager)
mgr.init_kv_gather = lambda *a, **k: None
return SimpleNamespace(
non_causal_multi_token_decode=False,
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
v_head_dim=QK_NOPE_HEAD_DIM,
dcp_manager=mgr,
prefill_backend=_PrefillBackendStub(),
)
def _ensure_workspace_manager(device: torch.device) -> None:
"""Init the global workspace manager, which GPUModelRunner does in production.
``AiterMLAMetadataBuilder.__init__`` reserves the fp8 prefill
persistent-scheduling scratch through ``current_workspace_manager()``, so
merely constructing a builder outside a serve trips its assert. Idempotent:
one builder is constructed per DCP shard. Deliberately does NOT lock the
workspace -- production locks only after warmup, and this keeps allocating
across shards.
"""
from vllm.v1.worker.workspace import (
init_workspace_manager,
is_workspace_manager_initialized,
)
if not is_workspace_manager_initialized():
init_workspace_manager(device)
class _Layer:
def __init__(self, device):
self._q_scale = torch.tensor(1.0, device=device)
self._k_scale = torch.tensor(1.0, device=device)
self._v_scale = torch.tensor(1.0, device=device)
def _build_impl(num_heads: int, dcp: int, cp_rank: int):
"""Real AiterMLAImpl.forward_mqa bound to a minimally-populated instance.
Constructing the full impl would drag in kv_b_proj and the rest of the
weight-dependent MLA setup -- i.e. model loading, which this test exists to
avoid. forward_mqa's fold path reads only the attributes set below.
"""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
impl = object.__new__(AiterMLAImpl)
impl.kv_lora_rank = KV_LORA_RANK
impl.qk_rope_head_dim = QK_ROPE_HEAD_DIM
impl.num_heads = num_heads
impl.dcp_world_size = dcp
impl.dcp_rank = cp_rank
impl.pcp_world_size = 1
impl.kv_cache_dtype = "fp8"
impl.scale = SCALE
# `_decode_num_heads` is a read-only property deriving num_heads *
# dcp_world_size, both set above; assert rather than assign.
assert impl._decode_num_heads == num_heads * dcp
return impl
def _make_vllm_config(heads_per_rank: int):
"""A DCP8 MLA config with no weights and no distributed init.
DeepSeek-R1's config supplies the MLA dims (kv_lora_rank 512, rope 64,
nope 128) that the aiter asm kernel is built for; only the head count is
overridden, to place the GATHERED count (heads_per_rank * DCP) on the
kernel variant under test. ``get_num_attention_heads`` reads
model_arch_config, a snapshot taken at config-creation time, so patching
hf_config alone would be inert.
"""
from tests.v1.attention.utils import create_vllm_config
vllm_config = create_vllm_config(
model_name=MODEL,
tensor_parallel_size=1,
max_model_len=2048,
max_num_seqs=NUM_REQS * 2,
max_num_batched_tokens=4096,
)
vllm_config.model_config.model_arch_config.total_num_attention_heads = (
heads_per_rank
)
# Without a speculative config the batch reorder threshold stays at 1 and a
# qlen-QLEN row is classified as a PREFILL, so the decode path under test is
# never reached. _init_reorder_batch_threshold reads exactly these two
# fields, so a stand-in is enough and avoids pulling in a draft model.
vllm_config.speculative_config = SimpleNamespace(
num_speculative_tokens=QLEN - 1, parallel_drafting=False
)
vllm_config.cache_config.cache_dtype = "fp8"
vllm_config.parallel_config.decode_context_parallel_size = DCP
vllm_config.parallel_config.cp_kv_cache_interleave_size = 1
return vllm_config
def _run_shard(
vllm_config,
spec,
layer_name,
device,
cp_rank,
kern_rank,
q_flat,
kv_flat,
nr,
qlen,
ctx,
dcp,
pos,
):
"""Build real metadata for one DCP rank and run the real asm kernel.
``kern_rank`` is separate from ``cp_rank`` so the positive control can hand
rank r's pages to the kernel while telling it that it is rank r+1.
"""
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
L = ctx + qlen
local_len = int(pos.numel())
# One GLOBAL kv cache: page (req*L + p) holds request req position p. Shards
# select pages by residue class, so every shard reads the same underlying
# bytes and no re-quantisation noise is introduced between arms.
block_table = (
torch.arange(nr, device=device).view(nr, 1) * L + pos.view(1, -1)
).to(torch.int32)
# seq_lens is the GLOBAL context; dcp_local_seq_lens is what this rank
# actually holds. Passing the global length as the local one walks the
# kernel off the end of the page table.
seq_lens_cpu = torch.full((nr,), L, dtype=torch.int32)
local_cpu = torch.full((nr,), local_len, dtype=torch.int32)
common = CommonAttentionMetadata(
query_start_loc=torch.arange(
0, nr * qlen + 1, qlen, dtype=torch.int32, device=device
),
query_start_loc_cpu=torch.arange(0, nr * qlen + 1, qlen, dtype=torch.int32),
seq_lens=seq_lens_cpu.to(device),
num_reqs=nr,
num_actual_tokens=nr * qlen,
max_query_len=qlen,
max_seq_len=L,
block_table_tensor=block_table,
slot_mapping=torch.arange(nr * qlen, dtype=torch.int64, device=device),
causal=True,
dcp_local_seq_lens=local_cpu.to(device),
dcp_local_seq_lens_cpu_upper_bound=local_cpu,
)
_install_fake_groups(dcp, kern_rank)
builder = AiterMLAMetadataBuilder(spec, [layer_name], vllm_config, device)
md = builder.build(0, common)
impl = _build_impl(builder.num_heads, dcp, kern_rank)
o, lse = impl.forward_mqa(q_flat, kv_flat, md, _Layer(device))
torch.accelerator.synchronize()
return md, o, lse, local_len
def _numerics(heads_per_rank: int, sabotage: bool = False):
"""Run every DCP shard and return (per-shard max rel err, merged rel err)."""
import aiter
from vllm.config import set_current_vllm_config
device = torch.device("cuda", 0)
torch.accelerator.set_device_index(device.index)
torch.manual_seed(0)
_ensure_workspace_manager(device)
_install_fake_groups(DCP, 0)
vllm_config = _make_vllm_config(heads_per_rank)
with set_current_vllm_config(vllm_config):
from vllm.v1.kv_cache_interface import MLAAttentionSpec
layer_name = "model.layers.0.self_attn.attn"
vllm_config.compilation_config.static_forward_context[layer_name] = (
_make_ctx_layer(vllm_config)
)
# block_size 1: one page per token, so page ids are positions and the
# residue-class page table is expressible directly.
spec = MLAAttentionSpec(
block_size=1,
num_kv_heads=1,
head_size=HEAD_SIZE,
dtype=vllm_config.model_config.dtype,
non_causal_multi_token_decode=False,
)
nr, ctx, qlen = NUM_REQS, CTX, QLEN
L = ctx + qlen
gathered_heads = heads_per_rank * DCP
kv_dtype = aiter.dtypes.fp8
kv_flat = torch.randn(
nr * L, 1, HEAD_SIZE, dtype=torch.float32, device=device
).to(kv_dtype)
q_flat = (
torch.randn(
nr * qlen,
gathered_heads,
HEAD_SIZE,
dtype=torch.float32,
device=device,
)
.mul_(0.5)
.to(kv_dtype)
)
# Dequantise for the reference so the comparison measures the kernel,
# not the fp8 cast that both sides share.
kv_f32 = kv_flat.float().view(nr, L, HEAD_SIZE)
q_f32 = q_flat.float().view(nr, qlen, gathered_heads, HEAD_SIZE)
ref_o, _ = _torch_reference(
q_f32, kv_f32, ctx, qlen, SCALE, torch.arange(L, device=device)
)
shard_o, shard_lse = [], []
max_shard_rel = 0.0
for r in range(DCP):
pos = torch.arange(r, L, DCP, device=device) # residue class r
kern_rank = (r + 1) % DCP if sabotage else r
md, o, lse, _ = _run_shard(
vllm_config,
spec,
layer_name,
device,
r,
kern_rank,
q_flat,
kv_flat,
nr,
qlen,
ctx,
DCP,
pos,
)
# The route actually under test must have been taken; a silent
# fall-out to Triton/segmented would otherwise pass quietly.
assert md.decode.dcp_route is rocm_aiter_mla._DCPDecodeRoute.CPRR
assert md.decode.asm_decode_num_heads, "cprr asm route was not taken"
o = o.float().view(nr, qlen, gathered_heads, KV_LORA_RANK)
shard_lse.append(lse.float().view(nr, qlen, gathered_heads))
shard_o.append(o)
po, _ = _torch_reference(q_f32, kv_f32, ctx, qlen, SCALE, pos)
rel = (o - po).abs().max().item() / max(po.abs().max().item(), 1e-6)
max_shard_rel = max(max_shard_rel, rel)
merged, _ = _merge_partials(shard_o, shard_lse)
merge_rel = (merged - ref_o).abs().max().item() / max(
ref_o.abs().max().item(), 1e-6
)
return max_shard_rel, merge_rel
# --------------------------------------------------------------------------
# tests
# --------------------------------------------------------------------------
@requires_cprr
@pytest.mark.parametrize(
"heads_per_rank",
[
16, # 128 gathered heads -- a NATIVE cprr kernel, no padding
12, # 96 gathered heads -- K3's DSpark target, must pad up to 128
],
ids=["native-128", "padded-96-to-128"],
)
@torch.inference_mode()
def test_cprr_shards_match_reference_and_merge_to_full_context(heads_per_rank):
"""Every DCP shard must be right, and their LSE merge must reproduce
full-context attention.
A shard that ignored global positions (applying a local causal tail
instead) still produces plausible-looking output; it only diverges from the
reference on the tokens whose causal window is truncated. Checking the merge
alone would hide that, so both are asserted.
"""
shard_rel, merge_rel = _numerics(heads_per_rank)
assert shard_rel < SHARD_RTOL, f"per-shard rel err {shard_rel:.3e}"
assert merge_rel < MERGE_RTOL, f"merged rel err {merge_rel:.3e}"
@requires_cprr
@torch.inference_mode()
def test_wrong_cp_rank_breaks_the_result():
"""Positive control: without it, a passing run above proves nothing.
Feeding shard r's pages to the kernel while telling it that it is rank r+1
shifts every reconstructed global position by one, so both the causal mask
and the round-robin reconstruction are wrong. If the error does NOT blow up,
the kernel is not using cp_rank and the test above has no power.
"""
shard_rel, merge_rel = _numerics(16, sabotage=True)
assert shard_rel > SHARD_RTOL or merge_rel > MERGE_RTOL, (
f"sabotaged run still passed (shard {shard_rel:.3e}, merge "
f"{merge_rel:.3e}) -- cp_rank is not reaching the kernel"
)
@requires_cprr
def test_cprr_kernel_is_selected_for_the_k3_shape():
"""The K3 DSpark target shape must actually reach the cprr route.
Guards against a silent fall-out to segmented/Triton, which is what the
stock configuration does and what this route exists to avoid.
"""
from vllm.v1.attention.backends.mla import rocm_aiter_mla as m
assert (
m._asm_dcp_verify_configured(
dcp_world_size=DCP, cp_interleave=1, multi_token_decode=True
)
is True
)
assert QLEN >= m._MIN_CPRR_QLEN
assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR
assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS
@@ -87,21 +87,31 @@ def _builder(
kv_cache_dtype: str = "auto",
dcp_world_size: int = 1,
dcp_rank: int = 0,
asm_dcp_verify: bool = False,
decode_causal: bool = True,
supports_segmented_dcp_verify: bool | None = None,
):
decode_num_heads = num_heads * dcp_world_size
asm_heads = (
rocm_aiter_mla._asm_dcp_verify_heads(decode_num_heads) if asm_dcp_verify else 0
)
if supports_segmented_dcp_verify is None:
supports_segmented_dcp_verify = rocm_aiter_mla._segmented_dcp_verify_supported(
dcp_world_size, 1
)
stub = SimpleNamespace(
device=torch.device("cpu"),
num_heads=num_heads,
# DCP gathers every rank's head shard before decode, and the routing
# predicates read the gathered count.
_decode_num_heads=num_heads * dcp_world_size,
_decode_num_heads=decode_num_heads,
dcp_world_size=dcp_world_size,
dcp_rank=dcp_rank,
cp_kv_cache_interleave_size=1,
# Mirrors the production constructor: configuration decides whether the
# segmented route is available, query length decides per batch.
_supports_segmented_dcp_verify=rocm_aiter_mla._segmented_dcp_verify_supported(
dcp_world_size, 1
),
# segmented route is available, query length decides per batch. CPRR
# preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back.
_supports_segmented_dcp_verify=supports_segmented_dcp_verify,
# Derived once in the real constructor, so derive it once here too.
_segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size),
_dcp_verify_buffers=None,
@@ -122,7 +132,11 @@ def _builder(
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
_use_persistent_metadata=False,
kernel_block_size=kernel_block_size,
_num_attention_heads=AiterMLAHelper.get_actual_mla_num_heads(num_heads),
_num_attention_heads=(
asm_heads
if asm_dcp_verify
else AiterMLAHelper.get_actual_mla_num_heads(num_heads)
),
_mla_work_meta_data=torch.empty(1, dtype=torch.int32),
_mla_work_info_set=torch.empty(1, dtype=torch.int32),
_mla_work_indptr=torch.empty(1, dtype=torch.int32),
@@ -132,7 +146,19 @@ def _builder(
_mla_q_dtype=torch.bfloat16,
_mla_kv_dtype=torch.bfloat16,
decode_attn_out_dtype=torch.bfloat16,
_decode_causal=True,
_decode_causal=decode_causal,
# Default off so existing cases keep exercising segmented/persistent.
# Pass asm_dcp_verify=True for the CPRR route and its qlen-floor fallback.
_asm_dcp_verify=asm_dcp_verify,
_asm_dcp_verify_heads=asm_heads,
_g_kv_indptr_buf=(
torch.zeros(max_decode_rows + 1, dtype=torch.int32)
if asm_dcp_verify and dcp_world_size > 1
else None
),
# Only read when the asm route is on; the real builder derives it from
# the device CU count.
_mla_max_split_per_batch=256,
)
# Bound to the stub rather than faked: the verify flatten's per-row view is
# part of what _build_decode is being tested for.
@@ -287,6 +313,209 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch):
assert metadata.dcp_verify is not None
def _cprr_decode_batch(builder, *, qlen: int, num_reqs: int = 1):
seq_lens = torch.full((num_reqs,), 16, dtype=torch.int32)
query_start_loc = torch.arange(0, (num_reqs + 1) * qlen, qlen, dtype=torch.int32)
return AiterMLAMetadataBuilder._build_decode(
builder,
block_table_tensor=torch.zeros(num_reqs, 8, dtype=torch.int32),
seq_lens_device=seq_lens,
max_seq_len=16,
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=num_reqs * qlen,
max_query_len=qlen,
dcp_tot_seq_lens_device=seq_lens * 8,
)
def test_asm_preferred_qlen2_falls_back_to_segmented(monkeypatch):
"""K=4 can clamp to qlen 2; CPRR has no kernel, so use segmented MLA."""
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
metadata = _cprr_decode_batch(
_builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
kernel_block_size=2,
),
qlen=2,
)
assert metadata.dcp_verify is not None
assert metadata.g_kv_indptr is None
assert not metadata.has_persistent_metadata
get_mla_metadata_v1.assert_not_called()
def test_asm_preferred_qlen_at_cprr_floor_stays_on_cprr(monkeypatch):
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
metadata = _cprr_decode_batch(
_builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
),
qlen=rocm_aiter_mla._MIN_CPRR_QLEN,
)
assert metadata.dcp_verify is None
assert metadata.g_kv_indptr is not None
assert metadata.has_persistent_metadata
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is True
def test_cprr_capable_single_token_decode_keeps_persistent_metadata(monkeypatch):
"""The CPRR qlen floor must not suppress the ordinary qlen-1 schedule."""
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
metadata = _cprr_decode_batch(
_builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
),
qlen=1,
)
assert metadata.g_kv_indptr is None
assert metadata.has_persistent_metadata
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None
def test_non_causal_dcp_block_uses_plain_mask0_metadata(monkeypatch):
"""A non-causal block needs no CPRR global-position window or head pad."""
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
builder = _builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
)
builder._decode_causal = False
metadata = _cprr_decode_batch(builder, qlen=4)
assert metadata.dcp_verify is None
assert metadata.g_kv_indptr is None
assert metadata.has_persistent_metadata
assert get_mla_metadata_v1.call_args.args[3] == 96
assert get_mla_metadata_v1.call_args.args[5] is False
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None
def test_asm_qlen2_without_segmented_fails_during_build(monkeypatch):
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=mock.MagicMock()),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
_cprr_decode_batch(
_builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
supports_segmented_dcp_verify=False,
),
qlen=2,
)
def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
"""Process-level CPRR capability must not pad or route a non-causal batch."""
captured = {}
def fake_aiter_decode(q, kv_buffer, out, *args, **kwargs):
captured["q_heads"] = q.shape[1]
captured["kwargs"] = kwargs
return None, torch.zeros(q.shape[0], q.shape[1])
monkeypatch.setattr(
rocm_aiter_mla, "_get_aiter_mla_decode", lambda: fake_aiter_decode
)
impl = object.__new__(AiterMLAImpl)
impl.num_heads = 12
impl.dcp_world_size = 8
impl.kv_cache_dtype = "auto"
impl.kv_lora_rank = 512
impl.qk_rope_head_dim = 64
impl.scale = 576**-0.5
decode = SimpleNamespace(
max_qo_len=4,
qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
paged_kv_indices=torch.zeros(1, dtype=torch.int32),
paged_kv_last_page_len=torch.ones(1, dtype=torch.int32),
use_gluon_decode=False,
use_gluon_verify=False,
dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN,
dcp_verify=None,
g_kv_indptr=None,
has_persistent_metadata=False,
attn_out_dtype=torch.bfloat16,
asm_decode_num_heads=0,
mla_num_kv_splits=0,
cp_world_size=8,
cp_rank=0,
min_kv_seq_len=1,
)
attn_metadata = SimpleNamespace(decode=decode, causal=False, work_meta_data=None)
layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0))
q = torch.zeros(4, 96, 576, dtype=torch.bfloat16)
output, lse = impl.forward_mqa(q, torch.zeros(1, 1, 576), attn_metadata, layer)
assert captured["q_heads"] == 96
assert captured["kwargs"]["causal"] is False
assert "g_kv_indptr" not in captured["kwargs"]
assert "num_kv_splits" not in captured["kwargs"]
assert output.shape[1] == 96
assert lse is not None and lse.shape == (4, 96)
def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
"""Single-token DCP decode must come back with an LSE the merge can use.
@@ -325,9 +554,14 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
paged_kv_last_page_len=torch.ones(num_tokens, dtype=torch.int32),
use_gluon_decode=False,
use_gluon_verify=False,
dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN,
dcp_verify=None,
has_persistent_metadata=False,
attn_out_dtype=torch.bfloat16,
# 0 == the round-robin (cprr) asm route is off, this rank's decode takes
# the existing path. Mirrors the AiterMLADecodeMetadata default.
asm_decode_num_heads=0,
g_kv_indptr=None,
)
attn_metadata = SimpleNamespace(decode=decode, causal=True, work_meta_data=None)
layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0))
@@ -453,6 +687,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
),
use_gluon_decode=False,
use_gluon_verify=False,
dcp_route=rocm_aiter_mla._DCPDecodeRoute.SEGMENTED,
dcp_verify=view,
attn_out_dtype=torch.bfloat16,
)
+10
View File
@@ -145,6 +145,7 @@ if TYPE_CHECKING:
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
VLLM_ROCM_USE_AITER_MLA: bool = True
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["asm", "segmented"] = "segmented"
VLLM_ROCM_USE_AITER_MHA: bool = True
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False
VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False
@@ -1309,6 +1310,15 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ROCM_USE_AITER_MLA": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MLA", "True").lower() in ("true", "1")
),
# Kernel for causal multi-token (spec-decode) verify steps under decode
# context parallelism on gfx950: "asm" uses AITER's round-robin ASM
# decode, "segmented" the Triton segmented MLA path.
"VLLM_ROCM_AITER_MLA_DCP_VERIFY": env_with_choices(
"VLLM_ROCM_AITER_MLA_DCP_VERIFY",
"segmented",
["asm", "segmented"],
case_sensitive=False,
),
# Small-head (<16) AITER MLA decode kernel selection. Small head counts
# (e.g. Kimi-K3: 12 heads/rank at TP8, 6 at TP16) can decode either through
# the Gluon small-head kernel or through the padded persistent-scheduling
+258 -23
View File
@@ -3,11 +3,13 @@
import functools
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from typing import TYPE_CHECKING, ClassVar, Final
import torch
from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
@@ -239,6 +241,78 @@ def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) ->
)
# DCP decode routing
#
# With decode context parallelism each rank holds a round-robin shard of the KV
# cache (token t on rank t % dcp_world_size; requires interleave size 1) and
# attends over it with all DCP-gathered query heads. Partial outputs are merged
# across ranks by LSE. Each decode batch takes one route:
#
# * plain: qlen == 1 or a non-causal block. Every query sees every local token.
# * cprr: causal qlen >= _MIN_CPRR_QLEN, gfx950, speculative decoding. AITER's
# round-robin ASM decode rebuilds global positions from g_kv_indptr, cp_rank
# and cp_world_size and applies the causal window in-kernel. Kernels exist only
# at _NATIVE_CPRR_HEADS; other gathered head counts are padded up.
# * segmented: causal qlen > 1 when cprr is not selected or qlen is below
# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV length
# encodes its causal window.
#
# The plain kernel must never serve a causal qlen > 1 block: it applies
# causality on local indices, which is wrong over a round-robin shard.
_NATIVE_CPRR_HEADS: Final = (16, 32, 64, 128)
_MIN_CPRR_QLEN: Final = 3
class _DCPDecodeRoute(Enum):
PLAIN = "plain"
SEGMENTED = "segmented"
CPRR = "cprr"
def _asm_dcp_verify_heads(decode_num_heads: int) -> int:
"""Head count the cprr asm decode runs at, or 0 if it cannot serve this shape."""
if decode_num_heads in _NATIVE_CPRR_HEADS:
return decode_num_heads
for nat in _NATIVE_CPRR_HEADS:
if nat >= decode_num_heads:
return nat
return 0
def _select_dcp_decode_route(
supports_segmented: bool,
causal: bool,
max_qo_len: int,
asm_selected: bool,
) -> _DCPDecodeRoute:
"""Select one DCP decode route for this batch; see DCP decode routing."""
if not causal or max_qo_len <= 1:
return _DCPDecodeRoute.PLAIN
if asm_selected and max_qo_len >= _MIN_CPRR_QLEN:
return _DCPDecodeRoute.CPRR
if supports_segmented:
return _DCPDecodeRoute.SEGMENTED
raise RuntimeError(
"ROCM_AITER_MLA DCP multi-token verify requires either segmented "
"MLA or the round-robin asm decode."
)
def _asm_dcp_verify_configured(
dcp_world_size: int, cp_interleave: int, multi_token_decode: bool
) -> bool:
"""Whether asm cprr DCP verify is reachable in this process.
Requires DCP with interleave size 1, speculative decoding, and gfx950 (the
only arch AITER builds cprr kernels for).
"""
if not (dcp_world_size > 1 and cp_interleave == 1 and multi_token_decode):
return False
from vllm.platforms.rocm import on_gfx950
return bool(on_gfx950())
def _aiter_mla_small_head_mode() -> str:
"""Small-head (<16) MLA decode kernel selection.
@@ -410,8 +484,19 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
max_qo_len: int | None = None
# Minimum KV length used by Gluon to choose a safe split count.
min_kv_seq_len: int = 1
# Exactly one DCP decode route is selected by the metadata builder.
dcp_route: _DCPDecodeRoute = _DCPDecodeRoute.PLAIN
# Set exactly when this batch routes to segmented DCP verification.
dcp_verify: AiterMLADCPVerifyMetadata | None = None
# cprr only: global per-request page indptr and this rank's position.
g_kv_indptr: torch.Tensor | None = None
cp_world_size: int = 1
cp_rank: int = 0
# Padded head count cprr runs at; 0 when this batch is not on cprr.
asm_decode_num_heads: int = 0
# Must equal the split cap the reduce scratch was sized with, or the kernel
# writes past it and faults.
mla_num_kv_splits: int = 0
# Small-head decode uses Gluon (avoids padding to 16).
use_gluon_decode: bool = False
# Small-head non-DCP multi-token verify uses Gluon's native MTP entry.
@@ -460,6 +545,9 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
# Set from the common metadata every build; a batch is causal unless the
# drafter says otherwise.
_decode_causal: bool = True
# A non-causal draft needs no cross-shard causal window: the ordinary
# mask0 decode returns per-row LSE for the DCP merge.
supports_non_causal_multi_token_dcp: ClassVar[bool] = True
@staticmethod
def _uniform_padded_mtp_qo_len(
@@ -513,15 +601,48 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
parallel_config.decode_context_parallel_size,
parallel_config.cp_kv_cache_interleave_size,
)
asm_dcp_verify_config = (
_asm_dcp_verify_configured(
parallel_config.decode_context_parallel_size,
parallel_config.cp_kv_cache_interleave_size,
multi_token_decode=vllm_config.speculative_config is not None,
)
and envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "asm"
)
super().__init__(
kv_cache_spec,
layer_names,
vllm_config,
device,
AiterMLAMetadata,
supports_dcp_with_varlen=supports_segmented_dcp_verify,
supports_dcp_with_varlen=(
supports_segmented_dcp_verify or asm_dcp_verify_config
),
)
self._asm_dcp_verify = False
self._asm_dcp_verify_heads = 0
# The head-count half of the route decision needs self.num_heads, so only
# the config half is known before super().__init__().
if asm_dcp_verify_config:
self._asm_dcp_verify_heads = _asm_dcp_verify_heads(
self.num_heads * self.dcp_world_size
)
if not self._asm_dcp_verify_heads:
raise ValueError(
"VLLM_ROCM_AITER_MLA_DCP_VERIFY=asm, but the round-robin asm "
f"decode has no kernel for {self.num_heads * self.dcp_world_size} "
f"DCP-gathered heads (native counts: {_NATIVE_CPRR_HEADS}). "
"Set VLLM_ROCM_AITER_MLA_DCP_VERIFY=segmented."
)
self._asm_dcp_verify = True
self._supports_segmented_dcp_verify = supports_segmented_dcp_verify
self._mla_max_split_per_batch = 0
if self._asm_dcp_verify:
# Cap on KV splits per batch; at low batch the KV axis is the only
# parallelism. Buffer sizing and every runtime call must see this value.
self._mla_max_split_per_batch = torch.cuda.get_device_properties(
device
).multi_processor_count
self.compilation_config = vllm_config.compilation_config
self.decode_attn_out_dtype = vllm_config.model_config.dtype
@@ -576,6 +697,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
self._num_attention_heads = AiterMLAHelper.get_actual_mla_num_heads(
self._decode_num_heads
)
# the cprr kernel exists only at the native head
# counts, so run (and size the persistent schedule) at the padded count.
# 96 is 16-aligned, so get_actual_mla_num_heads leaves it alone and the
# pad has to be applied here explicitly.
if self._asm_dcp_verify:
self._num_attention_heads = self._asm_dcp_verify_heads
kv_cache_dtype_str = getattr(vllm_config.cache_config, "cache_dtype", "auto")
if kv_cache_dtype_str in ("fp8", "fp8_e4m3", "fp8_e5m2"):
kv_cache_dtype_str = "fp8"
@@ -619,6 +746,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
kv_dtype,
is_sparse=False,
fast_mode=True,
# Must match max_split_per_batch in get_mla_metadata_v1; without it
# aiter sizes the reduce scratch from capacity (~9 GiB).
**(
dict(max_split_per_batch=self._mla_max_split_per_batch)
if self._asm_dcp_verify
else {}
),
)
self._mla_work_meta_data = torch.empty(
work_meta_data_size, dtype=work_meta_data_type, device=device
@@ -664,6 +798,14 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
device,
)
# Full cudagraphs need a stable address for the cprr global page indptr.
# Zero-initialized; element 0 is the indptr base and is never rewritten.
self._g_kv_indptr_buf: torch.Tensor | None = None
if self._asm_dcp_verify:
self._g_kv_indptr_buf = torch.zeros(
max_num_reqs + 1, dtype=torch.int32, device=device
)
# Persistent buffers for segmented DCP verification. Captured graphs
# require stable addresses while row lengths vary between replays.
self._dcp_verify_buffers: AiterMLADCPVerifyMetadata | None = None
@@ -683,8 +825,11 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
)
if self._supports_segmented_dcp_verify and self._mtp_decode_qlen > 1:
# A DCP rank's shard of the longest sequence bounds every verify
# row, and full graphs need that bound to be constant.
# Allocate even when CPRR is the preferred route: a later
# replay can fall below _MIN_CPRR_QLEN and needs these
# addresses. A DCP rank's shard of the longest sequence
# bounds every verify row, and full graphs need that bound
# to be constant.
num_dcp_partitions = (
self.dcp_world_size * self.cp_kv_cache_interleave_size
)
@@ -1063,6 +1208,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
pad_uniform_mtp = padded_mtp_qo_len > 0
seq_lens_for_kernel = seq_lens_device
# the global lengths need the same dummy-row
# treatment as the local ones below, but only for g_kv_indptr -- the
# metadata field keeps the true values, which is what the segmented
# route wants (it derives its own row lens and gives a dummy row
# length 0, correctly).
g_tot_seq_lens = dcp_tot_seq_lens_device
num_kernel_reqs = num_reqs
if pad_uniform_mtp:
qo_lens_device = (
@@ -1074,6 +1225,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
seq_lens_for_kernel,
seq_lens_for_kernel.new_full((), max_qo_len),
)
# A dummy row just got a LOCAL length of max_qo_len, so its GLOBAL
# length has to say the same thing or the cprr mask is degenerate:
# global_len 0 puts the row query positions at {-3,-2,-1} while its
# local shard sits at {rank, W+rank, 2W+rank}, so every entry is
# masked and the row softmaxes over nothing. max_qo_len * W is
# exactly the global length whose round-robin shard is max_qo_len
# on every rank.
if g_tot_seq_lens is not None:
g_tot_seq_lens = torch.where(
qo_lens_device > 0,
g_tot_seq_lens,
g_tot_seq_lens.new_full((), max_qo_len * self.dcp_world_size),
)
if self._graph_seq_lens is not None:
self._graph_seq_lens[:num_kernel_reqs].copy_(
@@ -1106,16 +1270,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
causal,
self._kv_cache_bytes,
)
use_segmented_dcp_verify = (
self._supports_segmented_dcp_verify and max_qo_len > 1
)
dcp_route = _DCPDecodeRoute.PLAIN
if self.dcp_world_size > 1:
dcp_route = _select_dcp_decode_route(
self._supports_segmented_dcp_verify,
causal,
int(max_qo_len),
self._asm_dcp_verify,
)
# Segmented DCP verify carries its own per-row subpage table, so the
# flat per-token view is dead work for it. Leave the buffer alone and
# hand the metadata None, so a future reader cannot pick up whatever
# the previous batch left behind.
paged_kv_indices = None
if not use_segmented_dcp_verify:
if dcp_route is not _DCPDecodeRoute.SEGMENTED:
if self.compilation_config.cudagraph_mode.has_full_cudagraphs():
self.paged_kv_indices.fill_(-1)
@@ -1169,6 +1338,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
else:
qo_indptr = query_start_loc_device[: 1 + num_kernel_reqs]
g_kv_indptr = None
if dcp_route is _DCPDecodeRoute.CPRR and g_tot_seq_lens is not None:
assert self._g_kv_indptr_buf is not None
_ngk = g_tot_seq_lens.shape[0]
g_kv_indptr = self._g_kv_indptr_buf[: _ngk + 1]
g_kv_indptr[1:].copy_(g_tot_seq_lens.cumsum(dim=0, dtype=torch.int32))
# A FULL cudagraph replays at its capture-time request count, so a
# smaller real batch leaves the tail holding the PREVIOUS step's
# cumsum. Flatten it the way paged_kv_indptr is flattened below, so
# padding rows read global_len == 0 and agree with their zero-length
# local shard instead of a stale global length.
self._g_kv_indptr_buf[_ngk + 1 :].fill_(g_kv_indptr[-1])
has_persistent_metadata = False
# Only the asm decode consumes the schedule, so gate on the routing
# rather than on num_heads >= 16, which denies it to a padded rank
@@ -1178,7 +1360,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
use_persistent_metadata = (
not use_gluon_decode
and not use_gluon_verify
and not use_segmented_dcp_verify
and dcp_route is not _DCPDecodeRoute.SEGMENTED
# A padded rank has no bf16 persistent kernel past qlen 4 where the
# gfx950 fold is absent; the non-persistent entry covers it. fp8
# keeps the schedule -- its fold rejects non-persistent outright.
@@ -1218,13 +1400,28 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
uni_qo_len = (
max_qo_len if pad_uniform_mtp or torch.all(qo_len == max_qo_len) else -1
)
cprr_kwargs: dict = {}
is_causal = causal
if self._asm_dcp_verify:
cprr_kwargs = dict(
max_split_per_batch=self._mla_max_split_per_batch,
intra_batch_mode=False,
)
if dcp_route is _DCPDecodeRoute.CPRR:
is_causal = False
cprr_kwargs["is_cp_round_robin"] = True
metadata_num_heads = (
self._asm_dcp_verify_heads
if dcp_route is _DCPDecodeRoute.CPRR
else AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads)
)
get_mla_metadata_v1(
qo_indptr,
paged_kv_indptr,
paged_kv_last_page_len,
self._num_attention_heads,
metadata_num_heads,
1,
causal,
is_causal,
self._mla_work_meta_data,
self._mla_work_info_set,
self._mla_work_indptr,
@@ -1238,6 +1435,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
fast_mode=True,
dtype_q=self._mla_q_dtype,
dtype_kv=self._mla_kv_dtype,
**cprr_kwargs,
)
has_persistent_metadata = True
@@ -1261,7 +1459,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
min_kv_seq_len = int(per_req_len.min().item())
dcp_verify = None
if use_segmented_dcp_verify:
if dcp_route is _DCPDecodeRoute.SEGMENTED:
assert dcp_tot_seq_lens_device is not None
dcp_verify = self._build_dcp_verify_row_view(
int(max_qo_len),
@@ -1279,7 +1477,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
dcp_tot_seq_lens=dcp_tot_seq_lens_device,
max_qo_len=max_qo_len,
min_kv_seq_len=min_kv_seq_len,
dcp_route=dcp_route,
dcp_verify=dcp_verify,
g_kv_indptr=g_kv_indptr,
cp_world_size=self.dcp_world_size,
cp_rank=self.dcp_rank,
asm_decode_num_heads=(
self._asm_dcp_verify_heads if dcp_route is _DCPDecodeRoute.CPRR else 0
),
mla_num_kv_splits=(
self._mla_max_split_per_batch
if dcp_route is _DCPDecodeRoute.CPRR
else 0
),
use_gluon_decode=use_gluon_decode,
use_gluon_verify=use_gluon_verify,
attn_out_dtype=self.decode_attn_out_dtype,
@@ -2062,8 +2272,10 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
)
return o, None
verify = decode.dcp_verify
if verify is not None:
dcp_route = decode.dcp_route
if dcp_route is _DCPDecodeRoute.SEGMENTED:
verify = decode.dcp_verify
assert verify is not None
if type(q) is tuple:
q_nope, q_pe = q
else:
@@ -2085,11 +2297,6 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
decode.attn_out_dtype,
)
if self.dcp_world_size > 1 and int(decode.max_qo_len) > 1:
raise RuntimeError(
"ROCM_AITER_MLA DCP multi-token verify requires segmented MLA."
)
if type(q) is tuple:
q = torch.cat(q, dim=-1)
@@ -2102,8 +2309,19 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
"ROCM_AITER_MLA decode expected the DCP-gathered query head count "
f"{self._decode_num_heads}, got {q.shape[1]}"
)
mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q)
mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads(self._decode_num_heads)
# cprr has kernels only at native head counts, so pad q up (e.g. 96 ->
# 128). Heads are independent in MLA; the extras are sliced off below.
asm_dcp_heads = decode.asm_decode_num_heads
if asm_dcp_heads > self._decode_num_heads:
mla_num_heads = asm_dcp_heads
mla_padded_q = AiterMLAHelper.get_mla_padded_q(
self._decode_num_heads, q, target_heads=mla_num_heads
)
else:
mla_padded_q = AiterMLAHelper.get_mla_padded_q(self._decode_num_heads, q)
mla_num_heads = AiterMLAHelper.get_actual_mla_num_heads(
self._decode_num_heads
)
o = torch.empty(
B,
mla_num_heads,
@@ -2137,6 +2355,16 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
lse = None
if self.dcp_world_size > 1:
if dcp_route is _DCPDecodeRoute.CPRR:
# cprr only: global positions for the in-kernel causal window.
assert decode.g_kv_indptr is not None
mla_kwargs["g_kv_indptr"] = decode.g_kv_indptr
mla_kwargs["cp_world_size"] = decode.cp_world_size
mla_kwargs["cp_rank"] = decode.cp_rank
# Must match the schedule get_mla_metadata_v1 was built with.
assert decode.mla_num_kv_splits > 0
mla_kwargs["num_kv_splits"] = decode.mla_num_kv_splits
mla_kwargs["intra_batch_mode"] = False
# The vLLM custom-op wrapper exposes only the in-place output and
# drops aiter's final LSE, which the cross-shard merge needs, so go
# through aiter's native entry point on the DCP path.
@@ -2151,6 +2379,7 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
decode.max_qo_len,
sm_scale=self.scale,
return_lse=True,
causal=attn_metadata.causal,
**mla_kwargs,
)
assert lse is not None, (
@@ -2172,7 +2401,13 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
**mla_kwargs,
)
output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o)
if lse is not None:
lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse)
if asm_dcp_heads > self._decode_num_heads:
# Keep a view: a post-kernel copy races the symm-mem a2a combine.
output = o[:, : self._decode_num_heads]
if lse is not None:
lse = lse[:, : self._decode_num_heads, ...]
else:
output = AiterMLAHelper.get_mla_unpadded_o(self._decode_num_heads, o)
if lse is not None:
lse = AiterMLAHelper.get_mla_unpadded_lse(self._decode_num_heads, lse)
return output, lse