[ROCm][MLA] Add an AITER ASM round-robin decode route for DCP multi-token verify (#56861)

Signed-off-by: Xiaohu Guo <Xiaohu.Guo@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: seungrokj <144636725+seungrokj@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
xiaohuguo2023
2026-09-28 14:52:21 -04:00
committed by GitHub
co-authored by Claude Opus 5 seungrokj Cursor
parent 5ba2e3090b
commit 25624f65cf
5 changed files with 1202 additions and 30 deletions
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routing/selection tests for the AITER ASM cprr DCP-verify route.
Companion to the upstream segmented-verify tests. Everything here is pure CPU:
it pins the *decisions* (which route a KV group takes, which head count the
kernel runs at, which configurations are refused) rather than kernel numerics.
The numerics parity test against the segmented route needs a GPU and both
routes built; it is marked and skipped when unavailable.
"""
from __future__ import annotations
import pytest
rocm_aiter_mla = pytest.importorskip(
"vllm.v1.attention.backends.mla.rocm_aiter_mla",
reason="ROCm AITER MLA backend not available",
)
_heads = rocm_aiter_mla._asm_dcp_verify_heads
_configured = rocm_aiter_mla._asm_dcp_verify_configured
_select_route = rocm_aiter_mla._select_dcp_decode_route
Route = rocm_aiter_mla._DCPDecodeRoute
NATIVE = rocm_aiter_mla._NATIVE_CPRR_HEADS
MIN_QLEN = rocm_aiter_mla._MIN_CPRR_QLEN
def test_dcp_verify_env_defaults_to_segmented(monkeypatch):
monkeypatch.delenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", raising=False)
assert rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY == "segmented"
def test_dcp_verify_env_rejects_unknown_route(monkeypatch):
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "unknown")
with pytest.raises(ValueError, match="Invalid value"):
_ = rocm_aiter_mla.envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY
# --------------------------------------------------------------------------
# _asm_dcp_verify_heads -- which head count the cprr kernel runs at
# --------------------------------------------------------------------------
@pytest.mark.parametrize("n", NATIVE)
def test_native_head_counts_are_used_as_is(n):
assert _heads(n) == n
def test_non_native_head_count_pads_up_to_the_next_native():
"""K3 at TP8/DCP8 gathers 96 heads for the target; the kernel has no 96
variant, so it must run at 128, not silently fall out of the asm path."""
assert _heads(96) == 128
assert _heads(1) == 16
assert _heads(17) == 32
assert _heads(65) == 128
def test_head_count_above_the_largest_native_is_unservable():
"""0 means 'cannot serve'; anything else would route a shape the kernel
has no build for."""
assert _heads(max(NATIVE) + 1) == 0
assert _heads(256) == 0
# --------------------------------------------------------------------------
# _asm_dcp_verify_configured -- reachability
# --------------------------------------------------------------------------
@pytest.fixture
def on_gfx950(monkeypatch):
"""The cprr kernels are built only for gfx950, so the gate checks the arch.
Patched rather than detected, so both branches run on any runner."""
import vllm.platforms.rocm as rocm
monkeypatch.setattr(rocm, "on_gfx950", lambda: True)
def test_route_needs_dcp(on_gfx950):
assert _configured(dcp_world_size=1, cp_interleave=1, multi_token_decode=True) is (
False
)
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is (
True
)
def test_round_robin_interleave_other_than_one_is_excluded(on_gfx950):
"""The kernel's global-position causal window assumes interleave 1, the
same restriction the segmented gate carries."""
assert _configured(dcp_world_size=8, cp_interleave=2, multi_token_decode=True) is (
False
)
assert _configured(dcp_world_size=8, cp_interleave=16, multi_token_decode=True) is (
False
)
def test_route_needs_multi_token_decode(on_gfx950):
"""Single-token decode is already served. Enabling the route for it would
pad the head count on every step and buy nothing."""
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=False) is (
False
)
def test_route_needs_gfx950(monkeypatch):
"""hsa/gfx942/mla and hsa/gfx1250/mla carry no cprr rows, so off gfx950 the
kernel lookup fails at the first verify step rather than falling back."""
import vllm.platforms.rocm as rocm
monkeypatch.setattr(rocm, "on_gfx950", lambda: False)
assert _configured(dcp_world_size=8, cp_interleave=1, multi_token_decode=True) is (
False
)
# --------------------------------------------------------------------------
# qlen floor
# --------------------------------------------------------------------------
def test_min_cprr_qlen_is_above_two():
"""Qlen 2 is below CPRR's floor and must use segmented verification."""
assert MIN_QLEN > 2
@pytest.mark.parametrize(
"supports,causal,qlen,asm,expected",
[
(True, True, 1, True, Route.PLAIN),
(True, True, 2, True, Route.SEGMENTED),
(True, True, 3, True, Route.CPRR),
(True, True, 4, True, Route.CPRR),
(True, True, 4, False, Route.SEGMENTED),
(True, False, 2, True, Route.PLAIN),
(True, False, 4, False, Route.PLAIN),
(False, True, 3, True, Route.CPRR),
],
)
def test_dcp_decode_route(supports, causal, qlen, asm, expected):
assert _select_route(supports, causal, qlen, asm) is expected
def test_causal_multi_token_batch_without_a_valid_route_fails():
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
_select_route(False, True, 2, True)
# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs
# the real builder and the real asm kernel on a gfx950 GPU.
@@ -0,0 +1,540 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GPU numerics for the AITER ASM cprr DCP-verify route.
Runs the REAL metadata builder (``AiterMLAMetadataBuilder.build``, so
``get_mla_metadata_v1`` is exercised with the cprr kwargs and the persistent
schedule buffers are produced) and the REAL asm kernel, on ONE GPU with the DCP
ranks simulated one after another.
That is legitimate because the aiter MLA backend contains no collective on this
path: the query all-gather happens in the *layer*, strictly before
``forward_mqa``, so the head fold, the round-robin kernel call and the un-fold
are pure per-rank local arithmetic in which ``cp_world_size``/``cp_rank`` are
just numbers feeding the causal-mask maths. Restricting a shard to a residue
class of global positions is exactly what DCP rank r sees.
Three properties are checked:
* each shard matches an exact torch reference over the positions it holds,
with causality applied on the GLOBAL position (the thing cprr reconstructs
in-kernel and the segmented route instead carries in per-row lengths);
* the LSE merge of all shards reproduces full-context attention;
* a shard told the wrong ``cp_rank`` fails both -- without this positive
control a passing run proves nothing, since a kernel that ignored global
positions still produces plausible-looking output.
"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("ROCm AITER MLA tests", allow_module_level=True)
from vllm._aiter_ops import is_aiter_found # noqa: E402
from vllm.v1.attention.backends.mla import rocm_aiter_mla # noqa: E402
KV_LORA_RANK = 512
QK_ROPE_HEAD_DIM = 64
QK_NOPE_HEAD_DIM = 128
HEAD_SIZE = KV_LORA_RANK + QK_ROPE_HEAD_DIM
SCALE = (QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM) ** -0.5
DCP = 8
NUM_REQS = 8
CTX = 256
QLEN = 5 # 1 + num_speculative_tokens; must clear _MIN_CPRR_QLEN
# fp8-e4m3 keeps 3 mantissa bits, so the relative error floor is ~6.2e-2. Gates
# are relative for the same reason: an absolute bound would just be a bound on
# the output magnitude.
SHARD_RTOL = 1.0e-1
MERGE_RTOL = 5.0e-2
MODEL = "deepseek-ai/DeepSeek-R1"
def _gpu_available() -> bool:
return is_aiter_found() and torch.accelerator.is_available()
def _on_gfx950() -> bool:
if not is_aiter_found():
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
requires_cprr = pytest.mark.skipif(
not (_gpu_available() and _on_gfx950()),
reason="cprr DCP verify needs ROCm + AITER on gfx950",
)
@pytest.fixture(autouse=True)
def select_asm_route(monkeypatch):
monkeypatch.setenv("VLLM_ROCM_AITER_MLA_DCP_VERIFY", "asm")
# --------------------------------------------------------------------------
# reference maths
# --------------------------------------------------------------------------
def _torch_reference(q_f32, kv_f32, ctx, qlen, scale, positions):
"""Exact absorbed-MLA decode over an explicit set of GLOBAL KV positions.
A shard sees only ``positions``, but causality still uses the global
position: query token i (global position ctx + i) may attend to any p in
positions with p <= ctx + i. Restricting ``positions`` to a round-robin
residue class is precisely DCP rank r's view.
"""
nr, _, heads, _ = q_f32.shape
dev = q_f32.device
o = torch.zeros(nr, qlen, heads, KV_LORA_RANK, dtype=torch.float32, device=dev)
lse = torch.full((nr, qlen, heads), float("-inf"), dtype=torch.float32, device=dev)
for r in range(nr):
kv_sel = kv_f32[r].index_select(0, positions)
for i in range(qlen):
valid = positions <= (ctx + i)
if not bool(valid.any()):
continue
k = kv_sel[valid]
s = (q_f32[r, i] @ k.T) * scale
m = s.max(dim=-1, keepdim=True).values
p = torch.exp(s - m)
denom = p.sum(dim=-1, keepdim=True)
o[r, i] = (p / denom) @ k[:, :KV_LORA_RANK]
lse[r, i] = m.squeeze(-1) + torch.log(denom.squeeze(-1))
return o, lse
def _merge_partials(os_, lses):
"""Cross-shard softmax merge (natural-log LSE)."""
stack_lse = torch.stack(lses)
m = stack_lse.max(dim=0).values
w = torch.exp(stack_lse - m)
num = (w.unsqueeze(-1) * torch.stack(os_)).sum(dim=0)
den = w.sum(dim=0).unsqueeze(-1)
return num / den, m + torch.log(w.sum(dim=0))
# --------------------------------------------------------------------------
# single-process rank simulation
# --------------------------------------------------------------------------
class _FakeGroup:
def __init__(self, world_size: int, rank: int):
self.world_size = world_size
self.rank_in_group = rank
self.rank = rank
self.local_rank = rank
def all_gather(self, tensor, dim=0):
raise RuntimeError(
"collective called in a single-process test -- the path under test "
"is supposed to be collective-free; this indicates a real change"
)
def _install_fake_groups(dcp_size: int, cp_rank: int, tp_size: int = 1) -> None:
"""Point vLLM's parallel_state at rank-simulating stubs (no NCCL)."""
import vllm.distributed.parallel_state as ps
import vllm.model_executor.layers.attention.mla_attention as mla_attention
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
dcp = _FakeGroup(dcp_size, cp_rank)
tp = _FakeGroup(tp_size, cp_rank)
ps.get_dcp_group = lambda: dcp
ps.get_tp_group = lambda: tp
# Both of these imported the symbol directly at module import time, so
# patching parallel_state alone leaves them resolving the real group (and
# the builder then reports dcp_world_size 1, silently skipping the route).
backend.get_dcp_group = lambda: dcp
mla_attention.get_dcp_group = lambda: dcp
@pytest.fixture(autouse=True)
def _restore_parallel_groups():
"""Undo _install_fake_groups after every test.
It patches module globals in three modules, so without this the fake DCP
group leaks into the rest of the pytest session and silently breaks
unrelated tests that expect a real (or absent) group.
"""
import vllm.distributed.parallel_state as ps
import vllm.model_executor.layers.attention.mla_attention as mla_attention
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
saved = [
(mod, name, getattr(mod, name))
for mod, name in (
(ps, "get_dcp_group"),
(ps, "get_tp_group"),
(backend, "get_dcp_group"),
(mla_attention, "get_dcp_group"),
)
]
yield
for mod, name, value in saved:
setattr(mod, name, value)
class _PrefillBackendStub:
"""The base builder does ``attention_layer.prefill_backend.clone()``.
prefill_backend is consulted only from the chunked-context prefill path,
which a decode-only test never enters.
"""
def clone(self):
return self
def _make_ctx_layer(vllm_config):
"""Stand-in for the registered attention layer in static_forward_context.
``MLACommonMetadataBuilder.__init__`` reads the MLA dims, ``dcp_manager``
and ``prefill_backend`` off this object. ``dcp_manager`` is the Direct-DCP
symmetric-memory workspace, used ONLY by the chunked-prefill KV all-gather;
it is constructed without running __init__ so no peer pointers are exchanged
and no collective is issued. That is what keeps this test single-process.
"""
from vllm.v1.attention.ops.dcp import MLADCPManager
mgr = object.__new__(MLADCPManager)
mgr.init_kv_gather = lambda *a, **k: None
return SimpleNamespace(
non_causal_multi_token_decode=False,
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
v_head_dim=QK_NOPE_HEAD_DIM,
dcp_manager=mgr,
prefill_backend=_PrefillBackendStub(),
)
def _ensure_workspace_manager(device: torch.device) -> None:
"""Init the global workspace manager, which GPUModelRunner does in production.
``AiterMLAMetadataBuilder.__init__`` reserves the fp8 prefill
persistent-scheduling scratch through ``current_workspace_manager()``, so
merely constructing a builder outside a serve trips its assert. Idempotent:
one builder is constructed per DCP shard. Deliberately does NOT lock the
workspace -- production locks only after warmup, and this keeps allocating
across shards.
"""
from vllm.v1.worker.workspace import (
init_workspace_manager,
is_workspace_manager_initialized,
)
if not is_workspace_manager_initialized():
init_workspace_manager(device)
class _Layer:
def __init__(self, device):
self._q_scale = torch.tensor(1.0, device=device)
self._k_scale = torch.tensor(1.0, device=device)
self._v_scale = torch.tensor(1.0, device=device)
def _build_impl(num_heads: int, dcp: int, cp_rank: int):
"""Real AiterMLAImpl.forward_mqa bound to a minimally-populated instance.
Constructing the full impl would drag in kv_b_proj and the rest of the
weight-dependent MLA setup -- i.e. model loading, which this test exists to
avoid. forward_mqa's fold path reads only the attributes set below.
"""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
impl = object.__new__(AiterMLAImpl)
impl.kv_lora_rank = KV_LORA_RANK
impl.qk_rope_head_dim = QK_ROPE_HEAD_DIM
impl.num_heads = num_heads
impl.dcp_world_size = dcp
impl.dcp_rank = cp_rank
impl.pcp_world_size = 1
impl.kv_cache_dtype = "fp8"
impl.scale = SCALE
# `_decode_num_heads` is a read-only property deriving num_heads *
# dcp_world_size, both set above; assert rather than assign.
assert impl._decode_num_heads == num_heads * dcp
return impl
def _make_vllm_config(heads_per_rank: int):
"""A DCP8 MLA config with no weights and no distributed init.
DeepSeek-R1's config supplies the MLA dims (kv_lora_rank 512, rope 64,
nope 128) that the aiter asm kernel is built for; only the head count is
overridden, to place the GATHERED count (heads_per_rank * DCP) on the
kernel variant under test. ``get_num_attention_heads`` reads
model_arch_config, a snapshot taken at config-creation time, so patching
hf_config alone would be inert.
"""
from tests.v1.attention.utils import create_vllm_config
vllm_config = create_vllm_config(
model_name=MODEL,
tensor_parallel_size=1,
max_model_len=2048,
max_num_seqs=NUM_REQS * 2,
max_num_batched_tokens=4096,
)
vllm_config.model_config.model_arch_config.total_num_attention_heads = (
heads_per_rank
)
# Without a speculative config the batch reorder threshold stays at 1 and a
# qlen-QLEN row is classified as a PREFILL, so the decode path under test is
# never reached. _init_reorder_batch_threshold reads exactly these two
# fields, so a stand-in is enough and avoids pulling in a draft model.
vllm_config.speculative_config = SimpleNamespace(
num_speculative_tokens=QLEN - 1, parallel_drafting=False
)
vllm_config.cache_config.cache_dtype = "fp8"
vllm_config.parallel_config.decode_context_parallel_size = DCP
vllm_config.parallel_config.cp_kv_cache_interleave_size = 1
return vllm_config
def _run_shard(
vllm_config,
spec,
layer_name,
device,
cp_rank,
kern_rank,
q_flat,
kv_flat,
nr,
qlen,
ctx,
dcp,
pos,
):
"""Build real metadata for one DCP rank and run the real asm kernel.
``kern_rank`` is separate from ``cp_rank`` so the positive control can hand
rank r's pages to the kernel while telling it that it is rank r+1.
"""
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
L = ctx + qlen
local_len = int(pos.numel())
# One GLOBAL kv cache: page (req*L + p) holds request req position p. Shards
# select pages by residue class, so every shard reads the same underlying
# bytes and no re-quantisation noise is introduced between arms.
block_table = (
torch.arange(nr, device=device).view(nr, 1) * L + pos.view(1, -1)
).to(torch.int32)
# seq_lens is the GLOBAL context; dcp_local_seq_lens is what this rank
# actually holds. Passing the global length as the local one walks the
# kernel off the end of the page table.
seq_lens_cpu = torch.full((nr,), L, dtype=torch.int32)
local_cpu = torch.full((nr,), local_len, dtype=torch.int32)
common = CommonAttentionMetadata(
query_start_loc=torch.arange(
0, nr * qlen + 1, qlen, dtype=torch.int32, device=device
),
query_start_loc_cpu=torch.arange(0, nr * qlen + 1, qlen, dtype=torch.int32),
seq_lens=seq_lens_cpu.to(device),
num_reqs=nr,
num_actual_tokens=nr * qlen,
max_query_len=qlen,
max_seq_len=L,
block_table_tensor=block_table,
slot_mapping=torch.arange(nr * qlen, dtype=torch.int64, device=device),
causal=True,
dcp_local_seq_lens=local_cpu.to(device),
dcp_local_seq_lens_cpu_upper_bound=local_cpu,
)
_install_fake_groups(dcp, kern_rank)
builder = AiterMLAMetadataBuilder(spec, [layer_name], vllm_config, device)
md = builder.build(0, common)
impl = _build_impl(builder.num_heads, dcp, kern_rank)
o, lse = impl.forward_mqa(q_flat, kv_flat, md, _Layer(device))
torch.accelerator.synchronize()
return md, o, lse, local_len
def _numerics(heads_per_rank: int, sabotage: bool = False):
"""Run every DCP shard and return (per-shard max rel err, merged rel err)."""
import aiter
from vllm.config import set_current_vllm_config
device = torch.device("cuda", 0)
torch.accelerator.set_device_index(device.index)
torch.manual_seed(0)
_ensure_workspace_manager(device)
_install_fake_groups(DCP, 0)
vllm_config = _make_vllm_config(heads_per_rank)
with set_current_vllm_config(vllm_config):
from vllm.v1.kv_cache_interface import MLAAttentionSpec
layer_name = "model.layers.0.self_attn.attn"
vllm_config.compilation_config.static_forward_context[layer_name] = (
_make_ctx_layer(vllm_config)
)
# block_size 1: one page per token, so page ids are positions and the
# residue-class page table is expressible directly.
spec = MLAAttentionSpec(
block_size=1,
num_kv_heads=1,
head_size=HEAD_SIZE,
dtype=vllm_config.model_config.dtype,
non_causal_multi_token_decode=False,
)
nr, ctx, qlen = NUM_REQS, CTX, QLEN
L = ctx + qlen
gathered_heads = heads_per_rank * DCP
kv_dtype = aiter.dtypes.fp8
kv_flat = torch.randn(
nr * L, 1, HEAD_SIZE, dtype=torch.float32, device=device
).to(kv_dtype)
q_flat = (
torch.randn(
nr * qlen,
gathered_heads,
HEAD_SIZE,
dtype=torch.float32,
device=device,
)
.mul_(0.5)
.to(kv_dtype)
)
# Dequantise for the reference so the comparison measures the kernel,
# not the fp8 cast that both sides share.
kv_f32 = kv_flat.float().view(nr, L, HEAD_SIZE)
q_f32 = q_flat.float().view(nr, qlen, gathered_heads, HEAD_SIZE)
ref_o, _ = _torch_reference(
q_f32, kv_f32, ctx, qlen, SCALE, torch.arange(L, device=device)
)
shard_o, shard_lse = [], []
max_shard_rel = 0.0
for r in range(DCP):
pos = torch.arange(r, L, DCP, device=device) # residue class r
kern_rank = (r + 1) % DCP if sabotage else r
md, o, lse, _ = _run_shard(
vllm_config,
spec,
layer_name,
device,
r,
kern_rank,
q_flat,
kv_flat,
nr,
qlen,
ctx,
DCP,
pos,
)
# The route actually under test must have been taken; a silent
# fall-out to Triton/segmented would otherwise pass quietly.
assert md.decode.dcp_route is rocm_aiter_mla._DCPDecodeRoute.CPRR
assert md.decode.asm_decode_num_heads, "cprr asm route was not taken"
o = o.float().view(nr, qlen, gathered_heads, KV_LORA_RANK)
shard_lse.append(lse.float().view(nr, qlen, gathered_heads))
shard_o.append(o)
po, _ = _torch_reference(q_f32, kv_f32, ctx, qlen, SCALE, pos)
rel = (o - po).abs().max().item() / max(po.abs().max().item(), 1e-6)
max_shard_rel = max(max_shard_rel, rel)
merged, _ = _merge_partials(shard_o, shard_lse)
merge_rel = (merged - ref_o).abs().max().item() / max(
ref_o.abs().max().item(), 1e-6
)
return max_shard_rel, merge_rel
# --------------------------------------------------------------------------
# tests
# --------------------------------------------------------------------------
@requires_cprr
@pytest.mark.parametrize(
"heads_per_rank",
[
16, # 128 gathered heads -- a NATIVE cprr kernel, no padding
12, # 96 gathered heads -- K3's DSpark target, must pad up to 128
],
ids=["native-128", "padded-96-to-128"],
)
@torch.inference_mode()
def test_cprr_shards_match_reference_and_merge_to_full_context(heads_per_rank):
"""Every DCP shard must be right, and their LSE merge must reproduce
full-context attention.
A shard that ignored global positions (applying a local causal tail
instead) still produces plausible-looking output; it only diverges from the
reference on the tokens whose causal window is truncated. Checking the merge
alone would hide that, so both are asserted.
"""
shard_rel, merge_rel = _numerics(heads_per_rank)
assert shard_rel < SHARD_RTOL, f"per-shard rel err {shard_rel:.3e}"
assert merge_rel < MERGE_RTOL, f"merged rel err {merge_rel:.3e}"
@requires_cprr
@torch.inference_mode()
def test_wrong_cp_rank_breaks_the_result():
"""Positive control: without it, a passing run above proves nothing.
Feeding shard r's pages to the kernel while telling it that it is rank r+1
shifts every reconstructed global position by one, so both the causal mask
and the round-robin reconstruction are wrong. If the error does NOT blow up,
the kernel is not using cp_rank and the test above has no power.
"""
shard_rel, merge_rel = _numerics(16, sabotage=True)
assert shard_rel > SHARD_RTOL or merge_rel > MERGE_RTOL, (
f"sabotaged run still passed (shard {shard_rel:.3e}, merge "
f"{merge_rel:.3e}) -- cp_rank is not reaching the kernel"
)
@requires_cprr
def test_cprr_kernel_is_selected_for_the_k3_shape():
"""The K3 DSpark target shape must actually reach the cprr route.
Guards against a silent fall-out to segmented/Triton, which is what the
stock configuration does and what this route exists to avoid.
"""
from vllm.v1.attention.backends.mla import rocm_aiter_mla as m
assert (
m._asm_dcp_verify_configured(
dcp_world_size=DCP, cp_interleave=1, multi_token_decode=True
)
is True
)
assert QLEN >= m._MIN_CPRR_QLEN
assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR
assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS
@@ -87,21 +87,31 @@ def _builder(
kv_cache_dtype: str = "auto",
dcp_world_size: int = 1,
dcp_rank: int = 0,
asm_dcp_verify: bool = False,
decode_causal: bool = True,
supports_segmented_dcp_verify: bool | None = None,
):
decode_num_heads = num_heads * dcp_world_size
asm_heads = (
rocm_aiter_mla._asm_dcp_verify_heads(decode_num_heads) if asm_dcp_verify else 0
)
if supports_segmented_dcp_verify is None:
supports_segmented_dcp_verify = rocm_aiter_mla._segmented_dcp_verify_supported(
dcp_world_size, 1
)
stub = SimpleNamespace(
device=torch.device("cpu"),
num_heads=num_heads,
# DCP gathers every rank's head shard before decode, and the routing
# predicates read the gathered count.
_decode_num_heads=num_heads * dcp_world_size,
_decode_num_heads=decode_num_heads,
dcp_world_size=dcp_world_size,
dcp_rank=dcp_rank,
cp_kv_cache_interleave_size=1,
# Mirrors the production constructor: configuration decides whether the
# segmented route is available, query length decides per batch.
_supports_segmented_dcp_verify=rocm_aiter_mla._segmented_dcp_verify_supported(
dcp_world_size, 1
),
# segmented route is available, query length decides per batch. CPRR
# preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back.
_supports_segmented_dcp_verify=supports_segmented_dcp_verify,
# Derived once in the real constructor, so derive it once here too.
_segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size),
_dcp_verify_buffers=None,
@@ -122,7 +132,11 @@ def _builder(
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
_use_persistent_metadata=False,
kernel_block_size=kernel_block_size,
_num_attention_heads=AiterMLAHelper.get_actual_mla_num_heads(num_heads),
_num_attention_heads=(
asm_heads
if asm_dcp_verify
else AiterMLAHelper.get_actual_mla_num_heads(num_heads)
),
_mla_work_meta_data=torch.empty(1, dtype=torch.int32),
_mla_work_info_set=torch.empty(1, dtype=torch.int32),
_mla_work_indptr=torch.empty(1, dtype=torch.int32),
@@ -132,7 +146,19 @@ def _builder(
_mla_q_dtype=torch.bfloat16,
_mla_kv_dtype=torch.bfloat16,
decode_attn_out_dtype=torch.bfloat16,
_decode_causal=True,
_decode_causal=decode_causal,
# Default off so existing cases keep exercising segmented/persistent.
# Pass asm_dcp_verify=True for the CPRR route and its qlen-floor fallback.
_asm_dcp_verify=asm_dcp_verify,
_asm_dcp_verify_heads=asm_heads,
_g_kv_indptr_buf=(
torch.zeros(max_decode_rows + 1, dtype=torch.int32)
if asm_dcp_verify and dcp_world_size > 1
else None
),
# Only read when the asm route is on; the real builder derives it from
# the device CU count.
_mla_max_split_per_batch=256,
)
# Bound to the stub rather than faked: the verify flatten's per-row view is
# part of what _build_decode is being tested for.
@@ -287,6 +313,209 @@ def test_dcp_fp8_verify_build_uses_segmented(monkeypatch):
assert metadata.dcp_verify is not None
def _cprr_decode_batch(builder, *, qlen: int, num_reqs: int = 1):
seq_lens = torch.full((num_reqs,), 16, dtype=torch.int32)
query_start_loc = torch.arange(0, (num_reqs + 1) * qlen, qlen, dtype=torch.int32)
return AiterMLAMetadataBuilder._build_decode(
builder,
block_table_tensor=torch.zeros(num_reqs, 8, dtype=torch.int32),
seq_lens_device=seq_lens,
max_seq_len=16,
query_start_loc_cpu=query_start_loc,
query_start_loc_device=query_start_loc,
num_decode_tokens=num_reqs * qlen,
max_query_len=qlen,
dcp_tot_seq_lens_device=seq_lens * 8,
)
def test_asm_preferred_qlen2_falls_back_to_segmented(monkeypatch):
"""K=4 can clamp to qlen 2; CPRR has no kernel, so use segmented MLA."""
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
metadata = _cprr_decode_batch(
_builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
kernel_block_size=2,
),
qlen=2,
)
assert metadata.dcp_verify is not None
assert metadata.g_kv_indptr is None
assert not metadata.has_persistent_metadata
get_mla_metadata_v1.assert_not_called()
def test_asm_preferred_qlen_at_cprr_floor_stays_on_cprr(monkeypatch):
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
metadata = _cprr_decode_batch(
_builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
),
qlen=rocm_aiter_mla._MIN_CPRR_QLEN,
)
assert metadata.dcp_verify is None
assert metadata.g_kv_indptr is not None
assert metadata.has_persistent_metadata
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is True
def test_cprr_capable_single_token_decode_keeps_persistent_metadata(monkeypatch):
"""The CPRR qlen floor must not suppress the ordinary qlen-1 schedule."""
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
metadata = _cprr_decode_batch(
_builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
),
qlen=1,
)
assert metadata.g_kv_indptr is None
assert metadata.has_persistent_metadata
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None
def test_non_causal_dcp_block_uses_plain_mask0_metadata(monkeypatch):
"""A non-causal block needs no CPRR global-position window or head pad."""
get_mla_metadata_v1 = mock.MagicMock()
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=get_mla_metadata_v1),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
builder = _builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
)
builder._decode_causal = False
metadata = _cprr_decode_batch(builder, qlen=4)
assert metadata.dcp_verify is None
assert metadata.g_kv_indptr is None
assert metadata.has_persistent_metadata
assert get_mla_metadata_v1.call_args.args[3] == 96
assert get_mla_metadata_v1.call_args.args[5] is False
assert get_mla_metadata_v1.call_args.kwargs.get("is_cp_round_robin") is None
def test_asm_qlen2_without_segmented_fails_during_build(monkeypatch):
monkeypatch.setitem(
sys.modules,
"aiter",
SimpleNamespace(get_mla_metadata_v1=mock.MagicMock()),
)
monkeypatch.setattr(
rocm_aiter_mla, "_expand_page_indices_kernel", _NoOpTritonKernel()
)
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
_cprr_decode_batch(
_builder(
mtp_decode_qlen=5,
dcp_world_size=8,
num_heads=12,
asm_dcp_verify=True,
supports_segmented_dcp_verify=False,
),
qlen=2,
)
def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
"""Process-level CPRR capability must not pad or route a non-causal batch."""
captured = {}
def fake_aiter_decode(q, kv_buffer, out, *args, **kwargs):
captured["q_heads"] = q.shape[1]
captured["kwargs"] = kwargs
return None, torch.zeros(q.shape[0], q.shape[1])
monkeypatch.setattr(
rocm_aiter_mla, "_get_aiter_mla_decode", lambda: fake_aiter_decode
)
impl = object.__new__(AiterMLAImpl)
impl.num_heads = 12
impl.dcp_world_size = 8
impl.kv_cache_dtype = "auto"
impl.kv_lora_rank = 512
impl.qk_rope_head_dim = 64
impl.scale = 576**-0.5
decode = SimpleNamespace(
max_qo_len=4,
qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
paged_kv_indices=torch.zeros(1, dtype=torch.int32),
paged_kv_last_page_len=torch.ones(1, dtype=torch.int32),
use_gluon_decode=False,
use_gluon_verify=False,
dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN,
dcp_verify=None,
g_kv_indptr=None,
has_persistent_metadata=False,
attn_out_dtype=torch.bfloat16,
asm_decode_num_heads=0,
mla_num_kv_splits=0,
cp_world_size=8,
cp_rank=0,
min_kv_seq_len=1,
)
attn_metadata = SimpleNamespace(decode=decode, causal=False, work_meta_data=None)
layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0))
q = torch.zeros(4, 96, 576, dtype=torch.bfloat16)
output, lse = impl.forward_mqa(q, torch.zeros(1, 1, 576), attn_metadata, layer)
assert captured["q_heads"] == 96
assert captured["kwargs"]["causal"] is False
assert "g_kv_indptr" not in captured["kwargs"]
assert "num_kv_splits" not in captured["kwargs"]
assert output.shape[1] == 96
assert lse is not None and lse.shape == (4, 96)
def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
"""Single-token DCP decode must come back with an LSE the merge can use.
@@ -325,9 +554,14 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
paged_kv_last_page_len=torch.ones(num_tokens, dtype=torch.int32),
use_gluon_decode=False,
use_gluon_verify=False,
dcp_route=rocm_aiter_mla._DCPDecodeRoute.PLAIN,
dcp_verify=None,
has_persistent_metadata=False,
attn_out_dtype=torch.bfloat16,
# 0 == the round-robin (cprr) asm route is off, this rank's decode takes
# the existing path. Mirrors the AiterMLADecodeMetadata default.
asm_decode_num_heads=0,
g_kv_indptr=None,
)
attn_metadata = SimpleNamespace(decode=decode, causal=True, work_meta_data=None)
layer = SimpleNamespace(_q_scale=torch.tensor(1.0), _k_scale=torch.tensor(1.0))
@@ -453,6 +687,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
),
use_gluon_decode=False,
use_gluon_verify=False,
dcp_route=rocm_aiter_mla._DCPDecodeRoute.SEGMENTED,
dcp_verify=view,
attn_out_dtype=torch.bfloat16,
)