[ROCm][MLA] Route gfx942 DCP causal multi-token verify to Triton split-KV decode (#58555)

Signed-off-by: Xavier Aguilar <xavier.aguilarfruto@amd.com>
This commit is contained in:
xaguilar-amd
2026-10-14 10:17:15 +08:00
committed by GitHub
parent 449c05a0ca
commit a1f51a0226
5 changed files with 744 additions and 65 deletions
@@ -161,6 +161,11 @@ def _build(monkeypatch, route, gathered_heads, segmented_supported=True):
lambda *args: segmented_supported,
)
monkeypatch.setattr(rocm_aiter_mla, "_fp8_mla_prefill_supported", lambda: False)
# These cases pick between cprr and segmented. On a gfx942 runner the real
# arch would add the Triton verify route, which this stub cannot build.
import vllm.platforms.rocm as rocm
monkeypatch.setattr(rocm, "on_gfx942", lambda: False)
monkeypatch.setattr(
rocm_aiter_mla, "get_dcp_group", lambda: SimpleNamespace(rank_in_group=0)
)
@@ -264,25 +269,46 @@ def test_min_cprr_qlen_is_above_two():
@pytest.mark.parametrize(
"supports,causal,qlen,asm,expected",
"segmented,triton,causal,qlen,asm,expected",
[
(True, True, 1, True, Route.PLAIN),
(True, True, 2, True, Route.SEGMENTED),
(True, True, 3, True, Route.CPRR),
(True, True, 4, True, Route.CPRR),
(True, True, 4, False, Route.SEGMENTED),
(True, False, 2, True, Route.PLAIN),
(True, False, 4, False, Route.PLAIN),
(False, True, 3, True, Route.CPRR),
(True, False, True, 1, True, Route.PLAIN),
(True, False, True, 2, True, Route.SEGMENTED),
(True, False, True, 3, True, Route.CPRR),
(True, False, True, 4, True, Route.CPRR),
(True, False, True, 4, False, Route.SEGMENTED),
(True, False, False, 2, True, Route.PLAIN),
(True, False, False, 4, False, Route.PLAIN),
(False, False, True, 3, True, Route.CPRR),
# gfx942: no cprr and no segmented, so causal verify goes to Triton.
(False, True, True, 1, False, Route.PLAIN),
(False, True, True, 2, False, Route.TRITON),
(False, True, True, 4, False, Route.TRITON),
(False, True, False, 4, False, Route.PLAIN),
# cprr still wins at or above its qlen floor.
(False, True, True, 4, True, Route.CPRR),
(False, True, True, 2, True, Route.TRITON),
],
)
def test_dcp_decode_route(supports, causal, qlen, asm, expected):
assert _select_route(supports, causal, qlen, asm) is expected
def test_dcp_decode_route(segmented, triton, causal, qlen, asm, expected):
route = _select_route(
supports_segmented=segmented,
supports_triton=triton,
causal=causal,
max_qo_len=qlen,
asm_selected=asm,
)
assert route is expected
def test_causal_multi_token_batch_without_a_valid_route_fails():
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
_select_route(False, True, 2, True)
_select_route(
supports_segmented=False,
supports_triton=False,
causal=True,
max_qo_len=2,
asm_selected=True,
)
# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs
@@ -536,5 +536,12 @@ def test_cprr_kernel_is_selected_for_the_k3_shape():
is True
)
assert QLEN >= m._MIN_CPRR_QLEN
assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR
route = m._select_dcp_decode_route(
supports_segmented=True,
supports_triton=False,
causal=True,
max_qo_len=QLEN,
asm_selected=True,
)
assert route is m._DCPDecodeRoute.CPRR
assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS
@@ -0,0 +1,331 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Causal target verify under DCP on gfx942 (MI300/MI325).
Reproduces the attention shape of
vllm serve moonshotai/Kimi-K3 --tensor-parallel-size 8 \
--decode-context-parallel-size 8 \
--speculative-config '{"method": "dspark", "num_speculative_tokens": 5}'
96 gathered heads, bf16 KV, qlen 6 and the 768-token blocks the hybrid KDA
layers impose. Segmented MLA tiles those blocks at 128 tokens and needs 128 KiB
of LDS (gfx942 has 64 KiB), so the builder must route this shape to the generic
Triton split-KV kernel. DeepSeek-V3 with MTP and an fp8 KV cache at DCP 8 hits
the same limit through its 128 gathered heads. Shapes segmented MLA fits stay
on it.
The real metadata builder and ``forward_mqa`` run on one GPU with the DCP ranks
simulated one after another. That is sound because this path has no
collective: the query all-gather happens in the layer, before ``forward_mqa``.
"""
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
if not current_platform.is_rocm():
pytest.skip("ROCm AITER MLA tests", allow_module_level=True)
from vllm._aiter_ops import is_aiter_found # noqa: E402
# DeepSeek-R1 supplies the MLA dims, which Kimi-K3 and DeepSeek-V3 share; only
# the head count is overridden.
MODEL = "deepseek-ai/DeepSeek-R1"
KV_LORA_RANK, ROPE_DIM, NOPE_DIM = 512, 64, 128
HEAD_SIZE = KV_LORA_RANK + ROPE_DIM
SCALE = (NOPE_DIM + ROPE_DIM) ** -0.5
DCP = 8
# Each rank holds ~800 tokens per request, so its shard spans several blocks.
NUM_REQS, CTX = 4, 6400
FP8_KV_SCALE = 0.05
def _on_gfx942() -> bool:
if not (is_aiter_found() and torch.accelerator.is_available()):
return False
from vllm.platforms.rocm import on_gfx942
return on_gfx942()
pytestmark = pytest.mark.skipif(not _on_gfx942(), reason="needs ROCm + AITER on gfx942")
class _FakeGroup:
def __init__(self, world_size: int, rank: int):
self.world_size = world_size
self.rank_in_group = self.rank = self.local_rank = rank
def all_gather(self, tensor, dim=0):
raise RuntimeError("DCP verify must not issue a collective")
@pytest.fixture
def fake_dcp_groups(monkeypatch):
"""Point every importer of get_dcp_group at a settable rank stub."""
import vllm.distributed.parallel_state as ps
import vllm.model_executor.layers.attention.mla_attention as mla_attention
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
group = _FakeGroup(DCP, 0)
for module in (ps, mla_attention, backend):
monkeypatch.setattr(module, "get_dcp_group", lambda: group)
monkeypatch.setattr(ps, "get_tp_group", lambda: _FakeGroup(1, 0))
return group
def _vllm_config(heads_per_rank: int, qlen: int, fp8_kv: bool):
from tests.v1.attention.utils import create_vllm_config
config = create_vllm_config(
model_name=MODEL,
max_model_len=CTX + qlen,
max_num_seqs=NUM_REQS,
max_num_batched_tokens=4096,
)
config.model_config.model_arch_config.total_num_attention_heads = heads_per_rank
# The reorder threshold reads only these two fields; without them a
# multi-token row is classified as prefill and never reaches decode.
config.speculative_config = SimpleNamespace(
num_speculative_tokens=qlen - 1, parallel_drafting=False
)
config.parallel_config.decode_context_parallel_size = DCP
config.parallel_config.cp_kv_cache_interleave_size = 1
if fp8_kv:
config.cache_config.cache_dtype = "fp8"
return config
def _ctx_layer():
"""The registered-layer fields MLACommonMetadataBuilder.__init__ reads."""
from vllm.v1.attention.ops.dcp import MLADCPManager
dcp_manager = object.__new__(MLADCPManager)
dcp_manager.init_kv_gather = lambda *args, **kwargs: None
# Only chunked-context prefill uses the cloned backend.
prefill_backend = SimpleNamespace()
prefill_backend.clone = lambda: prefill_backend
return SimpleNamespace(
non_causal_multi_token_decode=False,
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=NOPE_DIM,
qk_rope_head_dim=ROPE_DIM,
v_head_dim=NOPE_DIM,
dcp_manager=dcp_manager,
prefill_backend=prefill_backend,
)
def _impl(dcp_rank: int, heads_per_rank: int, fp8_kv: bool):
"""forward_mqa needs these attributes, not the weight-loading __init__."""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
impl = object.__new__(AiterMLAImpl)
impl.num_heads = heads_per_rank
impl.kv_lora_rank = KV_LORA_RANK
impl.qk_rope_head_dim = ROPE_DIM
impl.dcp_world_size = DCP
impl.dcp_rank = dcp_rank
impl.pcp_world_size = 1
impl.kv_cache_dtype = "fp8" if fp8_kv else "auto"
impl.scale = SCALE
impl._sm_count = current_platform.num_compute_units()
return impl
def _run_rank(config, spec, layer_name, group, rank, q, kv_cache_src, k_scale):
"""Shard the KV round-robin onto ``rank`` and run its verify step."""
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
device = q.device
qlen, heads = q.shape[1], q.shape[2] // DCP
seq_len = CTX + qlen
positions = torch.arange(rank, seq_len, DCP, device=device)
local_len = positions.numel()
# Production block tables span max_model_len, so every rank's table is at
# least as wide as the largest shard needs.
blocks_per_req = cdiv(cdiv(seq_len, DCP), spec.block_size)
kv_cache = torch.zeros(
NUM_REQS * blocks_per_req,
spec.block_size,
HEAD_SIZE,
dtype=kv_cache_src.dtype,
device=device,
)
kv_cache.view(NUM_REQS, -1, HEAD_SIZE)[:, :local_len] = kv_cache_src[:, positions]
block_table = torch.arange(
NUM_REQS * blocks_per_req, dtype=torch.int32, device=device
).view(NUM_REQS, blocks_per_req)
query_start_loc = torch.arange(
0, NUM_REQS * qlen + 1, qlen, dtype=torch.int32, device=device
)
local_lens = torch.full((NUM_REQS,), local_len, dtype=torch.int32)
common = CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.full((NUM_REQS,), seq_len, dtype=torch.int32, device=device),
num_reqs=NUM_REQS,
num_actual_tokens=NUM_REQS * qlen,
max_query_len=qlen,
max_seq_len=seq_len,
block_table_tensor=block_table,
slot_mapping=torch.arange(NUM_REQS * qlen, dtype=torch.int64, device=device),
causal=True,
dcp_local_seq_lens=local_lens.to(device),
dcp_local_seq_lens_cpu_upper_bound=local_lens,
)
group.rank_in_group = group.rank = group.local_rank = rank
builder = AiterMLAMetadataBuilder(spec, [layer_name], config, device)
metadata = builder.build(0, common)
layer = SimpleNamespace(
_q_scale=torch.tensor(1.0, device=device),
_k_scale=torch.tensor(k_scale, device=device),
)
impl = _impl(rank, heads, kv_cache_src.dtype != torch.bfloat16)
output, lse = impl.forward_mqa(q.flatten(0, 1), kv_cache, metadata, layer)
return metadata.decode.dcp_route.name, output, lse
@pytest.mark.parametrize(
"heads_per_rank, qlen, block_size, fp8_kv, route",
[
(12, 6, 768, False, "TRITON"),
(12, 6, 64, False, "SEGMENTED"),
(16, 2, 16, True, "TRITON"),
],
ids=["kimi-k3-hybrid-kda-blocks", "segmented-fits", "deepseek-v3-mtp-fp8-kv"],
)
@torch.inference_mode()
def test_dcp_verify_matches_attention(
fake_dcp_groups, heads_per_rank, qlen, block_size, fp8_kv, route
):
from vllm.config import set_current_vllm_config
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.workspace import (
init_workspace_manager,
is_workspace_manager_initialized,
)
device = torch.device("cuda", torch.accelerator.current_device_index())
if not is_workspace_manager_initialized():
init_workspace_manager(device)
torch.manual_seed(0)
seq_len = CTX + qlen
q = torch.randn(
NUM_REQS,
qlen,
heads_per_rank * DCP,
HEAD_SIZE,
dtype=torch.bfloat16,
device=device,
)
kv = torch.randn(NUM_REQS, seq_len, HEAD_SIZE, device=device).clamp_(-4, 4)
if fp8_kv:
k_scale = FP8_KV_SCALE
kv_cache_src = (kv / k_scale).to(current_platform.fp8_dtype())
else:
k_scale = 1.0
kv_cache_src = kv.to(torch.bfloat16)
kv_ref = kv_cache_src.float() * k_scale
config = _vllm_config(heads_per_rank, qlen, fp8_kv)
layer_name = "model.layers.0.self_attn.attn"
with set_current_vllm_config(config):
config.compilation_config.static_forward_context[layer_name] = _ctx_layer()
spec = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=HEAD_SIZE,
dtype=kv_cache_src.dtype,
cache_dtype_str="fp8" if fp8_kv else None,
non_causal_multi_token_decode=False,
)
shards = [
_run_rank(
config,
spec,
layer_name,
fake_dcp_groups,
rank,
q,
kv_cache_src,
k_scale,
)
for rank in range(DCP)
]
assert {shard_route for shard_route, _, _ in shards} == {route}
lses = torch.stack([lse.float() for _, _, lse in shards])
weights = torch.exp(lses - lses.max(dim=0).values)
merged = (
weights.unsqueeze(-1) * torch.stack([out.float() for _, out, _ in shards])
).sum(dim=0) / weights.sum(dim=0).unsqueeze(-1)
# Query token t of a request sees the context plus verify tokens 0..t.
scores = torch.einsum("rthd,rld->rthl", q.float(), kv_ref) * SCALE
visible = torch.arange(seq_len, device=device) <= CTX + torch.arange(
qlen, device=device
).view(-1, 1)
scores.masked_fill_(~visible[None, :, None, :], float("-inf"))
reference = torch.einsum(
"rthl,rld->rthd", scores.softmax(dim=-1), kv_ref[..., :KV_LORA_RANK]
)
torch.testing.assert_close(
merged.view_as(reference), reference, rtol=2e-2, atol=2e-2
)
@pytest.mark.parametrize(
"fp8_kv, num_heads, block_size",
[
(False, 96, 768),
(False, 128, 64),
(True, 64, 128),
(True, 128, 16),
],
)
def test_segmented_lds_gate_matches_the_kernel(fp8_kv, num_heads, block_size):
"""The gate must flag exactly the shapes AITER segmented MLA cannot compile."""
from triton.runtime.errors import OutOfResources
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
_get_segmented_mla_decode,
_segmented_mla_fits_lds,
_segmented_mla_page_size,
)
page = _segmented_mla_page_size(block_size)
kv_dtype = current_platform.fp8_dtype() if fp8_kv else torch.bfloat16
q = torch.zeros(1, num_heads, HEAD_SIZE, dtype=torch.bfloat16, device="cuda")
kv = torch.zeros(1, page, 1, HEAD_SIZE, dtype=kv_dtype, device="cuda")
one_row = torch.tensor([0, 1], dtype=torch.int32, device="cuda")
try:
_get_segmented_mla_decode()(
q,
kv,
None,
one_row,
torch.tensor([page], dtype=torch.int32, device="cuda"),
page,
torch.zeros(1, 1, dtype=torch.int32, device="cuda"),
SCALE,
KV_LORA_RANK,
ROPE_DIM,
causal=True,
q_descale=None,
kv_descale=torch.tensor(1.0, device="cuda"),
skip_reduce=True,
)
compiles = True
except OutOfResources:
compiles = False
assert _segmented_mla_fits_lds(num_heads, block_size, fp8_kv) == compiles
@@ -20,6 +20,10 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( # noqa: E402
AiterMLAImpl,
AiterMLAMetadataBuilder,
)
from vllm.v1.attention.ops.dcp import ( # noqa: E402
CPTritonContext,
correct_attn_out,
)
from vllm.v1.attention.ops.rocm_aiter_mla_merge import ( # noqa: E402
merge_mla_segments_triton,
)
@@ -121,8 +125,11 @@ def _builder(
# segmented route is available, query length decides per batch. CPRR
# preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back.
_supports_segmented_dcp_verify=supports_segmented_dcp_verify,
_use_triton_dcp_verify=False,
# Derived once in the real constructor, so derive it once here too.
_segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size),
_dcp_verify_page_size=rocm_aiter_mla._segmented_mla_page_size(
kernel_block_size
),
_dcp_verify_buffers=None,
_graph_seq_lens=None,
_kv_cache_dtype_str=kv_cache_dtype,
@@ -630,6 +637,99 @@ def test_segmented_verify_reduce_returns_natural_lse_and_masks_empty_rows():
assert lse[1].item() == float("-inf")
def test_triton_verify_empty_local_shard_merges_through_dcp_combine():
"""Empty rows need no masking: -inf LSE gives them zero combine weight.
Row 0 has KV on both ranks, row 1 only on rank 0, row 2 on neither (graph
padding). The raw rows come back NaN; the production AG+RS combine must
still produce full-context attention and zeros for the padded row.
"""
torch.manual_seed(0)
num_ranks, num_rows, num_heads = 2, 3, 16
kv_lora_rank, head_dim, block_size = 512, 576, 16
scale = head_dim**-0.5
local_lens = [[5, 4, 0], [5, 0, 0]]
q = torch.randn(num_rows, num_heads, head_dim, dtype=torch.bfloat16, device="cuda")
kv_caches = [
torch.randn(1, block_size, head_dim, dtype=torch.bfloat16, device="cuda")
for _ in range(num_ranks)
]
impl = object.__new__(AiterMLAImpl)
impl.scale = scale
impl.kv_lora_rank = kv_lora_rank
impl._sm_count = current_platform.num_compute_units()
layer = SimpleNamespace(_k_scale=torch.tensor(1.0, device="cuda"))
outputs, lses = [], []
for rank in range(num_ranks):
verify = SimpleNamespace(
block_table=torch.zeros(num_rows, 1, dtype=torch.int32, device="cuda"),
row_lens=torch.tensor(local_lens[rank], dtype=torch.int32, device="cuda"),
max_kv_seq_len=max(local_lens[rank]),
)
output, lse = impl._forward_triton_dcp_verify(
q, verify, kv_caches[rank], layer, torch.bfloat16
)
outputs.append(output)
lses.append(lse)
assert torch.isneginf(lses[1][1]).all()
assert torch.isneginf(lses[0][2]).all() and torch.isneginf(lses[1][2]).all()
all_lses = torch.stack(lses)
merged = sum(
correct_attn_out(outputs[rank], all_lses, rank, CPTritonContext())[0].float()
for rank in range(num_ranks)
)
reference = torch.zeros(num_rows, num_heads, kv_lora_rank, device="cuda")
for row in range(num_rows):
keys = torch.cat(
[kv_caches[rank][0, : local_lens[rank][row]] for rank in range(num_ranks)]
).float()
if keys.shape[0]:
probs = torch.softmax(q[row].float() @ keys.T * scale, dim=-1)
reference[row] = probs @ keys[:, :kv_lora_rank]
assert torch.isfinite(merged).all()
torch.testing.assert_close(merged, reference, rtol=2e-2, atol=2e-2)
def test_batch_aware_kv_splits_fit_an_int32_safe_reservation(monkeypatch):
# MI325X DCP8 verify at 128K context: 128 gathered heads, 16K local tokens.
sm_count, heads, local_seq_len, kv_lora_rank = 304, 128, 16384, 512
max_rows = 4096
reserved = []
monkeypatch.setattr(
rocm_aiter_mla, "is_workspace_manager_initialized", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla,
"current_workspace_manager",
lambda: SimpleNamespace(get_simultaneous=lambda *specs: reserved.extend(specs)),
)
rocm_aiter_mla._reserve_triton_dcp_verify_workspace(
max_rows, heads, local_seq_len, kv_lora_rank, sm_count
)
((shape, _),) = reserved
# The kernel's attn_logits offsets are int32; seq-only splits overflow them.
seq_only = rocm_aiter_mla._compute_num_kv_splits(local_seq_len, sm_count)
assert max_rows * heads * seq_only * (kv_lora_rank + 1) >= 2**31
assert math.prod(shape) < 2**31
counts = [
rocm_aiter_mla._triton_dcp_verify_num_kv_splits(
local_seq_len, rows, heads, sm_count
)
for rows in range(1, max_rows + 1)
]
# Small batches keep the seq-only split count.
assert counts[:4] == [seq_only] * 4
assert counts == sorted(counts, reverse=True)
for rows, splits in enumerate(counts, start=1):
assert rows * heads * splits * (kv_lora_rank + 1) <= math.prod(shape)
def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
"""Exercise the target qlen>1 path and merge two rank-local results."""
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)