[ROCm][MLA] Route gfx942 DCP causal multi-token verify to Triton split-KV decode (#58555)

Signed-off-by: Xavier Aguilar <xavier.aguilarfruto@amd.com>
This commit is contained in:
xaguilar-amd
2026-10-14 10:17:15 +08:00
committed by GitHub
parent 449c05a0ca
commit a1f51a0226
5 changed files with 744 additions and 65 deletions
@@ -161,6 +161,11 @@ def _build(monkeypatch, route, gathered_heads, segmented_supported=True):
lambda *args: segmented_supported,
)
monkeypatch.setattr(rocm_aiter_mla, "_fp8_mla_prefill_supported", lambda: False)
# These cases pick between cprr and segmented. On a gfx942 runner the real
# arch would add the Triton verify route, which this stub cannot build.
import vllm.platforms.rocm as rocm
monkeypatch.setattr(rocm, "on_gfx942", lambda: False)
monkeypatch.setattr(
rocm_aiter_mla, "get_dcp_group", lambda: SimpleNamespace(rank_in_group=0)
)
@@ -264,25 +269,46 @@ def test_min_cprr_qlen_is_above_two():
@pytest.mark.parametrize(
"supports,causal,qlen,asm,expected",
"segmented,triton,causal,qlen,asm,expected",
[
(True, True, 1, True, Route.PLAIN),
(True, True, 2, True, Route.SEGMENTED),
(True, True, 3, True, Route.CPRR),
(True, True, 4, True, Route.CPRR),
(True, True, 4, False, Route.SEGMENTED),
(True, False, 2, True, Route.PLAIN),
(True, False, 4, False, Route.PLAIN),
(False, True, 3, True, Route.CPRR),
(True, False, True, 1, True, Route.PLAIN),
(True, False, True, 2, True, Route.SEGMENTED),
(True, False, True, 3, True, Route.CPRR),
(True, False, True, 4, True, Route.CPRR),
(True, False, True, 4, False, Route.SEGMENTED),
(True, False, False, 2, True, Route.PLAIN),
(True, False, False, 4, False, Route.PLAIN),
(False, False, True, 3, True, Route.CPRR),
# gfx942: no cprr and no segmented, so causal verify goes to Triton.
(False, True, True, 1, False, Route.PLAIN),
(False, True, True, 2, False, Route.TRITON),
(False, True, True, 4, False, Route.TRITON),
(False, True, False, 4, False, Route.PLAIN),
# cprr still wins at or above its qlen floor.
(False, True, True, 4, True, Route.CPRR),
(False, True, True, 2, True, Route.TRITON),
],
)
def test_dcp_decode_route(supports, causal, qlen, asm, expected):
assert _select_route(supports, causal, qlen, asm) is expected
def test_dcp_decode_route(segmented, triton, causal, qlen, asm, expected):
route = _select_route(
supports_segmented=segmented,
supports_triton=triton,
causal=causal,
max_qo_len=qlen,
asm_selected=asm,
)
assert route is expected
def test_causal_multi_token_batch_without_a_valid_route_fails():
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
_select_route(False, True, 2, True)
_select_route(
supports_segmented=False,
supports_triton=False,
causal=True,
max_qo_len=2,
asm_selected=True,
)
# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs
@@ -536,5 +536,12 @@ def test_cprr_kernel_is_selected_for_the_k3_shape():
is True
)
assert QLEN >= m._MIN_CPRR_QLEN
assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR
route = m._select_dcp_decode_route(
supports_segmented=True,
supports_triton=False,
causal=True,
max_qo_len=QLEN,
asm_selected=True,
)
assert route is m._DCPDecodeRoute.CPRR
assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS
@@ -0,0 +1,331 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Causal target verify under DCP on gfx942 (MI300/MI325).
Reproduces the attention shape of
vllm serve moonshotai/Kimi-K3 --tensor-parallel-size 8 \
--decode-context-parallel-size 8 \
--speculative-config '{"method": "dspark", "num_speculative_tokens": 5}'
96 gathered heads, bf16 KV, qlen 6 and the 768-token blocks the hybrid KDA
layers impose. Segmented MLA tiles those blocks at 128 tokens and needs 128 KiB
of LDS (gfx942 has 64 KiB), so the builder must route this shape to the generic
Triton split-KV kernel. DeepSeek-V3 with MTP and an fp8 KV cache at DCP 8 hits
the same limit through its 128 gathered heads. Shapes segmented MLA fits stay
on it.
The real metadata builder and ``forward_mqa`` run on one GPU with the DCP ranks
simulated one after another. That is sound because this path has no
collective: the query all-gather happens in the layer, before ``forward_mqa``.
"""
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
if not current_platform.is_rocm():
pytest.skip("ROCm AITER MLA tests", allow_module_level=True)
from vllm._aiter_ops import is_aiter_found # noqa: E402
# DeepSeek-R1 supplies the MLA dims, which Kimi-K3 and DeepSeek-V3 share; only
# the head count is overridden.
MODEL = "deepseek-ai/DeepSeek-R1"
KV_LORA_RANK, ROPE_DIM, NOPE_DIM = 512, 64, 128
HEAD_SIZE = KV_LORA_RANK + ROPE_DIM
SCALE = (NOPE_DIM + ROPE_DIM) ** -0.5
DCP = 8
# Each rank holds ~800 tokens per request, so its shard spans several blocks.
NUM_REQS, CTX = 4, 6400
FP8_KV_SCALE = 0.05
def _on_gfx942() -> bool:
if not (is_aiter_found() and torch.accelerator.is_available()):
return False
from vllm.platforms.rocm import on_gfx942
return on_gfx942()
pytestmark = pytest.mark.skipif(not _on_gfx942(), reason="needs ROCm + AITER on gfx942")
class _FakeGroup:
def __init__(self, world_size: int, rank: int):
self.world_size = world_size
self.rank_in_group = self.rank = self.local_rank = rank
def all_gather(self, tensor, dim=0):
raise RuntimeError("DCP verify must not issue a collective")
@pytest.fixture
def fake_dcp_groups(monkeypatch):
"""Point every importer of get_dcp_group at a settable rank stub."""
import vllm.distributed.parallel_state as ps
import vllm.model_executor.layers.attention.mla_attention as mla_attention
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
group = _FakeGroup(DCP, 0)
for module in (ps, mla_attention, backend):
monkeypatch.setattr(module, "get_dcp_group", lambda: group)
monkeypatch.setattr(ps, "get_tp_group", lambda: _FakeGroup(1, 0))
return group
def _vllm_config(heads_per_rank: int, qlen: int, fp8_kv: bool):
from tests.v1.attention.utils import create_vllm_config
config = create_vllm_config(
model_name=MODEL,
max_model_len=CTX + qlen,
max_num_seqs=NUM_REQS,
max_num_batched_tokens=4096,
)
config.model_config.model_arch_config.total_num_attention_heads = heads_per_rank
# The reorder threshold reads only these two fields; without them a
# multi-token row is classified as prefill and never reaches decode.
config.speculative_config = SimpleNamespace(
num_speculative_tokens=qlen - 1, parallel_drafting=False
)
config.parallel_config.decode_context_parallel_size = DCP
config.parallel_config.cp_kv_cache_interleave_size = 1
if fp8_kv:
config.cache_config.cache_dtype = "fp8"
return config
def _ctx_layer():
"""The registered-layer fields MLACommonMetadataBuilder.__init__ reads."""
from vllm.v1.attention.ops.dcp import MLADCPManager
dcp_manager = object.__new__(MLADCPManager)
dcp_manager.init_kv_gather = lambda *args, **kwargs: None
# Only chunked-context prefill uses the cloned backend.
prefill_backend = SimpleNamespace()
prefill_backend.clone = lambda: prefill_backend
return SimpleNamespace(
non_causal_multi_token_decode=False,
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=NOPE_DIM,
qk_rope_head_dim=ROPE_DIM,
v_head_dim=NOPE_DIM,
dcp_manager=dcp_manager,
prefill_backend=prefill_backend,
)
def _impl(dcp_rank: int, heads_per_rank: int, fp8_kv: bool):
"""forward_mqa needs these attributes, not the weight-loading __init__."""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
impl = object.__new__(AiterMLAImpl)
impl.num_heads = heads_per_rank
impl.kv_lora_rank = KV_LORA_RANK
impl.qk_rope_head_dim = ROPE_DIM
impl.dcp_world_size = DCP
impl.dcp_rank = dcp_rank
impl.pcp_world_size = 1
impl.kv_cache_dtype = "fp8" if fp8_kv else "auto"
impl.scale = SCALE
impl._sm_count = current_platform.num_compute_units()
return impl
def _run_rank(config, spec, layer_name, group, rank, q, kv_cache_src, k_scale):
"""Shard the KV round-robin onto ``rank`` and run its verify step."""
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
device = q.device
qlen, heads = q.shape[1], q.shape[2] // DCP
seq_len = CTX + qlen
positions = torch.arange(rank, seq_len, DCP, device=device)
local_len = positions.numel()
# Production block tables span max_model_len, so every rank's table is at
# least as wide as the largest shard needs.
blocks_per_req = cdiv(cdiv(seq_len, DCP), spec.block_size)
kv_cache = torch.zeros(
NUM_REQS * blocks_per_req,
spec.block_size,
HEAD_SIZE,
dtype=kv_cache_src.dtype,
device=device,
)
kv_cache.view(NUM_REQS, -1, HEAD_SIZE)[:, :local_len] = kv_cache_src[:, positions]
block_table = torch.arange(
NUM_REQS * blocks_per_req, dtype=torch.int32, device=device
).view(NUM_REQS, blocks_per_req)
query_start_loc = torch.arange(
0, NUM_REQS * qlen + 1, qlen, dtype=torch.int32, device=device
)
local_lens = torch.full((NUM_REQS,), local_len, dtype=torch.int32)
common = CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.full((NUM_REQS,), seq_len, dtype=torch.int32, device=device),
num_reqs=NUM_REQS,
num_actual_tokens=NUM_REQS * qlen,
max_query_len=qlen,
max_seq_len=seq_len,
block_table_tensor=block_table,
slot_mapping=torch.arange(NUM_REQS * qlen, dtype=torch.int64, device=device),
causal=True,
dcp_local_seq_lens=local_lens.to(device),
dcp_local_seq_lens_cpu_upper_bound=local_lens,
)
group.rank_in_group = group.rank = group.local_rank = rank
builder = AiterMLAMetadataBuilder(spec, [layer_name], config, device)
metadata = builder.build(0, common)
layer = SimpleNamespace(
_q_scale=torch.tensor(1.0, device=device),
_k_scale=torch.tensor(k_scale, device=device),
)
impl = _impl(rank, heads, kv_cache_src.dtype != torch.bfloat16)
output, lse = impl.forward_mqa(q.flatten(0, 1), kv_cache, metadata, layer)
return metadata.decode.dcp_route.name, output, lse
@pytest.mark.parametrize(
"heads_per_rank, qlen, block_size, fp8_kv, route",
[
(12, 6, 768, False, "TRITON"),
(12, 6, 64, False, "SEGMENTED"),
(16, 2, 16, True, "TRITON"),
],
ids=["kimi-k3-hybrid-kda-blocks", "segmented-fits", "deepseek-v3-mtp-fp8-kv"],
)
@torch.inference_mode()
def test_dcp_verify_matches_attention(
fake_dcp_groups, heads_per_rank, qlen, block_size, fp8_kv, route
):
from vllm.config import set_current_vllm_config
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.workspace import (
init_workspace_manager,
is_workspace_manager_initialized,
)
device = torch.device("cuda", torch.accelerator.current_device_index())
if not is_workspace_manager_initialized():
init_workspace_manager(device)
torch.manual_seed(0)
seq_len = CTX + qlen
q = torch.randn(
NUM_REQS,
qlen,
heads_per_rank * DCP,
HEAD_SIZE,
dtype=torch.bfloat16,
device=device,
)
kv = torch.randn(NUM_REQS, seq_len, HEAD_SIZE, device=device).clamp_(-4, 4)
if fp8_kv:
k_scale = FP8_KV_SCALE
kv_cache_src = (kv / k_scale).to(current_platform.fp8_dtype())
else:
k_scale = 1.0
kv_cache_src = kv.to(torch.bfloat16)
kv_ref = kv_cache_src.float() * k_scale
config = _vllm_config(heads_per_rank, qlen, fp8_kv)
layer_name = "model.layers.0.self_attn.attn"
with set_current_vllm_config(config):
config.compilation_config.static_forward_context[layer_name] = _ctx_layer()
spec = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=HEAD_SIZE,
dtype=kv_cache_src.dtype,
cache_dtype_str="fp8" if fp8_kv else None,
non_causal_multi_token_decode=False,
)
shards = [
_run_rank(
config,
spec,
layer_name,
fake_dcp_groups,
rank,
q,
kv_cache_src,
k_scale,
)
for rank in range(DCP)
]
assert {shard_route for shard_route, _, _ in shards} == {route}
lses = torch.stack([lse.float() for _, _, lse in shards])
weights = torch.exp(lses - lses.max(dim=0).values)
merged = (
weights.unsqueeze(-1) * torch.stack([out.float() for _, out, _ in shards])
).sum(dim=0) / weights.sum(dim=0).unsqueeze(-1)
# Query token t of a request sees the context plus verify tokens 0..t.
scores = torch.einsum("rthd,rld->rthl", q.float(), kv_ref) * SCALE
visible = torch.arange(seq_len, device=device) <= CTX + torch.arange(
qlen, device=device
).view(-1, 1)
scores.masked_fill_(~visible[None, :, None, :], float("-inf"))
reference = torch.einsum(
"rthl,rld->rthd", scores.softmax(dim=-1), kv_ref[..., :KV_LORA_RANK]
)
torch.testing.assert_close(
merged.view_as(reference), reference, rtol=2e-2, atol=2e-2
)
@pytest.mark.parametrize(
"fp8_kv, num_heads, block_size",
[
(False, 96, 768),
(False, 128, 64),
(True, 64, 128),
(True, 128, 16),
],
)
def test_segmented_lds_gate_matches_the_kernel(fp8_kv, num_heads, block_size):
"""The gate must flag exactly the shapes AITER segmented MLA cannot compile."""
from triton.runtime.errors import OutOfResources
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
_get_segmented_mla_decode,
_segmented_mla_fits_lds,
_segmented_mla_page_size,
)
page = _segmented_mla_page_size(block_size)
kv_dtype = current_platform.fp8_dtype() if fp8_kv else torch.bfloat16
q = torch.zeros(1, num_heads, HEAD_SIZE, dtype=torch.bfloat16, device="cuda")
kv = torch.zeros(1, page, 1, HEAD_SIZE, dtype=kv_dtype, device="cuda")
one_row = torch.tensor([0, 1], dtype=torch.int32, device="cuda")
try:
_get_segmented_mla_decode()(
q,
kv,
None,
one_row,
torch.tensor([page], dtype=torch.int32, device="cuda"),
page,
torch.zeros(1, 1, dtype=torch.int32, device="cuda"),
SCALE,
KV_LORA_RANK,
ROPE_DIM,
causal=True,
q_descale=None,
kv_descale=torch.tensor(1.0, device="cuda"),
skip_reduce=True,
)
compiles = True
except OutOfResources:
compiles = False
assert _segmented_mla_fits_lds(num_heads, block_size, fp8_kv) == compiles
@@ -20,6 +20,10 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( # noqa: E402
AiterMLAImpl,
AiterMLAMetadataBuilder,
)
from vllm.v1.attention.ops.dcp import ( # noqa: E402
CPTritonContext,
correct_attn_out,
)
from vllm.v1.attention.ops.rocm_aiter_mla_merge import ( # noqa: E402
merge_mla_segments_triton,
)
@@ -121,8 +125,11 @@ def _builder(
# segmented route is available, query length decides per batch. CPRR
# preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back.
_supports_segmented_dcp_verify=supports_segmented_dcp_verify,
_use_triton_dcp_verify=False,
# Derived once in the real constructor, so derive it once here too.
_segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size),
_dcp_verify_page_size=rocm_aiter_mla._segmented_mla_page_size(
kernel_block_size
),
_dcp_verify_buffers=None,
_graph_seq_lens=None,
_kv_cache_dtype_str=kv_cache_dtype,
@@ -630,6 +637,99 @@ def test_segmented_verify_reduce_returns_natural_lse_and_masks_empty_rows():
assert lse[1].item() == float("-inf")
def test_triton_verify_empty_local_shard_merges_through_dcp_combine():
"""Empty rows need no masking: -inf LSE gives them zero combine weight.
Row 0 has KV on both ranks, row 1 only on rank 0, row 2 on neither (graph
padding). The raw rows come back NaN; the production AG+RS combine must
still produce full-context attention and zeros for the padded row.
"""
torch.manual_seed(0)
num_ranks, num_rows, num_heads = 2, 3, 16
kv_lora_rank, head_dim, block_size = 512, 576, 16
scale = head_dim**-0.5
local_lens = [[5, 4, 0], [5, 0, 0]]
q = torch.randn(num_rows, num_heads, head_dim, dtype=torch.bfloat16, device="cuda")
kv_caches = [
torch.randn(1, block_size, head_dim, dtype=torch.bfloat16, device="cuda")
for _ in range(num_ranks)
]
impl = object.__new__(AiterMLAImpl)
impl.scale = scale
impl.kv_lora_rank = kv_lora_rank
impl._sm_count = current_platform.num_compute_units()
layer = SimpleNamespace(_k_scale=torch.tensor(1.0, device="cuda"))
outputs, lses = [], []
for rank in range(num_ranks):
verify = SimpleNamespace(
block_table=torch.zeros(num_rows, 1, dtype=torch.int32, device="cuda"),
row_lens=torch.tensor(local_lens[rank], dtype=torch.int32, device="cuda"),
max_kv_seq_len=max(local_lens[rank]),
)
output, lse = impl._forward_triton_dcp_verify(
q, verify, kv_caches[rank], layer, torch.bfloat16
)
outputs.append(output)
lses.append(lse)
assert torch.isneginf(lses[1][1]).all()
assert torch.isneginf(lses[0][2]).all() and torch.isneginf(lses[1][2]).all()
all_lses = torch.stack(lses)
merged = sum(
correct_attn_out(outputs[rank], all_lses, rank, CPTritonContext())[0].float()
for rank in range(num_ranks)
)
reference = torch.zeros(num_rows, num_heads, kv_lora_rank, device="cuda")
for row in range(num_rows):
keys = torch.cat(
[kv_caches[rank][0, : local_lens[rank][row]] for rank in range(num_ranks)]
).float()
if keys.shape[0]:
probs = torch.softmax(q[row].float() @ keys.T * scale, dim=-1)
reference[row] = probs @ keys[:, :kv_lora_rank]
assert torch.isfinite(merged).all()
torch.testing.assert_close(merged, reference, rtol=2e-2, atol=2e-2)
def test_batch_aware_kv_splits_fit_an_int32_safe_reservation(monkeypatch):
# MI325X DCP8 verify at 128K context: 128 gathered heads, 16K local tokens.
sm_count, heads, local_seq_len, kv_lora_rank = 304, 128, 16384, 512
max_rows = 4096
reserved = []
monkeypatch.setattr(
rocm_aiter_mla, "is_workspace_manager_initialized", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla,
"current_workspace_manager",
lambda: SimpleNamespace(get_simultaneous=lambda *specs: reserved.extend(specs)),
)
rocm_aiter_mla._reserve_triton_dcp_verify_workspace(
max_rows, heads, local_seq_len, kv_lora_rank, sm_count
)
((shape, _),) = reserved
# The kernel's attn_logits offsets are int32; seq-only splits overflow them.
seq_only = rocm_aiter_mla._compute_num_kv_splits(local_seq_len, sm_count)
assert max_rows * heads * seq_only * (kv_lora_rank + 1) >= 2**31
assert math.prod(shape) < 2**31
counts = [
rocm_aiter_mla._triton_dcp_verify_num_kv_splits(
local_seq_len, rows, heads, sm_count
)
for rows in range(1, max_rows + 1)
]
# Small batches keep the seq-only split count.
assert counts[:4] == [seq_only] * 4
assert counts == sorted(counts, reverse=True)
for rows, splits in enumerate(counts, start=1):
assert rows * heads * splits * (kv_lora_rank + 1) <= math.prod(shape)
def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
"""Exercise the target qlen>1 path and merge two rank-local results."""
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
+266 -51
View File
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
accumulate_mla_context_chunk,
init_mla_context_partial,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.math_utils import cdiv, largest_power_of_2_divisor
@@ -35,6 +36,7 @@ from vllm.v1.attention.backend import (
CommonAttentionMetadata,
MultipleOf,
)
from vllm.v1.attention.backends.mla.triton_mla import _compute_num_kv_splits
from vllm.v1.attention.backends.utils import (
get_dcp_local_seq_lens,
)
@@ -46,7 +48,12 @@ from vllm.v1.attention.ops.rocm_aiter_mla_prefill import (
expand_context,
gather_compressed_context,
)
from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd
from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache
from vllm.v1.worker.workspace import (
current_workspace_manager,
is_workspace_manager_initialized,
)
if TYPE_CHECKING:
from vllm.platforms.interface import DeviceCapability
@@ -269,6 +276,75 @@ def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) ->
)
def _segmented_mla_fits_lds(num_heads: int, block_size: int, fp8_kv: bool) -> bool:
"""Whether AITER segmented MLA compiles for this DCP verify shape.
On gfx942 it needs 128 KiB of LDS, over the 64 KiB available, for bf16 KV
at TILE-128 and for fp8 KV past 64 heads, where its query block widens to
128 heads. ``num_heads`` is the DCP-gathered count.
"""
from vllm.platforms.rocm import on_gfx942
if not on_gfx942():
return True
if fp8_kv:
return num_heads <= 64
return _segmented_mla_page_size(block_size) < 128
def _triton_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) -> bool:
"""Whether gfx942 can fall back to generic split-KV causal target verify.
The batch shape is causal qlen>1 over a DCP shard. Eagle MTP and DSpark
target verify are that same shape, so this is not limited to one
speculative method. gfx942 has no cprr kernel, so this serves the shapes
segmented MLA cannot (see _segmented_mla_fits_lds). qlen==1 and non-causal
stay on the plain ASM decode.
"""
from vllm.platforms.rocm import on_gfx942
return on_gfx942() and dcp_world_size > 1 and cp_interleave == 1
# Heads per stage-1 program in the grouped MLA decode kernel (its BLOCK_H).
_DECODE_HEADS_PER_PROGRAM: Final = 16
# Tuned on MI325X with microbenchmarks
_BATCH_OCCUPANCY_MULTIPLIER: Final = 4
def _triton_dcp_verify_num_kv_splits(
max_seq_len: int, num_rows: int, num_heads: int, sm_count: int
) -> int:
# Split only until the stage-1 grid fills the GPU. Past that, splits add
# scratch and combine work, and large batches overflow the kernel's int32
# attn_logits offsets.
programs_per_split = max(1, num_rows) * cdiv(num_heads, _DECODE_HEADS_PER_PROGRAM)
occupancy_splits = triton.next_power_of_2(
cdiv(_BATCH_OCCUPANCY_MULTIPLIER * sm_count, programs_per_split)
)
return min(_compute_num_kv_splits(max_seq_len, sm_count), occupancy_splits)
def _reserve_triton_dcp_verify_workspace(
max_rows: int,
num_heads: int,
max_seq_len: int,
kv_lora_rank: int,
sm_count: int,
) -> None:
"""Reserve split-KV scratch before warmup locks the workspace manager."""
if not is_workspace_manager_initialized():
return
# Fewer rows may take more splits, so size for the largest product.
max_row_splits = max(
rows * _triton_dcp_verify_num_kv_splits(max_seq_len, rows, num_heads, sm_count)
for rows in range(1, max_rows + 1)
)
current_workspace_manager().get_simultaneous(
((max_row_splits, num_heads, kv_lora_rank + 1), torch.float32),
)
# DCP decode routing
#
# With decode context parallelism each rank holds a round-robin shard of the KV
@@ -281,9 +357,12 @@ def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) ->
# round-robin ASM decode rebuilds global positions from g_kv_indptr, cp_rank
# and cp_world_size and applies the causal window in-kernel. Kernels exist only
# at _NATIVE_CPRR_HEADS; other gathered head counts are padded up.
# * triton: causal qlen > 1 on gfx942 for shapes segmented MLA cannot fit in
# LDS (_segmented_mla_fits_lds). Generic split-KV decode over the same
# per-row local window segmented uses.
# * segmented: causal qlen > 1 when cprr is not selected or qlen is below
# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV length
# encodes its causal window.
# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV
# length encodes its causal window.
#
# The plain kernel must never serve a causal qlen > 1 block: it applies
# causality on local indices, which is wrong over a round-robin shard.
@@ -295,6 +374,12 @@ class _DCPDecodeRoute(Enum):
PLAIN = "plain"
SEGMENTED = "segmented"
CPRR = "cprr"
TRITON = "triton"
def _is_row_view_dcp_route(route: _DCPDecodeRoute) -> bool:
"""Routes that attend one causal window per verify token."""
return route in (_DCPDecodeRoute.SEGMENTED, _DCPDecodeRoute.TRITON)
def _asm_dcp_verify_heads(decode_num_heads: int) -> int:
@@ -308,7 +393,9 @@ def _asm_dcp_verify_heads(decode_num_heads: int) -> int:
def _select_dcp_decode_route(
*,
supports_segmented: bool,
supports_triton: bool,
causal: bool,
max_qo_len: int,
asm_selected: bool,
@@ -318,11 +405,13 @@ def _select_dcp_decode_route(
return _DCPDecodeRoute.PLAIN
if asm_selected and max_qo_len >= _MIN_CPRR_QLEN:
return _DCPDecodeRoute.CPRR
if supports_triton:
return _DCPDecodeRoute.TRITON
if supports_segmented:
return _DCPDecodeRoute.SEGMENTED
raise RuntimeError(
"ROCM_AITER_MLA DCP multi-token verify requires either segmented "
"MLA or the round-robin asm decode."
"MLA, the round-robin asm decode, or the gfx942 Triton verify path."
)
@@ -472,11 +561,10 @@ class AiterMLABackend(MLACommonBackend):
@dataclass
class AiterMLADCPVerifyMetadata:
"""One paged-KV row per verify token, for segmented DCP verification.
"""One paged-KV row per verify token for causal DCP verification.
These are produced together or not at all, so they travel as one value: its
presence on the decode metadata *is* the routing decision the builder made,
and the impl does not re-derive it.
Segmented MLA and the gfx942 Triton path both consume this view. Which
kernel runs is ``AiterMLADecodeMetadata.dcp_route``.
"""
# Local KV length of each verify row, already causally bounded.
@@ -514,7 +602,8 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata):
min_kv_seq_len: int = 1
# Exactly one DCP decode route is selected by the metadata builder.
dcp_route: _DCPDecodeRoute = _DCPDecodeRoute.PLAIN
# Set exactly when this batch routes to segmented DCP verification.
# Set when this batch takes a per-row DCP verify kernel (segmented MLA or
# gfx942 Triton). ``dcp_route`` selects which kernel.
dcp_verify: AiterMLADCPVerifyMetadata | None = None
# cprr only: global per-request page indptr and this rank's position.
g_kv_indptr: torch.Tensor | None = None
@@ -643,6 +732,10 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
parallel_config.decode_context_parallel_size,
parallel_config.cp_kv_cache_interleave_size,
)
supports_triton_dcp_verify = _triton_dcp_verify_supported(
parallel_config.decode_context_parallel_size,
parallel_config.cp_kv_cache_interleave_size,
)
verify_route = envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY.lower()
asm_dcp_verify_config = (
_asm_dcp_verify_configured(
@@ -659,7 +752,9 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
device,
AiterMLAMetadata,
supports_dcp_with_varlen=(
supports_segmented_dcp_verify or asm_dcp_verify_config
supports_segmented_dcp_verify
or asm_dcp_verify_config
or supports_triton_dcp_verify
),
)
self._asm_dcp_verify = False
@@ -684,7 +779,17 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
else "This AITER build also lacks segmented MLA decode."
)
)
self._supports_segmented_dcp_verify = supports_segmented_dcp_verify
self._supports_segmented_dcp_verify = (
supports_segmented_dcp_verify
and _segmented_mla_fits_lds(
self.num_heads * self.dcp_world_size,
kv_cache_spec.block_size,
is_quantized_kv_cache(vllm_config.cache_config.cache_dtype),
)
)
self._use_triton_dcp_verify = (
supports_triton_dcp_verify and not self._supports_segmented_dcp_verify
)
self._mla_max_split_per_batch = 0
if self._asm_dcp_verify:
# Cap on KV splits per batch; at low batch the KV axis is the only
@@ -723,7 +828,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
# (e.g. 16 with Eagle3), we expand block-level indices into per-token
# flat indices since the aiter kernel always uses page_size=1 internally.
self.kernel_block_size = kv_cache_spec.block_size
self._segmented_page_size = _segmented_mla_page_size(self.kernel_block_size)
# Generic Triton indexes physical KV blocks; segmented MLA indexes
# TILE subpages of them.
self._dcp_verify_page_size = (
kv_cache_spec.block_size
if self._use_triton_dcp_verify
else _segmented_mla_page_size(kv_cache_spec.block_size)
)
# A DCP rank's shard of the longest sequence bounds every verify row.
dcp_local_max_seq_len = (
cdiv(
vllm_config.model_config.max_model_len,
self.dcp_world_size * parallel_config.cp_kv_cache_interleave_size,
)
* parallel_config.cp_kv_cache_interleave_size
)
# In the flat view (.view(-1,1,1,H)), each token is its own page,
# so max_num_pages_per_req = max_model_len regardless of
@@ -886,21 +1005,16 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
max_num_reqs, dtype=torch.int32, device=device
)
if self._supports_segmented_dcp_verify and self._mtp_decode_qlen > 1:
if (
self._supports_segmented_dcp_verify or self._use_triton_dcp_verify
) and self._mtp_decode_qlen > 1:
# Allocate even when CPRR is the preferred route: a later
# replay can fall below _MIN_CPRR_QLEN and needs these
# addresses. A DCP rank's shard of the longest sequence
# bounds every verify row, and full graphs need that bound
# to be constant.
num_dcp_partitions = (
self.dcp_world_size * self.cp_kv_cache_interleave_size
)
graph_max_kv_seq_len = (
cdiv(vllm_config.model_config.max_model_len, num_dcp_partitions)
* self.cp_kv_cache_interleave_size
)
# addresses. Full graphs need the row bound to be constant.
max_verify_rows = max_num_reqs * self._mtp_decode_qlen
max_local_pages = cdiv(graph_max_kv_seq_len, self._segmented_page_size)
max_local_pages = cdiv(
dcp_local_max_seq_len, self._dcp_verify_page_size
)
self._dcp_verify_buffers = AiterMLADCPVerifyMetadata(
row_lens=torch.zeros(
max_verify_rows, dtype=torch.int32, device=device
@@ -913,9 +1027,20 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
qo_indptr=torch.arange(
max_verify_rows + 1, dtype=torch.int32, device=device
),
page_size=self._segmented_page_size,
max_kv_seq_len=graph_max_kv_seq_len,
page_size=self._dcp_verify_page_size,
max_kv_seq_len=dcp_local_max_seq_len,
)
# gfx942 DCP verify launches the generic split-KV kernel, which draws
# its partials from the shared workspace. Reserve it before warmup
# locks the pool.
if self._use_triton_dcp_verify and self._mtp_decode_qlen > 1:
_reserve_triton_dcp_verify_workspace(
max_num_reqs * self._mtp_decode_qlen,
self._decode_num_heads,
dcp_local_max_seq_len,
self.mla_dims.kv_lora_rank,
current_platform.num_compute_units(),
)
def _init_fp8_prefill_ps_buffers(
self,
@@ -1193,7 +1318,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
block_table: torch.Tensor,
dcp_tot_seq_lens: torch.Tensor,
) -> AiterMLADCPVerifyMetadata:
"""Build one paged-KV row per verify token for segmented DCP verification.
"""Build one paged-KV row per verify token for DCP verification.
Every row is a single query (``qo_indptr`` is an arange), so the whole
causal structure is carried by ``dcp_local_verify_row_lens`` and the
@@ -1217,7 +1342,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
if buffers is not None
else max(1, int(row_lens.max().item()))
)
page_size = self._segmented_page_size
page_size = self._dcp_verify_page_size
max_local_pages = cdiv(max_kv_seq_len, page_size)
if buffers is not None:
row_block_table = buffers.block_table[:num_rows]
@@ -1235,7 +1360,9 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
dtype=torch.int32,
device=block_table.device,
)
self._fill_dcp_verify_page_table(row_block_table, block_table, num_reqs, qlen)
self._fill_dcp_verify_page_table(
row_block_table, block_table, num_reqs, qlen, page_size
)
return AiterMLADCPVerifyMetadata(
row_lens=row_lens,
block_table=row_block_table,
@@ -1250,8 +1377,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
block_table: torch.Tensor,
num_reqs: int,
qlen: int,
page_size: int,
) -> None:
"""Expand each request's blocks into the subpages one verify row reads.
"""Fill one page list per request and broadcast it across qlen rows.
``page_size`` is the unit the kernel indexes. Segmented MLA uses TILE
subpages; generic Triton uses the physical KV block. When those are
equal, pages_per_block is 1 and this copies block IDs as-is.
Every row of a request shares the request's shard, so the page list is
built once per request and repeated; only ``row_lens`` distinguishes the
@@ -1259,9 +1391,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
request's block table is always wider than the pages a row can reach.
"""
kernel_block_size = self.kernel_block_size
page_size = self._segmented_page_size
assert kernel_block_size is not None
assert page_size is not None
assert page_size > 0
assert kernel_block_size % page_size == 0, (
f"DCP verify page_size {page_size} must divide kernel_block_size "
f"{kernel_block_size}"
)
pages_per_block = kernel_block_size // page_size
max_local_pages = row_block_table.shape[1]
max_local_blocks = cdiv(max_local_pages, pages_per_block)
@@ -1269,15 +1404,18 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
f"DCP verify needs {max_local_blocks} blocks per request but the "
f"block table only has {block_table.shape[1]}"
)
subpage_offsets = torch.arange(
pages_per_block,
dtype=block_table.dtype,
device=block_table.device,
)
per_req_page_table = (
block_table[:num_reqs, :max_local_blocks, None] * pages_per_block
+ subpage_offsets
).flatten(1)[:, :max_local_pages]
if pages_per_block == 1:
per_req_page_table = block_table[:num_reqs, :max_local_pages]
else:
subpage_offsets = torch.arange(
pages_per_block,
dtype=block_table.dtype,
device=block_table.device,
)
per_req_page_table = (
block_table[:num_reqs, :max_local_blocks, None] * pages_per_block
+ subpage_offsets
).flatten(1)[:, :max_local_pages]
# Broadcast the request's page list across its rows instead of
# materializing a repeat_interleave copy.
row_block_table.unflatten(0, (num_reqs, qlen)).copy_(
@@ -1373,18 +1511,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
dcp_route = _DCPDecodeRoute.PLAIN
if self.dcp_world_size > 1:
dcp_route = _select_dcp_decode_route(
self._supports_segmented_dcp_verify,
causal,
int(max_qo_len),
self._asm_dcp_verify,
supports_segmented=self._supports_segmented_dcp_verify,
supports_triton=self._use_triton_dcp_verify,
causal=causal,
max_qo_len=int(max_qo_len),
asm_selected=self._asm_dcp_verify,
)
# Segmented DCP verify carries its own per-row subpage table, so the
# Row-view DCP verify carries its own per-row page table, so the
# flat per-token view is dead work for it. Leave the buffer alone and
# hand the metadata None, so a future reader cannot pick up whatever
# the previous batch left behind.
paged_kv_indices = None
if dcp_route is not _DCPDecodeRoute.SEGMENTED:
if not _is_row_view_dcp_route(dcp_route):
if self.compilation_config.cudagraph_mode.has_full_cudagraphs():
self.paged_kv_indices.fill_(-1)
@@ -1461,13 +1600,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
has_persistent_metadata = False
# Only the asm decode consumes the schedule, so gate on the routing
# rather than on num_heads >= 16, which denies it to a padded rank
# running the same asm kernels. The predicates are disjoint -- decode
# is qlen==1, verify is qlen>1 -- and cover both Gluon entries plus the
# segmented DCP verify.
# running the same asm kernels. Gluon and the row-view DCP routes
# (segmented and gfx942 Triton) cover the non-ASM decode/verify paths.
use_persistent_metadata = (
not use_gluon_decode
and not use_gluon_verify
and dcp_route is not _DCPDecodeRoute.SEGMENTED
and not _is_row_view_dcp_route(dcp_route)
# A padded rank has no bf16 persistent kernel past qlen 4 where the
# gfx950 fold is absent; the non-persistent entry covers it. fp8
# keeps the schedule -- its fold rejects non-persistent outright.
@@ -1566,7 +1704,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]):
min_kv_seq_len = int(per_req_len.min().item())
dcp_verify = None
if dcp_route is _DCPDecodeRoute.SEGMENTED:
if _is_row_view_dcp_route(dcp_route):
assert causal
assert dcp_tot_seq_lens_device is not None
dcp_verify = self._build_dcp_verify_row_view(
int(max_qo_len),
@@ -2000,6 +2139,7 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
)
AiterMLAHelper.check_num_heads_validity(num_heads)
AiterMLAHelper.check_num_heads_validity(self._decode_num_heads)
self._sm_count = current_platform.num_compute_units()
unsupported_features = [alibi_slopes, sliding_window, logits_soft_cap]
if any(unsupported_features):
@@ -2389,6 +2529,64 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
out_dtype,
)
def _forward_triton_dcp_verify(
self,
q_mla: torch.Tensor,
verify: AiterMLADCPVerifyMetadata,
kv_c_and_k_pe_cache: torch.Tensor,
layer: AttentionLayer,
out_dtype: torch.dtype,
) -> tuple[torch.Tensor, torch.Tensor]:
"""gfx942 causal verify using the LDS-safe generic split-KV kernel.
Each verify token is its own row whose causal window is its
``row_lens`` entry. A zero-length row (an empty DCP shard or graph
padding) comes back as a NaN output with an LSE of -inf; the DCP
combine gives it zero weight and never reads the output.
"""
num_rows, num_heads = q_mla.shape[:2]
# Stage 2 stores every output and LSE element, empty rows included.
output = torch.empty(
num_rows, num_heads, self.kv_lora_rank, dtype=out_dtype, device=q_mla.device
)
# Matches merge_mla_segments_triton and the fp32 LSE the DCP combine
# kernel is warmed up for.
lse = torch.empty(num_rows, num_heads, dtype=torch.float32, device=q_mla.device)
if envs.VLLM_BATCH_INVARIANT:
num_kv_splits = 1
else:
num_kv_splits = _triton_dcp_verify_num_kv_splits(
verify.max_kv_seq_len, num_rows, num_heads, self._sm_count
)
logits_shape = (num_rows, num_heads, num_kv_splits, self.kv_lora_rank + 1)
if is_workspace_manager_initialized():
(attn_logits,) = current_workspace_manager().get_simultaneous(
(logits_shape, torch.float32),
)
else:
attn_logits = torch.empty(
logits_shape, dtype=torch.float32, device=q_mla.device
)
paged_kv = kv_c_and_k_pe_cache.unsqueeze(2)
decode_attention_fwd(
q_mla,
paged_kv,
paged_kv[..., : self.kv_lora_rank],
output,
lse,
verify.block_table,
verify.row_lens,
attn_logits,
num_kv_splits,
self.scale,
paged_kv.size(1),
k_scale=layer._k_scale,
v_scale=layer._k_scale,
is_mla=True,
)
return output, lse
def forward_mqa(
self,
q: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
@@ -2499,6 +2697,23 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]):
return o, None
dcp_route = decode.dcp_route
if dcp_route is _DCPDecodeRoute.TRITON:
verify = decode.dcp_verify
assert verify is not None
q_mla = torch.cat(q, dim=-1) if isinstance(q, tuple) else q
if (
is_quantized_kv_cache(self.kv_cache_dtype)
and q_mla.dtype != torch.bfloat16
):
q_mla = q_mla.to(torch.bfloat16) * layer._q_scale
return self._forward_triton_dcp_verify(
q_mla,
verify,
kv_c_and_k_pe_cache,
layer,
decode.attn_out_dtype,
)
if dcp_route is _DCPDecodeRoute.SEGMENTED:
verify = decode.dcp_verify
assert verify is not None