From a1f51a0226a8f5e97a5ed340e091a447c8b35911 Mon Sep 17 00:00:00 2001 From: xaguilar-amd Date: Wed, 14 Oct 2026 04:17:15 +0200 Subject: [PATCH] [ROCm][MLA] Route gfx942 DCP causal multi-token verify to Triton split-KV decode (#58555) Signed-off-by: Xavier Aguilar --- .../attention/test_rocm_aiter_mla_dcp_cprr.py | 50 ++- .../test_rocm_aiter_mla_dcp_cprr_numerics.py | 9 +- .../test_rocm_aiter_mla_dcp_verify_gfx942.py | 331 ++++++++++++++++++ .../test_rocm_aiter_mla_mtp_split.py | 102 +++++- .../attention/backends/mla/rocm_aiter_mla.py | 317 ++++++++++++++--- 5 files changed, 744 insertions(+), 65 deletions(-) create mode 100644 tests/v1/attention/test_rocm_aiter_mla_dcp_verify_gfx942.py diff --git a/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py index 56886872ec..7c75898015 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py +++ b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr.py @@ -161,6 +161,11 @@ def _build(monkeypatch, route, gathered_heads, segmented_supported=True): lambda *args: segmented_supported, ) monkeypatch.setattr(rocm_aiter_mla, "_fp8_mla_prefill_supported", lambda: False) + # These cases pick between cprr and segmented. On a gfx942 runner the real + # arch would add the Triton verify route, which this stub cannot build. + import vllm.platforms.rocm as rocm + + monkeypatch.setattr(rocm, "on_gfx942", lambda: False) monkeypatch.setattr( rocm_aiter_mla, "get_dcp_group", lambda: SimpleNamespace(rank_in_group=0) ) @@ -264,25 +269,46 @@ def test_min_cprr_qlen_is_above_two(): @pytest.mark.parametrize( - "supports,causal,qlen,asm,expected", + "segmented,triton,causal,qlen,asm,expected", [ - (True, True, 1, True, Route.PLAIN), - (True, True, 2, True, Route.SEGMENTED), - (True, True, 3, True, Route.CPRR), - (True, True, 4, True, Route.CPRR), - (True, True, 4, False, Route.SEGMENTED), - (True, False, 2, True, Route.PLAIN), - (True, False, 4, False, Route.PLAIN), - (False, True, 3, True, Route.CPRR), + (True, False, True, 1, True, Route.PLAIN), + (True, False, True, 2, True, Route.SEGMENTED), + (True, False, True, 3, True, Route.CPRR), + (True, False, True, 4, True, Route.CPRR), + (True, False, True, 4, False, Route.SEGMENTED), + (True, False, False, 2, True, Route.PLAIN), + (True, False, False, 4, False, Route.PLAIN), + (False, False, True, 3, True, Route.CPRR), + # gfx942: no cprr and no segmented, so causal verify goes to Triton. + (False, True, True, 1, False, Route.PLAIN), + (False, True, True, 2, False, Route.TRITON), + (False, True, True, 4, False, Route.TRITON), + (False, True, False, 4, False, Route.PLAIN), + # cprr still wins at or above its qlen floor. + (False, True, True, 4, True, Route.CPRR), + (False, True, True, 2, True, Route.TRITON), ], ) -def test_dcp_decode_route(supports, causal, qlen, asm, expected): - assert _select_route(supports, causal, qlen, asm) is expected +def test_dcp_decode_route(segmented, triton, causal, qlen, asm, expected): + route = _select_route( + supports_segmented=segmented, + supports_triton=triton, + causal=causal, + max_qo_len=qlen, + asm_selected=asm, + ) + assert route is expected def test_causal_multi_token_batch_without_a_valid_route_fails(): with pytest.raises(RuntimeError, match="requires either segmented MLA"): - _select_route(False, True, 2, True) + _select_route( + supports_segmented=False, + supports_triton=False, + causal=True, + max_qo_len=2, + asm_selected=True, + ) # Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs diff --git a/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr_numerics.py b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr_numerics.py index 44323dde41..18abfa9438 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr_numerics.py +++ b/tests/v1/attention/test_rocm_aiter_mla_dcp_cprr_numerics.py @@ -536,5 +536,12 @@ def test_cprr_kernel_is_selected_for_the_k3_shape(): is True ) assert QLEN >= m._MIN_CPRR_QLEN - assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR + route = m._select_dcp_decode_route( + supports_segmented=True, + supports_triton=False, + causal=True, + max_qo_len=QLEN, + asm_selected=True, + ) + assert route is m._DCPDecodeRoute.CPRR assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS diff --git a/tests/v1/attention/test_rocm_aiter_mla_dcp_verify_gfx942.py b/tests/v1/attention/test_rocm_aiter_mla_dcp_verify_gfx942.py new file mode 100644 index 0000000000..6b61688666 --- /dev/null +++ b/tests/v1/attention/test_rocm_aiter_mla_dcp_verify_gfx942.py @@ -0,0 +1,331 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Causal target verify under DCP on gfx942 (MI300/MI325). + +Reproduces the attention shape of + + vllm serve moonshotai/Kimi-K3 --tensor-parallel-size 8 \ + --decode-context-parallel-size 8 \ + --speculative-config '{"method": "dspark", "num_speculative_tokens": 5}' + +96 gathered heads, bf16 KV, qlen 6 and the 768-token blocks the hybrid KDA +layers impose. Segmented MLA tiles those blocks at 128 tokens and needs 128 KiB +of LDS (gfx942 has 64 KiB), so the builder must route this shape to the generic +Triton split-KV kernel. DeepSeek-V3 with MTP and an fp8 KV cache at DCP 8 hits +the same limit through its 128 gathered heads. Shapes segmented MLA fits stay +on it. + +The real metadata builder and ``forward_mqa`` run on one GPU with the DCP ranks +simulated one after another. That is sound because this path has no +collective: the query all-gather happens in the layer, before ``forward_mqa``. +""" + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.platforms import current_platform +from vllm.utils.math_utils import cdiv + +if not current_platform.is_rocm(): + pytest.skip("ROCm AITER MLA tests", allow_module_level=True) + +from vllm._aiter_ops import is_aiter_found # noqa: E402 + +# DeepSeek-R1 supplies the MLA dims, which Kimi-K3 and DeepSeek-V3 share; only +# the head count is overridden. +MODEL = "deepseek-ai/DeepSeek-R1" +KV_LORA_RANK, ROPE_DIM, NOPE_DIM = 512, 64, 128 +HEAD_SIZE = KV_LORA_RANK + ROPE_DIM +SCALE = (NOPE_DIM + ROPE_DIM) ** -0.5 +DCP = 8 +# Each rank holds ~800 tokens per request, so its shard spans several blocks. +NUM_REQS, CTX = 4, 6400 +FP8_KV_SCALE = 0.05 + + +def _on_gfx942() -> bool: + if not (is_aiter_found() and torch.accelerator.is_available()): + return False + from vllm.platforms.rocm import on_gfx942 + + return on_gfx942() + + +pytestmark = pytest.mark.skipif(not _on_gfx942(), reason="needs ROCm + AITER on gfx942") + + +class _FakeGroup: + def __init__(self, world_size: int, rank: int): + self.world_size = world_size + self.rank_in_group = self.rank = self.local_rank = rank + + def all_gather(self, tensor, dim=0): + raise RuntimeError("DCP verify must not issue a collective") + + +@pytest.fixture +def fake_dcp_groups(monkeypatch): + """Point every importer of get_dcp_group at a settable rank stub.""" + import vllm.distributed.parallel_state as ps + import vllm.model_executor.layers.attention.mla_attention as mla_attention + import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend + + group = _FakeGroup(DCP, 0) + for module in (ps, mla_attention, backend): + monkeypatch.setattr(module, "get_dcp_group", lambda: group) + monkeypatch.setattr(ps, "get_tp_group", lambda: _FakeGroup(1, 0)) + return group + + +def _vllm_config(heads_per_rank: int, qlen: int, fp8_kv: bool): + from tests.v1.attention.utils import create_vllm_config + + config = create_vllm_config( + model_name=MODEL, + max_model_len=CTX + qlen, + max_num_seqs=NUM_REQS, + max_num_batched_tokens=4096, + ) + config.model_config.model_arch_config.total_num_attention_heads = heads_per_rank + # The reorder threshold reads only these two fields; without them a + # multi-token row is classified as prefill and never reaches decode. + config.speculative_config = SimpleNamespace( + num_speculative_tokens=qlen - 1, parallel_drafting=False + ) + config.parallel_config.decode_context_parallel_size = DCP + config.parallel_config.cp_kv_cache_interleave_size = 1 + if fp8_kv: + config.cache_config.cache_dtype = "fp8" + return config + + +def _ctx_layer(): + """The registered-layer fields MLACommonMetadataBuilder.__init__ reads.""" + from vllm.v1.attention.ops.dcp import MLADCPManager + + dcp_manager = object.__new__(MLADCPManager) + dcp_manager.init_kv_gather = lambda *args, **kwargs: None + # Only chunked-context prefill uses the cloned backend. + prefill_backend = SimpleNamespace() + prefill_backend.clone = lambda: prefill_backend + return SimpleNamespace( + non_causal_multi_token_decode=False, + q_lora_rank=None, + kv_lora_rank=KV_LORA_RANK, + qk_nope_head_dim=NOPE_DIM, + qk_rope_head_dim=ROPE_DIM, + v_head_dim=NOPE_DIM, + dcp_manager=dcp_manager, + prefill_backend=prefill_backend, + ) + + +def _impl(dcp_rank: int, heads_per_rank: int, fp8_kv: bool): + """forward_mqa needs these attributes, not the weight-loading __init__.""" + from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl + + impl = object.__new__(AiterMLAImpl) + impl.num_heads = heads_per_rank + impl.kv_lora_rank = KV_LORA_RANK + impl.qk_rope_head_dim = ROPE_DIM + impl.dcp_world_size = DCP + impl.dcp_rank = dcp_rank + impl.pcp_world_size = 1 + impl.kv_cache_dtype = "fp8" if fp8_kv else "auto" + impl.scale = SCALE + impl._sm_count = current_platform.num_compute_units() + return impl + + +def _run_rank(config, spec, layer_name, group, rank, q, kv_cache_src, k_scale): + """Shard the KV round-robin onto ``rank`` and run its verify step.""" + from vllm.v1.attention.backend import CommonAttentionMetadata + from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder + + device = q.device + qlen, heads = q.shape[1], q.shape[2] // DCP + seq_len = CTX + qlen + positions = torch.arange(rank, seq_len, DCP, device=device) + local_len = positions.numel() + # Production block tables span max_model_len, so every rank's table is at + # least as wide as the largest shard needs. + blocks_per_req = cdiv(cdiv(seq_len, DCP), spec.block_size) + kv_cache = torch.zeros( + NUM_REQS * blocks_per_req, + spec.block_size, + HEAD_SIZE, + dtype=kv_cache_src.dtype, + device=device, + ) + kv_cache.view(NUM_REQS, -1, HEAD_SIZE)[:, :local_len] = kv_cache_src[:, positions] + block_table = torch.arange( + NUM_REQS * blocks_per_req, dtype=torch.int32, device=device + ).view(NUM_REQS, blocks_per_req) + + query_start_loc = torch.arange( + 0, NUM_REQS * qlen + 1, qlen, dtype=torch.int32, device=device + ) + local_lens = torch.full((NUM_REQS,), local_len, dtype=torch.int32) + common = CommonAttentionMetadata( + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc.cpu(), + seq_lens=torch.full((NUM_REQS,), seq_len, dtype=torch.int32, device=device), + num_reqs=NUM_REQS, + num_actual_tokens=NUM_REQS * qlen, + max_query_len=qlen, + max_seq_len=seq_len, + block_table_tensor=block_table, + slot_mapping=torch.arange(NUM_REQS * qlen, dtype=torch.int64, device=device), + causal=True, + dcp_local_seq_lens=local_lens.to(device), + dcp_local_seq_lens_cpu_upper_bound=local_lens, + ) + + group.rank_in_group = group.rank = group.local_rank = rank + builder = AiterMLAMetadataBuilder(spec, [layer_name], config, device) + metadata = builder.build(0, common) + layer = SimpleNamespace( + _q_scale=torch.tensor(1.0, device=device), + _k_scale=torch.tensor(k_scale, device=device), + ) + impl = _impl(rank, heads, kv_cache_src.dtype != torch.bfloat16) + output, lse = impl.forward_mqa(q.flatten(0, 1), kv_cache, metadata, layer) + return metadata.decode.dcp_route.name, output, lse + + +@pytest.mark.parametrize( + "heads_per_rank, qlen, block_size, fp8_kv, route", + [ + (12, 6, 768, False, "TRITON"), + (12, 6, 64, False, "SEGMENTED"), + (16, 2, 16, True, "TRITON"), + ], + ids=["kimi-k3-hybrid-kda-blocks", "segmented-fits", "deepseek-v3-mtp-fp8-kv"], +) +@torch.inference_mode() +def test_dcp_verify_matches_attention( + fake_dcp_groups, heads_per_rank, qlen, block_size, fp8_kv, route +): + from vllm.config import set_current_vllm_config + from vllm.v1.kv_cache_interface import MLAAttentionSpec + from vllm.v1.worker.workspace import ( + init_workspace_manager, + is_workspace_manager_initialized, + ) + + device = torch.device("cuda", torch.accelerator.current_device_index()) + if not is_workspace_manager_initialized(): + init_workspace_manager(device) + torch.manual_seed(0) + seq_len = CTX + qlen + q = torch.randn( + NUM_REQS, + qlen, + heads_per_rank * DCP, + HEAD_SIZE, + dtype=torch.bfloat16, + device=device, + ) + kv = torch.randn(NUM_REQS, seq_len, HEAD_SIZE, device=device).clamp_(-4, 4) + if fp8_kv: + k_scale = FP8_KV_SCALE + kv_cache_src = (kv / k_scale).to(current_platform.fp8_dtype()) + else: + k_scale = 1.0 + kv_cache_src = kv.to(torch.bfloat16) + kv_ref = kv_cache_src.float() * k_scale + + config = _vllm_config(heads_per_rank, qlen, fp8_kv) + layer_name = "model.layers.0.self_attn.attn" + with set_current_vllm_config(config): + config.compilation_config.static_forward_context[layer_name] = _ctx_layer() + spec = MLAAttentionSpec( + block_size=block_size, + num_kv_heads=1, + head_size=HEAD_SIZE, + dtype=kv_cache_src.dtype, + cache_dtype_str="fp8" if fp8_kv else None, + non_causal_multi_token_decode=False, + ) + shards = [ + _run_rank( + config, + spec, + layer_name, + fake_dcp_groups, + rank, + q, + kv_cache_src, + k_scale, + ) + for rank in range(DCP) + ] + + assert {shard_route for shard_route, _, _ in shards} == {route} + lses = torch.stack([lse.float() for _, _, lse in shards]) + weights = torch.exp(lses - lses.max(dim=0).values) + merged = ( + weights.unsqueeze(-1) * torch.stack([out.float() for _, out, _ in shards]) + ).sum(dim=0) / weights.sum(dim=0).unsqueeze(-1) + + # Query token t of a request sees the context plus verify tokens 0..t. + scores = torch.einsum("rthd,rld->rthl", q.float(), kv_ref) * SCALE + visible = torch.arange(seq_len, device=device) <= CTX + torch.arange( + qlen, device=device + ).view(-1, 1) + scores.masked_fill_(~visible[None, :, None, :], float("-inf")) + reference = torch.einsum( + "rthl,rld->rthd", scores.softmax(dim=-1), kv_ref[..., :KV_LORA_RANK] + ) + torch.testing.assert_close( + merged.view_as(reference), reference, rtol=2e-2, atol=2e-2 + ) + + +@pytest.mark.parametrize( + "fp8_kv, num_heads, block_size", + [ + (False, 96, 768), + (False, 128, 64), + (True, 64, 128), + (True, 128, 16), + ], +) +def test_segmented_lds_gate_matches_the_kernel(fp8_kv, num_heads, block_size): + """The gate must flag exactly the shapes AITER segmented MLA cannot compile.""" + from triton.runtime.errors import OutOfResources + + from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( + _get_segmented_mla_decode, + _segmented_mla_fits_lds, + _segmented_mla_page_size, + ) + + page = _segmented_mla_page_size(block_size) + kv_dtype = current_platform.fp8_dtype() if fp8_kv else torch.bfloat16 + q = torch.zeros(1, num_heads, HEAD_SIZE, dtype=torch.bfloat16, device="cuda") + kv = torch.zeros(1, page, 1, HEAD_SIZE, dtype=kv_dtype, device="cuda") + one_row = torch.tensor([0, 1], dtype=torch.int32, device="cuda") + try: + _get_segmented_mla_decode()( + q, + kv, + None, + one_row, + torch.tensor([page], dtype=torch.int32, device="cuda"), + page, + torch.zeros(1, 1, dtype=torch.int32, device="cuda"), + SCALE, + KV_LORA_RANK, + ROPE_DIM, + causal=True, + q_descale=None, + kv_descale=torch.tensor(1.0, device="cuda"), + skip_reduce=True, + ) + compiles = True + except OutOfResources: + compiles = False + assert _segmented_mla_fits_lds(num_heads, block_size, fp8_kv) == compiles diff --git a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py index 1583995ac9..187bbea4bf 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py +++ b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py @@ -20,6 +20,10 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( # noqa: E402 AiterMLAImpl, AiterMLAMetadataBuilder, ) +from vllm.v1.attention.ops.dcp import ( # noqa: E402 + CPTritonContext, + correct_attn_out, +) from vllm.v1.attention.ops.rocm_aiter_mla_merge import ( # noqa: E402 merge_mla_segments_triton, ) @@ -121,8 +125,11 @@ def _builder( # segmented route is available, query length decides per batch. CPRR # preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back. _supports_segmented_dcp_verify=supports_segmented_dcp_verify, + _use_triton_dcp_verify=False, # Derived once in the real constructor, so derive it once here too. - _segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size), + _dcp_verify_page_size=rocm_aiter_mla._segmented_mla_page_size( + kernel_block_size + ), _dcp_verify_buffers=None, _graph_seq_lens=None, _kv_cache_dtype_str=kv_cache_dtype, @@ -630,6 +637,99 @@ def test_segmented_verify_reduce_returns_natural_lse_and_masks_empty_rows(): assert lse[1].item() == float("-inf") +def test_triton_verify_empty_local_shard_merges_through_dcp_combine(): + """Empty rows need no masking: -inf LSE gives them zero combine weight. + + Row 0 has KV on both ranks, row 1 only on rank 0, row 2 on neither (graph + padding). The raw rows come back NaN; the production AG+RS combine must + still produce full-context attention and zeros for the padded row. + """ + torch.manual_seed(0) + num_ranks, num_rows, num_heads = 2, 3, 16 + kv_lora_rank, head_dim, block_size = 512, 576, 16 + scale = head_dim**-0.5 + local_lens = [[5, 4, 0], [5, 0, 0]] + + q = torch.randn(num_rows, num_heads, head_dim, dtype=torch.bfloat16, device="cuda") + kv_caches = [ + torch.randn(1, block_size, head_dim, dtype=torch.bfloat16, device="cuda") + for _ in range(num_ranks) + ] + impl = object.__new__(AiterMLAImpl) + impl.scale = scale + impl.kv_lora_rank = kv_lora_rank + impl._sm_count = current_platform.num_compute_units() + layer = SimpleNamespace(_k_scale=torch.tensor(1.0, device="cuda")) + outputs, lses = [], [] + for rank in range(num_ranks): + verify = SimpleNamespace( + block_table=torch.zeros(num_rows, 1, dtype=torch.int32, device="cuda"), + row_lens=torch.tensor(local_lens[rank], dtype=torch.int32, device="cuda"), + max_kv_seq_len=max(local_lens[rank]), + ) + output, lse = impl._forward_triton_dcp_verify( + q, verify, kv_caches[rank], layer, torch.bfloat16 + ) + outputs.append(output) + lses.append(lse) + + assert torch.isneginf(lses[1][1]).all() + assert torch.isneginf(lses[0][2]).all() and torch.isneginf(lses[1][2]).all() + + all_lses = torch.stack(lses) + merged = sum( + correct_attn_out(outputs[rank], all_lses, rank, CPTritonContext())[0].float() + for rank in range(num_ranks) + ) + + reference = torch.zeros(num_rows, num_heads, kv_lora_rank, device="cuda") + for row in range(num_rows): + keys = torch.cat( + [kv_caches[rank][0, : local_lens[rank][row]] for rank in range(num_ranks)] + ).float() + if keys.shape[0]: + probs = torch.softmax(q[row].float() @ keys.T * scale, dim=-1) + reference[row] = probs @ keys[:, :kv_lora_rank] + + assert torch.isfinite(merged).all() + torch.testing.assert_close(merged, reference, rtol=2e-2, atol=2e-2) + + +def test_batch_aware_kv_splits_fit_an_int32_safe_reservation(monkeypatch): + # MI325X DCP8 verify at 128K context: 128 gathered heads, 16K local tokens. + sm_count, heads, local_seq_len, kv_lora_rank = 304, 128, 16384, 512 + max_rows = 4096 + reserved = [] + monkeypatch.setattr( + rocm_aiter_mla, "is_workspace_manager_initialized", lambda: True + ) + monkeypatch.setattr( + rocm_aiter_mla, + "current_workspace_manager", + lambda: SimpleNamespace(get_simultaneous=lambda *specs: reserved.extend(specs)), + ) + rocm_aiter_mla._reserve_triton_dcp_verify_workspace( + max_rows, heads, local_seq_len, kv_lora_rank, sm_count + ) + ((shape, _),) = reserved + # The kernel's attn_logits offsets are int32; seq-only splits overflow them. + seq_only = rocm_aiter_mla._compute_num_kv_splits(local_seq_len, sm_count) + assert max_rows * heads * seq_only * (kv_lora_rank + 1) >= 2**31 + assert math.prod(shape) < 2**31 + + counts = [ + rocm_aiter_mla._triton_dcp_verify_num_kv_splits( + local_seq_len, rows, heads, sm_count + ) + for rows in range(1, max_rows + 1) + ] + # Small batches keep the seq-only split count. + assert counts[:4] == [seq_only] * 4 + assert counts == sorted(counts, reverse=True) + for rows, splits in enumerate(counts, start=1): + assert rows * heads * splits * (kv_lora_rank + 1) <= math.prod(shape) + + def test_segmented_dcp_verify_matches_causal_attention(monkeypatch): """Exercise the target qlen>1 path and merge two rank-local results.""" monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True) diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index d64b4797c0..dfd5dfabb0 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -26,6 +26,7 @@ from vllm.model_executor.layers.attention.mla_attention import ( accumulate_mla_context_chunk, init_mla_context_partial, ) +from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.math_utils import cdiv, largest_power_of_2_divisor @@ -35,6 +36,7 @@ from vllm.v1.attention.backend import ( CommonAttentionMetadata, MultipleOf, ) +from vllm.v1.attention.backends.mla.triton_mla import _compute_num_kv_splits from vllm.v1.attention.backends.utils import ( get_dcp_local_seq_lens, ) @@ -46,7 +48,12 @@ from vllm.v1.attention.ops.rocm_aiter_mla_prefill import ( expand_context, gather_compressed_context, ) +from vllm.v1.attention.ops.triton_decode_attention import decode_attention_fwd from vllm.v1.kv_cache_interface import AttentionSpec, is_quantized_kv_cache +from vllm.v1.worker.workspace import ( + current_workspace_manager, + is_workspace_manager_initialized, +) if TYPE_CHECKING: from vllm.platforms.interface import DeviceCapability @@ -269,6 +276,75 @@ def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) -> ) +def _segmented_mla_fits_lds(num_heads: int, block_size: int, fp8_kv: bool) -> bool: + """Whether AITER segmented MLA compiles for this DCP verify shape. + + On gfx942 it needs 128 KiB of LDS, over the 64 KiB available, for bf16 KV + at TILE-128 and for fp8 KV past 64 heads, where its query block widens to + 128 heads. ``num_heads`` is the DCP-gathered count. + """ + from vllm.platforms.rocm import on_gfx942 + + if not on_gfx942(): + return True + if fp8_kv: + return num_heads <= 64 + return _segmented_mla_page_size(block_size) < 128 + + +def _triton_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) -> bool: + """Whether gfx942 can fall back to generic split-KV causal target verify. + + The batch shape is causal qlen>1 over a DCP shard. Eagle MTP and DSpark + target verify are that same shape, so this is not limited to one + speculative method. gfx942 has no cprr kernel, so this serves the shapes + segmented MLA cannot (see _segmented_mla_fits_lds). qlen==1 and non-causal + stay on the plain ASM decode. + """ + from vllm.platforms.rocm import on_gfx942 + + return on_gfx942() and dcp_world_size > 1 and cp_interleave == 1 + + +# Heads per stage-1 program in the grouped MLA decode kernel (its BLOCK_H). +_DECODE_HEADS_PER_PROGRAM: Final = 16 +# Tuned on MI325X with microbenchmarks +_BATCH_OCCUPANCY_MULTIPLIER: Final = 4 + + +def _triton_dcp_verify_num_kv_splits( + max_seq_len: int, num_rows: int, num_heads: int, sm_count: int +) -> int: + # Split only until the stage-1 grid fills the GPU. Past that, splits add + # scratch and combine work, and large batches overflow the kernel's int32 + # attn_logits offsets. + programs_per_split = max(1, num_rows) * cdiv(num_heads, _DECODE_HEADS_PER_PROGRAM) + occupancy_splits = triton.next_power_of_2( + cdiv(_BATCH_OCCUPANCY_MULTIPLIER * sm_count, programs_per_split) + ) + return min(_compute_num_kv_splits(max_seq_len, sm_count), occupancy_splits) + + +def _reserve_triton_dcp_verify_workspace( + max_rows: int, + num_heads: int, + max_seq_len: int, + kv_lora_rank: int, + sm_count: int, +) -> None: + """Reserve split-KV scratch before warmup locks the workspace manager.""" + if not is_workspace_manager_initialized(): + return + # Fewer rows may take more splits, so size for the largest product. + max_row_splits = max( + rows * _triton_dcp_verify_num_kv_splits(max_seq_len, rows, num_heads, sm_count) + for rows in range(1, max_rows + 1) + ) + current_workspace_manager().get_simultaneous( + ((max_row_splits, num_heads, kv_lora_rank + 1), torch.float32), + ) + + # DCP decode routing # # With decode context parallelism each rank holds a round-robin shard of the KV @@ -281,9 +357,12 @@ def _segmented_dcp_verify_supported(dcp_world_size: int, cp_interleave: int) -> # round-robin ASM decode rebuilds global positions from g_kv_indptr, cp_rank # and cp_world_size and applies the causal window in-kernel. Kernels exist only # at _NATIVE_CPRR_HEADS; other gathered head counts are padded up. +# * triton: causal qlen > 1 on gfx942 for shapes segmented MLA cannot fit in +# LDS (_segmented_mla_fits_lds). Generic split-KV decode over the same +# per-row local window segmented uses. # * segmented: causal qlen > 1 when cprr is not selected or qlen is below -# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV length -# encodes its causal window. +# _MIN_CPRR_QLEN. Each verify token becomes a single-query row whose KV +# length encodes its causal window. # # The plain kernel must never serve a causal qlen > 1 block: it applies # causality on local indices, which is wrong over a round-robin shard. @@ -295,6 +374,12 @@ class _DCPDecodeRoute(Enum): PLAIN = "plain" SEGMENTED = "segmented" CPRR = "cprr" + TRITON = "triton" + + +def _is_row_view_dcp_route(route: _DCPDecodeRoute) -> bool: + """Routes that attend one causal window per verify token.""" + return route in (_DCPDecodeRoute.SEGMENTED, _DCPDecodeRoute.TRITON) def _asm_dcp_verify_heads(decode_num_heads: int) -> int: @@ -308,7 +393,9 @@ def _asm_dcp_verify_heads(decode_num_heads: int) -> int: def _select_dcp_decode_route( + *, supports_segmented: bool, + supports_triton: bool, causal: bool, max_qo_len: int, asm_selected: bool, @@ -318,11 +405,13 @@ def _select_dcp_decode_route( return _DCPDecodeRoute.PLAIN if asm_selected and max_qo_len >= _MIN_CPRR_QLEN: return _DCPDecodeRoute.CPRR + if supports_triton: + return _DCPDecodeRoute.TRITON if supports_segmented: return _DCPDecodeRoute.SEGMENTED raise RuntimeError( "ROCM_AITER_MLA DCP multi-token verify requires either segmented " - "MLA or the round-robin asm decode." + "MLA, the round-robin asm decode, or the gfx942 Triton verify path." ) @@ -472,11 +561,10 @@ class AiterMLABackend(MLACommonBackend): @dataclass class AiterMLADCPVerifyMetadata: - """One paged-KV row per verify token, for segmented DCP verification. + """One paged-KV row per verify token for causal DCP verification. - These are produced together or not at all, so they travel as one value: its - presence on the decode metadata *is* the routing decision the builder made, - and the impl does not re-derive it. + Segmented MLA and the gfx942 Triton path both consume this view. Which + kernel runs is ``AiterMLADecodeMetadata.dcp_route``. """ # Local KV length of each verify row, already causally bounded. @@ -514,7 +602,8 @@ class AiterMLADecodeMetadata(MLACommonDecodeMetadata): min_kv_seq_len: int = 1 # Exactly one DCP decode route is selected by the metadata builder. dcp_route: _DCPDecodeRoute = _DCPDecodeRoute.PLAIN - # Set exactly when this batch routes to segmented DCP verification. + # Set when this batch takes a per-row DCP verify kernel (segmented MLA or + # gfx942 Triton). ``dcp_route`` selects which kernel. dcp_verify: AiterMLADCPVerifyMetadata | None = None # cprr only: global per-request page indptr and this rank's position. g_kv_indptr: torch.Tensor | None = None @@ -643,6 +732,10 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): parallel_config.decode_context_parallel_size, parallel_config.cp_kv_cache_interleave_size, ) + supports_triton_dcp_verify = _triton_dcp_verify_supported( + parallel_config.decode_context_parallel_size, + parallel_config.cp_kv_cache_interleave_size, + ) verify_route = envs.VLLM_ROCM_AITER_MLA_DCP_VERIFY.lower() asm_dcp_verify_config = ( _asm_dcp_verify_configured( @@ -659,7 +752,9 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): device, AiterMLAMetadata, supports_dcp_with_varlen=( - supports_segmented_dcp_verify or asm_dcp_verify_config + supports_segmented_dcp_verify + or asm_dcp_verify_config + or supports_triton_dcp_verify ), ) self._asm_dcp_verify = False @@ -684,7 +779,17 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): else "This AITER build also lacks segmented MLA decode." ) ) - self._supports_segmented_dcp_verify = supports_segmented_dcp_verify + self._supports_segmented_dcp_verify = ( + supports_segmented_dcp_verify + and _segmented_mla_fits_lds( + self.num_heads * self.dcp_world_size, + kv_cache_spec.block_size, + is_quantized_kv_cache(vllm_config.cache_config.cache_dtype), + ) + ) + self._use_triton_dcp_verify = ( + supports_triton_dcp_verify and not self._supports_segmented_dcp_verify + ) self._mla_max_split_per_batch = 0 if self._asm_dcp_verify: # Cap on KV splits per batch; at low batch the KV axis is the only @@ -723,7 +828,21 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): # (e.g. 16 with Eagle3), we expand block-level indices into per-token # flat indices since the aiter kernel always uses page_size=1 internally. self.kernel_block_size = kv_cache_spec.block_size - self._segmented_page_size = _segmented_mla_page_size(self.kernel_block_size) + # Generic Triton indexes physical KV blocks; segmented MLA indexes + # TILE subpages of them. + self._dcp_verify_page_size = ( + kv_cache_spec.block_size + if self._use_triton_dcp_verify + else _segmented_mla_page_size(kv_cache_spec.block_size) + ) + # A DCP rank's shard of the longest sequence bounds every verify row. + dcp_local_max_seq_len = ( + cdiv( + vllm_config.model_config.max_model_len, + self.dcp_world_size * parallel_config.cp_kv_cache_interleave_size, + ) + * parallel_config.cp_kv_cache_interleave_size + ) # In the flat view (.view(-1,1,1,H)), each token is its own page, # so max_num_pages_per_req = max_model_len regardless of @@ -886,21 +1005,16 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): max_num_reqs, dtype=torch.int32, device=device ) - if self._supports_segmented_dcp_verify and self._mtp_decode_qlen > 1: + if ( + self._supports_segmented_dcp_verify or self._use_triton_dcp_verify + ) and self._mtp_decode_qlen > 1: # Allocate even when CPRR is the preferred route: a later # replay can fall below _MIN_CPRR_QLEN and needs these - # addresses. A DCP rank's shard of the longest sequence - # bounds every verify row, and full graphs need that bound - # to be constant. - num_dcp_partitions = ( - self.dcp_world_size * self.cp_kv_cache_interleave_size - ) - graph_max_kv_seq_len = ( - cdiv(vllm_config.model_config.max_model_len, num_dcp_partitions) - * self.cp_kv_cache_interleave_size - ) + # addresses. Full graphs need the row bound to be constant. max_verify_rows = max_num_reqs * self._mtp_decode_qlen - max_local_pages = cdiv(graph_max_kv_seq_len, self._segmented_page_size) + max_local_pages = cdiv( + dcp_local_max_seq_len, self._dcp_verify_page_size + ) self._dcp_verify_buffers = AiterMLADCPVerifyMetadata( row_lens=torch.zeros( max_verify_rows, dtype=torch.int32, device=device @@ -913,9 +1027,20 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): qo_indptr=torch.arange( max_verify_rows + 1, dtype=torch.int32, device=device ), - page_size=self._segmented_page_size, - max_kv_seq_len=graph_max_kv_seq_len, + page_size=self._dcp_verify_page_size, + max_kv_seq_len=dcp_local_max_seq_len, ) + # gfx942 DCP verify launches the generic split-KV kernel, which draws + # its partials from the shared workspace. Reserve it before warmup + # locks the pool. + if self._use_triton_dcp_verify and self._mtp_decode_qlen > 1: + _reserve_triton_dcp_verify_workspace( + max_num_reqs * self._mtp_decode_qlen, + self._decode_num_heads, + dcp_local_max_seq_len, + self.mla_dims.kv_lora_rank, + current_platform.num_compute_units(), + ) def _init_fp8_prefill_ps_buffers( self, @@ -1193,7 +1318,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): block_table: torch.Tensor, dcp_tot_seq_lens: torch.Tensor, ) -> AiterMLADCPVerifyMetadata: - """Build one paged-KV row per verify token for segmented DCP verification. + """Build one paged-KV row per verify token for DCP verification. Every row is a single query (``qo_indptr`` is an arange), so the whole causal structure is carried by ``dcp_local_verify_row_lens`` and the @@ -1217,7 +1342,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): if buffers is not None else max(1, int(row_lens.max().item())) ) - page_size = self._segmented_page_size + page_size = self._dcp_verify_page_size max_local_pages = cdiv(max_kv_seq_len, page_size) if buffers is not None: row_block_table = buffers.block_table[:num_rows] @@ -1235,7 +1360,9 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): dtype=torch.int32, device=block_table.device, ) - self._fill_dcp_verify_page_table(row_block_table, block_table, num_reqs, qlen) + self._fill_dcp_verify_page_table( + row_block_table, block_table, num_reqs, qlen, page_size + ) return AiterMLADCPVerifyMetadata( row_lens=row_lens, block_table=row_block_table, @@ -1250,8 +1377,13 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): block_table: torch.Tensor, num_reqs: int, qlen: int, + page_size: int, ) -> None: - """Expand each request's blocks into the subpages one verify row reads. + """Fill one page list per request and broadcast it across qlen rows. + + ``page_size`` is the unit the kernel indexes. Segmented MLA uses TILE + subpages; generic Triton uses the physical KV block. When those are + equal, pages_per_block is 1 and this copies block IDs as-is. Every row of a request shares the request's shard, so the page list is built once per request and repeated; only ``row_lens`` distinguishes the @@ -1259,9 +1391,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): request's block table is always wider than the pages a row can reach. """ kernel_block_size = self.kernel_block_size - page_size = self._segmented_page_size assert kernel_block_size is not None - assert page_size is not None + assert page_size > 0 + assert kernel_block_size % page_size == 0, ( + f"DCP verify page_size {page_size} must divide kernel_block_size " + f"{kernel_block_size}" + ) pages_per_block = kernel_block_size // page_size max_local_pages = row_block_table.shape[1] max_local_blocks = cdiv(max_local_pages, pages_per_block) @@ -1269,15 +1404,18 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): f"DCP verify needs {max_local_blocks} blocks per request but the " f"block table only has {block_table.shape[1]}" ) - subpage_offsets = torch.arange( - pages_per_block, - dtype=block_table.dtype, - device=block_table.device, - ) - per_req_page_table = ( - block_table[:num_reqs, :max_local_blocks, None] * pages_per_block - + subpage_offsets - ).flatten(1)[:, :max_local_pages] + if pages_per_block == 1: + per_req_page_table = block_table[:num_reqs, :max_local_pages] + else: + subpage_offsets = torch.arange( + pages_per_block, + dtype=block_table.dtype, + device=block_table.device, + ) + per_req_page_table = ( + block_table[:num_reqs, :max_local_blocks, None] * pages_per_block + + subpage_offsets + ).flatten(1)[:, :max_local_pages] # Broadcast the request's page list across its rows instead of # materializing a repeat_interleave copy. row_block_table.unflatten(0, (num_reqs, qlen)).copy_( @@ -1373,18 +1511,19 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): dcp_route = _DCPDecodeRoute.PLAIN if self.dcp_world_size > 1: dcp_route = _select_dcp_decode_route( - self._supports_segmented_dcp_verify, - causal, - int(max_qo_len), - self._asm_dcp_verify, + supports_segmented=self._supports_segmented_dcp_verify, + supports_triton=self._use_triton_dcp_verify, + causal=causal, + max_qo_len=int(max_qo_len), + asm_selected=self._asm_dcp_verify, ) - # Segmented DCP verify carries its own per-row subpage table, so the + # Row-view DCP verify carries its own per-row page table, so the # flat per-token view is dead work for it. Leave the buffer alone and # hand the metadata None, so a future reader cannot pick up whatever # the previous batch left behind. paged_kv_indices = None - if dcp_route is not _DCPDecodeRoute.SEGMENTED: + if not _is_row_view_dcp_route(dcp_route): if self.compilation_config.cudagraph_mode.has_full_cudagraphs(): self.paged_kv_indices.fill_(-1) @@ -1461,13 +1600,12 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): has_persistent_metadata = False # Only the asm decode consumes the schedule, so gate on the routing # rather than on num_heads >= 16, which denies it to a padded rank - # running the same asm kernels. The predicates are disjoint -- decode - # is qlen==1, verify is qlen>1 -- and cover both Gluon entries plus the - # segmented DCP verify. + # running the same asm kernels. Gluon and the row-view DCP routes + # (segmented and gfx942 Triton) cover the non-ASM decode/verify paths. use_persistent_metadata = ( not use_gluon_decode and not use_gluon_verify - and dcp_route is not _DCPDecodeRoute.SEGMENTED + and not _is_row_view_dcp_route(dcp_route) # A padded rank has no bf16 persistent kernel past qlen 4 where the # gfx950 fold is absent; the non-persistent entry covers it. fp8 # keeps the schedule -- its fold rejects non-persistent outright. @@ -1566,7 +1704,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): min_kv_seq_len = int(per_req_len.min().item()) dcp_verify = None - if dcp_route is _DCPDecodeRoute.SEGMENTED: + if _is_row_view_dcp_route(dcp_route): + assert causal assert dcp_tot_seq_lens_device is not None dcp_verify = self._build_dcp_verify_row_view( int(max_qo_len), @@ -2000,6 +2139,7 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): ) AiterMLAHelper.check_num_heads_validity(num_heads) AiterMLAHelper.check_num_heads_validity(self._decode_num_heads) + self._sm_count = current_platform.num_compute_units() unsupported_features = [alibi_slopes, sliding_window, logits_soft_cap] if any(unsupported_features): @@ -2389,6 +2529,64 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): out_dtype, ) + def _forward_triton_dcp_verify( + self, + q_mla: torch.Tensor, + verify: AiterMLADCPVerifyMetadata, + kv_c_and_k_pe_cache: torch.Tensor, + layer: AttentionLayer, + out_dtype: torch.dtype, + ) -> tuple[torch.Tensor, torch.Tensor]: + """gfx942 causal verify using the LDS-safe generic split-KV kernel. + + Each verify token is its own row whose causal window is its + ``row_lens`` entry. A zero-length row (an empty DCP shard or graph + padding) comes back as a NaN output with an LSE of -inf; the DCP + combine gives it zero weight and never reads the output. + """ + num_rows, num_heads = q_mla.shape[:2] + # Stage 2 stores every output and LSE element, empty rows included. + output = torch.empty( + num_rows, num_heads, self.kv_lora_rank, dtype=out_dtype, device=q_mla.device + ) + # Matches merge_mla_segments_triton and the fp32 LSE the DCP combine + # kernel is warmed up for. + lse = torch.empty(num_rows, num_heads, dtype=torch.float32, device=q_mla.device) + if envs.VLLM_BATCH_INVARIANT: + num_kv_splits = 1 + else: + num_kv_splits = _triton_dcp_verify_num_kv_splits( + verify.max_kv_seq_len, num_rows, num_heads, self._sm_count + ) + logits_shape = (num_rows, num_heads, num_kv_splits, self.kv_lora_rank + 1) + if is_workspace_manager_initialized(): + (attn_logits,) = current_workspace_manager().get_simultaneous( + (logits_shape, torch.float32), + ) + else: + attn_logits = torch.empty( + logits_shape, dtype=torch.float32, device=q_mla.device + ) + + paged_kv = kv_c_and_k_pe_cache.unsqueeze(2) + decode_attention_fwd( + q_mla, + paged_kv, + paged_kv[..., : self.kv_lora_rank], + output, + lse, + verify.block_table, + verify.row_lens, + attn_logits, + num_kv_splits, + self.scale, + paged_kv.size(1), + k_scale=layer._k_scale, + v_scale=layer._k_scale, + is_mla=True, + ) + return output, lse + def forward_mqa( self, q: torch.Tensor | tuple[torch.Tensor, torch.Tensor], @@ -2499,6 +2697,23 @@ class AiterMLAImpl(MLACommonImpl[AiterMLAMetadata]): return o, None dcp_route = decode.dcp_route + if dcp_route is _DCPDecodeRoute.TRITON: + verify = decode.dcp_verify + assert verify is not None + q_mla = torch.cat(q, dim=-1) if isinstance(q, tuple) else q + if ( + is_quantized_kv_cache(self.kv_cache_dtype) + and q_mla.dtype != torch.bfloat16 + ): + q_mla = q_mla.to(torch.bfloat16) * layer._q_scale + return self._forward_triton_dcp_verify( + q_mla, + verify, + kv_c_and_k_pe_cache, + layer, + decode.attn_out_dtype, + ) + if dcp_route is _DCPDecodeRoute.SEGMENTED: verify = decode.dcp_verify assert verify is not None